1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2007 - 2018 Intel Corporation. */
3 
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5 
6 #include <linux/module.h>
7 #include <linux/types.h>
8 #include <linux/init.h>
9 #include <linux/bitops.h>
10 #include <linux/vmalloc.h>
11 #include <linux/pagemap.h>
12 #include <linux/netdevice.h>
13 #include <linux/ipv6.h>
14 #include <linux/slab.h>
15 #include <net/checksum.h>
16 #include <net/ip6_checksum.h>
17 #include <net/pkt_sched.h>
18 #include <net/pkt_cls.h>
19 #include <linux/net_tstamp.h>
20 #include <linux/mii.h>
21 #include <linux/ethtool.h>
22 #include <linux/if.h>
23 #include <linux/if_vlan.h>
24 #include <linux/pci.h>
25 #include <linux/delay.h>
26 #include <linux/interrupt.h>
27 #include <linux/ip.h>
28 #include <linux/tcp.h>
29 #include <linux/sctp.h>
30 #include <linux/if_ether.h>
31 #include <linux/prefetch.h>
32 #include <linux/bpf.h>
33 #include <linux/bpf_trace.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/etherdevice.h>
36 #include <linux/lockdep.h>
37 #ifdef CONFIG_IGB_DCA
38 #include <linux/dca.h>
39 #endif
40 #include <linux/i2c.h>
41 #include "igb.h"
42 
43 enum queue_mode {
44 	QUEUE_MODE_STRICT_PRIORITY,
45 	QUEUE_MODE_STREAM_RESERVATION,
46 };
47 
48 enum tx_queue_prio {
49 	TX_QUEUE_PRIO_HIGH,
50 	TX_QUEUE_PRIO_LOW,
51 };
52 
53 char igb_driver_name[] = "igb";
54 static const char igb_driver_string[] =
55 				"Intel(R) Gigabit Ethernet Network Driver";
56 static const char igb_copyright[] =
57 				"Copyright (c) 2007-2014 Intel Corporation.";
58 
59 static const struct e1000_info *igb_info_tbl[] = {
60 	[board_82575] = &e1000_82575_info,
61 };
62 
63 static const struct pci_device_id igb_pci_tbl[] = {
64 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
65 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
66 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
67 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
68 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
69 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
70 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
71 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
72 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
73 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
74 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
75 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
76 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
77 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
78 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
79 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
80 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
81 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
82 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
83 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
84 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
85 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
86 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
87 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
88 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
89 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
90 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
91 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
92 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
93 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
94 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
95 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
96 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
97 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
98 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
99 	/* required last entry */
100 	{0, }
101 };
102 
103 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
104 
105 static int igb_setup_all_tx_resources(struct igb_adapter *);
106 static int igb_setup_all_rx_resources(struct igb_adapter *);
107 static void igb_free_all_tx_resources(struct igb_adapter *);
108 static void igb_free_all_rx_resources(struct igb_adapter *);
109 static void igb_setup_mrqc(struct igb_adapter *);
110 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
111 static void igb_remove(struct pci_dev *pdev);
112 static void igb_init_queue_configuration(struct igb_adapter *adapter);
113 static int igb_sw_init(struct igb_adapter *);
114 int igb_open(struct net_device *);
115 int igb_close(struct net_device *);
116 static void igb_configure(struct igb_adapter *);
117 static void igb_configure_tx(struct igb_adapter *);
118 static void igb_configure_rx(struct igb_adapter *);
119 static void igb_clean_all_tx_rings(struct igb_adapter *);
120 static void igb_clean_all_rx_rings(struct igb_adapter *);
121 static void igb_clean_tx_ring(struct igb_ring *);
122 static void igb_clean_rx_ring(struct igb_ring *);
123 static void igb_set_rx_mode(struct net_device *);
124 static void igb_update_phy_info(struct timer_list *);
125 static void igb_watchdog(struct timer_list *);
126 static void igb_watchdog_task(struct work_struct *);
127 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
128 static void igb_get_stats64(struct net_device *dev,
129 			    struct rtnl_link_stats64 *stats);
130 static int igb_change_mtu(struct net_device *, int);
131 static int igb_set_mac(struct net_device *, void *);
132 static void igb_set_uta(struct igb_adapter *adapter, bool set);
133 static irqreturn_t igb_intr(int irq, void *);
134 static irqreturn_t igb_intr_msi(int irq, void *);
135 static irqreturn_t igb_msix_other(int irq, void *);
136 static irqreturn_t igb_msix_ring(int irq, void *);
137 #ifdef CONFIG_IGB_DCA
138 static void igb_update_dca(struct igb_q_vector *);
139 static void igb_setup_dca(struct igb_adapter *);
140 #endif /* CONFIG_IGB_DCA */
141 static int igb_poll(struct napi_struct *, int);
142 static bool igb_clean_tx_irq(struct igb_q_vector *, int);
143 static int igb_clean_rx_irq(struct igb_q_vector *, int);
144 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
145 static void igb_tx_timeout(struct net_device *, unsigned int txqueue);
146 static void igb_reset_task(struct work_struct *);
147 static void igb_vlan_mode(struct net_device *netdev,
148 			  netdev_features_t features);
149 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
150 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
151 static void igb_restore_vlan(struct igb_adapter *);
152 static void igb_rar_set_index(struct igb_adapter *, u32);
153 static void igb_ping_all_vfs(struct igb_adapter *);
154 static void igb_msg_task(struct igb_adapter *);
155 static void igb_vmm_control(struct igb_adapter *);
156 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
157 static void igb_flush_mac_table(struct igb_adapter *);
158 static int igb_available_rars(struct igb_adapter *, u8);
159 static void igb_set_default_mac_filter(struct igb_adapter *);
160 static int igb_uc_sync(struct net_device *, const unsigned char *);
161 static int igb_uc_unsync(struct net_device *, const unsigned char *);
162 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
163 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
164 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
165 			       int vf, u16 vlan, u8 qos, __be16 vlan_proto);
166 static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
167 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
168 				   bool setting);
169 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf,
170 				bool setting);
171 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
172 				 struct ifla_vf_info *ivi);
173 static void igb_check_vf_rate_limit(struct igb_adapter *);
174 static void igb_nfc_filter_exit(struct igb_adapter *adapter);
175 static void igb_nfc_filter_restore(struct igb_adapter *adapter);
176 
177 #ifdef CONFIG_PCI_IOV
178 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
179 static int igb_disable_sriov(struct pci_dev *dev, bool reinit);
180 #endif
181 
182 static int igb_suspend(struct device *);
183 static int igb_resume(struct device *);
184 static int igb_runtime_suspend(struct device *dev);
185 static int igb_runtime_resume(struct device *dev);
186 static int igb_runtime_idle(struct device *dev);
187 #ifdef CONFIG_PM
188 static const struct dev_pm_ops igb_pm_ops = {
189 	SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
190 	SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
191 			igb_runtime_idle)
192 };
193 #endif
194 static void igb_shutdown(struct pci_dev *);
195 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
196 #ifdef CONFIG_IGB_DCA
197 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
198 static struct notifier_block dca_notifier = {
199 	.notifier_call	= igb_notify_dca,
200 	.next		= NULL,
201 	.priority	= 0
202 };
203 #endif
204 #ifdef CONFIG_PCI_IOV
205 static unsigned int max_vfs;
206 module_param(max_vfs, uint, 0);
207 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
208 #endif /* CONFIG_PCI_IOV */
209 
210 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
211 		     pci_channel_state_t);
212 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
213 static void igb_io_resume(struct pci_dev *);
214 
215 static const struct pci_error_handlers igb_err_handler = {
216 	.error_detected = igb_io_error_detected,
217 	.slot_reset = igb_io_slot_reset,
218 	.resume = igb_io_resume,
219 };
220 
221 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
222 
223 static struct pci_driver igb_driver = {
224 	.name     = igb_driver_name,
225 	.id_table = igb_pci_tbl,
226 	.probe    = igb_probe,
227 	.remove   = igb_remove,
228 #ifdef CONFIG_PM
229 	.driver.pm = &igb_pm_ops,
230 #endif
231 	.shutdown = igb_shutdown,
232 	.sriov_configure = igb_pci_sriov_configure,
233 	.err_handler = &igb_err_handler
234 };
235 
236 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
237 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
238 MODULE_LICENSE("GPL v2");
239 
240 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
241 static int debug = -1;
242 module_param(debug, int, 0);
243 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
244 
245 struct igb_reg_info {
246 	u32 ofs;
247 	char *name;
248 };
249 
250 static const struct igb_reg_info igb_reg_info_tbl[] = {
251 
252 	/* General Registers */
253 	{E1000_CTRL, "CTRL"},
254 	{E1000_STATUS, "STATUS"},
255 	{E1000_CTRL_EXT, "CTRL_EXT"},
256 
257 	/* Interrupt Registers */
258 	{E1000_ICR, "ICR"},
259 
260 	/* RX Registers */
261 	{E1000_RCTL, "RCTL"},
262 	{E1000_RDLEN(0), "RDLEN"},
263 	{E1000_RDH(0), "RDH"},
264 	{E1000_RDT(0), "RDT"},
265 	{E1000_RXDCTL(0), "RXDCTL"},
266 	{E1000_RDBAL(0), "RDBAL"},
267 	{E1000_RDBAH(0), "RDBAH"},
268 
269 	/* TX Registers */
270 	{E1000_TCTL, "TCTL"},
271 	{E1000_TDBAL(0), "TDBAL"},
272 	{E1000_TDBAH(0), "TDBAH"},
273 	{E1000_TDLEN(0), "TDLEN"},
274 	{E1000_TDH(0), "TDH"},
275 	{E1000_TDT(0), "TDT"},
276 	{E1000_TXDCTL(0), "TXDCTL"},
277 	{E1000_TDFH, "TDFH"},
278 	{E1000_TDFT, "TDFT"},
279 	{E1000_TDFHS, "TDFHS"},
280 	{E1000_TDFPC, "TDFPC"},
281 
282 	/* List Terminator */
283 	{}
284 };
285 
286 /* igb_regdump - register printout routine */
igb_regdump(struct e1000_hw * hw,struct igb_reg_info * reginfo)287 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
288 {
289 	int n = 0;
290 	char rname[16];
291 	u32 regs[8];
292 
293 	switch (reginfo->ofs) {
294 	case E1000_RDLEN(0):
295 		for (n = 0; n < 4; n++)
296 			regs[n] = rd32(E1000_RDLEN(n));
297 		break;
298 	case E1000_RDH(0):
299 		for (n = 0; n < 4; n++)
300 			regs[n] = rd32(E1000_RDH(n));
301 		break;
302 	case E1000_RDT(0):
303 		for (n = 0; n < 4; n++)
304 			regs[n] = rd32(E1000_RDT(n));
305 		break;
306 	case E1000_RXDCTL(0):
307 		for (n = 0; n < 4; n++)
308 			regs[n] = rd32(E1000_RXDCTL(n));
309 		break;
310 	case E1000_RDBAL(0):
311 		for (n = 0; n < 4; n++)
312 			regs[n] = rd32(E1000_RDBAL(n));
313 		break;
314 	case E1000_RDBAH(0):
315 		for (n = 0; n < 4; n++)
316 			regs[n] = rd32(E1000_RDBAH(n));
317 		break;
318 	case E1000_TDBAL(0):
319 		for (n = 0; n < 4; n++)
320 			regs[n] = rd32(E1000_TDBAL(n));
321 		break;
322 	case E1000_TDBAH(0):
323 		for (n = 0; n < 4; n++)
324 			regs[n] = rd32(E1000_TDBAH(n));
325 		break;
326 	case E1000_TDLEN(0):
327 		for (n = 0; n < 4; n++)
328 			regs[n] = rd32(E1000_TDLEN(n));
329 		break;
330 	case E1000_TDH(0):
331 		for (n = 0; n < 4; n++)
332 			regs[n] = rd32(E1000_TDH(n));
333 		break;
334 	case E1000_TDT(0):
335 		for (n = 0; n < 4; n++)
336 			regs[n] = rd32(E1000_TDT(n));
337 		break;
338 	case E1000_TXDCTL(0):
339 		for (n = 0; n < 4; n++)
340 			regs[n] = rd32(E1000_TXDCTL(n));
341 		break;
342 	default:
343 		pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
344 		return;
345 	}
346 
347 	snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
348 	pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
349 		regs[2], regs[3]);
350 }
351 
352 /* igb_dump - Print registers, Tx-rings and Rx-rings */
igb_dump(struct igb_adapter * adapter)353 static void igb_dump(struct igb_adapter *adapter)
354 {
355 	struct net_device *netdev = adapter->netdev;
356 	struct e1000_hw *hw = &adapter->hw;
357 	struct igb_reg_info *reginfo;
358 	struct igb_ring *tx_ring;
359 	union e1000_adv_tx_desc *tx_desc;
360 	struct my_u0 { __le64 a; __le64 b; } *u0;
361 	struct igb_ring *rx_ring;
362 	union e1000_adv_rx_desc *rx_desc;
363 	u32 staterr;
364 	u16 i, n;
365 
366 	if (!netif_msg_hw(adapter))
367 		return;
368 
369 	/* Print netdevice Info */
370 	if (netdev) {
371 		dev_info(&adapter->pdev->dev, "Net device Info\n");
372 		pr_info("Device Name     state            trans_start\n");
373 		pr_info("%-15s %016lX %016lX\n", netdev->name,
374 			netdev->state, dev_trans_start(netdev));
375 	}
376 
377 	/* Print Registers */
378 	dev_info(&adapter->pdev->dev, "Register Dump\n");
379 	pr_info(" Register Name   Value\n");
380 	for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
381 	     reginfo->name; reginfo++) {
382 		igb_regdump(hw, reginfo);
383 	}
384 
385 	/* Print TX Ring Summary */
386 	if (!netdev || !netif_running(netdev))
387 		goto exit;
388 
389 	dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
390 	pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
391 	for (n = 0; n < adapter->num_tx_queues; n++) {
392 		struct igb_tx_buffer *buffer_info;
393 		tx_ring = adapter->tx_ring[n];
394 		buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
395 		pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
396 			n, tx_ring->next_to_use, tx_ring->next_to_clean,
397 			(u64)dma_unmap_addr(buffer_info, dma),
398 			dma_unmap_len(buffer_info, len),
399 			buffer_info->next_to_watch,
400 			(u64)buffer_info->time_stamp);
401 	}
402 
403 	/* Print TX Rings */
404 	if (!netif_msg_tx_done(adapter))
405 		goto rx_ring_summary;
406 
407 	dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
408 
409 	/* Transmit Descriptor Formats
410 	 *
411 	 * Advanced Transmit Descriptor
412 	 *   +--------------------------------------------------------------+
413 	 * 0 |         Buffer Address [63:0]                                |
414 	 *   +--------------------------------------------------------------+
415 	 * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
416 	 *   +--------------------------------------------------------------+
417 	 *   63      46 45    40 39 38 36 35 32 31   24             15       0
418 	 */
419 
420 	for (n = 0; n < adapter->num_tx_queues; n++) {
421 		tx_ring = adapter->tx_ring[n];
422 		pr_info("------------------------------------\n");
423 		pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
424 		pr_info("------------------------------------\n");
425 		pr_info("T [desc]     [address 63:0  ] [PlPOCIStDDM Ln] [bi->dma       ] leng  ntw timestamp        bi->skb\n");
426 
427 		for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
428 			const char *next_desc;
429 			struct igb_tx_buffer *buffer_info;
430 			tx_desc = IGB_TX_DESC(tx_ring, i);
431 			buffer_info = &tx_ring->tx_buffer_info[i];
432 			u0 = (struct my_u0 *)tx_desc;
433 			if (i == tx_ring->next_to_use &&
434 			    i == tx_ring->next_to_clean)
435 				next_desc = " NTC/U";
436 			else if (i == tx_ring->next_to_use)
437 				next_desc = " NTU";
438 			else if (i == tx_ring->next_to_clean)
439 				next_desc = " NTC";
440 			else
441 				next_desc = "";
442 
443 			pr_info("T [0x%03X]    %016llX %016llX %016llX %04X  %p %016llX %p%s\n",
444 				i, le64_to_cpu(u0->a),
445 				le64_to_cpu(u0->b),
446 				(u64)dma_unmap_addr(buffer_info, dma),
447 				dma_unmap_len(buffer_info, len),
448 				buffer_info->next_to_watch,
449 				(u64)buffer_info->time_stamp,
450 				buffer_info->skb, next_desc);
451 
452 			if (netif_msg_pktdata(adapter) && buffer_info->skb)
453 				print_hex_dump(KERN_INFO, "",
454 					DUMP_PREFIX_ADDRESS,
455 					16, 1, buffer_info->skb->data,
456 					dma_unmap_len(buffer_info, len),
457 					true);
458 		}
459 	}
460 
461 	/* Print RX Rings Summary */
462 rx_ring_summary:
463 	dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
464 	pr_info("Queue [NTU] [NTC]\n");
465 	for (n = 0; n < adapter->num_rx_queues; n++) {
466 		rx_ring = adapter->rx_ring[n];
467 		pr_info(" %5d %5X %5X\n",
468 			n, rx_ring->next_to_use, rx_ring->next_to_clean);
469 	}
470 
471 	/* Print RX Rings */
472 	if (!netif_msg_rx_status(adapter))
473 		goto exit;
474 
475 	dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
476 
477 	/* Advanced Receive Descriptor (Read) Format
478 	 *    63                                           1        0
479 	 *    +-----------------------------------------------------+
480 	 *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
481 	 *    +----------------------------------------------+------+
482 	 *  8 |       Header Buffer Address [63:1]           |  DD  |
483 	 *    +-----------------------------------------------------+
484 	 *
485 	 *
486 	 * Advanced Receive Descriptor (Write-Back) Format
487 	 *
488 	 *   63       48 47    32 31  30      21 20 17 16   4 3     0
489 	 *   +------------------------------------------------------+
490 	 * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
491 	 *   | Checksum   Ident  |   |           |    | Type | Type |
492 	 *   +------------------------------------------------------+
493 	 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
494 	 *   +------------------------------------------------------+
495 	 *   63       48 47    32 31            20 19               0
496 	 */
497 
498 	for (n = 0; n < adapter->num_rx_queues; n++) {
499 		rx_ring = adapter->rx_ring[n];
500 		pr_info("------------------------------------\n");
501 		pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
502 		pr_info("------------------------------------\n");
503 		pr_info("R  [desc]      [ PktBuf     A0] [  HeadBuf   DD] [bi->dma       ] [bi->skb] <-- Adv Rx Read format\n");
504 		pr_info("RWB[desc]      [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
505 
506 		for (i = 0; i < rx_ring->count; i++) {
507 			const char *next_desc;
508 			struct igb_rx_buffer *buffer_info;
509 			buffer_info = &rx_ring->rx_buffer_info[i];
510 			rx_desc = IGB_RX_DESC(rx_ring, i);
511 			u0 = (struct my_u0 *)rx_desc;
512 			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
513 
514 			if (i == rx_ring->next_to_use)
515 				next_desc = " NTU";
516 			else if (i == rx_ring->next_to_clean)
517 				next_desc = " NTC";
518 			else
519 				next_desc = "";
520 
521 			if (staterr & E1000_RXD_STAT_DD) {
522 				/* Descriptor Done */
523 				pr_info("%s[0x%03X]     %016llX %016llX ---------------- %s\n",
524 					"RWB", i,
525 					le64_to_cpu(u0->a),
526 					le64_to_cpu(u0->b),
527 					next_desc);
528 			} else {
529 				pr_info("%s[0x%03X]     %016llX %016llX %016llX %s\n",
530 					"R  ", i,
531 					le64_to_cpu(u0->a),
532 					le64_to_cpu(u0->b),
533 					(u64)buffer_info->dma,
534 					next_desc);
535 
536 				if (netif_msg_pktdata(adapter) &&
537 				    buffer_info->dma && buffer_info->page) {
538 					print_hex_dump(KERN_INFO, "",
539 					  DUMP_PREFIX_ADDRESS,
540 					  16, 1,
541 					  page_address(buffer_info->page) +
542 						      buffer_info->page_offset,
543 					  igb_rx_bufsz(rx_ring), true);
544 				}
545 			}
546 		}
547 	}
548 
549 exit:
550 	return;
551 }
552 
553 /**
554  *  igb_get_i2c_data - Reads the I2C SDA data bit
555  *  @data: opaque pointer to adapter struct
556  *
557  *  Returns the I2C data bit value
558  **/
igb_get_i2c_data(void * data)559 static int igb_get_i2c_data(void *data)
560 {
561 	struct igb_adapter *adapter = (struct igb_adapter *)data;
562 	struct e1000_hw *hw = &adapter->hw;
563 	s32 i2cctl = rd32(E1000_I2CPARAMS);
564 
565 	return !!(i2cctl & E1000_I2C_DATA_IN);
566 }
567 
568 /**
569  *  igb_set_i2c_data - Sets the I2C data bit
570  *  @data: pointer to hardware structure
571  *  @state: I2C data value (0 or 1) to set
572  *
573  *  Sets the I2C data bit
574  **/
igb_set_i2c_data(void * data,int state)575 static void igb_set_i2c_data(void *data, int state)
576 {
577 	struct igb_adapter *adapter = (struct igb_adapter *)data;
578 	struct e1000_hw *hw = &adapter->hw;
579 	s32 i2cctl = rd32(E1000_I2CPARAMS);
580 
581 	if (state) {
582 		i2cctl |= E1000_I2C_DATA_OUT | E1000_I2C_DATA_OE_N;
583 	} else {
584 		i2cctl &= ~E1000_I2C_DATA_OE_N;
585 		i2cctl &= ~E1000_I2C_DATA_OUT;
586 	}
587 
588 	wr32(E1000_I2CPARAMS, i2cctl);
589 	wrfl();
590 }
591 
592 /**
593  *  igb_set_i2c_clk - Sets the I2C SCL clock
594  *  @data: pointer to hardware structure
595  *  @state: state to set clock
596  *
597  *  Sets the I2C clock line to state
598  **/
igb_set_i2c_clk(void * data,int state)599 static void igb_set_i2c_clk(void *data, int state)
600 {
601 	struct igb_adapter *adapter = (struct igb_adapter *)data;
602 	struct e1000_hw *hw = &adapter->hw;
603 	s32 i2cctl = rd32(E1000_I2CPARAMS);
604 
605 	if (state) {
606 		i2cctl |= E1000_I2C_CLK_OUT | E1000_I2C_CLK_OE_N;
607 	} else {
608 		i2cctl &= ~E1000_I2C_CLK_OUT;
609 		i2cctl &= ~E1000_I2C_CLK_OE_N;
610 	}
611 	wr32(E1000_I2CPARAMS, i2cctl);
612 	wrfl();
613 }
614 
615 /**
616  *  igb_get_i2c_clk - Gets the I2C SCL clock state
617  *  @data: pointer to hardware structure
618  *
619  *  Gets the I2C clock state
620  **/
igb_get_i2c_clk(void * data)621 static int igb_get_i2c_clk(void *data)
622 {
623 	struct igb_adapter *adapter = (struct igb_adapter *)data;
624 	struct e1000_hw *hw = &adapter->hw;
625 	s32 i2cctl = rd32(E1000_I2CPARAMS);
626 
627 	return !!(i2cctl & E1000_I2C_CLK_IN);
628 }
629 
630 static const struct i2c_algo_bit_data igb_i2c_algo = {
631 	.setsda		= igb_set_i2c_data,
632 	.setscl		= igb_set_i2c_clk,
633 	.getsda		= igb_get_i2c_data,
634 	.getscl		= igb_get_i2c_clk,
635 	.udelay		= 5,
636 	.timeout	= 20,
637 };
638 
639 /**
640  *  igb_get_hw_dev - return device
641  *  @hw: pointer to hardware structure
642  *
643  *  used by hardware layer to print debugging information
644  **/
igb_get_hw_dev(struct e1000_hw * hw)645 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
646 {
647 	struct igb_adapter *adapter = hw->back;
648 	return adapter->netdev;
649 }
650 
651 /**
652  *  igb_init_module - Driver Registration Routine
653  *
654  *  igb_init_module is the first routine called when the driver is
655  *  loaded. All it does is register with the PCI subsystem.
656  **/
igb_init_module(void)657 static int __init igb_init_module(void)
658 {
659 	int ret;
660 
661 	pr_info("%s\n", igb_driver_string);
662 	pr_info("%s\n", igb_copyright);
663 
664 #ifdef CONFIG_IGB_DCA
665 	dca_register_notify(&dca_notifier);
666 #endif
667 	ret = pci_register_driver(&igb_driver);
668 	return ret;
669 }
670 
671 module_init(igb_init_module);
672 
673 /**
674  *  igb_exit_module - Driver Exit Cleanup Routine
675  *
676  *  igb_exit_module is called just before the driver is removed
677  *  from memory.
678  **/
igb_exit_module(void)679 static void __exit igb_exit_module(void)
680 {
681 #ifdef CONFIG_IGB_DCA
682 	dca_unregister_notify(&dca_notifier);
683 #endif
684 	pci_unregister_driver(&igb_driver);
685 }
686 
687 module_exit(igb_exit_module);
688 
689 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
690 /**
691  *  igb_cache_ring_register - Descriptor ring to register mapping
692  *  @adapter: board private structure to initialize
693  *
694  *  Once we know the feature-set enabled for the device, we'll cache
695  *  the register offset the descriptor ring is assigned to.
696  **/
igb_cache_ring_register(struct igb_adapter * adapter)697 static void igb_cache_ring_register(struct igb_adapter *adapter)
698 {
699 	int i = 0, j = 0;
700 	u32 rbase_offset = adapter->vfs_allocated_count;
701 
702 	switch (adapter->hw.mac.type) {
703 	case e1000_82576:
704 		/* The queues are allocated for virtualization such that VF 0
705 		 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
706 		 * In order to avoid collision we start at the first free queue
707 		 * and continue consuming queues in the same sequence
708 		 */
709 		if (adapter->vfs_allocated_count) {
710 			for (; i < adapter->rss_queues; i++)
711 				adapter->rx_ring[i]->reg_idx = rbase_offset +
712 							       Q_IDX_82576(i);
713 		}
714 		fallthrough;
715 	case e1000_82575:
716 	case e1000_82580:
717 	case e1000_i350:
718 	case e1000_i354:
719 	case e1000_i210:
720 	case e1000_i211:
721 	default:
722 		for (; i < adapter->num_rx_queues; i++)
723 			adapter->rx_ring[i]->reg_idx = rbase_offset + i;
724 		for (; j < adapter->num_tx_queues; j++)
725 			adapter->tx_ring[j]->reg_idx = rbase_offset + j;
726 		break;
727 	}
728 }
729 
igb_rd32(struct e1000_hw * hw,u32 reg)730 u32 igb_rd32(struct e1000_hw *hw, u32 reg)
731 {
732 	struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
733 	u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
734 	u32 value = 0;
735 
736 	if (E1000_REMOVED(hw_addr))
737 		return ~value;
738 
739 	value = readl(&hw_addr[reg]);
740 
741 	/* reads should not return all F's */
742 	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
743 		struct net_device *netdev = igb->netdev;
744 		hw->hw_addr = NULL;
745 		netdev_err(netdev, "PCIe link lost\n");
746 		WARN(pci_device_is_present(igb->pdev),
747 		     "igb: Failed to read reg 0x%x!\n", reg);
748 	}
749 
750 	return value;
751 }
752 
753 /**
754  *  igb_write_ivar - configure ivar for given MSI-X vector
755  *  @hw: pointer to the HW structure
756  *  @msix_vector: vector number we are allocating to a given ring
757  *  @index: row index of IVAR register to write within IVAR table
758  *  @offset: column offset of in IVAR, should be multiple of 8
759  *
760  *  This function is intended to handle the writing of the IVAR register
761  *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
762  *  each containing an cause allocation for an Rx and Tx ring, and a
763  *  variable number of rows depending on the number of queues supported.
764  **/
igb_write_ivar(struct e1000_hw * hw,int msix_vector,int index,int offset)765 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
766 			   int index, int offset)
767 {
768 	u32 ivar = array_rd32(E1000_IVAR0, index);
769 
770 	/* clear any bits that are currently set */
771 	ivar &= ~((u32)0xFF << offset);
772 
773 	/* write vector and valid bit */
774 	ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
775 
776 	array_wr32(E1000_IVAR0, index, ivar);
777 }
778 
779 #define IGB_N0_QUEUE -1
igb_assign_vector(struct igb_q_vector * q_vector,int msix_vector)780 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
781 {
782 	struct igb_adapter *adapter = q_vector->adapter;
783 	struct e1000_hw *hw = &adapter->hw;
784 	int rx_queue = IGB_N0_QUEUE;
785 	int tx_queue = IGB_N0_QUEUE;
786 	u32 msixbm = 0;
787 
788 	if (q_vector->rx.ring)
789 		rx_queue = q_vector->rx.ring->reg_idx;
790 	if (q_vector->tx.ring)
791 		tx_queue = q_vector->tx.ring->reg_idx;
792 
793 	switch (hw->mac.type) {
794 	case e1000_82575:
795 		/* The 82575 assigns vectors using a bitmask, which matches the
796 		 * bitmask for the EICR/EIMS/EIMC registers.  To assign one
797 		 * or more queues to a vector, we write the appropriate bits
798 		 * into the MSIXBM register for that vector.
799 		 */
800 		if (rx_queue > IGB_N0_QUEUE)
801 			msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
802 		if (tx_queue > IGB_N0_QUEUE)
803 			msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
804 		if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
805 			msixbm |= E1000_EIMS_OTHER;
806 		array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
807 		q_vector->eims_value = msixbm;
808 		break;
809 	case e1000_82576:
810 		/* 82576 uses a table that essentially consists of 2 columns
811 		 * with 8 rows.  The ordering is column-major so we use the
812 		 * lower 3 bits as the row index, and the 4th bit as the
813 		 * column offset.
814 		 */
815 		if (rx_queue > IGB_N0_QUEUE)
816 			igb_write_ivar(hw, msix_vector,
817 				       rx_queue & 0x7,
818 				       (rx_queue & 0x8) << 1);
819 		if (tx_queue > IGB_N0_QUEUE)
820 			igb_write_ivar(hw, msix_vector,
821 				       tx_queue & 0x7,
822 				       ((tx_queue & 0x8) << 1) + 8);
823 		q_vector->eims_value = BIT(msix_vector);
824 		break;
825 	case e1000_82580:
826 	case e1000_i350:
827 	case e1000_i354:
828 	case e1000_i210:
829 	case e1000_i211:
830 		/* On 82580 and newer adapters the scheme is similar to 82576
831 		 * however instead of ordering column-major we have things
832 		 * ordered row-major.  So we traverse the table by using
833 		 * bit 0 as the column offset, and the remaining bits as the
834 		 * row index.
835 		 */
836 		if (rx_queue > IGB_N0_QUEUE)
837 			igb_write_ivar(hw, msix_vector,
838 				       rx_queue >> 1,
839 				       (rx_queue & 0x1) << 4);
840 		if (tx_queue > IGB_N0_QUEUE)
841 			igb_write_ivar(hw, msix_vector,
842 				       tx_queue >> 1,
843 				       ((tx_queue & 0x1) << 4) + 8);
844 		q_vector->eims_value = BIT(msix_vector);
845 		break;
846 	default:
847 		BUG();
848 		break;
849 	}
850 
851 	/* add q_vector eims value to global eims_enable_mask */
852 	adapter->eims_enable_mask |= q_vector->eims_value;
853 
854 	/* configure q_vector to set itr on first interrupt */
855 	q_vector->set_itr = 1;
856 }
857 
858 /**
859  *  igb_configure_msix - Configure MSI-X hardware
860  *  @adapter: board private structure to initialize
861  *
862  *  igb_configure_msix sets up the hardware to properly
863  *  generate MSI-X interrupts.
864  **/
igb_configure_msix(struct igb_adapter * adapter)865 static void igb_configure_msix(struct igb_adapter *adapter)
866 {
867 	u32 tmp;
868 	int i, vector = 0;
869 	struct e1000_hw *hw = &adapter->hw;
870 
871 	adapter->eims_enable_mask = 0;
872 
873 	/* set vector for other causes, i.e. link changes */
874 	switch (hw->mac.type) {
875 	case e1000_82575:
876 		tmp = rd32(E1000_CTRL_EXT);
877 		/* enable MSI-X PBA support*/
878 		tmp |= E1000_CTRL_EXT_PBA_CLR;
879 
880 		/* Auto-Mask interrupts upon ICR read. */
881 		tmp |= E1000_CTRL_EXT_EIAME;
882 		tmp |= E1000_CTRL_EXT_IRCA;
883 
884 		wr32(E1000_CTRL_EXT, tmp);
885 
886 		/* enable msix_other interrupt */
887 		array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
888 		adapter->eims_other = E1000_EIMS_OTHER;
889 
890 		break;
891 
892 	case e1000_82576:
893 	case e1000_82580:
894 	case e1000_i350:
895 	case e1000_i354:
896 	case e1000_i210:
897 	case e1000_i211:
898 		/* Turn on MSI-X capability first, or our settings
899 		 * won't stick.  And it will take days to debug.
900 		 */
901 		wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
902 		     E1000_GPIE_PBA | E1000_GPIE_EIAME |
903 		     E1000_GPIE_NSICR);
904 
905 		/* enable msix_other interrupt */
906 		adapter->eims_other = BIT(vector);
907 		tmp = (vector++ | E1000_IVAR_VALID) << 8;
908 
909 		wr32(E1000_IVAR_MISC, tmp);
910 		break;
911 	default:
912 		/* do nothing, since nothing else supports MSI-X */
913 		break;
914 	} /* switch (hw->mac.type) */
915 
916 	adapter->eims_enable_mask |= adapter->eims_other;
917 
918 	for (i = 0; i < adapter->num_q_vectors; i++)
919 		igb_assign_vector(adapter->q_vector[i], vector++);
920 
921 	wrfl();
922 }
923 
924 /**
925  *  igb_request_msix - Initialize MSI-X interrupts
926  *  @adapter: board private structure to initialize
927  *
928  *  igb_request_msix allocates MSI-X vectors and requests interrupts from the
929  *  kernel.
930  **/
igb_request_msix(struct igb_adapter * adapter)931 static int igb_request_msix(struct igb_adapter *adapter)
932 {
933 	unsigned int num_q_vectors = adapter->num_q_vectors;
934 	struct net_device *netdev = adapter->netdev;
935 	int i, err = 0, vector = 0, free_vector = 0;
936 
937 	err = request_irq(adapter->msix_entries[vector].vector,
938 			  igb_msix_other, 0, netdev->name, adapter);
939 	if (err)
940 		goto err_out;
941 
942 	if (num_q_vectors > MAX_Q_VECTORS) {
943 		num_q_vectors = MAX_Q_VECTORS;
944 		dev_warn(&adapter->pdev->dev,
945 			 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
946 			 adapter->num_q_vectors, MAX_Q_VECTORS);
947 	}
948 	for (i = 0; i < num_q_vectors; i++) {
949 		struct igb_q_vector *q_vector = adapter->q_vector[i];
950 
951 		vector++;
952 
953 		q_vector->itr_register = adapter->io_addr + E1000_EITR(vector);
954 
955 		if (q_vector->rx.ring && q_vector->tx.ring)
956 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
957 				q_vector->rx.ring->queue_index);
958 		else if (q_vector->tx.ring)
959 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,
960 				q_vector->tx.ring->queue_index);
961 		else if (q_vector->rx.ring)
962 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,
963 				q_vector->rx.ring->queue_index);
964 		else
965 			sprintf(q_vector->name, "%s-unused", netdev->name);
966 
967 		err = request_irq(adapter->msix_entries[vector].vector,
968 				  igb_msix_ring, 0, q_vector->name,
969 				  q_vector);
970 		if (err)
971 			goto err_free;
972 	}
973 
974 	igb_configure_msix(adapter);
975 	return 0;
976 
977 err_free:
978 	/* free already assigned IRQs */
979 	free_irq(adapter->msix_entries[free_vector++].vector, adapter);
980 
981 	vector--;
982 	for (i = 0; i < vector; i++) {
983 		free_irq(adapter->msix_entries[free_vector++].vector,
984 			 adapter->q_vector[i]);
985 	}
986 err_out:
987 	return err;
988 }
989 
990 /**
991  *  igb_free_q_vector - Free memory allocated for specific interrupt vector
992  *  @adapter: board private structure to initialize
993  *  @v_idx: Index of vector to be freed
994  *
995  *  This function frees the memory allocated to the q_vector.
996  **/
igb_free_q_vector(struct igb_adapter * adapter,int v_idx)997 static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
998 {
999 	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1000 
1001 	adapter->q_vector[v_idx] = NULL;
1002 
1003 	/* igb_get_stats64() might access the rings on this vector,
1004 	 * we must wait a grace period before freeing it.
1005 	 */
1006 	if (q_vector)
1007 		kfree_rcu(q_vector, rcu);
1008 }
1009 
1010 /**
1011  *  igb_reset_q_vector - Reset config for interrupt vector
1012  *  @adapter: board private structure to initialize
1013  *  @v_idx: Index of vector to be reset
1014  *
1015  *  If NAPI is enabled it will delete any references to the
1016  *  NAPI struct. This is preparation for igb_free_q_vector.
1017  **/
igb_reset_q_vector(struct igb_adapter * adapter,int v_idx)1018 static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
1019 {
1020 	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1021 
1022 	/* Coming from igb_set_interrupt_capability, the vectors are not yet
1023 	 * allocated. So, q_vector is NULL so we should stop here.
1024 	 */
1025 	if (!q_vector)
1026 		return;
1027 
1028 	if (q_vector->tx.ring)
1029 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
1030 
1031 	if (q_vector->rx.ring)
1032 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
1033 
1034 	netif_napi_del(&q_vector->napi);
1035 
1036 }
1037 
igb_reset_interrupt_capability(struct igb_adapter * adapter)1038 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
1039 {
1040 	int v_idx = adapter->num_q_vectors;
1041 
1042 	if (adapter->flags & IGB_FLAG_HAS_MSIX)
1043 		pci_disable_msix(adapter->pdev);
1044 	else if (adapter->flags & IGB_FLAG_HAS_MSI)
1045 		pci_disable_msi(adapter->pdev);
1046 
1047 	while (v_idx--)
1048 		igb_reset_q_vector(adapter, v_idx);
1049 }
1050 
1051 /**
1052  *  igb_free_q_vectors - Free memory allocated for interrupt vectors
1053  *  @adapter: board private structure to initialize
1054  *
1055  *  This function frees the memory allocated to the q_vectors.  In addition if
1056  *  NAPI is enabled it will delete any references to the NAPI struct prior
1057  *  to freeing the q_vector.
1058  **/
igb_free_q_vectors(struct igb_adapter * adapter)1059 static void igb_free_q_vectors(struct igb_adapter *adapter)
1060 {
1061 	int v_idx = adapter->num_q_vectors;
1062 
1063 	adapter->num_tx_queues = 0;
1064 	adapter->num_rx_queues = 0;
1065 	adapter->num_q_vectors = 0;
1066 
1067 	while (v_idx--) {
1068 		igb_reset_q_vector(adapter, v_idx);
1069 		igb_free_q_vector(adapter, v_idx);
1070 	}
1071 }
1072 
1073 /**
1074  *  igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1075  *  @adapter: board private structure to initialize
1076  *
1077  *  This function resets the device so that it has 0 Rx queues, Tx queues, and
1078  *  MSI-X interrupts allocated.
1079  */
igb_clear_interrupt_scheme(struct igb_adapter * adapter)1080 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1081 {
1082 	igb_free_q_vectors(adapter);
1083 	igb_reset_interrupt_capability(adapter);
1084 }
1085 
1086 /**
1087  *  igb_set_interrupt_capability - set MSI or MSI-X if supported
1088  *  @adapter: board private structure to initialize
1089  *  @msix: boolean value of MSIX capability
1090  *
1091  *  Attempt to configure interrupts using the best available
1092  *  capabilities of the hardware and kernel.
1093  **/
igb_set_interrupt_capability(struct igb_adapter * adapter,bool msix)1094 static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
1095 {
1096 	int err;
1097 	int numvecs, i;
1098 
1099 	if (!msix)
1100 		goto msi_only;
1101 	adapter->flags |= IGB_FLAG_HAS_MSIX;
1102 
1103 	/* Number of supported queues. */
1104 	adapter->num_rx_queues = adapter->rss_queues;
1105 	if (adapter->vfs_allocated_count)
1106 		adapter->num_tx_queues = 1;
1107 	else
1108 		adapter->num_tx_queues = adapter->rss_queues;
1109 
1110 	/* start with one vector for every Rx queue */
1111 	numvecs = adapter->num_rx_queues;
1112 
1113 	/* if Tx handler is separate add 1 for every Tx queue */
1114 	if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1115 		numvecs += adapter->num_tx_queues;
1116 
1117 	/* store the number of vectors reserved for queues */
1118 	adapter->num_q_vectors = numvecs;
1119 
1120 	/* add 1 vector for link status interrupts */
1121 	numvecs++;
1122 	for (i = 0; i < numvecs; i++)
1123 		adapter->msix_entries[i].entry = i;
1124 
1125 	err = pci_enable_msix_range(adapter->pdev,
1126 				    adapter->msix_entries,
1127 				    numvecs,
1128 				    numvecs);
1129 	if (err > 0)
1130 		return;
1131 
1132 	igb_reset_interrupt_capability(adapter);
1133 
1134 	/* If we can't do MSI-X, try MSI */
1135 msi_only:
1136 	adapter->flags &= ~IGB_FLAG_HAS_MSIX;
1137 #ifdef CONFIG_PCI_IOV
1138 	/* disable SR-IOV for non MSI-X configurations */
1139 	if (adapter->vf_data) {
1140 		struct e1000_hw *hw = &adapter->hw;
1141 		/* disable iov and allow time for transactions to clear */
1142 		pci_disable_sriov(adapter->pdev);
1143 		msleep(500);
1144 
1145 		kfree(adapter->vf_mac_list);
1146 		adapter->vf_mac_list = NULL;
1147 		kfree(adapter->vf_data);
1148 		adapter->vf_data = NULL;
1149 		wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1150 		wrfl();
1151 		msleep(100);
1152 		dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1153 	}
1154 #endif
1155 	adapter->vfs_allocated_count = 0;
1156 	adapter->rss_queues = 1;
1157 	adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1158 	adapter->num_rx_queues = 1;
1159 	adapter->num_tx_queues = 1;
1160 	adapter->num_q_vectors = 1;
1161 	if (!pci_enable_msi(adapter->pdev))
1162 		adapter->flags |= IGB_FLAG_HAS_MSI;
1163 }
1164 
igb_add_ring(struct igb_ring * ring,struct igb_ring_container * head)1165 static void igb_add_ring(struct igb_ring *ring,
1166 			 struct igb_ring_container *head)
1167 {
1168 	head->ring = ring;
1169 	head->count++;
1170 }
1171 
1172 /**
1173  *  igb_alloc_q_vector - Allocate memory for a single interrupt vector
1174  *  @adapter: board private structure to initialize
1175  *  @v_count: q_vectors allocated on adapter, used for ring interleaving
1176  *  @v_idx: index of vector in adapter struct
1177  *  @txr_count: total number of Tx rings to allocate
1178  *  @txr_idx: index of first Tx ring to allocate
1179  *  @rxr_count: total number of Rx rings to allocate
1180  *  @rxr_idx: index of first Rx ring to allocate
1181  *
1182  *  We allocate one q_vector.  If allocation fails we return -ENOMEM.
1183  **/
igb_alloc_q_vector(struct igb_adapter * adapter,int v_count,int v_idx,int txr_count,int txr_idx,int rxr_count,int rxr_idx)1184 static int igb_alloc_q_vector(struct igb_adapter *adapter,
1185 			      int v_count, int v_idx,
1186 			      int txr_count, int txr_idx,
1187 			      int rxr_count, int rxr_idx)
1188 {
1189 	struct igb_q_vector *q_vector;
1190 	struct igb_ring *ring;
1191 	int ring_count;
1192 	size_t size;
1193 
1194 	/* igb only supports 1 Tx and/or 1 Rx queue per vector */
1195 	if (txr_count > 1 || rxr_count > 1)
1196 		return -ENOMEM;
1197 
1198 	ring_count = txr_count + rxr_count;
1199 	size = kmalloc_size_roundup(struct_size(q_vector, ring, ring_count));
1200 
1201 	/* allocate q_vector and rings */
1202 	q_vector = adapter->q_vector[v_idx];
1203 	if (!q_vector) {
1204 		q_vector = kzalloc(size, GFP_KERNEL);
1205 	} else if (size > ksize(q_vector)) {
1206 		struct igb_q_vector *new_q_vector;
1207 
1208 		new_q_vector = kzalloc(size, GFP_KERNEL);
1209 		if (new_q_vector)
1210 			kfree_rcu(q_vector, rcu);
1211 		q_vector = new_q_vector;
1212 	} else {
1213 		memset(q_vector, 0, size);
1214 	}
1215 	if (!q_vector)
1216 		return -ENOMEM;
1217 
1218 	/* initialize NAPI */
1219 	netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll);
1220 
1221 	/* tie q_vector and adapter together */
1222 	adapter->q_vector[v_idx] = q_vector;
1223 	q_vector->adapter = adapter;
1224 
1225 	/* initialize work limits */
1226 	q_vector->tx.work_limit = adapter->tx_work_limit;
1227 
1228 	/* initialize ITR configuration */
1229 	q_vector->itr_register = adapter->io_addr + E1000_EITR(0);
1230 	q_vector->itr_val = IGB_START_ITR;
1231 
1232 	/* initialize pointer to rings */
1233 	ring = q_vector->ring;
1234 
1235 	/* intialize ITR */
1236 	if (rxr_count) {
1237 		/* rx or rx/tx vector */
1238 		if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
1239 			q_vector->itr_val = adapter->rx_itr_setting;
1240 	} else {
1241 		/* tx only vector */
1242 		if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
1243 			q_vector->itr_val = adapter->tx_itr_setting;
1244 	}
1245 
1246 	if (txr_count) {
1247 		/* assign generic ring traits */
1248 		ring->dev = &adapter->pdev->dev;
1249 		ring->netdev = adapter->netdev;
1250 
1251 		/* configure backlink on ring */
1252 		ring->q_vector = q_vector;
1253 
1254 		/* update q_vector Tx values */
1255 		igb_add_ring(ring, &q_vector->tx);
1256 
1257 		/* For 82575, context index must be unique per ring. */
1258 		if (adapter->hw.mac.type == e1000_82575)
1259 			set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
1260 
1261 		/* apply Tx specific ring traits */
1262 		ring->count = adapter->tx_ring_count;
1263 		ring->queue_index = txr_idx;
1264 
1265 		ring->cbs_enable = false;
1266 		ring->idleslope = 0;
1267 		ring->sendslope = 0;
1268 		ring->hicredit = 0;
1269 		ring->locredit = 0;
1270 
1271 		u64_stats_init(&ring->tx_syncp);
1272 		u64_stats_init(&ring->tx_syncp2);
1273 
1274 		/* assign ring to adapter */
1275 		adapter->tx_ring[txr_idx] = ring;
1276 
1277 		/* push pointer to next ring */
1278 		ring++;
1279 	}
1280 
1281 	if (rxr_count) {
1282 		/* assign generic ring traits */
1283 		ring->dev = &adapter->pdev->dev;
1284 		ring->netdev = adapter->netdev;
1285 
1286 		/* configure backlink on ring */
1287 		ring->q_vector = q_vector;
1288 
1289 		/* update q_vector Rx values */
1290 		igb_add_ring(ring, &q_vector->rx);
1291 
1292 		/* set flag indicating ring supports SCTP checksum offload */
1293 		if (adapter->hw.mac.type >= e1000_82576)
1294 			set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1295 
1296 		/* On i350, i354, i210, and i211, loopback VLAN packets
1297 		 * have the tag byte-swapped.
1298 		 */
1299 		if (adapter->hw.mac.type >= e1000_i350)
1300 			set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1301 
1302 		/* apply Rx specific ring traits */
1303 		ring->count = adapter->rx_ring_count;
1304 		ring->queue_index = rxr_idx;
1305 
1306 		u64_stats_init(&ring->rx_syncp);
1307 
1308 		/* assign ring to adapter */
1309 		adapter->rx_ring[rxr_idx] = ring;
1310 	}
1311 
1312 	return 0;
1313 }
1314 
1315 
1316 /**
1317  *  igb_alloc_q_vectors - Allocate memory for interrupt vectors
1318  *  @adapter: board private structure to initialize
1319  *
1320  *  We allocate one q_vector per queue interrupt.  If allocation fails we
1321  *  return -ENOMEM.
1322  **/
igb_alloc_q_vectors(struct igb_adapter * adapter)1323 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1324 {
1325 	int q_vectors = adapter->num_q_vectors;
1326 	int rxr_remaining = adapter->num_rx_queues;
1327 	int txr_remaining = adapter->num_tx_queues;
1328 	int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1329 	int err;
1330 
1331 	if (q_vectors >= (rxr_remaining + txr_remaining)) {
1332 		for (; rxr_remaining; v_idx++) {
1333 			err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1334 						 0, 0, 1, rxr_idx);
1335 
1336 			if (err)
1337 				goto err_out;
1338 
1339 			/* update counts and index */
1340 			rxr_remaining--;
1341 			rxr_idx++;
1342 		}
1343 	}
1344 
1345 	for (; v_idx < q_vectors; v_idx++) {
1346 		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1347 		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1348 
1349 		err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1350 					 tqpv, txr_idx, rqpv, rxr_idx);
1351 
1352 		if (err)
1353 			goto err_out;
1354 
1355 		/* update counts and index */
1356 		rxr_remaining -= rqpv;
1357 		txr_remaining -= tqpv;
1358 		rxr_idx++;
1359 		txr_idx++;
1360 	}
1361 
1362 	return 0;
1363 
1364 err_out:
1365 	adapter->num_tx_queues = 0;
1366 	adapter->num_rx_queues = 0;
1367 	adapter->num_q_vectors = 0;
1368 
1369 	while (v_idx--)
1370 		igb_free_q_vector(adapter, v_idx);
1371 
1372 	return -ENOMEM;
1373 }
1374 
1375 /**
1376  *  igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1377  *  @adapter: board private structure to initialize
1378  *  @msix: boolean value of MSIX capability
1379  *
1380  *  This function initializes the interrupts and allocates all of the queues.
1381  **/
igb_init_interrupt_scheme(struct igb_adapter * adapter,bool msix)1382 static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1383 {
1384 	struct pci_dev *pdev = adapter->pdev;
1385 	int err;
1386 
1387 	igb_set_interrupt_capability(adapter, msix);
1388 
1389 	err = igb_alloc_q_vectors(adapter);
1390 	if (err) {
1391 		dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1392 		goto err_alloc_q_vectors;
1393 	}
1394 
1395 	igb_cache_ring_register(adapter);
1396 
1397 	return 0;
1398 
1399 err_alloc_q_vectors:
1400 	igb_reset_interrupt_capability(adapter);
1401 	return err;
1402 }
1403 
1404 /**
1405  *  igb_request_irq - initialize interrupts
1406  *  @adapter: board private structure to initialize
1407  *
1408  *  Attempts to configure interrupts using the best available
1409  *  capabilities of the hardware and kernel.
1410  **/
igb_request_irq(struct igb_adapter * adapter)1411 static int igb_request_irq(struct igb_adapter *adapter)
1412 {
1413 	struct net_device *netdev = adapter->netdev;
1414 	struct pci_dev *pdev = adapter->pdev;
1415 	int err = 0;
1416 
1417 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1418 		err = igb_request_msix(adapter);
1419 		if (!err)
1420 			goto request_done;
1421 		/* fall back to MSI */
1422 		igb_free_all_tx_resources(adapter);
1423 		igb_free_all_rx_resources(adapter);
1424 
1425 		igb_clear_interrupt_scheme(adapter);
1426 		err = igb_init_interrupt_scheme(adapter, false);
1427 		if (err)
1428 			goto request_done;
1429 
1430 		igb_setup_all_tx_resources(adapter);
1431 		igb_setup_all_rx_resources(adapter);
1432 		igb_configure(adapter);
1433 	}
1434 
1435 	igb_assign_vector(adapter->q_vector[0], 0);
1436 
1437 	if (adapter->flags & IGB_FLAG_HAS_MSI) {
1438 		err = request_irq(pdev->irq, igb_intr_msi, 0,
1439 				  netdev->name, adapter);
1440 		if (!err)
1441 			goto request_done;
1442 
1443 		/* fall back to legacy interrupts */
1444 		igb_reset_interrupt_capability(adapter);
1445 		adapter->flags &= ~IGB_FLAG_HAS_MSI;
1446 	}
1447 
1448 	err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1449 			  netdev->name, adapter);
1450 
1451 	if (err)
1452 		dev_err(&pdev->dev, "Error %d getting interrupt\n",
1453 			err);
1454 
1455 request_done:
1456 	return err;
1457 }
1458 
igb_free_irq(struct igb_adapter * adapter)1459 static void igb_free_irq(struct igb_adapter *adapter)
1460 {
1461 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1462 		int vector = 0, i;
1463 
1464 		free_irq(adapter->msix_entries[vector++].vector, adapter);
1465 
1466 		for (i = 0; i < adapter->num_q_vectors; i++)
1467 			free_irq(adapter->msix_entries[vector++].vector,
1468 				 adapter->q_vector[i]);
1469 	} else {
1470 		free_irq(adapter->pdev->irq, adapter);
1471 	}
1472 }
1473 
1474 /**
1475  *  igb_irq_disable - Mask off interrupt generation on the NIC
1476  *  @adapter: board private structure
1477  **/
igb_irq_disable(struct igb_adapter * adapter)1478 static void igb_irq_disable(struct igb_adapter *adapter)
1479 {
1480 	struct e1000_hw *hw = &adapter->hw;
1481 
1482 	/* we need to be careful when disabling interrupts.  The VFs are also
1483 	 * mapped into these registers and so clearing the bits can cause
1484 	 * issues on the VF drivers so we only need to clear what we set
1485 	 */
1486 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1487 		u32 regval = rd32(E1000_EIAM);
1488 
1489 		wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1490 		wr32(E1000_EIMC, adapter->eims_enable_mask);
1491 		regval = rd32(E1000_EIAC);
1492 		wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1493 	}
1494 
1495 	wr32(E1000_IAM, 0);
1496 	wr32(E1000_IMC, ~0);
1497 	wrfl();
1498 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1499 		int i;
1500 
1501 		for (i = 0; i < adapter->num_q_vectors; i++)
1502 			synchronize_irq(adapter->msix_entries[i].vector);
1503 	} else {
1504 		synchronize_irq(adapter->pdev->irq);
1505 	}
1506 }
1507 
1508 /**
1509  *  igb_irq_enable - Enable default interrupt generation settings
1510  *  @adapter: board private structure
1511  **/
igb_irq_enable(struct igb_adapter * adapter)1512 static void igb_irq_enable(struct igb_adapter *adapter)
1513 {
1514 	struct e1000_hw *hw = &adapter->hw;
1515 
1516 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1517 		u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1518 		u32 regval = rd32(E1000_EIAC);
1519 
1520 		wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1521 		regval = rd32(E1000_EIAM);
1522 		wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1523 		wr32(E1000_EIMS, adapter->eims_enable_mask);
1524 		if (adapter->vfs_allocated_count) {
1525 			wr32(E1000_MBVFIMR, 0xFF);
1526 			ims |= E1000_IMS_VMMB;
1527 		}
1528 		wr32(E1000_IMS, ims);
1529 	} else {
1530 		wr32(E1000_IMS, IMS_ENABLE_MASK |
1531 				E1000_IMS_DRSTA);
1532 		wr32(E1000_IAM, IMS_ENABLE_MASK |
1533 				E1000_IMS_DRSTA);
1534 	}
1535 }
1536 
igb_update_mng_vlan(struct igb_adapter * adapter)1537 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1538 {
1539 	struct e1000_hw *hw = &adapter->hw;
1540 	u16 pf_id = adapter->vfs_allocated_count;
1541 	u16 vid = adapter->hw.mng_cookie.vlan_id;
1542 	u16 old_vid = adapter->mng_vlan_id;
1543 
1544 	if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1545 		/* add VID to filter table */
1546 		igb_vfta_set(hw, vid, pf_id, true, true);
1547 		adapter->mng_vlan_id = vid;
1548 	} else {
1549 		adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1550 	}
1551 
1552 	if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1553 	    (vid != old_vid) &&
1554 	    !test_bit(old_vid, adapter->active_vlans)) {
1555 		/* remove VID from filter table */
1556 		igb_vfta_set(hw, vid, pf_id, false, true);
1557 	}
1558 }
1559 
1560 /**
1561  *  igb_release_hw_control - release control of the h/w to f/w
1562  *  @adapter: address of board private structure
1563  *
1564  *  igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1565  *  For ASF and Pass Through versions of f/w this means that the
1566  *  driver is no longer loaded.
1567  **/
igb_release_hw_control(struct igb_adapter * adapter)1568 static void igb_release_hw_control(struct igb_adapter *adapter)
1569 {
1570 	struct e1000_hw *hw = &adapter->hw;
1571 	u32 ctrl_ext;
1572 
1573 	/* Let firmware take over control of h/w */
1574 	ctrl_ext = rd32(E1000_CTRL_EXT);
1575 	wr32(E1000_CTRL_EXT,
1576 			ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1577 }
1578 
1579 /**
1580  *  igb_get_hw_control - get control of the h/w from f/w
1581  *  @adapter: address of board private structure
1582  *
1583  *  igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1584  *  For ASF and Pass Through versions of f/w this means that
1585  *  the driver is loaded.
1586  **/
igb_get_hw_control(struct igb_adapter * adapter)1587 static void igb_get_hw_control(struct igb_adapter *adapter)
1588 {
1589 	struct e1000_hw *hw = &adapter->hw;
1590 	u32 ctrl_ext;
1591 
1592 	/* Let firmware know the driver has taken over */
1593 	ctrl_ext = rd32(E1000_CTRL_EXT);
1594 	wr32(E1000_CTRL_EXT,
1595 			ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1596 }
1597 
enable_fqtss(struct igb_adapter * adapter,bool enable)1598 static void enable_fqtss(struct igb_adapter *adapter, bool enable)
1599 {
1600 	struct net_device *netdev = adapter->netdev;
1601 	struct e1000_hw *hw = &adapter->hw;
1602 
1603 	WARN_ON(hw->mac.type != e1000_i210);
1604 
1605 	if (enable)
1606 		adapter->flags |= IGB_FLAG_FQTSS;
1607 	else
1608 		adapter->flags &= ~IGB_FLAG_FQTSS;
1609 
1610 	if (netif_running(netdev))
1611 		schedule_work(&adapter->reset_task);
1612 }
1613 
is_fqtss_enabled(struct igb_adapter * adapter)1614 static bool is_fqtss_enabled(struct igb_adapter *adapter)
1615 {
1616 	return (adapter->flags & IGB_FLAG_FQTSS) ? true : false;
1617 }
1618 
set_tx_desc_fetch_prio(struct e1000_hw * hw,int queue,enum tx_queue_prio prio)1619 static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue,
1620 				   enum tx_queue_prio prio)
1621 {
1622 	u32 val;
1623 
1624 	WARN_ON(hw->mac.type != e1000_i210);
1625 	WARN_ON(queue < 0 || queue > 4);
1626 
1627 	val = rd32(E1000_I210_TXDCTL(queue));
1628 
1629 	if (prio == TX_QUEUE_PRIO_HIGH)
1630 		val |= E1000_TXDCTL_PRIORITY;
1631 	else
1632 		val &= ~E1000_TXDCTL_PRIORITY;
1633 
1634 	wr32(E1000_I210_TXDCTL(queue), val);
1635 }
1636 
set_queue_mode(struct e1000_hw * hw,int queue,enum queue_mode mode)1637 static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode)
1638 {
1639 	u32 val;
1640 
1641 	WARN_ON(hw->mac.type != e1000_i210);
1642 	WARN_ON(queue < 0 || queue > 1);
1643 
1644 	val = rd32(E1000_I210_TQAVCC(queue));
1645 
1646 	if (mode == QUEUE_MODE_STREAM_RESERVATION)
1647 		val |= E1000_TQAVCC_QUEUEMODE;
1648 	else
1649 		val &= ~E1000_TQAVCC_QUEUEMODE;
1650 
1651 	wr32(E1000_I210_TQAVCC(queue), val);
1652 }
1653 
is_any_cbs_enabled(struct igb_adapter * adapter)1654 static bool is_any_cbs_enabled(struct igb_adapter *adapter)
1655 {
1656 	int i;
1657 
1658 	for (i = 0; i < adapter->num_tx_queues; i++) {
1659 		if (adapter->tx_ring[i]->cbs_enable)
1660 			return true;
1661 	}
1662 
1663 	return false;
1664 }
1665 
is_any_txtime_enabled(struct igb_adapter * adapter)1666 static bool is_any_txtime_enabled(struct igb_adapter *adapter)
1667 {
1668 	int i;
1669 
1670 	for (i = 0; i < adapter->num_tx_queues; i++) {
1671 		if (adapter->tx_ring[i]->launchtime_enable)
1672 			return true;
1673 	}
1674 
1675 	return false;
1676 }
1677 
1678 /**
1679  *  igb_config_tx_modes - Configure "Qav Tx mode" features on igb
1680  *  @adapter: pointer to adapter struct
1681  *  @queue: queue number
1682  *
1683  *  Configure CBS and Launchtime for a given hardware queue.
1684  *  Parameters are retrieved from the correct Tx ring, so
1685  *  igb_save_cbs_params() and igb_save_txtime_params() should be used
1686  *  for setting those correctly prior to this function being called.
1687  **/
igb_config_tx_modes(struct igb_adapter * adapter,int queue)1688 static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
1689 {
1690 	struct net_device *netdev = adapter->netdev;
1691 	struct e1000_hw *hw = &adapter->hw;
1692 	struct igb_ring *ring;
1693 	u32 tqavcc, tqavctrl;
1694 	u16 value;
1695 
1696 	WARN_ON(hw->mac.type != e1000_i210);
1697 	WARN_ON(queue < 0 || queue > 1);
1698 	ring = adapter->tx_ring[queue];
1699 
1700 	/* If any of the Qav features is enabled, configure queues as SR and
1701 	 * with HIGH PRIO. If none is, then configure them with LOW PRIO and
1702 	 * as SP.
1703 	 */
1704 	if (ring->cbs_enable || ring->launchtime_enable) {
1705 		set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
1706 		set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
1707 	} else {
1708 		set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
1709 		set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
1710 	}
1711 
1712 	/* If CBS is enabled, set DataTranARB and config its parameters. */
1713 	if (ring->cbs_enable || queue == 0) {
1714 		/* i210 does not allow the queue 0 to be in the Strict
1715 		 * Priority mode while the Qav mode is enabled, so,
1716 		 * instead of disabling strict priority mode, we give
1717 		 * queue 0 the maximum of credits possible.
1718 		 *
1719 		 * See section 8.12.19 of the i210 datasheet, "Note:
1720 		 * Queue0 QueueMode must be set to 1b when
1721 		 * TransmitMode is set to Qav."
1722 		 */
1723 		if (queue == 0 && !ring->cbs_enable) {
1724 			/* max "linkspeed" idleslope in kbps */
1725 			ring->idleslope = 1000000;
1726 			ring->hicredit = ETH_FRAME_LEN;
1727 		}
1728 
1729 		/* Always set data transfer arbitration to credit-based
1730 		 * shaper algorithm on TQAVCTRL if CBS is enabled for any of
1731 		 * the queues.
1732 		 */
1733 		tqavctrl = rd32(E1000_I210_TQAVCTRL);
1734 		tqavctrl |= E1000_TQAVCTRL_DATATRANARB;
1735 		wr32(E1000_I210_TQAVCTRL, tqavctrl);
1736 
1737 		/* According to i210 datasheet section 7.2.7.7, we should set
1738 		 * the 'idleSlope' field from TQAVCC register following the
1739 		 * equation:
1740 		 *
1741 		 * For 100 Mbps link speed:
1742 		 *
1743 		 *     value = BW * 0x7735 * 0.2                          (E1)
1744 		 *
1745 		 * For 1000Mbps link speed:
1746 		 *
1747 		 *     value = BW * 0x7735 * 2                            (E2)
1748 		 *
1749 		 * E1 and E2 can be merged into one equation as shown below.
1750 		 * Note that 'link-speed' is in Mbps.
1751 		 *
1752 		 *     value = BW * 0x7735 * 2 * link-speed
1753 		 *                           --------------               (E3)
1754 		 *                                1000
1755 		 *
1756 		 * 'BW' is the percentage bandwidth out of full link speed
1757 		 * which can be found with the following equation. Note that
1758 		 * idleSlope here is the parameter from this function which
1759 		 * is in kbps.
1760 		 *
1761 		 *     BW =     idleSlope
1762 		 *          -----------------                             (E4)
1763 		 *          link-speed * 1000
1764 		 *
1765 		 * That said, we can come up with a generic equation to
1766 		 * calculate the value we should set it TQAVCC register by
1767 		 * replacing 'BW' in E3 by E4. The resulting equation is:
1768 		 *
1769 		 * value =     idleSlope     * 0x7735 * 2 * link-speed
1770 		 *         -----------------            --------------    (E5)
1771 		 *         link-speed * 1000                 1000
1772 		 *
1773 		 * 'link-speed' is present in both sides of the fraction so
1774 		 * it is canceled out. The final equation is the following:
1775 		 *
1776 		 *     value = idleSlope * 61034
1777 		 *             -----------------                          (E6)
1778 		 *                  1000000
1779 		 *
1780 		 * NOTE: For i210, given the above, we can see that idleslope
1781 		 *       is represented in 16.38431 kbps units by the value at
1782 		 *       the TQAVCC register (1Gbps / 61034), which reduces
1783 		 *       the granularity for idleslope increments.
1784 		 *       For instance, if you want to configure a 2576kbps
1785 		 *       idleslope, the value to be written on the register
1786 		 *       would have to be 157.23. If rounded down, you end
1787 		 *       up with less bandwidth available than originally
1788 		 *       required (~2572 kbps). If rounded up, you end up
1789 		 *       with a higher bandwidth (~2589 kbps). Below the
1790 		 *       approach we take is to always round up the
1791 		 *       calculated value, so the resulting bandwidth might
1792 		 *       be slightly higher for some configurations.
1793 		 */
1794 		value = DIV_ROUND_UP_ULL(ring->idleslope * 61034ULL, 1000000);
1795 
1796 		tqavcc = rd32(E1000_I210_TQAVCC(queue));
1797 		tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1798 		tqavcc |= value;
1799 		wr32(E1000_I210_TQAVCC(queue), tqavcc);
1800 
1801 		wr32(E1000_I210_TQAVHC(queue),
1802 		     0x80000000 + ring->hicredit * 0x7735);
1803 	} else {
1804 
1805 		/* Set idleSlope to zero. */
1806 		tqavcc = rd32(E1000_I210_TQAVCC(queue));
1807 		tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1808 		wr32(E1000_I210_TQAVCC(queue), tqavcc);
1809 
1810 		/* Set hiCredit to zero. */
1811 		wr32(E1000_I210_TQAVHC(queue), 0);
1812 
1813 		/* If CBS is not enabled for any queues anymore, then return to
1814 		 * the default state of Data Transmission Arbitration on
1815 		 * TQAVCTRL.
1816 		 */
1817 		if (!is_any_cbs_enabled(adapter)) {
1818 			tqavctrl = rd32(E1000_I210_TQAVCTRL);
1819 			tqavctrl &= ~E1000_TQAVCTRL_DATATRANARB;
1820 			wr32(E1000_I210_TQAVCTRL, tqavctrl);
1821 		}
1822 	}
1823 
1824 	/* If LaunchTime is enabled, set DataTranTIM. */
1825 	if (ring->launchtime_enable) {
1826 		/* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled
1827 		 * for any of the SR queues, and configure fetchtime delta.
1828 		 * XXX NOTE:
1829 		 *     - LaunchTime will be enabled for all SR queues.
1830 		 *     - A fixed offset can be added relative to the launch
1831 		 *       time of all packets if configured at reg LAUNCH_OS0.
1832 		 *       We are keeping it as 0 for now (default value).
1833 		 */
1834 		tqavctrl = rd32(E1000_I210_TQAVCTRL);
1835 		tqavctrl |= E1000_TQAVCTRL_DATATRANTIM |
1836 		       E1000_TQAVCTRL_FETCHTIME_DELTA;
1837 		wr32(E1000_I210_TQAVCTRL, tqavctrl);
1838 	} else {
1839 		/* If Launchtime is not enabled for any SR queues anymore,
1840 		 * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta,
1841 		 * effectively disabling Launchtime.
1842 		 */
1843 		if (!is_any_txtime_enabled(adapter)) {
1844 			tqavctrl = rd32(E1000_I210_TQAVCTRL);
1845 			tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM;
1846 			tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA;
1847 			wr32(E1000_I210_TQAVCTRL, tqavctrl);
1848 		}
1849 	}
1850 
1851 	/* XXX: In i210 controller the sendSlope and loCredit parameters from
1852 	 * CBS are not configurable by software so we don't do any 'controller
1853 	 * configuration' in respect to these parameters.
1854 	 */
1855 
1856 	netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n",
1857 		   ring->cbs_enable ? "enabled" : "disabled",
1858 		   ring->launchtime_enable ? "enabled" : "disabled",
1859 		   queue,
1860 		   ring->idleslope, ring->sendslope,
1861 		   ring->hicredit, ring->locredit);
1862 }
1863 
igb_save_txtime_params(struct igb_adapter * adapter,int queue,bool enable)1864 static int igb_save_txtime_params(struct igb_adapter *adapter, int queue,
1865 				  bool enable)
1866 {
1867 	struct igb_ring *ring;
1868 
1869 	if (queue < 0 || queue > adapter->num_tx_queues)
1870 		return -EINVAL;
1871 
1872 	ring = adapter->tx_ring[queue];
1873 	ring->launchtime_enable = enable;
1874 
1875 	return 0;
1876 }
1877 
igb_save_cbs_params(struct igb_adapter * adapter,int queue,bool enable,int idleslope,int sendslope,int hicredit,int locredit)1878 static int igb_save_cbs_params(struct igb_adapter *adapter, int queue,
1879 			       bool enable, int idleslope, int sendslope,
1880 			       int hicredit, int locredit)
1881 {
1882 	struct igb_ring *ring;
1883 
1884 	if (queue < 0 || queue > adapter->num_tx_queues)
1885 		return -EINVAL;
1886 
1887 	ring = adapter->tx_ring[queue];
1888 
1889 	ring->cbs_enable = enable;
1890 	ring->idleslope = idleslope;
1891 	ring->sendslope = sendslope;
1892 	ring->hicredit = hicredit;
1893 	ring->locredit = locredit;
1894 
1895 	return 0;
1896 }
1897 
1898 /**
1899  *  igb_setup_tx_mode - Switch to/from Qav Tx mode when applicable
1900  *  @adapter: pointer to adapter struct
1901  *
1902  *  Configure TQAVCTRL register switching the controller's Tx mode
1903  *  if FQTSS mode is enabled or disabled. Additionally, will issue
1904  *  a call to igb_config_tx_modes() per queue so any previously saved
1905  *  Tx parameters are applied.
1906  **/
igb_setup_tx_mode(struct igb_adapter * adapter)1907 static void igb_setup_tx_mode(struct igb_adapter *adapter)
1908 {
1909 	struct net_device *netdev = adapter->netdev;
1910 	struct e1000_hw *hw = &adapter->hw;
1911 	u32 val;
1912 
1913 	/* Only i210 controller supports changing the transmission mode. */
1914 	if (hw->mac.type != e1000_i210)
1915 		return;
1916 
1917 	if (is_fqtss_enabled(adapter)) {
1918 		int i, max_queue;
1919 
1920 		/* Configure TQAVCTRL register: set transmit mode to 'Qav',
1921 		 * set data fetch arbitration to 'round robin', set SP_WAIT_SR
1922 		 * so SP queues wait for SR ones.
1923 		 */
1924 		val = rd32(E1000_I210_TQAVCTRL);
1925 		val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR;
1926 		val &= ~E1000_TQAVCTRL_DATAFETCHARB;
1927 		wr32(E1000_I210_TQAVCTRL, val);
1928 
1929 		/* Configure Tx and Rx packet buffers sizes as described in
1930 		 * i210 datasheet section 7.2.7.7.
1931 		 */
1932 		val = rd32(E1000_TXPBS);
1933 		val &= ~I210_TXPBSIZE_MASK;
1934 		val |= I210_TXPBSIZE_PB0_6KB | I210_TXPBSIZE_PB1_6KB |
1935 			I210_TXPBSIZE_PB2_6KB | I210_TXPBSIZE_PB3_6KB;
1936 		wr32(E1000_TXPBS, val);
1937 
1938 		val = rd32(E1000_RXPBS);
1939 		val &= ~I210_RXPBSIZE_MASK;
1940 		val |= I210_RXPBSIZE_PB_30KB;
1941 		wr32(E1000_RXPBS, val);
1942 
1943 		/* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ
1944 		 * register should not exceed the buffer size programmed in
1945 		 * TXPBS. The smallest buffer size programmed in TXPBS is 4kB
1946 		 * so according to the datasheet we should set MAX_TPKT_SIZE to
1947 		 * 4kB / 64.
1948 		 *
1949 		 * However, when we do so, no frame from queue 2 and 3 are
1950 		 * transmitted.  It seems the MAX_TPKT_SIZE should not be great
1951 		 * or _equal_ to the buffer size programmed in TXPBS. For this
1952 		 * reason, we set MAX_ TPKT_SIZE to (4kB - 1) / 64.
1953 		 */
1954 		val = (4096 - 1) / 64;
1955 		wr32(E1000_I210_DTXMXPKTSZ, val);
1956 
1957 		/* Since FQTSS mode is enabled, apply any CBS configuration
1958 		 * previously set. If no previous CBS configuration has been
1959 		 * done, then the initial configuration is applied, which means
1960 		 * CBS is disabled.
1961 		 */
1962 		max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ?
1963 			    adapter->num_tx_queues : I210_SR_QUEUES_NUM;
1964 
1965 		for (i = 0; i < max_queue; i++) {
1966 			igb_config_tx_modes(adapter, i);
1967 		}
1968 	} else {
1969 		wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
1970 		wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
1971 		wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT);
1972 
1973 		val = rd32(E1000_I210_TQAVCTRL);
1974 		/* According to Section 8.12.21, the other flags we've set when
1975 		 * enabling FQTSS are not relevant when disabling FQTSS so we
1976 		 * don't set they here.
1977 		 */
1978 		val &= ~E1000_TQAVCTRL_XMIT_MODE;
1979 		wr32(E1000_I210_TQAVCTRL, val);
1980 	}
1981 
1982 	netdev_dbg(netdev, "FQTSS %s\n", (is_fqtss_enabled(adapter)) ?
1983 		   "enabled" : "disabled");
1984 }
1985 
1986 /**
1987  *  igb_configure - configure the hardware for RX and TX
1988  *  @adapter: private board structure
1989  **/
igb_configure(struct igb_adapter * adapter)1990 static void igb_configure(struct igb_adapter *adapter)
1991 {
1992 	struct net_device *netdev = adapter->netdev;
1993 	int i;
1994 
1995 	igb_get_hw_control(adapter);
1996 	igb_set_rx_mode(netdev);
1997 	igb_setup_tx_mode(adapter);
1998 
1999 	igb_restore_vlan(adapter);
2000 
2001 	igb_setup_tctl(adapter);
2002 	igb_setup_mrqc(adapter);
2003 	igb_setup_rctl(adapter);
2004 
2005 	igb_nfc_filter_restore(adapter);
2006 	igb_configure_tx(adapter);
2007 	igb_configure_rx(adapter);
2008 
2009 	igb_rx_fifo_flush_82575(&adapter->hw);
2010 
2011 	/* call igb_desc_unused which always leaves
2012 	 * at least 1 descriptor unused to make sure
2013 	 * next_to_use != next_to_clean
2014 	 */
2015 	for (i = 0; i < adapter->num_rx_queues; i++) {
2016 		struct igb_ring *ring = adapter->rx_ring[i];
2017 		igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
2018 	}
2019 }
2020 
2021 /**
2022  *  igb_power_up_link - Power up the phy/serdes link
2023  *  @adapter: address of board private structure
2024  **/
igb_power_up_link(struct igb_adapter * adapter)2025 void igb_power_up_link(struct igb_adapter *adapter)
2026 {
2027 	igb_reset_phy(&adapter->hw);
2028 
2029 	if (adapter->hw.phy.media_type == e1000_media_type_copper)
2030 		igb_power_up_phy_copper(&adapter->hw);
2031 	else
2032 		igb_power_up_serdes_link_82575(&adapter->hw);
2033 
2034 	igb_setup_link(&adapter->hw);
2035 }
2036 
2037 /**
2038  *  igb_power_down_link - Power down the phy/serdes link
2039  *  @adapter: address of board private structure
2040  */
igb_power_down_link(struct igb_adapter * adapter)2041 static void igb_power_down_link(struct igb_adapter *adapter)
2042 {
2043 	if (adapter->hw.phy.media_type == e1000_media_type_copper)
2044 		igb_power_down_phy_copper_82575(&adapter->hw);
2045 	else
2046 		igb_shutdown_serdes_link_82575(&adapter->hw);
2047 }
2048 
2049 /**
2050  * igb_check_swap_media -  Detect and switch function for Media Auto Sense
2051  * @adapter: address of the board private structure
2052  **/
igb_check_swap_media(struct igb_adapter * adapter)2053 static void igb_check_swap_media(struct igb_adapter *adapter)
2054 {
2055 	struct e1000_hw *hw = &adapter->hw;
2056 	u32 ctrl_ext, connsw;
2057 	bool swap_now = false;
2058 
2059 	ctrl_ext = rd32(E1000_CTRL_EXT);
2060 	connsw = rd32(E1000_CONNSW);
2061 
2062 	/* need to live swap if current media is copper and we have fiber/serdes
2063 	 * to go to.
2064 	 */
2065 
2066 	if ((hw->phy.media_type == e1000_media_type_copper) &&
2067 	    (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
2068 		swap_now = true;
2069 	} else if ((hw->phy.media_type != e1000_media_type_copper) &&
2070 		   !(connsw & E1000_CONNSW_SERDESD)) {
2071 		/* copper signal takes time to appear */
2072 		if (adapter->copper_tries < 4) {
2073 			adapter->copper_tries++;
2074 			connsw |= E1000_CONNSW_AUTOSENSE_CONF;
2075 			wr32(E1000_CONNSW, connsw);
2076 			return;
2077 		} else {
2078 			adapter->copper_tries = 0;
2079 			if ((connsw & E1000_CONNSW_PHYSD) &&
2080 			    (!(connsw & E1000_CONNSW_PHY_PDN))) {
2081 				swap_now = true;
2082 				connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
2083 				wr32(E1000_CONNSW, connsw);
2084 			}
2085 		}
2086 	}
2087 
2088 	if (!swap_now)
2089 		return;
2090 
2091 	switch (hw->phy.media_type) {
2092 	case e1000_media_type_copper:
2093 		netdev_info(adapter->netdev,
2094 			"MAS: changing media to fiber/serdes\n");
2095 		ctrl_ext |=
2096 			E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2097 		adapter->flags |= IGB_FLAG_MEDIA_RESET;
2098 		adapter->copper_tries = 0;
2099 		break;
2100 	case e1000_media_type_internal_serdes:
2101 	case e1000_media_type_fiber:
2102 		netdev_info(adapter->netdev,
2103 			"MAS: changing media to copper\n");
2104 		ctrl_ext &=
2105 			~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2106 		adapter->flags |= IGB_FLAG_MEDIA_RESET;
2107 		break;
2108 	default:
2109 		/* shouldn't get here during regular operation */
2110 		netdev_err(adapter->netdev,
2111 			"AMS: Invalid media type found, returning\n");
2112 		break;
2113 	}
2114 	wr32(E1000_CTRL_EXT, ctrl_ext);
2115 }
2116 
2117 /**
2118  *  igb_up - Open the interface and prepare it to handle traffic
2119  *  @adapter: board private structure
2120  **/
igb_up(struct igb_adapter * adapter)2121 int igb_up(struct igb_adapter *adapter)
2122 {
2123 	struct e1000_hw *hw = &adapter->hw;
2124 	int i;
2125 
2126 	/* hardware has been reset, we need to reload some things */
2127 	igb_configure(adapter);
2128 
2129 	clear_bit(__IGB_DOWN, &adapter->state);
2130 
2131 	for (i = 0; i < adapter->num_q_vectors; i++)
2132 		napi_enable(&(adapter->q_vector[i]->napi));
2133 
2134 	if (adapter->flags & IGB_FLAG_HAS_MSIX)
2135 		igb_configure_msix(adapter);
2136 	else
2137 		igb_assign_vector(adapter->q_vector[0], 0);
2138 
2139 	/* Clear any pending interrupts. */
2140 	rd32(E1000_TSICR);
2141 	rd32(E1000_ICR);
2142 	igb_irq_enable(adapter);
2143 
2144 	/* notify VFs that reset has been completed */
2145 	if (adapter->vfs_allocated_count) {
2146 		u32 reg_data = rd32(E1000_CTRL_EXT);
2147 
2148 		reg_data |= E1000_CTRL_EXT_PFRSTD;
2149 		wr32(E1000_CTRL_EXT, reg_data);
2150 	}
2151 
2152 	netif_tx_start_all_queues(adapter->netdev);
2153 
2154 	/* start the watchdog. */
2155 	hw->mac.get_link_status = 1;
2156 	schedule_work(&adapter->watchdog_task);
2157 
2158 	if ((adapter->flags & IGB_FLAG_EEE) &&
2159 	    (!hw->dev_spec._82575.eee_disable))
2160 		adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;
2161 
2162 	return 0;
2163 }
2164 
igb_down(struct igb_adapter * adapter)2165 void igb_down(struct igb_adapter *adapter)
2166 {
2167 	struct net_device *netdev = adapter->netdev;
2168 	struct e1000_hw *hw = &adapter->hw;
2169 	u32 tctl, rctl;
2170 	int i;
2171 
2172 	/* signal that we're down so the interrupt handler does not
2173 	 * reschedule our watchdog timer
2174 	 */
2175 	set_bit(__IGB_DOWN, &adapter->state);
2176 
2177 	/* disable receives in the hardware */
2178 	rctl = rd32(E1000_RCTL);
2179 	wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
2180 	/* flush and sleep below */
2181 
2182 	igb_nfc_filter_exit(adapter);
2183 
2184 	netif_carrier_off(netdev);
2185 	netif_tx_stop_all_queues(netdev);
2186 
2187 	/* disable transmits in the hardware */
2188 	tctl = rd32(E1000_TCTL);
2189 	tctl &= ~E1000_TCTL_EN;
2190 	wr32(E1000_TCTL, tctl);
2191 	/* flush both disables and wait for them to finish */
2192 	wrfl();
2193 	usleep_range(10000, 11000);
2194 
2195 	igb_irq_disable(adapter);
2196 
2197 	adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
2198 
2199 	for (i = 0; i < adapter->num_q_vectors; i++) {
2200 		if (adapter->q_vector[i]) {
2201 			napi_synchronize(&adapter->q_vector[i]->napi);
2202 			napi_disable(&adapter->q_vector[i]->napi);
2203 		}
2204 	}
2205 
2206 	del_timer_sync(&adapter->watchdog_timer);
2207 	del_timer_sync(&adapter->phy_info_timer);
2208 
2209 	/* record the stats before reset*/
2210 	spin_lock(&adapter->stats64_lock);
2211 	igb_update_stats(adapter);
2212 	spin_unlock(&adapter->stats64_lock);
2213 
2214 	adapter->link_speed = 0;
2215 	adapter->link_duplex = 0;
2216 
2217 	if (!pci_channel_offline(adapter->pdev))
2218 		igb_reset(adapter);
2219 
2220 	/* clear VLAN promisc flag so VFTA will be updated if necessary */
2221 	adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
2222 
2223 	igb_clean_all_tx_rings(adapter);
2224 	igb_clean_all_rx_rings(adapter);
2225 #ifdef CONFIG_IGB_DCA
2226 
2227 	/* since we reset the hardware DCA settings were cleared */
2228 	igb_setup_dca(adapter);
2229 #endif
2230 }
2231 
igb_reinit_locked(struct igb_adapter * adapter)2232 void igb_reinit_locked(struct igb_adapter *adapter)
2233 {
2234 	while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
2235 		usleep_range(1000, 2000);
2236 	igb_down(adapter);
2237 	igb_up(adapter);
2238 	clear_bit(__IGB_RESETTING, &adapter->state);
2239 }
2240 
2241 /** igb_enable_mas - Media Autosense re-enable after swap
2242  *
2243  * @adapter: adapter struct
2244  **/
igb_enable_mas(struct igb_adapter * adapter)2245 static void igb_enable_mas(struct igb_adapter *adapter)
2246 {
2247 	struct e1000_hw *hw = &adapter->hw;
2248 	u32 connsw = rd32(E1000_CONNSW);
2249 
2250 	/* configure for SerDes media detect */
2251 	if ((hw->phy.media_type == e1000_media_type_copper) &&
2252 	    (!(connsw & E1000_CONNSW_SERDESD))) {
2253 		connsw |= E1000_CONNSW_ENRGSRC;
2254 		connsw |= E1000_CONNSW_AUTOSENSE_EN;
2255 		wr32(E1000_CONNSW, connsw);
2256 		wrfl();
2257 	}
2258 }
2259 
2260 #ifdef CONFIG_IGB_HWMON
2261 /**
2262  *  igb_set_i2c_bb - Init I2C interface
2263  *  @hw: pointer to hardware structure
2264  **/
igb_set_i2c_bb(struct e1000_hw * hw)2265 static void igb_set_i2c_bb(struct e1000_hw *hw)
2266 {
2267 	u32 ctrl_ext;
2268 	s32 i2cctl;
2269 
2270 	ctrl_ext = rd32(E1000_CTRL_EXT);
2271 	ctrl_ext |= E1000_CTRL_I2C_ENA;
2272 	wr32(E1000_CTRL_EXT, ctrl_ext);
2273 	wrfl();
2274 
2275 	i2cctl = rd32(E1000_I2CPARAMS);
2276 	i2cctl |= E1000_I2CBB_EN
2277 		| E1000_I2C_CLK_OE_N
2278 		| E1000_I2C_DATA_OE_N;
2279 	wr32(E1000_I2CPARAMS, i2cctl);
2280 	wrfl();
2281 }
2282 #endif
2283 
igb_reset(struct igb_adapter * adapter)2284 void igb_reset(struct igb_adapter *adapter)
2285 {
2286 	struct pci_dev *pdev = adapter->pdev;
2287 	struct e1000_hw *hw = &adapter->hw;
2288 	struct e1000_mac_info *mac = &hw->mac;
2289 	struct e1000_fc_info *fc = &hw->fc;
2290 	u32 pba, hwm;
2291 
2292 	/* Repartition Pba for greater than 9k mtu
2293 	 * To take effect CTRL.RST is required.
2294 	 */
2295 	switch (mac->type) {
2296 	case e1000_i350:
2297 	case e1000_i354:
2298 	case e1000_82580:
2299 		pba = rd32(E1000_RXPBS);
2300 		pba = igb_rxpbs_adjust_82580(pba);
2301 		break;
2302 	case e1000_82576:
2303 		pba = rd32(E1000_RXPBS);
2304 		pba &= E1000_RXPBS_SIZE_MASK_82576;
2305 		break;
2306 	case e1000_82575:
2307 	case e1000_i210:
2308 	case e1000_i211:
2309 	default:
2310 		pba = E1000_PBA_34K;
2311 		break;
2312 	}
2313 
2314 	if (mac->type == e1000_82575) {
2315 		u32 min_rx_space, min_tx_space, needed_tx_space;
2316 
2317 		/* write Rx PBA so that hardware can report correct Tx PBA */
2318 		wr32(E1000_PBA, pba);
2319 
2320 		/* To maintain wire speed transmits, the Tx FIFO should be
2321 		 * large enough to accommodate two full transmit packets,
2322 		 * rounded up to the next 1KB and expressed in KB.  Likewise,
2323 		 * the Rx FIFO should be large enough to accommodate at least
2324 		 * one full receive packet and is similarly rounded up and
2325 		 * expressed in KB.
2326 		 */
2327 		min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024);
2328 
2329 		/* The Tx FIFO also stores 16 bytes of information about the Tx
2330 		 * but don't include Ethernet FCS because hardware appends it.
2331 		 * We only need to round down to the nearest 512 byte block
2332 		 * count since the value we care about is 2 frames, not 1.
2333 		 */
2334 		min_tx_space = adapter->max_frame_size;
2335 		min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN;
2336 		min_tx_space = DIV_ROUND_UP(min_tx_space, 512);
2337 
2338 		/* upper 16 bits has Tx packet buffer allocation size in KB */
2339 		needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16);
2340 
2341 		/* If current Tx allocation is less than the min Tx FIFO size,
2342 		 * and the min Tx FIFO size is less than the current Rx FIFO
2343 		 * allocation, take space away from current Rx allocation.
2344 		 */
2345 		if (needed_tx_space < pba) {
2346 			pba -= needed_tx_space;
2347 
2348 			/* if short on Rx space, Rx wins and must trump Tx
2349 			 * adjustment
2350 			 */
2351 			if (pba < min_rx_space)
2352 				pba = min_rx_space;
2353 		}
2354 
2355 		/* adjust PBA for jumbo frames */
2356 		wr32(E1000_PBA, pba);
2357 	}
2358 
2359 	/* flow control settings
2360 	 * The high water mark must be low enough to fit one full frame
2361 	 * after transmitting the pause frame.  As such we must have enough
2362 	 * space to allow for us to complete our current transmit and then
2363 	 * receive the frame that is in progress from the link partner.
2364 	 * Set it to:
2365 	 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
2366 	 */
2367 	hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
2368 
2369 	fc->high_water = hwm & 0xFFFFFFF0;	/* 16-byte granularity */
2370 	fc->low_water = fc->high_water - 16;
2371 	fc->pause_time = 0xFFFF;
2372 	fc->send_xon = 1;
2373 	fc->current_mode = fc->requested_mode;
2374 
2375 	/* disable receive for all VFs and wait one second */
2376 	if (adapter->vfs_allocated_count) {
2377 		int i;
2378 
2379 		for (i = 0 ; i < adapter->vfs_allocated_count; i++)
2380 			adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
2381 
2382 		/* ping all the active vfs to let them know we are going down */
2383 		igb_ping_all_vfs(adapter);
2384 
2385 		/* disable transmits and receives */
2386 		wr32(E1000_VFRE, 0);
2387 		wr32(E1000_VFTE, 0);
2388 	}
2389 
2390 	/* Allow time for pending master requests to run */
2391 	hw->mac.ops.reset_hw(hw);
2392 	wr32(E1000_WUC, 0);
2393 
2394 	if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
2395 		/* need to resetup here after media swap */
2396 		adapter->ei.get_invariants(hw);
2397 		adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
2398 	}
2399 	if ((mac->type == e1000_82575 || mac->type == e1000_i350) &&
2400 	    (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
2401 		igb_enable_mas(adapter);
2402 	}
2403 	if (hw->mac.ops.init_hw(hw))
2404 		dev_err(&pdev->dev, "Hardware Error\n");
2405 
2406 	/* RAR registers were cleared during init_hw, clear mac table */
2407 	igb_flush_mac_table(adapter);
2408 	__dev_uc_unsync(adapter->netdev, NULL);
2409 
2410 	/* Recover default RAR entry */
2411 	igb_set_default_mac_filter(adapter);
2412 
2413 	/* Flow control settings reset on hardware reset, so guarantee flow
2414 	 * control is off when forcing speed.
2415 	 */
2416 	if (!hw->mac.autoneg)
2417 		igb_force_mac_fc(hw);
2418 
2419 	igb_init_dmac(adapter, pba);
2420 #ifdef CONFIG_IGB_HWMON
2421 	/* Re-initialize the thermal sensor on i350 devices. */
2422 	if (!test_bit(__IGB_DOWN, &adapter->state)) {
2423 		if (mac->type == e1000_i350 && hw->bus.func == 0) {
2424 			/* If present, re-initialize the external thermal sensor
2425 			 * interface.
2426 			 */
2427 			if (adapter->ets)
2428 				igb_set_i2c_bb(hw);
2429 			mac->ops.init_thermal_sensor_thresh(hw);
2430 		}
2431 	}
2432 #endif
2433 	/* Re-establish EEE setting */
2434 	if (hw->phy.media_type == e1000_media_type_copper) {
2435 		switch (mac->type) {
2436 		case e1000_i350:
2437 		case e1000_i210:
2438 		case e1000_i211:
2439 			igb_set_eee_i350(hw, true, true);
2440 			break;
2441 		case e1000_i354:
2442 			igb_set_eee_i354(hw, true, true);
2443 			break;
2444 		default:
2445 			break;
2446 		}
2447 	}
2448 	if (!netif_running(adapter->netdev))
2449 		igb_power_down_link(adapter);
2450 
2451 	igb_update_mng_vlan(adapter);
2452 
2453 	/* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
2454 	wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
2455 
2456 	/* Re-enable PTP, where applicable. */
2457 	if (adapter->ptp_flags & IGB_PTP_ENABLED)
2458 		igb_ptp_reset(adapter);
2459 
2460 	igb_get_phy_info(hw);
2461 }
2462 
igb_fix_features(struct net_device * netdev,netdev_features_t features)2463 static netdev_features_t igb_fix_features(struct net_device *netdev,
2464 	netdev_features_t features)
2465 {
2466 	/* Since there is no support for separate Rx/Tx vlan accel
2467 	 * enable/disable make sure Tx flag is always in same state as Rx.
2468 	 */
2469 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
2470 		features |= NETIF_F_HW_VLAN_CTAG_TX;
2471 	else
2472 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2473 
2474 	return features;
2475 }
2476 
igb_set_features(struct net_device * netdev,netdev_features_t features)2477 static int igb_set_features(struct net_device *netdev,
2478 	netdev_features_t features)
2479 {
2480 	netdev_features_t changed = netdev->features ^ features;
2481 	struct igb_adapter *adapter = netdev_priv(netdev);
2482 
2483 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2484 		igb_vlan_mode(netdev, features);
2485 
2486 	if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2487 		return 0;
2488 
2489 	if (!(features & NETIF_F_NTUPLE)) {
2490 		struct hlist_node *node2;
2491 		struct igb_nfc_filter *rule;
2492 
2493 		spin_lock(&adapter->nfc_lock);
2494 		hlist_for_each_entry_safe(rule, node2,
2495 					  &adapter->nfc_filter_list, nfc_node) {
2496 			igb_erase_filter(adapter, rule);
2497 			hlist_del(&rule->nfc_node);
2498 			kfree(rule);
2499 		}
2500 		spin_unlock(&adapter->nfc_lock);
2501 		adapter->nfc_filter_count = 0;
2502 	}
2503 
2504 	netdev->features = features;
2505 
2506 	if (netif_running(netdev))
2507 		igb_reinit_locked(adapter);
2508 	else
2509 		igb_reset(adapter);
2510 
2511 	return 1;
2512 }
2513 
igb_ndo_fdb_add(struct ndmsg * ndm,struct nlattr * tb[],struct net_device * dev,const unsigned char * addr,u16 vid,u16 flags,struct netlink_ext_ack * extack)2514 static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
2515 			   struct net_device *dev,
2516 			   const unsigned char *addr, u16 vid,
2517 			   u16 flags,
2518 			   struct netlink_ext_ack *extack)
2519 {
2520 	/* guarantee we can provide a unique filter for the unicast address */
2521 	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
2522 		struct igb_adapter *adapter = netdev_priv(dev);
2523 		int vfn = adapter->vfs_allocated_count;
2524 
2525 		if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn))
2526 			return -ENOMEM;
2527 	}
2528 
2529 	return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
2530 }
2531 
2532 #define IGB_MAX_MAC_HDR_LEN	127
2533 #define IGB_MAX_NETWORK_HDR_LEN	511
2534 
2535 static netdev_features_t
igb_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2536 igb_features_check(struct sk_buff *skb, struct net_device *dev,
2537 		   netdev_features_t features)
2538 {
2539 	unsigned int network_hdr_len, mac_hdr_len;
2540 
2541 	/* Make certain the headers can be described by a context descriptor */
2542 	mac_hdr_len = skb_network_header(skb) - skb->data;
2543 	if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN))
2544 		return features & ~(NETIF_F_HW_CSUM |
2545 				    NETIF_F_SCTP_CRC |
2546 				    NETIF_F_GSO_UDP_L4 |
2547 				    NETIF_F_HW_VLAN_CTAG_TX |
2548 				    NETIF_F_TSO |
2549 				    NETIF_F_TSO6);
2550 
2551 	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2552 	if (unlikely(network_hdr_len >  IGB_MAX_NETWORK_HDR_LEN))
2553 		return features & ~(NETIF_F_HW_CSUM |
2554 				    NETIF_F_SCTP_CRC |
2555 				    NETIF_F_GSO_UDP_L4 |
2556 				    NETIF_F_TSO |
2557 				    NETIF_F_TSO6);
2558 
2559 	/* We can only support IPV4 TSO in tunnels if we can mangle the
2560 	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2561 	 */
2562 	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2563 		features &= ~NETIF_F_TSO;
2564 
2565 	return features;
2566 }
2567 
igb_offload_apply(struct igb_adapter * adapter,s32 queue)2568 static void igb_offload_apply(struct igb_adapter *adapter, s32 queue)
2569 {
2570 	if (!is_fqtss_enabled(adapter)) {
2571 		enable_fqtss(adapter, true);
2572 		return;
2573 	}
2574 
2575 	igb_config_tx_modes(adapter, queue);
2576 
2577 	if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter))
2578 		enable_fqtss(adapter, false);
2579 }
2580 
igb_offload_cbs(struct igb_adapter * adapter,struct tc_cbs_qopt_offload * qopt)2581 static int igb_offload_cbs(struct igb_adapter *adapter,
2582 			   struct tc_cbs_qopt_offload *qopt)
2583 {
2584 	struct e1000_hw *hw = &adapter->hw;
2585 	int err;
2586 
2587 	/* CBS offloading is only supported by i210 controller. */
2588 	if (hw->mac.type != e1000_i210)
2589 		return -EOPNOTSUPP;
2590 
2591 	/* CBS offloading is only supported by queue 0 and queue 1. */
2592 	if (qopt->queue < 0 || qopt->queue > 1)
2593 		return -EINVAL;
2594 
2595 	err = igb_save_cbs_params(adapter, qopt->queue, qopt->enable,
2596 				  qopt->idleslope, qopt->sendslope,
2597 				  qopt->hicredit, qopt->locredit);
2598 	if (err)
2599 		return err;
2600 
2601 	igb_offload_apply(adapter, qopt->queue);
2602 
2603 	return 0;
2604 }
2605 
2606 #define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
2607 #define VLAN_PRIO_FULL_MASK (0x07)
2608 
igb_parse_cls_flower(struct igb_adapter * adapter,struct flow_cls_offload * f,int traffic_class,struct igb_nfc_filter * input)2609 static int igb_parse_cls_flower(struct igb_adapter *adapter,
2610 				struct flow_cls_offload *f,
2611 				int traffic_class,
2612 				struct igb_nfc_filter *input)
2613 {
2614 	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
2615 	struct flow_dissector *dissector = rule->match.dissector;
2616 	struct netlink_ext_ack *extack = f->common.extack;
2617 
2618 	if (dissector->used_keys &
2619 	    ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
2620 	      BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
2621 	      BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2622 	      BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) {
2623 		NL_SET_ERR_MSG_MOD(extack,
2624 				   "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported");
2625 		return -EOPNOTSUPP;
2626 	}
2627 
2628 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2629 		struct flow_match_eth_addrs match;
2630 
2631 		flow_rule_match_eth_addrs(rule, &match);
2632 		if (!is_zero_ether_addr(match.mask->dst)) {
2633 			if (!is_broadcast_ether_addr(match.mask->dst)) {
2634 				NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address");
2635 				return -EINVAL;
2636 			}
2637 
2638 			input->filter.match_flags |=
2639 				IGB_FILTER_FLAG_DST_MAC_ADDR;
2640 			ether_addr_copy(input->filter.dst_addr, match.key->dst);
2641 		}
2642 
2643 		if (!is_zero_ether_addr(match.mask->src)) {
2644 			if (!is_broadcast_ether_addr(match.mask->src)) {
2645 				NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address");
2646 				return -EINVAL;
2647 			}
2648 
2649 			input->filter.match_flags |=
2650 				IGB_FILTER_FLAG_SRC_MAC_ADDR;
2651 			ether_addr_copy(input->filter.src_addr, match.key->src);
2652 		}
2653 	}
2654 
2655 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
2656 		struct flow_match_basic match;
2657 
2658 		flow_rule_match_basic(rule, &match);
2659 		if (match.mask->n_proto) {
2660 			if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) {
2661 				NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter");
2662 				return -EINVAL;
2663 			}
2664 
2665 			input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE;
2666 			input->filter.etype = match.key->n_proto;
2667 		}
2668 	}
2669 
2670 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
2671 		struct flow_match_vlan match;
2672 
2673 		flow_rule_match_vlan(rule, &match);
2674 		if (match.mask->vlan_priority) {
2675 			if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) {
2676 				NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority");
2677 				return -EINVAL;
2678 			}
2679 
2680 			input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI;
2681 			input->filter.vlan_tci =
2682 				(__force __be16)match.key->vlan_priority;
2683 		}
2684 	}
2685 
2686 	input->action = traffic_class;
2687 	input->cookie = f->cookie;
2688 
2689 	return 0;
2690 }
2691 
igb_configure_clsflower(struct igb_adapter * adapter,struct flow_cls_offload * cls_flower)2692 static int igb_configure_clsflower(struct igb_adapter *adapter,
2693 				   struct flow_cls_offload *cls_flower)
2694 {
2695 	struct netlink_ext_ack *extack = cls_flower->common.extack;
2696 	struct igb_nfc_filter *filter, *f;
2697 	int err, tc;
2698 
2699 	tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
2700 	if (tc < 0) {
2701 		NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class");
2702 		return -EINVAL;
2703 	}
2704 
2705 	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
2706 	if (!filter)
2707 		return -ENOMEM;
2708 
2709 	err = igb_parse_cls_flower(adapter, cls_flower, tc, filter);
2710 	if (err < 0)
2711 		goto err_parse;
2712 
2713 	spin_lock(&adapter->nfc_lock);
2714 
2715 	hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) {
2716 		if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2717 			err = -EEXIST;
2718 			NL_SET_ERR_MSG_MOD(extack,
2719 					   "This filter is already set in ethtool");
2720 			goto err_locked;
2721 		}
2722 	}
2723 
2724 	hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) {
2725 		if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2726 			err = -EEXIST;
2727 			NL_SET_ERR_MSG_MOD(extack,
2728 					   "This filter is already set in cls_flower");
2729 			goto err_locked;
2730 		}
2731 	}
2732 
2733 	err = igb_add_filter(adapter, filter);
2734 	if (err < 0) {
2735 		NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter");
2736 		goto err_locked;
2737 	}
2738 
2739 	hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list);
2740 
2741 	spin_unlock(&adapter->nfc_lock);
2742 
2743 	return 0;
2744 
2745 err_locked:
2746 	spin_unlock(&adapter->nfc_lock);
2747 
2748 err_parse:
2749 	kfree(filter);
2750 
2751 	return err;
2752 }
2753 
igb_delete_clsflower(struct igb_adapter * adapter,struct flow_cls_offload * cls_flower)2754 static int igb_delete_clsflower(struct igb_adapter *adapter,
2755 				struct flow_cls_offload *cls_flower)
2756 {
2757 	struct igb_nfc_filter *filter;
2758 	int err;
2759 
2760 	spin_lock(&adapter->nfc_lock);
2761 
2762 	hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node)
2763 		if (filter->cookie == cls_flower->cookie)
2764 			break;
2765 
2766 	if (!filter) {
2767 		err = -ENOENT;
2768 		goto out;
2769 	}
2770 
2771 	err = igb_erase_filter(adapter, filter);
2772 	if (err < 0)
2773 		goto out;
2774 
2775 	hlist_del(&filter->nfc_node);
2776 	kfree(filter);
2777 
2778 out:
2779 	spin_unlock(&adapter->nfc_lock);
2780 
2781 	return err;
2782 }
2783 
igb_setup_tc_cls_flower(struct igb_adapter * adapter,struct flow_cls_offload * cls_flower)2784 static int igb_setup_tc_cls_flower(struct igb_adapter *adapter,
2785 				   struct flow_cls_offload *cls_flower)
2786 {
2787 	switch (cls_flower->command) {
2788 	case FLOW_CLS_REPLACE:
2789 		return igb_configure_clsflower(adapter, cls_flower);
2790 	case FLOW_CLS_DESTROY:
2791 		return igb_delete_clsflower(adapter, cls_flower);
2792 	case FLOW_CLS_STATS:
2793 		return -EOPNOTSUPP;
2794 	default:
2795 		return -EOPNOTSUPP;
2796 	}
2797 }
2798 
igb_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)2799 static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2800 				 void *cb_priv)
2801 {
2802 	struct igb_adapter *adapter = cb_priv;
2803 
2804 	if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
2805 		return -EOPNOTSUPP;
2806 
2807 	switch (type) {
2808 	case TC_SETUP_CLSFLOWER:
2809 		return igb_setup_tc_cls_flower(adapter, type_data);
2810 
2811 	default:
2812 		return -EOPNOTSUPP;
2813 	}
2814 }
2815 
igb_offload_txtime(struct igb_adapter * adapter,struct tc_etf_qopt_offload * qopt)2816 static int igb_offload_txtime(struct igb_adapter *adapter,
2817 			      struct tc_etf_qopt_offload *qopt)
2818 {
2819 	struct e1000_hw *hw = &adapter->hw;
2820 	int err;
2821 
2822 	/* Launchtime offloading is only supported by i210 controller. */
2823 	if (hw->mac.type != e1000_i210)
2824 		return -EOPNOTSUPP;
2825 
2826 	/* Launchtime offloading is only supported by queues 0 and 1. */
2827 	if (qopt->queue < 0 || qopt->queue > 1)
2828 		return -EINVAL;
2829 
2830 	err = igb_save_txtime_params(adapter, qopt->queue, qopt->enable);
2831 	if (err)
2832 		return err;
2833 
2834 	igb_offload_apply(adapter, qopt->queue);
2835 
2836 	return 0;
2837 }
2838 
igb_tc_query_caps(struct igb_adapter * adapter,struct tc_query_caps_base * base)2839 static int igb_tc_query_caps(struct igb_adapter *adapter,
2840 			     struct tc_query_caps_base *base)
2841 {
2842 	switch (base->type) {
2843 	case TC_SETUP_QDISC_TAPRIO: {
2844 		struct tc_taprio_caps *caps = base->caps;
2845 
2846 		caps->broken_mqprio = true;
2847 
2848 		return 0;
2849 	}
2850 	default:
2851 		return -EOPNOTSUPP;
2852 	}
2853 }
2854 
2855 static LIST_HEAD(igb_block_cb_list);
2856 
igb_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)2857 static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
2858 			void *type_data)
2859 {
2860 	struct igb_adapter *adapter = netdev_priv(dev);
2861 
2862 	switch (type) {
2863 	case TC_QUERY_CAPS:
2864 		return igb_tc_query_caps(adapter, type_data);
2865 	case TC_SETUP_QDISC_CBS:
2866 		return igb_offload_cbs(adapter, type_data);
2867 	case TC_SETUP_BLOCK:
2868 		return flow_block_cb_setup_simple(type_data,
2869 						  &igb_block_cb_list,
2870 						  igb_setup_tc_block_cb,
2871 						  adapter, adapter, true);
2872 
2873 	case TC_SETUP_QDISC_ETF:
2874 		return igb_offload_txtime(adapter, type_data);
2875 
2876 	default:
2877 		return -EOPNOTSUPP;
2878 	}
2879 }
2880 
igb_xdp_setup(struct net_device * dev,struct netdev_bpf * bpf)2881 static int igb_xdp_setup(struct net_device *dev, struct netdev_bpf *bpf)
2882 {
2883 	int i, frame_size = dev->mtu + IGB_ETH_PKT_HDR_PAD;
2884 	struct igb_adapter *adapter = netdev_priv(dev);
2885 	struct bpf_prog *prog = bpf->prog, *old_prog;
2886 	bool running = netif_running(dev);
2887 	bool need_reset;
2888 
2889 	/* verify igb ring attributes are sufficient for XDP */
2890 	for (i = 0; i < adapter->num_rx_queues; i++) {
2891 		struct igb_ring *ring = adapter->rx_ring[i];
2892 
2893 		if (frame_size > igb_rx_bufsz(ring)) {
2894 			NL_SET_ERR_MSG_MOD(bpf->extack,
2895 					   "The RX buffer size is too small for the frame size");
2896 			netdev_warn(dev, "XDP RX buffer size %d is too small for the frame size %d\n",
2897 				    igb_rx_bufsz(ring), frame_size);
2898 			return -EINVAL;
2899 		}
2900 	}
2901 
2902 	old_prog = xchg(&adapter->xdp_prog, prog);
2903 	need_reset = (!!prog != !!old_prog);
2904 
2905 	/* device is up and bpf is added/removed, must setup the RX queues */
2906 	if (need_reset && running) {
2907 		igb_close(dev);
2908 	} else {
2909 		for (i = 0; i < adapter->num_rx_queues; i++)
2910 			(void)xchg(&adapter->rx_ring[i]->xdp_prog,
2911 			    adapter->xdp_prog);
2912 	}
2913 
2914 	if (old_prog)
2915 		bpf_prog_put(old_prog);
2916 
2917 	/* bpf is just replaced, RXQ and MTU are already setup */
2918 	if (!need_reset) {
2919 		return 0;
2920 	} else {
2921 		if (prog)
2922 			xdp_features_set_redirect_target(dev, true);
2923 		else
2924 			xdp_features_clear_redirect_target(dev);
2925 	}
2926 
2927 	if (running)
2928 		igb_open(dev);
2929 
2930 	return 0;
2931 }
2932 
igb_xdp(struct net_device * dev,struct netdev_bpf * xdp)2933 static int igb_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2934 {
2935 	switch (xdp->command) {
2936 	case XDP_SETUP_PROG:
2937 		return igb_xdp_setup(dev, xdp);
2938 	default:
2939 		return -EINVAL;
2940 	}
2941 }
2942 
2943 /* This function assumes __netif_tx_lock is held by the caller. */
igb_xdp_ring_update_tail(struct igb_ring * ring)2944 static void igb_xdp_ring_update_tail(struct igb_ring *ring)
2945 {
2946 	lockdep_assert_held(&txring_txq(ring)->_xmit_lock);
2947 
2948 	/* Force memory writes to complete before letting h/w know there
2949 	 * are new descriptors to fetch.
2950 	 */
2951 	wmb();
2952 	writel(ring->next_to_use, ring->tail);
2953 }
2954 
igb_xdp_tx_queue_mapping(struct igb_adapter * adapter)2955 static struct igb_ring *igb_xdp_tx_queue_mapping(struct igb_adapter *adapter)
2956 {
2957 	unsigned int r_idx = smp_processor_id();
2958 
2959 	if (r_idx >= adapter->num_tx_queues)
2960 		r_idx = r_idx % adapter->num_tx_queues;
2961 
2962 	return adapter->tx_ring[r_idx];
2963 }
2964 
igb_xdp_xmit_back(struct igb_adapter * adapter,struct xdp_buff * xdp)2965 static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp)
2966 {
2967 	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2968 	int cpu = smp_processor_id();
2969 	struct igb_ring *tx_ring;
2970 	struct netdev_queue *nq;
2971 	u32 ret;
2972 
2973 	if (unlikely(!xdpf))
2974 		return IGB_XDP_CONSUMED;
2975 
2976 	/* During program transitions its possible adapter->xdp_prog is assigned
2977 	 * but ring has not been configured yet. In this case simply abort xmit.
2978 	 */
2979 	tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL;
2980 	if (unlikely(!tx_ring))
2981 		return IGB_XDP_CONSUMED;
2982 
2983 	nq = txring_txq(tx_ring);
2984 	__netif_tx_lock(nq, cpu);
2985 	/* Avoid transmit queue timeout since we share it with the slow path */
2986 	txq_trans_cond_update(nq);
2987 	ret = igb_xmit_xdp_ring(adapter, tx_ring, xdpf);
2988 	__netif_tx_unlock(nq);
2989 
2990 	return ret;
2991 }
2992 
igb_xdp_xmit(struct net_device * dev,int n,struct xdp_frame ** frames,u32 flags)2993 static int igb_xdp_xmit(struct net_device *dev, int n,
2994 			struct xdp_frame **frames, u32 flags)
2995 {
2996 	struct igb_adapter *adapter = netdev_priv(dev);
2997 	int cpu = smp_processor_id();
2998 	struct igb_ring *tx_ring;
2999 	struct netdev_queue *nq;
3000 	int nxmit = 0;
3001 	int i;
3002 
3003 	if (unlikely(test_bit(__IGB_DOWN, &adapter->state)))
3004 		return -ENETDOWN;
3005 
3006 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
3007 		return -EINVAL;
3008 
3009 	/* During program transitions its possible adapter->xdp_prog is assigned
3010 	 * but ring has not been configured yet. In this case simply abort xmit.
3011 	 */
3012 	tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL;
3013 	if (unlikely(!tx_ring))
3014 		return -ENXIO;
3015 
3016 	nq = txring_txq(tx_ring);
3017 	__netif_tx_lock(nq, cpu);
3018 
3019 	/* Avoid transmit queue timeout since we share it with the slow path */
3020 	txq_trans_cond_update(nq);
3021 
3022 	for (i = 0; i < n; i++) {
3023 		struct xdp_frame *xdpf = frames[i];
3024 		int err;
3025 
3026 		err = igb_xmit_xdp_ring(adapter, tx_ring, xdpf);
3027 		if (err != IGB_XDP_TX)
3028 			break;
3029 		nxmit++;
3030 	}
3031 
3032 	if (unlikely(flags & XDP_XMIT_FLUSH))
3033 		igb_xdp_ring_update_tail(tx_ring);
3034 
3035 	__netif_tx_unlock(nq);
3036 
3037 	return nxmit;
3038 }
3039 
3040 static const struct net_device_ops igb_netdev_ops = {
3041 	.ndo_open		= igb_open,
3042 	.ndo_stop		= igb_close,
3043 	.ndo_start_xmit		= igb_xmit_frame,
3044 	.ndo_get_stats64	= igb_get_stats64,
3045 	.ndo_set_rx_mode	= igb_set_rx_mode,
3046 	.ndo_set_mac_address	= igb_set_mac,
3047 	.ndo_change_mtu		= igb_change_mtu,
3048 	.ndo_eth_ioctl		= igb_ioctl,
3049 	.ndo_tx_timeout		= igb_tx_timeout,
3050 	.ndo_validate_addr	= eth_validate_addr,
3051 	.ndo_vlan_rx_add_vid	= igb_vlan_rx_add_vid,
3052 	.ndo_vlan_rx_kill_vid	= igb_vlan_rx_kill_vid,
3053 	.ndo_set_vf_mac		= igb_ndo_set_vf_mac,
3054 	.ndo_set_vf_vlan	= igb_ndo_set_vf_vlan,
3055 	.ndo_set_vf_rate	= igb_ndo_set_vf_bw,
3056 	.ndo_set_vf_spoofchk	= igb_ndo_set_vf_spoofchk,
3057 	.ndo_set_vf_trust	= igb_ndo_set_vf_trust,
3058 	.ndo_get_vf_config	= igb_ndo_get_vf_config,
3059 	.ndo_fix_features	= igb_fix_features,
3060 	.ndo_set_features	= igb_set_features,
3061 	.ndo_fdb_add		= igb_ndo_fdb_add,
3062 	.ndo_features_check	= igb_features_check,
3063 	.ndo_setup_tc		= igb_setup_tc,
3064 	.ndo_bpf		= igb_xdp,
3065 	.ndo_xdp_xmit		= igb_xdp_xmit,
3066 };
3067 
3068 /**
3069  * igb_set_fw_version - Configure version string for ethtool
3070  * @adapter: adapter struct
3071  **/
igb_set_fw_version(struct igb_adapter * adapter)3072 void igb_set_fw_version(struct igb_adapter *adapter)
3073 {
3074 	struct e1000_hw *hw = &adapter->hw;
3075 	struct e1000_fw_version fw;
3076 
3077 	igb_get_fw_version(hw, &fw);
3078 
3079 	switch (hw->mac.type) {
3080 	case e1000_i210:
3081 	case e1000_i211:
3082 		if (!(igb_get_flash_presence_i210(hw))) {
3083 			snprintf(adapter->fw_version,
3084 				 sizeof(adapter->fw_version),
3085 				 "%2d.%2d-%d",
3086 				 fw.invm_major, fw.invm_minor,
3087 				 fw.invm_img_type);
3088 			break;
3089 		}
3090 		fallthrough;
3091 	default:
3092 		/* if option is rom valid, display its version too */
3093 		if (fw.or_valid) {
3094 			snprintf(adapter->fw_version,
3095 				 sizeof(adapter->fw_version),
3096 				 "%d.%d, 0x%08x, %d.%d.%d",
3097 				 fw.eep_major, fw.eep_minor, fw.etrack_id,
3098 				 fw.or_major, fw.or_build, fw.or_patch);
3099 		/* no option rom */
3100 		} else if (fw.etrack_id != 0X0000) {
3101 			snprintf(adapter->fw_version,
3102 			    sizeof(adapter->fw_version),
3103 			    "%d.%d, 0x%08x",
3104 			    fw.eep_major, fw.eep_minor, fw.etrack_id);
3105 		} else {
3106 		snprintf(adapter->fw_version,
3107 		    sizeof(adapter->fw_version),
3108 		    "%d.%d.%d",
3109 		    fw.eep_major, fw.eep_minor, fw.eep_build);
3110 		}
3111 		break;
3112 	}
3113 }
3114 
3115 /**
3116  * igb_init_mas - init Media Autosense feature if enabled in the NVM
3117  *
3118  * @adapter: adapter struct
3119  **/
igb_init_mas(struct igb_adapter * adapter)3120 static void igb_init_mas(struct igb_adapter *adapter)
3121 {
3122 	struct e1000_hw *hw = &adapter->hw;
3123 	u16 eeprom_data;
3124 
3125 	hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data);
3126 	switch (hw->bus.func) {
3127 	case E1000_FUNC_0:
3128 		if (eeprom_data & IGB_MAS_ENABLE_0) {
3129 			adapter->flags |= IGB_FLAG_MAS_ENABLE;
3130 			netdev_info(adapter->netdev,
3131 				"MAS: Enabling Media Autosense for port %d\n",
3132 				hw->bus.func);
3133 		}
3134 		break;
3135 	case E1000_FUNC_1:
3136 		if (eeprom_data & IGB_MAS_ENABLE_1) {
3137 			adapter->flags |= IGB_FLAG_MAS_ENABLE;
3138 			netdev_info(adapter->netdev,
3139 				"MAS: Enabling Media Autosense for port %d\n",
3140 				hw->bus.func);
3141 		}
3142 		break;
3143 	case E1000_FUNC_2:
3144 		if (eeprom_data & IGB_MAS_ENABLE_2) {
3145 			adapter->flags |= IGB_FLAG_MAS_ENABLE;
3146 			netdev_info(adapter->netdev,
3147 				"MAS: Enabling Media Autosense for port %d\n",
3148 				hw->bus.func);
3149 		}
3150 		break;
3151 	case E1000_FUNC_3:
3152 		if (eeprom_data & IGB_MAS_ENABLE_3) {
3153 			adapter->flags |= IGB_FLAG_MAS_ENABLE;
3154 			netdev_info(adapter->netdev,
3155 				"MAS: Enabling Media Autosense for port %d\n",
3156 				hw->bus.func);
3157 		}
3158 		break;
3159 	default:
3160 		/* Shouldn't get here */
3161 		netdev_err(adapter->netdev,
3162 			"MAS: Invalid port configuration, returning\n");
3163 		break;
3164 	}
3165 }
3166 
3167 /**
3168  *  igb_init_i2c - Init I2C interface
3169  *  @adapter: pointer to adapter structure
3170  **/
igb_init_i2c(struct igb_adapter * adapter)3171 static s32 igb_init_i2c(struct igb_adapter *adapter)
3172 {
3173 	s32 status = 0;
3174 
3175 	/* I2C interface supported on i350 devices */
3176 	if (adapter->hw.mac.type != e1000_i350)
3177 		return 0;
3178 
3179 	/* Initialize the i2c bus which is controlled by the registers.
3180 	 * This bus will use the i2c_algo_bit structure that implements
3181 	 * the protocol through toggling of the 4 bits in the register.
3182 	 */
3183 	adapter->i2c_adap.owner = THIS_MODULE;
3184 	adapter->i2c_algo = igb_i2c_algo;
3185 	adapter->i2c_algo.data = adapter;
3186 	adapter->i2c_adap.algo_data = &adapter->i2c_algo;
3187 	adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
3188 	strscpy(adapter->i2c_adap.name, "igb BB",
3189 		sizeof(adapter->i2c_adap.name));
3190 	status = i2c_bit_add_bus(&adapter->i2c_adap);
3191 	return status;
3192 }
3193 
3194 /**
3195  *  igb_probe - Device Initialization Routine
3196  *  @pdev: PCI device information struct
3197  *  @ent: entry in igb_pci_tbl
3198  *
3199  *  Returns 0 on success, negative on failure
3200  *
3201  *  igb_probe initializes an adapter identified by a pci_dev structure.
3202  *  The OS initialization, configuring of the adapter private structure,
3203  *  and a hardware reset occur.
3204  **/
igb_probe(struct pci_dev * pdev,const struct pci_device_id * ent)3205 static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3206 {
3207 	struct net_device *netdev;
3208 	struct igb_adapter *adapter;
3209 	struct e1000_hw *hw;
3210 	u16 eeprom_data = 0;
3211 	s32 ret_val;
3212 	static int global_quad_port_a; /* global quad port a indication */
3213 	const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
3214 	u8 part_str[E1000_PBANUM_LENGTH];
3215 	int err;
3216 
3217 	/* Catch broken hardware that put the wrong VF device ID in
3218 	 * the PCIe SR-IOV capability.
3219 	 */
3220 	if (pdev->is_virtfn) {
3221 		WARN(1, KERN_ERR "%s (%x:%x) should not be a VF!\n",
3222 			pci_name(pdev), pdev->vendor, pdev->device);
3223 		return -EINVAL;
3224 	}
3225 
3226 	err = pci_enable_device_mem(pdev);
3227 	if (err)
3228 		return err;
3229 
3230 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3231 	if (err) {
3232 		dev_err(&pdev->dev,
3233 			"No usable DMA configuration, aborting\n");
3234 		goto err_dma;
3235 	}
3236 
3237 	err = pci_request_mem_regions(pdev, igb_driver_name);
3238 	if (err)
3239 		goto err_pci_reg;
3240 
3241 	pci_set_master(pdev);
3242 	pci_save_state(pdev);
3243 
3244 	err = -ENOMEM;
3245 	netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
3246 				   IGB_MAX_TX_QUEUES);
3247 	if (!netdev)
3248 		goto err_alloc_etherdev;
3249 
3250 	SET_NETDEV_DEV(netdev, &pdev->dev);
3251 
3252 	pci_set_drvdata(pdev, netdev);
3253 	adapter = netdev_priv(netdev);
3254 	adapter->netdev = netdev;
3255 	adapter->pdev = pdev;
3256 	hw = &adapter->hw;
3257 	hw->back = adapter;
3258 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3259 
3260 	err = -EIO;
3261 	adapter->io_addr = pci_iomap(pdev, 0, 0);
3262 	if (!adapter->io_addr)
3263 		goto err_ioremap;
3264 	/* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
3265 	hw->hw_addr = adapter->io_addr;
3266 
3267 	netdev->netdev_ops = &igb_netdev_ops;
3268 	igb_set_ethtool_ops(netdev);
3269 	netdev->watchdog_timeo = 5 * HZ;
3270 
3271 	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
3272 
3273 	netdev->mem_start = pci_resource_start(pdev, 0);
3274 	netdev->mem_end = pci_resource_end(pdev, 0);
3275 
3276 	/* PCI config space info */
3277 	hw->vendor_id = pdev->vendor;
3278 	hw->device_id = pdev->device;
3279 	hw->revision_id = pdev->revision;
3280 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
3281 	hw->subsystem_device_id = pdev->subsystem_device;
3282 
3283 	/* Copy the default MAC, PHY and NVM function pointers */
3284 	memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
3285 	memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
3286 	memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
3287 	/* Initialize skew-specific constants */
3288 	err = ei->get_invariants(hw);
3289 	if (err)
3290 		goto err_sw_init;
3291 
3292 	/* setup the private structure */
3293 	err = igb_sw_init(adapter);
3294 	if (err)
3295 		goto err_sw_init;
3296 
3297 	igb_get_bus_info_pcie(hw);
3298 
3299 	hw->phy.autoneg_wait_to_complete = false;
3300 
3301 	/* Copper options */
3302 	if (hw->phy.media_type == e1000_media_type_copper) {
3303 		hw->phy.mdix = AUTO_ALL_MODES;
3304 		hw->phy.disable_polarity_correction = false;
3305 		hw->phy.ms_type = e1000_ms_hw_default;
3306 	}
3307 
3308 	if (igb_check_reset_block(hw))
3309 		dev_info(&pdev->dev,
3310 			"PHY reset is blocked due to SOL/IDER session.\n");
3311 
3312 	/* features is initialized to 0 in allocation, it might have bits
3313 	 * set by igb_sw_init so we should use an or instead of an
3314 	 * assignment.
3315 	 */
3316 	netdev->features |= NETIF_F_SG |
3317 			    NETIF_F_TSO |
3318 			    NETIF_F_TSO6 |
3319 			    NETIF_F_RXHASH |
3320 			    NETIF_F_RXCSUM |
3321 			    NETIF_F_HW_CSUM;
3322 
3323 	if (hw->mac.type >= e1000_82576)
3324 		netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;
3325 
3326 	if (hw->mac.type >= e1000_i350)
3327 		netdev->features |= NETIF_F_HW_TC;
3328 
3329 #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
3330 				  NETIF_F_GSO_GRE_CSUM | \
3331 				  NETIF_F_GSO_IPXIP4 | \
3332 				  NETIF_F_GSO_IPXIP6 | \
3333 				  NETIF_F_GSO_UDP_TUNNEL | \
3334 				  NETIF_F_GSO_UDP_TUNNEL_CSUM)
3335 
3336 	netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES;
3337 	netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES;
3338 
3339 	/* copy netdev features into list of user selectable features */
3340 	netdev->hw_features |= netdev->features |
3341 			       NETIF_F_HW_VLAN_CTAG_RX |
3342 			       NETIF_F_HW_VLAN_CTAG_TX |
3343 			       NETIF_F_RXALL;
3344 
3345 	if (hw->mac.type >= e1000_i350)
3346 		netdev->hw_features |= NETIF_F_NTUPLE;
3347 
3348 	netdev->features |= NETIF_F_HIGHDMA;
3349 
3350 	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
3351 	netdev->mpls_features |= NETIF_F_HW_CSUM;
3352 	netdev->hw_enc_features |= netdev->vlan_features;
3353 
3354 	/* set this bit last since it cannot be part of vlan_features */
3355 	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
3356 			    NETIF_F_HW_VLAN_CTAG_RX |
3357 			    NETIF_F_HW_VLAN_CTAG_TX;
3358 
3359 	netdev->priv_flags |= IFF_SUPP_NOFCS;
3360 
3361 	netdev->priv_flags |= IFF_UNICAST_FLT;
3362 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
3363 
3364 	/* MTU range: 68 - 9216 */
3365 	netdev->min_mtu = ETH_MIN_MTU;
3366 	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
3367 
3368 	adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
3369 
3370 	/* before reading the NVM, reset the controller to put the device in a
3371 	 * known good starting state
3372 	 */
3373 	hw->mac.ops.reset_hw(hw);
3374 
3375 	/* make sure the NVM is good , i211/i210 parts can have special NVM
3376 	 * that doesn't contain a checksum
3377 	 */
3378 	switch (hw->mac.type) {
3379 	case e1000_i210:
3380 	case e1000_i211:
3381 		if (igb_get_flash_presence_i210(hw)) {
3382 			if (hw->nvm.ops.validate(hw) < 0) {
3383 				dev_err(&pdev->dev,
3384 					"The NVM Checksum Is Not Valid\n");
3385 				err = -EIO;
3386 				goto err_eeprom;
3387 			}
3388 		}
3389 		break;
3390 	default:
3391 		if (hw->nvm.ops.validate(hw) < 0) {
3392 			dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
3393 			err = -EIO;
3394 			goto err_eeprom;
3395 		}
3396 		break;
3397 	}
3398 
3399 	if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
3400 		/* copy the MAC address out of the NVM */
3401 		if (hw->mac.ops.read_mac_addr(hw))
3402 			dev_err(&pdev->dev, "NVM Read Error\n");
3403 	}
3404 
3405 	eth_hw_addr_set(netdev, hw->mac.addr);
3406 
3407 	if (!is_valid_ether_addr(netdev->dev_addr)) {
3408 		dev_err(&pdev->dev, "Invalid MAC Address\n");
3409 		err = -EIO;
3410 		goto err_eeprom;
3411 	}
3412 
3413 	igb_set_default_mac_filter(adapter);
3414 
3415 	/* get firmware version for ethtool -i */
3416 	igb_set_fw_version(adapter);
3417 
3418 	/* configure RXPBSIZE and TXPBSIZE */
3419 	if (hw->mac.type == e1000_i210) {
3420 		wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
3421 		wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
3422 	}
3423 
3424 	timer_setup(&adapter->watchdog_timer, igb_watchdog, 0);
3425 	timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0);
3426 
3427 	INIT_WORK(&adapter->reset_task, igb_reset_task);
3428 	INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
3429 
3430 	/* Initialize link properties that are user-changeable */
3431 	adapter->fc_autoneg = true;
3432 	hw->mac.autoneg = true;
3433 	hw->phy.autoneg_advertised = 0x2f;
3434 
3435 	hw->fc.requested_mode = e1000_fc_default;
3436 	hw->fc.current_mode = e1000_fc_default;
3437 
3438 	igb_validate_mdi_setting(hw);
3439 
3440 	/* By default, support wake on port A */
3441 	if (hw->bus.func == 0)
3442 		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3443 
3444 	/* Check the NVM for wake support on non-port A ports */
3445 	if (hw->mac.type >= e1000_82580)
3446 		hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
3447 				 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
3448 				 &eeprom_data);
3449 	else if (hw->bus.func == 1)
3450 		hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
3451 
3452 	if (eeprom_data & IGB_EEPROM_APME)
3453 		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3454 
3455 	/* now that we have the eeprom settings, apply the special cases where
3456 	 * the eeprom may be wrong or the board simply won't support wake on
3457 	 * lan on a particular port
3458 	 */
3459 	switch (pdev->device) {
3460 	case E1000_DEV_ID_82575GB_QUAD_COPPER:
3461 		adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3462 		break;
3463 	case E1000_DEV_ID_82575EB_FIBER_SERDES:
3464 	case E1000_DEV_ID_82576_FIBER:
3465 	case E1000_DEV_ID_82576_SERDES:
3466 		/* Wake events only supported on port A for dual fiber
3467 		 * regardless of eeprom setting
3468 		 */
3469 		if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
3470 			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3471 		break;
3472 	case E1000_DEV_ID_82576_QUAD_COPPER:
3473 	case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
3474 		/* if quad port adapter, disable WoL on all but port A */
3475 		if (global_quad_port_a != 0)
3476 			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3477 		else
3478 			adapter->flags |= IGB_FLAG_QUAD_PORT_A;
3479 		/* Reset for multiple quad port adapters */
3480 		if (++global_quad_port_a == 4)
3481 			global_quad_port_a = 0;
3482 		break;
3483 	default:
3484 		/* If the device can't wake, don't set software support */
3485 		if (!device_can_wakeup(&adapter->pdev->dev))
3486 			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3487 	}
3488 
3489 	/* initialize the wol settings based on the eeprom settings */
3490 	if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
3491 		adapter->wol |= E1000_WUFC_MAG;
3492 
3493 	/* Some vendors want WoL disabled by default, but still supported */
3494 	if ((hw->mac.type == e1000_i350) &&
3495 	    (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
3496 		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3497 		adapter->wol = 0;
3498 	}
3499 
3500 	/* Some vendors want the ability to Use the EEPROM setting as
3501 	 * enable/disable only, and not for capability
3502 	 */
3503 	if (((hw->mac.type == e1000_i350) ||
3504 	     (hw->mac.type == e1000_i354)) &&
3505 	    (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) {
3506 		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3507 		adapter->wol = 0;
3508 	}
3509 	if (hw->mac.type == e1000_i350) {
3510 		if (((pdev->subsystem_device == 0x5001) ||
3511 		     (pdev->subsystem_device == 0x5002)) &&
3512 				(hw->bus.func == 0)) {
3513 			adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3514 			adapter->wol = 0;
3515 		}
3516 		if (pdev->subsystem_device == 0x1F52)
3517 			adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3518 	}
3519 
3520 	device_set_wakeup_enable(&adapter->pdev->dev,
3521 				 adapter->flags & IGB_FLAG_WOL_SUPPORTED);
3522 
3523 	/* reset the hardware with the new settings */
3524 	igb_reset(adapter);
3525 
3526 	/* Init the I2C interface */
3527 	err = igb_init_i2c(adapter);
3528 	if (err) {
3529 		dev_err(&pdev->dev, "failed to init i2c interface\n");
3530 		goto err_eeprom;
3531 	}
3532 
3533 	/* let the f/w know that the h/w is now under the control of the
3534 	 * driver.
3535 	 */
3536 	igb_get_hw_control(adapter);
3537 
3538 	strcpy(netdev->name, "eth%d");
3539 	err = register_netdev(netdev);
3540 	if (err)
3541 		goto err_register;
3542 
3543 	/* carrier off reporting is important to ethtool even BEFORE open */
3544 	netif_carrier_off(netdev);
3545 
3546 #ifdef CONFIG_IGB_DCA
3547 	if (dca_add_requester(&pdev->dev) == 0) {
3548 		adapter->flags |= IGB_FLAG_DCA_ENABLED;
3549 		dev_info(&pdev->dev, "DCA enabled\n");
3550 		igb_setup_dca(adapter);
3551 	}
3552 
3553 #endif
3554 #ifdef CONFIG_IGB_HWMON
3555 	/* Initialize the thermal sensor on i350 devices. */
3556 	if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
3557 		u16 ets_word;
3558 
3559 		/* Read the NVM to determine if this i350 device supports an
3560 		 * external thermal sensor.
3561 		 */
3562 		hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
3563 		if (ets_word != 0x0000 && ets_word != 0xFFFF)
3564 			adapter->ets = true;
3565 		else
3566 			adapter->ets = false;
3567 		/* Only enable I2C bit banging if an external thermal
3568 		 * sensor is supported.
3569 		 */
3570 		if (adapter->ets)
3571 			igb_set_i2c_bb(hw);
3572 		hw->mac.ops.init_thermal_sensor_thresh(hw);
3573 		if (igb_sysfs_init(adapter))
3574 			dev_err(&pdev->dev,
3575 				"failed to allocate sysfs resources\n");
3576 	} else {
3577 		adapter->ets = false;
3578 	}
3579 #endif
3580 	/* Check if Media Autosense is enabled */
3581 	adapter->ei = *ei;
3582 	if (hw->dev_spec._82575.mas_capable)
3583 		igb_init_mas(adapter);
3584 
3585 	/* do hw tstamp init after resetting */
3586 	igb_ptp_init(adapter);
3587 
3588 	dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
3589 	/* print bus type/speed/width info, not applicable to i354 */
3590 	if (hw->mac.type != e1000_i354) {
3591 		dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
3592 			 netdev->name,
3593 			 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
3594 			  (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
3595 			   "unknown"),
3596 			 ((hw->bus.width == e1000_bus_width_pcie_x4) ?
3597 			  "Width x4" :
3598 			  (hw->bus.width == e1000_bus_width_pcie_x2) ?
3599 			  "Width x2" :
3600 			  (hw->bus.width == e1000_bus_width_pcie_x1) ?
3601 			  "Width x1" : "unknown"), netdev->dev_addr);
3602 	}
3603 
3604 	if ((hw->mac.type == e1000_82576 &&
3605 	     rd32(E1000_EECD) & E1000_EECD_PRES) ||
3606 	    (hw->mac.type >= e1000_i210 ||
3607 	     igb_get_flash_presence_i210(hw))) {
3608 		ret_val = igb_read_part_string(hw, part_str,
3609 					       E1000_PBANUM_LENGTH);
3610 	} else {
3611 		ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
3612 	}
3613 
3614 	if (ret_val)
3615 		strcpy(part_str, "Unknown");
3616 	dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
3617 	dev_info(&pdev->dev,
3618 		"Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
3619 		(adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" :
3620 		(adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
3621 		adapter->num_rx_queues, adapter->num_tx_queues);
3622 	if (hw->phy.media_type == e1000_media_type_copper) {
3623 		switch (hw->mac.type) {
3624 		case e1000_i350:
3625 		case e1000_i210:
3626 		case e1000_i211:
3627 			/* Enable EEE for internal copper PHY devices */
3628 			err = igb_set_eee_i350(hw, true, true);
3629 			if ((!err) &&
3630 			    (!hw->dev_spec._82575.eee_disable)) {
3631 				adapter->eee_advert =
3632 					MDIO_EEE_100TX | MDIO_EEE_1000T;
3633 				adapter->flags |= IGB_FLAG_EEE;
3634 			}
3635 			break;
3636 		case e1000_i354:
3637 			if ((rd32(E1000_CTRL_EXT) &
3638 			    E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3639 				err = igb_set_eee_i354(hw, true, true);
3640 				if ((!err) &&
3641 					(!hw->dev_spec._82575.eee_disable)) {
3642 					adapter->eee_advert =
3643 					   MDIO_EEE_100TX | MDIO_EEE_1000T;
3644 					adapter->flags |= IGB_FLAG_EEE;
3645 				}
3646 			}
3647 			break;
3648 		default:
3649 			break;
3650 		}
3651 	}
3652 
3653 	dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
3654 
3655 	pm_runtime_put_noidle(&pdev->dev);
3656 	return 0;
3657 
3658 err_register:
3659 	igb_release_hw_control(adapter);
3660 	memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
3661 err_eeprom:
3662 	if (!igb_check_reset_block(hw))
3663 		igb_reset_phy(hw);
3664 
3665 	if (hw->flash_address)
3666 		iounmap(hw->flash_address);
3667 err_sw_init:
3668 	kfree(adapter->mac_table);
3669 	kfree(adapter->shadow_vfta);
3670 	igb_clear_interrupt_scheme(adapter);
3671 #ifdef CONFIG_PCI_IOV
3672 	igb_disable_sriov(pdev, false);
3673 #endif
3674 	pci_iounmap(pdev, adapter->io_addr);
3675 err_ioremap:
3676 	free_netdev(netdev);
3677 err_alloc_etherdev:
3678 	pci_release_mem_regions(pdev);
3679 err_pci_reg:
3680 err_dma:
3681 	pci_disable_device(pdev);
3682 	return err;
3683 }
3684 
3685 #ifdef CONFIG_PCI_IOV
igb_sriov_reinit(struct pci_dev * dev)3686 static int igb_sriov_reinit(struct pci_dev *dev)
3687 {
3688 	struct net_device *netdev = pci_get_drvdata(dev);
3689 	struct igb_adapter *adapter = netdev_priv(netdev);
3690 	struct pci_dev *pdev = adapter->pdev;
3691 
3692 	rtnl_lock();
3693 
3694 	if (netif_running(netdev))
3695 		igb_close(netdev);
3696 	else
3697 		igb_reset(adapter);
3698 
3699 	igb_clear_interrupt_scheme(adapter);
3700 
3701 	igb_init_queue_configuration(adapter);
3702 
3703 	if (igb_init_interrupt_scheme(adapter, true)) {
3704 		rtnl_unlock();
3705 		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
3706 		return -ENOMEM;
3707 	}
3708 
3709 	if (netif_running(netdev))
3710 		igb_open(netdev);
3711 
3712 	rtnl_unlock();
3713 
3714 	return 0;
3715 }
3716 
igb_disable_sriov(struct pci_dev * pdev,bool reinit)3717 static int igb_disable_sriov(struct pci_dev *pdev, bool reinit)
3718 {
3719 	struct net_device *netdev = pci_get_drvdata(pdev);
3720 	struct igb_adapter *adapter = netdev_priv(netdev);
3721 	struct e1000_hw *hw = &adapter->hw;
3722 	unsigned long flags;
3723 
3724 	/* reclaim resources allocated to VFs */
3725 	if (adapter->vf_data) {
3726 		/* disable iov and allow time for transactions to clear */
3727 		if (pci_vfs_assigned(pdev)) {
3728 			dev_warn(&pdev->dev,
3729 				 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
3730 			return -EPERM;
3731 		} else {
3732 			pci_disable_sriov(pdev);
3733 			msleep(500);
3734 		}
3735 		spin_lock_irqsave(&adapter->vfs_lock, flags);
3736 		kfree(adapter->vf_mac_list);
3737 		adapter->vf_mac_list = NULL;
3738 		kfree(adapter->vf_data);
3739 		adapter->vf_data = NULL;
3740 		adapter->vfs_allocated_count = 0;
3741 		spin_unlock_irqrestore(&adapter->vfs_lock, flags);
3742 		wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
3743 		wrfl();
3744 		msleep(100);
3745 		dev_info(&pdev->dev, "IOV Disabled\n");
3746 
3747 		/* Re-enable DMA Coalescing flag since IOV is turned off */
3748 		adapter->flags |= IGB_FLAG_DMAC;
3749 	}
3750 
3751 	return reinit ? igb_sriov_reinit(pdev) : 0;
3752 }
3753 
igb_enable_sriov(struct pci_dev * pdev,int num_vfs,bool reinit)3754 static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs, bool reinit)
3755 {
3756 	struct net_device *netdev = pci_get_drvdata(pdev);
3757 	struct igb_adapter *adapter = netdev_priv(netdev);
3758 	int old_vfs = pci_num_vf(pdev);
3759 	struct vf_mac_filter *mac_list;
3760 	int err = 0;
3761 	int num_vf_mac_filters, i;
3762 
3763 	if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) {
3764 		err = -EPERM;
3765 		goto out;
3766 	}
3767 	if (!num_vfs)
3768 		goto out;
3769 
3770 	if (old_vfs) {
3771 		dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
3772 			 old_vfs, max_vfs);
3773 		adapter->vfs_allocated_count = old_vfs;
3774 	} else
3775 		adapter->vfs_allocated_count = num_vfs;
3776 
3777 	adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
3778 				sizeof(struct vf_data_storage), GFP_KERNEL);
3779 
3780 	/* if allocation failed then we do not support SR-IOV */
3781 	if (!adapter->vf_data) {
3782 		adapter->vfs_allocated_count = 0;
3783 		err = -ENOMEM;
3784 		goto out;
3785 	}
3786 
3787 	/* Due to the limited number of RAR entries calculate potential
3788 	 * number of MAC filters available for the VFs. Reserve entries
3789 	 * for PF default MAC, PF MAC filters and at least one RAR entry
3790 	 * for each VF for VF MAC.
3791 	 */
3792 	num_vf_mac_filters = adapter->hw.mac.rar_entry_count -
3793 			     (1 + IGB_PF_MAC_FILTERS_RESERVED +
3794 			      adapter->vfs_allocated_count);
3795 
3796 	adapter->vf_mac_list = kcalloc(num_vf_mac_filters,
3797 				       sizeof(struct vf_mac_filter),
3798 				       GFP_KERNEL);
3799 
3800 	mac_list = adapter->vf_mac_list;
3801 	INIT_LIST_HEAD(&adapter->vf_macs.l);
3802 
3803 	if (adapter->vf_mac_list) {
3804 		/* Initialize list of VF MAC filters */
3805 		for (i = 0; i < num_vf_mac_filters; i++) {
3806 			mac_list->vf = -1;
3807 			mac_list->free = true;
3808 			list_add(&mac_list->l, &adapter->vf_macs.l);
3809 			mac_list++;
3810 		}
3811 	} else {
3812 		/* If we could not allocate memory for the VF MAC filters
3813 		 * we can continue without this feature but warn user.
3814 		 */
3815 		dev_err(&pdev->dev,
3816 			"Unable to allocate memory for VF MAC filter list\n");
3817 	}
3818 
3819 	dev_info(&pdev->dev, "%d VFs allocated\n",
3820 		 adapter->vfs_allocated_count);
3821 	for (i = 0; i < adapter->vfs_allocated_count; i++)
3822 		igb_vf_configure(adapter, i);
3823 
3824 	/* DMA Coalescing is not supported in IOV mode. */
3825 	adapter->flags &= ~IGB_FLAG_DMAC;
3826 
3827 	if (reinit) {
3828 		err = igb_sriov_reinit(pdev);
3829 		if (err)
3830 			goto err_out;
3831 	}
3832 
3833 	/* only call pci_enable_sriov() if no VFs are allocated already */
3834 	if (!old_vfs) {
3835 		err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
3836 		if (err)
3837 			goto err_out;
3838 	}
3839 
3840 	goto out;
3841 
3842 err_out:
3843 	kfree(adapter->vf_mac_list);
3844 	adapter->vf_mac_list = NULL;
3845 	kfree(adapter->vf_data);
3846 	adapter->vf_data = NULL;
3847 	adapter->vfs_allocated_count = 0;
3848 out:
3849 	return err;
3850 }
3851 
3852 #endif
3853 /**
3854  *  igb_remove_i2c - Cleanup  I2C interface
3855  *  @adapter: pointer to adapter structure
3856  **/
igb_remove_i2c(struct igb_adapter * adapter)3857 static void igb_remove_i2c(struct igb_adapter *adapter)
3858 {
3859 	/* free the adapter bus structure */
3860 	i2c_del_adapter(&adapter->i2c_adap);
3861 }
3862 
3863 /**
3864  *  igb_remove - Device Removal Routine
3865  *  @pdev: PCI device information struct
3866  *
3867  *  igb_remove is called by the PCI subsystem to alert the driver
3868  *  that it should release a PCI device.  The could be caused by a
3869  *  Hot-Plug event, or because the driver is going to be removed from
3870  *  memory.
3871  **/
igb_remove(struct pci_dev * pdev)3872 static void igb_remove(struct pci_dev *pdev)
3873 {
3874 	struct net_device *netdev = pci_get_drvdata(pdev);
3875 	struct igb_adapter *adapter = netdev_priv(netdev);
3876 	struct e1000_hw *hw = &adapter->hw;
3877 
3878 	pm_runtime_get_noresume(&pdev->dev);
3879 #ifdef CONFIG_IGB_HWMON
3880 	igb_sysfs_exit(adapter);
3881 #endif
3882 	igb_remove_i2c(adapter);
3883 	igb_ptp_stop(adapter);
3884 	/* The watchdog timer may be rescheduled, so explicitly
3885 	 * disable watchdog from being rescheduled.
3886 	 */
3887 	set_bit(__IGB_DOWN, &adapter->state);
3888 	del_timer_sync(&adapter->watchdog_timer);
3889 	del_timer_sync(&adapter->phy_info_timer);
3890 
3891 	cancel_work_sync(&adapter->reset_task);
3892 	cancel_work_sync(&adapter->watchdog_task);
3893 
3894 #ifdef CONFIG_IGB_DCA
3895 	if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
3896 		dev_info(&pdev->dev, "DCA disabled\n");
3897 		dca_remove_requester(&pdev->dev);
3898 		adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
3899 		wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
3900 	}
3901 #endif
3902 
3903 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
3904 	 * would have already happened in close and is redundant.
3905 	 */
3906 	igb_release_hw_control(adapter);
3907 
3908 #ifdef CONFIG_PCI_IOV
3909 	igb_disable_sriov(pdev, false);
3910 #endif
3911 
3912 	unregister_netdev(netdev);
3913 
3914 	igb_clear_interrupt_scheme(adapter);
3915 
3916 	pci_iounmap(pdev, adapter->io_addr);
3917 	if (hw->flash_address)
3918 		iounmap(hw->flash_address);
3919 	pci_release_mem_regions(pdev);
3920 
3921 	kfree(adapter->mac_table);
3922 	kfree(adapter->shadow_vfta);
3923 	free_netdev(netdev);
3924 
3925 	pci_disable_device(pdev);
3926 }
3927 
3928 /**
3929  *  igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
3930  *  @adapter: board private structure to initialize
3931  *
3932  *  This function initializes the vf specific data storage and then attempts to
3933  *  allocate the VFs.  The reason for ordering it this way is because it is much
3934  *  mor expensive time wise to disable SR-IOV than it is to allocate and free
3935  *  the memory for the VFs.
3936  **/
igb_probe_vfs(struct igb_adapter * adapter)3937 static void igb_probe_vfs(struct igb_adapter *adapter)
3938 {
3939 #ifdef CONFIG_PCI_IOV
3940 	struct pci_dev *pdev = adapter->pdev;
3941 	struct e1000_hw *hw = &adapter->hw;
3942 
3943 	/* Virtualization features not supported on i210 and 82580 family. */
3944 	if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211) ||
3945 	    (hw->mac.type == e1000_82580))
3946 		return;
3947 
3948 	/* Of the below we really only want the effect of getting
3949 	 * IGB_FLAG_HAS_MSIX set (if available), without which
3950 	 * igb_enable_sriov() has no effect.
3951 	 */
3952 	igb_set_interrupt_capability(adapter, true);
3953 	igb_reset_interrupt_capability(adapter);
3954 
3955 	pci_sriov_set_totalvfs(pdev, 7);
3956 	igb_enable_sriov(pdev, max_vfs, false);
3957 
3958 #endif /* CONFIG_PCI_IOV */
3959 }
3960 
igb_get_max_rss_queues(struct igb_adapter * adapter)3961 unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter)
3962 {
3963 	struct e1000_hw *hw = &adapter->hw;
3964 	unsigned int max_rss_queues;
3965 
3966 	/* Determine the maximum number of RSS queues supported. */
3967 	switch (hw->mac.type) {
3968 	case e1000_i211:
3969 		max_rss_queues = IGB_MAX_RX_QUEUES_I211;
3970 		break;
3971 	case e1000_82575:
3972 	case e1000_i210:
3973 		max_rss_queues = IGB_MAX_RX_QUEUES_82575;
3974 		break;
3975 	case e1000_i350:
3976 		/* I350 cannot do RSS and SR-IOV at the same time */
3977 		if (!!adapter->vfs_allocated_count) {
3978 			max_rss_queues = 1;
3979 			break;
3980 		}
3981 		fallthrough;
3982 	case e1000_82576:
3983 		if (!!adapter->vfs_allocated_count) {
3984 			max_rss_queues = 2;
3985 			break;
3986 		}
3987 		fallthrough;
3988 	case e1000_82580:
3989 	case e1000_i354:
3990 	default:
3991 		max_rss_queues = IGB_MAX_RX_QUEUES;
3992 		break;
3993 	}
3994 
3995 	return max_rss_queues;
3996 }
3997 
igb_init_queue_configuration(struct igb_adapter * adapter)3998 static void igb_init_queue_configuration(struct igb_adapter *adapter)
3999 {
4000 	u32 max_rss_queues;
4001 
4002 	max_rss_queues = igb_get_max_rss_queues(adapter);
4003 	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4004 
4005 	igb_set_flag_queue_pairs(adapter, max_rss_queues);
4006 }
4007 
igb_set_flag_queue_pairs(struct igb_adapter * adapter,const u32 max_rss_queues)4008 void igb_set_flag_queue_pairs(struct igb_adapter *adapter,
4009 			      const u32 max_rss_queues)
4010 {
4011 	struct e1000_hw *hw = &adapter->hw;
4012 
4013 	/* Determine if we need to pair queues. */
4014 	switch (hw->mac.type) {
4015 	case e1000_82575:
4016 	case e1000_i211:
4017 		/* Device supports enough interrupts without queue pairing. */
4018 		break;
4019 	case e1000_82576:
4020 	case e1000_82580:
4021 	case e1000_i350:
4022 	case e1000_i354:
4023 	case e1000_i210:
4024 	default:
4025 		/* If rss_queues > half of max_rss_queues, pair the queues in
4026 		 * order to conserve interrupts due to limited supply.
4027 		 */
4028 		if (adapter->rss_queues > (max_rss_queues / 2))
4029 			adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
4030 		else
4031 			adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS;
4032 		break;
4033 	}
4034 }
4035 
4036 /**
4037  *  igb_sw_init - Initialize general software structures (struct igb_adapter)
4038  *  @adapter: board private structure to initialize
4039  *
4040  *  igb_sw_init initializes the Adapter private data structure.
4041  *  Fields are initialized based on PCI device information and
4042  *  OS network device settings (MTU size).
4043  **/
igb_sw_init(struct igb_adapter * adapter)4044 static int igb_sw_init(struct igb_adapter *adapter)
4045 {
4046 	struct e1000_hw *hw = &adapter->hw;
4047 	struct net_device *netdev = adapter->netdev;
4048 	struct pci_dev *pdev = adapter->pdev;
4049 
4050 	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4051 
4052 	/* set default ring sizes */
4053 	adapter->tx_ring_count = IGB_DEFAULT_TXD;
4054 	adapter->rx_ring_count = IGB_DEFAULT_RXD;
4055 
4056 	/* set default ITR values */
4057 	adapter->rx_itr_setting = IGB_DEFAULT_ITR;
4058 	adapter->tx_itr_setting = IGB_DEFAULT_ITR;
4059 
4060 	/* set default work limits */
4061 	adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
4062 
4063 	adapter->max_frame_size = netdev->mtu + IGB_ETH_PKT_HDR_PAD;
4064 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4065 
4066 	spin_lock_init(&adapter->nfc_lock);
4067 	spin_lock_init(&adapter->stats64_lock);
4068 
4069 	/* init spinlock to avoid concurrency of VF resources */
4070 	spin_lock_init(&adapter->vfs_lock);
4071 #ifdef CONFIG_PCI_IOV
4072 	switch (hw->mac.type) {
4073 	case e1000_82576:
4074 	case e1000_i350:
4075 		if (max_vfs > 7) {
4076 			dev_warn(&pdev->dev,
4077 				 "Maximum of 7 VFs per PF, using max\n");
4078 			max_vfs = adapter->vfs_allocated_count = 7;
4079 		} else
4080 			adapter->vfs_allocated_count = max_vfs;
4081 		if (adapter->vfs_allocated_count)
4082 			dev_warn(&pdev->dev,
4083 				 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
4084 		break;
4085 	default:
4086 		break;
4087 	}
4088 #endif /* CONFIG_PCI_IOV */
4089 
4090 	/* Assume MSI-X interrupts, will be checked during IRQ allocation */
4091 	adapter->flags |= IGB_FLAG_HAS_MSIX;
4092 
4093 	adapter->mac_table = kcalloc(hw->mac.rar_entry_count,
4094 				     sizeof(struct igb_mac_addr),
4095 				     GFP_KERNEL);
4096 	if (!adapter->mac_table)
4097 		return -ENOMEM;
4098 
4099 	igb_probe_vfs(adapter);
4100 
4101 	igb_init_queue_configuration(adapter);
4102 
4103 	/* Setup and initialize a copy of the hw vlan table array */
4104 	adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
4105 				       GFP_KERNEL);
4106 	if (!adapter->shadow_vfta)
4107 		return -ENOMEM;
4108 
4109 	/* This call may decrease the number of queues */
4110 	if (igb_init_interrupt_scheme(adapter, true)) {
4111 		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4112 		return -ENOMEM;
4113 	}
4114 
4115 	/* Explicitly disable IRQ since the NIC can be in any state. */
4116 	igb_irq_disable(adapter);
4117 
4118 	if (hw->mac.type >= e1000_i350)
4119 		adapter->flags &= ~IGB_FLAG_DMAC;
4120 
4121 	set_bit(__IGB_DOWN, &adapter->state);
4122 	return 0;
4123 }
4124 
4125 /**
4126  *  __igb_open - Called when a network interface is made active
4127  *  @netdev: network interface device structure
4128  *  @resuming: indicates whether we are in a resume call
4129  *
4130  *  Returns 0 on success, negative value on failure
4131  *
4132  *  The open entry point is called when a network interface is made
4133  *  active by the system (IFF_UP).  At this point all resources needed
4134  *  for transmit and receive operations are allocated, the interrupt
4135  *  handler is registered with the OS, the watchdog timer is started,
4136  *  and the stack is notified that the interface is ready.
4137  **/
__igb_open(struct net_device * netdev,bool resuming)4138 static int __igb_open(struct net_device *netdev, bool resuming)
4139 {
4140 	struct igb_adapter *adapter = netdev_priv(netdev);
4141 	struct e1000_hw *hw = &adapter->hw;
4142 	struct pci_dev *pdev = adapter->pdev;
4143 	int err;
4144 	int i;
4145 
4146 	/* disallow open during test */
4147 	if (test_bit(__IGB_TESTING, &adapter->state)) {
4148 		WARN_ON(resuming);
4149 		return -EBUSY;
4150 	}
4151 
4152 	if (!resuming)
4153 		pm_runtime_get_sync(&pdev->dev);
4154 
4155 	netif_carrier_off(netdev);
4156 
4157 	/* allocate transmit descriptors */
4158 	err = igb_setup_all_tx_resources(adapter);
4159 	if (err)
4160 		goto err_setup_tx;
4161 
4162 	/* allocate receive descriptors */
4163 	err = igb_setup_all_rx_resources(adapter);
4164 	if (err)
4165 		goto err_setup_rx;
4166 
4167 	igb_power_up_link(adapter);
4168 
4169 	/* before we allocate an interrupt, we must be ready to handle it.
4170 	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
4171 	 * as soon as we call pci_request_irq, so we have to setup our
4172 	 * clean_rx handler before we do so.
4173 	 */
4174 	igb_configure(adapter);
4175 
4176 	err = igb_request_irq(adapter);
4177 	if (err)
4178 		goto err_req_irq;
4179 
4180 	/* Notify the stack of the actual queue counts. */
4181 	err = netif_set_real_num_tx_queues(adapter->netdev,
4182 					   adapter->num_tx_queues);
4183 	if (err)
4184 		goto err_set_queues;
4185 
4186 	err = netif_set_real_num_rx_queues(adapter->netdev,
4187 					   adapter->num_rx_queues);
4188 	if (err)
4189 		goto err_set_queues;
4190 
4191 	/* From here on the code is the same as igb_up() */
4192 	clear_bit(__IGB_DOWN, &adapter->state);
4193 
4194 	for (i = 0; i < adapter->num_q_vectors; i++)
4195 		napi_enable(&(adapter->q_vector[i]->napi));
4196 
4197 	/* Clear any pending interrupts. */
4198 	rd32(E1000_TSICR);
4199 	rd32(E1000_ICR);
4200 
4201 	igb_irq_enable(adapter);
4202 
4203 	/* notify VFs that reset has been completed */
4204 	if (adapter->vfs_allocated_count) {
4205 		u32 reg_data = rd32(E1000_CTRL_EXT);
4206 
4207 		reg_data |= E1000_CTRL_EXT_PFRSTD;
4208 		wr32(E1000_CTRL_EXT, reg_data);
4209 	}
4210 
4211 	netif_tx_start_all_queues(netdev);
4212 
4213 	if (!resuming)
4214 		pm_runtime_put(&pdev->dev);
4215 
4216 	/* start the watchdog. */
4217 	hw->mac.get_link_status = 1;
4218 	schedule_work(&adapter->watchdog_task);
4219 
4220 	return 0;
4221 
4222 err_set_queues:
4223 	igb_free_irq(adapter);
4224 err_req_irq:
4225 	igb_release_hw_control(adapter);
4226 	igb_power_down_link(adapter);
4227 	igb_free_all_rx_resources(adapter);
4228 err_setup_rx:
4229 	igb_free_all_tx_resources(adapter);
4230 err_setup_tx:
4231 	igb_reset(adapter);
4232 	if (!resuming)
4233 		pm_runtime_put(&pdev->dev);
4234 
4235 	return err;
4236 }
4237 
igb_open(struct net_device * netdev)4238 int igb_open(struct net_device *netdev)
4239 {
4240 	return __igb_open(netdev, false);
4241 }
4242 
4243 /**
4244  *  __igb_close - Disables a network interface
4245  *  @netdev: network interface device structure
4246  *  @suspending: indicates we are in a suspend call
4247  *
4248  *  Returns 0, this is not allowed to fail
4249  *
4250  *  The close entry point is called when an interface is de-activated
4251  *  by the OS.  The hardware is still under the driver's control, but
4252  *  needs to be disabled.  A global MAC reset is issued to stop the
4253  *  hardware, and all transmit and receive resources are freed.
4254  **/
__igb_close(struct net_device * netdev,bool suspending)4255 static int __igb_close(struct net_device *netdev, bool suspending)
4256 {
4257 	struct igb_adapter *adapter = netdev_priv(netdev);
4258 	struct pci_dev *pdev = adapter->pdev;
4259 
4260 	WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
4261 
4262 	if (!suspending)
4263 		pm_runtime_get_sync(&pdev->dev);
4264 
4265 	igb_down(adapter);
4266 	igb_free_irq(adapter);
4267 
4268 	igb_free_all_tx_resources(adapter);
4269 	igb_free_all_rx_resources(adapter);
4270 
4271 	if (!suspending)
4272 		pm_runtime_put_sync(&pdev->dev);
4273 	return 0;
4274 }
4275 
igb_close(struct net_device * netdev)4276 int igb_close(struct net_device *netdev)
4277 {
4278 	if (netif_device_present(netdev) || netdev->dismantle)
4279 		return __igb_close(netdev, false);
4280 	return 0;
4281 }
4282 
4283 /**
4284  *  igb_setup_tx_resources - allocate Tx resources (Descriptors)
4285  *  @tx_ring: tx descriptor ring (for a specific queue) to setup
4286  *
4287  *  Return 0 on success, negative on failure
4288  **/
igb_setup_tx_resources(struct igb_ring * tx_ring)4289 int igb_setup_tx_resources(struct igb_ring *tx_ring)
4290 {
4291 	struct device *dev = tx_ring->dev;
4292 	int size;
4293 
4294 	size = sizeof(struct igb_tx_buffer) * tx_ring->count;
4295 
4296 	tx_ring->tx_buffer_info = vmalloc(size);
4297 	if (!tx_ring->tx_buffer_info)
4298 		goto err;
4299 
4300 	/* round up to nearest 4K */
4301 	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
4302 	tx_ring->size = ALIGN(tx_ring->size, 4096);
4303 
4304 	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
4305 					   &tx_ring->dma, GFP_KERNEL);
4306 	if (!tx_ring->desc)
4307 		goto err;
4308 
4309 	tx_ring->next_to_use = 0;
4310 	tx_ring->next_to_clean = 0;
4311 
4312 	return 0;
4313 
4314 err:
4315 	vfree(tx_ring->tx_buffer_info);
4316 	tx_ring->tx_buffer_info = NULL;
4317 	dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
4318 	return -ENOMEM;
4319 }
4320 
4321 /**
4322  *  igb_setup_all_tx_resources - wrapper to allocate Tx resources
4323  *				 (Descriptors) for all queues
4324  *  @adapter: board private structure
4325  *
4326  *  Return 0 on success, negative on failure
4327  **/
igb_setup_all_tx_resources(struct igb_adapter * adapter)4328 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
4329 {
4330 	struct pci_dev *pdev = adapter->pdev;
4331 	int i, err = 0;
4332 
4333 	for (i = 0; i < adapter->num_tx_queues; i++) {
4334 		err = igb_setup_tx_resources(adapter->tx_ring[i]);
4335 		if (err) {
4336 			dev_err(&pdev->dev,
4337 				"Allocation for Tx Queue %u failed\n", i);
4338 			for (i--; i >= 0; i--)
4339 				igb_free_tx_resources(adapter->tx_ring[i]);
4340 			break;
4341 		}
4342 	}
4343 
4344 	return err;
4345 }
4346 
4347 /**
4348  *  igb_setup_tctl - configure the transmit control registers
4349  *  @adapter: Board private structure
4350  **/
igb_setup_tctl(struct igb_adapter * adapter)4351 void igb_setup_tctl(struct igb_adapter *adapter)
4352 {
4353 	struct e1000_hw *hw = &adapter->hw;
4354 	u32 tctl;
4355 
4356 	/* disable queue 0 which is enabled by default on 82575 and 82576 */
4357 	wr32(E1000_TXDCTL(0), 0);
4358 
4359 	/* Program the Transmit Control Register */
4360 	tctl = rd32(E1000_TCTL);
4361 	tctl &= ~E1000_TCTL_CT;
4362 	tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
4363 		(E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
4364 
4365 	igb_config_collision_dist(hw);
4366 
4367 	/* Enable transmits */
4368 	tctl |= E1000_TCTL_EN;
4369 
4370 	wr32(E1000_TCTL, tctl);
4371 }
4372 
4373 /**
4374  *  igb_configure_tx_ring - Configure transmit ring after Reset
4375  *  @adapter: board private structure
4376  *  @ring: tx ring to configure
4377  *
4378  *  Configure a transmit ring after a reset.
4379  **/
igb_configure_tx_ring(struct igb_adapter * adapter,struct igb_ring * ring)4380 void igb_configure_tx_ring(struct igb_adapter *adapter,
4381 			   struct igb_ring *ring)
4382 {
4383 	struct e1000_hw *hw = &adapter->hw;
4384 	u32 txdctl = 0;
4385 	u64 tdba = ring->dma;
4386 	int reg_idx = ring->reg_idx;
4387 
4388 	wr32(E1000_TDLEN(reg_idx),
4389 	     ring->count * sizeof(union e1000_adv_tx_desc));
4390 	wr32(E1000_TDBAL(reg_idx),
4391 	     tdba & 0x00000000ffffffffULL);
4392 	wr32(E1000_TDBAH(reg_idx), tdba >> 32);
4393 
4394 	ring->tail = adapter->io_addr + E1000_TDT(reg_idx);
4395 	wr32(E1000_TDH(reg_idx), 0);
4396 	writel(0, ring->tail);
4397 
4398 	txdctl |= IGB_TX_PTHRESH;
4399 	txdctl |= IGB_TX_HTHRESH << 8;
4400 	txdctl |= IGB_TX_WTHRESH << 16;
4401 
4402 	/* reinitialize tx_buffer_info */
4403 	memset(ring->tx_buffer_info, 0,
4404 	       sizeof(struct igb_tx_buffer) * ring->count);
4405 
4406 	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
4407 	wr32(E1000_TXDCTL(reg_idx), txdctl);
4408 }
4409 
4410 /**
4411  *  igb_configure_tx - Configure transmit Unit after Reset
4412  *  @adapter: board private structure
4413  *
4414  *  Configure the Tx unit of the MAC after a reset.
4415  **/
igb_configure_tx(struct igb_adapter * adapter)4416 static void igb_configure_tx(struct igb_adapter *adapter)
4417 {
4418 	struct e1000_hw *hw = &adapter->hw;
4419 	int i;
4420 
4421 	/* disable the queues */
4422 	for (i = 0; i < adapter->num_tx_queues; i++)
4423 		wr32(E1000_TXDCTL(adapter->tx_ring[i]->reg_idx), 0);
4424 
4425 	wrfl();
4426 	usleep_range(10000, 20000);
4427 
4428 	for (i = 0; i < adapter->num_tx_queues; i++)
4429 		igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
4430 }
4431 
4432 /**
4433  *  igb_setup_rx_resources - allocate Rx resources (Descriptors)
4434  *  @rx_ring: Rx descriptor ring (for a specific queue) to setup
4435  *
4436  *  Returns 0 on success, negative on failure
4437  **/
igb_setup_rx_resources(struct igb_ring * rx_ring)4438 int igb_setup_rx_resources(struct igb_ring *rx_ring)
4439 {
4440 	struct igb_adapter *adapter = netdev_priv(rx_ring->netdev);
4441 	struct device *dev = rx_ring->dev;
4442 	int size, res;
4443 
4444 	/* XDP RX-queue info */
4445 	if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
4446 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4447 	res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
4448 			       rx_ring->queue_index, 0);
4449 	if (res < 0) {
4450 		dev_err(dev, "Failed to register xdp_rxq index %u\n",
4451 			rx_ring->queue_index);
4452 		return res;
4453 	}
4454 
4455 	size = sizeof(struct igb_rx_buffer) * rx_ring->count;
4456 
4457 	rx_ring->rx_buffer_info = vmalloc(size);
4458 	if (!rx_ring->rx_buffer_info)
4459 		goto err;
4460 
4461 	/* Round up to nearest 4K */
4462 	rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
4463 	rx_ring->size = ALIGN(rx_ring->size, 4096);
4464 
4465 	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
4466 					   &rx_ring->dma, GFP_KERNEL);
4467 	if (!rx_ring->desc)
4468 		goto err;
4469 
4470 	rx_ring->next_to_alloc = 0;
4471 	rx_ring->next_to_clean = 0;
4472 	rx_ring->next_to_use = 0;
4473 
4474 	rx_ring->xdp_prog = adapter->xdp_prog;
4475 
4476 	return 0;
4477 
4478 err:
4479 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4480 	vfree(rx_ring->rx_buffer_info);
4481 	rx_ring->rx_buffer_info = NULL;
4482 	dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
4483 	return -ENOMEM;
4484 }
4485 
4486 /**
4487  *  igb_setup_all_rx_resources - wrapper to allocate Rx resources
4488  *				 (Descriptors) for all queues
4489  *  @adapter: board private structure
4490  *
4491  *  Return 0 on success, negative on failure
4492  **/
igb_setup_all_rx_resources(struct igb_adapter * adapter)4493 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
4494 {
4495 	struct pci_dev *pdev = adapter->pdev;
4496 	int i, err = 0;
4497 
4498 	for (i = 0; i < adapter->num_rx_queues; i++) {
4499 		err = igb_setup_rx_resources(adapter->rx_ring[i]);
4500 		if (err) {
4501 			dev_err(&pdev->dev,
4502 				"Allocation for Rx Queue %u failed\n", i);
4503 			for (i--; i >= 0; i--)
4504 				igb_free_rx_resources(adapter->rx_ring[i]);
4505 			break;
4506 		}
4507 	}
4508 
4509 	return err;
4510 }
4511 
4512 /**
4513  *  igb_setup_mrqc - configure the multiple receive queue control registers
4514  *  @adapter: Board private structure
4515  **/
igb_setup_mrqc(struct igb_adapter * adapter)4516 static void igb_setup_mrqc(struct igb_adapter *adapter)
4517 {
4518 	struct e1000_hw *hw = &adapter->hw;
4519 	u32 mrqc, rxcsum;
4520 	u32 j, num_rx_queues;
4521 	u32 rss_key[10];
4522 
4523 	netdev_rss_key_fill(rss_key, sizeof(rss_key));
4524 	for (j = 0; j < 10; j++)
4525 		wr32(E1000_RSSRK(j), rss_key[j]);
4526 
4527 	num_rx_queues = adapter->rss_queues;
4528 
4529 	switch (hw->mac.type) {
4530 	case e1000_82576:
4531 		/* 82576 supports 2 RSS queues for SR-IOV */
4532 		if (adapter->vfs_allocated_count)
4533 			num_rx_queues = 2;
4534 		break;
4535 	default:
4536 		break;
4537 	}
4538 
4539 	if (adapter->rss_indir_tbl_init != num_rx_queues) {
4540 		for (j = 0; j < IGB_RETA_SIZE; j++)
4541 			adapter->rss_indir_tbl[j] =
4542 			(j * num_rx_queues) / IGB_RETA_SIZE;
4543 		adapter->rss_indir_tbl_init = num_rx_queues;
4544 	}
4545 	igb_write_rss_indir_tbl(adapter);
4546 
4547 	/* Disable raw packet checksumming so that RSS hash is placed in
4548 	 * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
4549 	 * offloads as they are enabled by default
4550 	 */
4551 	rxcsum = rd32(E1000_RXCSUM);
4552 	rxcsum |= E1000_RXCSUM_PCSD;
4553 
4554 	if (adapter->hw.mac.type >= e1000_82576)
4555 		/* Enable Receive Checksum Offload for SCTP */
4556 		rxcsum |= E1000_RXCSUM_CRCOFL;
4557 
4558 	/* Don't need to set TUOFL or IPOFL, they default to 1 */
4559 	wr32(E1000_RXCSUM, rxcsum);
4560 
4561 	/* Generate RSS hash based on packet types, TCP/UDP
4562 	 * port numbers and/or IPv4/v6 src and dst addresses
4563 	 */
4564 	mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
4565 	       E1000_MRQC_RSS_FIELD_IPV4_TCP |
4566 	       E1000_MRQC_RSS_FIELD_IPV6 |
4567 	       E1000_MRQC_RSS_FIELD_IPV6_TCP |
4568 	       E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
4569 
4570 	if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
4571 		mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
4572 	if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
4573 		mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;
4574 
4575 	/* If VMDq is enabled then we set the appropriate mode for that, else
4576 	 * we default to RSS so that an RSS hash is calculated per packet even
4577 	 * if we are only using one queue
4578 	 */
4579 	if (adapter->vfs_allocated_count) {
4580 		if (hw->mac.type > e1000_82575) {
4581 			/* Set the default pool for the PF's first queue */
4582 			u32 vtctl = rd32(E1000_VT_CTL);
4583 
4584 			vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
4585 				   E1000_VT_CTL_DISABLE_DEF_POOL);
4586 			vtctl |= adapter->vfs_allocated_count <<
4587 				E1000_VT_CTL_DEFAULT_POOL_SHIFT;
4588 			wr32(E1000_VT_CTL, vtctl);
4589 		}
4590 		if (adapter->rss_queues > 1)
4591 			mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ;
4592 		else
4593 			mrqc |= E1000_MRQC_ENABLE_VMDQ;
4594 	} else {
4595 		mrqc |= E1000_MRQC_ENABLE_RSS_MQ;
4596 	}
4597 	igb_vmm_control(adapter);
4598 
4599 	wr32(E1000_MRQC, mrqc);
4600 }
4601 
4602 /**
4603  *  igb_setup_rctl - configure the receive control registers
4604  *  @adapter: Board private structure
4605  **/
igb_setup_rctl(struct igb_adapter * adapter)4606 void igb_setup_rctl(struct igb_adapter *adapter)
4607 {
4608 	struct e1000_hw *hw = &adapter->hw;
4609 	u32 rctl;
4610 
4611 	rctl = rd32(E1000_RCTL);
4612 
4613 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4614 	rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
4615 
4616 	rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
4617 		(hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4618 
4619 	/* enable stripping of CRC. It's unlikely this will break BMC
4620 	 * redirection as it did with e1000. Newer features require
4621 	 * that the HW strips the CRC.
4622 	 */
4623 	rctl |= E1000_RCTL_SECRC;
4624 
4625 	/* disable store bad packets and clear size bits. */
4626 	rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
4627 
4628 	/* enable LPE to allow for reception of jumbo frames */
4629 	rctl |= E1000_RCTL_LPE;
4630 
4631 	/* disable queue 0 to prevent tail write w/o re-config */
4632 	wr32(E1000_RXDCTL(0), 0);
4633 
4634 	/* Attention!!!  For SR-IOV PF driver operations you must enable
4635 	 * queue drop for all VF and PF queues to prevent head of line blocking
4636 	 * if an un-trusted VF does not provide descriptors to hardware.
4637 	 */
4638 	if (adapter->vfs_allocated_count) {
4639 		/* set all queue drop enable bits */
4640 		wr32(E1000_QDE, ALL_QUEUES);
4641 	}
4642 
4643 	/* This is useful for sniffing bad packets. */
4644 	if (adapter->netdev->features & NETIF_F_RXALL) {
4645 		/* UPE and MPE will be handled by normal PROMISC logic
4646 		 * in e1000e_set_rx_mode
4647 		 */
4648 		rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
4649 			 E1000_RCTL_BAM | /* RX All Bcast Pkts */
4650 			 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
4651 
4652 		rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */
4653 			  E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
4654 		/* Do not mess with E1000_CTRL_VME, it affects transmit as well,
4655 		 * and that breaks VLANs.
4656 		 */
4657 	}
4658 
4659 	wr32(E1000_RCTL, rctl);
4660 }
4661 
igb_set_vf_rlpml(struct igb_adapter * adapter,int size,int vfn)4662 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
4663 				   int vfn)
4664 {
4665 	struct e1000_hw *hw = &adapter->hw;
4666 	u32 vmolr;
4667 
4668 	if (size > MAX_JUMBO_FRAME_SIZE)
4669 		size = MAX_JUMBO_FRAME_SIZE;
4670 
4671 	vmolr = rd32(E1000_VMOLR(vfn));
4672 	vmolr &= ~E1000_VMOLR_RLPML_MASK;
4673 	vmolr |= size | E1000_VMOLR_LPE;
4674 	wr32(E1000_VMOLR(vfn), vmolr);
4675 
4676 	return 0;
4677 }
4678 
igb_set_vf_vlan_strip(struct igb_adapter * adapter,int vfn,bool enable)4679 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
4680 					 int vfn, bool enable)
4681 {
4682 	struct e1000_hw *hw = &adapter->hw;
4683 	u32 val, reg;
4684 
4685 	if (hw->mac.type < e1000_82576)
4686 		return;
4687 
4688 	if (hw->mac.type == e1000_i350)
4689 		reg = E1000_DVMOLR(vfn);
4690 	else
4691 		reg = E1000_VMOLR(vfn);
4692 
4693 	val = rd32(reg);
4694 	if (enable)
4695 		val |= E1000_VMOLR_STRVLAN;
4696 	else
4697 		val &= ~(E1000_VMOLR_STRVLAN);
4698 	wr32(reg, val);
4699 }
4700 
igb_set_vmolr(struct igb_adapter * adapter,int vfn,bool aupe)4701 static inline void igb_set_vmolr(struct igb_adapter *adapter,
4702 				 int vfn, bool aupe)
4703 {
4704 	struct e1000_hw *hw = &adapter->hw;
4705 	u32 vmolr;
4706 
4707 	/* This register exists only on 82576 and newer so if we are older then
4708 	 * we should exit and do nothing
4709 	 */
4710 	if (hw->mac.type < e1000_82576)
4711 		return;
4712 
4713 	vmolr = rd32(E1000_VMOLR(vfn));
4714 	if (aupe)
4715 		vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
4716 	else
4717 		vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
4718 
4719 	/* clear all bits that might not be set */
4720 	vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
4721 
4722 	if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
4723 		vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
4724 	/* for VMDq only allow the VFs and pool 0 to accept broadcast and
4725 	 * multicast packets
4726 	 */
4727 	if (vfn <= adapter->vfs_allocated_count)
4728 		vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
4729 
4730 	wr32(E1000_VMOLR(vfn), vmolr);
4731 }
4732 
4733 /**
4734  *  igb_setup_srrctl - configure the split and replication receive control
4735  *                     registers
4736  *  @adapter: Board private structure
4737  *  @ring: receive ring to be configured
4738  **/
igb_setup_srrctl(struct igb_adapter * adapter,struct igb_ring * ring)4739 void igb_setup_srrctl(struct igb_adapter *adapter, struct igb_ring *ring)
4740 {
4741 	struct e1000_hw *hw = &adapter->hw;
4742 	int reg_idx = ring->reg_idx;
4743 	u32 srrctl = 0;
4744 
4745 	srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4746 	if (ring_uses_large_buffer(ring))
4747 		srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4748 	else
4749 		srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4750 	srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4751 	if (hw->mac.type >= e1000_82580)
4752 		srrctl |= E1000_SRRCTL_TIMESTAMP;
4753 	/* Only set Drop Enable if VFs allocated, or we are supporting multiple
4754 	 * queues and rx flow control is disabled
4755 	 */
4756 	if (adapter->vfs_allocated_count ||
4757 	    (!(hw->fc.current_mode & e1000_fc_rx_pause) &&
4758 	     adapter->num_rx_queues > 1))
4759 		srrctl |= E1000_SRRCTL_DROP_EN;
4760 
4761 	wr32(E1000_SRRCTL(reg_idx), srrctl);
4762 }
4763 
4764 /**
4765  *  igb_configure_rx_ring - Configure a receive ring after Reset
4766  *  @adapter: board private structure
4767  *  @ring: receive ring to be configured
4768  *
4769  *  Configure the Rx unit of the MAC after a reset.
4770  **/
igb_configure_rx_ring(struct igb_adapter * adapter,struct igb_ring * ring)4771 void igb_configure_rx_ring(struct igb_adapter *adapter,
4772 			   struct igb_ring *ring)
4773 {
4774 	struct e1000_hw *hw = &adapter->hw;
4775 	union e1000_adv_rx_desc *rx_desc;
4776 	u64 rdba = ring->dma;
4777 	int reg_idx = ring->reg_idx;
4778 	u32 rxdctl = 0;
4779 
4780 	xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
4781 	WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
4782 					   MEM_TYPE_PAGE_SHARED, NULL));
4783 
4784 	/* disable the queue */
4785 	wr32(E1000_RXDCTL(reg_idx), 0);
4786 
4787 	/* Set DMA base address registers */
4788 	wr32(E1000_RDBAL(reg_idx),
4789 	     rdba & 0x00000000ffffffffULL);
4790 	wr32(E1000_RDBAH(reg_idx), rdba >> 32);
4791 	wr32(E1000_RDLEN(reg_idx),
4792 	     ring->count * sizeof(union e1000_adv_rx_desc));
4793 
4794 	/* initialize head and tail */
4795 	ring->tail = adapter->io_addr + E1000_RDT(reg_idx);
4796 	wr32(E1000_RDH(reg_idx), 0);
4797 	writel(0, ring->tail);
4798 
4799 	/* set descriptor configuration */
4800 	igb_setup_srrctl(adapter, ring);
4801 
4802 	/* set filtering for VMDQ pools */
4803 	igb_set_vmolr(adapter, reg_idx & 0x7, true);
4804 
4805 	rxdctl |= IGB_RX_PTHRESH;
4806 	rxdctl |= IGB_RX_HTHRESH << 8;
4807 	rxdctl |= IGB_RX_WTHRESH << 16;
4808 
4809 	/* initialize rx_buffer_info */
4810 	memset(ring->rx_buffer_info, 0,
4811 	       sizeof(struct igb_rx_buffer) * ring->count);
4812 
4813 	/* initialize Rx descriptor 0 */
4814 	rx_desc = IGB_RX_DESC(ring, 0);
4815 	rx_desc->wb.upper.length = 0;
4816 
4817 	/* enable receive descriptor fetching */
4818 	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4819 	wr32(E1000_RXDCTL(reg_idx), rxdctl);
4820 }
4821 
igb_set_rx_buffer_len(struct igb_adapter * adapter,struct igb_ring * rx_ring)4822 static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
4823 				  struct igb_ring *rx_ring)
4824 {
4825 #if (PAGE_SIZE < 8192)
4826 	struct e1000_hw *hw = &adapter->hw;
4827 #endif
4828 
4829 	/* set build_skb and buffer size flags */
4830 	clear_ring_build_skb_enabled(rx_ring);
4831 	clear_ring_uses_large_buffer(rx_ring);
4832 
4833 	if (adapter->flags & IGB_FLAG_RX_LEGACY)
4834 		return;
4835 
4836 	set_ring_build_skb_enabled(rx_ring);
4837 
4838 #if (PAGE_SIZE < 8192)
4839 	if (adapter->max_frame_size > IGB_MAX_FRAME_BUILD_SKB ||
4840 	    IGB_2K_TOO_SMALL_WITH_PADDING ||
4841 	    rd32(E1000_RCTL) & E1000_RCTL_SBP)
4842 		set_ring_uses_large_buffer(rx_ring);
4843 #endif
4844 }
4845 
4846 /**
4847  *  igb_configure_rx - Configure receive Unit after Reset
4848  *  @adapter: board private structure
4849  *
4850  *  Configure the Rx unit of the MAC after a reset.
4851  **/
igb_configure_rx(struct igb_adapter * adapter)4852 static void igb_configure_rx(struct igb_adapter *adapter)
4853 {
4854 	int i;
4855 
4856 	/* set the correct pool for the PF default MAC address in entry 0 */
4857 	igb_set_default_mac_filter(adapter);
4858 
4859 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
4860 	 * the Base and Length of the Rx Descriptor Ring
4861 	 */
4862 	for (i = 0; i < adapter->num_rx_queues; i++) {
4863 		struct igb_ring *rx_ring = adapter->rx_ring[i];
4864 
4865 		igb_set_rx_buffer_len(adapter, rx_ring);
4866 		igb_configure_rx_ring(adapter, rx_ring);
4867 	}
4868 }
4869 
4870 /**
4871  *  igb_free_tx_resources - Free Tx Resources per Queue
4872  *  @tx_ring: Tx descriptor ring for a specific queue
4873  *
4874  *  Free all transmit software resources
4875  **/
igb_free_tx_resources(struct igb_ring * tx_ring)4876 void igb_free_tx_resources(struct igb_ring *tx_ring)
4877 {
4878 	igb_clean_tx_ring(tx_ring);
4879 
4880 	vfree(tx_ring->tx_buffer_info);
4881 	tx_ring->tx_buffer_info = NULL;
4882 
4883 	/* if not set, then don't free */
4884 	if (!tx_ring->desc)
4885 		return;
4886 
4887 	dma_free_coherent(tx_ring->dev, tx_ring->size,
4888 			  tx_ring->desc, tx_ring->dma);
4889 
4890 	tx_ring->desc = NULL;
4891 }
4892 
4893 /**
4894  *  igb_free_all_tx_resources - Free Tx Resources for All Queues
4895  *  @adapter: board private structure
4896  *
4897  *  Free all transmit software resources
4898  **/
igb_free_all_tx_resources(struct igb_adapter * adapter)4899 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
4900 {
4901 	int i;
4902 
4903 	for (i = 0; i < adapter->num_tx_queues; i++)
4904 		if (adapter->tx_ring[i])
4905 			igb_free_tx_resources(adapter->tx_ring[i]);
4906 }
4907 
4908 /**
4909  *  igb_clean_tx_ring - Free Tx Buffers
4910  *  @tx_ring: ring to be cleaned
4911  **/
igb_clean_tx_ring(struct igb_ring * tx_ring)4912 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
4913 {
4914 	u16 i = tx_ring->next_to_clean;
4915 	struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
4916 
4917 	while (i != tx_ring->next_to_use) {
4918 		union e1000_adv_tx_desc *eop_desc, *tx_desc;
4919 
4920 		/* Free all the Tx ring sk_buffs or xdp frames */
4921 		if (tx_buffer->type == IGB_TYPE_SKB)
4922 			dev_kfree_skb_any(tx_buffer->skb);
4923 		else
4924 			xdp_return_frame(tx_buffer->xdpf);
4925 
4926 		/* unmap skb header data */
4927 		dma_unmap_single(tx_ring->dev,
4928 				 dma_unmap_addr(tx_buffer, dma),
4929 				 dma_unmap_len(tx_buffer, len),
4930 				 DMA_TO_DEVICE);
4931 
4932 		/* check for eop_desc to determine the end of the packet */
4933 		eop_desc = tx_buffer->next_to_watch;
4934 		tx_desc = IGB_TX_DESC(tx_ring, i);
4935 
4936 		/* unmap remaining buffers */
4937 		while (tx_desc != eop_desc) {
4938 			tx_buffer++;
4939 			tx_desc++;
4940 			i++;
4941 			if (unlikely(i == tx_ring->count)) {
4942 				i = 0;
4943 				tx_buffer = tx_ring->tx_buffer_info;
4944 				tx_desc = IGB_TX_DESC(tx_ring, 0);
4945 			}
4946 
4947 			/* unmap any remaining paged data */
4948 			if (dma_unmap_len(tx_buffer, len))
4949 				dma_unmap_page(tx_ring->dev,
4950 					       dma_unmap_addr(tx_buffer, dma),
4951 					       dma_unmap_len(tx_buffer, len),
4952 					       DMA_TO_DEVICE);
4953 		}
4954 
4955 		tx_buffer->next_to_watch = NULL;
4956 
4957 		/* move us one more past the eop_desc for start of next pkt */
4958 		tx_buffer++;
4959 		i++;
4960 		if (unlikely(i == tx_ring->count)) {
4961 			i = 0;
4962 			tx_buffer = tx_ring->tx_buffer_info;
4963 		}
4964 	}
4965 
4966 	/* reset BQL for queue */
4967 	netdev_tx_reset_queue(txring_txq(tx_ring));
4968 
4969 	/* reset next_to_use and next_to_clean */
4970 	tx_ring->next_to_use = 0;
4971 	tx_ring->next_to_clean = 0;
4972 }
4973 
4974 /**
4975  *  igb_clean_all_tx_rings - Free Tx Buffers for all queues
4976  *  @adapter: board private structure
4977  **/
igb_clean_all_tx_rings(struct igb_adapter * adapter)4978 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
4979 {
4980 	int i;
4981 
4982 	for (i = 0; i < adapter->num_tx_queues; i++)
4983 		if (adapter->tx_ring[i])
4984 			igb_clean_tx_ring(adapter->tx_ring[i]);
4985 }
4986 
4987 /**
4988  *  igb_free_rx_resources - Free Rx Resources
4989  *  @rx_ring: ring to clean the resources from
4990  *
4991  *  Free all receive software resources
4992  **/
igb_free_rx_resources(struct igb_ring * rx_ring)4993 void igb_free_rx_resources(struct igb_ring *rx_ring)
4994 {
4995 	igb_clean_rx_ring(rx_ring);
4996 
4997 	rx_ring->xdp_prog = NULL;
4998 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4999 	vfree(rx_ring->rx_buffer_info);
5000 	rx_ring->rx_buffer_info = NULL;
5001 
5002 	/* if not set, then don't free */
5003 	if (!rx_ring->desc)
5004 		return;
5005 
5006 	dma_free_coherent(rx_ring->dev, rx_ring->size,
5007 			  rx_ring->desc, rx_ring->dma);
5008 
5009 	rx_ring->desc = NULL;
5010 }
5011 
5012 /**
5013  *  igb_free_all_rx_resources - Free Rx Resources for All Queues
5014  *  @adapter: board private structure
5015  *
5016  *  Free all receive software resources
5017  **/
igb_free_all_rx_resources(struct igb_adapter * adapter)5018 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
5019 {
5020 	int i;
5021 
5022 	for (i = 0; i < adapter->num_rx_queues; i++)
5023 		if (adapter->rx_ring[i])
5024 			igb_free_rx_resources(adapter->rx_ring[i]);
5025 }
5026 
5027 /**
5028  *  igb_clean_rx_ring - Free Rx Buffers per Queue
5029  *  @rx_ring: ring to free buffers from
5030  **/
igb_clean_rx_ring(struct igb_ring * rx_ring)5031 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
5032 {
5033 	u16 i = rx_ring->next_to_clean;
5034 
5035 	dev_kfree_skb(rx_ring->skb);
5036 	rx_ring->skb = NULL;
5037 
5038 	/* Free all the Rx ring sk_buffs */
5039 	while (i != rx_ring->next_to_alloc) {
5040 		struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
5041 
5042 		/* Invalidate cache lines that may have been written to by
5043 		 * device so that we avoid corrupting memory.
5044 		 */
5045 		dma_sync_single_range_for_cpu(rx_ring->dev,
5046 					      buffer_info->dma,
5047 					      buffer_info->page_offset,
5048 					      igb_rx_bufsz(rx_ring),
5049 					      DMA_FROM_DEVICE);
5050 
5051 		/* free resources associated with mapping */
5052 		dma_unmap_page_attrs(rx_ring->dev,
5053 				     buffer_info->dma,
5054 				     igb_rx_pg_size(rx_ring),
5055 				     DMA_FROM_DEVICE,
5056 				     IGB_RX_DMA_ATTR);
5057 		__page_frag_cache_drain(buffer_info->page,
5058 					buffer_info->pagecnt_bias);
5059 
5060 		i++;
5061 		if (i == rx_ring->count)
5062 			i = 0;
5063 	}
5064 
5065 	rx_ring->next_to_alloc = 0;
5066 	rx_ring->next_to_clean = 0;
5067 	rx_ring->next_to_use = 0;
5068 }
5069 
5070 /**
5071  *  igb_clean_all_rx_rings - Free Rx Buffers for all queues
5072  *  @adapter: board private structure
5073  **/
igb_clean_all_rx_rings(struct igb_adapter * adapter)5074 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
5075 {
5076 	int i;
5077 
5078 	for (i = 0; i < adapter->num_rx_queues; i++)
5079 		if (adapter->rx_ring[i])
5080 			igb_clean_rx_ring(adapter->rx_ring[i]);
5081 }
5082 
5083 /**
5084  *  igb_set_mac - Change the Ethernet Address of the NIC
5085  *  @netdev: network interface device structure
5086  *  @p: pointer to an address structure
5087  *
5088  *  Returns 0 on success, negative on failure
5089  **/
igb_set_mac(struct net_device * netdev,void * p)5090 static int igb_set_mac(struct net_device *netdev, void *p)
5091 {
5092 	struct igb_adapter *adapter = netdev_priv(netdev);
5093 	struct e1000_hw *hw = &adapter->hw;
5094 	struct sockaddr *addr = p;
5095 
5096 	if (!is_valid_ether_addr(addr->sa_data))
5097 		return -EADDRNOTAVAIL;
5098 
5099 	eth_hw_addr_set(netdev, addr->sa_data);
5100 	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
5101 
5102 	/* set the correct pool for the new PF MAC address in entry 0 */
5103 	igb_set_default_mac_filter(adapter);
5104 
5105 	return 0;
5106 }
5107 
5108 /**
5109  *  igb_write_mc_addr_list - write multicast addresses to MTA
5110  *  @netdev: network interface device structure
5111  *
5112  *  Writes multicast address list to the MTA hash table.
5113  *  Returns: -ENOMEM on failure
5114  *           0 on no addresses written
5115  *           X on writing X addresses to MTA
5116  **/
igb_write_mc_addr_list(struct net_device * netdev)5117 static int igb_write_mc_addr_list(struct net_device *netdev)
5118 {
5119 	struct igb_adapter *adapter = netdev_priv(netdev);
5120 	struct e1000_hw *hw = &adapter->hw;
5121 	struct netdev_hw_addr *ha;
5122 	u8  *mta_list;
5123 	int i;
5124 
5125 	if (netdev_mc_empty(netdev)) {
5126 		/* nothing to program, so clear mc list */
5127 		igb_update_mc_addr_list(hw, NULL, 0);
5128 		igb_restore_vf_multicasts(adapter);
5129 		return 0;
5130 	}
5131 
5132 	mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
5133 	if (!mta_list)
5134 		return -ENOMEM;
5135 
5136 	/* The shared function expects a packed array of only addresses. */
5137 	i = 0;
5138 	netdev_for_each_mc_addr(ha, netdev)
5139 		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
5140 
5141 	igb_update_mc_addr_list(hw, mta_list, i);
5142 	kfree(mta_list);
5143 
5144 	return netdev_mc_count(netdev);
5145 }
5146 
igb_vlan_promisc_enable(struct igb_adapter * adapter)5147 static int igb_vlan_promisc_enable(struct igb_adapter *adapter)
5148 {
5149 	struct e1000_hw *hw = &adapter->hw;
5150 	u32 i, pf_id;
5151 
5152 	switch (hw->mac.type) {
5153 	case e1000_i210:
5154 	case e1000_i211:
5155 	case e1000_i350:
5156 		/* VLAN filtering needed for VLAN prio filter */
5157 		if (adapter->netdev->features & NETIF_F_NTUPLE)
5158 			break;
5159 		fallthrough;
5160 	case e1000_82576:
5161 	case e1000_82580:
5162 	case e1000_i354:
5163 		/* VLAN filtering needed for pool filtering */
5164 		if (adapter->vfs_allocated_count)
5165 			break;
5166 		fallthrough;
5167 	default:
5168 		return 1;
5169 	}
5170 
5171 	/* We are already in VLAN promisc, nothing to do */
5172 	if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
5173 		return 0;
5174 
5175 	if (!adapter->vfs_allocated_count)
5176 		goto set_vfta;
5177 
5178 	/* Add PF to all active pools */
5179 	pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
5180 
5181 	for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
5182 		u32 vlvf = rd32(E1000_VLVF(i));
5183 
5184 		vlvf |= BIT(pf_id);
5185 		wr32(E1000_VLVF(i), vlvf);
5186 	}
5187 
5188 set_vfta:
5189 	/* Set all bits in the VLAN filter table array */
5190 	for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;)
5191 		hw->mac.ops.write_vfta(hw, i, ~0U);
5192 
5193 	/* Set flag so we don't redo unnecessary work */
5194 	adapter->flags |= IGB_FLAG_VLAN_PROMISC;
5195 
5196 	return 0;
5197 }
5198 
5199 #define VFTA_BLOCK_SIZE 8
igb_scrub_vfta(struct igb_adapter * adapter,u32 vfta_offset)5200 static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset)
5201 {
5202 	struct e1000_hw *hw = &adapter->hw;
5203 	u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
5204 	u32 vid_start = vfta_offset * 32;
5205 	u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
5206 	u32 i, vid, word, bits, pf_id;
5207 
5208 	/* guarantee that we don't scrub out management VLAN */
5209 	vid = adapter->mng_vlan_id;
5210 	if (vid >= vid_start && vid < vid_end)
5211 		vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5212 
5213 	if (!adapter->vfs_allocated_count)
5214 		goto set_vfta;
5215 
5216 	pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
5217 
5218 	for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
5219 		u32 vlvf = rd32(E1000_VLVF(i));
5220 
5221 		/* pull VLAN ID from VLVF */
5222 		vid = vlvf & VLAN_VID_MASK;
5223 
5224 		/* only concern ourselves with a certain range */
5225 		if (vid < vid_start || vid >= vid_end)
5226 			continue;
5227 
5228 		if (vlvf & E1000_VLVF_VLANID_ENABLE) {
5229 			/* record VLAN ID in VFTA */
5230 			vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5231 
5232 			/* if PF is part of this then continue */
5233 			if (test_bit(vid, adapter->active_vlans))
5234 				continue;
5235 		}
5236 
5237 		/* remove PF from the pool */
5238 		bits = ~BIT(pf_id);
5239 		bits &= rd32(E1000_VLVF(i));
5240 		wr32(E1000_VLVF(i), bits);
5241 	}
5242 
5243 set_vfta:
5244 	/* extract values from active_vlans and write back to VFTA */
5245 	for (i = VFTA_BLOCK_SIZE; i--;) {
5246 		vid = (vfta_offset + i) * 32;
5247 		word = vid / BITS_PER_LONG;
5248 		bits = vid % BITS_PER_LONG;
5249 
5250 		vfta[i] |= adapter->active_vlans[word] >> bits;
5251 
5252 		hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]);
5253 	}
5254 }
5255 
igb_vlan_promisc_disable(struct igb_adapter * adapter)5256 static void igb_vlan_promisc_disable(struct igb_adapter *adapter)
5257 {
5258 	u32 i;
5259 
5260 	/* We are not in VLAN promisc, nothing to do */
5261 	if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC))
5262 		return;
5263 
5264 	/* Set flag so we don't redo unnecessary work */
5265 	adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
5266 
5267 	for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE)
5268 		igb_scrub_vfta(adapter, i);
5269 }
5270 
5271 /**
5272  *  igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
5273  *  @netdev: network interface device structure
5274  *
5275  *  The set_rx_mode entry point is called whenever the unicast or multicast
5276  *  address lists or the network interface flags are updated.  This routine is
5277  *  responsible for configuring the hardware for proper unicast, multicast,
5278  *  promiscuous mode, and all-multi behavior.
5279  **/
igb_set_rx_mode(struct net_device * netdev)5280 static void igb_set_rx_mode(struct net_device *netdev)
5281 {
5282 	struct igb_adapter *adapter = netdev_priv(netdev);
5283 	struct e1000_hw *hw = &adapter->hw;
5284 	unsigned int vfn = adapter->vfs_allocated_count;
5285 	u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
5286 	int count;
5287 
5288 	/* Check for Promiscuous and All Multicast modes */
5289 	if (netdev->flags & IFF_PROMISC) {
5290 		rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE;
5291 		vmolr |= E1000_VMOLR_MPME;
5292 
5293 		/* enable use of UTA filter to force packets to default pool */
5294 		if (hw->mac.type == e1000_82576)
5295 			vmolr |= E1000_VMOLR_ROPE;
5296 	} else {
5297 		if (netdev->flags & IFF_ALLMULTI) {
5298 			rctl |= E1000_RCTL_MPE;
5299 			vmolr |= E1000_VMOLR_MPME;
5300 		} else {
5301 			/* Write addresses to the MTA, if the attempt fails
5302 			 * then we should just turn on promiscuous mode so
5303 			 * that we can at least receive multicast traffic
5304 			 */
5305 			count = igb_write_mc_addr_list(netdev);
5306 			if (count < 0) {
5307 				rctl |= E1000_RCTL_MPE;
5308 				vmolr |= E1000_VMOLR_MPME;
5309 			} else if (count) {
5310 				vmolr |= E1000_VMOLR_ROMPE;
5311 			}
5312 		}
5313 	}
5314 
5315 	/* Write addresses to available RAR registers, if there is not
5316 	 * sufficient space to store all the addresses then enable
5317 	 * unicast promiscuous mode
5318 	 */
5319 	if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) {
5320 		rctl |= E1000_RCTL_UPE;
5321 		vmolr |= E1000_VMOLR_ROPE;
5322 	}
5323 
5324 	/* enable VLAN filtering by default */
5325 	rctl |= E1000_RCTL_VFE;
5326 
5327 	/* disable VLAN filtering for modes that require it */
5328 	if ((netdev->flags & IFF_PROMISC) ||
5329 	    (netdev->features & NETIF_F_RXALL)) {
5330 		/* if we fail to set all rules then just clear VFE */
5331 		if (igb_vlan_promisc_enable(adapter))
5332 			rctl &= ~E1000_RCTL_VFE;
5333 	} else {
5334 		igb_vlan_promisc_disable(adapter);
5335 	}
5336 
5337 	/* update state of unicast, multicast, and VLAN filtering modes */
5338 	rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE |
5339 				     E1000_RCTL_VFE);
5340 	wr32(E1000_RCTL, rctl);
5341 
5342 #if (PAGE_SIZE < 8192)
5343 	if (!adapter->vfs_allocated_count) {
5344 		if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5345 			rlpml = IGB_MAX_FRAME_BUILD_SKB;
5346 	}
5347 #endif
5348 	wr32(E1000_RLPML, rlpml);
5349 
5350 	/* In order to support SR-IOV and eventually VMDq it is necessary to set
5351 	 * the VMOLR to enable the appropriate modes.  Without this workaround
5352 	 * we will have issues with VLAN tag stripping not being done for frames
5353 	 * that are only arriving because we are the default pool
5354 	 */
5355 	if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
5356 		return;
5357 
5358 	/* set UTA to appropriate mode */
5359 	igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE));
5360 
5361 	vmolr |= rd32(E1000_VMOLR(vfn)) &
5362 		 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
5363 
5364 	/* enable Rx jumbo frames, restrict as needed to support build_skb */
5365 	vmolr &= ~E1000_VMOLR_RLPML_MASK;
5366 #if (PAGE_SIZE < 8192)
5367 	if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5368 		vmolr |= IGB_MAX_FRAME_BUILD_SKB;
5369 	else
5370 #endif
5371 		vmolr |= MAX_JUMBO_FRAME_SIZE;
5372 	vmolr |= E1000_VMOLR_LPE;
5373 
5374 	wr32(E1000_VMOLR(vfn), vmolr);
5375 
5376 	igb_restore_vf_multicasts(adapter);
5377 }
5378 
igb_check_wvbr(struct igb_adapter * adapter)5379 static void igb_check_wvbr(struct igb_adapter *adapter)
5380 {
5381 	struct e1000_hw *hw = &adapter->hw;
5382 	u32 wvbr = 0;
5383 
5384 	switch (hw->mac.type) {
5385 	case e1000_82576:
5386 	case e1000_i350:
5387 		wvbr = rd32(E1000_WVBR);
5388 		if (!wvbr)
5389 			return;
5390 		break;
5391 	default:
5392 		break;
5393 	}
5394 
5395 	adapter->wvbr |= wvbr;
5396 }
5397 
5398 #define IGB_STAGGERED_QUEUE_OFFSET 8
5399 
igb_spoof_check(struct igb_adapter * adapter)5400 static void igb_spoof_check(struct igb_adapter *adapter)
5401 {
5402 	int j;
5403 
5404 	if (!adapter->wvbr)
5405 		return;
5406 
5407 	for (j = 0; j < adapter->vfs_allocated_count; j++) {
5408 		if (adapter->wvbr & BIT(j) ||
5409 		    adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) {
5410 			dev_warn(&adapter->pdev->dev,
5411 				"Spoof event(s) detected on VF %d\n", j);
5412 			adapter->wvbr &=
5413 				~(BIT(j) |
5414 				  BIT(j + IGB_STAGGERED_QUEUE_OFFSET));
5415 		}
5416 	}
5417 }
5418 
5419 /* Need to wait a few seconds after link up to get diagnostic information from
5420  * the phy
5421  */
igb_update_phy_info(struct timer_list * t)5422 static void igb_update_phy_info(struct timer_list *t)
5423 {
5424 	struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5425 	igb_get_phy_info(&adapter->hw);
5426 }
5427 
5428 /**
5429  *  igb_has_link - check shared code for link and determine up/down
5430  *  @adapter: pointer to driver private info
5431  **/
igb_has_link(struct igb_adapter * adapter)5432 bool igb_has_link(struct igb_adapter *adapter)
5433 {
5434 	struct e1000_hw *hw = &adapter->hw;
5435 	bool link_active = false;
5436 
5437 	/* get_link_status is set on LSC (link status) interrupt or
5438 	 * rx sequence error interrupt.  get_link_status will stay
5439 	 * false until the e1000_check_for_link establishes link
5440 	 * for copper adapters ONLY
5441 	 */
5442 	switch (hw->phy.media_type) {
5443 	case e1000_media_type_copper:
5444 		if (!hw->mac.get_link_status)
5445 			return true;
5446 		fallthrough;
5447 	case e1000_media_type_internal_serdes:
5448 		hw->mac.ops.check_for_link(hw);
5449 		link_active = !hw->mac.get_link_status;
5450 		break;
5451 	default:
5452 	case e1000_media_type_unknown:
5453 		break;
5454 	}
5455 
5456 	if (((hw->mac.type == e1000_i210) ||
5457 	     (hw->mac.type == e1000_i211)) &&
5458 	     (hw->phy.id == I210_I_PHY_ID)) {
5459 		if (!netif_carrier_ok(adapter->netdev)) {
5460 			adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5461 		} else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
5462 			adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
5463 			adapter->link_check_timeout = jiffies;
5464 		}
5465 	}
5466 
5467 	return link_active;
5468 }
5469 
igb_thermal_sensor_event(struct e1000_hw * hw,u32 event)5470 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
5471 {
5472 	bool ret = false;
5473 	u32 ctrl_ext, thstat;
5474 
5475 	/* check for thermal sensor event on i350 copper only */
5476 	if (hw->mac.type == e1000_i350) {
5477 		thstat = rd32(E1000_THSTAT);
5478 		ctrl_ext = rd32(E1000_CTRL_EXT);
5479 
5480 		if ((hw->phy.media_type == e1000_media_type_copper) &&
5481 		    !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
5482 			ret = !!(thstat & event);
5483 	}
5484 
5485 	return ret;
5486 }
5487 
5488 /**
5489  *  igb_check_lvmmc - check for malformed packets received
5490  *  and indicated in LVMMC register
5491  *  @adapter: pointer to adapter
5492  **/
igb_check_lvmmc(struct igb_adapter * adapter)5493 static void igb_check_lvmmc(struct igb_adapter *adapter)
5494 {
5495 	struct e1000_hw *hw = &adapter->hw;
5496 	u32 lvmmc;
5497 
5498 	lvmmc = rd32(E1000_LVMMC);
5499 	if (lvmmc) {
5500 		if (unlikely(net_ratelimit())) {
5501 			netdev_warn(adapter->netdev,
5502 				    "malformed Tx packet detected and dropped, LVMMC:0x%08x\n",
5503 				    lvmmc);
5504 		}
5505 	}
5506 }
5507 
5508 /**
5509  *  igb_watchdog - Timer Call-back
5510  *  @t: pointer to timer_list containing our private info pointer
5511  **/
igb_watchdog(struct timer_list * t)5512 static void igb_watchdog(struct timer_list *t)
5513 {
5514 	struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5515 	/* Do the rest outside of interrupt context */
5516 	schedule_work(&adapter->watchdog_task);
5517 }
5518 
igb_watchdog_task(struct work_struct * work)5519 static void igb_watchdog_task(struct work_struct *work)
5520 {
5521 	struct igb_adapter *adapter = container_of(work,
5522 						   struct igb_adapter,
5523 						   watchdog_task);
5524 	struct e1000_hw *hw = &adapter->hw;
5525 	struct e1000_phy_info *phy = &hw->phy;
5526 	struct net_device *netdev = adapter->netdev;
5527 	u32 link;
5528 	int i;
5529 	u32 connsw;
5530 	u16 phy_data, retry_count = 20;
5531 
5532 	link = igb_has_link(adapter);
5533 
5534 	if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
5535 		if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5536 			adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5537 		else
5538 			link = false;
5539 	}
5540 
5541 	/* Force link down if we have fiber to swap to */
5542 	if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5543 		if (hw->phy.media_type == e1000_media_type_copper) {
5544 			connsw = rd32(E1000_CONNSW);
5545 			if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
5546 				link = 0;
5547 		}
5548 	}
5549 	if (link) {
5550 		/* Perform a reset if the media type changed. */
5551 		if (hw->dev_spec._82575.media_changed) {
5552 			hw->dev_spec._82575.media_changed = false;
5553 			adapter->flags |= IGB_FLAG_MEDIA_RESET;
5554 			igb_reset(adapter);
5555 		}
5556 		/* Cancel scheduled suspend requests. */
5557 		pm_runtime_resume(netdev->dev.parent);
5558 
5559 		if (!netif_carrier_ok(netdev)) {
5560 			u32 ctrl;
5561 
5562 			hw->mac.ops.get_speed_and_duplex(hw,
5563 							 &adapter->link_speed,
5564 							 &adapter->link_duplex);
5565 
5566 			ctrl = rd32(E1000_CTRL);
5567 			/* Links status message must follow this format */
5568 			netdev_info(netdev,
5569 			       "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5570 			       netdev->name,
5571 			       adapter->link_speed,
5572 			       adapter->link_duplex == FULL_DUPLEX ?
5573 			       "Full" : "Half",
5574 			       (ctrl & E1000_CTRL_TFCE) &&
5575 			       (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
5576 			       (ctrl & E1000_CTRL_RFCE) ?  "RX" :
5577 			       (ctrl & E1000_CTRL_TFCE) ?  "TX" : "None");
5578 
5579 			/* disable EEE if enabled */
5580 			if ((adapter->flags & IGB_FLAG_EEE) &&
5581 				(adapter->link_duplex == HALF_DUPLEX)) {
5582 				dev_info(&adapter->pdev->dev,
5583 				"EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
5584 				adapter->hw.dev_spec._82575.eee_disable = true;
5585 				adapter->flags &= ~IGB_FLAG_EEE;
5586 			}
5587 
5588 			/* check if SmartSpeed worked */
5589 			igb_check_downshift(hw);
5590 			if (phy->speed_downgraded)
5591 				netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5592 
5593 			/* check for thermal sensor event */
5594 			if (igb_thermal_sensor_event(hw,
5595 			    E1000_THSTAT_LINK_THROTTLE))
5596 				netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n");
5597 
5598 			/* adjust timeout factor according to speed/duplex */
5599 			adapter->tx_timeout_factor = 1;
5600 			switch (adapter->link_speed) {
5601 			case SPEED_10:
5602 				adapter->tx_timeout_factor = 14;
5603 				break;
5604 			case SPEED_100:
5605 				/* maybe add some timeout factor ? */
5606 				break;
5607 			}
5608 
5609 			if (adapter->link_speed != SPEED_1000 ||
5610 			    !hw->phy.ops.read_reg)
5611 				goto no_wait;
5612 
5613 			/* wait for Remote receiver status OK */
5614 retry_read_status:
5615 			if (!igb_read_phy_reg(hw, PHY_1000T_STATUS,
5616 					      &phy_data)) {
5617 				if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5618 				    retry_count) {
5619 					msleep(100);
5620 					retry_count--;
5621 					goto retry_read_status;
5622 				} else if (!retry_count) {
5623 					dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
5624 				}
5625 			} else {
5626 				dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
5627 			}
5628 no_wait:
5629 			netif_carrier_on(netdev);
5630 
5631 			igb_ping_all_vfs(adapter);
5632 			igb_check_vf_rate_limit(adapter);
5633 
5634 			/* link state has changed, schedule phy info update */
5635 			if (!test_bit(__IGB_DOWN, &adapter->state))
5636 				mod_timer(&adapter->phy_info_timer,
5637 					  round_jiffies(jiffies + 2 * HZ));
5638 		}
5639 	} else {
5640 		if (netif_carrier_ok(netdev)) {
5641 			adapter->link_speed = 0;
5642 			adapter->link_duplex = 0;
5643 
5644 			/* check for thermal sensor event */
5645 			if (igb_thermal_sensor_event(hw,
5646 			    E1000_THSTAT_PWR_DOWN)) {
5647 				netdev_err(netdev, "The network adapter was stopped because it overheated\n");
5648 			}
5649 
5650 			/* Links status message must follow this format */
5651 			netdev_info(netdev, "igb: %s NIC Link is Down\n",
5652 			       netdev->name);
5653 			netif_carrier_off(netdev);
5654 
5655 			igb_ping_all_vfs(adapter);
5656 
5657 			/* link state has changed, schedule phy info update */
5658 			if (!test_bit(__IGB_DOWN, &adapter->state))
5659 				mod_timer(&adapter->phy_info_timer,
5660 					  round_jiffies(jiffies + 2 * HZ));
5661 
5662 			/* link is down, time to check for alternate media */
5663 			if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5664 				igb_check_swap_media(adapter);
5665 				if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5666 					schedule_work(&adapter->reset_task);
5667 					/* return immediately */
5668 					return;
5669 				}
5670 			}
5671 			pm_schedule_suspend(netdev->dev.parent,
5672 					    MSEC_PER_SEC * 5);
5673 
5674 		/* also check for alternate media here */
5675 		} else if (!netif_carrier_ok(netdev) &&
5676 			   (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
5677 			igb_check_swap_media(adapter);
5678 			if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5679 				schedule_work(&adapter->reset_task);
5680 				/* return immediately */
5681 				return;
5682 			}
5683 		}
5684 	}
5685 
5686 	spin_lock(&adapter->stats64_lock);
5687 	igb_update_stats(adapter);
5688 	spin_unlock(&adapter->stats64_lock);
5689 
5690 	for (i = 0; i < adapter->num_tx_queues; i++) {
5691 		struct igb_ring *tx_ring = adapter->tx_ring[i];
5692 		if (!netif_carrier_ok(netdev)) {
5693 			/* We've lost link, so the controller stops DMA,
5694 			 * but we've got queued Tx work that's never going
5695 			 * to get done, so reset controller to flush Tx.
5696 			 * (Do the reset outside of interrupt context).
5697 			 */
5698 			if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
5699 				adapter->tx_timeout_count++;
5700 				schedule_work(&adapter->reset_task);
5701 				/* return immediately since reset is imminent */
5702 				return;
5703 			}
5704 		}
5705 
5706 		/* Force detection of hung controller every watchdog period */
5707 		set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5708 	}
5709 
5710 	/* Cause software interrupt to ensure Rx ring is cleaned */
5711 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
5712 		u32 eics = 0;
5713 
5714 		for (i = 0; i < adapter->num_q_vectors; i++)
5715 			eics |= adapter->q_vector[i]->eims_value;
5716 		wr32(E1000_EICS, eics);
5717 	} else {
5718 		wr32(E1000_ICS, E1000_ICS_RXDMT0);
5719 	}
5720 
5721 	igb_spoof_check(adapter);
5722 	igb_ptp_rx_hang(adapter);
5723 	igb_ptp_tx_hang(adapter);
5724 
5725 	/* Check LVMMC register on i350/i354 only */
5726 	if ((adapter->hw.mac.type == e1000_i350) ||
5727 	    (adapter->hw.mac.type == e1000_i354))
5728 		igb_check_lvmmc(adapter);
5729 
5730 	/* Reset the timer */
5731 	if (!test_bit(__IGB_DOWN, &adapter->state)) {
5732 		if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
5733 			mod_timer(&adapter->watchdog_timer,
5734 				  round_jiffies(jiffies +  HZ));
5735 		else
5736 			mod_timer(&adapter->watchdog_timer,
5737 				  round_jiffies(jiffies + 2 * HZ));
5738 	}
5739 }
5740 
5741 enum latency_range {
5742 	lowest_latency = 0,
5743 	low_latency = 1,
5744 	bulk_latency = 2,
5745 	latency_invalid = 255
5746 };
5747 
5748 /**
5749  *  igb_update_ring_itr - update the dynamic ITR value based on packet size
5750  *  @q_vector: pointer to q_vector
5751  *
5752  *  Stores a new ITR value based on strictly on packet size.  This
5753  *  algorithm is less sophisticated than that used in igb_update_itr,
5754  *  due to the difficulty of synchronizing statistics across multiple
5755  *  receive rings.  The divisors and thresholds used by this function
5756  *  were determined based on theoretical maximum wire speed and testing
5757  *  data, in order to minimize response time while increasing bulk
5758  *  throughput.
5759  *  This functionality is controlled by ethtool's coalescing settings.
5760  *  NOTE:  This function is called only when operating in a multiqueue
5761  *         receive environment.
5762  **/
igb_update_ring_itr(struct igb_q_vector * q_vector)5763 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
5764 {
5765 	int new_val = q_vector->itr_val;
5766 	int avg_wire_size = 0;
5767 	struct igb_adapter *adapter = q_vector->adapter;
5768 	unsigned int packets;
5769 
5770 	/* For non-gigabit speeds, just fix the interrupt rate at 4000
5771 	 * ints/sec - ITR timer value of 120 ticks.
5772 	 */
5773 	if (adapter->link_speed != SPEED_1000) {
5774 		new_val = IGB_4K_ITR;
5775 		goto set_itr_val;
5776 	}
5777 
5778 	packets = q_vector->rx.total_packets;
5779 	if (packets)
5780 		avg_wire_size = q_vector->rx.total_bytes / packets;
5781 
5782 	packets = q_vector->tx.total_packets;
5783 	if (packets)
5784 		avg_wire_size = max_t(u32, avg_wire_size,
5785 				      q_vector->tx.total_bytes / packets);
5786 
5787 	/* if avg_wire_size isn't set no work was done */
5788 	if (!avg_wire_size)
5789 		goto clear_counts;
5790 
5791 	/* Add 24 bytes to size to account for CRC, preamble, and gap */
5792 	avg_wire_size += 24;
5793 
5794 	/* Don't starve jumbo frames */
5795 	avg_wire_size = min(avg_wire_size, 3000);
5796 
5797 	/* Give a little boost to mid-size frames */
5798 	if ((avg_wire_size > 300) && (avg_wire_size < 1200))
5799 		new_val = avg_wire_size / 3;
5800 	else
5801 		new_val = avg_wire_size / 2;
5802 
5803 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
5804 	if (new_val < IGB_20K_ITR &&
5805 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5806 	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5807 		new_val = IGB_20K_ITR;
5808 
5809 set_itr_val:
5810 	if (new_val != q_vector->itr_val) {
5811 		q_vector->itr_val = new_val;
5812 		q_vector->set_itr = 1;
5813 	}
5814 clear_counts:
5815 	q_vector->rx.total_bytes = 0;
5816 	q_vector->rx.total_packets = 0;
5817 	q_vector->tx.total_bytes = 0;
5818 	q_vector->tx.total_packets = 0;
5819 }
5820 
5821 /**
5822  *  igb_update_itr - update the dynamic ITR value based on statistics
5823  *  @q_vector: pointer to q_vector
5824  *  @ring_container: ring info to update the itr for
5825  *
5826  *  Stores a new ITR value based on packets and byte
5827  *  counts during the last interrupt.  The advantage of per interrupt
5828  *  computation is faster updates and more accurate ITR for the current
5829  *  traffic pattern.  Constants in this function were computed
5830  *  based on theoretical maximum wire speed and thresholds were set based
5831  *  on testing data as well as attempting to minimize response time
5832  *  while increasing bulk throughput.
5833  *  This functionality is controlled by ethtool's coalescing settings.
5834  *  NOTE:  These calculations are only valid when operating in a single-
5835  *         queue environment.
5836  **/
igb_update_itr(struct igb_q_vector * q_vector,struct igb_ring_container * ring_container)5837 static void igb_update_itr(struct igb_q_vector *q_vector,
5838 			   struct igb_ring_container *ring_container)
5839 {
5840 	unsigned int packets = ring_container->total_packets;
5841 	unsigned int bytes = ring_container->total_bytes;
5842 	u8 itrval = ring_container->itr;
5843 
5844 	/* no packets, exit with status unchanged */
5845 	if (packets == 0)
5846 		return;
5847 
5848 	switch (itrval) {
5849 	case lowest_latency:
5850 		/* handle TSO and jumbo frames */
5851 		if (bytes/packets > 8000)
5852 			itrval = bulk_latency;
5853 		else if ((packets < 5) && (bytes > 512))
5854 			itrval = low_latency;
5855 		break;
5856 	case low_latency:  /* 50 usec aka 20000 ints/s */
5857 		if (bytes > 10000) {
5858 			/* this if handles the TSO accounting */
5859 			if (bytes/packets > 8000)
5860 				itrval = bulk_latency;
5861 			else if ((packets < 10) || ((bytes/packets) > 1200))
5862 				itrval = bulk_latency;
5863 			else if ((packets > 35))
5864 				itrval = lowest_latency;
5865 		} else if (bytes/packets > 2000) {
5866 			itrval = bulk_latency;
5867 		} else if (packets <= 2 && bytes < 512) {
5868 			itrval = lowest_latency;
5869 		}
5870 		break;
5871 	case bulk_latency: /* 250 usec aka 4000 ints/s */
5872 		if (bytes > 25000) {
5873 			if (packets > 35)
5874 				itrval = low_latency;
5875 		} else if (bytes < 1500) {
5876 			itrval = low_latency;
5877 		}
5878 		break;
5879 	}
5880 
5881 	/* clear work counters since we have the values we need */
5882 	ring_container->total_bytes = 0;
5883 	ring_container->total_packets = 0;
5884 
5885 	/* write updated itr to ring container */
5886 	ring_container->itr = itrval;
5887 }
5888 
igb_set_itr(struct igb_q_vector * q_vector)5889 static void igb_set_itr(struct igb_q_vector *q_vector)
5890 {
5891 	struct igb_adapter *adapter = q_vector->adapter;
5892 	u32 new_itr = q_vector->itr_val;
5893 	u8 current_itr = 0;
5894 
5895 	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
5896 	if (adapter->link_speed != SPEED_1000) {
5897 		current_itr = 0;
5898 		new_itr = IGB_4K_ITR;
5899 		goto set_itr_now;
5900 	}
5901 
5902 	igb_update_itr(q_vector, &q_vector->tx);
5903 	igb_update_itr(q_vector, &q_vector->rx);
5904 
5905 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
5906 
5907 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
5908 	if (current_itr == lowest_latency &&
5909 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5910 	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5911 		current_itr = low_latency;
5912 
5913 	switch (current_itr) {
5914 	/* counts and packets in update_itr are dependent on these numbers */
5915 	case lowest_latency:
5916 		new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
5917 		break;
5918 	case low_latency:
5919 		new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
5920 		break;
5921 	case bulk_latency:
5922 		new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
5923 		break;
5924 	default:
5925 		break;
5926 	}
5927 
5928 set_itr_now:
5929 	if (new_itr != q_vector->itr_val) {
5930 		/* this attempts to bias the interrupt rate towards Bulk
5931 		 * by adding intermediate steps when interrupt rate is
5932 		 * increasing
5933 		 */
5934 		new_itr = new_itr > q_vector->itr_val ?
5935 			  max((new_itr * q_vector->itr_val) /
5936 			  (new_itr + (q_vector->itr_val >> 2)),
5937 			  new_itr) : new_itr;
5938 		/* Don't write the value here; it resets the adapter's
5939 		 * internal timer, and causes us to delay far longer than
5940 		 * we should between interrupts.  Instead, we write the ITR
5941 		 * value at the beginning of the next interrupt so the timing
5942 		 * ends up being correct.
5943 		 */
5944 		q_vector->itr_val = new_itr;
5945 		q_vector->set_itr = 1;
5946 	}
5947 }
5948 
igb_tx_ctxtdesc(struct igb_ring * tx_ring,struct igb_tx_buffer * first,u32 vlan_macip_lens,u32 type_tucmd,u32 mss_l4len_idx)5949 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring,
5950 			    struct igb_tx_buffer *first,
5951 			    u32 vlan_macip_lens, u32 type_tucmd,
5952 			    u32 mss_l4len_idx)
5953 {
5954 	struct e1000_adv_tx_context_desc *context_desc;
5955 	u16 i = tx_ring->next_to_use;
5956 	struct timespec64 ts;
5957 
5958 	context_desc = IGB_TX_CTXTDESC(tx_ring, i);
5959 
5960 	i++;
5961 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
5962 
5963 	/* set bits to identify this as an advanced context descriptor */
5964 	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
5965 
5966 	/* For 82575, context index must be unique per ring. */
5967 	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5968 		mss_l4len_idx |= tx_ring->reg_idx << 4;
5969 
5970 	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
5971 	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
5972 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
5973 
5974 	/* We assume there is always a valid tx time available. Invalid times
5975 	 * should have been handled by the upper layers.
5976 	 */
5977 	if (tx_ring->launchtime_enable) {
5978 		ts = ktime_to_timespec64(first->skb->tstamp);
5979 		skb_txtime_consumed(first->skb);
5980 		context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32);
5981 	} else {
5982 		context_desc->seqnum_seed = 0;
5983 	}
5984 }
5985 
igb_tso(struct igb_ring * tx_ring,struct igb_tx_buffer * first,u8 * hdr_len)5986 static int igb_tso(struct igb_ring *tx_ring,
5987 		   struct igb_tx_buffer *first,
5988 		   u8 *hdr_len)
5989 {
5990 	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
5991 	struct sk_buff *skb = first->skb;
5992 	union {
5993 		struct iphdr *v4;
5994 		struct ipv6hdr *v6;
5995 		unsigned char *hdr;
5996 	} ip;
5997 	union {
5998 		struct tcphdr *tcp;
5999 		struct udphdr *udp;
6000 		unsigned char *hdr;
6001 	} l4;
6002 	u32 paylen, l4_offset;
6003 	int err;
6004 
6005 	if (skb->ip_summed != CHECKSUM_PARTIAL)
6006 		return 0;
6007 
6008 	if (!skb_is_gso(skb))
6009 		return 0;
6010 
6011 	err = skb_cow_head(skb, 0);
6012 	if (err < 0)
6013 		return err;
6014 
6015 	ip.hdr = skb_network_header(skb);
6016 	l4.hdr = skb_checksum_start(skb);
6017 
6018 	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
6019 	type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?
6020 		      E1000_ADVTXD_TUCMD_L4T_UDP : E1000_ADVTXD_TUCMD_L4T_TCP;
6021 
6022 	/* initialize outer IP header fields */
6023 	if (ip.v4->version == 4) {
6024 		unsigned char *csum_start = skb_checksum_start(skb);
6025 		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
6026 
6027 		/* IP header will have to cancel out any data that
6028 		 * is not a part of the outer IP header
6029 		 */
6030 		ip.v4->check = csum_fold(csum_partial(trans_start,
6031 						      csum_start - trans_start,
6032 						      0));
6033 		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
6034 
6035 		ip.v4->tot_len = 0;
6036 		first->tx_flags |= IGB_TX_FLAGS_TSO |
6037 				   IGB_TX_FLAGS_CSUM |
6038 				   IGB_TX_FLAGS_IPV4;
6039 	} else {
6040 		ip.v6->payload_len = 0;
6041 		first->tx_flags |= IGB_TX_FLAGS_TSO |
6042 				   IGB_TX_FLAGS_CSUM;
6043 	}
6044 
6045 	/* determine offset of inner transport header */
6046 	l4_offset = l4.hdr - skb->data;
6047 
6048 	/* remove payload length from inner checksum */
6049 	paylen = skb->len - l4_offset;
6050 	if (type_tucmd & E1000_ADVTXD_TUCMD_L4T_TCP) {
6051 		/* compute length of segmentation header */
6052 		*hdr_len = (l4.tcp->doff * 4) + l4_offset;
6053 		csum_replace_by_diff(&l4.tcp->check,
6054 			(__force __wsum)htonl(paylen));
6055 	} else {
6056 		/* compute length of segmentation header */
6057 		*hdr_len = sizeof(*l4.udp) + l4_offset;
6058 		csum_replace_by_diff(&l4.udp->check,
6059 				     (__force __wsum)htonl(paylen));
6060 	}
6061 
6062 	/* update gso size and bytecount with header size */
6063 	first->gso_segs = skb_shinfo(skb)->gso_segs;
6064 	first->bytecount += (first->gso_segs - 1) * *hdr_len;
6065 
6066 	/* MSS L4LEN IDX */
6067 	mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
6068 	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
6069 
6070 	/* VLAN MACLEN IPLEN */
6071 	vlan_macip_lens = l4.hdr - ip.hdr;
6072 	vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
6073 	vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
6074 
6075 	igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens,
6076 			type_tucmd, mss_l4len_idx);
6077 
6078 	return 1;
6079 }
6080 
igb_tx_csum(struct igb_ring * tx_ring,struct igb_tx_buffer * first)6081 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
6082 {
6083 	struct sk_buff *skb = first->skb;
6084 	u32 vlan_macip_lens = 0;
6085 	u32 type_tucmd = 0;
6086 
6087 	if (skb->ip_summed != CHECKSUM_PARTIAL) {
6088 csum_failed:
6089 		if (!(first->tx_flags & IGB_TX_FLAGS_VLAN) &&
6090 		    !tx_ring->launchtime_enable)
6091 			return;
6092 		goto no_csum;
6093 	}
6094 
6095 	switch (skb->csum_offset) {
6096 	case offsetof(struct tcphdr, check):
6097 		type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
6098 		fallthrough;
6099 	case offsetof(struct udphdr, check):
6100 		break;
6101 	case offsetof(struct sctphdr, checksum):
6102 		/* validate that this is actually an SCTP request */
6103 		if (skb_csum_is_sctp(skb)) {
6104 			type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
6105 			break;
6106 		}
6107 		fallthrough;
6108 	default:
6109 		skb_checksum_help(skb);
6110 		goto csum_failed;
6111 	}
6112 
6113 	/* update TX checksum flag */
6114 	first->tx_flags |= IGB_TX_FLAGS_CSUM;
6115 	vlan_macip_lens = skb_checksum_start_offset(skb) -
6116 			  skb_network_offset(skb);
6117 no_csum:
6118 	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
6119 	vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
6120 
6121 	igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
6122 }
6123 
6124 #define IGB_SET_FLAG(_input, _flag, _result) \
6125 	((_flag <= _result) ? \
6126 	 ((u32)(_input & _flag) * (_result / _flag)) : \
6127 	 ((u32)(_input & _flag) / (_flag / _result)))
6128 
igb_tx_cmd_type(struct sk_buff * skb,u32 tx_flags)6129 static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
6130 {
6131 	/* set type for advanced descriptor with frame checksum insertion */
6132 	u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
6133 		       E1000_ADVTXD_DCMD_DEXT |
6134 		       E1000_ADVTXD_DCMD_IFCS;
6135 
6136 	/* set HW vlan bit if vlan is present */
6137 	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
6138 				 (E1000_ADVTXD_DCMD_VLE));
6139 
6140 	/* set segmentation bits for TSO */
6141 	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
6142 				 (E1000_ADVTXD_DCMD_TSE));
6143 
6144 	/* set timestamp bit if present */
6145 	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
6146 				 (E1000_ADVTXD_MAC_TSTAMP));
6147 
6148 	/* insert frame checksum */
6149 	cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
6150 
6151 	return cmd_type;
6152 }
6153 
igb_tx_olinfo_status(struct igb_ring * tx_ring,union e1000_adv_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)6154 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
6155 				 union e1000_adv_tx_desc *tx_desc,
6156 				 u32 tx_flags, unsigned int paylen)
6157 {
6158 	u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
6159 
6160 	/* 82575 requires a unique index per ring */
6161 	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
6162 		olinfo_status |= tx_ring->reg_idx << 4;
6163 
6164 	/* insert L4 checksum */
6165 	olinfo_status |= IGB_SET_FLAG(tx_flags,
6166 				      IGB_TX_FLAGS_CSUM,
6167 				      (E1000_TXD_POPTS_TXSM << 8));
6168 
6169 	/* insert IPv4 checksum */
6170 	olinfo_status |= IGB_SET_FLAG(tx_flags,
6171 				      IGB_TX_FLAGS_IPV4,
6172 				      (E1000_TXD_POPTS_IXSM << 8));
6173 
6174 	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6175 }
6176 
__igb_maybe_stop_tx(struct igb_ring * tx_ring,const u16 size)6177 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
6178 {
6179 	struct net_device *netdev = tx_ring->netdev;
6180 
6181 	netif_stop_subqueue(netdev, tx_ring->queue_index);
6182 
6183 	/* Herbert's original patch had:
6184 	 *  smp_mb__after_netif_stop_queue();
6185 	 * but since that doesn't exist yet, just open code it.
6186 	 */
6187 	smp_mb();
6188 
6189 	/* We need to check again in a case another CPU has just
6190 	 * made room available.
6191 	 */
6192 	if (igb_desc_unused(tx_ring) < size)
6193 		return -EBUSY;
6194 
6195 	/* A reprieve! */
6196 	netif_wake_subqueue(netdev, tx_ring->queue_index);
6197 
6198 	u64_stats_update_begin(&tx_ring->tx_syncp2);
6199 	tx_ring->tx_stats.restart_queue2++;
6200 	u64_stats_update_end(&tx_ring->tx_syncp2);
6201 
6202 	return 0;
6203 }
6204 
igb_maybe_stop_tx(struct igb_ring * tx_ring,const u16 size)6205 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
6206 {
6207 	if (igb_desc_unused(tx_ring) >= size)
6208 		return 0;
6209 	return __igb_maybe_stop_tx(tx_ring, size);
6210 }
6211 
igb_tx_map(struct igb_ring * tx_ring,struct igb_tx_buffer * first,const u8 hdr_len)6212 static int igb_tx_map(struct igb_ring *tx_ring,
6213 		      struct igb_tx_buffer *first,
6214 		      const u8 hdr_len)
6215 {
6216 	struct sk_buff *skb = first->skb;
6217 	struct igb_tx_buffer *tx_buffer;
6218 	union e1000_adv_tx_desc *tx_desc;
6219 	skb_frag_t *frag;
6220 	dma_addr_t dma;
6221 	unsigned int data_len, size;
6222 	u32 tx_flags = first->tx_flags;
6223 	u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
6224 	u16 i = tx_ring->next_to_use;
6225 
6226 	tx_desc = IGB_TX_DESC(tx_ring, i);
6227 
6228 	igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
6229 
6230 	size = skb_headlen(skb);
6231 	data_len = skb->data_len;
6232 
6233 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
6234 
6235 	tx_buffer = first;
6236 
6237 	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
6238 		if (dma_mapping_error(tx_ring->dev, dma))
6239 			goto dma_error;
6240 
6241 		/* record length, and DMA address */
6242 		dma_unmap_len_set(tx_buffer, len, size);
6243 		dma_unmap_addr_set(tx_buffer, dma, dma);
6244 
6245 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
6246 
6247 		while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
6248 			tx_desc->read.cmd_type_len =
6249 				cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
6250 
6251 			i++;
6252 			tx_desc++;
6253 			if (i == tx_ring->count) {
6254 				tx_desc = IGB_TX_DESC(tx_ring, 0);
6255 				i = 0;
6256 			}
6257 			tx_desc->read.olinfo_status = 0;
6258 
6259 			dma += IGB_MAX_DATA_PER_TXD;
6260 			size -= IGB_MAX_DATA_PER_TXD;
6261 
6262 			tx_desc->read.buffer_addr = cpu_to_le64(dma);
6263 		}
6264 
6265 		if (likely(!data_len))
6266 			break;
6267 
6268 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
6269 
6270 		i++;
6271 		tx_desc++;
6272 		if (i == tx_ring->count) {
6273 			tx_desc = IGB_TX_DESC(tx_ring, 0);
6274 			i = 0;
6275 		}
6276 		tx_desc->read.olinfo_status = 0;
6277 
6278 		size = skb_frag_size(frag);
6279 		data_len -= size;
6280 
6281 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
6282 				       size, DMA_TO_DEVICE);
6283 
6284 		tx_buffer = &tx_ring->tx_buffer_info[i];
6285 	}
6286 
6287 	/* write last descriptor with RS and EOP bits */
6288 	cmd_type |= size | IGB_TXD_DCMD;
6289 	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6290 
6291 	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
6292 
6293 	/* set the timestamp */
6294 	first->time_stamp = jiffies;
6295 
6296 	skb_tx_timestamp(skb);
6297 
6298 	/* Force memory writes to complete before letting h/w know there
6299 	 * are new descriptors to fetch.  (Only applicable for weak-ordered
6300 	 * memory model archs, such as IA-64).
6301 	 *
6302 	 * We also need this memory barrier to make certain all of the
6303 	 * status bits have been updated before next_to_watch is written.
6304 	 */
6305 	dma_wmb();
6306 
6307 	/* set next_to_watch value indicating a packet is present */
6308 	first->next_to_watch = tx_desc;
6309 
6310 	i++;
6311 	if (i == tx_ring->count)
6312 		i = 0;
6313 
6314 	tx_ring->next_to_use = i;
6315 
6316 	/* Make sure there is space in the ring for the next send. */
6317 	igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6318 
6319 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
6320 		writel(i, tx_ring->tail);
6321 	}
6322 	return 0;
6323 
6324 dma_error:
6325 	dev_err(tx_ring->dev, "TX DMA map failed\n");
6326 	tx_buffer = &tx_ring->tx_buffer_info[i];
6327 
6328 	/* clear dma mappings for failed tx_buffer_info map */
6329 	while (tx_buffer != first) {
6330 		if (dma_unmap_len(tx_buffer, len))
6331 			dma_unmap_page(tx_ring->dev,
6332 				       dma_unmap_addr(tx_buffer, dma),
6333 				       dma_unmap_len(tx_buffer, len),
6334 				       DMA_TO_DEVICE);
6335 		dma_unmap_len_set(tx_buffer, len, 0);
6336 
6337 		if (i-- == 0)
6338 			i += tx_ring->count;
6339 		tx_buffer = &tx_ring->tx_buffer_info[i];
6340 	}
6341 
6342 	if (dma_unmap_len(tx_buffer, len))
6343 		dma_unmap_single(tx_ring->dev,
6344 				 dma_unmap_addr(tx_buffer, dma),
6345 				 dma_unmap_len(tx_buffer, len),
6346 				 DMA_TO_DEVICE);
6347 	dma_unmap_len_set(tx_buffer, len, 0);
6348 
6349 	dev_kfree_skb_any(tx_buffer->skb);
6350 	tx_buffer->skb = NULL;
6351 
6352 	tx_ring->next_to_use = i;
6353 
6354 	return -1;
6355 }
6356 
igb_xmit_xdp_ring(struct igb_adapter * adapter,struct igb_ring * tx_ring,struct xdp_frame * xdpf)6357 int igb_xmit_xdp_ring(struct igb_adapter *adapter,
6358 		      struct igb_ring *tx_ring,
6359 		      struct xdp_frame *xdpf)
6360 {
6361 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
6362 	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
6363 	u16 count, i, index = tx_ring->next_to_use;
6364 	struct igb_tx_buffer *tx_head = &tx_ring->tx_buffer_info[index];
6365 	struct igb_tx_buffer *tx_buffer = tx_head;
6366 	union e1000_adv_tx_desc *tx_desc = IGB_TX_DESC(tx_ring, index);
6367 	u32 len = xdpf->len, cmd_type, olinfo_status;
6368 	void *data = xdpf->data;
6369 
6370 	count = TXD_USE_COUNT(len);
6371 	for (i = 0; i < nr_frags; i++)
6372 		count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
6373 
6374 	if (igb_maybe_stop_tx(tx_ring, count + 3))
6375 		return IGB_XDP_CONSUMED;
6376 
6377 	i = 0;
6378 	/* record the location of the first descriptor for this packet */
6379 	tx_head->bytecount = xdp_get_frame_len(xdpf);
6380 	tx_head->type = IGB_TYPE_XDP;
6381 	tx_head->gso_segs = 1;
6382 	tx_head->xdpf = xdpf;
6383 
6384 	olinfo_status = tx_head->bytecount << E1000_ADVTXD_PAYLEN_SHIFT;
6385 	/* 82575 requires a unique index per ring */
6386 	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
6387 		olinfo_status |= tx_ring->reg_idx << 4;
6388 	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6389 
6390 	for (;;) {
6391 		dma_addr_t dma;
6392 
6393 		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
6394 		if (dma_mapping_error(tx_ring->dev, dma))
6395 			goto unmap;
6396 
6397 		/* record length, and DMA address */
6398 		dma_unmap_len_set(tx_buffer, len, len);
6399 		dma_unmap_addr_set(tx_buffer, dma, dma);
6400 
6401 		/* put descriptor type bits */
6402 		cmd_type = E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_DEXT |
6403 			   E1000_ADVTXD_DCMD_IFCS | len;
6404 
6405 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6406 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
6407 
6408 		tx_buffer->protocol = 0;
6409 
6410 		if (++index == tx_ring->count)
6411 			index = 0;
6412 
6413 		if (i == nr_frags)
6414 			break;
6415 
6416 		tx_buffer = &tx_ring->tx_buffer_info[index];
6417 		tx_desc = IGB_TX_DESC(tx_ring, index);
6418 		tx_desc->read.olinfo_status = 0;
6419 
6420 		data = skb_frag_address(&sinfo->frags[i]);
6421 		len = skb_frag_size(&sinfo->frags[i]);
6422 		i++;
6423 	}
6424 	tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_TXD_DCMD);
6425 
6426 	netdev_tx_sent_queue(txring_txq(tx_ring), tx_head->bytecount);
6427 	/* set the timestamp */
6428 	tx_head->time_stamp = jiffies;
6429 
6430 	/* Avoid any potential race with xdp_xmit and cleanup */
6431 	smp_wmb();
6432 
6433 	/* set next_to_watch value indicating a packet is present */
6434 	tx_head->next_to_watch = tx_desc;
6435 	tx_ring->next_to_use = index;
6436 
6437 	/* Make sure there is space in the ring for the next send. */
6438 	igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6439 
6440 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more())
6441 		writel(index, tx_ring->tail);
6442 
6443 	return IGB_XDP_TX;
6444 
6445 unmap:
6446 	for (;;) {
6447 		tx_buffer = &tx_ring->tx_buffer_info[index];
6448 		if (dma_unmap_len(tx_buffer, len))
6449 			dma_unmap_page(tx_ring->dev,
6450 				       dma_unmap_addr(tx_buffer, dma),
6451 				       dma_unmap_len(tx_buffer, len),
6452 				       DMA_TO_DEVICE);
6453 		dma_unmap_len_set(tx_buffer, len, 0);
6454 		if (tx_buffer == tx_head)
6455 			break;
6456 
6457 		if (!index)
6458 			index += tx_ring->count;
6459 		index--;
6460 	}
6461 
6462 	return IGB_XDP_CONSUMED;
6463 }
6464 
igb_xmit_frame_ring(struct sk_buff * skb,struct igb_ring * tx_ring)6465 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
6466 				struct igb_ring *tx_ring)
6467 {
6468 	struct igb_tx_buffer *first;
6469 	int tso;
6470 	u32 tx_flags = 0;
6471 	unsigned short f;
6472 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
6473 	__be16 protocol = vlan_get_protocol(skb);
6474 	u8 hdr_len = 0;
6475 
6476 	/* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
6477 	 *       + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
6478 	 *       + 2 desc gap to keep tail from touching head,
6479 	 *       + 1 desc for context descriptor,
6480 	 * otherwise try next time
6481 	 */
6482 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
6483 		count += TXD_USE_COUNT(skb_frag_size(
6484 						&skb_shinfo(skb)->frags[f]));
6485 
6486 	if (igb_maybe_stop_tx(tx_ring, count + 3)) {
6487 		/* this is a hard error */
6488 		return NETDEV_TX_BUSY;
6489 	}
6490 
6491 	/* record the location of the first descriptor for this packet */
6492 	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
6493 	first->type = IGB_TYPE_SKB;
6494 	first->skb = skb;
6495 	first->bytecount = skb->len;
6496 	first->gso_segs = 1;
6497 
6498 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
6499 		struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6500 
6501 		if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
6502 		    !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
6503 					   &adapter->state)) {
6504 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
6505 			tx_flags |= IGB_TX_FLAGS_TSTAMP;
6506 
6507 			adapter->ptp_tx_skb = skb_get(skb);
6508 			adapter->ptp_tx_start = jiffies;
6509 			if (adapter->hw.mac.type == e1000_82576)
6510 				schedule_work(&adapter->ptp_tx_work);
6511 		} else {
6512 			adapter->tx_hwtstamp_skipped++;
6513 		}
6514 	}
6515 
6516 	if (skb_vlan_tag_present(skb)) {
6517 		tx_flags |= IGB_TX_FLAGS_VLAN;
6518 		tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
6519 	}
6520 
6521 	/* record initial flags and protocol */
6522 	first->tx_flags = tx_flags;
6523 	first->protocol = protocol;
6524 
6525 	tso = igb_tso(tx_ring, first, &hdr_len);
6526 	if (tso < 0)
6527 		goto out_drop;
6528 	else if (!tso)
6529 		igb_tx_csum(tx_ring, first);
6530 
6531 	if (igb_tx_map(tx_ring, first, hdr_len))
6532 		goto cleanup_tx_tstamp;
6533 
6534 	return NETDEV_TX_OK;
6535 
6536 out_drop:
6537 	dev_kfree_skb_any(first->skb);
6538 	first->skb = NULL;
6539 cleanup_tx_tstamp:
6540 	if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) {
6541 		struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6542 
6543 		dev_kfree_skb_any(adapter->ptp_tx_skb);
6544 		adapter->ptp_tx_skb = NULL;
6545 		if (adapter->hw.mac.type == e1000_82576)
6546 			cancel_work_sync(&adapter->ptp_tx_work);
6547 		clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
6548 	}
6549 
6550 	return NETDEV_TX_OK;
6551 }
6552 
igb_tx_queue_mapping(struct igb_adapter * adapter,struct sk_buff * skb)6553 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
6554 						    struct sk_buff *skb)
6555 {
6556 	unsigned int r_idx = skb->queue_mapping;
6557 
6558 	if (r_idx >= adapter->num_tx_queues)
6559 		r_idx = r_idx % adapter->num_tx_queues;
6560 
6561 	return adapter->tx_ring[r_idx];
6562 }
6563 
igb_xmit_frame(struct sk_buff * skb,struct net_device * netdev)6564 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
6565 				  struct net_device *netdev)
6566 {
6567 	struct igb_adapter *adapter = netdev_priv(netdev);
6568 
6569 	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
6570 	 * in order to meet this minimum size requirement.
6571 	 */
6572 	if (skb_put_padto(skb, 17))
6573 		return NETDEV_TX_OK;
6574 
6575 	return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
6576 }
6577 
6578 /**
6579  *  igb_tx_timeout - Respond to a Tx Hang
6580  *  @netdev: network interface device structure
6581  *  @txqueue: number of the Tx queue that hung (unused)
6582  **/
igb_tx_timeout(struct net_device * netdev,unsigned int __always_unused txqueue)6583 static void igb_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
6584 {
6585 	struct igb_adapter *adapter = netdev_priv(netdev);
6586 	struct e1000_hw *hw = &adapter->hw;
6587 
6588 	/* Do the reset outside of interrupt context */
6589 	adapter->tx_timeout_count++;
6590 
6591 	if (hw->mac.type >= e1000_82580)
6592 		hw->dev_spec._82575.global_device_reset = true;
6593 
6594 	schedule_work(&adapter->reset_task);
6595 	wr32(E1000_EICS,
6596 	     (adapter->eims_enable_mask & ~adapter->eims_other));
6597 }
6598 
igb_reset_task(struct work_struct * work)6599 static void igb_reset_task(struct work_struct *work)
6600 {
6601 	struct igb_adapter *adapter;
6602 	adapter = container_of(work, struct igb_adapter, reset_task);
6603 
6604 	rtnl_lock();
6605 	/* If we're already down or resetting, just bail */
6606 	if (test_bit(__IGB_DOWN, &adapter->state) ||
6607 	    test_bit(__IGB_RESETTING, &adapter->state)) {
6608 		rtnl_unlock();
6609 		return;
6610 	}
6611 
6612 	igb_dump(adapter);
6613 	netdev_err(adapter->netdev, "Reset adapter\n");
6614 	igb_reinit_locked(adapter);
6615 	rtnl_unlock();
6616 }
6617 
6618 /**
6619  *  igb_get_stats64 - Get System Network Statistics
6620  *  @netdev: network interface device structure
6621  *  @stats: rtnl_link_stats64 pointer
6622  **/
igb_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)6623 static void igb_get_stats64(struct net_device *netdev,
6624 			    struct rtnl_link_stats64 *stats)
6625 {
6626 	struct igb_adapter *adapter = netdev_priv(netdev);
6627 
6628 	spin_lock(&adapter->stats64_lock);
6629 	igb_update_stats(adapter);
6630 	memcpy(stats, &adapter->stats64, sizeof(*stats));
6631 	spin_unlock(&adapter->stats64_lock);
6632 }
6633 
6634 /**
6635  *  igb_change_mtu - Change the Maximum Transfer Unit
6636  *  @netdev: network interface device structure
6637  *  @new_mtu: new value for maximum frame size
6638  *
6639  *  Returns 0 on success, negative on failure
6640  **/
igb_change_mtu(struct net_device * netdev,int new_mtu)6641 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
6642 {
6643 	struct igb_adapter *adapter = netdev_priv(netdev);
6644 	int max_frame = new_mtu + IGB_ETH_PKT_HDR_PAD;
6645 
6646 	if (adapter->xdp_prog) {
6647 		int i;
6648 
6649 		for (i = 0; i < adapter->num_rx_queues; i++) {
6650 			struct igb_ring *ring = adapter->rx_ring[i];
6651 
6652 			if (max_frame > igb_rx_bufsz(ring)) {
6653 				netdev_warn(adapter->netdev,
6654 					    "Requested MTU size is not supported with XDP. Max frame size is %d\n",
6655 					    max_frame);
6656 				return -EINVAL;
6657 			}
6658 		}
6659 	}
6660 
6661 	/* adjust max frame to be at least the size of a standard frame */
6662 	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
6663 		max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
6664 
6665 	while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
6666 		usleep_range(1000, 2000);
6667 
6668 	/* igb_down has a dependency on max_frame_size */
6669 	adapter->max_frame_size = max_frame;
6670 
6671 	if (netif_running(netdev))
6672 		igb_down(adapter);
6673 
6674 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
6675 		   netdev->mtu, new_mtu);
6676 	netdev->mtu = new_mtu;
6677 
6678 	if (netif_running(netdev))
6679 		igb_up(adapter);
6680 	else
6681 		igb_reset(adapter);
6682 
6683 	clear_bit(__IGB_RESETTING, &adapter->state);
6684 
6685 	return 0;
6686 }
6687 
6688 /**
6689  *  igb_update_stats - Update the board statistics counters
6690  *  @adapter: board private structure
6691  **/
igb_update_stats(struct igb_adapter * adapter)6692 void igb_update_stats(struct igb_adapter *adapter)
6693 {
6694 	struct rtnl_link_stats64 *net_stats = &adapter->stats64;
6695 	struct e1000_hw *hw = &adapter->hw;
6696 	struct pci_dev *pdev = adapter->pdev;
6697 	u32 reg, mpc;
6698 	int i;
6699 	u64 bytes, packets;
6700 	unsigned int start;
6701 	u64 _bytes, _packets;
6702 
6703 	/* Prevent stats update while adapter is being reset, or if the pci
6704 	 * connection is down.
6705 	 */
6706 	if (adapter->link_speed == 0)
6707 		return;
6708 	if (pci_channel_offline(pdev))
6709 		return;
6710 
6711 	bytes = 0;
6712 	packets = 0;
6713 
6714 	rcu_read_lock();
6715 	for (i = 0; i < adapter->num_rx_queues; i++) {
6716 		struct igb_ring *ring = adapter->rx_ring[i];
6717 		u32 rqdpc = rd32(E1000_RQDPC(i));
6718 		if (hw->mac.type >= e1000_i210)
6719 			wr32(E1000_RQDPC(i), 0);
6720 
6721 		if (rqdpc) {
6722 			ring->rx_stats.drops += rqdpc;
6723 			net_stats->rx_fifo_errors += rqdpc;
6724 		}
6725 
6726 		do {
6727 			start = u64_stats_fetch_begin(&ring->rx_syncp);
6728 			_bytes = ring->rx_stats.bytes;
6729 			_packets = ring->rx_stats.packets;
6730 		} while (u64_stats_fetch_retry(&ring->rx_syncp, start));
6731 		bytes += _bytes;
6732 		packets += _packets;
6733 	}
6734 
6735 	net_stats->rx_bytes = bytes;
6736 	net_stats->rx_packets = packets;
6737 
6738 	bytes = 0;
6739 	packets = 0;
6740 	for (i = 0; i < adapter->num_tx_queues; i++) {
6741 		struct igb_ring *ring = adapter->tx_ring[i];
6742 		do {
6743 			start = u64_stats_fetch_begin(&ring->tx_syncp);
6744 			_bytes = ring->tx_stats.bytes;
6745 			_packets = ring->tx_stats.packets;
6746 		} while (u64_stats_fetch_retry(&ring->tx_syncp, start));
6747 		bytes += _bytes;
6748 		packets += _packets;
6749 	}
6750 	net_stats->tx_bytes = bytes;
6751 	net_stats->tx_packets = packets;
6752 	rcu_read_unlock();
6753 
6754 	/* read stats registers */
6755 	adapter->stats.crcerrs += rd32(E1000_CRCERRS);
6756 	adapter->stats.gprc += rd32(E1000_GPRC);
6757 	adapter->stats.gorc += rd32(E1000_GORCL);
6758 	rd32(E1000_GORCH); /* clear GORCL */
6759 	adapter->stats.bprc += rd32(E1000_BPRC);
6760 	adapter->stats.mprc += rd32(E1000_MPRC);
6761 	adapter->stats.roc += rd32(E1000_ROC);
6762 
6763 	adapter->stats.prc64 += rd32(E1000_PRC64);
6764 	adapter->stats.prc127 += rd32(E1000_PRC127);
6765 	adapter->stats.prc255 += rd32(E1000_PRC255);
6766 	adapter->stats.prc511 += rd32(E1000_PRC511);
6767 	adapter->stats.prc1023 += rd32(E1000_PRC1023);
6768 	adapter->stats.prc1522 += rd32(E1000_PRC1522);
6769 	adapter->stats.symerrs += rd32(E1000_SYMERRS);
6770 	adapter->stats.sec += rd32(E1000_SEC);
6771 
6772 	mpc = rd32(E1000_MPC);
6773 	adapter->stats.mpc += mpc;
6774 	net_stats->rx_fifo_errors += mpc;
6775 	adapter->stats.scc += rd32(E1000_SCC);
6776 	adapter->stats.ecol += rd32(E1000_ECOL);
6777 	adapter->stats.mcc += rd32(E1000_MCC);
6778 	adapter->stats.latecol += rd32(E1000_LATECOL);
6779 	adapter->stats.dc += rd32(E1000_DC);
6780 	adapter->stats.rlec += rd32(E1000_RLEC);
6781 	adapter->stats.xonrxc += rd32(E1000_XONRXC);
6782 	adapter->stats.xontxc += rd32(E1000_XONTXC);
6783 	adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
6784 	adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
6785 	adapter->stats.fcruc += rd32(E1000_FCRUC);
6786 	adapter->stats.gptc += rd32(E1000_GPTC);
6787 	adapter->stats.gotc += rd32(E1000_GOTCL);
6788 	rd32(E1000_GOTCH); /* clear GOTCL */
6789 	adapter->stats.rnbc += rd32(E1000_RNBC);
6790 	adapter->stats.ruc += rd32(E1000_RUC);
6791 	adapter->stats.rfc += rd32(E1000_RFC);
6792 	adapter->stats.rjc += rd32(E1000_RJC);
6793 	adapter->stats.tor += rd32(E1000_TORH);
6794 	adapter->stats.tot += rd32(E1000_TOTH);
6795 	adapter->stats.tpr += rd32(E1000_TPR);
6796 
6797 	adapter->stats.ptc64 += rd32(E1000_PTC64);
6798 	adapter->stats.ptc127 += rd32(E1000_PTC127);
6799 	adapter->stats.ptc255 += rd32(E1000_PTC255);
6800 	adapter->stats.ptc511 += rd32(E1000_PTC511);
6801 	adapter->stats.ptc1023 += rd32(E1000_PTC1023);
6802 	adapter->stats.ptc1522 += rd32(E1000_PTC1522);
6803 
6804 	adapter->stats.mptc += rd32(E1000_MPTC);
6805 	adapter->stats.bptc += rd32(E1000_BPTC);
6806 
6807 	adapter->stats.tpt += rd32(E1000_TPT);
6808 	adapter->stats.colc += rd32(E1000_COLC);
6809 
6810 	adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
6811 	/* read internal phy specific stats */
6812 	reg = rd32(E1000_CTRL_EXT);
6813 	if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
6814 		adapter->stats.rxerrc += rd32(E1000_RXERRC);
6815 
6816 		/* this stat has invalid values on i210/i211 */
6817 		if ((hw->mac.type != e1000_i210) &&
6818 		    (hw->mac.type != e1000_i211))
6819 			adapter->stats.tncrs += rd32(E1000_TNCRS);
6820 	}
6821 
6822 	adapter->stats.tsctc += rd32(E1000_TSCTC);
6823 	adapter->stats.tsctfc += rd32(E1000_TSCTFC);
6824 
6825 	adapter->stats.iac += rd32(E1000_IAC);
6826 	adapter->stats.icrxoc += rd32(E1000_ICRXOC);
6827 	adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
6828 	adapter->stats.icrxatc += rd32(E1000_ICRXATC);
6829 	adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
6830 	adapter->stats.ictxatc += rd32(E1000_ICTXATC);
6831 	adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
6832 	adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
6833 	adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
6834 
6835 	/* Fill out the OS statistics structure */
6836 	net_stats->multicast = adapter->stats.mprc;
6837 	net_stats->collisions = adapter->stats.colc;
6838 
6839 	/* Rx Errors */
6840 
6841 	/* RLEC on some newer hardware can be incorrect so build
6842 	 * our own version based on RUC and ROC
6843 	 */
6844 	net_stats->rx_errors = adapter->stats.rxerrc +
6845 		adapter->stats.crcerrs + adapter->stats.algnerrc +
6846 		adapter->stats.ruc + adapter->stats.roc +
6847 		adapter->stats.cexterr;
6848 	net_stats->rx_length_errors = adapter->stats.ruc +
6849 				      adapter->stats.roc;
6850 	net_stats->rx_crc_errors = adapter->stats.crcerrs;
6851 	net_stats->rx_frame_errors = adapter->stats.algnerrc;
6852 	net_stats->rx_missed_errors = adapter->stats.mpc;
6853 
6854 	/* Tx Errors */
6855 	net_stats->tx_errors = adapter->stats.ecol +
6856 			       adapter->stats.latecol;
6857 	net_stats->tx_aborted_errors = adapter->stats.ecol;
6858 	net_stats->tx_window_errors = adapter->stats.latecol;
6859 	net_stats->tx_carrier_errors = adapter->stats.tncrs;
6860 
6861 	/* Tx Dropped needs to be maintained elsewhere */
6862 
6863 	/* Management Stats */
6864 	adapter->stats.mgptc += rd32(E1000_MGTPTC);
6865 	adapter->stats.mgprc += rd32(E1000_MGTPRC);
6866 	adapter->stats.mgpdc += rd32(E1000_MGTPDC);
6867 
6868 	/* OS2BMC Stats */
6869 	reg = rd32(E1000_MANC);
6870 	if (reg & E1000_MANC_EN_BMC2OS) {
6871 		adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
6872 		adapter->stats.o2bspc += rd32(E1000_O2BSPC);
6873 		adapter->stats.b2ospc += rd32(E1000_B2OSPC);
6874 		adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
6875 	}
6876 }
6877 
igb_perout(struct igb_adapter * adapter,int tsintr_tt)6878 static void igb_perout(struct igb_adapter *adapter, int tsintr_tt)
6879 {
6880 	int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_PEROUT, tsintr_tt);
6881 	struct e1000_hw *hw = &adapter->hw;
6882 	struct timespec64 ts;
6883 	u32 tsauxc;
6884 
6885 	if (pin < 0 || pin >= IGB_N_SDP)
6886 		return;
6887 
6888 	spin_lock(&adapter->tmreg_lock);
6889 
6890 	if (hw->mac.type == e1000_82580 ||
6891 	    hw->mac.type == e1000_i354 ||
6892 	    hw->mac.type == e1000_i350) {
6893 		s64 ns = timespec64_to_ns(&adapter->perout[tsintr_tt].period);
6894 		u32 systiml, systimh, level_mask, level, rem;
6895 		u64 systim, now;
6896 
6897 		/* read systim registers in sequence */
6898 		rd32(E1000_SYSTIMR);
6899 		systiml = rd32(E1000_SYSTIML);
6900 		systimh = rd32(E1000_SYSTIMH);
6901 		systim = (((u64)(systimh & 0xFF)) << 32) | ((u64)systiml);
6902 		now = timecounter_cyc2time(&adapter->tc, systim);
6903 
6904 		if (pin < 2) {
6905 			level_mask = (tsintr_tt == 1) ? 0x80000 : 0x40000;
6906 			level = (rd32(E1000_CTRL) & level_mask) ? 1 : 0;
6907 		} else {
6908 			level_mask = (tsintr_tt == 1) ? 0x80 : 0x40;
6909 			level = (rd32(E1000_CTRL_EXT) & level_mask) ? 1 : 0;
6910 		}
6911 
6912 		div_u64_rem(now, ns, &rem);
6913 		systim = systim + (ns - rem);
6914 
6915 		/* synchronize pin level with rising/falling edges */
6916 		div_u64_rem(now, ns << 1, &rem);
6917 		if (rem < ns) {
6918 			/* first half of period */
6919 			if (level == 0) {
6920 				/* output is already low, skip this period */
6921 				systim += ns;
6922 				pr_notice("igb: periodic output on %s missed falling edge\n",
6923 					  adapter->sdp_config[pin].name);
6924 			}
6925 		} else {
6926 			/* second half of period */
6927 			if (level == 1) {
6928 				/* output is already high, skip this period */
6929 				systim += ns;
6930 				pr_notice("igb: periodic output on %s missed rising edge\n",
6931 					  adapter->sdp_config[pin].name);
6932 			}
6933 		}
6934 
6935 		/* for this chip family tv_sec is the upper part of the binary value,
6936 		 * so not seconds
6937 		 */
6938 		ts.tv_nsec = (u32)systim;
6939 		ts.tv_sec  = ((u32)(systim >> 32)) & 0xFF;
6940 	} else {
6941 		ts = timespec64_add(adapter->perout[tsintr_tt].start,
6942 				    adapter->perout[tsintr_tt].period);
6943 	}
6944 
6945 	/* u32 conversion of tv_sec is safe until y2106 */
6946 	wr32((tsintr_tt == 1) ? E1000_TRGTTIML1 : E1000_TRGTTIML0, ts.tv_nsec);
6947 	wr32((tsintr_tt == 1) ? E1000_TRGTTIMH1 : E1000_TRGTTIMH0, (u32)ts.tv_sec);
6948 	tsauxc = rd32(E1000_TSAUXC);
6949 	tsauxc |= TSAUXC_EN_TT0;
6950 	wr32(E1000_TSAUXC, tsauxc);
6951 	adapter->perout[tsintr_tt].start = ts;
6952 
6953 	spin_unlock(&adapter->tmreg_lock);
6954 }
6955 
igb_extts(struct igb_adapter * adapter,int tsintr_tt)6956 static void igb_extts(struct igb_adapter *adapter, int tsintr_tt)
6957 {
6958 	int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_EXTTS, tsintr_tt);
6959 	int auxstmpl = (tsintr_tt == 1) ? E1000_AUXSTMPL1 : E1000_AUXSTMPL0;
6960 	int auxstmph = (tsintr_tt == 1) ? E1000_AUXSTMPH1 : E1000_AUXSTMPH0;
6961 	struct e1000_hw *hw = &adapter->hw;
6962 	struct ptp_clock_event event;
6963 	struct timespec64 ts;
6964 	unsigned long flags;
6965 
6966 	if (pin < 0 || pin >= IGB_N_SDP)
6967 		return;
6968 
6969 	if (hw->mac.type == e1000_82580 ||
6970 	    hw->mac.type == e1000_i354 ||
6971 	    hw->mac.type == e1000_i350) {
6972 		u64 ns = rd32(auxstmpl);
6973 
6974 		ns += ((u64)(rd32(auxstmph) & 0xFF)) << 32;
6975 		spin_lock_irqsave(&adapter->tmreg_lock, flags);
6976 		ns = timecounter_cyc2time(&adapter->tc, ns);
6977 		spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
6978 		ts = ns_to_timespec64(ns);
6979 	} else {
6980 		ts.tv_nsec = rd32(auxstmpl);
6981 		ts.tv_sec  = rd32(auxstmph);
6982 	}
6983 
6984 	event.type = PTP_CLOCK_EXTTS;
6985 	event.index = tsintr_tt;
6986 	event.timestamp = ts.tv_sec * 1000000000ULL + ts.tv_nsec;
6987 	ptp_clock_event(adapter->ptp_clock, &event);
6988 }
6989 
igb_tsync_interrupt(struct igb_adapter * adapter)6990 static void igb_tsync_interrupt(struct igb_adapter *adapter)
6991 {
6992 	const u32 mask = (TSINTR_SYS_WRAP | E1000_TSICR_TXTS |
6993 			  TSINTR_TT0 | TSINTR_TT1 |
6994 			  TSINTR_AUTT0 | TSINTR_AUTT1);
6995 	struct e1000_hw *hw = &adapter->hw;
6996 	u32 tsicr = rd32(E1000_TSICR);
6997 	struct ptp_clock_event event;
6998 
6999 	if (hw->mac.type == e1000_82580) {
7000 		/* 82580 has a hardware bug that requires an explicit
7001 		 * write to clear the TimeSync interrupt cause.
7002 		 */
7003 		wr32(E1000_TSICR, tsicr & mask);
7004 	}
7005 
7006 	if (tsicr & TSINTR_SYS_WRAP) {
7007 		event.type = PTP_CLOCK_PPS;
7008 		if (adapter->ptp_caps.pps)
7009 			ptp_clock_event(adapter->ptp_clock, &event);
7010 	}
7011 
7012 	if (tsicr & E1000_TSICR_TXTS) {
7013 		/* retrieve hardware timestamp */
7014 		schedule_work(&adapter->ptp_tx_work);
7015 	}
7016 
7017 	if (tsicr & TSINTR_TT0)
7018 		igb_perout(adapter, 0);
7019 
7020 	if (tsicr & TSINTR_TT1)
7021 		igb_perout(adapter, 1);
7022 
7023 	if (tsicr & TSINTR_AUTT0)
7024 		igb_extts(adapter, 0);
7025 
7026 	if (tsicr & TSINTR_AUTT1)
7027 		igb_extts(adapter, 1);
7028 }
7029 
igb_msix_other(int irq,void * data)7030 static irqreturn_t igb_msix_other(int irq, void *data)
7031 {
7032 	struct igb_adapter *adapter = data;
7033 	struct e1000_hw *hw = &adapter->hw;
7034 	u32 icr = rd32(E1000_ICR);
7035 	/* reading ICR causes bit 31 of EICR to be cleared */
7036 
7037 	if (icr & E1000_ICR_DRSTA)
7038 		schedule_work(&adapter->reset_task);
7039 
7040 	if (icr & E1000_ICR_DOUTSYNC) {
7041 		/* HW is reporting DMA is out of sync */
7042 		adapter->stats.doosync++;
7043 		/* The DMA Out of Sync is also indication of a spoof event
7044 		 * in IOV mode. Check the Wrong VM Behavior register to
7045 		 * see if it is really a spoof event.
7046 		 */
7047 		igb_check_wvbr(adapter);
7048 	}
7049 
7050 	/* Check for a mailbox event */
7051 	if (icr & E1000_ICR_VMMB)
7052 		igb_msg_task(adapter);
7053 
7054 	if (icr & E1000_ICR_LSC) {
7055 		hw->mac.get_link_status = 1;
7056 		/* guard against interrupt when we're going down */
7057 		if (!test_bit(__IGB_DOWN, &adapter->state))
7058 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
7059 	}
7060 
7061 	if (icr & E1000_ICR_TS)
7062 		igb_tsync_interrupt(adapter);
7063 
7064 	wr32(E1000_EIMS, adapter->eims_other);
7065 
7066 	return IRQ_HANDLED;
7067 }
7068 
igb_write_itr(struct igb_q_vector * q_vector)7069 static void igb_write_itr(struct igb_q_vector *q_vector)
7070 {
7071 	struct igb_adapter *adapter = q_vector->adapter;
7072 	u32 itr_val = q_vector->itr_val & 0x7FFC;
7073 
7074 	if (!q_vector->set_itr)
7075 		return;
7076 
7077 	if (!itr_val)
7078 		itr_val = 0x4;
7079 
7080 	if (adapter->hw.mac.type == e1000_82575)
7081 		itr_val |= itr_val << 16;
7082 	else
7083 		itr_val |= E1000_EITR_CNT_IGNR;
7084 
7085 	writel(itr_val, q_vector->itr_register);
7086 	q_vector->set_itr = 0;
7087 }
7088 
igb_msix_ring(int irq,void * data)7089 static irqreturn_t igb_msix_ring(int irq, void *data)
7090 {
7091 	struct igb_q_vector *q_vector = data;
7092 
7093 	/* Write the ITR value calculated from the previous interrupt. */
7094 	igb_write_itr(q_vector);
7095 
7096 	napi_schedule(&q_vector->napi);
7097 
7098 	return IRQ_HANDLED;
7099 }
7100 
7101 #ifdef CONFIG_IGB_DCA
igb_update_tx_dca(struct igb_adapter * adapter,struct igb_ring * tx_ring,int cpu)7102 static void igb_update_tx_dca(struct igb_adapter *adapter,
7103 			      struct igb_ring *tx_ring,
7104 			      int cpu)
7105 {
7106 	struct e1000_hw *hw = &adapter->hw;
7107 	u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
7108 
7109 	if (hw->mac.type != e1000_82575)
7110 		txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;
7111 
7112 	/* We can enable relaxed ordering for reads, but not writes when
7113 	 * DCA is enabled.  This is due to a known issue in some chipsets
7114 	 * which will cause the DCA tag to be cleared.
7115 	 */
7116 	txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
7117 		  E1000_DCA_TXCTRL_DATA_RRO_EN |
7118 		  E1000_DCA_TXCTRL_DESC_DCA_EN;
7119 
7120 	wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
7121 }
7122 
igb_update_rx_dca(struct igb_adapter * adapter,struct igb_ring * rx_ring,int cpu)7123 static void igb_update_rx_dca(struct igb_adapter *adapter,
7124 			      struct igb_ring *rx_ring,
7125 			      int cpu)
7126 {
7127 	struct e1000_hw *hw = &adapter->hw;
7128 	u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);
7129 
7130 	if (hw->mac.type != e1000_82575)
7131 		rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;
7132 
7133 	/* We can enable relaxed ordering for reads, but not writes when
7134 	 * DCA is enabled.  This is due to a known issue in some chipsets
7135 	 * which will cause the DCA tag to be cleared.
7136 	 */
7137 	rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
7138 		  E1000_DCA_RXCTRL_DESC_DCA_EN;
7139 
7140 	wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
7141 }
7142 
igb_update_dca(struct igb_q_vector * q_vector)7143 static void igb_update_dca(struct igb_q_vector *q_vector)
7144 {
7145 	struct igb_adapter *adapter = q_vector->adapter;
7146 	int cpu = get_cpu();
7147 
7148 	if (q_vector->cpu == cpu)
7149 		goto out_no_update;
7150 
7151 	if (q_vector->tx.ring)
7152 		igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);
7153 
7154 	if (q_vector->rx.ring)
7155 		igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);
7156 
7157 	q_vector->cpu = cpu;
7158 out_no_update:
7159 	put_cpu();
7160 }
7161 
igb_setup_dca(struct igb_adapter * adapter)7162 static void igb_setup_dca(struct igb_adapter *adapter)
7163 {
7164 	struct e1000_hw *hw = &adapter->hw;
7165 	int i;
7166 
7167 	if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
7168 		return;
7169 
7170 	/* Always use CB2 mode, difference is masked in the CB driver. */
7171 	wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
7172 
7173 	for (i = 0; i < adapter->num_q_vectors; i++) {
7174 		adapter->q_vector[i]->cpu = -1;
7175 		igb_update_dca(adapter->q_vector[i]);
7176 	}
7177 }
7178 
__igb_notify_dca(struct device * dev,void * data)7179 static int __igb_notify_dca(struct device *dev, void *data)
7180 {
7181 	struct net_device *netdev = dev_get_drvdata(dev);
7182 	struct igb_adapter *adapter = netdev_priv(netdev);
7183 	struct pci_dev *pdev = adapter->pdev;
7184 	struct e1000_hw *hw = &adapter->hw;
7185 	unsigned long event = *(unsigned long *)data;
7186 
7187 	switch (event) {
7188 	case DCA_PROVIDER_ADD:
7189 		/* if already enabled, don't do it again */
7190 		if (adapter->flags & IGB_FLAG_DCA_ENABLED)
7191 			break;
7192 		if (dca_add_requester(dev) == 0) {
7193 			adapter->flags |= IGB_FLAG_DCA_ENABLED;
7194 			dev_info(&pdev->dev, "DCA enabled\n");
7195 			igb_setup_dca(adapter);
7196 			break;
7197 		}
7198 		fallthrough; /* since DCA is disabled. */
7199 	case DCA_PROVIDER_REMOVE:
7200 		if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
7201 			/* without this a class_device is left
7202 			 * hanging around in the sysfs model
7203 			 */
7204 			dca_remove_requester(dev);
7205 			dev_info(&pdev->dev, "DCA disabled\n");
7206 			adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
7207 			wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
7208 		}
7209 		break;
7210 	}
7211 
7212 	return 0;
7213 }
7214 
igb_notify_dca(struct notifier_block * nb,unsigned long event,void * p)7215 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
7216 			  void *p)
7217 {
7218 	int ret_val;
7219 
7220 	ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
7221 					 __igb_notify_dca);
7222 
7223 	return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
7224 }
7225 #endif /* CONFIG_IGB_DCA */
7226 
7227 #ifdef CONFIG_PCI_IOV
igb_vf_configure(struct igb_adapter * adapter,int vf)7228 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
7229 {
7230 	unsigned char mac_addr[ETH_ALEN];
7231 
7232 	eth_zero_addr(mac_addr);
7233 	igb_set_vf_mac(adapter, vf, mac_addr);
7234 
7235 	/* By default spoof check is enabled for all VFs */
7236 	adapter->vf_data[vf].spoofchk_enabled = true;
7237 
7238 	/* By default VFs are not trusted */
7239 	adapter->vf_data[vf].trusted = false;
7240 
7241 	return 0;
7242 }
7243 
7244 #endif
igb_ping_all_vfs(struct igb_adapter * adapter)7245 static void igb_ping_all_vfs(struct igb_adapter *adapter)
7246 {
7247 	struct e1000_hw *hw = &adapter->hw;
7248 	u32 ping;
7249 	int i;
7250 
7251 	for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
7252 		ping = E1000_PF_CONTROL_MSG;
7253 		if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
7254 			ping |= E1000_VT_MSGTYPE_CTS;
7255 		igb_write_mbx(hw, &ping, 1, i);
7256 	}
7257 }
7258 
igb_set_vf_promisc(struct igb_adapter * adapter,u32 * msgbuf,u32 vf)7259 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7260 {
7261 	struct e1000_hw *hw = &adapter->hw;
7262 	u32 vmolr = rd32(E1000_VMOLR(vf));
7263 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7264 
7265 	vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7266 			    IGB_VF_FLAG_MULTI_PROMISC);
7267 	vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
7268 
7269 	if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
7270 		vmolr |= E1000_VMOLR_MPME;
7271 		vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7272 		*msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
7273 	} else {
7274 		/* if we have hashes and we are clearing a multicast promisc
7275 		 * flag we need to write the hashes to the MTA as this step
7276 		 * was previously skipped
7277 		 */
7278 		if (vf_data->num_vf_mc_hashes > 30) {
7279 			vmolr |= E1000_VMOLR_MPME;
7280 		} else if (vf_data->num_vf_mc_hashes) {
7281 			int j;
7282 
7283 			vmolr |= E1000_VMOLR_ROMPE;
7284 			for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
7285 				igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
7286 		}
7287 	}
7288 
7289 	wr32(E1000_VMOLR(vf), vmolr);
7290 
7291 	/* there are flags left unprocessed, likely not supported */
7292 	if (*msgbuf & E1000_VT_MSGINFO_MASK)
7293 		return -EINVAL;
7294 
7295 	return 0;
7296 }
7297 
igb_set_vf_multicasts(struct igb_adapter * adapter,u32 * msgbuf,u32 vf)7298 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
7299 				  u32 *msgbuf, u32 vf)
7300 {
7301 	int n = FIELD_GET(E1000_VT_MSGINFO_MASK, msgbuf[0]);
7302 	u16 *hash_list = (u16 *)&msgbuf[1];
7303 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7304 	int i;
7305 
7306 	/* salt away the number of multicast addresses assigned
7307 	 * to this VF for later use to restore when the PF multi cast
7308 	 * list changes
7309 	 */
7310 	vf_data->num_vf_mc_hashes = n;
7311 
7312 	/* only up to 30 hash values supported */
7313 	if (n > 30)
7314 		n = 30;
7315 
7316 	/* store the hashes for later use */
7317 	for (i = 0; i < n; i++)
7318 		vf_data->vf_mc_hashes[i] = hash_list[i];
7319 
7320 	/* Flush and reset the mta with the new values */
7321 	igb_set_rx_mode(adapter->netdev);
7322 
7323 	return 0;
7324 }
7325 
igb_restore_vf_multicasts(struct igb_adapter * adapter)7326 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
7327 {
7328 	struct e1000_hw *hw = &adapter->hw;
7329 	struct vf_data_storage *vf_data;
7330 	int i, j;
7331 
7332 	for (i = 0; i < adapter->vfs_allocated_count; i++) {
7333 		u32 vmolr = rd32(E1000_VMOLR(i));
7334 
7335 		vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
7336 
7337 		vf_data = &adapter->vf_data[i];
7338 
7339 		if ((vf_data->num_vf_mc_hashes > 30) ||
7340 		    (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
7341 			vmolr |= E1000_VMOLR_MPME;
7342 		} else if (vf_data->num_vf_mc_hashes) {
7343 			vmolr |= E1000_VMOLR_ROMPE;
7344 			for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
7345 				igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
7346 		}
7347 		wr32(E1000_VMOLR(i), vmolr);
7348 	}
7349 }
7350 
igb_clear_vf_vfta(struct igb_adapter * adapter,u32 vf)7351 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
7352 {
7353 	struct e1000_hw *hw = &adapter->hw;
7354 	u32 pool_mask, vlvf_mask, i;
7355 
7356 	/* create mask for VF and other pools */
7357 	pool_mask = E1000_VLVF_POOLSEL_MASK;
7358 	vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf);
7359 
7360 	/* drop PF from pool bits */
7361 	pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT +
7362 			     adapter->vfs_allocated_count);
7363 
7364 	/* Find the vlan filter for this id */
7365 	for (i = E1000_VLVF_ARRAY_SIZE; i--;) {
7366 		u32 vlvf = rd32(E1000_VLVF(i));
7367 		u32 vfta_mask, vid, vfta;
7368 
7369 		/* remove the vf from the pool */
7370 		if (!(vlvf & vlvf_mask))
7371 			continue;
7372 
7373 		/* clear out bit from VLVF */
7374 		vlvf ^= vlvf_mask;
7375 
7376 		/* if other pools are present, just remove ourselves */
7377 		if (vlvf & pool_mask)
7378 			goto update_vlvfb;
7379 
7380 		/* if PF is present, leave VFTA */
7381 		if (vlvf & E1000_VLVF_POOLSEL_MASK)
7382 			goto update_vlvf;
7383 
7384 		vid = vlvf & E1000_VLVF_VLANID_MASK;
7385 		vfta_mask = BIT(vid % 32);
7386 
7387 		/* clear bit from VFTA */
7388 		vfta = adapter->shadow_vfta[vid / 32];
7389 		if (vfta & vfta_mask)
7390 			hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask);
7391 update_vlvf:
7392 		/* clear pool selection enable */
7393 		if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
7394 			vlvf &= E1000_VLVF_POOLSEL_MASK;
7395 		else
7396 			vlvf = 0;
7397 update_vlvfb:
7398 		/* clear pool bits */
7399 		wr32(E1000_VLVF(i), vlvf);
7400 	}
7401 }
7402 
igb_find_vlvf_entry(struct e1000_hw * hw,u32 vlan)7403 static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan)
7404 {
7405 	u32 vlvf;
7406 	int idx;
7407 
7408 	/* short cut the special case */
7409 	if (vlan == 0)
7410 		return 0;
7411 
7412 	/* Search for the VLAN id in the VLVF entries */
7413 	for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) {
7414 		vlvf = rd32(E1000_VLVF(idx));
7415 		if ((vlvf & VLAN_VID_MASK) == vlan)
7416 			break;
7417 	}
7418 
7419 	return idx;
7420 }
7421 
igb_update_pf_vlvf(struct igb_adapter * adapter,u32 vid)7422 static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid)
7423 {
7424 	struct e1000_hw *hw = &adapter->hw;
7425 	u32 bits, pf_id;
7426 	int idx;
7427 
7428 	idx = igb_find_vlvf_entry(hw, vid);
7429 	if (!idx)
7430 		return;
7431 
7432 	/* See if any other pools are set for this VLAN filter
7433 	 * entry other than the PF.
7434 	 */
7435 	pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
7436 	bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK;
7437 	bits &= rd32(E1000_VLVF(idx));
7438 
7439 	/* Disable the filter so this falls into the default pool. */
7440 	if (!bits) {
7441 		if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
7442 			wr32(E1000_VLVF(idx), BIT(pf_id));
7443 		else
7444 			wr32(E1000_VLVF(idx), 0);
7445 	}
7446 }
7447 
igb_set_vf_vlan(struct igb_adapter * adapter,u32 vid,bool add,u32 vf)7448 static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid,
7449 			   bool add, u32 vf)
7450 {
7451 	int pf_id = adapter->vfs_allocated_count;
7452 	struct e1000_hw *hw = &adapter->hw;
7453 	int err;
7454 
7455 	/* If VLAN overlaps with one the PF is currently monitoring make
7456 	 * sure that we are able to allocate a VLVF entry.  This may be
7457 	 * redundant but it guarantees PF will maintain visibility to
7458 	 * the VLAN.
7459 	 */
7460 	if (add && test_bit(vid, adapter->active_vlans)) {
7461 		err = igb_vfta_set(hw, vid, pf_id, true, false);
7462 		if (err)
7463 			return err;
7464 	}
7465 
7466 	err = igb_vfta_set(hw, vid, vf, add, false);
7467 
7468 	if (add && !err)
7469 		return err;
7470 
7471 	/* If we failed to add the VF VLAN or we are removing the VF VLAN
7472 	 * we may need to drop the PF pool bit in order to allow us to free
7473 	 * up the VLVF resources.
7474 	 */
7475 	if (test_bit(vid, adapter->active_vlans) ||
7476 	    (adapter->flags & IGB_FLAG_VLAN_PROMISC))
7477 		igb_update_pf_vlvf(adapter, vid);
7478 
7479 	return err;
7480 }
7481 
igb_set_vmvir(struct igb_adapter * adapter,u32 vid,u32 vf)7482 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
7483 {
7484 	struct e1000_hw *hw = &adapter->hw;
7485 
7486 	if (vid)
7487 		wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
7488 	else
7489 		wr32(E1000_VMVIR(vf), 0);
7490 }
7491 
igb_enable_port_vlan(struct igb_adapter * adapter,int vf,u16 vlan,u8 qos)7492 static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf,
7493 				u16 vlan, u8 qos)
7494 {
7495 	int err;
7496 
7497 	err = igb_set_vf_vlan(adapter, vlan, true, vf);
7498 	if (err)
7499 		return err;
7500 
7501 	igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
7502 	igb_set_vmolr(adapter, vf, !vlan);
7503 
7504 	/* revoke access to previous VLAN */
7505 	if (vlan != adapter->vf_data[vf].pf_vlan)
7506 		igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7507 				false, vf);
7508 
7509 	adapter->vf_data[vf].pf_vlan = vlan;
7510 	adapter->vf_data[vf].pf_qos = qos;
7511 	igb_set_vf_vlan_strip(adapter, vf, true);
7512 	dev_info(&adapter->pdev->dev,
7513 		 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
7514 	if (test_bit(__IGB_DOWN, &adapter->state)) {
7515 		dev_warn(&adapter->pdev->dev,
7516 			 "The VF VLAN has been set, but the PF device is not up.\n");
7517 		dev_warn(&adapter->pdev->dev,
7518 			 "Bring the PF device up before attempting to use the VF device.\n");
7519 	}
7520 
7521 	return err;
7522 }
7523 
igb_disable_port_vlan(struct igb_adapter * adapter,int vf)7524 static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf)
7525 {
7526 	/* Restore tagless access via VLAN 0 */
7527 	igb_set_vf_vlan(adapter, 0, true, vf);
7528 
7529 	igb_set_vmvir(adapter, 0, vf);
7530 	igb_set_vmolr(adapter, vf, true);
7531 
7532 	/* Remove any PF assigned VLAN */
7533 	if (adapter->vf_data[vf].pf_vlan)
7534 		igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7535 				false, vf);
7536 
7537 	adapter->vf_data[vf].pf_vlan = 0;
7538 	adapter->vf_data[vf].pf_qos = 0;
7539 	igb_set_vf_vlan_strip(adapter, vf, false);
7540 
7541 	return 0;
7542 }
7543 
igb_ndo_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 vlan_proto)7544 static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf,
7545 			       u16 vlan, u8 qos, __be16 vlan_proto)
7546 {
7547 	struct igb_adapter *adapter = netdev_priv(netdev);
7548 
7549 	if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
7550 		return -EINVAL;
7551 
7552 	if (vlan_proto != htons(ETH_P_8021Q))
7553 		return -EPROTONOSUPPORT;
7554 
7555 	return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) :
7556 			       igb_disable_port_vlan(adapter, vf);
7557 }
7558 
igb_set_vf_vlan_msg(struct igb_adapter * adapter,u32 * msgbuf,u32 vf)7559 static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7560 {
7561 	int add = FIELD_GET(E1000_VT_MSGINFO_MASK, msgbuf[0]);
7562 	int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
7563 	int ret;
7564 
7565 	if (adapter->vf_data[vf].pf_vlan)
7566 		return -1;
7567 
7568 	/* VLAN 0 is a special case, don't allow it to be removed */
7569 	if (!vid && !add)
7570 		return 0;
7571 
7572 	ret = igb_set_vf_vlan(adapter, vid, !!add, vf);
7573 	if (!ret)
7574 		igb_set_vf_vlan_strip(adapter, vf, !!vid);
7575 	return ret;
7576 }
7577 
igb_vf_reset(struct igb_adapter * adapter,u32 vf)7578 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
7579 {
7580 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7581 
7582 	/* clear flags - except flag that indicates PF has set the MAC */
7583 	vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC;
7584 	vf_data->last_nack = jiffies;
7585 
7586 	/* reset vlans for device */
7587 	igb_clear_vf_vfta(adapter, vf);
7588 	igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf);
7589 	igb_set_vmvir(adapter, vf_data->pf_vlan |
7590 			       (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf);
7591 	igb_set_vmolr(adapter, vf, !vf_data->pf_vlan);
7592 	igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan));
7593 
7594 	/* reset multicast table array for vf */
7595 	adapter->vf_data[vf].num_vf_mc_hashes = 0;
7596 
7597 	/* Flush and reset the mta with the new values */
7598 	igb_set_rx_mode(adapter->netdev);
7599 }
7600 
igb_vf_reset_event(struct igb_adapter * adapter,u32 vf)7601 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
7602 {
7603 	unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7604 
7605 	/* clear mac address as we were hotplug removed/added */
7606 	if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
7607 		eth_zero_addr(vf_mac);
7608 
7609 	/* process remaining reset events */
7610 	igb_vf_reset(adapter, vf);
7611 }
7612 
igb_vf_reset_msg(struct igb_adapter * adapter,u32 vf)7613 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
7614 {
7615 	struct e1000_hw *hw = &adapter->hw;
7616 	unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7617 	u32 reg, msgbuf[3] = {};
7618 	u8 *addr = (u8 *)(&msgbuf[1]);
7619 
7620 	/* process all the same items cleared in a function level reset */
7621 	igb_vf_reset(adapter, vf);
7622 
7623 	/* set vf mac address */
7624 	igb_set_vf_mac(adapter, vf, vf_mac);
7625 
7626 	/* enable transmit and receive for vf */
7627 	reg = rd32(E1000_VFTE);
7628 	wr32(E1000_VFTE, reg | BIT(vf));
7629 	reg = rd32(E1000_VFRE);
7630 	wr32(E1000_VFRE, reg | BIT(vf));
7631 
7632 	adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
7633 
7634 	/* reply to reset with ack and vf mac address */
7635 	if (!is_zero_ether_addr(vf_mac)) {
7636 		msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
7637 		memcpy(addr, vf_mac, ETH_ALEN);
7638 	} else {
7639 		msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
7640 	}
7641 	igb_write_mbx(hw, msgbuf, 3, vf);
7642 }
7643 
igb_flush_mac_table(struct igb_adapter * adapter)7644 static void igb_flush_mac_table(struct igb_adapter *adapter)
7645 {
7646 	struct e1000_hw *hw = &adapter->hw;
7647 	int i;
7648 
7649 	for (i = 0; i < hw->mac.rar_entry_count; i++) {
7650 		adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
7651 		eth_zero_addr(adapter->mac_table[i].addr);
7652 		adapter->mac_table[i].queue = 0;
7653 		igb_rar_set_index(adapter, i);
7654 	}
7655 }
7656 
igb_available_rars(struct igb_adapter * adapter,u8 queue)7657 static int igb_available_rars(struct igb_adapter *adapter, u8 queue)
7658 {
7659 	struct e1000_hw *hw = &adapter->hw;
7660 	/* do not count rar entries reserved for VFs MAC addresses */
7661 	int rar_entries = hw->mac.rar_entry_count -
7662 			  adapter->vfs_allocated_count;
7663 	int i, count = 0;
7664 
7665 	for (i = 0; i < rar_entries; i++) {
7666 		/* do not count default entries */
7667 		if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT)
7668 			continue;
7669 
7670 		/* do not count "in use" entries for different queues */
7671 		if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) &&
7672 		    (adapter->mac_table[i].queue != queue))
7673 			continue;
7674 
7675 		count++;
7676 	}
7677 
7678 	return count;
7679 }
7680 
7681 /* Set default MAC address for the PF in the first RAR entry */
igb_set_default_mac_filter(struct igb_adapter * adapter)7682 static void igb_set_default_mac_filter(struct igb_adapter *adapter)
7683 {
7684 	struct igb_mac_addr *mac_table = &adapter->mac_table[0];
7685 
7686 	ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
7687 	mac_table->queue = adapter->vfs_allocated_count;
7688 	mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7689 
7690 	igb_rar_set_index(adapter, 0);
7691 }
7692 
7693 /* If the filter to be added and an already existing filter express
7694  * the same address and address type, it should be possible to only
7695  * override the other configurations, for example the queue to steer
7696  * traffic.
7697  */
igb_mac_entry_can_be_used(const struct igb_mac_addr * entry,const u8 * addr,const u8 flags)7698 static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry,
7699 				      const u8 *addr, const u8 flags)
7700 {
7701 	if (!(entry->state & IGB_MAC_STATE_IN_USE))
7702 		return true;
7703 
7704 	if ((entry->state & IGB_MAC_STATE_SRC_ADDR) !=
7705 	    (flags & IGB_MAC_STATE_SRC_ADDR))
7706 		return false;
7707 
7708 	if (!ether_addr_equal(addr, entry->addr))
7709 		return false;
7710 
7711 	return true;
7712 }
7713 
7714 /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
7715  * 'flags' is used to indicate what kind of match is made, match is by
7716  * default for the destination address, if matching by source address
7717  * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used.
7718  */
igb_add_mac_filter_flags(struct igb_adapter * adapter,const u8 * addr,const u8 queue,const u8 flags)7719 static int igb_add_mac_filter_flags(struct igb_adapter *adapter,
7720 				    const u8 *addr, const u8 queue,
7721 				    const u8 flags)
7722 {
7723 	struct e1000_hw *hw = &adapter->hw;
7724 	int rar_entries = hw->mac.rar_entry_count -
7725 			  adapter->vfs_allocated_count;
7726 	int i;
7727 
7728 	if (is_zero_ether_addr(addr))
7729 		return -EINVAL;
7730 
7731 	/* Search for the first empty entry in the MAC table.
7732 	 * Do not touch entries at the end of the table reserved for the VF MAC
7733 	 * addresses.
7734 	 */
7735 	for (i = 0; i < rar_entries; i++) {
7736 		if (!igb_mac_entry_can_be_used(&adapter->mac_table[i],
7737 					       addr, flags))
7738 			continue;
7739 
7740 		ether_addr_copy(adapter->mac_table[i].addr, addr);
7741 		adapter->mac_table[i].queue = queue;
7742 		adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags;
7743 
7744 		igb_rar_set_index(adapter, i);
7745 		return i;
7746 	}
7747 
7748 	return -ENOSPC;
7749 }
7750 
igb_add_mac_filter(struct igb_adapter * adapter,const u8 * addr,const u8 queue)7751 static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7752 			      const u8 queue)
7753 {
7754 	return igb_add_mac_filter_flags(adapter, addr, queue, 0);
7755 }
7756 
7757 /* Remove a MAC filter for 'addr' directing matching traffic to
7758  * 'queue', 'flags' is used to indicate what kind of match need to be
7759  * removed, match is by default for the destination address, if
7760  * matching by source address is to be removed the flag
7761  * IGB_MAC_STATE_SRC_ADDR can be used.
7762  */
igb_del_mac_filter_flags(struct igb_adapter * adapter,const u8 * addr,const u8 queue,const u8 flags)7763 static int igb_del_mac_filter_flags(struct igb_adapter *adapter,
7764 				    const u8 *addr, const u8 queue,
7765 				    const u8 flags)
7766 {
7767 	struct e1000_hw *hw = &adapter->hw;
7768 	int rar_entries = hw->mac.rar_entry_count -
7769 			  adapter->vfs_allocated_count;
7770 	int i;
7771 
7772 	if (is_zero_ether_addr(addr))
7773 		return -EINVAL;
7774 
7775 	/* Search for matching entry in the MAC table based on given address
7776 	 * and queue. Do not touch entries at the end of the table reserved
7777 	 * for the VF MAC addresses.
7778 	 */
7779 	for (i = 0; i < rar_entries; i++) {
7780 		if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE))
7781 			continue;
7782 		if ((adapter->mac_table[i].state & flags) != flags)
7783 			continue;
7784 		if (adapter->mac_table[i].queue != queue)
7785 			continue;
7786 		if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
7787 			continue;
7788 
7789 		/* When a filter for the default address is "deleted",
7790 		 * we return it to its initial configuration
7791 		 */
7792 		if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) {
7793 			adapter->mac_table[i].state =
7794 				IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7795 			adapter->mac_table[i].queue =
7796 				adapter->vfs_allocated_count;
7797 		} else {
7798 			adapter->mac_table[i].state = 0;
7799 			adapter->mac_table[i].queue = 0;
7800 			eth_zero_addr(adapter->mac_table[i].addr);
7801 		}
7802 
7803 		igb_rar_set_index(adapter, i);
7804 		return 0;
7805 	}
7806 
7807 	return -ENOENT;
7808 }
7809 
igb_del_mac_filter(struct igb_adapter * adapter,const u8 * addr,const u8 queue)7810 static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7811 			      const u8 queue)
7812 {
7813 	return igb_del_mac_filter_flags(adapter, addr, queue, 0);
7814 }
7815 
igb_add_mac_steering_filter(struct igb_adapter * adapter,const u8 * addr,u8 queue,u8 flags)7816 int igb_add_mac_steering_filter(struct igb_adapter *adapter,
7817 				const u8 *addr, u8 queue, u8 flags)
7818 {
7819 	struct e1000_hw *hw = &adapter->hw;
7820 
7821 	/* In theory, this should be supported on 82575 as well, but
7822 	 * that part wasn't easily accessible during development.
7823 	 */
7824 	if (hw->mac.type != e1000_i210)
7825 		return -EOPNOTSUPP;
7826 
7827 	return igb_add_mac_filter_flags(adapter, addr, queue,
7828 					IGB_MAC_STATE_QUEUE_STEERING | flags);
7829 }
7830 
igb_del_mac_steering_filter(struct igb_adapter * adapter,const u8 * addr,u8 queue,u8 flags)7831 int igb_del_mac_steering_filter(struct igb_adapter *adapter,
7832 				const u8 *addr, u8 queue, u8 flags)
7833 {
7834 	return igb_del_mac_filter_flags(adapter, addr, queue,
7835 					IGB_MAC_STATE_QUEUE_STEERING | flags);
7836 }
7837 
igb_uc_sync(struct net_device * netdev,const unsigned char * addr)7838 static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr)
7839 {
7840 	struct igb_adapter *adapter = netdev_priv(netdev);
7841 	int ret;
7842 
7843 	ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7844 
7845 	return min_t(int, ret, 0);
7846 }
7847 
igb_uc_unsync(struct net_device * netdev,const unsigned char * addr)7848 static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr)
7849 {
7850 	struct igb_adapter *adapter = netdev_priv(netdev);
7851 
7852 	igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7853 
7854 	return 0;
7855 }
7856 
igb_set_vf_mac_filter(struct igb_adapter * adapter,const int vf,const u32 info,const u8 * addr)7857 static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
7858 				 const u32 info, const u8 *addr)
7859 {
7860 	struct pci_dev *pdev = adapter->pdev;
7861 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7862 	struct list_head *pos;
7863 	struct vf_mac_filter *entry = NULL;
7864 	int ret = 0;
7865 
7866 	if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7867 	    !vf_data->trusted) {
7868 		dev_warn(&pdev->dev,
7869 			 "VF %d requested MAC filter but is administratively denied\n",
7870 			  vf);
7871 		return -EINVAL;
7872 	}
7873 	if (!is_valid_ether_addr(addr)) {
7874 		dev_warn(&pdev->dev,
7875 			 "VF %d attempted to set invalid MAC filter\n",
7876 			  vf);
7877 		return -EINVAL;
7878 	}
7879 
7880 	switch (info) {
7881 	case E1000_VF_MAC_FILTER_CLR:
7882 		/* remove all unicast MAC filters related to the current VF */
7883 		list_for_each(pos, &adapter->vf_macs.l) {
7884 			entry = list_entry(pos, struct vf_mac_filter, l);
7885 			if (entry->vf == vf) {
7886 				entry->vf = -1;
7887 				entry->free = true;
7888 				igb_del_mac_filter(adapter, entry->vf_mac, vf);
7889 			}
7890 		}
7891 		break;
7892 	case E1000_VF_MAC_FILTER_ADD:
7893 		/* try to find empty slot in the list */
7894 		list_for_each(pos, &adapter->vf_macs.l) {
7895 			entry = list_entry(pos, struct vf_mac_filter, l);
7896 			if (entry->free)
7897 				break;
7898 		}
7899 
7900 		if (entry && entry->free) {
7901 			entry->free = false;
7902 			entry->vf = vf;
7903 			ether_addr_copy(entry->vf_mac, addr);
7904 
7905 			ret = igb_add_mac_filter(adapter, addr, vf);
7906 			ret = min_t(int, ret, 0);
7907 		} else {
7908 			ret = -ENOSPC;
7909 		}
7910 
7911 		if (ret == -ENOSPC)
7912 			dev_warn(&pdev->dev,
7913 				 "VF %d has requested MAC filter but there is no space for it\n",
7914 				 vf);
7915 		break;
7916 	default:
7917 		ret = -EINVAL;
7918 		break;
7919 	}
7920 
7921 	return ret;
7922 }
7923 
igb_set_vf_mac_addr(struct igb_adapter * adapter,u32 * msg,int vf)7924 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
7925 {
7926 	struct pci_dev *pdev = adapter->pdev;
7927 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7928 	u32 info = msg[0] & E1000_VT_MSGINFO_MASK;
7929 
7930 	/* The VF MAC Address is stored in a packed array of bytes
7931 	 * starting at the second 32 bit word of the msg array
7932 	 */
7933 	unsigned char *addr = (unsigned char *)&msg[1];
7934 	int ret = 0;
7935 
7936 	if (!info) {
7937 		if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7938 		    !vf_data->trusted) {
7939 			dev_warn(&pdev->dev,
7940 				 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
7941 				 vf);
7942 			return -EINVAL;
7943 		}
7944 
7945 		if (!is_valid_ether_addr(addr)) {
7946 			dev_warn(&pdev->dev,
7947 				 "VF %d attempted to set invalid MAC\n",
7948 				 vf);
7949 			return -EINVAL;
7950 		}
7951 
7952 		ret = igb_set_vf_mac(adapter, vf, addr);
7953 	} else {
7954 		ret = igb_set_vf_mac_filter(adapter, vf, info, addr);
7955 	}
7956 
7957 	return ret;
7958 }
7959 
igb_rcv_ack_from_vf(struct igb_adapter * adapter,u32 vf)7960 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
7961 {
7962 	struct e1000_hw *hw = &adapter->hw;
7963 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7964 	u32 msg = E1000_VT_MSGTYPE_NACK;
7965 
7966 	/* if device isn't clear to send it shouldn't be reading either */
7967 	if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
7968 	    time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
7969 		igb_write_mbx(hw, &msg, 1, vf);
7970 		vf_data->last_nack = jiffies;
7971 	}
7972 }
7973 
igb_rcv_msg_from_vf(struct igb_adapter * adapter,u32 vf)7974 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
7975 {
7976 	struct pci_dev *pdev = adapter->pdev;
7977 	u32 msgbuf[E1000_VFMAILBOX_SIZE];
7978 	struct e1000_hw *hw = &adapter->hw;
7979 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7980 	s32 retval;
7981 
7982 	retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false);
7983 
7984 	if (retval) {
7985 		/* if receive failed revoke VF CTS stats and restart init */
7986 		dev_err(&pdev->dev, "Error receiving message from VF\n");
7987 		vf_data->flags &= ~IGB_VF_FLAG_CTS;
7988 		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
7989 			goto unlock;
7990 		goto out;
7991 	}
7992 
7993 	/* this is a message we already processed, do nothing */
7994 	if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
7995 		goto unlock;
7996 
7997 	/* until the vf completes a reset it should not be
7998 	 * allowed to start any configuration.
7999 	 */
8000 	if (msgbuf[0] == E1000_VF_RESET) {
8001 		/* unlocks mailbox */
8002 		igb_vf_reset_msg(adapter, vf);
8003 		return;
8004 	}
8005 
8006 	if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
8007 		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
8008 			goto unlock;
8009 		retval = -1;
8010 		goto out;
8011 	}
8012 
8013 	switch ((msgbuf[0] & 0xFFFF)) {
8014 	case E1000_VF_SET_MAC_ADDR:
8015 		retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
8016 		break;
8017 	case E1000_VF_SET_PROMISC:
8018 		retval = igb_set_vf_promisc(adapter, msgbuf, vf);
8019 		break;
8020 	case E1000_VF_SET_MULTICAST:
8021 		retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
8022 		break;
8023 	case E1000_VF_SET_LPE:
8024 		retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
8025 		break;
8026 	case E1000_VF_SET_VLAN:
8027 		retval = -1;
8028 		if (vf_data->pf_vlan)
8029 			dev_warn(&pdev->dev,
8030 				 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
8031 				 vf);
8032 		else
8033 			retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf);
8034 		break;
8035 	default:
8036 		dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
8037 		retval = -1;
8038 		break;
8039 	}
8040 
8041 	msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
8042 out:
8043 	/* notify the VF of the results of what it sent us */
8044 	if (retval)
8045 		msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
8046 	else
8047 		msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
8048 
8049 	/* unlocks mailbox */
8050 	igb_write_mbx(hw, msgbuf, 1, vf);
8051 	return;
8052 
8053 unlock:
8054 	igb_unlock_mbx(hw, vf);
8055 }
8056 
igb_msg_task(struct igb_adapter * adapter)8057 static void igb_msg_task(struct igb_adapter *adapter)
8058 {
8059 	struct e1000_hw *hw = &adapter->hw;
8060 	unsigned long flags;
8061 	u32 vf;
8062 
8063 	spin_lock_irqsave(&adapter->vfs_lock, flags);
8064 	for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
8065 		/* process any reset requests */
8066 		if (!igb_check_for_rst(hw, vf))
8067 			igb_vf_reset_event(adapter, vf);
8068 
8069 		/* process any messages pending */
8070 		if (!igb_check_for_msg(hw, vf))
8071 			igb_rcv_msg_from_vf(adapter, vf);
8072 
8073 		/* process any acks */
8074 		if (!igb_check_for_ack(hw, vf))
8075 			igb_rcv_ack_from_vf(adapter, vf);
8076 	}
8077 	spin_unlock_irqrestore(&adapter->vfs_lock, flags);
8078 }
8079 
8080 /**
8081  *  igb_set_uta - Set unicast filter table address
8082  *  @adapter: board private structure
8083  *  @set: boolean indicating if we are setting or clearing bits
8084  *
8085  *  The unicast table address is a register array of 32-bit registers.
8086  *  The table is meant to be used in a way similar to how the MTA is used
8087  *  however due to certain limitations in the hardware it is necessary to
8088  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
8089  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
8090  **/
igb_set_uta(struct igb_adapter * adapter,bool set)8091 static void igb_set_uta(struct igb_adapter *adapter, bool set)
8092 {
8093 	struct e1000_hw *hw = &adapter->hw;
8094 	u32 uta = set ? ~0 : 0;
8095 	int i;
8096 
8097 	/* we only need to do this if VMDq is enabled */
8098 	if (!adapter->vfs_allocated_count)
8099 		return;
8100 
8101 	for (i = hw->mac.uta_reg_count; i--;)
8102 		array_wr32(E1000_UTA, i, uta);
8103 }
8104 
8105 /**
8106  *  igb_intr_msi - Interrupt Handler
8107  *  @irq: interrupt number
8108  *  @data: pointer to a network interface device structure
8109  **/
igb_intr_msi(int irq,void * data)8110 static irqreturn_t igb_intr_msi(int irq, void *data)
8111 {
8112 	struct igb_adapter *adapter = data;
8113 	struct igb_q_vector *q_vector = adapter->q_vector[0];
8114 	struct e1000_hw *hw = &adapter->hw;
8115 	/* read ICR disables interrupts using IAM */
8116 	u32 icr = rd32(E1000_ICR);
8117 
8118 	igb_write_itr(q_vector);
8119 
8120 	if (icr & E1000_ICR_DRSTA)
8121 		schedule_work(&adapter->reset_task);
8122 
8123 	if (icr & E1000_ICR_DOUTSYNC) {
8124 		/* HW is reporting DMA is out of sync */
8125 		adapter->stats.doosync++;
8126 	}
8127 
8128 	if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
8129 		hw->mac.get_link_status = 1;
8130 		if (!test_bit(__IGB_DOWN, &adapter->state))
8131 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
8132 	}
8133 
8134 	if (icr & E1000_ICR_TS)
8135 		igb_tsync_interrupt(adapter);
8136 
8137 	napi_schedule(&q_vector->napi);
8138 
8139 	return IRQ_HANDLED;
8140 }
8141 
8142 /**
8143  *  igb_intr - Legacy Interrupt Handler
8144  *  @irq: interrupt number
8145  *  @data: pointer to a network interface device structure
8146  **/
igb_intr(int irq,void * data)8147 static irqreturn_t igb_intr(int irq, void *data)
8148 {
8149 	struct igb_adapter *adapter = data;
8150 	struct igb_q_vector *q_vector = adapter->q_vector[0];
8151 	struct e1000_hw *hw = &adapter->hw;
8152 	/* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
8153 	 * need for the IMC write
8154 	 */
8155 	u32 icr = rd32(E1000_ICR);
8156 
8157 	/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
8158 	 * not set, then the adapter didn't send an interrupt
8159 	 */
8160 	if (!(icr & E1000_ICR_INT_ASSERTED))
8161 		return IRQ_NONE;
8162 
8163 	igb_write_itr(q_vector);
8164 
8165 	if (icr & E1000_ICR_DRSTA)
8166 		schedule_work(&adapter->reset_task);
8167 
8168 	if (icr & E1000_ICR_DOUTSYNC) {
8169 		/* HW is reporting DMA is out of sync */
8170 		adapter->stats.doosync++;
8171 	}
8172 
8173 	if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
8174 		hw->mac.get_link_status = 1;
8175 		/* guard against interrupt when we're going down */
8176 		if (!test_bit(__IGB_DOWN, &adapter->state))
8177 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
8178 	}
8179 
8180 	if (icr & E1000_ICR_TS)
8181 		igb_tsync_interrupt(adapter);
8182 
8183 	napi_schedule(&q_vector->napi);
8184 
8185 	return IRQ_HANDLED;
8186 }
8187 
igb_ring_irq_enable(struct igb_q_vector * q_vector)8188 static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
8189 {
8190 	struct igb_adapter *adapter = q_vector->adapter;
8191 	struct e1000_hw *hw = &adapter->hw;
8192 
8193 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
8194 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
8195 		if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
8196 			igb_set_itr(q_vector);
8197 		else
8198 			igb_update_ring_itr(q_vector);
8199 	}
8200 
8201 	if (!test_bit(__IGB_DOWN, &adapter->state)) {
8202 		if (adapter->flags & IGB_FLAG_HAS_MSIX)
8203 			wr32(E1000_EIMS, q_vector->eims_value);
8204 		else
8205 			igb_irq_enable(adapter);
8206 	}
8207 }
8208 
8209 /**
8210  *  igb_poll - NAPI Rx polling callback
8211  *  @napi: napi polling structure
8212  *  @budget: count of how many packets we should handle
8213  **/
igb_poll(struct napi_struct * napi,int budget)8214 static int igb_poll(struct napi_struct *napi, int budget)
8215 {
8216 	struct igb_q_vector *q_vector = container_of(napi,
8217 						     struct igb_q_vector,
8218 						     napi);
8219 	bool clean_complete = true;
8220 	int work_done = 0;
8221 
8222 #ifdef CONFIG_IGB_DCA
8223 	if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
8224 		igb_update_dca(q_vector);
8225 #endif
8226 	if (q_vector->tx.ring)
8227 		clean_complete = igb_clean_tx_irq(q_vector, budget);
8228 
8229 	if (q_vector->rx.ring) {
8230 		int cleaned = igb_clean_rx_irq(q_vector, budget);
8231 
8232 		work_done += cleaned;
8233 		if (cleaned >= budget)
8234 			clean_complete = false;
8235 	}
8236 
8237 	/* If all work not completed, return budget and keep polling */
8238 	if (!clean_complete)
8239 		return budget;
8240 
8241 	/* Exit the polling mode, but don't re-enable interrupts if stack might
8242 	 * poll us due to busy-polling
8243 	 */
8244 	if (likely(napi_complete_done(napi, work_done)))
8245 		igb_ring_irq_enable(q_vector);
8246 
8247 	return work_done;
8248 }
8249 
8250 /**
8251  *  igb_clean_tx_irq - Reclaim resources after transmit completes
8252  *  @q_vector: pointer to q_vector containing needed info
8253  *  @napi_budget: Used to determine if we are in netpoll
8254  *
8255  *  returns true if ring is completely cleaned
8256  **/
igb_clean_tx_irq(struct igb_q_vector * q_vector,int napi_budget)8257 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget)
8258 {
8259 	struct igb_adapter *adapter = q_vector->adapter;
8260 	struct igb_ring *tx_ring = q_vector->tx.ring;
8261 	struct igb_tx_buffer *tx_buffer;
8262 	union e1000_adv_tx_desc *tx_desc;
8263 	unsigned int total_bytes = 0, total_packets = 0;
8264 	unsigned int budget = q_vector->tx.work_limit;
8265 	unsigned int i = tx_ring->next_to_clean;
8266 
8267 	if (test_bit(__IGB_DOWN, &adapter->state))
8268 		return true;
8269 
8270 	tx_buffer = &tx_ring->tx_buffer_info[i];
8271 	tx_desc = IGB_TX_DESC(tx_ring, i);
8272 	i -= tx_ring->count;
8273 
8274 	do {
8275 		union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
8276 
8277 		/* if next_to_watch is not set then there is no work pending */
8278 		if (!eop_desc)
8279 			break;
8280 
8281 		/* prevent any other reads prior to eop_desc */
8282 		smp_rmb();
8283 
8284 		/* if DD is not set pending work has not been completed */
8285 		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
8286 			break;
8287 
8288 		/* clear next_to_watch to prevent false hangs */
8289 		tx_buffer->next_to_watch = NULL;
8290 
8291 		/* update the statistics for this packet */
8292 		total_bytes += tx_buffer->bytecount;
8293 		total_packets += tx_buffer->gso_segs;
8294 
8295 		/* free the skb */
8296 		if (tx_buffer->type == IGB_TYPE_SKB)
8297 			napi_consume_skb(tx_buffer->skb, napi_budget);
8298 		else
8299 			xdp_return_frame(tx_buffer->xdpf);
8300 
8301 		/* unmap skb header data */
8302 		dma_unmap_single(tx_ring->dev,
8303 				 dma_unmap_addr(tx_buffer, dma),
8304 				 dma_unmap_len(tx_buffer, len),
8305 				 DMA_TO_DEVICE);
8306 
8307 		/* clear tx_buffer data */
8308 		dma_unmap_len_set(tx_buffer, len, 0);
8309 
8310 		/* clear last DMA location and unmap remaining buffers */
8311 		while (tx_desc != eop_desc) {
8312 			tx_buffer++;
8313 			tx_desc++;
8314 			i++;
8315 			if (unlikely(!i)) {
8316 				i -= tx_ring->count;
8317 				tx_buffer = tx_ring->tx_buffer_info;
8318 				tx_desc = IGB_TX_DESC(tx_ring, 0);
8319 			}
8320 
8321 			/* unmap any remaining paged data */
8322 			if (dma_unmap_len(tx_buffer, len)) {
8323 				dma_unmap_page(tx_ring->dev,
8324 					       dma_unmap_addr(tx_buffer, dma),
8325 					       dma_unmap_len(tx_buffer, len),
8326 					       DMA_TO_DEVICE);
8327 				dma_unmap_len_set(tx_buffer, len, 0);
8328 			}
8329 		}
8330 
8331 		/* move us one more past the eop_desc for start of next pkt */
8332 		tx_buffer++;
8333 		tx_desc++;
8334 		i++;
8335 		if (unlikely(!i)) {
8336 			i -= tx_ring->count;
8337 			tx_buffer = tx_ring->tx_buffer_info;
8338 			tx_desc = IGB_TX_DESC(tx_ring, 0);
8339 		}
8340 
8341 		/* issue prefetch for next Tx descriptor */
8342 		prefetch(tx_desc);
8343 
8344 		/* update budget accounting */
8345 		budget--;
8346 	} while (likely(budget));
8347 
8348 	netdev_tx_completed_queue(txring_txq(tx_ring),
8349 				  total_packets, total_bytes);
8350 	i += tx_ring->count;
8351 	tx_ring->next_to_clean = i;
8352 	u64_stats_update_begin(&tx_ring->tx_syncp);
8353 	tx_ring->tx_stats.bytes += total_bytes;
8354 	tx_ring->tx_stats.packets += total_packets;
8355 	u64_stats_update_end(&tx_ring->tx_syncp);
8356 	q_vector->tx.total_bytes += total_bytes;
8357 	q_vector->tx.total_packets += total_packets;
8358 
8359 	if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
8360 		struct e1000_hw *hw = &adapter->hw;
8361 
8362 		/* Detect a transmit hang in hardware, this serializes the
8363 		 * check with the clearing of time_stamp and movement of i
8364 		 */
8365 		clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8366 		if (tx_buffer->next_to_watch &&
8367 		    time_after(jiffies, tx_buffer->time_stamp +
8368 			       (adapter->tx_timeout_factor * HZ)) &&
8369 		    !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
8370 
8371 			/* detected Tx unit hang */
8372 			dev_err(tx_ring->dev,
8373 				"Detected Tx Unit Hang\n"
8374 				"  Tx Queue             <%d>\n"
8375 				"  TDH                  <%x>\n"
8376 				"  TDT                  <%x>\n"
8377 				"  next_to_use          <%x>\n"
8378 				"  next_to_clean        <%x>\n"
8379 				"buffer_info[next_to_clean]\n"
8380 				"  time_stamp           <%lx>\n"
8381 				"  next_to_watch        <%p>\n"
8382 				"  jiffies              <%lx>\n"
8383 				"  desc.status          <%x>\n",
8384 				tx_ring->queue_index,
8385 				rd32(E1000_TDH(tx_ring->reg_idx)),
8386 				readl(tx_ring->tail),
8387 				tx_ring->next_to_use,
8388 				tx_ring->next_to_clean,
8389 				tx_buffer->time_stamp,
8390 				tx_buffer->next_to_watch,
8391 				jiffies,
8392 				tx_buffer->next_to_watch->wb.status);
8393 			netif_stop_subqueue(tx_ring->netdev,
8394 					    tx_ring->queue_index);
8395 
8396 			/* we are about to reset, no point in enabling stuff */
8397 			return true;
8398 		}
8399 	}
8400 
8401 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
8402 	if (unlikely(total_packets &&
8403 	    netif_carrier_ok(tx_ring->netdev) &&
8404 	    igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
8405 		/* Make sure that anybody stopping the queue after this
8406 		 * sees the new next_to_clean.
8407 		 */
8408 		smp_mb();
8409 		if (__netif_subqueue_stopped(tx_ring->netdev,
8410 					     tx_ring->queue_index) &&
8411 		    !(test_bit(__IGB_DOWN, &adapter->state))) {
8412 			netif_wake_subqueue(tx_ring->netdev,
8413 					    tx_ring->queue_index);
8414 
8415 			u64_stats_update_begin(&tx_ring->tx_syncp);
8416 			tx_ring->tx_stats.restart_queue++;
8417 			u64_stats_update_end(&tx_ring->tx_syncp);
8418 		}
8419 	}
8420 
8421 	return !!budget;
8422 }
8423 
8424 /**
8425  *  igb_reuse_rx_page - page flip buffer and store it back on the ring
8426  *  @rx_ring: rx descriptor ring to store buffers on
8427  *  @old_buff: donor buffer to have page reused
8428  *
8429  *  Synchronizes page for reuse by the adapter
8430  **/
igb_reuse_rx_page(struct igb_ring * rx_ring,struct igb_rx_buffer * old_buff)8431 static void igb_reuse_rx_page(struct igb_ring *rx_ring,
8432 			      struct igb_rx_buffer *old_buff)
8433 {
8434 	struct igb_rx_buffer *new_buff;
8435 	u16 nta = rx_ring->next_to_alloc;
8436 
8437 	new_buff = &rx_ring->rx_buffer_info[nta];
8438 
8439 	/* update, and store next to alloc */
8440 	nta++;
8441 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
8442 
8443 	/* Transfer page from old buffer to new buffer.
8444 	 * Move each member individually to avoid possible store
8445 	 * forwarding stalls.
8446 	 */
8447 	new_buff->dma		= old_buff->dma;
8448 	new_buff->page		= old_buff->page;
8449 	new_buff->page_offset	= old_buff->page_offset;
8450 	new_buff->pagecnt_bias	= old_buff->pagecnt_bias;
8451 }
8452 
igb_can_reuse_rx_page(struct igb_rx_buffer * rx_buffer,int rx_buf_pgcnt)8453 static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
8454 				  int rx_buf_pgcnt)
8455 {
8456 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
8457 	struct page *page = rx_buffer->page;
8458 
8459 	/* avoid re-using remote and pfmemalloc pages */
8460 	if (!dev_page_is_reusable(page))
8461 		return false;
8462 
8463 #if (PAGE_SIZE < 8192)
8464 	/* if we are only owner of page we can reuse it */
8465 	if (unlikely((rx_buf_pgcnt - pagecnt_bias) > 1))
8466 		return false;
8467 #else
8468 #define IGB_LAST_OFFSET \
8469 	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)
8470 
8471 	if (rx_buffer->page_offset > IGB_LAST_OFFSET)
8472 		return false;
8473 #endif
8474 
8475 	/* If we have drained the page fragment pool we need to update
8476 	 * the pagecnt_bias and page count so that we fully restock the
8477 	 * number of references the driver holds.
8478 	 */
8479 	if (unlikely(pagecnt_bias == 1)) {
8480 		page_ref_add(page, USHRT_MAX - 1);
8481 		rx_buffer->pagecnt_bias = USHRT_MAX;
8482 	}
8483 
8484 	return true;
8485 }
8486 
8487 /**
8488  *  igb_add_rx_frag - Add contents of Rx buffer to sk_buff
8489  *  @rx_ring: rx descriptor ring to transact packets on
8490  *  @rx_buffer: buffer containing page to add
8491  *  @skb: sk_buff to place the data into
8492  *  @size: size of buffer to be added
8493  *
8494  *  This function will add the data contained in rx_buffer->page to the skb.
8495  **/
igb_add_rx_frag(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,struct sk_buff * skb,unsigned int size)8496 static void igb_add_rx_frag(struct igb_ring *rx_ring,
8497 			    struct igb_rx_buffer *rx_buffer,
8498 			    struct sk_buff *skb,
8499 			    unsigned int size)
8500 {
8501 #if (PAGE_SIZE < 8192)
8502 	unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8503 #else
8504 	unsigned int truesize = ring_uses_build_skb(rx_ring) ?
8505 				SKB_DATA_ALIGN(IGB_SKB_PAD + size) :
8506 				SKB_DATA_ALIGN(size);
8507 #endif
8508 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
8509 			rx_buffer->page_offset, size, truesize);
8510 #if (PAGE_SIZE < 8192)
8511 	rx_buffer->page_offset ^= truesize;
8512 #else
8513 	rx_buffer->page_offset += truesize;
8514 #endif
8515 }
8516 
igb_construct_skb(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,struct xdp_buff * xdp,ktime_t timestamp)8517 static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,
8518 					 struct igb_rx_buffer *rx_buffer,
8519 					 struct xdp_buff *xdp,
8520 					 ktime_t timestamp)
8521 {
8522 #if (PAGE_SIZE < 8192)
8523 	unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8524 #else
8525 	unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
8526 					       xdp->data_hard_start);
8527 #endif
8528 	unsigned int size = xdp->data_end - xdp->data;
8529 	unsigned int headlen;
8530 	struct sk_buff *skb;
8531 
8532 	/* prefetch first cache line of first page */
8533 	net_prefetch(xdp->data);
8534 
8535 	/* allocate a skb to store the frags */
8536 	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
8537 	if (unlikely(!skb))
8538 		return NULL;
8539 
8540 	if (timestamp)
8541 		skb_hwtstamps(skb)->hwtstamp = timestamp;
8542 
8543 	/* Determine available headroom for copy */
8544 	headlen = size;
8545 	if (headlen > IGB_RX_HDR_LEN)
8546 		headlen = eth_get_headlen(skb->dev, xdp->data, IGB_RX_HDR_LEN);
8547 
8548 	/* align pull length to size of long to optimize memcpy performance */
8549 	memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, sizeof(long)));
8550 
8551 	/* update all of the pointers */
8552 	size -= headlen;
8553 	if (size) {
8554 		skb_add_rx_frag(skb, 0, rx_buffer->page,
8555 				(xdp->data + headlen) - page_address(rx_buffer->page),
8556 				size, truesize);
8557 #if (PAGE_SIZE < 8192)
8558 		rx_buffer->page_offset ^= truesize;
8559 #else
8560 		rx_buffer->page_offset += truesize;
8561 #endif
8562 	} else {
8563 		rx_buffer->pagecnt_bias++;
8564 	}
8565 
8566 	return skb;
8567 }
8568 
igb_build_skb(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,struct xdp_buff * xdp,ktime_t timestamp)8569 static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring,
8570 				     struct igb_rx_buffer *rx_buffer,
8571 				     struct xdp_buff *xdp,
8572 				     ktime_t timestamp)
8573 {
8574 #if (PAGE_SIZE < 8192)
8575 	unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8576 #else
8577 	unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
8578 				SKB_DATA_ALIGN(xdp->data_end -
8579 					       xdp->data_hard_start);
8580 #endif
8581 	unsigned int metasize = xdp->data - xdp->data_meta;
8582 	struct sk_buff *skb;
8583 
8584 	/* prefetch first cache line of first page */
8585 	net_prefetch(xdp->data_meta);
8586 
8587 	/* build an skb around the page buffer */
8588 	skb = napi_build_skb(xdp->data_hard_start, truesize);
8589 	if (unlikely(!skb))
8590 		return NULL;
8591 
8592 	/* update pointers within the skb to store the data */
8593 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
8594 	__skb_put(skb, xdp->data_end - xdp->data);
8595 
8596 	if (metasize)
8597 		skb_metadata_set(skb, metasize);
8598 
8599 	if (timestamp)
8600 		skb_hwtstamps(skb)->hwtstamp = timestamp;
8601 
8602 	/* update buffer offset */
8603 #if (PAGE_SIZE < 8192)
8604 	rx_buffer->page_offset ^= truesize;
8605 #else
8606 	rx_buffer->page_offset += truesize;
8607 #endif
8608 
8609 	return skb;
8610 }
8611 
igb_run_xdp(struct igb_adapter * adapter,struct igb_ring * rx_ring,struct xdp_buff * xdp)8612 static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter,
8613 				   struct igb_ring *rx_ring,
8614 				   struct xdp_buff *xdp)
8615 {
8616 	int err, result = IGB_XDP_PASS;
8617 	struct bpf_prog *xdp_prog;
8618 	u32 act;
8619 
8620 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
8621 
8622 	if (!xdp_prog)
8623 		goto xdp_out;
8624 
8625 	prefetchw(xdp->data_hard_start); /* xdp_frame write */
8626 
8627 	act = bpf_prog_run_xdp(xdp_prog, xdp);
8628 	switch (act) {
8629 	case XDP_PASS:
8630 		break;
8631 	case XDP_TX:
8632 		result = igb_xdp_xmit_back(adapter, xdp);
8633 		if (result == IGB_XDP_CONSUMED)
8634 			goto out_failure;
8635 		break;
8636 	case XDP_REDIRECT:
8637 		err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
8638 		if (err)
8639 			goto out_failure;
8640 		result = IGB_XDP_REDIR;
8641 		break;
8642 	default:
8643 		bpf_warn_invalid_xdp_action(adapter->netdev, xdp_prog, act);
8644 		fallthrough;
8645 	case XDP_ABORTED:
8646 out_failure:
8647 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
8648 		fallthrough;
8649 	case XDP_DROP:
8650 		result = IGB_XDP_CONSUMED;
8651 		break;
8652 	}
8653 xdp_out:
8654 	return ERR_PTR(-result);
8655 }
8656 
igb_rx_frame_truesize(struct igb_ring * rx_ring,unsigned int size)8657 static unsigned int igb_rx_frame_truesize(struct igb_ring *rx_ring,
8658 					  unsigned int size)
8659 {
8660 	unsigned int truesize;
8661 
8662 #if (PAGE_SIZE < 8192)
8663 	truesize = igb_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
8664 #else
8665 	truesize = ring_uses_build_skb(rx_ring) ?
8666 		SKB_DATA_ALIGN(IGB_SKB_PAD + size) +
8667 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
8668 		SKB_DATA_ALIGN(size);
8669 #endif
8670 	return truesize;
8671 }
8672 
igb_rx_buffer_flip(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,unsigned int size)8673 static void igb_rx_buffer_flip(struct igb_ring *rx_ring,
8674 			       struct igb_rx_buffer *rx_buffer,
8675 			       unsigned int size)
8676 {
8677 	unsigned int truesize = igb_rx_frame_truesize(rx_ring, size);
8678 #if (PAGE_SIZE < 8192)
8679 	rx_buffer->page_offset ^= truesize;
8680 #else
8681 	rx_buffer->page_offset += truesize;
8682 #endif
8683 }
8684 
igb_rx_checksum(struct igb_ring * ring,union e1000_adv_rx_desc * rx_desc,struct sk_buff * skb)8685 static inline void igb_rx_checksum(struct igb_ring *ring,
8686 				   union e1000_adv_rx_desc *rx_desc,
8687 				   struct sk_buff *skb)
8688 {
8689 	skb_checksum_none_assert(skb);
8690 
8691 	/* Ignore Checksum bit is set */
8692 	if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
8693 		return;
8694 
8695 	/* Rx checksum disabled via ethtool */
8696 	if (!(ring->netdev->features & NETIF_F_RXCSUM))
8697 		return;
8698 
8699 	/* TCP/UDP checksum error bit is set */
8700 	if (igb_test_staterr(rx_desc,
8701 			     E1000_RXDEXT_STATERR_TCPE |
8702 			     E1000_RXDEXT_STATERR_IPE)) {
8703 		/* work around errata with sctp packets where the TCPE aka
8704 		 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
8705 		 * packets, (aka let the stack check the crc32c)
8706 		 */
8707 		if (!((skb->len == 60) &&
8708 		      test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
8709 			u64_stats_update_begin(&ring->rx_syncp);
8710 			ring->rx_stats.csum_err++;
8711 			u64_stats_update_end(&ring->rx_syncp);
8712 		}
8713 		/* let the stack verify checksum errors */
8714 		return;
8715 	}
8716 	/* It must be a TCP or UDP packet with a valid checksum */
8717 	if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
8718 				      E1000_RXD_STAT_UDPCS))
8719 		skb->ip_summed = CHECKSUM_UNNECESSARY;
8720 
8721 	dev_dbg(ring->dev, "cksum success: bits %08X\n",
8722 		le32_to_cpu(rx_desc->wb.upper.status_error));
8723 }
8724 
igb_rx_hash(struct igb_ring * ring,union e1000_adv_rx_desc * rx_desc,struct sk_buff * skb)8725 static inline void igb_rx_hash(struct igb_ring *ring,
8726 			       union e1000_adv_rx_desc *rx_desc,
8727 			       struct sk_buff *skb)
8728 {
8729 	if (ring->netdev->features & NETIF_F_RXHASH)
8730 		skb_set_hash(skb,
8731 			     le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
8732 			     PKT_HASH_TYPE_L3);
8733 }
8734 
8735 /**
8736  *  igb_is_non_eop - process handling of non-EOP buffers
8737  *  @rx_ring: Rx ring being processed
8738  *  @rx_desc: Rx descriptor for current buffer
8739  *
8740  *  This function updates next to clean.  If the buffer is an EOP buffer
8741  *  this function exits returning false, otherwise it will place the
8742  *  sk_buff in the next buffer to be chained and return true indicating
8743  *  that this is in fact a non-EOP buffer.
8744  **/
igb_is_non_eop(struct igb_ring * rx_ring,union e1000_adv_rx_desc * rx_desc)8745 static bool igb_is_non_eop(struct igb_ring *rx_ring,
8746 			   union e1000_adv_rx_desc *rx_desc)
8747 {
8748 	u32 ntc = rx_ring->next_to_clean + 1;
8749 
8750 	/* fetch, update, and store next to clean */
8751 	ntc = (ntc < rx_ring->count) ? ntc : 0;
8752 	rx_ring->next_to_clean = ntc;
8753 
8754 	prefetch(IGB_RX_DESC(rx_ring, ntc));
8755 
8756 	if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
8757 		return false;
8758 
8759 	return true;
8760 }
8761 
8762 /**
8763  *  igb_cleanup_headers - Correct corrupted or empty headers
8764  *  @rx_ring: rx descriptor ring packet is being transacted on
8765  *  @rx_desc: pointer to the EOP Rx descriptor
8766  *  @skb: pointer to current skb being fixed
8767  *
8768  *  Address the case where we are pulling data in on pages only
8769  *  and as such no data is present in the skb header.
8770  *
8771  *  In addition if skb is not at least 60 bytes we need to pad it so that
8772  *  it is large enough to qualify as a valid Ethernet frame.
8773  *
8774  *  Returns true if an error was encountered and skb was freed.
8775  **/
igb_cleanup_headers(struct igb_ring * rx_ring,union e1000_adv_rx_desc * rx_desc,struct sk_buff * skb)8776 static bool igb_cleanup_headers(struct igb_ring *rx_ring,
8777 				union e1000_adv_rx_desc *rx_desc,
8778 				struct sk_buff *skb)
8779 {
8780 	/* XDP packets use error pointer so abort at this point */
8781 	if (IS_ERR(skb))
8782 		return true;
8783 
8784 	if (unlikely((igb_test_staterr(rx_desc,
8785 				       E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
8786 		struct net_device *netdev = rx_ring->netdev;
8787 		if (!(netdev->features & NETIF_F_RXALL)) {
8788 			dev_kfree_skb_any(skb);
8789 			return true;
8790 		}
8791 	}
8792 
8793 	/* if eth_skb_pad returns an error the skb was freed */
8794 	if (eth_skb_pad(skb))
8795 		return true;
8796 
8797 	return false;
8798 }
8799 
8800 /**
8801  *  igb_process_skb_fields - Populate skb header fields from Rx descriptor
8802  *  @rx_ring: rx descriptor ring packet is being transacted on
8803  *  @rx_desc: pointer to the EOP Rx descriptor
8804  *  @skb: pointer to current skb being populated
8805  *
8806  *  This function checks the ring, descriptor, and packet information in
8807  *  order to populate the hash, checksum, VLAN, timestamp, protocol, and
8808  *  other fields within the skb.
8809  **/
igb_process_skb_fields(struct igb_ring * rx_ring,union e1000_adv_rx_desc * rx_desc,struct sk_buff * skb)8810 static void igb_process_skb_fields(struct igb_ring *rx_ring,
8811 				   union e1000_adv_rx_desc *rx_desc,
8812 				   struct sk_buff *skb)
8813 {
8814 	struct net_device *dev = rx_ring->netdev;
8815 
8816 	igb_rx_hash(rx_ring, rx_desc, skb);
8817 
8818 	igb_rx_checksum(rx_ring, rx_desc, skb);
8819 
8820 	if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) &&
8821 	    !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))
8822 		igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);
8823 
8824 	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
8825 	    igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
8826 		u16 vid;
8827 
8828 		if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
8829 		    test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
8830 			vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
8831 		else
8832 			vid = le16_to_cpu(rx_desc->wb.upper.vlan);
8833 
8834 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
8835 	}
8836 
8837 	skb_record_rx_queue(skb, rx_ring->queue_index);
8838 
8839 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8840 }
8841 
igb_rx_offset(struct igb_ring * rx_ring)8842 static unsigned int igb_rx_offset(struct igb_ring *rx_ring)
8843 {
8844 	return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0;
8845 }
8846 
igb_get_rx_buffer(struct igb_ring * rx_ring,const unsigned int size,int * rx_buf_pgcnt)8847 static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring,
8848 					       const unsigned int size, int *rx_buf_pgcnt)
8849 {
8850 	struct igb_rx_buffer *rx_buffer;
8851 
8852 	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
8853 	*rx_buf_pgcnt =
8854 #if (PAGE_SIZE < 8192)
8855 		page_count(rx_buffer->page);
8856 #else
8857 		0;
8858 #endif
8859 	prefetchw(rx_buffer->page);
8860 
8861 	/* we are reusing so sync this buffer for CPU use */
8862 	dma_sync_single_range_for_cpu(rx_ring->dev,
8863 				      rx_buffer->dma,
8864 				      rx_buffer->page_offset,
8865 				      size,
8866 				      DMA_FROM_DEVICE);
8867 
8868 	rx_buffer->pagecnt_bias--;
8869 
8870 	return rx_buffer;
8871 }
8872 
igb_put_rx_buffer(struct igb_ring * rx_ring,struct igb_rx_buffer * rx_buffer,int rx_buf_pgcnt)8873 static void igb_put_rx_buffer(struct igb_ring *rx_ring,
8874 			      struct igb_rx_buffer *rx_buffer, int rx_buf_pgcnt)
8875 {
8876 	if (igb_can_reuse_rx_page(rx_buffer, rx_buf_pgcnt)) {
8877 		/* hand second half of page back to the ring */
8878 		igb_reuse_rx_page(rx_ring, rx_buffer);
8879 	} else {
8880 		/* We are not reusing the buffer so unmap it and free
8881 		 * any references we are holding to it
8882 		 */
8883 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
8884 				     igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
8885 				     IGB_RX_DMA_ATTR);
8886 		__page_frag_cache_drain(rx_buffer->page,
8887 					rx_buffer->pagecnt_bias);
8888 	}
8889 
8890 	/* clear contents of rx_buffer */
8891 	rx_buffer->page = NULL;
8892 }
8893 
igb_clean_rx_irq(struct igb_q_vector * q_vector,const int budget)8894 static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
8895 {
8896 	unsigned int total_bytes = 0, total_packets = 0;
8897 	struct igb_adapter *adapter = q_vector->adapter;
8898 	struct igb_ring *rx_ring = q_vector->rx.ring;
8899 	u16 cleaned_count = igb_desc_unused(rx_ring);
8900 	struct sk_buff *skb = rx_ring->skb;
8901 	int cpu = smp_processor_id();
8902 	unsigned int xdp_xmit = 0;
8903 	struct netdev_queue *nq;
8904 	struct xdp_buff xdp;
8905 	u32 frame_sz = 0;
8906 	int rx_buf_pgcnt;
8907 
8908 	/* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
8909 #if (PAGE_SIZE < 8192)
8910 	frame_sz = igb_rx_frame_truesize(rx_ring, 0);
8911 #endif
8912 	xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
8913 
8914 	while (likely(total_packets < budget)) {
8915 		union e1000_adv_rx_desc *rx_desc;
8916 		struct igb_rx_buffer *rx_buffer;
8917 		ktime_t timestamp = 0;
8918 		int pkt_offset = 0;
8919 		unsigned int size;
8920 		void *pktbuf;
8921 
8922 		/* return some buffers to hardware, one at a time is too slow */
8923 		if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
8924 			igb_alloc_rx_buffers(rx_ring, cleaned_count);
8925 			cleaned_count = 0;
8926 		}
8927 
8928 		rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
8929 		size = le16_to_cpu(rx_desc->wb.upper.length);
8930 		if (!size)
8931 			break;
8932 
8933 		/* This memory barrier is needed to keep us from reading
8934 		 * any other fields out of the rx_desc until we know the
8935 		 * descriptor has been written back
8936 		 */
8937 		dma_rmb();
8938 
8939 		rx_buffer = igb_get_rx_buffer(rx_ring, size, &rx_buf_pgcnt);
8940 		pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
8941 
8942 		/* pull rx packet timestamp if available and valid */
8943 		if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
8944 			int ts_hdr_len;
8945 
8946 			ts_hdr_len = igb_ptp_rx_pktstamp(rx_ring->q_vector,
8947 							 pktbuf, &timestamp);
8948 
8949 			pkt_offset += ts_hdr_len;
8950 			size -= ts_hdr_len;
8951 		}
8952 
8953 		/* retrieve a buffer from the ring */
8954 		if (!skb) {
8955 			unsigned char *hard_start = pktbuf - igb_rx_offset(rx_ring);
8956 			unsigned int offset = pkt_offset + igb_rx_offset(rx_ring);
8957 
8958 			xdp_prepare_buff(&xdp, hard_start, offset, size, true);
8959 			xdp_buff_clear_frags_flag(&xdp);
8960 #if (PAGE_SIZE > 4096)
8961 			/* At larger PAGE_SIZE, frame_sz depend on len size */
8962 			xdp.frame_sz = igb_rx_frame_truesize(rx_ring, size);
8963 #endif
8964 			skb = igb_run_xdp(adapter, rx_ring, &xdp);
8965 		}
8966 
8967 		if (IS_ERR(skb)) {
8968 			unsigned int xdp_res = -PTR_ERR(skb);
8969 
8970 			if (xdp_res & (IGB_XDP_TX | IGB_XDP_REDIR)) {
8971 				xdp_xmit |= xdp_res;
8972 				igb_rx_buffer_flip(rx_ring, rx_buffer, size);
8973 			} else {
8974 				rx_buffer->pagecnt_bias++;
8975 			}
8976 			total_packets++;
8977 			total_bytes += size;
8978 		} else if (skb)
8979 			igb_add_rx_frag(rx_ring, rx_buffer, skb, size);
8980 		else if (ring_uses_build_skb(rx_ring))
8981 			skb = igb_build_skb(rx_ring, rx_buffer, &xdp,
8982 					    timestamp);
8983 		else
8984 			skb = igb_construct_skb(rx_ring, rx_buffer,
8985 						&xdp, timestamp);
8986 
8987 		/* exit if we failed to retrieve a buffer */
8988 		if (!skb) {
8989 			rx_ring->rx_stats.alloc_failed++;
8990 			rx_buffer->pagecnt_bias++;
8991 			break;
8992 		}
8993 
8994 		igb_put_rx_buffer(rx_ring, rx_buffer, rx_buf_pgcnt);
8995 		cleaned_count++;
8996 
8997 		/* fetch next buffer in frame if non-eop */
8998 		if (igb_is_non_eop(rx_ring, rx_desc))
8999 			continue;
9000 
9001 		/* verify the packet layout is correct */
9002 		if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
9003 			skb = NULL;
9004 			continue;
9005 		}
9006 
9007 		/* probably a little skewed due to removing CRC */
9008 		total_bytes += skb->len;
9009 
9010 		/* populate checksum, timestamp, VLAN, and protocol */
9011 		igb_process_skb_fields(rx_ring, rx_desc, skb);
9012 
9013 		napi_gro_receive(&q_vector->napi, skb);
9014 
9015 		/* reset skb pointer */
9016 		skb = NULL;
9017 
9018 		/* update budget accounting */
9019 		total_packets++;
9020 	}
9021 
9022 	/* place incomplete frames back on ring for completion */
9023 	rx_ring->skb = skb;
9024 
9025 	if (xdp_xmit & IGB_XDP_REDIR)
9026 		xdp_do_flush();
9027 
9028 	if (xdp_xmit & IGB_XDP_TX) {
9029 		struct igb_ring *tx_ring = igb_xdp_tx_queue_mapping(adapter);
9030 
9031 		nq = txring_txq(tx_ring);
9032 		__netif_tx_lock(nq, cpu);
9033 		igb_xdp_ring_update_tail(tx_ring);
9034 		__netif_tx_unlock(nq);
9035 	}
9036 
9037 	u64_stats_update_begin(&rx_ring->rx_syncp);
9038 	rx_ring->rx_stats.packets += total_packets;
9039 	rx_ring->rx_stats.bytes += total_bytes;
9040 	u64_stats_update_end(&rx_ring->rx_syncp);
9041 	q_vector->rx.total_packets += total_packets;
9042 	q_vector->rx.total_bytes += total_bytes;
9043 
9044 	if (cleaned_count)
9045 		igb_alloc_rx_buffers(rx_ring, cleaned_count);
9046 
9047 	return total_packets;
9048 }
9049 
igb_alloc_mapped_page(struct igb_ring * rx_ring,struct igb_rx_buffer * bi)9050 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
9051 				  struct igb_rx_buffer *bi)
9052 {
9053 	struct page *page = bi->page;
9054 	dma_addr_t dma;
9055 
9056 	/* since we are recycling buffers we should seldom need to alloc */
9057 	if (likely(page))
9058 		return true;
9059 
9060 	/* alloc new page for storage */
9061 	page = dev_alloc_pages(igb_rx_pg_order(rx_ring));
9062 	if (unlikely(!page)) {
9063 		rx_ring->rx_stats.alloc_failed++;
9064 		return false;
9065 	}
9066 
9067 	/* map page for use */
9068 	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
9069 				 igb_rx_pg_size(rx_ring),
9070 				 DMA_FROM_DEVICE,
9071 				 IGB_RX_DMA_ATTR);
9072 
9073 	/* if mapping failed free memory back to system since
9074 	 * there isn't much point in holding memory we can't use
9075 	 */
9076 	if (dma_mapping_error(rx_ring->dev, dma)) {
9077 		__free_pages(page, igb_rx_pg_order(rx_ring));
9078 
9079 		rx_ring->rx_stats.alloc_failed++;
9080 		return false;
9081 	}
9082 
9083 	bi->dma = dma;
9084 	bi->page = page;
9085 	bi->page_offset = igb_rx_offset(rx_ring);
9086 	page_ref_add(page, USHRT_MAX - 1);
9087 	bi->pagecnt_bias = USHRT_MAX;
9088 
9089 	return true;
9090 }
9091 
9092 /**
9093  *  igb_alloc_rx_buffers - Replace used receive buffers
9094  *  @rx_ring: rx descriptor ring to allocate new receive buffers
9095  *  @cleaned_count: count of buffers to allocate
9096  **/
igb_alloc_rx_buffers(struct igb_ring * rx_ring,u16 cleaned_count)9097 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9098 {
9099 	union e1000_adv_rx_desc *rx_desc;
9100 	struct igb_rx_buffer *bi;
9101 	u16 i = rx_ring->next_to_use;
9102 	u16 bufsz;
9103 
9104 	/* nothing to do */
9105 	if (!cleaned_count)
9106 		return;
9107 
9108 	rx_desc = IGB_RX_DESC(rx_ring, i);
9109 	bi = &rx_ring->rx_buffer_info[i];
9110 	i -= rx_ring->count;
9111 
9112 	bufsz = igb_rx_bufsz(rx_ring);
9113 
9114 	do {
9115 		if (!igb_alloc_mapped_page(rx_ring, bi))
9116 			break;
9117 
9118 		/* sync the buffer for use by the device */
9119 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
9120 						 bi->page_offset, bufsz,
9121 						 DMA_FROM_DEVICE);
9122 
9123 		/* Refresh the desc even if buffer_addrs didn't change
9124 		 * because each write-back erases this info.
9125 		 */
9126 		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
9127 
9128 		rx_desc++;
9129 		bi++;
9130 		i++;
9131 		if (unlikely(!i)) {
9132 			rx_desc = IGB_RX_DESC(rx_ring, 0);
9133 			bi = rx_ring->rx_buffer_info;
9134 			i -= rx_ring->count;
9135 		}
9136 
9137 		/* clear the length for the next_to_use descriptor */
9138 		rx_desc->wb.upper.length = 0;
9139 
9140 		cleaned_count--;
9141 	} while (cleaned_count);
9142 
9143 	i += rx_ring->count;
9144 
9145 	if (rx_ring->next_to_use != i) {
9146 		/* record the next descriptor to use */
9147 		rx_ring->next_to_use = i;
9148 
9149 		/* update next to alloc since we have filled the ring */
9150 		rx_ring->next_to_alloc = i;
9151 
9152 		/* Force memory writes to complete before letting h/w
9153 		 * know there are new descriptors to fetch.  (Only
9154 		 * applicable for weak-ordered memory model archs,
9155 		 * such as IA-64).
9156 		 */
9157 		dma_wmb();
9158 		writel(i, rx_ring->tail);
9159 	}
9160 }
9161 
9162 /**
9163  * igb_mii_ioctl -
9164  * @netdev: pointer to netdev struct
9165  * @ifr: interface structure
9166  * @cmd: ioctl command to execute
9167  **/
igb_mii_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)9168 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
9169 {
9170 	struct igb_adapter *adapter = netdev_priv(netdev);
9171 	struct mii_ioctl_data *data = if_mii(ifr);
9172 
9173 	if (adapter->hw.phy.media_type != e1000_media_type_copper)
9174 		return -EOPNOTSUPP;
9175 
9176 	switch (cmd) {
9177 	case SIOCGMIIPHY:
9178 		data->phy_id = adapter->hw.phy.addr;
9179 		break;
9180 	case SIOCGMIIREG:
9181 		if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
9182 				     &data->val_out))
9183 			return -EIO;
9184 		break;
9185 	case SIOCSMIIREG:
9186 	default:
9187 		return -EOPNOTSUPP;
9188 	}
9189 	return 0;
9190 }
9191 
9192 /**
9193  * igb_ioctl -
9194  * @netdev: pointer to netdev struct
9195  * @ifr: interface structure
9196  * @cmd: ioctl command to execute
9197  **/
igb_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)9198 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
9199 {
9200 	switch (cmd) {
9201 	case SIOCGMIIPHY:
9202 	case SIOCGMIIREG:
9203 	case SIOCSMIIREG:
9204 		return igb_mii_ioctl(netdev, ifr, cmd);
9205 	case SIOCGHWTSTAMP:
9206 		return igb_ptp_get_ts_config(netdev, ifr);
9207 	case SIOCSHWTSTAMP:
9208 		return igb_ptp_set_ts_config(netdev, ifr);
9209 	default:
9210 		return -EOPNOTSUPP;
9211 	}
9212 }
9213 
igb_read_pci_cfg(struct e1000_hw * hw,u32 reg,u16 * value)9214 void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
9215 {
9216 	struct igb_adapter *adapter = hw->back;
9217 
9218 	pci_read_config_word(adapter->pdev, reg, value);
9219 }
9220 
igb_write_pci_cfg(struct e1000_hw * hw,u32 reg,u16 * value)9221 void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
9222 {
9223 	struct igb_adapter *adapter = hw->back;
9224 
9225 	pci_write_config_word(adapter->pdev, reg, *value);
9226 }
9227 
igb_read_pcie_cap_reg(struct e1000_hw * hw,u32 reg,u16 * value)9228 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
9229 {
9230 	struct igb_adapter *adapter = hw->back;
9231 
9232 	if (pcie_capability_read_word(adapter->pdev, reg, value))
9233 		return -E1000_ERR_CONFIG;
9234 
9235 	return 0;
9236 }
9237 
igb_write_pcie_cap_reg(struct e1000_hw * hw,u32 reg,u16 * value)9238 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
9239 {
9240 	struct igb_adapter *adapter = hw->back;
9241 
9242 	if (pcie_capability_write_word(adapter->pdev, reg, *value))
9243 		return -E1000_ERR_CONFIG;
9244 
9245 	return 0;
9246 }
9247 
igb_vlan_mode(struct net_device * netdev,netdev_features_t features)9248 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9249 {
9250 	struct igb_adapter *adapter = netdev_priv(netdev);
9251 	struct e1000_hw *hw = &adapter->hw;
9252 	u32 ctrl, rctl;
9253 	bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
9254 
9255 	if (enable) {
9256 		/* enable VLAN tag insert/strip */
9257 		ctrl = rd32(E1000_CTRL);
9258 		ctrl |= E1000_CTRL_VME;
9259 		wr32(E1000_CTRL, ctrl);
9260 
9261 		/* Disable CFI check */
9262 		rctl = rd32(E1000_RCTL);
9263 		rctl &= ~E1000_RCTL_CFIEN;
9264 		wr32(E1000_RCTL, rctl);
9265 	} else {
9266 		/* disable VLAN tag insert/strip */
9267 		ctrl = rd32(E1000_CTRL);
9268 		ctrl &= ~E1000_CTRL_VME;
9269 		wr32(E1000_CTRL, ctrl);
9270 	}
9271 
9272 	igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable);
9273 }
9274 
igb_vlan_rx_add_vid(struct net_device * netdev,__be16 proto,u16 vid)9275 static int igb_vlan_rx_add_vid(struct net_device *netdev,
9276 			       __be16 proto, u16 vid)
9277 {
9278 	struct igb_adapter *adapter = netdev_priv(netdev);
9279 	struct e1000_hw *hw = &adapter->hw;
9280 	int pf_id = adapter->vfs_allocated_count;
9281 
9282 	/* add the filter since PF can receive vlans w/o entry in vlvf */
9283 	if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
9284 		igb_vfta_set(hw, vid, pf_id, true, !!vid);
9285 
9286 	set_bit(vid, adapter->active_vlans);
9287 
9288 	return 0;
9289 }
9290 
igb_vlan_rx_kill_vid(struct net_device * netdev,__be16 proto,u16 vid)9291 static int igb_vlan_rx_kill_vid(struct net_device *netdev,
9292 				__be16 proto, u16 vid)
9293 {
9294 	struct igb_adapter *adapter = netdev_priv(netdev);
9295 	int pf_id = adapter->vfs_allocated_count;
9296 	struct e1000_hw *hw = &adapter->hw;
9297 
9298 	/* remove VID from filter table */
9299 	if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
9300 		igb_vfta_set(hw, vid, pf_id, false, true);
9301 
9302 	clear_bit(vid, adapter->active_vlans);
9303 
9304 	return 0;
9305 }
9306 
igb_restore_vlan(struct igb_adapter * adapter)9307 static void igb_restore_vlan(struct igb_adapter *adapter)
9308 {
9309 	u16 vid = 1;
9310 
9311 	igb_vlan_mode(adapter->netdev, adapter->netdev->features);
9312 	igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
9313 
9314 	for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
9315 		igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
9316 }
9317 
igb_set_spd_dplx(struct igb_adapter * adapter,u32 spd,u8 dplx)9318 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9319 {
9320 	struct pci_dev *pdev = adapter->pdev;
9321 	struct e1000_mac_info *mac = &adapter->hw.mac;
9322 
9323 	mac->autoneg = 0;
9324 
9325 	/* Make sure dplx is at most 1 bit and lsb of speed is not set
9326 	 * for the switch() below to work
9327 	 */
9328 	if ((spd & 1) || (dplx & ~1))
9329 		goto err_inval;
9330 
9331 	/* Fiber NIC's only allow 1000 gbps Full duplex
9332 	 * and 100Mbps Full duplex for 100baseFx sfp
9333 	 */
9334 	if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
9335 		switch (spd + dplx) {
9336 		case SPEED_10 + DUPLEX_HALF:
9337 		case SPEED_10 + DUPLEX_FULL:
9338 		case SPEED_100 + DUPLEX_HALF:
9339 			goto err_inval;
9340 		default:
9341 			break;
9342 		}
9343 	}
9344 
9345 	switch (spd + dplx) {
9346 	case SPEED_10 + DUPLEX_HALF:
9347 		mac->forced_speed_duplex = ADVERTISE_10_HALF;
9348 		break;
9349 	case SPEED_10 + DUPLEX_FULL:
9350 		mac->forced_speed_duplex = ADVERTISE_10_FULL;
9351 		break;
9352 	case SPEED_100 + DUPLEX_HALF:
9353 		mac->forced_speed_duplex = ADVERTISE_100_HALF;
9354 		break;
9355 	case SPEED_100 + DUPLEX_FULL:
9356 		mac->forced_speed_duplex = ADVERTISE_100_FULL;
9357 		break;
9358 	case SPEED_1000 + DUPLEX_FULL:
9359 		mac->autoneg = 1;
9360 		adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
9361 		break;
9362 	case SPEED_1000 + DUPLEX_HALF: /* not supported */
9363 	default:
9364 		goto err_inval;
9365 	}
9366 
9367 	/* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
9368 	adapter->hw.phy.mdix = AUTO_ALL_MODES;
9369 
9370 	return 0;
9371 
9372 err_inval:
9373 	dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9374 	return -EINVAL;
9375 }
9376 
__igb_shutdown(struct pci_dev * pdev,bool * enable_wake,bool runtime)9377 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
9378 			  bool runtime)
9379 {
9380 	struct net_device *netdev = pci_get_drvdata(pdev);
9381 	struct igb_adapter *adapter = netdev_priv(netdev);
9382 	struct e1000_hw *hw = &adapter->hw;
9383 	u32 ctrl, rctl, status;
9384 	u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9385 	bool wake;
9386 
9387 	rtnl_lock();
9388 	netif_device_detach(netdev);
9389 
9390 	if (netif_running(netdev))
9391 		__igb_close(netdev, true);
9392 
9393 	igb_ptp_suspend(adapter);
9394 
9395 	igb_clear_interrupt_scheme(adapter);
9396 	rtnl_unlock();
9397 
9398 	status = rd32(E1000_STATUS);
9399 	if (status & E1000_STATUS_LU)
9400 		wufc &= ~E1000_WUFC_LNKC;
9401 
9402 	if (wufc) {
9403 		igb_setup_rctl(adapter);
9404 		igb_set_rx_mode(netdev);
9405 
9406 		/* turn on all-multi mode if wake on multicast is enabled */
9407 		if (wufc & E1000_WUFC_MC) {
9408 			rctl = rd32(E1000_RCTL);
9409 			rctl |= E1000_RCTL_MPE;
9410 			wr32(E1000_RCTL, rctl);
9411 		}
9412 
9413 		ctrl = rd32(E1000_CTRL);
9414 		ctrl |= E1000_CTRL_ADVD3WUC;
9415 		wr32(E1000_CTRL, ctrl);
9416 
9417 		/* Allow time for pending master requests to run */
9418 		igb_disable_pcie_master(hw);
9419 
9420 		wr32(E1000_WUC, E1000_WUC_PME_EN);
9421 		wr32(E1000_WUFC, wufc);
9422 	} else {
9423 		wr32(E1000_WUC, 0);
9424 		wr32(E1000_WUFC, 0);
9425 	}
9426 
9427 	wake = wufc || adapter->en_mng_pt;
9428 	if (!wake)
9429 		igb_power_down_link(adapter);
9430 	else
9431 		igb_power_up_link(adapter);
9432 
9433 	if (enable_wake)
9434 		*enable_wake = wake;
9435 
9436 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
9437 	 * would have already happened in close and is redundant.
9438 	 */
9439 	igb_release_hw_control(adapter);
9440 
9441 	pci_disable_device(pdev);
9442 
9443 	return 0;
9444 }
9445 
igb_deliver_wake_packet(struct net_device * netdev)9446 static void igb_deliver_wake_packet(struct net_device *netdev)
9447 {
9448 	struct igb_adapter *adapter = netdev_priv(netdev);
9449 	struct e1000_hw *hw = &adapter->hw;
9450 	struct sk_buff *skb;
9451 	u32 wupl;
9452 
9453 	wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK;
9454 
9455 	/* WUPM stores only the first 128 bytes of the wake packet.
9456 	 * Read the packet only if we have the whole thing.
9457 	 */
9458 	if ((wupl == 0) || (wupl > E1000_WUPM_BYTES))
9459 		return;
9460 
9461 	skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES);
9462 	if (!skb)
9463 		return;
9464 
9465 	skb_put(skb, wupl);
9466 
9467 	/* Ensure reads are 32-bit aligned */
9468 	wupl = roundup(wupl, 4);
9469 
9470 	memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl);
9471 
9472 	skb->protocol = eth_type_trans(skb, netdev);
9473 	netif_rx(skb);
9474 }
9475 
igb_suspend(struct device * dev)9476 static int __maybe_unused igb_suspend(struct device *dev)
9477 {
9478 	return __igb_shutdown(to_pci_dev(dev), NULL, 0);
9479 }
9480 
__igb_resume(struct device * dev,bool rpm)9481 static int __maybe_unused __igb_resume(struct device *dev, bool rpm)
9482 {
9483 	struct pci_dev *pdev = to_pci_dev(dev);
9484 	struct net_device *netdev = pci_get_drvdata(pdev);
9485 	struct igb_adapter *adapter = netdev_priv(netdev);
9486 	struct e1000_hw *hw = &adapter->hw;
9487 	u32 err, val;
9488 
9489 	pci_set_power_state(pdev, PCI_D0);
9490 	pci_restore_state(pdev);
9491 	pci_save_state(pdev);
9492 
9493 	if (!pci_device_is_present(pdev))
9494 		return -ENODEV;
9495 	err = pci_enable_device_mem(pdev);
9496 	if (err) {
9497 		dev_err(&pdev->dev,
9498 			"igb: Cannot enable PCI device from suspend\n");
9499 		return err;
9500 	}
9501 	pci_set_master(pdev);
9502 
9503 	pci_enable_wake(pdev, PCI_D3hot, 0);
9504 	pci_enable_wake(pdev, PCI_D3cold, 0);
9505 
9506 	if (igb_init_interrupt_scheme(adapter, true)) {
9507 		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
9508 		return -ENOMEM;
9509 	}
9510 
9511 	igb_reset(adapter);
9512 
9513 	/* let the f/w know that the h/w is now under the control of the
9514 	 * driver.
9515 	 */
9516 	igb_get_hw_control(adapter);
9517 
9518 	val = rd32(E1000_WUS);
9519 	if (val & WAKE_PKT_WUS)
9520 		igb_deliver_wake_packet(netdev);
9521 
9522 	wr32(E1000_WUS, ~0);
9523 
9524 	if (!rpm)
9525 		rtnl_lock();
9526 	if (!err && netif_running(netdev))
9527 		err = __igb_open(netdev, true);
9528 
9529 	if (!err)
9530 		netif_device_attach(netdev);
9531 	if (!rpm)
9532 		rtnl_unlock();
9533 
9534 	return err;
9535 }
9536 
igb_resume(struct device * dev)9537 static int __maybe_unused igb_resume(struct device *dev)
9538 {
9539 	return __igb_resume(dev, false);
9540 }
9541 
igb_runtime_idle(struct device * dev)9542 static int __maybe_unused igb_runtime_idle(struct device *dev)
9543 {
9544 	struct net_device *netdev = dev_get_drvdata(dev);
9545 	struct igb_adapter *adapter = netdev_priv(netdev);
9546 
9547 	if (!igb_has_link(adapter))
9548 		pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
9549 
9550 	return -EBUSY;
9551 }
9552 
igb_runtime_suspend(struct device * dev)9553 static int __maybe_unused igb_runtime_suspend(struct device *dev)
9554 {
9555 	return __igb_shutdown(to_pci_dev(dev), NULL, 1);
9556 }
9557 
igb_runtime_resume(struct device * dev)9558 static int __maybe_unused igb_runtime_resume(struct device *dev)
9559 {
9560 	return __igb_resume(dev, true);
9561 }
9562 
igb_shutdown(struct pci_dev * pdev)9563 static void igb_shutdown(struct pci_dev *pdev)
9564 {
9565 	bool wake;
9566 
9567 	__igb_shutdown(pdev, &wake, 0);
9568 
9569 	if (system_state == SYSTEM_POWER_OFF) {
9570 		pci_wake_from_d3(pdev, wake);
9571 		pci_set_power_state(pdev, PCI_D3hot);
9572 	}
9573 }
9574 
igb_pci_sriov_configure(struct pci_dev * dev,int num_vfs)9575 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
9576 {
9577 #ifdef CONFIG_PCI_IOV
9578 	int err;
9579 
9580 	if (num_vfs == 0) {
9581 		return igb_disable_sriov(dev, true);
9582 	} else {
9583 		err = igb_enable_sriov(dev, num_vfs, true);
9584 		return err ? err : num_vfs;
9585 	}
9586 #endif
9587 	return 0;
9588 }
9589 
9590 /**
9591  *  igb_io_error_detected - called when PCI error is detected
9592  *  @pdev: Pointer to PCI device
9593  *  @state: The current pci connection state
9594  *
9595  *  This function is called after a PCI bus error affecting
9596  *  this device has been detected.
9597  **/
igb_io_error_detected(struct pci_dev * pdev,pci_channel_state_t state)9598 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
9599 					      pci_channel_state_t state)
9600 {
9601 	struct net_device *netdev = pci_get_drvdata(pdev);
9602 	struct igb_adapter *adapter = netdev_priv(netdev);
9603 
9604 	if (state == pci_channel_io_normal) {
9605 		dev_warn(&pdev->dev, "Non-correctable non-fatal error reported.\n");
9606 		return PCI_ERS_RESULT_CAN_RECOVER;
9607 	}
9608 
9609 	netif_device_detach(netdev);
9610 
9611 	if (state == pci_channel_io_perm_failure)
9612 		return PCI_ERS_RESULT_DISCONNECT;
9613 
9614 	if (netif_running(netdev))
9615 		igb_down(adapter);
9616 	pci_disable_device(pdev);
9617 
9618 	/* Request a slot reset. */
9619 	return PCI_ERS_RESULT_NEED_RESET;
9620 }
9621 
9622 /**
9623  *  igb_io_slot_reset - called after the pci bus has been reset.
9624  *  @pdev: Pointer to PCI device
9625  *
9626  *  Restart the card from scratch, as if from a cold-boot. Implementation
9627  *  resembles the first-half of the __igb_resume routine.
9628  **/
igb_io_slot_reset(struct pci_dev * pdev)9629 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
9630 {
9631 	struct net_device *netdev = pci_get_drvdata(pdev);
9632 	struct igb_adapter *adapter = netdev_priv(netdev);
9633 	struct e1000_hw *hw = &adapter->hw;
9634 	pci_ers_result_t result;
9635 
9636 	if (pci_enable_device_mem(pdev)) {
9637 		dev_err(&pdev->dev,
9638 			"Cannot re-enable PCI device after reset.\n");
9639 		result = PCI_ERS_RESULT_DISCONNECT;
9640 	} else {
9641 		pci_set_master(pdev);
9642 		pci_restore_state(pdev);
9643 		pci_save_state(pdev);
9644 
9645 		pci_enable_wake(pdev, PCI_D3hot, 0);
9646 		pci_enable_wake(pdev, PCI_D3cold, 0);
9647 
9648 		/* In case of PCI error, adapter lose its HW address
9649 		 * so we should re-assign it here.
9650 		 */
9651 		hw->hw_addr = adapter->io_addr;
9652 
9653 		igb_reset(adapter);
9654 		wr32(E1000_WUS, ~0);
9655 		result = PCI_ERS_RESULT_RECOVERED;
9656 	}
9657 
9658 	return result;
9659 }
9660 
9661 /**
9662  *  igb_io_resume - called when traffic can start flowing again.
9663  *  @pdev: Pointer to PCI device
9664  *
9665  *  This callback is called when the error recovery driver tells us that
9666  *  its OK to resume normal operation. Implementation resembles the
9667  *  second-half of the __igb_resume routine.
9668  */
igb_io_resume(struct pci_dev * pdev)9669 static void igb_io_resume(struct pci_dev *pdev)
9670 {
9671 	struct net_device *netdev = pci_get_drvdata(pdev);
9672 	struct igb_adapter *adapter = netdev_priv(netdev);
9673 
9674 	if (netif_running(netdev)) {
9675 		if (!test_bit(__IGB_DOWN, &adapter->state)) {
9676 			dev_dbg(&pdev->dev, "Resuming from non-fatal error, do nothing.\n");
9677 			return;
9678 		}
9679 		if (igb_up(adapter)) {
9680 			dev_err(&pdev->dev, "igb_up failed after reset\n");
9681 			return;
9682 		}
9683 	}
9684 
9685 	netif_device_attach(netdev);
9686 
9687 	/* let the f/w know that the h/w is now under the control of the
9688 	 * driver.
9689 	 */
9690 	igb_get_hw_control(adapter);
9691 }
9692 
9693 /**
9694  *  igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
9695  *  @adapter: Pointer to adapter structure
9696  *  @index: Index of the RAR entry which need to be synced with MAC table
9697  **/
igb_rar_set_index(struct igb_adapter * adapter,u32 index)9698 static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
9699 {
9700 	struct e1000_hw *hw = &adapter->hw;
9701 	u32 rar_low, rar_high;
9702 	u8 *addr = adapter->mac_table[index].addr;
9703 
9704 	/* HW expects these to be in network order when they are plugged
9705 	 * into the registers which are little endian.  In order to guarantee
9706 	 * that ordering we need to do an leXX_to_cpup here in order to be
9707 	 * ready for the byteswap that occurs with writel
9708 	 */
9709 	rar_low = le32_to_cpup((__le32 *)(addr));
9710 	rar_high = le16_to_cpup((__le16 *)(addr + 4));
9711 
9712 	/* Indicate to hardware the Address is Valid. */
9713 	if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) {
9714 		if (is_valid_ether_addr(addr))
9715 			rar_high |= E1000_RAH_AV;
9716 
9717 		if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR)
9718 			rar_high |= E1000_RAH_ASEL_SRC_ADDR;
9719 
9720 		switch (hw->mac.type) {
9721 		case e1000_82575:
9722 		case e1000_i210:
9723 			if (adapter->mac_table[index].state &
9724 			    IGB_MAC_STATE_QUEUE_STEERING)
9725 				rar_high |= E1000_RAH_QSEL_ENABLE;
9726 
9727 			rar_high |= E1000_RAH_POOL_1 *
9728 				    adapter->mac_table[index].queue;
9729 			break;
9730 		default:
9731 			rar_high |= E1000_RAH_POOL_1 <<
9732 				    adapter->mac_table[index].queue;
9733 			break;
9734 		}
9735 	}
9736 
9737 	wr32(E1000_RAL(index), rar_low);
9738 	wrfl();
9739 	wr32(E1000_RAH(index), rar_high);
9740 	wrfl();
9741 }
9742 
igb_set_vf_mac(struct igb_adapter * adapter,int vf,unsigned char * mac_addr)9743 static int igb_set_vf_mac(struct igb_adapter *adapter,
9744 			  int vf, unsigned char *mac_addr)
9745 {
9746 	struct e1000_hw *hw = &adapter->hw;
9747 	/* VF MAC addresses start at end of receive addresses and moves
9748 	 * towards the first, as a result a collision should not be possible
9749 	 */
9750 	int rar_entry = hw->mac.rar_entry_count - (vf + 1);
9751 	unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses;
9752 
9753 	ether_addr_copy(vf_mac_addr, mac_addr);
9754 	ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr);
9755 	adapter->mac_table[rar_entry].queue = vf;
9756 	adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE;
9757 	igb_rar_set_index(adapter, rar_entry);
9758 
9759 	return 0;
9760 }
9761 
igb_ndo_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)9762 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
9763 {
9764 	struct igb_adapter *adapter = netdev_priv(netdev);
9765 
9766 	if (vf >= adapter->vfs_allocated_count)
9767 		return -EINVAL;
9768 
9769 	/* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC
9770 	 * flag and allows to overwrite the MAC via VF netdev.  This
9771 	 * is necessary to allow libvirt a way to restore the original
9772 	 * MAC after unbinding vfio-pci and reloading igbvf after shutting
9773 	 * down a VM.
9774 	 */
9775 	if (is_zero_ether_addr(mac)) {
9776 		adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC;
9777 		dev_info(&adapter->pdev->dev,
9778 			 "remove administratively set MAC on VF %d\n",
9779 			 vf);
9780 	} else if (is_valid_ether_addr(mac)) {
9781 		adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
9782 		dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n",
9783 			 mac, vf);
9784 		dev_info(&adapter->pdev->dev,
9785 			 "Reload the VF driver to make this change effective.");
9786 		/* Generate additional warning if PF is down */
9787 		if (test_bit(__IGB_DOWN, &adapter->state)) {
9788 			dev_warn(&adapter->pdev->dev,
9789 				 "The VF MAC address has been set, but the PF device is not up.\n");
9790 			dev_warn(&adapter->pdev->dev,
9791 				 "Bring the PF device up before attempting to use the VF device.\n");
9792 		}
9793 	} else {
9794 		return -EINVAL;
9795 	}
9796 	return igb_set_vf_mac(adapter, vf, mac);
9797 }
9798 
igb_link_mbps(int internal_link_speed)9799 static int igb_link_mbps(int internal_link_speed)
9800 {
9801 	switch (internal_link_speed) {
9802 	case SPEED_100:
9803 		return 100;
9804 	case SPEED_1000:
9805 		return 1000;
9806 	default:
9807 		return 0;
9808 	}
9809 }
9810 
igb_set_vf_rate_limit(struct e1000_hw * hw,int vf,int tx_rate,int link_speed)9811 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
9812 				  int link_speed)
9813 {
9814 	int rf_dec, rf_int;
9815 	u32 bcnrc_val;
9816 
9817 	if (tx_rate != 0) {
9818 		/* Calculate the rate factor values to set */
9819 		rf_int = link_speed / tx_rate;
9820 		rf_dec = (link_speed - (rf_int * tx_rate));
9821 		rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) /
9822 			 tx_rate;
9823 
9824 		bcnrc_val = E1000_RTTBCNRC_RS_ENA;
9825 		bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) &
9826 			      E1000_RTTBCNRC_RF_INT_MASK);
9827 		bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
9828 	} else {
9829 		bcnrc_val = 0;
9830 	}
9831 
9832 	wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
9833 	/* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
9834 	 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
9835 	 */
9836 	wr32(E1000_RTTBCNRM, 0x14);
9837 	wr32(E1000_RTTBCNRC, bcnrc_val);
9838 }
9839 
igb_check_vf_rate_limit(struct igb_adapter * adapter)9840 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
9841 {
9842 	int actual_link_speed, i;
9843 	bool reset_rate = false;
9844 
9845 	/* VF TX rate limit was not set or not supported */
9846 	if ((adapter->vf_rate_link_speed == 0) ||
9847 	    (adapter->hw.mac.type != e1000_82576))
9848 		return;
9849 
9850 	actual_link_speed = igb_link_mbps(adapter->link_speed);
9851 	if (actual_link_speed != adapter->vf_rate_link_speed) {
9852 		reset_rate = true;
9853 		adapter->vf_rate_link_speed = 0;
9854 		dev_info(&adapter->pdev->dev,
9855 			 "Link speed has been changed. VF Transmit rate is disabled\n");
9856 	}
9857 
9858 	for (i = 0; i < adapter->vfs_allocated_count; i++) {
9859 		if (reset_rate)
9860 			adapter->vf_data[i].tx_rate = 0;
9861 
9862 		igb_set_vf_rate_limit(&adapter->hw, i,
9863 				      adapter->vf_data[i].tx_rate,
9864 				      actual_link_speed);
9865 	}
9866 }
9867 
igb_ndo_set_vf_bw(struct net_device * netdev,int vf,int min_tx_rate,int max_tx_rate)9868 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf,
9869 			     int min_tx_rate, int max_tx_rate)
9870 {
9871 	struct igb_adapter *adapter = netdev_priv(netdev);
9872 	struct e1000_hw *hw = &adapter->hw;
9873 	int actual_link_speed;
9874 
9875 	if (hw->mac.type != e1000_82576)
9876 		return -EOPNOTSUPP;
9877 
9878 	if (min_tx_rate)
9879 		return -EINVAL;
9880 
9881 	actual_link_speed = igb_link_mbps(adapter->link_speed);
9882 	if ((vf >= adapter->vfs_allocated_count) ||
9883 	    (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
9884 	    (max_tx_rate < 0) ||
9885 	    (max_tx_rate > actual_link_speed))
9886 		return -EINVAL;
9887 
9888 	adapter->vf_rate_link_speed = actual_link_speed;
9889 	adapter->vf_data[vf].tx_rate = (u16)max_tx_rate;
9890 	igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed);
9891 
9892 	return 0;
9893 }
9894 
igb_ndo_set_vf_spoofchk(struct net_device * netdev,int vf,bool setting)9895 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
9896 				   bool setting)
9897 {
9898 	struct igb_adapter *adapter = netdev_priv(netdev);
9899 	struct e1000_hw *hw = &adapter->hw;
9900 	u32 reg_val, reg_offset;
9901 
9902 	if (!adapter->vfs_allocated_count)
9903 		return -EOPNOTSUPP;
9904 
9905 	if (vf >= adapter->vfs_allocated_count)
9906 		return -EINVAL;
9907 
9908 	reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
9909 	reg_val = rd32(reg_offset);
9910 	if (setting)
9911 		reg_val |= (BIT(vf) |
9912 			    BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9913 	else
9914 		reg_val &= ~(BIT(vf) |
9915 			     BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9916 	wr32(reg_offset, reg_val);
9917 
9918 	adapter->vf_data[vf].spoofchk_enabled = setting;
9919 	return 0;
9920 }
9921 
igb_ndo_set_vf_trust(struct net_device * netdev,int vf,bool setting)9922 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting)
9923 {
9924 	struct igb_adapter *adapter = netdev_priv(netdev);
9925 
9926 	if (vf >= adapter->vfs_allocated_count)
9927 		return -EINVAL;
9928 	if (adapter->vf_data[vf].trusted == setting)
9929 		return 0;
9930 
9931 	adapter->vf_data[vf].trusted = setting;
9932 
9933 	dev_info(&adapter->pdev->dev, "VF %u is %strusted\n",
9934 		 vf, setting ? "" : "not ");
9935 	return 0;
9936 }
9937 
igb_ndo_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivi)9938 static int igb_ndo_get_vf_config(struct net_device *netdev,
9939 				 int vf, struct ifla_vf_info *ivi)
9940 {
9941 	struct igb_adapter *adapter = netdev_priv(netdev);
9942 	if (vf >= adapter->vfs_allocated_count)
9943 		return -EINVAL;
9944 	ivi->vf = vf;
9945 	memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
9946 	ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
9947 	ivi->min_tx_rate = 0;
9948 	ivi->vlan = adapter->vf_data[vf].pf_vlan;
9949 	ivi->qos = adapter->vf_data[vf].pf_qos;
9950 	ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
9951 	ivi->trusted = adapter->vf_data[vf].trusted;
9952 	return 0;
9953 }
9954 
igb_vmm_control(struct igb_adapter * adapter)9955 static void igb_vmm_control(struct igb_adapter *adapter)
9956 {
9957 	struct e1000_hw *hw = &adapter->hw;
9958 	u32 reg;
9959 
9960 	switch (hw->mac.type) {
9961 	case e1000_82575:
9962 	case e1000_i210:
9963 	case e1000_i211:
9964 	case e1000_i354:
9965 	default:
9966 		/* replication is not supported for 82575 */
9967 		return;
9968 	case e1000_82576:
9969 		/* notify HW that the MAC is adding vlan tags */
9970 		reg = rd32(E1000_DTXCTL);
9971 		reg |= E1000_DTXCTL_VLAN_ADDED;
9972 		wr32(E1000_DTXCTL, reg);
9973 		fallthrough;
9974 	case e1000_82580:
9975 		/* enable replication vlan tag stripping */
9976 		reg = rd32(E1000_RPLOLR);
9977 		reg |= E1000_RPLOLR_STRVLAN;
9978 		wr32(E1000_RPLOLR, reg);
9979 		fallthrough;
9980 	case e1000_i350:
9981 		/* none of the above registers are supported by i350 */
9982 		break;
9983 	}
9984 
9985 	if (adapter->vfs_allocated_count) {
9986 		igb_vmdq_set_loopback_pf(hw, true);
9987 		igb_vmdq_set_replication_pf(hw, true);
9988 		igb_vmdq_set_anti_spoofing_pf(hw, true,
9989 					      adapter->vfs_allocated_count);
9990 	} else {
9991 		igb_vmdq_set_loopback_pf(hw, false);
9992 		igb_vmdq_set_replication_pf(hw, false);
9993 	}
9994 }
9995 
igb_init_dmac(struct igb_adapter * adapter,u32 pba)9996 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
9997 {
9998 	struct e1000_hw *hw = &adapter->hw;
9999 	u32 dmac_thr;
10000 	u16 hwm;
10001 	u32 reg;
10002 
10003 	if (hw->mac.type > e1000_82580) {
10004 		if (adapter->flags & IGB_FLAG_DMAC) {
10005 			/* force threshold to 0. */
10006 			wr32(E1000_DMCTXTH, 0);
10007 
10008 			/* DMA Coalescing high water mark needs to be greater
10009 			 * than the Rx threshold. Set hwm to PBA - max frame
10010 			 * size in 16B units, capping it at PBA - 6KB.
10011 			 */
10012 			hwm = 64 * (pba - 6);
10013 			reg = rd32(E1000_FCRTC);
10014 			reg &= ~E1000_FCRTC_RTH_COAL_MASK;
10015 			reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
10016 				& E1000_FCRTC_RTH_COAL_MASK);
10017 			wr32(E1000_FCRTC, reg);
10018 
10019 			/* Set the DMA Coalescing Rx threshold to PBA - 2 * max
10020 			 * frame size, capping it at PBA - 10KB.
10021 			 */
10022 			dmac_thr = pba - 10;
10023 			reg = rd32(E1000_DMACR);
10024 			reg &= ~E1000_DMACR_DMACTHR_MASK;
10025 			reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
10026 				& E1000_DMACR_DMACTHR_MASK);
10027 
10028 			/* transition to L0x or L1 if available..*/
10029 			reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
10030 
10031 			/* watchdog timer= +-1000 usec in 32usec intervals */
10032 			reg |= (1000 >> 5);
10033 
10034 			/* Disable BMC-to-OS Watchdog Enable */
10035 			if (hw->mac.type != e1000_i354)
10036 				reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
10037 			wr32(E1000_DMACR, reg);
10038 
10039 			/* no lower threshold to disable
10040 			 * coalescing(smart fifb)-UTRESH=0
10041 			 */
10042 			wr32(E1000_DMCRTRH, 0);
10043 
10044 			reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
10045 
10046 			wr32(E1000_DMCTLX, reg);
10047 
10048 			/* free space in tx packet buffer to wake from
10049 			 * DMA coal
10050 			 */
10051 			wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
10052 			     (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
10053 		}
10054 
10055 		if (hw->mac.type >= e1000_i210 ||
10056 		    (adapter->flags & IGB_FLAG_DMAC)) {
10057 			reg = rd32(E1000_PCIEMISC);
10058 			reg |= E1000_PCIEMISC_LX_DECISION;
10059 			wr32(E1000_PCIEMISC, reg);
10060 		} /* endif adapter->dmac is not disabled */
10061 	} else if (hw->mac.type == e1000_82580) {
10062 		u32 reg = rd32(E1000_PCIEMISC);
10063 
10064 		wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
10065 		wr32(E1000_DMACR, 0);
10066 	}
10067 }
10068 
10069 /**
10070  *  igb_read_i2c_byte - Reads 8 bit word over I2C
10071  *  @hw: pointer to hardware structure
10072  *  @byte_offset: byte offset to read
10073  *  @dev_addr: device address
10074  *  @data: value read
10075  *
10076  *  Performs byte read operation over I2C interface at
10077  *  a specified device address.
10078  **/
igb_read_i2c_byte(struct e1000_hw * hw,u8 byte_offset,u8 dev_addr,u8 * data)10079 s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
10080 		      u8 dev_addr, u8 *data)
10081 {
10082 	struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
10083 	struct i2c_client *this_client = adapter->i2c_client;
10084 	s32 status;
10085 	u16 swfw_mask = 0;
10086 
10087 	if (!this_client)
10088 		return E1000_ERR_I2C;
10089 
10090 	swfw_mask = E1000_SWFW_PHY0_SM;
10091 
10092 	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
10093 		return E1000_ERR_SWFW_SYNC;
10094 
10095 	status = i2c_smbus_read_byte_data(this_client, byte_offset);
10096 	hw->mac.ops.release_swfw_sync(hw, swfw_mask);
10097 
10098 	if (status < 0)
10099 		return E1000_ERR_I2C;
10100 	else {
10101 		*data = status;
10102 		return 0;
10103 	}
10104 }
10105 
10106 /**
10107  *  igb_write_i2c_byte - Writes 8 bit word over I2C
10108  *  @hw: pointer to hardware structure
10109  *  @byte_offset: byte offset to write
10110  *  @dev_addr: device address
10111  *  @data: value to write
10112  *
10113  *  Performs byte write operation over I2C interface at
10114  *  a specified device address.
10115  **/
igb_write_i2c_byte(struct e1000_hw * hw,u8 byte_offset,u8 dev_addr,u8 data)10116 s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
10117 		       u8 dev_addr, u8 data)
10118 {
10119 	struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
10120 	struct i2c_client *this_client = adapter->i2c_client;
10121 	s32 status;
10122 	u16 swfw_mask = E1000_SWFW_PHY0_SM;
10123 
10124 	if (!this_client)
10125 		return E1000_ERR_I2C;
10126 
10127 	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
10128 		return E1000_ERR_SWFW_SYNC;
10129 	status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
10130 	hw->mac.ops.release_swfw_sync(hw, swfw_mask);
10131 
10132 	if (status)
10133 		return E1000_ERR_I2C;
10134 	else
10135 		return 0;
10136 
10137 }
10138 
igb_reinit_queues(struct igb_adapter * adapter)10139 int igb_reinit_queues(struct igb_adapter *adapter)
10140 {
10141 	struct net_device *netdev = adapter->netdev;
10142 	struct pci_dev *pdev = adapter->pdev;
10143 	int err = 0;
10144 
10145 	if (netif_running(netdev))
10146 		igb_close(netdev);
10147 
10148 	igb_reset_interrupt_capability(adapter);
10149 
10150 	if (igb_init_interrupt_scheme(adapter, true)) {
10151 		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
10152 		return -ENOMEM;
10153 	}
10154 
10155 	if (netif_running(netdev))
10156 		err = igb_open(netdev);
10157 
10158 	return err;
10159 }
10160 
igb_nfc_filter_exit(struct igb_adapter * adapter)10161 static void igb_nfc_filter_exit(struct igb_adapter *adapter)
10162 {
10163 	struct igb_nfc_filter *rule;
10164 
10165 	spin_lock(&adapter->nfc_lock);
10166 
10167 	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
10168 		igb_erase_filter(adapter, rule);
10169 
10170 	hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
10171 		igb_erase_filter(adapter, rule);
10172 
10173 	spin_unlock(&adapter->nfc_lock);
10174 }
10175 
igb_nfc_filter_restore(struct igb_adapter * adapter)10176 static void igb_nfc_filter_restore(struct igb_adapter *adapter)
10177 {
10178 	struct igb_nfc_filter *rule;
10179 
10180 	spin_lock(&adapter->nfc_lock);
10181 
10182 	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
10183 		igb_add_filter(adapter, rule);
10184 
10185 	spin_unlock(&adapter->nfc_lock);
10186 }
10187 /* igb_main.c */
10188