1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 1999 - 2006 Intel Corporation. */
3 
4 #include "e1000.h"
5 #include <net/ip6_checksum.h>
6 #include <linux/io.h>
7 #include <linux/prefetch.h>
8 #include <linux/bitops.h>
9 #include <linux/if_vlan.h>
10 
11 char e1000_driver_name[] = "e1000";
12 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
13 #define DRV_VERSION "7.3.21-k8-NAPI"
14 const char e1000_driver_version[] = DRV_VERSION;
15 static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
16 
17 /* e1000_pci_tbl - PCI Device ID Table
18  *
19  * Last entry must be all 0s
20  *
21  * Macro expands to...
22  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
23  */
24 static const struct pci_device_id e1000_pci_tbl[] = {
25 	INTEL_E1000_ETHERNET_DEVICE(0x1000),
26 	INTEL_E1000_ETHERNET_DEVICE(0x1001),
27 	INTEL_E1000_ETHERNET_DEVICE(0x1004),
28 	INTEL_E1000_ETHERNET_DEVICE(0x1008),
29 	INTEL_E1000_ETHERNET_DEVICE(0x1009),
30 	INTEL_E1000_ETHERNET_DEVICE(0x100C),
31 	INTEL_E1000_ETHERNET_DEVICE(0x100D),
32 	INTEL_E1000_ETHERNET_DEVICE(0x100E),
33 	INTEL_E1000_ETHERNET_DEVICE(0x100F),
34 	INTEL_E1000_ETHERNET_DEVICE(0x1010),
35 	INTEL_E1000_ETHERNET_DEVICE(0x1011),
36 	INTEL_E1000_ETHERNET_DEVICE(0x1012),
37 	INTEL_E1000_ETHERNET_DEVICE(0x1013),
38 	INTEL_E1000_ETHERNET_DEVICE(0x1014),
39 	INTEL_E1000_ETHERNET_DEVICE(0x1015),
40 	INTEL_E1000_ETHERNET_DEVICE(0x1016),
41 	INTEL_E1000_ETHERNET_DEVICE(0x1017),
42 	INTEL_E1000_ETHERNET_DEVICE(0x1018),
43 	INTEL_E1000_ETHERNET_DEVICE(0x1019),
44 	INTEL_E1000_ETHERNET_DEVICE(0x101A),
45 	INTEL_E1000_ETHERNET_DEVICE(0x101D),
46 	INTEL_E1000_ETHERNET_DEVICE(0x101E),
47 	INTEL_E1000_ETHERNET_DEVICE(0x1026),
48 	INTEL_E1000_ETHERNET_DEVICE(0x1027),
49 	INTEL_E1000_ETHERNET_DEVICE(0x1028),
50 	INTEL_E1000_ETHERNET_DEVICE(0x1075),
51 	INTEL_E1000_ETHERNET_DEVICE(0x1076),
52 	INTEL_E1000_ETHERNET_DEVICE(0x1077),
53 	INTEL_E1000_ETHERNET_DEVICE(0x1078),
54 	INTEL_E1000_ETHERNET_DEVICE(0x1079),
55 	INTEL_E1000_ETHERNET_DEVICE(0x107A),
56 	INTEL_E1000_ETHERNET_DEVICE(0x107B),
57 	INTEL_E1000_ETHERNET_DEVICE(0x107C),
58 	INTEL_E1000_ETHERNET_DEVICE(0x108A),
59 	INTEL_E1000_ETHERNET_DEVICE(0x1099),
60 	INTEL_E1000_ETHERNET_DEVICE(0x10B5),
61 	INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
62 	/* required last entry */
63 	{0,}
64 };
65 
66 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
67 
68 int e1000_up(struct e1000_adapter *adapter);
69 void e1000_down(struct e1000_adapter *adapter);
70 void e1000_reinit_locked(struct e1000_adapter *adapter);
71 void e1000_reset(struct e1000_adapter *adapter);
72 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
73 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
74 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
75 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
76 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
77 				    struct e1000_tx_ring *txdr);
78 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
79 				    struct e1000_rx_ring *rxdr);
80 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
81 				    struct e1000_tx_ring *tx_ring);
82 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
83 				    struct e1000_rx_ring *rx_ring);
84 void e1000_update_stats(struct e1000_adapter *adapter);
85 
86 static int e1000_init_module(void);
87 static void e1000_exit_module(void);
88 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
89 static void e1000_remove(struct pci_dev *pdev);
90 static int e1000_alloc_queues(struct e1000_adapter *adapter);
91 static int e1000_sw_init(struct e1000_adapter *adapter);
92 int e1000_open(struct net_device *netdev);
93 int e1000_close(struct net_device *netdev);
94 static void e1000_configure_tx(struct e1000_adapter *adapter);
95 static void e1000_configure_rx(struct e1000_adapter *adapter);
96 static void e1000_setup_rctl(struct e1000_adapter *adapter);
97 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
98 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
99 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
100 				struct e1000_tx_ring *tx_ring);
101 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
102 				struct e1000_rx_ring *rx_ring);
103 static void e1000_set_rx_mode(struct net_device *netdev);
104 static void e1000_update_phy_info_task(struct work_struct *work);
105 static void e1000_watchdog(struct work_struct *work);
106 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
107 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
108 				    struct net_device *netdev);
109 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
110 static int e1000_set_mac(struct net_device *netdev, void *p);
111 static irqreturn_t e1000_intr(int irq, void *data);
112 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
113 			       struct e1000_tx_ring *tx_ring);
114 static int e1000_clean(struct napi_struct *napi, int budget);
115 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
116 			       struct e1000_rx_ring *rx_ring,
117 			       int *work_done, int work_to_do);
118 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
119 				     struct e1000_rx_ring *rx_ring,
120 				     int *work_done, int work_to_do);
121 static void e1000_alloc_dummy_rx_buffers(struct e1000_adapter *adapter,
122 					 struct e1000_rx_ring *rx_ring,
123 					 int cleaned_count)
124 {
125 }
126 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
127 				   struct e1000_rx_ring *rx_ring,
128 				   int cleaned_count);
129 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
130 					 struct e1000_rx_ring *rx_ring,
131 					 int cleaned_count);
132 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
133 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
134 			   int cmd);
135 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
136 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
137 static void e1000_tx_timeout(struct net_device *dev);
138 static void e1000_reset_task(struct work_struct *work);
139 static void e1000_smartspeed(struct e1000_adapter *adapter);
140 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
141 				       struct sk_buff *skb);
142 
143 static bool e1000_vlan_used(struct e1000_adapter *adapter);
144 static void e1000_vlan_mode(struct net_device *netdev,
145 			    netdev_features_t features);
146 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
147 				     bool filter_on);
148 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
149 				 __be16 proto, u16 vid);
150 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
151 				  __be16 proto, u16 vid);
152 static void e1000_restore_vlan(struct e1000_adapter *adapter);
153 
154 #ifdef CONFIG_PM
155 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state);
156 static int e1000_resume(struct pci_dev *pdev);
157 #endif
158 static void e1000_shutdown(struct pci_dev *pdev);
159 
160 #ifdef CONFIG_NET_POLL_CONTROLLER
161 /* for netdump / net console */
162 static void e1000_netpoll (struct net_device *netdev);
163 #endif
164 
165 #define COPYBREAK_DEFAULT 256
166 static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
167 module_param(copybreak, uint, 0644);
168 MODULE_PARM_DESC(copybreak,
169 	"Maximum size of packet that is copied to a new buffer on receive");
170 
171 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
172 						pci_channel_state_t state);
173 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
174 static void e1000_io_resume(struct pci_dev *pdev);
175 
176 static const struct pci_error_handlers e1000_err_handler = {
177 	.error_detected = e1000_io_error_detected,
178 	.slot_reset = e1000_io_slot_reset,
179 	.resume = e1000_io_resume,
180 };
181 
182 static struct pci_driver e1000_driver = {
183 	.name     = e1000_driver_name,
184 	.id_table = e1000_pci_tbl,
185 	.probe    = e1000_probe,
186 	.remove   = e1000_remove,
187 #ifdef CONFIG_PM
188 	/* Power Management Hooks */
189 	.suspend  = e1000_suspend,
190 	.resume   = e1000_resume,
191 #endif
192 	.shutdown = e1000_shutdown,
193 	.err_handler = &e1000_err_handler
194 };
195 
196 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
197 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
198 MODULE_LICENSE("GPL v2");
199 MODULE_VERSION(DRV_VERSION);
200 
201 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
202 static int debug = -1;
203 module_param(debug, int, 0);
204 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
205 
206 /**
207  * e1000_get_hw_dev - return device
208  * used by hardware layer to print debugging information
209  *
210  **/
211 struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
212 {
213 	struct e1000_adapter *adapter = hw->back;
214 	return adapter->netdev;
215 }
216 
217 /**
218  * e1000_init_module - Driver Registration Routine
219  *
220  * e1000_init_module is the first routine called when the driver is
221  * loaded. All it does is register with the PCI subsystem.
222  **/
223 static int __init e1000_init_module(void)
224 {
225 	int ret;
226 	pr_info("%s - version %s\n", e1000_driver_string, e1000_driver_version);
227 
228 	pr_info("%s\n", e1000_copyright);
229 
230 	ret = pci_register_driver(&e1000_driver);
231 	if (copybreak != COPYBREAK_DEFAULT) {
232 		if (copybreak == 0)
233 			pr_info("copybreak disabled\n");
234 		else
235 			pr_info("copybreak enabled for "
236 				   "packets <= %u bytes\n", copybreak);
237 	}
238 	return ret;
239 }
240 
241 module_init(e1000_init_module);
242 
243 /**
244  * e1000_exit_module - Driver Exit Cleanup Routine
245  *
246  * e1000_exit_module is called just before the driver is removed
247  * from memory.
248  **/
249 static void __exit e1000_exit_module(void)
250 {
251 	pci_unregister_driver(&e1000_driver);
252 }
253 
254 module_exit(e1000_exit_module);
255 
256 static int e1000_request_irq(struct e1000_adapter *adapter)
257 {
258 	struct net_device *netdev = adapter->netdev;
259 	irq_handler_t handler = e1000_intr;
260 	int irq_flags = IRQF_SHARED;
261 	int err;
262 
263 	err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
264 			  netdev);
265 	if (err) {
266 		e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
267 	}
268 
269 	return err;
270 }
271 
272 static void e1000_free_irq(struct e1000_adapter *adapter)
273 {
274 	struct net_device *netdev = adapter->netdev;
275 
276 	free_irq(adapter->pdev->irq, netdev);
277 }
278 
279 /**
280  * e1000_irq_disable - Mask off interrupt generation on the NIC
281  * @adapter: board private structure
282  **/
283 static void e1000_irq_disable(struct e1000_adapter *adapter)
284 {
285 	struct e1000_hw *hw = &adapter->hw;
286 
287 	ew32(IMC, ~0);
288 	E1000_WRITE_FLUSH();
289 	synchronize_irq(adapter->pdev->irq);
290 }
291 
292 /**
293  * e1000_irq_enable - Enable default interrupt generation settings
294  * @adapter: board private structure
295  **/
296 static void e1000_irq_enable(struct e1000_adapter *adapter)
297 {
298 	struct e1000_hw *hw = &adapter->hw;
299 
300 	ew32(IMS, IMS_ENABLE_MASK);
301 	E1000_WRITE_FLUSH();
302 }
303 
304 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
305 {
306 	struct e1000_hw *hw = &adapter->hw;
307 	struct net_device *netdev = adapter->netdev;
308 	u16 vid = hw->mng_cookie.vlan_id;
309 	u16 old_vid = adapter->mng_vlan_id;
310 
311 	if (!e1000_vlan_used(adapter))
312 		return;
313 
314 	if (!test_bit(vid, adapter->active_vlans)) {
315 		if (hw->mng_cookie.status &
316 		    E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
317 			e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
318 			adapter->mng_vlan_id = vid;
319 		} else {
320 			adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
321 		}
322 		if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
323 		    (vid != old_vid) &&
324 		    !test_bit(old_vid, adapter->active_vlans))
325 			e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
326 					       old_vid);
327 	} else {
328 		adapter->mng_vlan_id = vid;
329 	}
330 }
331 
332 static void e1000_init_manageability(struct e1000_adapter *adapter)
333 {
334 	struct e1000_hw *hw = &adapter->hw;
335 
336 	if (adapter->en_mng_pt) {
337 		u32 manc = er32(MANC);
338 
339 		/* disable hardware interception of ARP */
340 		manc &= ~(E1000_MANC_ARP_EN);
341 
342 		ew32(MANC, manc);
343 	}
344 }
345 
346 static void e1000_release_manageability(struct e1000_adapter *adapter)
347 {
348 	struct e1000_hw *hw = &adapter->hw;
349 
350 	if (adapter->en_mng_pt) {
351 		u32 manc = er32(MANC);
352 
353 		/* re-enable hardware interception of ARP */
354 		manc |= E1000_MANC_ARP_EN;
355 
356 		ew32(MANC, manc);
357 	}
358 }
359 
360 /**
361  * e1000_configure - configure the hardware for RX and TX
362  * @adapter = private board structure
363  **/
364 static void e1000_configure(struct e1000_adapter *adapter)
365 {
366 	struct net_device *netdev = adapter->netdev;
367 	int i;
368 
369 	e1000_set_rx_mode(netdev);
370 
371 	e1000_restore_vlan(adapter);
372 	e1000_init_manageability(adapter);
373 
374 	e1000_configure_tx(adapter);
375 	e1000_setup_rctl(adapter);
376 	e1000_configure_rx(adapter);
377 	/* call E1000_DESC_UNUSED which always leaves
378 	 * at least 1 descriptor unused to make sure
379 	 * next_to_use != next_to_clean
380 	 */
381 	for (i = 0; i < adapter->num_rx_queues; i++) {
382 		struct e1000_rx_ring *ring = &adapter->rx_ring[i];
383 		adapter->alloc_rx_buf(adapter, ring,
384 				      E1000_DESC_UNUSED(ring));
385 	}
386 }
387 
388 int e1000_up(struct e1000_adapter *adapter)
389 {
390 	struct e1000_hw *hw = &adapter->hw;
391 
392 	/* hardware has been reset, we need to reload some things */
393 	e1000_configure(adapter);
394 
395 	clear_bit(__E1000_DOWN, &adapter->flags);
396 
397 	napi_enable(&adapter->napi);
398 
399 	e1000_irq_enable(adapter);
400 
401 	netif_wake_queue(adapter->netdev);
402 
403 	/* fire a link change interrupt to start the watchdog */
404 	ew32(ICS, E1000_ICS_LSC);
405 	return 0;
406 }
407 
408 /**
409  * e1000_power_up_phy - restore link in case the phy was powered down
410  * @adapter: address of board private structure
411  *
412  * The phy may be powered down to save power and turn off link when the
413  * driver is unloaded and wake on lan is not enabled (among others)
414  * *** this routine MUST be followed by a call to e1000_reset ***
415  **/
416 void e1000_power_up_phy(struct e1000_adapter *adapter)
417 {
418 	struct e1000_hw *hw = &adapter->hw;
419 	u16 mii_reg = 0;
420 
421 	/* Just clear the power down bit to wake the phy back up */
422 	if (hw->media_type == e1000_media_type_copper) {
423 		/* according to the manual, the phy will retain its
424 		 * settings across a power-down/up cycle
425 		 */
426 		e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
427 		mii_reg &= ~MII_CR_POWER_DOWN;
428 		e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
429 	}
430 }
431 
432 static void e1000_power_down_phy(struct e1000_adapter *adapter)
433 {
434 	struct e1000_hw *hw = &adapter->hw;
435 
436 	/* Power down the PHY so no link is implied when interface is down *
437 	 * The PHY cannot be powered down if any of the following is true *
438 	 * (a) WoL is enabled
439 	 * (b) AMT is active
440 	 * (c) SoL/IDER session is active
441 	 */
442 	if (!adapter->wol && hw->mac_type >= e1000_82540 &&
443 	   hw->media_type == e1000_media_type_copper) {
444 		u16 mii_reg = 0;
445 
446 		switch (hw->mac_type) {
447 		case e1000_82540:
448 		case e1000_82545:
449 		case e1000_82545_rev_3:
450 		case e1000_82546:
451 		case e1000_ce4100:
452 		case e1000_82546_rev_3:
453 		case e1000_82541:
454 		case e1000_82541_rev_2:
455 		case e1000_82547:
456 		case e1000_82547_rev_2:
457 			if (er32(MANC) & E1000_MANC_SMBUS_EN)
458 				goto out;
459 			break;
460 		default:
461 			goto out;
462 		}
463 		e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
464 		mii_reg |= MII_CR_POWER_DOWN;
465 		e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
466 		msleep(1);
467 	}
468 out:
469 	return;
470 }
471 
472 static void e1000_down_and_stop(struct e1000_adapter *adapter)
473 {
474 	set_bit(__E1000_DOWN, &adapter->flags);
475 
476 	cancel_delayed_work_sync(&adapter->watchdog_task);
477 
478 	/*
479 	 * Since the watchdog task can reschedule other tasks, we should cancel
480 	 * it first, otherwise we can run into the situation when a work is
481 	 * still running after the adapter has been turned down.
482 	 */
483 
484 	cancel_delayed_work_sync(&adapter->phy_info_task);
485 	cancel_delayed_work_sync(&adapter->fifo_stall_task);
486 
487 	/* Only kill reset task if adapter is not resetting */
488 	if (!test_bit(__E1000_RESETTING, &adapter->flags))
489 		cancel_work_sync(&adapter->reset_task);
490 }
491 
492 void e1000_down(struct e1000_adapter *adapter)
493 {
494 	struct e1000_hw *hw = &adapter->hw;
495 	struct net_device *netdev = adapter->netdev;
496 	u32 rctl, tctl;
497 
498 	/* disable receives in the hardware */
499 	rctl = er32(RCTL);
500 	ew32(RCTL, rctl & ~E1000_RCTL_EN);
501 	/* flush and sleep below */
502 
503 	netif_tx_disable(netdev);
504 
505 	/* disable transmits in the hardware */
506 	tctl = er32(TCTL);
507 	tctl &= ~E1000_TCTL_EN;
508 	ew32(TCTL, tctl);
509 	/* flush both disables and wait for them to finish */
510 	E1000_WRITE_FLUSH();
511 	msleep(10);
512 
513 	/* Set the carrier off after transmits have been disabled in the
514 	 * hardware, to avoid race conditions with e1000_watchdog() (which
515 	 * may be running concurrently to us, checking for the carrier
516 	 * bit to decide whether it should enable transmits again). Such
517 	 * a race condition would result into transmission being disabled
518 	 * in the hardware until the next IFF_DOWN+IFF_UP cycle.
519 	 */
520 	netif_carrier_off(netdev);
521 
522 	napi_disable(&adapter->napi);
523 
524 	e1000_irq_disable(adapter);
525 
526 	/* Setting DOWN must be after irq_disable to prevent
527 	 * a screaming interrupt.  Setting DOWN also prevents
528 	 * tasks from rescheduling.
529 	 */
530 	e1000_down_and_stop(adapter);
531 
532 	adapter->link_speed = 0;
533 	adapter->link_duplex = 0;
534 
535 	e1000_reset(adapter);
536 	e1000_clean_all_tx_rings(adapter);
537 	e1000_clean_all_rx_rings(adapter);
538 }
539 
540 void e1000_reinit_locked(struct e1000_adapter *adapter)
541 {
542 	WARN_ON(in_interrupt());
543 	while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
544 		msleep(1);
545 	e1000_down(adapter);
546 	e1000_up(adapter);
547 	clear_bit(__E1000_RESETTING, &adapter->flags);
548 }
549 
550 void e1000_reset(struct e1000_adapter *adapter)
551 {
552 	struct e1000_hw *hw = &adapter->hw;
553 	u32 pba = 0, tx_space, min_tx_space, min_rx_space;
554 	bool legacy_pba_adjust = false;
555 	u16 hwm;
556 
557 	/* Repartition Pba for greater than 9k mtu
558 	 * To take effect CTRL.RST is required.
559 	 */
560 
561 	switch (hw->mac_type) {
562 	case e1000_82542_rev2_0:
563 	case e1000_82542_rev2_1:
564 	case e1000_82543:
565 	case e1000_82544:
566 	case e1000_82540:
567 	case e1000_82541:
568 	case e1000_82541_rev_2:
569 		legacy_pba_adjust = true;
570 		pba = E1000_PBA_48K;
571 		break;
572 	case e1000_82545:
573 	case e1000_82545_rev_3:
574 	case e1000_82546:
575 	case e1000_ce4100:
576 	case e1000_82546_rev_3:
577 		pba = E1000_PBA_48K;
578 		break;
579 	case e1000_82547:
580 	case e1000_82547_rev_2:
581 		legacy_pba_adjust = true;
582 		pba = E1000_PBA_30K;
583 		break;
584 	case e1000_undefined:
585 	case e1000_num_macs:
586 		break;
587 	}
588 
589 	if (legacy_pba_adjust) {
590 		if (hw->max_frame_size > E1000_RXBUFFER_8192)
591 			pba -= 8; /* allocate more FIFO for Tx */
592 
593 		if (hw->mac_type == e1000_82547) {
594 			adapter->tx_fifo_head = 0;
595 			adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
596 			adapter->tx_fifo_size =
597 				(E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
598 			atomic_set(&adapter->tx_fifo_stall, 0);
599 		}
600 	} else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
601 		/* adjust PBA for jumbo frames */
602 		ew32(PBA, pba);
603 
604 		/* To maintain wire speed transmits, the Tx FIFO should be
605 		 * large enough to accommodate two full transmit packets,
606 		 * rounded up to the next 1KB and expressed in KB.  Likewise,
607 		 * the Rx FIFO should be large enough to accommodate at least
608 		 * one full receive packet and is similarly rounded up and
609 		 * expressed in KB.
610 		 */
611 		pba = er32(PBA);
612 		/* upper 16 bits has Tx packet buffer allocation size in KB */
613 		tx_space = pba >> 16;
614 		/* lower 16 bits has Rx packet buffer allocation size in KB */
615 		pba &= 0xffff;
616 		/* the Tx fifo also stores 16 bytes of information about the Tx
617 		 * but don't include ethernet FCS because hardware appends it
618 		 */
619 		min_tx_space = (hw->max_frame_size +
620 				sizeof(struct e1000_tx_desc) -
621 				ETH_FCS_LEN) * 2;
622 		min_tx_space = ALIGN(min_tx_space, 1024);
623 		min_tx_space >>= 10;
624 		/* software strips receive CRC, so leave room for it */
625 		min_rx_space = hw->max_frame_size;
626 		min_rx_space = ALIGN(min_rx_space, 1024);
627 		min_rx_space >>= 10;
628 
629 		/* If current Tx allocation is less than the min Tx FIFO size,
630 		 * and the min Tx FIFO size is less than the current Rx FIFO
631 		 * allocation, take space away from current Rx allocation
632 		 */
633 		if (tx_space < min_tx_space &&
634 		    ((min_tx_space - tx_space) < pba)) {
635 			pba = pba - (min_tx_space - tx_space);
636 
637 			/* PCI/PCIx hardware has PBA alignment constraints */
638 			switch (hw->mac_type) {
639 			case e1000_82545 ... e1000_82546_rev_3:
640 				pba &= ~(E1000_PBA_8K - 1);
641 				break;
642 			default:
643 				break;
644 			}
645 
646 			/* if short on Rx space, Rx wins and must trump Tx
647 			 * adjustment or use Early Receive if available
648 			 */
649 			if (pba < min_rx_space)
650 				pba = min_rx_space;
651 		}
652 	}
653 
654 	ew32(PBA, pba);
655 
656 	/* flow control settings:
657 	 * The high water mark must be low enough to fit one full frame
658 	 * (or the size used for early receive) above it in the Rx FIFO.
659 	 * Set it to the lower of:
660 	 * - 90% of the Rx FIFO size, and
661 	 * - the full Rx FIFO size minus the early receive size (for parts
662 	 *   with ERT support assuming ERT set to E1000_ERT_2048), or
663 	 * - the full Rx FIFO size minus one full frame
664 	 */
665 	hwm = min(((pba << 10) * 9 / 10),
666 		  ((pba << 10) - hw->max_frame_size));
667 
668 	hw->fc_high_water = hwm & 0xFFF8;	/* 8-byte granularity */
669 	hw->fc_low_water = hw->fc_high_water - 8;
670 	hw->fc_pause_time = E1000_FC_PAUSE_TIME;
671 	hw->fc_send_xon = 1;
672 	hw->fc = hw->original_fc;
673 
674 	/* Allow time for pending master requests to run */
675 	e1000_reset_hw(hw);
676 	if (hw->mac_type >= e1000_82544)
677 		ew32(WUC, 0);
678 
679 	if (e1000_init_hw(hw))
680 		e_dev_err("Hardware Error\n");
681 	e1000_update_mng_vlan(adapter);
682 
683 	/* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
684 	if (hw->mac_type >= e1000_82544 &&
685 	    hw->autoneg == 1 &&
686 	    hw->autoneg_advertised == ADVERTISE_1000_FULL) {
687 		u32 ctrl = er32(CTRL);
688 		/* clear phy power management bit if we are in gig only mode,
689 		 * which if enabled will attempt negotiation to 100Mb, which
690 		 * can cause a loss of link at power off or driver unload
691 		 */
692 		ctrl &= ~E1000_CTRL_SWDPIN3;
693 		ew32(CTRL, ctrl);
694 	}
695 
696 	/* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
697 	ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
698 
699 	e1000_reset_adaptive(hw);
700 	e1000_phy_get_info(hw, &adapter->phy_info);
701 
702 	e1000_release_manageability(adapter);
703 }
704 
705 /* Dump the eeprom for users having checksum issues */
706 static void e1000_dump_eeprom(struct e1000_adapter *adapter)
707 {
708 	struct net_device *netdev = adapter->netdev;
709 	struct ethtool_eeprom eeprom;
710 	const struct ethtool_ops *ops = netdev->ethtool_ops;
711 	u8 *data;
712 	int i;
713 	u16 csum_old, csum_new = 0;
714 
715 	eeprom.len = ops->get_eeprom_len(netdev);
716 	eeprom.offset = 0;
717 
718 	data = kmalloc(eeprom.len, GFP_KERNEL);
719 	if (!data)
720 		return;
721 
722 	ops->get_eeprom(netdev, &eeprom, data);
723 
724 	csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
725 		   (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
726 	for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
727 		csum_new += data[i] + (data[i + 1] << 8);
728 	csum_new = EEPROM_SUM - csum_new;
729 
730 	pr_err("/*********************/\n");
731 	pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
732 	pr_err("Calculated              : 0x%04x\n", csum_new);
733 
734 	pr_err("Offset    Values\n");
735 	pr_err("========  ======\n");
736 	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);
737 
738 	pr_err("Include this output when contacting your support provider.\n");
739 	pr_err("This is not a software error! Something bad happened to\n");
740 	pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
741 	pr_err("result in further problems, possibly loss of data,\n");
742 	pr_err("corruption or system hangs!\n");
743 	pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
744 	pr_err("which is invalid and requires you to set the proper MAC\n");
745 	pr_err("address manually before continuing to enable this network\n");
746 	pr_err("device. Please inspect the EEPROM dump and report the\n");
747 	pr_err("issue to your hardware vendor or Intel Customer Support.\n");
748 	pr_err("/*********************/\n");
749 
750 	kfree(data);
751 }
752 
753 /**
754  * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
755  * @pdev: PCI device information struct
756  *
757  * Return true if an adapter needs ioport resources
758  **/
759 static int e1000_is_need_ioport(struct pci_dev *pdev)
760 {
761 	switch (pdev->device) {
762 	case E1000_DEV_ID_82540EM:
763 	case E1000_DEV_ID_82540EM_LOM:
764 	case E1000_DEV_ID_82540EP:
765 	case E1000_DEV_ID_82540EP_LOM:
766 	case E1000_DEV_ID_82540EP_LP:
767 	case E1000_DEV_ID_82541EI:
768 	case E1000_DEV_ID_82541EI_MOBILE:
769 	case E1000_DEV_ID_82541ER:
770 	case E1000_DEV_ID_82541ER_LOM:
771 	case E1000_DEV_ID_82541GI:
772 	case E1000_DEV_ID_82541GI_LF:
773 	case E1000_DEV_ID_82541GI_MOBILE:
774 	case E1000_DEV_ID_82544EI_COPPER:
775 	case E1000_DEV_ID_82544EI_FIBER:
776 	case E1000_DEV_ID_82544GC_COPPER:
777 	case E1000_DEV_ID_82544GC_LOM:
778 	case E1000_DEV_ID_82545EM_COPPER:
779 	case E1000_DEV_ID_82545EM_FIBER:
780 	case E1000_DEV_ID_82546EB_COPPER:
781 	case E1000_DEV_ID_82546EB_FIBER:
782 	case E1000_DEV_ID_82546EB_QUAD_COPPER:
783 		return true;
784 	default:
785 		return false;
786 	}
787 }
788 
789 static netdev_features_t e1000_fix_features(struct net_device *netdev,
790 	netdev_features_t features)
791 {
792 	/* Since there is no support for separate Rx/Tx vlan accel
793 	 * enable/disable make sure Tx flag is always in same state as Rx.
794 	 */
795 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
796 		features |= NETIF_F_HW_VLAN_CTAG_TX;
797 	else
798 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
799 
800 	return features;
801 }
802 
803 static int e1000_set_features(struct net_device *netdev,
804 	netdev_features_t features)
805 {
806 	struct e1000_adapter *adapter = netdev_priv(netdev);
807 	netdev_features_t changed = features ^ netdev->features;
808 
809 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
810 		e1000_vlan_mode(netdev, features);
811 
812 	if (!(changed & (NETIF_F_RXCSUM | NETIF_F_RXALL)))
813 		return 0;
814 
815 	netdev->features = features;
816 	adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
817 
818 	if (netif_running(netdev))
819 		e1000_reinit_locked(adapter);
820 	else
821 		e1000_reset(adapter);
822 
823 	return 1;
824 }
825 
826 static const struct net_device_ops e1000_netdev_ops = {
827 	.ndo_open		= e1000_open,
828 	.ndo_stop		= e1000_close,
829 	.ndo_start_xmit		= e1000_xmit_frame,
830 	.ndo_set_rx_mode	= e1000_set_rx_mode,
831 	.ndo_set_mac_address	= e1000_set_mac,
832 	.ndo_tx_timeout		= e1000_tx_timeout,
833 	.ndo_change_mtu		= e1000_change_mtu,
834 	.ndo_do_ioctl		= e1000_ioctl,
835 	.ndo_validate_addr	= eth_validate_addr,
836 	.ndo_vlan_rx_add_vid	= e1000_vlan_rx_add_vid,
837 	.ndo_vlan_rx_kill_vid	= e1000_vlan_rx_kill_vid,
838 #ifdef CONFIG_NET_POLL_CONTROLLER
839 	.ndo_poll_controller	= e1000_netpoll,
840 #endif
841 	.ndo_fix_features	= e1000_fix_features,
842 	.ndo_set_features	= e1000_set_features,
843 };
844 
845 /**
846  * e1000_init_hw_struct - initialize members of hw struct
847  * @adapter: board private struct
848  * @hw: structure used by e1000_hw.c
849  *
850  * Factors out initialization of the e1000_hw struct to its own function
851  * that can be called very early at init (just after struct allocation).
852  * Fields are initialized based on PCI device information and
853  * OS network device settings (MTU size).
854  * Returns negative error codes if MAC type setup fails.
855  */
856 static int e1000_init_hw_struct(struct e1000_adapter *adapter,
857 				struct e1000_hw *hw)
858 {
859 	struct pci_dev *pdev = adapter->pdev;
860 
861 	/* PCI config space info */
862 	hw->vendor_id = pdev->vendor;
863 	hw->device_id = pdev->device;
864 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
865 	hw->subsystem_id = pdev->subsystem_device;
866 	hw->revision_id = pdev->revision;
867 
868 	pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
869 
870 	hw->max_frame_size = adapter->netdev->mtu +
871 			     ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
872 	hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
873 
874 	/* identify the MAC */
875 	if (e1000_set_mac_type(hw)) {
876 		e_err(probe, "Unknown MAC Type\n");
877 		return -EIO;
878 	}
879 
880 	switch (hw->mac_type) {
881 	default:
882 		break;
883 	case e1000_82541:
884 	case e1000_82547:
885 	case e1000_82541_rev_2:
886 	case e1000_82547_rev_2:
887 		hw->phy_init_script = 1;
888 		break;
889 	}
890 
891 	e1000_set_media_type(hw);
892 	e1000_get_bus_info(hw);
893 
894 	hw->wait_autoneg_complete = false;
895 	hw->tbi_compatibility_en = true;
896 	hw->adaptive_ifs = true;
897 
898 	/* Copper options */
899 
900 	if (hw->media_type == e1000_media_type_copper) {
901 		hw->mdix = AUTO_ALL_MODES;
902 		hw->disable_polarity_correction = false;
903 		hw->master_slave = E1000_MASTER_SLAVE;
904 	}
905 
906 	return 0;
907 }
908 
909 /**
910  * e1000_probe - Device Initialization Routine
911  * @pdev: PCI device information struct
912  * @ent: entry in e1000_pci_tbl
913  *
914  * Returns 0 on success, negative on failure
915  *
916  * e1000_probe initializes an adapter identified by a pci_dev structure.
917  * The OS initialization, configuring of the adapter private structure,
918  * and a hardware reset occur.
919  **/
920 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
921 {
922 	struct net_device *netdev;
923 	struct e1000_adapter *adapter = NULL;
924 	struct e1000_hw *hw;
925 
926 	static int cards_found;
927 	static int global_quad_port_a; /* global ksp3 port a indication */
928 	int i, err, pci_using_dac;
929 	u16 eeprom_data = 0;
930 	u16 tmp = 0;
931 	u16 eeprom_apme_mask = E1000_EEPROM_APME;
932 	int bars, need_ioport;
933 	bool disable_dev = false;
934 
935 	/* do not allocate ioport bars when not needed */
936 	need_ioport = e1000_is_need_ioport(pdev);
937 	if (need_ioport) {
938 		bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
939 		err = pci_enable_device(pdev);
940 	} else {
941 		bars = pci_select_bars(pdev, IORESOURCE_MEM);
942 		err = pci_enable_device_mem(pdev);
943 	}
944 	if (err)
945 		return err;
946 
947 	err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
948 	if (err)
949 		goto err_pci_reg;
950 
951 	pci_set_master(pdev);
952 	err = pci_save_state(pdev);
953 	if (err)
954 		goto err_alloc_etherdev;
955 
956 	err = -ENOMEM;
957 	netdev = alloc_etherdev(sizeof(struct e1000_adapter));
958 	if (!netdev)
959 		goto err_alloc_etherdev;
960 
961 	SET_NETDEV_DEV(netdev, &pdev->dev);
962 
963 	pci_set_drvdata(pdev, netdev);
964 	adapter = netdev_priv(netdev);
965 	adapter->netdev = netdev;
966 	adapter->pdev = pdev;
967 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
968 	adapter->bars = bars;
969 	adapter->need_ioport = need_ioport;
970 
971 	hw = &adapter->hw;
972 	hw->back = adapter;
973 
974 	err = -EIO;
975 	hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
976 	if (!hw->hw_addr)
977 		goto err_ioremap;
978 
979 	if (adapter->need_ioport) {
980 		for (i = BAR_1; i < PCI_STD_NUM_BARS; i++) {
981 			if (pci_resource_len(pdev, i) == 0)
982 				continue;
983 			if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
984 				hw->io_base = pci_resource_start(pdev, i);
985 				break;
986 			}
987 		}
988 	}
989 
990 	/* make ready for any if (hw->...) below */
991 	err = e1000_init_hw_struct(adapter, hw);
992 	if (err)
993 		goto err_sw_init;
994 
995 	/* there is a workaround being applied below that limits
996 	 * 64-bit DMA addresses to 64-bit hardware.  There are some
997 	 * 32-bit adapters that Tx hang when given 64-bit DMA addresses
998 	 */
999 	pci_using_dac = 0;
1000 	if ((hw->bus_type == e1000_bus_type_pcix) &&
1001 	    !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1002 		pci_using_dac = 1;
1003 	} else {
1004 		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1005 		if (err) {
1006 			pr_err("No usable DMA config, aborting\n");
1007 			goto err_dma;
1008 		}
1009 	}
1010 
1011 	netdev->netdev_ops = &e1000_netdev_ops;
1012 	e1000_set_ethtool_ops(netdev);
1013 	netdev->watchdog_timeo = 5 * HZ;
1014 	netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
1015 
1016 	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1017 
1018 	adapter->bd_number = cards_found;
1019 
1020 	/* setup the private structure */
1021 
1022 	err = e1000_sw_init(adapter);
1023 	if (err)
1024 		goto err_sw_init;
1025 
1026 	err = -EIO;
1027 	if (hw->mac_type == e1000_ce4100) {
1028 		hw->ce4100_gbe_mdio_base_virt =
1029 					ioremap(pci_resource_start(pdev, BAR_1),
1030 						pci_resource_len(pdev, BAR_1));
1031 
1032 		if (!hw->ce4100_gbe_mdio_base_virt)
1033 			goto err_mdio_ioremap;
1034 	}
1035 
1036 	if (hw->mac_type >= e1000_82543) {
1037 		netdev->hw_features = NETIF_F_SG |
1038 				   NETIF_F_HW_CSUM |
1039 				   NETIF_F_HW_VLAN_CTAG_RX;
1040 		netdev->features = NETIF_F_HW_VLAN_CTAG_TX |
1041 				   NETIF_F_HW_VLAN_CTAG_FILTER;
1042 	}
1043 
1044 	if ((hw->mac_type >= e1000_82544) &&
1045 	   (hw->mac_type != e1000_82547))
1046 		netdev->hw_features |= NETIF_F_TSO;
1047 
1048 	netdev->priv_flags |= IFF_SUPP_NOFCS;
1049 
1050 	netdev->features |= netdev->hw_features;
1051 	netdev->hw_features |= (NETIF_F_RXCSUM |
1052 				NETIF_F_RXALL |
1053 				NETIF_F_RXFCS);
1054 
1055 	if (pci_using_dac) {
1056 		netdev->features |= NETIF_F_HIGHDMA;
1057 		netdev->vlan_features |= NETIF_F_HIGHDMA;
1058 	}
1059 
1060 	netdev->vlan_features |= (NETIF_F_TSO |
1061 				  NETIF_F_HW_CSUM |
1062 				  NETIF_F_SG);
1063 
1064 	/* Do not set IFF_UNICAST_FLT for VMWare's 82545EM */
1065 	if (hw->device_id != E1000_DEV_ID_82545EM_COPPER ||
1066 	    hw->subsystem_vendor_id != PCI_VENDOR_ID_VMWARE)
1067 		netdev->priv_flags |= IFF_UNICAST_FLT;
1068 
1069 	/* MTU range: 46 - 16110 */
1070 	netdev->min_mtu = ETH_ZLEN - ETH_HLEN;
1071 	netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
1072 
1073 	adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1074 
1075 	/* initialize eeprom parameters */
1076 	if (e1000_init_eeprom_params(hw)) {
1077 		e_err(probe, "EEPROM initialization failed\n");
1078 		goto err_eeprom;
1079 	}
1080 
1081 	/* before reading the EEPROM, reset the controller to
1082 	 * put the device in a known good starting state
1083 	 */
1084 
1085 	e1000_reset_hw(hw);
1086 
1087 	/* make sure the EEPROM is good */
1088 	if (e1000_validate_eeprom_checksum(hw) < 0) {
1089 		e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1090 		e1000_dump_eeprom(adapter);
1091 		/* set MAC address to all zeroes to invalidate and temporary
1092 		 * disable this device for the user. This blocks regular
1093 		 * traffic while still permitting ethtool ioctls from reaching
1094 		 * the hardware as well as allowing the user to run the
1095 		 * interface after manually setting a hw addr using
1096 		 * `ip set address`
1097 		 */
1098 		memset(hw->mac_addr, 0, netdev->addr_len);
1099 	} else {
1100 		/* copy the MAC address out of the EEPROM */
1101 		if (e1000_read_mac_addr(hw))
1102 			e_err(probe, "EEPROM Read Error\n");
1103 	}
1104 	/* don't block initialization here due to bad MAC address */
1105 	memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
1106 
1107 	if (!is_valid_ether_addr(netdev->dev_addr))
1108 		e_err(probe, "Invalid MAC Address\n");
1109 
1110 
1111 	INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
1112 	INIT_DELAYED_WORK(&adapter->fifo_stall_task,
1113 			  e1000_82547_tx_fifo_stall_task);
1114 	INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1115 	INIT_WORK(&adapter->reset_task, e1000_reset_task);
1116 
1117 	e1000_check_options(adapter);
1118 
1119 	/* Initial Wake on LAN setting
1120 	 * If APM wake is enabled in the EEPROM,
1121 	 * enable the ACPI Magic Packet filter
1122 	 */
1123 
1124 	switch (hw->mac_type) {
1125 	case e1000_82542_rev2_0:
1126 	case e1000_82542_rev2_1:
1127 	case e1000_82543:
1128 		break;
1129 	case e1000_82544:
1130 		e1000_read_eeprom(hw,
1131 			EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
1132 		eeprom_apme_mask = E1000_EEPROM_82544_APM;
1133 		break;
1134 	case e1000_82546:
1135 	case e1000_82546_rev_3:
1136 		if (er32(STATUS) & E1000_STATUS_FUNC_1) {
1137 			e1000_read_eeprom(hw,
1138 				EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1139 			break;
1140 		}
1141 		/* Fall Through */
1142 	default:
1143 		e1000_read_eeprom(hw,
1144 			EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1145 		break;
1146 	}
1147 	if (eeprom_data & eeprom_apme_mask)
1148 		adapter->eeprom_wol |= E1000_WUFC_MAG;
1149 
1150 	/* now that we have the eeprom settings, apply the special cases
1151 	 * where the eeprom may be wrong or the board simply won't support
1152 	 * wake on lan on a particular port
1153 	 */
1154 	switch (pdev->device) {
1155 	case E1000_DEV_ID_82546GB_PCIE:
1156 		adapter->eeprom_wol = 0;
1157 		break;
1158 	case E1000_DEV_ID_82546EB_FIBER:
1159 	case E1000_DEV_ID_82546GB_FIBER:
1160 		/* Wake events only supported on port A for dual fiber
1161 		 * regardless of eeprom setting
1162 		 */
1163 		if (er32(STATUS) & E1000_STATUS_FUNC_1)
1164 			adapter->eeprom_wol = 0;
1165 		break;
1166 	case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
1167 		/* if quad port adapter, disable WoL on all but port A */
1168 		if (global_quad_port_a != 0)
1169 			adapter->eeprom_wol = 0;
1170 		else
1171 			adapter->quad_port_a = true;
1172 		/* Reset for multiple quad port adapters */
1173 		if (++global_quad_port_a == 4)
1174 			global_quad_port_a = 0;
1175 		break;
1176 	}
1177 
1178 	/* initialize the wol settings based on the eeprom settings */
1179 	adapter->wol = adapter->eeprom_wol;
1180 	device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1181 
1182 	/* Auto detect PHY address */
1183 	if (hw->mac_type == e1000_ce4100) {
1184 		for (i = 0; i < 32; i++) {
1185 			hw->phy_addr = i;
1186 			e1000_read_phy_reg(hw, PHY_ID2, &tmp);
1187 
1188 			if (tmp != 0 && tmp != 0xFF)
1189 				break;
1190 		}
1191 
1192 		if (i >= 32)
1193 			goto err_eeprom;
1194 	}
1195 
1196 	/* reset the hardware with the new settings */
1197 	e1000_reset(adapter);
1198 
1199 	strcpy(netdev->name, "eth%d");
1200 	err = register_netdev(netdev);
1201 	if (err)
1202 		goto err_register;
1203 
1204 	e1000_vlan_filter_on_off(adapter, false);
1205 
1206 	/* print bus type/speed/width info */
1207 	e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
1208 	       ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
1209 	       ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
1210 		(hw->bus_speed == e1000_bus_speed_120) ? 120 :
1211 		(hw->bus_speed == e1000_bus_speed_100) ? 100 :
1212 		(hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
1213 	       ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
1214 	       netdev->dev_addr);
1215 
1216 	/* carrier off reporting is important to ethtool even BEFORE open */
1217 	netif_carrier_off(netdev);
1218 
1219 	e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
1220 
1221 	cards_found++;
1222 	return 0;
1223 
1224 err_register:
1225 err_eeprom:
1226 	e1000_phy_hw_reset(hw);
1227 
1228 	if (hw->flash_address)
1229 		iounmap(hw->flash_address);
1230 	kfree(adapter->tx_ring);
1231 	kfree(adapter->rx_ring);
1232 err_dma:
1233 err_sw_init:
1234 err_mdio_ioremap:
1235 	iounmap(hw->ce4100_gbe_mdio_base_virt);
1236 	iounmap(hw->hw_addr);
1237 err_ioremap:
1238 	disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1239 	free_netdev(netdev);
1240 err_alloc_etherdev:
1241 	pci_release_selected_regions(pdev, bars);
1242 err_pci_reg:
1243 	if (!adapter || disable_dev)
1244 		pci_disable_device(pdev);
1245 	return err;
1246 }
1247 
1248 /**
1249  * e1000_remove - Device Removal Routine
1250  * @pdev: PCI device information struct
1251  *
1252  * e1000_remove is called by the PCI subsystem to alert the driver
1253  * that it should release a PCI device. That could be caused by a
1254  * Hot-Plug event, or because the driver is going to be removed from
1255  * memory.
1256  **/
1257 static void e1000_remove(struct pci_dev *pdev)
1258 {
1259 	struct net_device *netdev = pci_get_drvdata(pdev);
1260 	struct e1000_adapter *adapter = netdev_priv(netdev);
1261 	struct e1000_hw *hw = &adapter->hw;
1262 	bool disable_dev;
1263 
1264 	e1000_down_and_stop(adapter);
1265 	e1000_release_manageability(adapter);
1266 
1267 	unregister_netdev(netdev);
1268 
1269 	e1000_phy_hw_reset(hw);
1270 
1271 	kfree(adapter->tx_ring);
1272 	kfree(adapter->rx_ring);
1273 
1274 	if (hw->mac_type == e1000_ce4100)
1275 		iounmap(hw->ce4100_gbe_mdio_base_virt);
1276 	iounmap(hw->hw_addr);
1277 	if (hw->flash_address)
1278 		iounmap(hw->flash_address);
1279 	pci_release_selected_regions(pdev, adapter->bars);
1280 
1281 	disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1282 	free_netdev(netdev);
1283 
1284 	if (disable_dev)
1285 		pci_disable_device(pdev);
1286 }
1287 
1288 /**
1289  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
1290  * @adapter: board private structure to initialize
1291  *
1292  * e1000_sw_init initializes the Adapter private data structure.
1293  * e1000_init_hw_struct MUST be called before this function
1294  **/
1295 static int e1000_sw_init(struct e1000_adapter *adapter)
1296 {
1297 	adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1298 
1299 	adapter->num_tx_queues = 1;
1300 	adapter->num_rx_queues = 1;
1301 
1302 	if (e1000_alloc_queues(adapter)) {
1303 		e_err(probe, "Unable to allocate memory for queues\n");
1304 		return -ENOMEM;
1305 	}
1306 
1307 	/* Explicitly disable IRQ since the NIC can be in any state. */
1308 	e1000_irq_disable(adapter);
1309 
1310 	spin_lock_init(&adapter->stats_lock);
1311 
1312 	set_bit(__E1000_DOWN, &adapter->flags);
1313 
1314 	return 0;
1315 }
1316 
1317 /**
1318  * e1000_alloc_queues - Allocate memory for all rings
1319  * @adapter: board private structure to initialize
1320  *
1321  * We allocate one ring per queue at run-time since we don't know the
1322  * number of queues at compile-time.
1323  **/
1324 static int e1000_alloc_queues(struct e1000_adapter *adapter)
1325 {
1326 	adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1327 				   sizeof(struct e1000_tx_ring), GFP_KERNEL);
1328 	if (!adapter->tx_ring)
1329 		return -ENOMEM;
1330 
1331 	adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1332 				   sizeof(struct e1000_rx_ring), GFP_KERNEL);
1333 	if (!adapter->rx_ring) {
1334 		kfree(adapter->tx_ring);
1335 		return -ENOMEM;
1336 	}
1337 
1338 	return E1000_SUCCESS;
1339 }
1340 
1341 /**
1342  * e1000_open - Called when a network interface is made active
1343  * @netdev: network interface device structure
1344  *
1345  * Returns 0 on success, negative value on failure
1346  *
1347  * The open entry point is called when a network interface is made
1348  * active by the system (IFF_UP).  At this point all resources needed
1349  * for transmit and receive operations are allocated, the interrupt
1350  * handler is registered with the OS, the watchdog task is started,
1351  * and the stack is notified that the interface is ready.
1352  **/
1353 int e1000_open(struct net_device *netdev)
1354 {
1355 	struct e1000_adapter *adapter = netdev_priv(netdev);
1356 	struct e1000_hw *hw = &adapter->hw;
1357 	int err;
1358 
1359 	/* disallow open during test */
1360 	if (test_bit(__E1000_TESTING, &adapter->flags))
1361 		return -EBUSY;
1362 
1363 	netif_carrier_off(netdev);
1364 
1365 	/* allocate transmit descriptors */
1366 	err = e1000_setup_all_tx_resources(adapter);
1367 	if (err)
1368 		goto err_setup_tx;
1369 
1370 	/* allocate receive descriptors */
1371 	err = e1000_setup_all_rx_resources(adapter);
1372 	if (err)
1373 		goto err_setup_rx;
1374 
1375 	e1000_power_up_phy(adapter);
1376 
1377 	adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1378 	if ((hw->mng_cookie.status &
1379 			  E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1380 		e1000_update_mng_vlan(adapter);
1381 	}
1382 
1383 	/* before we allocate an interrupt, we must be ready to handle it.
1384 	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1385 	 * as soon as we call pci_request_irq, so we have to setup our
1386 	 * clean_rx handler before we do so.
1387 	 */
1388 	e1000_configure(adapter);
1389 
1390 	err = e1000_request_irq(adapter);
1391 	if (err)
1392 		goto err_req_irq;
1393 
1394 	/* From here on the code is the same as e1000_up() */
1395 	clear_bit(__E1000_DOWN, &adapter->flags);
1396 
1397 	napi_enable(&adapter->napi);
1398 
1399 	e1000_irq_enable(adapter);
1400 
1401 	netif_start_queue(netdev);
1402 
1403 	/* fire a link status change interrupt to start the watchdog */
1404 	ew32(ICS, E1000_ICS_LSC);
1405 
1406 	return E1000_SUCCESS;
1407 
1408 err_req_irq:
1409 	e1000_power_down_phy(adapter);
1410 	e1000_free_all_rx_resources(adapter);
1411 err_setup_rx:
1412 	e1000_free_all_tx_resources(adapter);
1413 err_setup_tx:
1414 	e1000_reset(adapter);
1415 
1416 	return err;
1417 }
1418 
1419 /**
1420  * e1000_close - Disables a network interface
1421  * @netdev: network interface device structure
1422  *
1423  * Returns 0, this is not allowed to fail
1424  *
1425  * The close entry point is called when an interface is de-activated
1426  * by the OS.  The hardware is still under the drivers control, but
1427  * needs to be disabled.  A global MAC reset is issued to stop the
1428  * hardware, and all transmit and receive resources are freed.
1429  **/
1430 int e1000_close(struct net_device *netdev)
1431 {
1432 	struct e1000_adapter *adapter = netdev_priv(netdev);
1433 	struct e1000_hw *hw = &adapter->hw;
1434 	int count = E1000_CHECK_RESET_COUNT;
1435 
1436 	while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
1437 		usleep_range(10000, 20000);
1438 
1439 	WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
1440 	e1000_down(adapter);
1441 	e1000_power_down_phy(adapter);
1442 	e1000_free_irq(adapter);
1443 
1444 	e1000_free_all_tx_resources(adapter);
1445 	e1000_free_all_rx_resources(adapter);
1446 
1447 	/* kill manageability vlan ID if supported, but not if a vlan with
1448 	 * the same ID is registered on the host OS (let 8021q kill it)
1449 	 */
1450 	if ((hw->mng_cookie.status &
1451 	     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
1452 	    !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1453 		e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
1454 				       adapter->mng_vlan_id);
1455 	}
1456 
1457 	return 0;
1458 }
1459 
1460 /**
1461  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1462  * @adapter: address of board private structure
1463  * @start: address of beginning of memory
1464  * @len: length of memory
1465  **/
1466 static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
1467 				  unsigned long len)
1468 {
1469 	struct e1000_hw *hw = &adapter->hw;
1470 	unsigned long begin = (unsigned long)start;
1471 	unsigned long end = begin + len;
1472 
1473 	/* First rev 82545 and 82546 need to not allow any memory
1474 	 * write location to cross 64k boundary due to errata 23
1475 	 */
1476 	if (hw->mac_type == e1000_82545 ||
1477 	    hw->mac_type == e1000_ce4100 ||
1478 	    hw->mac_type == e1000_82546) {
1479 		return ((begin ^ (end - 1)) >> 16) != 0 ? false : true;
1480 	}
1481 
1482 	return true;
1483 }
1484 
1485 /**
1486  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1487  * @adapter: board private structure
1488  * @txdr:    tx descriptor ring (for a specific queue) to setup
1489  *
1490  * Return 0 on success, negative on failure
1491  **/
1492 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
1493 				    struct e1000_tx_ring *txdr)
1494 {
1495 	struct pci_dev *pdev = adapter->pdev;
1496 	int size;
1497 
1498 	size = sizeof(struct e1000_tx_buffer) * txdr->count;
1499 	txdr->buffer_info = vzalloc(size);
1500 	if (!txdr->buffer_info)
1501 		return -ENOMEM;
1502 
1503 	/* round up to nearest 4K */
1504 
1505 	txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1506 	txdr->size = ALIGN(txdr->size, 4096);
1507 
1508 	txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
1509 					GFP_KERNEL);
1510 	if (!txdr->desc) {
1511 setup_tx_desc_die:
1512 		vfree(txdr->buffer_info);
1513 		return -ENOMEM;
1514 	}
1515 
1516 	/* Fix for errata 23, can't cross 64kB boundary */
1517 	if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1518 		void *olddesc = txdr->desc;
1519 		dma_addr_t olddma = txdr->dma;
1520 		e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1521 		      txdr->size, txdr->desc);
1522 		/* Try again, without freeing the previous */
1523 		txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
1524 						&txdr->dma, GFP_KERNEL);
1525 		/* Failed allocation, critical failure */
1526 		if (!txdr->desc) {
1527 			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1528 					  olddma);
1529 			goto setup_tx_desc_die;
1530 		}
1531 
1532 		if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1533 			/* give up */
1534 			dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
1535 					  txdr->dma);
1536 			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1537 					  olddma);
1538 			e_err(probe, "Unable to allocate aligned memory "
1539 			      "for the transmit descriptor ring\n");
1540 			vfree(txdr->buffer_info);
1541 			return -ENOMEM;
1542 		} else {
1543 			/* Free old allocation, new allocation was successful */
1544 			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1545 					  olddma);
1546 		}
1547 	}
1548 	memset(txdr->desc, 0, txdr->size);
1549 
1550 	txdr->next_to_use = 0;
1551 	txdr->next_to_clean = 0;
1552 
1553 	return 0;
1554 }
1555 
1556 /**
1557  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1558  * 				  (Descriptors) for all queues
1559  * @adapter: board private structure
1560  *
1561  * Return 0 on success, negative on failure
1562  **/
1563 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1564 {
1565 	int i, err = 0;
1566 
1567 	for (i = 0; i < adapter->num_tx_queues; i++) {
1568 		err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1569 		if (err) {
1570 			e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1571 			for (i-- ; i >= 0; i--)
1572 				e1000_free_tx_resources(adapter,
1573 							&adapter->tx_ring[i]);
1574 			break;
1575 		}
1576 	}
1577 
1578 	return err;
1579 }
1580 
1581 /**
1582  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1583  * @adapter: board private structure
1584  *
1585  * Configure the Tx unit of the MAC after a reset.
1586  **/
1587 static void e1000_configure_tx(struct e1000_adapter *adapter)
1588 {
1589 	u64 tdba;
1590 	struct e1000_hw *hw = &adapter->hw;
1591 	u32 tdlen, tctl, tipg;
1592 	u32 ipgr1, ipgr2;
1593 
1594 	/* Setup the HW Tx Head and Tail descriptor pointers */
1595 
1596 	switch (adapter->num_tx_queues) {
1597 	case 1:
1598 	default:
1599 		tdba = adapter->tx_ring[0].dma;
1600 		tdlen = adapter->tx_ring[0].count *
1601 			sizeof(struct e1000_tx_desc);
1602 		ew32(TDLEN, tdlen);
1603 		ew32(TDBAH, (tdba >> 32));
1604 		ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
1605 		ew32(TDT, 0);
1606 		ew32(TDH, 0);
1607 		adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ?
1608 					   E1000_TDH : E1000_82542_TDH);
1609 		adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ?
1610 					   E1000_TDT : E1000_82542_TDT);
1611 		break;
1612 	}
1613 
1614 	/* Set the default values for the Tx Inter Packet Gap timer */
1615 	if ((hw->media_type == e1000_media_type_fiber ||
1616 	     hw->media_type == e1000_media_type_internal_serdes))
1617 		tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1618 	else
1619 		tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1620 
1621 	switch (hw->mac_type) {
1622 	case e1000_82542_rev2_0:
1623 	case e1000_82542_rev2_1:
1624 		tipg = DEFAULT_82542_TIPG_IPGT;
1625 		ipgr1 = DEFAULT_82542_TIPG_IPGR1;
1626 		ipgr2 = DEFAULT_82542_TIPG_IPGR2;
1627 		break;
1628 	default:
1629 		ipgr1 = DEFAULT_82543_TIPG_IPGR1;
1630 		ipgr2 = DEFAULT_82543_TIPG_IPGR2;
1631 		break;
1632 	}
1633 	tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
1634 	tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
1635 	ew32(TIPG, tipg);
1636 
1637 	/* Set the Tx Interrupt Delay register */
1638 
1639 	ew32(TIDV, adapter->tx_int_delay);
1640 	if (hw->mac_type >= e1000_82540)
1641 		ew32(TADV, adapter->tx_abs_int_delay);
1642 
1643 	/* Program the Transmit Control Register */
1644 
1645 	tctl = er32(TCTL);
1646 	tctl &= ~E1000_TCTL_CT;
1647 	tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1648 		(E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1649 
1650 	e1000_config_collision_dist(hw);
1651 
1652 	/* Setup Transmit Descriptor Settings for eop descriptor */
1653 	adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
1654 
1655 	/* only set IDE if we are delaying interrupts using the timers */
1656 	if (adapter->tx_int_delay)
1657 		adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1658 
1659 	if (hw->mac_type < e1000_82543)
1660 		adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1661 	else
1662 		adapter->txd_cmd |= E1000_TXD_CMD_RS;
1663 
1664 	/* Cache if we're 82544 running in PCI-X because we'll
1665 	 * need this to apply a workaround later in the send path.
1666 	 */
1667 	if (hw->mac_type == e1000_82544 &&
1668 	    hw->bus_type == e1000_bus_type_pcix)
1669 		adapter->pcix_82544 = true;
1670 
1671 	ew32(TCTL, tctl);
1672 
1673 }
1674 
1675 /**
1676  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1677  * @adapter: board private structure
1678  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1679  *
1680  * Returns 0 on success, negative on failure
1681  **/
1682 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
1683 				    struct e1000_rx_ring *rxdr)
1684 {
1685 	struct pci_dev *pdev = adapter->pdev;
1686 	int size, desc_len;
1687 
1688 	size = sizeof(struct e1000_rx_buffer) * rxdr->count;
1689 	rxdr->buffer_info = vzalloc(size);
1690 	if (!rxdr->buffer_info)
1691 		return -ENOMEM;
1692 
1693 	desc_len = sizeof(struct e1000_rx_desc);
1694 
1695 	/* Round up to nearest 4K */
1696 
1697 	rxdr->size = rxdr->count * desc_len;
1698 	rxdr->size = ALIGN(rxdr->size, 4096);
1699 
1700 	rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
1701 					GFP_KERNEL);
1702 	if (!rxdr->desc) {
1703 setup_rx_desc_die:
1704 		vfree(rxdr->buffer_info);
1705 		return -ENOMEM;
1706 	}
1707 
1708 	/* Fix for errata 23, can't cross 64kB boundary */
1709 	if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1710 		void *olddesc = rxdr->desc;
1711 		dma_addr_t olddma = rxdr->dma;
1712 		e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1713 		      rxdr->size, rxdr->desc);
1714 		/* Try again, without freeing the previous */
1715 		rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
1716 						&rxdr->dma, GFP_KERNEL);
1717 		/* Failed allocation, critical failure */
1718 		if (!rxdr->desc) {
1719 			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1720 					  olddma);
1721 			goto setup_rx_desc_die;
1722 		}
1723 
1724 		if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1725 			/* give up */
1726 			dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
1727 					  rxdr->dma);
1728 			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1729 					  olddma);
1730 			e_err(probe, "Unable to allocate aligned memory for "
1731 			      "the Rx descriptor ring\n");
1732 			goto setup_rx_desc_die;
1733 		} else {
1734 			/* Free old allocation, new allocation was successful */
1735 			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1736 					  olddma);
1737 		}
1738 	}
1739 	memset(rxdr->desc, 0, rxdr->size);
1740 
1741 	rxdr->next_to_clean = 0;
1742 	rxdr->next_to_use = 0;
1743 	rxdr->rx_skb_top = NULL;
1744 
1745 	return 0;
1746 }
1747 
1748 /**
1749  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1750  * 				  (Descriptors) for all queues
1751  * @adapter: board private structure
1752  *
1753  * Return 0 on success, negative on failure
1754  **/
1755 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1756 {
1757 	int i, err = 0;
1758 
1759 	for (i = 0; i < adapter->num_rx_queues; i++) {
1760 		err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1761 		if (err) {
1762 			e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1763 			for (i-- ; i >= 0; i--)
1764 				e1000_free_rx_resources(adapter,
1765 							&adapter->rx_ring[i]);
1766 			break;
1767 		}
1768 	}
1769 
1770 	return err;
1771 }
1772 
1773 /**
1774  * e1000_setup_rctl - configure the receive control registers
1775  * @adapter: Board private structure
1776  **/
1777 static void e1000_setup_rctl(struct e1000_adapter *adapter)
1778 {
1779 	struct e1000_hw *hw = &adapter->hw;
1780 	u32 rctl;
1781 
1782 	rctl = er32(RCTL);
1783 
1784 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1785 
1786 	rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
1787 		E1000_RCTL_RDMTS_HALF |
1788 		(hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
1789 
1790 	if (hw->tbi_compatibility_on == 1)
1791 		rctl |= E1000_RCTL_SBP;
1792 	else
1793 		rctl &= ~E1000_RCTL_SBP;
1794 
1795 	if (adapter->netdev->mtu <= ETH_DATA_LEN)
1796 		rctl &= ~E1000_RCTL_LPE;
1797 	else
1798 		rctl |= E1000_RCTL_LPE;
1799 
1800 	/* Setup buffer sizes */
1801 	rctl &= ~E1000_RCTL_SZ_4096;
1802 	rctl |= E1000_RCTL_BSEX;
1803 	switch (adapter->rx_buffer_len) {
1804 	case E1000_RXBUFFER_2048:
1805 	default:
1806 		rctl |= E1000_RCTL_SZ_2048;
1807 		rctl &= ~E1000_RCTL_BSEX;
1808 		break;
1809 	case E1000_RXBUFFER_4096:
1810 		rctl |= E1000_RCTL_SZ_4096;
1811 		break;
1812 	case E1000_RXBUFFER_8192:
1813 		rctl |= E1000_RCTL_SZ_8192;
1814 		break;
1815 	case E1000_RXBUFFER_16384:
1816 		rctl |= E1000_RCTL_SZ_16384;
1817 		break;
1818 	}
1819 
1820 	/* This is useful for sniffing bad packets. */
1821 	if (adapter->netdev->features & NETIF_F_RXALL) {
1822 		/* UPE and MPE will be handled by normal PROMISC logic
1823 		 * in e1000e_set_rx_mode
1824 		 */
1825 		rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
1826 			 E1000_RCTL_BAM | /* RX All Bcast Pkts */
1827 			 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
1828 
1829 		rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
1830 			  E1000_RCTL_DPF | /* Allow filtered pause */
1831 			  E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
1832 		/* Do not mess with E1000_CTRL_VME, it affects transmit as well,
1833 		 * and that breaks VLANs.
1834 		 */
1835 	}
1836 
1837 	ew32(RCTL, rctl);
1838 }
1839 
1840 /**
1841  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1842  * @adapter: board private structure
1843  *
1844  * Configure the Rx unit of the MAC after a reset.
1845  **/
1846 static void e1000_configure_rx(struct e1000_adapter *adapter)
1847 {
1848 	u64 rdba;
1849 	struct e1000_hw *hw = &adapter->hw;
1850 	u32 rdlen, rctl, rxcsum;
1851 
1852 	if (adapter->netdev->mtu > ETH_DATA_LEN) {
1853 		rdlen = adapter->rx_ring[0].count *
1854 			sizeof(struct e1000_rx_desc);
1855 		adapter->clean_rx = e1000_clean_jumbo_rx_irq;
1856 		adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
1857 	} else {
1858 		rdlen = adapter->rx_ring[0].count *
1859 			sizeof(struct e1000_rx_desc);
1860 		adapter->clean_rx = e1000_clean_rx_irq;
1861 		adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1862 	}
1863 
1864 	/* disable receives while setting up the descriptors */
1865 	rctl = er32(RCTL);
1866 	ew32(RCTL, rctl & ~E1000_RCTL_EN);
1867 
1868 	/* set the Receive Delay Timer Register */
1869 	ew32(RDTR, adapter->rx_int_delay);
1870 
1871 	if (hw->mac_type >= e1000_82540) {
1872 		ew32(RADV, adapter->rx_abs_int_delay);
1873 		if (adapter->itr_setting != 0)
1874 			ew32(ITR, 1000000000 / (adapter->itr * 256));
1875 	}
1876 
1877 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
1878 	 * the Base and Length of the Rx Descriptor Ring
1879 	 */
1880 	switch (adapter->num_rx_queues) {
1881 	case 1:
1882 	default:
1883 		rdba = adapter->rx_ring[0].dma;
1884 		ew32(RDLEN, rdlen);
1885 		ew32(RDBAH, (rdba >> 32));
1886 		ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
1887 		ew32(RDT, 0);
1888 		ew32(RDH, 0);
1889 		adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ?
1890 					   E1000_RDH : E1000_82542_RDH);
1891 		adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ?
1892 					   E1000_RDT : E1000_82542_RDT);
1893 		break;
1894 	}
1895 
1896 	/* Enable 82543 Receive Checksum Offload for TCP and UDP */
1897 	if (hw->mac_type >= e1000_82543) {
1898 		rxcsum = er32(RXCSUM);
1899 		if (adapter->rx_csum)
1900 			rxcsum |= E1000_RXCSUM_TUOFL;
1901 		else
1902 			/* don't need to clear IPPCSE as it defaults to 0 */
1903 			rxcsum &= ~E1000_RXCSUM_TUOFL;
1904 		ew32(RXCSUM, rxcsum);
1905 	}
1906 
1907 	/* Enable Receives */
1908 	ew32(RCTL, rctl | E1000_RCTL_EN);
1909 }
1910 
1911 /**
1912  * e1000_free_tx_resources - Free Tx Resources per Queue
1913  * @adapter: board private structure
1914  * @tx_ring: Tx descriptor ring for a specific queue
1915  *
1916  * Free all transmit software resources
1917  **/
1918 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
1919 				    struct e1000_tx_ring *tx_ring)
1920 {
1921 	struct pci_dev *pdev = adapter->pdev;
1922 
1923 	e1000_clean_tx_ring(adapter, tx_ring);
1924 
1925 	vfree(tx_ring->buffer_info);
1926 	tx_ring->buffer_info = NULL;
1927 
1928 	dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
1929 			  tx_ring->dma);
1930 
1931 	tx_ring->desc = NULL;
1932 }
1933 
1934 /**
1935  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1936  * @adapter: board private structure
1937  *
1938  * Free all transmit software resources
1939  **/
1940 void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1941 {
1942 	int i;
1943 
1944 	for (i = 0; i < adapter->num_tx_queues; i++)
1945 		e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1946 }
1947 
1948 static void
1949 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1950 				 struct e1000_tx_buffer *buffer_info)
1951 {
1952 	if (buffer_info->dma) {
1953 		if (buffer_info->mapped_as_page)
1954 			dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1955 				       buffer_info->length, DMA_TO_DEVICE);
1956 		else
1957 			dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1958 					 buffer_info->length,
1959 					 DMA_TO_DEVICE);
1960 		buffer_info->dma = 0;
1961 	}
1962 	if (buffer_info->skb) {
1963 		dev_kfree_skb_any(buffer_info->skb);
1964 		buffer_info->skb = NULL;
1965 	}
1966 	buffer_info->time_stamp = 0;
1967 	/* buffer_info must be completely set up in the transmit path */
1968 }
1969 
1970 /**
1971  * e1000_clean_tx_ring - Free Tx Buffers
1972  * @adapter: board private structure
1973  * @tx_ring: ring to be cleaned
1974  **/
1975 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
1976 				struct e1000_tx_ring *tx_ring)
1977 {
1978 	struct e1000_hw *hw = &adapter->hw;
1979 	struct e1000_tx_buffer *buffer_info;
1980 	unsigned long size;
1981 	unsigned int i;
1982 
1983 	/* Free all the Tx ring sk_buffs */
1984 
1985 	for (i = 0; i < tx_ring->count; i++) {
1986 		buffer_info = &tx_ring->buffer_info[i];
1987 		e1000_unmap_and_free_tx_resource(adapter, buffer_info);
1988 	}
1989 
1990 	netdev_reset_queue(adapter->netdev);
1991 	size = sizeof(struct e1000_tx_buffer) * tx_ring->count;
1992 	memset(tx_ring->buffer_info, 0, size);
1993 
1994 	/* Zero out the descriptor ring */
1995 
1996 	memset(tx_ring->desc, 0, tx_ring->size);
1997 
1998 	tx_ring->next_to_use = 0;
1999 	tx_ring->next_to_clean = 0;
2000 	tx_ring->last_tx_tso = false;
2001 
2002 	writel(0, hw->hw_addr + tx_ring->tdh);
2003 	writel(0, hw->hw_addr + tx_ring->tdt);
2004 }
2005 
2006 /**
2007  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
2008  * @adapter: board private structure
2009  **/
2010 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2011 {
2012 	int i;
2013 
2014 	for (i = 0; i < adapter->num_tx_queues; i++)
2015 		e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
2016 }
2017 
2018 /**
2019  * e1000_free_rx_resources - Free Rx Resources
2020  * @adapter: board private structure
2021  * @rx_ring: ring to clean the resources from
2022  *
2023  * Free all receive software resources
2024  **/
2025 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
2026 				    struct e1000_rx_ring *rx_ring)
2027 {
2028 	struct pci_dev *pdev = adapter->pdev;
2029 
2030 	e1000_clean_rx_ring(adapter, rx_ring);
2031 
2032 	vfree(rx_ring->buffer_info);
2033 	rx_ring->buffer_info = NULL;
2034 
2035 	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2036 			  rx_ring->dma);
2037 
2038 	rx_ring->desc = NULL;
2039 }
2040 
2041 /**
2042  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
2043  * @adapter: board private structure
2044  *
2045  * Free all receive software resources
2046  **/
2047 void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2048 {
2049 	int i;
2050 
2051 	for (i = 0; i < adapter->num_rx_queues; i++)
2052 		e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
2053 }
2054 
2055 #define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
2056 static unsigned int e1000_frag_len(const struct e1000_adapter *a)
2057 {
2058 	return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) +
2059 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2060 }
2061 
2062 static void *e1000_alloc_frag(const struct e1000_adapter *a)
2063 {
2064 	unsigned int len = e1000_frag_len(a);
2065 	u8 *data = netdev_alloc_frag(len);
2066 
2067 	if (likely(data))
2068 		data += E1000_HEADROOM;
2069 	return data;
2070 }
2071 
2072 /**
2073  * e1000_clean_rx_ring - Free Rx Buffers per Queue
2074  * @adapter: board private structure
2075  * @rx_ring: ring to free buffers from
2076  **/
2077 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
2078 				struct e1000_rx_ring *rx_ring)
2079 {
2080 	struct e1000_hw *hw = &adapter->hw;
2081 	struct e1000_rx_buffer *buffer_info;
2082 	struct pci_dev *pdev = adapter->pdev;
2083 	unsigned long size;
2084 	unsigned int i;
2085 
2086 	/* Free all the Rx netfrags */
2087 	for (i = 0; i < rx_ring->count; i++) {
2088 		buffer_info = &rx_ring->buffer_info[i];
2089 		if (adapter->clean_rx == e1000_clean_rx_irq) {
2090 			if (buffer_info->dma)
2091 				dma_unmap_single(&pdev->dev, buffer_info->dma,
2092 						 adapter->rx_buffer_len,
2093 						 DMA_FROM_DEVICE);
2094 			if (buffer_info->rxbuf.data) {
2095 				skb_free_frag(buffer_info->rxbuf.data);
2096 				buffer_info->rxbuf.data = NULL;
2097 			}
2098 		} else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2099 			if (buffer_info->dma)
2100 				dma_unmap_page(&pdev->dev, buffer_info->dma,
2101 					       adapter->rx_buffer_len,
2102 					       DMA_FROM_DEVICE);
2103 			if (buffer_info->rxbuf.page) {
2104 				put_page(buffer_info->rxbuf.page);
2105 				buffer_info->rxbuf.page = NULL;
2106 			}
2107 		}
2108 
2109 		buffer_info->dma = 0;
2110 	}
2111 
2112 	/* there also may be some cached data from a chained receive */
2113 	napi_free_frags(&adapter->napi);
2114 	rx_ring->rx_skb_top = NULL;
2115 
2116 	size = sizeof(struct e1000_rx_buffer) * rx_ring->count;
2117 	memset(rx_ring->buffer_info, 0, size);
2118 
2119 	/* Zero out the descriptor ring */
2120 	memset(rx_ring->desc, 0, rx_ring->size);
2121 
2122 	rx_ring->next_to_clean = 0;
2123 	rx_ring->next_to_use = 0;
2124 
2125 	writel(0, hw->hw_addr + rx_ring->rdh);
2126 	writel(0, hw->hw_addr + rx_ring->rdt);
2127 }
2128 
2129 /**
2130  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2131  * @adapter: board private structure
2132  **/
2133 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2134 {
2135 	int i;
2136 
2137 	for (i = 0; i < adapter->num_rx_queues; i++)
2138 		e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2139 }
2140 
2141 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2142  * and memory write and invalidate disabled for certain operations
2143  */
2144 static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
2145 {
2146 	struct e1000_hw *hw = &adapter->hw;
2147 	struct net_device *netdev = adapter->netdev;
2148 	u32 rctl;
2149 
2150 	e1000_pci_clear_mwi(hw);
2151 
2152 	rctl = er32(RCTL);
2153 	rctl |= E1000_RCTL_RST;
2154 	ew32(RCTL, rctl);
2155 	E1000_WRITE_FLUSH();
2156 	mdelay(5);
2157 
2158 	if (netif_running(netdev))
2159 		e1000_clean_all_rx_rings(adapter);
2160 }
2161 
2162 static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
2163 {
2164 	struct e1000_hw *hw = &adapter->hw;
2165 	struct net_device *netdev = adapter->netdev;
2166 	u32 rctl;
2167 
2168 	rctl = er32(RCTL);
2169 	rctl &= ~E1000_RCTL_RST;
2170 	ew32(RCTL, rctl);
2171 	E1000_WRITE_FLUSH();
2172 	mdelay(5);
2173 
2174 	if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
2175 		e1000_pci_set_mwi(hw);
2176 
2177 	if (netif_running(netdev)) {
2178 		/* No need to loop, because 82542 supports only 1 queue */
2179 		struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2180 		e1000_configure_rx(adapter);
2181 		adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
2182 	}
2183 }
2184 
2185 /**
2186  * e1000_set_mac - Change the Ethernet Address of the NIC
2187  * @netdev: network interface device structure
2188  * @p: pointer to an address structure
2189  *
2190  * Returns 0 on success, negative on failure
2191  **/
2192 static int e1000_set_mac(struct net_device *netdev, void *p)
2193 {
2194 	struct e1000_adapter *adapter = netdev_priv(netdev);
2195 	struct e1000_hw *hw = &adapter->hw;
2196 	struct sockaddr *addr = p;
2197 
2198 	if (!is_valid_ether_addr(addr->sa_data))
2199 		return -EADDRNOTAVAIL;
2200 
2201 	/* 82542 2.0 needs to be in reset to write receive address registers */
2202 
2203 	if (hw->mac_type == e1000_82542_rev2_0)
2204 		e1000_enter_82542_rst(adapter);
2205 
2206 	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2207 	memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
2208 
2209 	e1000_rar_set(hw, hw->mac_addr, 0);
2210 
2211 	if (hw->mac_type == e1000_82542_rev2_0)
2212 		e1000_leave_82542_rst(adapter);
2213 
2214 	return 0;
2215 }
2216 
2217 /**
2218  * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2219  * @netdev: network interface device structure
2220  *
2221  * The set_rx_mode entry point is called whenever the unicast or multicast
2222  * address lists or the network interface flags are updated. This routine is
2223  * responsible for configuring the hardware for proper unicast, multicast,
2224  * promiscuous mode, and all-multi behavior.
2225  **/
2226 static void e1000_set_rx_mode(struct net_device *netdev)
2227 {
2228 	struct e1000_adapter *adapter = netdev_priv(netdev);
2229 	struct e1000_hw *hw = &adapter->hw;
2230 	struct netdev_hw_addr *ha;
2231 	bool use_uc = false;
2232 	u32 rctl;
2233 	u32 hash_value;
2234 	int i, rar_entries = E1000_RAR_ENTRIES;
2235 	int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2236 	u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);
2237 
2238 	if (!mcarray)
2239 		return;
2240 
2241 	/* Check for Promiscuous and All Multicast modes */
2242 
2243 	rctl = er32(RCTL);
2244 
2245 	if (netdev->flags & IFF_PROMISC) {
2246 		rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2247 		rctl &= ~E1000_RCTL_VFE;
2248 	} else {
2249 		if (netdev->flags & IFF_ALLMULTI)
2250 			rctl |= E1000_RCTL_MPE;
2251 		else
2252 			rctl &= ~E1000_RCTL_MPE;
2253 		/* Enable VLAN filter if there is a VLAN */
2254 		if (e1000_vlan_used(adapter))
2255 			rctl |= E1000_RCTL_VFE;
2256 	}
2257 
2258 	if (netdev_uc_count(netdev) > rar_entries - 1) {
2259 		rctl |= E1000_RCTL_UPE;
2260 	} else if (!(netdev->flags & IFF_PROMISC)) {
2261 		rctl &= ~E1000_RCTL_UPE;
2262 		use_uc = true;
2263 	}
2264 
2265 	ew32(RCTL, rctl);
2266 
2267 	/* 82542 2.0 needs to be in reset to write receive address registers */
2268 
2269 	if (hw->mac_type == e1000_82542_rev2_0)
2270 		e1000_enter_82542_rst(adapter);
2271 
2272 	/* load the first 14 addresses into the exact filters 1-14. Unicast
2273 	 * addresses take precedence to avoid disabling unicast filtering
2274 	 * when possible.
2275 	 *
2276 	 * RAR 0 is used for the station MAC address
2277 	 * if there are not 14 addresses, go ahead and clear the filters
2278 	 */
2279 	i = 1;
2280 	if (use_uc)
2281 		netdev_for_each_uc_addr(ha, netdev) {
2282 			if (i == rar_entries)
2283 				break;
2284 			e1000_rar_set(hw, ha->addr, i++);
2285 		}
2286 
2287 	netdev_for_each_mc_addr(ha, netdev) {
2288 		if (i == rar_entries) {
2289 			/* load any remaining addresses into the hash table */
2290 			u32 hash_reg, hash_bit, mta;
2291 			hash_value = e1000_hash_mc_addr(hw, ha->addr);
2292 			hash_reg = (hash_value >> 5) & 0x7F;
2293 			hash_bit = hash_value & 0x1F;
2294 			mta = (1 << hash_bit);
2295 			mcarray[hash_reg] |= mta;
2296 		} else {
2297 			e1000_rar_set(hw, ha->addr, i++);
2298 		}
2299 	}
2300 
2301 	for (; i < rar_entries; i++) {
2302 		E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2303 		E1000_WRITE_FLUSH();
2304 		E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2305 		E1000_WRITE_FLUSH();
2306 	}
2307 
2308 	/* write the hash table completely, write from bottom to avoid
2309 	 * both stupid write combining chipsets, and flushing each write
2310 	 */
2311 	for (i = mta_reg_count - 1; i >= 0 ; i--) {
2312 		/* If we are on an 82544 has an errata where writing odd
2313 		 * offsets overwrites the previous even offset, but writing
2314 		 * backwards over the range solves the issue by always
2315 		 * writing the odd offset first
2316 		 */
2317 		E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
2318 	}
2319 	E1000_WRITE_FLUSH();
2320 
2321 	if (hw->mac_type == e1000_82542_rev2_0)
2322 		e1000_leave_82542_rst(adapter);
2323 
2324 	kfree(mcarray);
2325 }
2326 
2327 /**
2328  * e1000_update_phy_info_task - get phy info
2329  * @work: work struct contained inside adapter struct
2330  *
2331  * Need to wait a few seconds after link up to get diagnostic information from
2332  * the phy
2333  */
2334 static void e1000_update_phy_info_task(struct work_struct *work)
2335 {
2336 	struct e1000_adapter *adapter = container_of(work,
2337 						     struct e1000_adapter,
2338 						     phy_info_task.work);
2339 
2340 	e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2341 }
2342 
2343 /**
2344  * e1000_82547_tx_fifo_stall_task - task to complete work
2345  * @work: work struct contained inside adapter struct
2346  **/
2347 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
2348 {
2349 	struct e1000_adapter *adapter = container_of(work,
2350 						     struct e1000_adapter,
2351 						     fifo_stall_task.work);
2352 	struct e1000_hw *hw = &adapter->hw;
2353 	struct net_device *netdev = adapter->netdev;
2354 	u32 tctl;
2355 
2356 	if (atomic_read(&adapter->tx_fifo_stall)) {
2357 		if ((er32(TDT) == er32(TDH)) &&
2358 		   (er32(TDFT) == er32(TDFH)) &&
2359 		   (er32(TDFTS) == er32(TDFHS))) {
2360 			tctl = er32(TCTL);
2361 			ew32(TCTL, tctl & ~E1000_TCTL_EN);
2362 			ew32(TDFT, adapter->tx_head_addr);
2363 			ew32(TDFH, adapter->tx_head_addr);
2364 			ew32(TDFTS, adapter->tx_head_addr);
2365 			ew32(TDFHS, adapter->tx_head_addr);
2366 			ew32(TCTL, tctl);
2367 			E1000_WRITE_FLUSH();
2368 
2369 			adapter->tx_fifo_head = 0;
2370 			atomic_set(&adapter->tx_fifo_stall, 0);
2371 			netif_wake_queue(netdev);
2372 		} else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2373 			schedule_delayed_work(&adapter->fifo_stall_task, 1);
2374 		}
2375 	}
2376 }
2377 
2378 bool e1000_has_link(struct e1000_adapter *adapter)
2379 {
2380 	struct e1000_hw *hw = &adapter->hw;
2381 	bool link_active = false;
2382 
2383 	/* get_link_status is set on LSC (link status) interrupt or rx
2384 	 * sequence error interrupt (except on intel ce4100).
2385 	 * get_link_status will stay false until the
2386 	 * e1000_check_for_link establishes link for copper adapters
2387 	 * ONLY
2388 	 */
2389 	switch (hw->media_type) {
2390 	case e1000_media_type_copper:
2391 		if (hw->mac_type == e1000_ce4100)
2392 			hw->get_link_status = 1;
2393 		if (hw->get_link_status) {
2394 			e1000_check_for_link(hw);
2395 			link_active = !hw->get_link_status;
2396 		} else {
2397 			link_active = true;
2398 		}
2399 		break;
2400 	case e1000_media_type_fiber:
2401 		e1000_check_for_link(hw);
2402 		link_active = !!(er32(STATUS) & E1000_STATUS_LU);
2403 		break;
2404 	case e1000_media_type_internal_serdes:
2405 		e1000_check_for_link(hw);
2406 		link_active = hw->serdes_has_link;
2407 		break;
2408 	default:
2409 		break;
2410 	}
2411 
2412 	return link_active;
2413 }
2414 
2415 /**
2416  * e1000_watchdog - work function
2417  * @work: work struct contained inside adapter struct
2418  **/
2419 static void e1000_watchdog(struct work_struct *work)
2420 {
2421 	struct e1000_adapter *adapter = container_of(work,
2422 						     struct e1000_adapter,
2423 						     watchdog_task.work);
2424 	struct e1000_hw *hw = &adapter->hw;
2425 	struct net_device *netdev = adapter->netdev;
2426 	struct e1000_tx_ring *txdr = adapter->tx_ring;
2427 	u32 link, tctl;
2428 
2429 	link = e1000_has_link(adapter);
2430 	if ((netif_carrier_ok(netdev)) && link)
2431 		goto link_up;
2432 
2433 	if (link) {
2434 		if (!netif_carrier_ok(netdev)) {
2435 			u32 ctrl;
2436 			/* update snapshot of PHY registers on LSC */
2437 			e1000_get_speed_and_duplex(hw,
2438 						   &adapter->link_speed,
2439 						   &adapter->link_duplex);
2440 
2441 			ctrl = er32(CTRL);
2442 			pr_info("%s NIC Link is Up %d Mbps %s, "
2443 				"Flow Control: %s\n",
2444 				netdev->name,
2445 				adapter->link_speed,
2446 				adapter->link_duplex == FULL_DUPLEX ?
2447 				"Full Duplex" : "Half Duplex",
2448 				((ctrl & E1000_CTRL_TFCE) && (ctrl &
2449 				E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2450 				E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2451 				E1000_CTRL_TFCE) ? "TX" : "None")));
2452 
2453 			/* adjust timeout factor according to speed/duplex */
2454 			adapter->tx_timeout_factor = 1;
2455 			switch (adapter->link_speed) {
2456 			case SPEED_10:
2457 				adapter->tx_timeout_factor = 16;
2458 				break;
2459 			case SPEED_100:
2460 				/* maybe add some timeout factor ? */
2461 				break;
2462 			}
2463 
2464 			/* enable transmits in the hardware */
2465 			tctl = er32(TCTL);
2466 			tctl |= E1000_TCTL_EN;
2467 			ew32(TCTL, tctl);
2468 
2469 			netif_carrier_on(netdev);
2470 			if (!test_bit(__E1000_DOWN, &adapter->flags))
2471 				schedule_delayed_work(&adapter->phy_info_task,
2472 						      2 * HZ);
2473 			adapter->smartspeed = 0;
2474 		}
2475 	} else {
2476 		if (netif_carrier_ok(netdev)) {
2477 			adapter->link_speed = 0;
2478 			adapter->link_duplex = 0;
2479 			pr_info("%s NIC Link is Down\n",
2480 				netdev->name);
2481 			netif_carrier_off(netdev);
2482 
2483 			if (!test_bit(__E1000_DOWN, &adapter->flags))
2484 				schedule_delayed_work(&adapter->phy_info_task,
2485 						      2 * HZ);
2486 		}
2487 
2488 		e1000_smartspeed(adapter);
2489 	}
2490 
2491 link_up:
2492 	e1000_update_stats(adapter);
2493 
2494 	hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2495 	adapter->tpt_old = adapter->stats.tpt;
2496 	hw->collision_delta = adapter->stats.colc - adapter->colc_old;
2497 	adapter->colc_old = adapter->stats.colc;
2498 
2499 	adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2500 	adapter->gorcl_old = adapter->stats.gorcl;
2501 	adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2502 	adapter->gotcl_old = adapter->stats.gotcl;
2503 
2504 	e1000_update_adaptive(hw);
2505 
2506 	if (!netif_carrier_ok(netdev)) {
2507 		if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2508 			/* We've lost link, so the controller stops DMA,
2509 			 * but we've got queued Tx work that's never going
2510 			 * to get done, so reset controller to flush Tx.
2511 			 * (Do the reset outside of interrupt context).
2512 			 */
2513 			adapter->tx_timeout_count++;
2514 			schedule_work(&adapter->reset_task);
2515 			/* exit immediately since reset is imminent */
2516 			return;
2517 		}
2518 	}
2519 
2520 	/* Simple mode for Interrupt Throttle Rate (ITR) */
2521 	if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
2522 		/* Symmetric Tx/Rx gets a reduced ITR=2000;
2523 		 * Total asymmetrical Tx or Rx gets ITR=8000;
2524 		 * everyone else is between 2000-8000.
2525 		 */
2526 		u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
2527 		u32 dif = (adapter->gotcl > adapter->gorcl ?
2528 			    adapter->gotcl - adapter->gorcl :
2529 			    adapter->gorcl - adapter->gotcl) / 10000;
2530 		u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2531 
2532 		ew32(ITR, 1000000000 / (itr * 256));
2533 	}
2534 
2535 	/* Cause software interrupt to ensure rx ring is cleaned */
2536 	ew32(ICS, E1000_ICS_RXDMT0);
2537 
2538 	/* Force detection of hung controller every watchdog period */
2539 	adapter->detect_tx_hung = true;
2540 
2541 	/* Reschedule the task */
2542 	if (!test_bit(__E1000_DOWN, &adapter->flags))
2543 		schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2544 }
2545 
2546 enum latency_range {
2547 	lowest_latency = 0,
2548 	low_latency = 1,
2549 	bulk_latency = 2,
2550 	latency_invalid = 255
2551 };
2552 
2553 /**
2554  * e1000_update_itr - update the dynamic ITR value based on statistics
2555  * @adapter: pointer to adapter
2556  * @itr_setting: current adapter->itr
2557  * @packets: the number of packets during this measurement interval
2558  * @bytes: the number of bytes during this measurement interval
2559  *
2560  *      Stores a new ITR value based on packets and byte
2561  *      counts during the last interrupt.  The advantage of per interrupt
2562  *      computation is faster updates and more accurate ITR for the current
2563  *      traffic pattern.  Constants in this function were computed
2564  *      based on theoretical maximum wire speed and thresholds were set based
2565  *      on testing data as well as attempting to minimize response time
2566  *      while increasing bulk throughput.
2567  *      this functionality is controlled by the InterruptThrottleRate module
2568  *      parameter (see e1000_param.c)
2569  **/
2570 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2571 				     u16 itr_setting, int packets, int bytes)
2572 {
2573 	unsigned int retval = itr_setting;
2574 	struct e1000_hw *hw = &adapter->hw;
2575 
2576 	if (unlikely(hw->mac_type < e1000_82540))
2577 		goto update_itr_done;
2578 
2579 	if (packets == 0)
2580 		goto update_itr_done;
2581 
2582 	switch (itr_setting) {
2583 	case lowest_latency:
2584 		/* jumbo frames get bulk treatment*/
2585 		if (bytes/packets > 8000)
2586 			retval = bulk_latency;
2587 		else if ((packets < 5) && (bytes > 512))
2588 			retval = low_latency;
2589 		break;
2590 	case low_latency:  /* 50 usec aka 20000 ints/s */
2591 		if (bytes > 10000) {
2592 			/* jumbo frames need bulk latency setting */
2593 			if (bytes/packets > 8000)
2594 				retval = bulk_latency;
2595 			else if ((packets < 10) || ((bytes/packets) > 1200))
2596 				retval = bulk_latency;
2597 			else if ((packets > 35))
2598 				retval = lowest_latency;
2599 		} else if (bytes/packets > 2000)
2600 			retval = bulk_latency;
2601 		else if (packets <= 2 && bytes < 512)
2602 			retval = lowest_latency;
2603 		break;
2604 	case bulk_latency: /* 250 usec aka 4000 ints/s */
2605 		if (bytes > 25000) {
2606 			if (packets > 35)
2607 				retval = low_latency;
2608 		} else if (bytes < 6000) {
2609 			retval = low_latency;
2610 		}
2611 		break;
2612 	}
2613 
2614 update_itr_done:
2615 	return retval;
2616 }
2617 
2618 static void e1000_set_itr(struct e1000_adapter *adapter)
2619 {
2620 	struct e1000_hw *hw = &adapter->hw;
2621 	u16 current_itr;
2622 	u32 new_itr = adapter->itr;
2623 
2624 	if (unlikely(hw->mac_type < e1000_82540))
2625 		return;
2626 
2627 	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2628 	if (unlikely(adapter->link_speed != SPEED_1000)) {
2629 		current_itr = 0;
2630 		new_itr = 4000;
2631 		goto set_itr_now;
2632 	}
2633 
2634 	adapter->tx_itr = e1000_update_itr(adapter, adapter->tx_itr,
2635 					   adapter->total_tx_packets,
2636 					   adapter->total_tx_bytes);
2637 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
2638 	if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2639 		adapter->tx_itr = low_latency;
2640 
2641 	adapter->rx_itr = e1000_update_itr(adapter, adapter->rx_itr,
2642 					   adapter->total_rx_packets,
2643 					   adapter->total_rx_bytes);
2644 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
2645 	if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2646 		adapter->rx_itr = low_latency;
2647 
2648 	current_itr = max(adapter->rx_itr, adapter->tx_itr);
2649 
2650 	switch (current_itr) {
2651 	/* counts and packets in update_itr are dependent on these numbers */
2652 	case lowest_latency:
2653 		new_itr = 70000;
2654 		break;
2655 	case low_latency:
2656 		new_itr = 20000; /* aka hwitr = ~200 */
2657 		break;
2658 	case bulk_latency:
2659 		new_itr = 4000;
2660 		break;
2661 	default:
2662 		break;
2663 	}
2664 
2665 set_itr_now:
2666 	if (new_itr != adapter->itr) {
2667 		/* this attempts to bias the interrupt rate towards Bulk
2668 		 * by adding intermediate steps when interrupt rate is
2669 		 * increasing
2670 		 */
2671 		new_itr = new_itr > adapter->itr ?
2672 			  min(adapter->itr + (new_itr >> 2), new_itr) :
2673 			  new_itr;
2674 		adapter->itr = new_itr;
2675 		ew32(ITR, 1000000000 / (new_itr * 256));
2676 	}
2677 }
2678 
2679 #define E1000_TX_FLAGS_CSUM		0x00000001
2680 #define E1000_TX_FLAGS_VLAN		0x00000002
2681 #define E1000_TX_FLAGS_TSO		0x00000004
2682 #define E1000_TX_FLAGS_IPV4		0x00000008
2683 #define E1000_TX_FLAGS_NO_FCS		0x00000010
2684 #define E1000_TX_FLAGS_VLAN_MASK	0xffff0000
2685 #define E1000_TX_FLAGS_VLAN_SHIFT	16
2686 
2687 static int e1000_tso(struct e1000_adapter *adapter,
2688 		     struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2689 		     __be16 protocol)
2690 {
2691 	struct e1000_context_desc *context_desc;
2692 	struct e1000_tx_buffer *buffer_info;
2693 	unsigned int i;
2694 	u32 cmd_length = 0;
2695 	u16 ipcse = 0, tucse, mss;
2696 	u8 ipcss, ipcso, tucss, tucso, hdr_len;
2697 
2698 	if (skb_is_gso(skb)) {
2699 		int err;
2700 
2701 		err = skb_cow_head(skb, 0);
2702 		if (err < 0)
2703 			return err;
2704 
2705 		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2706 		mss = skb_shinfo(skb)->gso_size;
2707 		if (protocol == htons(ETH_P_IP)) {
2708 			struct iphdr *iph = ip_hdr(skb);
2709 			iph->tot_len = 0;
2710 			iph->check = 0;
2711 			tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2712 								 iph->daddr, 0,
2713 								 IPPROTO_TCP,
2714 								 0);
2715 			cmd_length = E1000_TXD_CMD_IP;
2716 			ipcse = skb_transport_offset(skb) - 1;
2717 		} else if (skb_is_gso_v6(skb)) {
2718 			ipv6_hdr(skb)->payload_len = 0;
2719 			tcp_hdr(skb)->check =
2720 				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2721 						 &ipv6_hdr(skb)->daddr,
2722 						 0, IPPROTO_TCP, 0);
2723 			ipcse = 0;
2724 		}
2725 		ipcss = skb_network_offset(skb);
2726 		ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2727 		tucss = skb_transport_offset(skb);
2728 		tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
2729 		tucse = 0;
2730 
2731 		cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2732 			       E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2733 
2734 		i = tx_ring->next_to_use;
2735 		context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2736 		buffer_info = &tx_ring->buffer_info[i];
2737 
2738 		context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2739 		context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2740 		context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2741 		context_desc->upper_setup.tcp_fields.tucss = tucss;
2742 		context_desc->upper_setup.tcp_fields.tucso = tucso;
2743 		context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2744 		context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2745 		context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2746 		context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2747 
2748 		buffer_info->time_stamp = jiffies;
2749 		buffer_info->next_to_watch = i;
2750 
2751 		if (++i == tx_ring->count)
2752 			i = 0;
2753 
2754 		tx_ring->next_to_use = i;
2755 
2756 		return true;
2757 	}
2758 	return false;
2759 }
2760 
2761 static bool e1000_tx_csum(struct e1000_adapter *adapter,
2762 			  struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2763 			  __be16 protocol)
2764 {
2765 	struct e1000_context_desc *context_desc;
2766 	struct e1000_tx_buffer *buffer_info;
2767 	unsigned int i;
2768 	u8 css;
2769 	u32 cmd_len = E1000_TXD_CMD_DEXT;
2770 
2771 	if (skb->ip_summed != CHECKSUM_PARTIAL)
2772 		return false;
2773 
2774 	switch (protocol) {
2775 	case cpu_to_be16(ETH_P_IP):
2776 		if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2777 			cmd_len |= E1000_TXD_CMD_TCP;
2778 		break;
2779 	case cpu_to_be16(ETH_P_IPV6):
2780 		/* XXX not handling all IPV6 headers */
2781 		if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2782 			cmd_len |= E1000_TXD_CMD_TCP;
2783 		break;
2784 	default:
2785 		if (unlikely(net_ratelimit()))
2786 			e_warn(drv, "checksum_partial proto=%x!\n",
2787 			       skb->protocol);
2788 		break;
2789 	}
2790 
2791 	css = skb_checksum_start_offset(skb);
2792 
2793 	i = tx_ring->next_to_use;
2794 	buffer_info = &tx_ring->buffer_info[i];
2795 	context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2796 
2797 	context_desc->lower_setup.ip_config = 0;
2798 	context_desc->upper_setup.tcp_fields.tucss = css;
2799 	context_desc->upper_setup.tcp_fields.tucso =
2800 		css + skb->csum_offset;
2801 	context_desc->upper_setup.tcp_fields.tucse = 0;
2802 	context_desc->tcp_seg_setup.data = 0;
2803 	context_desc->cmd_and_length = cpu_to_le32(cmd_len);
2804 
2805 	buffer_info->time_stamp = jiffies;
2806 	buffer_info->next_to_watch = i;
2807 
2808 	if (unlikely(++i == tx_ring->count))
2809 		i = 0;
2810 
2811 	tx_ring->next_to_use = i;
2812 
2813 	return true;
2814 }
2815 
2816 #define E1000_MAX_TXD_PWR	12
2817 #define E1000_MAX_DATA_PER_TXD	(1<<E1000_MAX_TXD_PWR)
2818 
2819 static int e1000_tx_map(struct e1000_adapter *adapter,
2820 			struct e1000_tx_ring *tx_ring,
2821 			struct sk_buff *skb, unsigned int first,
2822 			unsigned int max_per_txd, unsigned int nr_frags,
2823 			unsigned int mss)
2824 {
2825 	struct e1000_hw *hw = &adapter->hw;
2826 	struct pci_dev *pdev = adapter->pdev;
2827 	struct e1000_tx_buffer *buffer_info;
2828 	unsigned int len = skb_headlen(skb);
2829 	unsigned int offset = 0, size, count = 0, i;
2830 	unsigned int f, bytecount, segs;
2831 
2832 	i = tx_ring->next_to_use;
2833 
2834 	while (len) {
2835 		buffer_info = &tx_ring->buffer_info[i];
2836 		size = min(len, max_per_txd);
2837 		/* Workaround for Controller erratum --
2838 		 * descriptor for non-tso packet in a linear SKB that follows a
2839 		 * tso gets written back prematurely before the data is fully
2840 		 * DMA'd to the controller
2841 		 */
2842 		if (!skb->data_len && tx_ring->last_tx_tso &&
2843 		    !skb_is_gso(skb)) {
2844 			tx_ring->last_tx_tso = false;
2845 			size -= 4;
2846 		}
2847 
2848 		/* Workaround for premature desc write-backs
2849 		 * in TSO mode.  Append 4-byte sentinel desc
2850 		 */
2851 		if (unlikely(mss && !nr_frags && size == len && size > 8))
2852 			size -= 4;
2853 		/* work-around for errata 10 and it applies
2854 		 * to all controllers in PCI-X mode
2855 		 * The fix is to make sure that the first descriptor of a
2856 		 * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2857 		 */
2858 		if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2859 			     (size > 2015) && count == 0))
2860 			size = 2015;
2861 
2862 		/* Workaround for potential 82544 hang in PCI-X.  Avoid
2863 		 * terminating buffers within evenly-aligned dwords.
2864 		 */
2865 		if (unlikely(adapter->pcix_82544 &&
2866 		   !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2867 		   size > 4))
2868 			size -= 4;
2869 
2870 		buffer_info->length = size;
2871 		/* set time_stamp *before* dma to help avoid a possible race */
2872 		buffer_info->time_stamp = jiffies;
2873 		buffer_info->mapped_as_page = false;
2874 		buffer_info->dma = dma_map_single(&pdev->dev,
2875 						  skb->data + offset,
2876 						  size, DMA_TO_DEVICE);
2877 		if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2878 			goto dma_error;
2879 		buffer_info->next_to_watch = i;
2880 
2881 		len -= size;
2882 		offset += size;
2883 		count++;
2884 		if (len) {
2885 			i++;
2886 			if (unlikely(i == tx_ring->count))
2887 				i = 0;
2888 		}
2889 	}
2890 
2891 	for (f = 0; f < nr_frags; f++) {
2892 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2893 
2894 		len = skb_frag_size(frag);
2895 		offset = 0;
2896 
2897 		while (len) {
2898 			unsigned long bufend;
2899 			i++;
2900 			if (unlikely(i == tx_ring->count))
2901 				i = 0;
2902 
2903 			buffer_info = &tx_ring->buffer_info[i];
2904 			size = min(len, max_per_txd);
2905 			/* Workaround for premature desc write-backs
2906 			 * in TSO mode.  Append 4-byte sentinel desc
2907 			 */
2908 			if (unlikely(mss && f == (nr_frags-1) &&
2909 			    size == len && size > 8))
2910 				size -= 4;
2911 			/* Workaround for potential 82544 hang in PCI-X.
2912 			 * Avoid terminating buffers within evenly-aligned
2913 			 * dwords.
2914 			 */
2915 			bufend = (unsigned long)
2916 				page_to_phys(skb_frag_page(frag));
2917 			bufend += offset + size - 1;
2918 			if (unlikely(adapter->pcix_82544 &&
2919 				     !(bufend & 4) &&
2920 				     size > 4))
2921 				size -= 4;
2922 
2923 			buffer_info->length = size;
2924 			buffer_info->time_stamp = jiffies;
2925 			buffer_info->mapped_as_page = true;
2926 			buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
2927 						offset, size, DMA_TO_DEVICE);
2928 			if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2929 				goto dma_error;
2930 			buffer_info->next_to_watch = i;
2931 
2932 			len -= size;
2933 			offset += size;
2934 			count++;
2935 		}
2936 	}
2937 
2938 	segs = skb_shinfo(skb)->gso_segs ?: 1;
2939 	/* multiply data chunks by size of headers */
2940 	bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
2941 
2942 	tx_ring->buffer_info[i].skb = skb;
2943 	tx_ring->buffer_info[i].segs = segs;
2944 	tx_ring->buffer_info[i].bytecount = bytecount;
2945 	tx_ring->buffer_info[first].next_to_watch = i;
2946 
2947 	return count;
2948 
2949 dma_error:
2950 	dev_err(&pdev->dev, "TX DMA map failed\n");
2951 	buffer_info->dma = 0;
2952 	if (count)
2953 		count--;
2954 
2955 	while (count--) {
2956 		if (i == 0)
2957 			i += tx_ring->count;
2958 		i--;
2959 		buffer_info = &tx_ring->buffer_info[i];
2960 		e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2961 	}
2962 
2963 	return 0;
2964 }
2965 
2966 static void e1000_tx_queue(struct e1000_adapter *adapter,
2967 			   struct e1000_tx_ring *tx_ring, int tx_flags,
2968 			   int count)
2969 {
2970 	struct e1000_tx_desc *tx_desc = NULL;
2971 	struct e1000_tx_buffer *buffer_info;
2972 	u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
2973 	unsigned int i;
2974 
2975 	if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
2976 		txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
2977 			     E1000_TXD_CMD_TSE;
2978 		txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2979 
2980 		if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
2981 			txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2982 	}
2983 
2984 	if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
2985 		txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2986 		txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2987 	}
2988 
2989 	if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
2990 		txd_lower |= E1000_TXD_CMD_VLE;
2991 		txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
2992 	}
2993 
2994 	if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
2995 		txd_lower &= ~(E1000_TXD_CMD_IFCS);
2996 
2997 	i = tx_ring->next_to_use;
2998 
2999 	while (count--) {
3000 		buffer_info = &tx_ring->buffer_info[i];
3001 		tx_desc = E1000_TX_DESC(*tx_ring, i);
3002 		tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3003 		tx_desc->lower.data =
3004 			cpu_to_le32(txd_lower | buffer_info->length);
3005 		tx_desc->upper.data = cpu_to_le32(txd_upper);
3006 		if (unlikely(++i == tx_ring->count))
3007 			i = 0;
3008 	}
3009 
3010 	tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
3011 
3012 	/* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
3013 	if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3014 		tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
3015 
3016 	/* Force memory writes to complete before letting h/w
3017 	 * know there are new descriptors to fetch.  (Only
3018 	 * applicable for weak-ordered memory model archs,
3019 	 * such as IA-64).
3020 	 */
3021 	dma_wmb();
3022 
3023 	tx_ring->next_to_use = i;
3024 }
3025 
3026 /* 82547 workaround to avoid controller hang in half-duplex environment.
3027  * The workaround is to avoid queuing a large packet that would span
3028  * the internal Tx FIFO ring boundary by notifying the stack to resend
3029  * the packet at a later time.  This gives the Tx FIFO an opportunity to
3030  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
3031  * to the beginning of the Tx FIFO.
3032  */
3033 
3034 #define E1000_FIFO_HDR			0x10
3035 #define E1000_82547_PAD_LEN		0x3E0
3036 
3037 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
3038 				       struct sk_buff *skb)
3039 {
3040 	u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
3041 	u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
3042 
3043 	skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
3044 
3045 	if (adapter->link_duplex != HALF_DUPLEX)
3046 		goto no_fifo_stall_required;
3047 
3048 	if (atomic_read(&adapter->tx_fifo_stall))
3049 		return 1;
3050 
3051 	if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
3052 		atomic_set(&adapter->tx_fifo_stall, 1);
3053 		return 1;
3054 	}
3055 
3056 no_fifo_stall_required:
3057 	adapter->tx_fifo_head += skb_fifo_len;
3058 	if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
3059 		adapter->tx_fifo_head -= adapter->tx_fifo_size;
3060 	return 0;
3061 }
3062 
3063 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
3064 {
3065 	struct e1000_adapter *adapter = netdev_priv(netdev);
3066 	struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3067 
3068 	netif_stop_queue(netdev);
3069 	/* Herbert's original patch had:
3070 	 *  smp_mb__after_netif_stop_queue();
3071 	 * but since that doesn't exist yet, just open code it.
3072 	 */
3073 	smp_mb();
3074 
3075 	/* We need to check again in a case another CPU has just
3076 	 * made room available.
3077 	 */
3078 	if (likely(E1000_DESC_UNUSED(tx_ring) < size))
3079 		return -EBUSY;
3080 
3081 	/* A reprieve! */
3082 	netif_start_queue(netdev);
3083 	++adapter->restart_queue;
3084 	return 0;
3085 }
3086 
3087 static int e1000_maybe_stop_tx(struct net_device *netdev,
3088 			       struct e1000_tx_ring *tx_ring, int size)
3089 {
3090 	if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
3091 		return 0;
3092 	return __e1000_maybe_stop_tx(netdev, size);
3093 }
3094 
3095 #define TXD_USE_COUNT(S, X) (((S) + ((1 << (X)) - 1)) >> (X))
3096 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
3097 				    struct net_device *netdev)
3098 {
3099 	struct e1000_adapter *adapter = netdev_priv(netdev);
3100 	struct e1000_hw *hw = &adapter->hw;
3101 	struct e1000_tx_ring *tx_ring;
3102 	unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
3103 	unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
3104 	unsigned int tx_flags = 0;
3105 	unsigned int len = skb_headlen(skb);
3106 	unsigned int nr_frags;
3107 	unsigned int mss;
3108 	int count = 0;
3109 	int tso;
3110 	unsigned int f;
3111 	__be16 protocol = vlan_get_protocol(skb);
3112 
3113 	/* This goes back to the question of how to logically map a Tx queue
3114 	 * to a flow.  Right now, performance is impacted slightly negatively
3115 	 * if using multiple Tx queues.  If the stack breaks away from a
3116 	 * single qdisc implementation, we can look at this again.
3117 	 */
3118 	tx_ring = adapter->tx_ring;
3119 
3120 	/* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN,
3121 	 * packets may get corrupted during padding by HW.
3122 	 * To WA this issue, pad all small packets manually.
3123 	 */
3124 	if (eth_skb_pad(skb))
3125 		return NETDEV_TX_OK;
3126 
3127 	mss = skb_shinfo(skb)->gso_size;
3128 	/* The controller does a simple calculation to
3129 	 * make sure there is enough room in the FIFO before
3130 	 * initiating the DMA for each buffer.  The calc is:
3131 	 * 4 = ceil(buffer len/mss).  To make sure we don't
3132 	 * overrun the FIFO, adjust the max buffer len if mss
3133 	 * drops.
3134 	 */
3135 	if (mss) {
3136 		u8 hdr_len;
3137 		max_per_txd = min(mss << 2, max_per_txd);
3138 		max_txd_pwr = fls(max_per_txd) - 1;
3139 
3140 		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3141 		if (skb->data_len && hdr_len == len) {
3142 			switch (hw->mac_type) {
3143 				unsigned int pull_size;
3144 			case e1000_82544:
3145 				/* Make sure we have room to chop off 4 bytes,
3146 				 * and that the end alignment will work out to
3147 				 * this hardware's requirements
3148 				 * NOTE: this is a TSO only workaround
3149 				 * if end byte alignment not correct move us
3150 				 * into the next dword
3151 				 */
3152 				if ((unsigned long)(skb_tail_pointer(skb) - 1)
3153 				    & 4)
3154 					break;
3155 				/* fall through */
3156 				pull_size = min((unsigned int)4, skb->data_len);
3157 				if (!__pskb_pull_tail(skb, pull_size)) {
3158 					e_err(drv, "__pskb_pull_tail "
3159 					      "failed.\n");
3160 					dev_kfree_skb_any(skb);
3161 					return NETDEV_TX_OK;
3162 				}
3163 				len = skb_headlen(skb);
3164 				break;
3165 			default:
3166 				/* do nothing */
3167 				break;
3168 			}
3169 		}
3170 	}
3171 
3172 	/* reserve a descriptor for the offload context */
3173 	if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
3174 		count++;
3175 	count++;
3176 
3177 	/* Controller Erratum workaround */
3178 	if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
3179 		count++;
3180 
3181 	count += TXD_USE_COUNT(len, max_txd_pwr);
3182 
3183 	if (adapter->pcix_82544)
3184 		count++;
3185 
3186 	/* work-around for errata 10 and it applies to all controllers
3187 	 * in PCI-X mode, so add one more descriptor to the count
3188 	 */
3189 	if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3190 			(len > 2015)))
3191 		count++;
3192 
3193 	nr_frags = skb_shinfo(skb)->nr_frags;
3194 	for (f = 0; f < nr_frags; f++)
3195 		count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
3196 				       max_txd_pwr);
3197 	if (adapter->pcix_82544)
3198 		count += nr_frags;
3199 
3200 	/* need: count + 2 desc gap to keep tail from touching
3201 	 * head, otherwise try next time
3202 	 */
3203 	if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
3204 		return NETDEV_TX_BUSY;
3205 
3206 	if (unlikely((hw->mac_type == e1000_82547) &&
3207 		     (e1000_82547_fifo_workaround(adapter, skb)))) {
3208 		netif_stop_queue(netdev);
3209 		if (!test_bit(__E1000_DOWN, &adapter->flags))
3210 			schedule_delayed_work(&adapter->fifo_stall_task, 1);
3211 		return NETDEV_TX_BUSY;
3212 	}
3213 
3214 	if (skb_vlan_tag_present(skb)) {
3215 		tx_flags |= E1000_TX_FLAGS_VLAN;
3216 		tx_flags |= (skb_vlan_tag_get(skb) <<
3217 			     E1000_TX_FLAGS_VLAN_SHIFT);
3218 	}
3219 
3220 	first = tx_ring->next_to_use;
3221 
3222 	tso = e1000_tso(adapter, tx_ring, skb, protocol);
3223 	if (tso < 0) {
3224 		dev_kfree_skb_any(skb);
3225 		return NETDEV_TX_OK;
3226 	}
3227 
3228 	if (likely(tso)) {
3229 		if (likely(hw->mac_type != e1000_82544))
3230 			tx_ring->last_tx_tso = true;
3231 		tx_flags |= E1000_TX_FLAGS_TSO;
3232 	} else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol)))
3233 		tx_flags |= E1000_TX_FLAGS_CSUM;
3234 
3235 	if (protocol == htons(ETH_P_IP))
3236 		tx_flags |= E1000_TX_FLAGS_IPV4;
3237 
3238 	if (unlikely(skb->no_fcs))
3239 		tx_flags |= E1000_TX_FLAGS_NO_FCS;
3240 
3241 	count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
3242 			     nr_frags, mss);
3243 
3244 	if (count) {
3245 		/* The descriptors needed is higher than other Intel drivers
3246 		 * due to a number of workarounds.  The breakdown is below:
3247 		 * Data descriptors: MAX_SKB_FRAGS + 1
3248 		 * Context Descriptor: 1
3249 		 * Keep head from touching tail: 2
3250 		 * Workarounds: 3
3251 		 */
3252 		int desc_needed = MAX_SKB_FRAGS + 7;
3253 
3254 		netdev_sent_queue(netdev, skb->len);
3255 		skb_tx_timestamp(skb);
3256 
3257 		e1000_tx_queue(adapter, tx_ring, tx_flags, count);
3258 
3259 		/* 82544 potentially requires twice as many data descriptors
3260 		 * in order to guarantee buffers don't end on evenly-aligned
3261 		 * dwords
3262 		 */
3263 		if (adapter->pcix_82544)
3264 			desc_needed += MAX_SKB_FRAGS + 1;
3265 
3266 		/* Make sure there is space in the ring for the next send. */
3267 		e1000_maybe_stop_tx(netdev, tx_ring, desc_needed);
3268 
3269 		if (!netdev_xmit_more() ||
3270 		    netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) {
3271 			writel(tx_ring->next_to_use, hw->hw_addr + tx_ring->tdt);
3272 		}
3273 	} else {
3274 		dev_kfree_skb_any(skb);
3275 		tx_ring->buffer_info[first].time_stamp = 0;
3276 		tx_ring->next_to_use = first;
3277 	}
3278 
3279 	return NETDEV_TX_OK;
3280 }
3281 
3282 #define NUM_REGS 38 /* 1 based count */
3283 static void e1000_regdump(struct e1000_adapter *adapter)
3284 {
3285 	struct e1000_hw *hw = &adapter->hw;
3286 	u32 regs[NUM_REGS];
3287 	u32 *regs_buff = regs;
3288 	int i = 0;
3289 
3290 	static const char * const reg_name[] = {
3291 		"CTRL",  "STATUS",
3292 		"RCTL", "RDLEN", "RDH", "RDT", "RDTR",
3293 		"TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
3294 		"TIDV", "TXDCTL", "TADV", "TARC0",
3295 		"TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
3296 		"TXDCTL1", "TARC1",
3297 		"CTRL_EXT", "ERT", "RDBAL", "RDBAH",
3298 		"TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
3299 		"RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
3300 	};
3301 
3302 	regs_buff[0]  = er32(CTRL);
3303 	regs_buff[1]  = er32(STATUS);
3304 
3305 	regs_buff[2]  = er32(RCTL);
3306 	regs_buff[3]  = er32(RDLEN);
3307 	regs_buff[4]  = er32(RDH);
3308 	regs_buff[5]  = er32(RDT);
3309 	regs_buff[6]  = er32(RDTR);
3310 
3311 	regs_buff[7]  = er32(TCTL);
3312 	regs_buff[8]  = er32(TDBAL);
3313 	regs_buff[9]  = er32(TDBAH);
3314 	regs_buff[10] = er32(TDLEN);
3315 	regs_buff[11] = er32(TDH);
3316 	regs_buff[12] = er32(TDT);
3317 	regs_buff[13] = er32(TIDV);
3318 	regs_buff[14] = er32(TXDCTL);
3319 	regs_buff[15] = er32(TADV);
3320 	regs_buff[16] = er32(TARC0);
3321 
3322 	regs_buff[17] = er32(TDBAL1);
3323 	regs_buff[18] = er32(TDBAH1);
3324 	regs_buff[19] = er32(TDLEN1);
3325 	regs_buff[20] = er32(TDH1);
3326 	regs_buff[21] = er32(TDT1);
3327 	regs_buff[22] = er32(TXDCTL1);
3328 	regs_buff[23] = er32(TARC1);
3329 	regs_buff[24] = er32(CTRL_EXT);
3330 	regs_buff[25] = er32(ERT);
3331 	regs_buff[26] = er32(RDBAL0);
3332 	regs_buff[27] = er32(RDBAH0);
3333 	regs_buff[28] = er32(TDFH);
3334 	regs_buff[29] = er32(TDFT);
3335 	regs_buff[30] = er32(TDFHS);
3336 	regs_buff[31] = er32(TDFTS);
3337 	regs_buff[32] = er32(TDFPC);
3338 	regs_buff[33] = er32(RDFH);
3339 	regs_buff[34] = er32(RDFT);
3340 	regs_buff[35] = er32(RDFHS);
3341 	regs_buff[36] = er32(RDFTS);
3342 	regs_buff[37] = er32(RDFPC);
3343 
3344 	pr_info("Register dump\n");
3345 	for (i = 0; i < NUM_REGS; i++)
3346 		pr_info("%-15s  %08x\n", reg_name[i], regs_buff[i]);
3347 }
3348 
3349 /*
3350  * e1000_dump: Print registers, tx ring and rx ring
3351  */
3352 static void e1000_dump(struct e1000_adapter *adapter)
3353 {
3354 	/* this code doesn't handle multiple rings */
3355 	struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3356 	struct e1000_rx_ring *rx_ring = adapter->rx_ring;
3357 	int i;
3358 
3359 	if (!netif_msg_hw(adapter))
3360 		return;
3361 
3362 	/* Print Registers */
3363 	e1000_regdump(adapter);
3364 
3365 	/* transmit dump */
3366 	pr_info("TX Desc ring0 dump\n");
3367 
3368 	/* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
3369 	 *
3370 	 * Legacy Transmit Descriptor
3371 	 *   +--------------------------------------------------------------+
3372 	 * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
3373 	 *   +--------------------------------------------------------------+
3374 	 * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
3375 	 *   +--------------------------------------------------------------+
3376 	 *   63       48 47        36 35    32 31     24 23    16 15        0
3377 	 *
3378 	 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
3379 	 *   63      48 47    40 39       32 31             16 15    8 7      0
3380 	 *   +----------------------------------------------------------------+
3381 	 * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
3382 	 *   +----------------------------------------------------------------+
3383 	 * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
3384 	 *   +----------------------------------------------------------------+
3385 	 *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
3386 	 *
3387 	 * Extended Data Descriptor (DTYP=0x1)
3388 	 *   +----------------------------------------------------------------+
3389 	 * 0 |                     Buffer Address [63:0]                      |
3390 	 *   +----------------------------------------------------------------+
3391 	 * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
3392 	 *   +----------------------------------------------------------------+
3393 	 *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
3394 	 */
3395 	pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3396 	pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3397 
3398 	if (!netif_msg_tx_done(adapter))
3399 		goto rx_ring_summary;
3400 
3401 	for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
3402 		struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
3403 		struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i];
3404 		struct my_u { __le64 a; __le64 b; };
3405 		struct my_u *u = (struct my_u *)tx_desc;
3406 		const char *type;
3407 
3408 		if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
3409 			type = "NTC/U";
3410 		else if (i == tx_ring->next_to_use)
3411 			type = "NTU";
3412 		else if (i == tx_ring->next_to_clean)
3413 			type = "NTC";
3414 		else
3415 			type = "";
3416 
3417 		pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p %s\n",
3418 			((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
3419 			le64_to_cpu(u->a), le64_to_cpu(u->b),
3420 			(u64)buffer_info->dma, buffer_info->length,
3421 			buffer_info->next_to_watch,
3422 			(u64)buffer_info->time_stamp, buffer_info->skb, type);
3423 	}
3424 
3425 rx_ring_summary:
3426 	/* receive dump */
3427 	pr_info("\nRX Desc ring dump\n");
3428 
3429 	/* Legacy Receive Descriptor Format
3430 	 *
3431 	 * +-----------------------------------------------------+
3432 	 * |                Buffer Address [63:0]                |
3433 	 * +-----------------------------------------------------+
3434 	 * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
3435 	 * +-----------------------------------------------------+
3436 	 * 63       48 47    40 39      32 31         16 15      0
3437 	 */
3438 	pr_info("R[desc]      [address 63:0  ] [vl er S cks ln] [bi->dma       ] [bi->skb]\n");
3439 
3440 	if (!netif_msg_rx_status(adapter))
3441 		goto exit;
3442 
3443 	for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
3444 		struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
3445 		struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i];
3446 		struct my_u { __le64 a; __le64 b; };
3447 		struct my_u *u = (struct my_u *)rx_desc;
3448 		const char *type;
3449 
3450 		if (i == rx_ring->next_to_use)
3451 			type = "NTU";
3452 		else if (i == rx_ring->next_to_clean)
3453 			type = "NTC";
3454 		else
3455 			type = "";
3456 
3457 		pr_info("R[0x%03X]     %016llX %016llX %016llX %p %s\n",
3458 			i, le64_to_cpu(u->a), le64_to_cpu(u->b),
3459 			(u64)buffer_info->dma, buffer_info->rxbuf.data, type);
3460 	} /* for */
3461 
3462 	/* dump the descriptor caches */
3463 	/* rx */
3464 	pr_info("Rx descriptor cache in 64bit format\n");
3465 	for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
3466 		pr_info("R%04X: %08X|%08X %08X|%08X\n",
3467 			i,
3468 			readl(adapter->hw.hw_addr + i+4),
3469 			readl(adapter->hw.hw_addr + i),
3470 			readl(adapter->hw.hw_addr + i+12),
3471 			readl(adapter->hw.hw_addr + i+8));
3472 	}
3473 	/* tx */
3474 	pr_info("Tx descriptor cache in 64bit format\n");
3475 	for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
3476 		pr_info("T%04X: %08X|%08X %08X|%08X\n",
3477 			i,
3478 			readl(adapter->hw.hw_addr + i+4),
3479 			readl(adapter->hw.hw_addr + i),
3480 			readl(adapter->hw.hw_addr + i+12),
3481 			readl(adapter->hw.hw_addr + i+8));
3482 	}
3483 exit:
3484 	return;
3485 }
3486 
3487 /**
3488  * e1000_tx_timeout - Respond to a Tx Hang
3489  * @netdev: network interface device structure
3490  **/
3491 static void e1000_tx_timeout(struct net_device *netdev)
3492 {
3493 	struct e1000_adapter *adapter = netdev_priv(netdev);
3494 
3495 	/* Do the reset outside of interrupt context */
3496 	adapter->tx_timeout_count++;
3497 	schedule_work(&adapter->reset_task);
3498 }
3499 
3500 static void e1000_reset_task(struct work_struct *work)
3501 {
3502 	struct e1000_adapter *adapter =
3503 		container_of(work, struct e1000_adapter, reset_task);
3504 
3505 	e_err(drv, "Reset adapter\n");
3506 	e1000_reinit_locked(adapter);
3507 }
3508 
3509 /**
3510  * e1000_change_mtu - Change the Maximum Transfer Unit
3511  * @netdev: network interface device structure
3512  * @new_mtu: new value for maximum frame size
3513  *
3514  * Returns 0 on success, negative on failure
3515  **/
3516 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
3517 {
3518 	struct e1000_adapter *adapter = netdev_priv(netdev);
3519 	struct e1000_hw *hw = &adapter->hw;
3520 	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3521 
3522 	/* Adapter-specific max frame size limits. */
3523 	switch (hw->mac_type) {
3524 	case e1000_undefined ... e1000_82542_rev2_1:
3525 		if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3526 			e_err(probe, "Jumbo Frames not supported.\n");
3527 			return -EINVAL;
3528 		}
3529 		break;
3530 	default:
3531 		/* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
3532 		break;
3533 	}
3534 
3535 	while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
3536 		msleep(1);
3537 	/* e1000_down has a dependency on max_frame_size */
3538 	hw->max_frame_size = max_frame;
3539 	if (netif_running(netdev)) {
3540 		/* prevent buffers from being reallocated */
3541 		adapter->alloc_rx_buf = e1000_alloc_dummy_rx_buffers;
3542 		e1000_down(adapter);
3543 	}
3544 
3545 	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3546 	 * means we reserve 2 more, this pushes us to allocate from the next
3547 	 * larger slab size.
3548 	 * i.e. RXBUFFER_2048 --> size-4096 slab
3549 	 * however with the new *_jumbo_rx* routines, jumbo receives will use
3550 	 * fragmented skbs
3551 	 */
3552 
3553 	if (max_frame <= E1000_RXBUFFER_2048)
3554 		adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3555 	else
3556 #if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3557 		adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3558 #elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
3559 		adapter->rx_buffer_len = PAGE_SIZE;
3560 #endif
3561 
3562 	/* adjust allocation if LPE protects us, and we aren't using SBP */
3563 	if (!hw->tbi_compatibility_on &&
3564 	    ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3565 	     (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
3566 		adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3567 
3568 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
3569 		   netdev->mtu, new_mtu);
3570 	netdev->mtu = new_mtu;
3571 
3572 	if (netif_running(netdev))
3573 		e1000_up(adapter);
3574 	else
3575 		e1000_reset(adapter);
3576 
3577 	clear_bit(__E1000_RESETTING, &adapter->flags);
3578 
3579 	return 0;
3580 }
3581 
3582 /**
3583  * e1000_update_stats - Update the board statistics counters
3584  * @adapter: board private structure
3585  **/
3586 void e1000_update_stats(struct e1000_adapter *adapter)
3587 {
3588 	struct net_device *netdev = adapter->netdev;
3589 	struct e1000_hw *hw = &adapter->hw;
3590 	struct pci_dev *pdev = adapter->pdev;
3591 	unsigned long flags;
3592 	u16 phy_tmp;
3593 
3594 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3595 
3596 	/* Prevent stats update while adapter is being reset, or if the pci
3597 	 * connection is down.
3598 	 */
3599 	if (adapter->link_speed == 0)
3600 		return;
3601 	if (pci_channel_offline(pdev))
3602 		return;
3603 
3604 	spin_lock_irqsave(&adapter->stats_lock, flags);
3605 
3606 	/* these counters are modified from e1000_tbi_adjust_stats,
3607 	 * called from the interrupt context, so they must only
3608 	 * be written while holding adapter->stats_lock
3609 	 */
3610 
3611 	adapter->stats.crcerrs += er32(CRCERRS);
3612 	adapter->stats.gprc += er32(GPRC);
3613 	adapter->stats.gorcl += er32(GORCL);
3614 	adapter->stats.gorch += er32(GORCH);
3615 	adapter->stats.bprc += er32(BPRC);
3616 	adapter->stats.mprc += er32(MPRC);
3617 	adapter->stats.roc += er32(ROC);
3618 
3619 	adapter->stats.prc64 += er32(PRC64);
3620 	adapter->stats.prc127 += er32(PRC127);
3621 	adapter->stats.prc255 += er32(PRC255);
3622 	adapter->stats.prc511 += er32(PRC511);
3623 	adapter->stats.prc1023 += er32(PRC1023);
3624 	adapter->stats.prc1522 += er32(PRC1522);
3625 
3626 	adapter->stats.symerrs += er32(SYMERRS);
3627 	adapter->stats.mpc += er32(MPC);
3628 	adapter->stats.scc += er32(SCC);
3629 	adapter->stats.ecol += er32(ECOL);
3630 	adapter->stats.mcc += er32(MCC);
3631 	adapter->stats.latecol += er32(LATECOL);
3632 	adapter->stats.dc += er32(DC);
3633 	adapter->stats.sec += er32(SEC);
3634 	adapter->stats.rlec += er32(RLEC);
3635 	adapter->stats.xonrxc += er32(XONRXC);
3636 	adapter->stats.xontxc += er32(XONTXC);
3637 	adapter->stats.xoffrxc += er32(XOFFRXC);
3638 	adapter->stats.xofftxc += er32(XOFFTXC);
3639 	adapter->stats.fcruc += er32(FCRUC);
3640 	adapter->stats.gptc += er32(GPTC);
3641 	adapter->stats.gotcl += er32(GOTCL);
3642 	adapter->stats.gotch += er32(GOTCH);
3643 	adapter->stats.rnbc += er32(RNBC);
3644 	adapter->stats.ruc += er32(RUC);
3645 	adapter->stats.rfc += er32(RFC);
3646 	adapter->stats.rjc += er32(RJC);
3647 	adapter->stats.torl += er32(TORL);
3648 	adapter->stats.torh += er32(TORH);
3649 	adapter->stats.totl += er32(TOTL);
3650 	adapter->stats.toth += er32(TOTH);
3651 	adapter->stats.tpr += er32(TPR);
3652 
3653 	adapter->stats.ptc64 += er32(PTC64);
3654 	adapter->stats.ptc127 += er32(PTC127);
3655 	adapter->stats.ptc255 += er32(PTC255);
3656 	adapter->stats.ptc511 += er32(PTC511);
3657 	adapter->stats.ptc1023 += er32(PTC1023);
3658 	adapter->stats.ptc1522 += er32(PTC1522);
3659 
3660 	adapter->stats.mptc += er32(MPTC);
3661 	adapter->stats.bptc += er32(BPTC);
3662 
3663 	/* used for adaptive IFS */
3664 
3665 	hw->tx_packet_delta = er32(TPT);
3666 	adapter->stats.tpt += hw->tx_packet_delta;
3667 	hw->collision_delta = er32(COLC);
3668 	adapter->stats.colc += hw->collision_delta;
3669 
3670 	if (hw->mac_type >= e1000_82543) {
3671 		adapter->stats.algnerrc += er32(ALGNERRC);
3672 		adapter->stats.rxerrc += er32(RXERRC);
3673 		adapter->stats.tncrs += er32(TNCRS);
3674 		adapter->stats.cexterr += er32(CEXTERR);
3675 		adapter->stats.tsctc += er32(TSCTC);
3676 		adapter->stats.tsctfc += er32(TSCTFC);
3677 	}
3678 
3679 	/* Fill out the OS statistics structure */
3680 	netdev->stats.multicast = adapter->stats.mprc;
3681 	netdev->stats.collisions = adapter->stats.colc;
3682 
3683 	/* Rx Errors */
3684 
3685 	/* RLEC on some newer hardware can be incorrect so build
3686 	 * our own version based on RUC and ROC
3687 	 */
3688 	netdev->stats.rx_errors = adapter->stats.rxerrc +
3689 		adapter->stats.crcerrs + adapter->stats.algnerrc +
3690 		adapter->stats.ruc + adapter->stats.roc +
3691 		adapter->stats.cexterr;
3692 	adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3693 	netdev->stats.rx_length_errors = adapter->stats.rlerrc;
3694 	netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3695 	netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3696 	netdev->stats.rx_missed_errors = adapter->stats.mpc;
3697 
3698 	/* Tx Errors */
3699 	adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3700 	netdev->stats.tx_errors = adapter->stats.txerrc;
3701 	netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3702 	netdev->stats.tx_window_errors = adapter->stats.latecol;
3703 	netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3704 	if (hw->bad_tx_carr_stats_fd &&
3705 	    adapter->link_duplex == FULL_DUPLEX) {
3706 		netdev->stats.tx_carrier_errors = 0;
3707 		adapter->stats.tncrs = 0;
3708 	}
3709 
3710 	/* Tx Dropped needs to be maintained elsewhere */
3711 
3712 	/* Phy Stats */
3713 	if (hw->media_type == e1000_media_type_copper) {
3714 		if ((adapter->link_speed == SPEED_1000) &&
3715 		   (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3716 			phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3717 			adapter->phy_stats.idle_errors += phy_tmp;
3718 		}
3719 
3720 		if ((hw->mac_type <= e1000_82546) &&
3721 		   (hw->phy_type == e1000_phy_m88) &&
3722 		   !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3723 			adapter->phy_stats.receive_errors += phy_tmp;
3724 	}
3725 
3726 	/* Management Stats */
3727 	if (hw->has_smbus) {
3728 		adapter->stats.mgptc += er32(MGTPTC);
3729 		adapter->stats.mgprc += er32(MGTPRC);
3730 		adapter->stats.mgpdc += er32(MGTPDC);
3731 	}
3732 
3733 	spin_unlock_irqrestore(&adapter->stats_lock, flags);
3734 }
3735 
3736 /**
3737  * e1000_intr - Interrupt Handler
3738  * @irq: interrupt number
3739  * @data: pointer to a network interface device structure
3740  **/
3741 static irqreturn_t e1000_intr(int irq, void *data)
3742 {
3743 	struct net_device *netdev = data;
3744 	struct e1000_adapter *adapter = netdev_priv(netdev);
3745 	struct e1000_hw *hw = &adapter->hw;
3746 	u32 icr = er32(ICR);
3747 
3748 	if (unlikely((!icr)))
3749 		return IRQ_NONE;  /* Not our interrupt */
3750 
3751 	/* we might have caused the interrupt, but the above
3752 	 * read cleared it, and just in case the driver is
3753 	 * down there is nothing to do so return handled
3754 	 */
3755 	if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
3756 		return IRQ_HANDLED;
3757 
3758 	if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3759 		hw->get_link_status = 1;
3760 		/* guard against interrupt when we're going down */
3761 		if (!test_bit(__E1000_DOWN, &adapter->flags))
3762 			schedule_delayed_work(&adapter->watchdog_task, 1);
3763 	}
3764 
3765 	/* disable interrupts, without the synchronize_irq bit */
3766 	ew32(IMC, ~0);
3767 	E1000_WRITE_FLUSH();
3768 
3769 	if (likely(napi_schedule_prep(&adapter->napi))) {
3770 		adapter->total_tx_bytes = 0;
3771 		adapter->total_tx_packets = 0;
3772 		adapter->total_rx_bytes = 0;
3773 		adapter->total_rx_packets = 0;
3774 		__napi_schedule(&adapter->napi);
3775 	} else {
3776 		/* this really should not happen! if it does it is basically a
3777 		 * bug, but not a hard error, so enable ints and continue
3778 		 */
3779 		if (!test_bit(__E1000_DOWN, &adapter->flags))
3780 			e1000_irq_enable(adapter);
3781 	}
3782 
3783 	return IRQ_HANDLED;
3784 }
3785 
3786 /**
3787  * e1000_clean - NAPI Rx polling callback
3788  * @adapter: board private structure
3789  **/
3790 static int e1000_clean(struct napi_struct *napi, int budget)
3791 {
3792 	struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
3793 						     napi);
3794 	int tx_clean_complete = 0, work_done = 0;
3795 
3796 	tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3797 
3798 	adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3799 
3800 	if (!tx_clean_complete || work_done == budget)
3801 		return budget;
3802 
3803 	/* Exit the polling mode, but don't re-enable interrupts if stack might
3804 	 * poll us due to busy-polling
3805 	 */
3806 	if (likely(napi_complete_done(napi, work_done))) {
3807 		if (likely(adapter->itr_setting & 3))
3808 			e1000_set_itr(adapter);
3809 		if (!test_bit(__E1000_DOWN, &adapter->flags))
3810 			e1000_irq_enable(adapter);
3811 	}
3812 
3813 	return work_done;
3814 }
3815 
3816 /**
3817  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3818  * @adapter: board private structure
3819  **/
3820 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
3821 			       struct e1000_tx_ring *tx_ring)
3822 {
3823 	struct e1000_hw *hw = &adapter->hw;
3824 	struct net_device *netdev = adapter->netdev;
3825 	struct e1000_tx_desc *tx_desc, *eop_desc;
3826 	struct e1000_tx_buffer *buffer_info;
3827 	unsigned int i, eop;
3828 	unsigned int count = 0;
3829 	unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3830 	unsigned int bytes_compl = 0, pkts_compl = 0;
3831 
3832 	i = tx_ring->next_to_clean;
3833 	eop = tx_ring->buffer_info[i].next_to_watch;
3834 	eop_desc = E1000_TX_DESC(*tx_ring, eop);
3835 
3836 	while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
3837 	       (count < tx_ring->count)) {
3838 		bool cleaned = false;
3839 		dma_rmb();	/* read buffer_info after eop_desc */
3840 		for ( ; !cleaned; count++) {
3841 			tx_desc = E1000_TX_DESC(*tx_ring, i);
3842 			buffer_info = &tx_ring->buffer_info[i];
3843 			cleaned = (i == eop);
3844 
3845 			if (cleaned) {
3846 				total_tx_packets += buffer_info->segs;
3847 				total_tx_bytes += buffer_info->bytecount;
3848 				if (buffer_info->skb) {
3849 					bytes_compl += buffer_info->skb->len;
3850 					pkts_compl++;
3851 				}
3852 
3853 			}
3854 			e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3855 			tx_desc->upper.data = 0;
3856 
3857 			if (unlikely(++i == tx_ring->count))
3858 				i = 0;
3859 		}
3860 
3861 		eop = tx_ring->buffer_info[i].next_to_watch;
3862 		eop_desc = E1000_TX_DESC(*tx_ring, eop);
3863 	}
3864 
3865 	/* Synchronize with E1000_DESC_UNUSED called from e1000_xmit_frame,
3866 	 * which will reuse the cleaned buffers.
3867 	 */
3868 	smp_store_release(&tx_ring->next_to_clean, i);
3869 
3870 	netdev_completed_queue(netdev, pkts_compl, bytes_compl);
3871 
3872 #define TX_WAKE_THRESHOLD 32
3873 	if (unlikely(count && netif_carrier_ok(netdev) &&
3874 		     E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
3875 		/* Make sure that anybody stopping the queue after this
3876 		 * sees the new next_to_clean.
3877 		 */
3878 		smp_mb();
3879 
3880 		if (netif_queue_stopped(netdev) &&
3881 		    !(test_bit(__E1000_DOWN, &adapter->flags))) {
3882 			netif_wake_queue(netdev);
3883 			++adapter->restart_queue;
3884 		}
3885 	}
3886 
3887 	if (adapter->detect_tx_hung) {
3888 		/* Detect a transmit hang in hardware, this serializes the
3889 		 * check with the clearing of time_stamp and movement of i
3890 		 */
3891 		adapter->detect_tx_hung = false;
3892 		if (tx_ring->buffer_info[eop].time_stamp &&
3893 		    time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3894 			       (adapter->tx_timeout_factor * HZ)) &&
3895 		    !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3896 
3897 			/* detected Tx unit hang */
3898 			e_err(drv, "Detected Tx Unit Hang\n"
3899 			      "  Tx Queue             <%lu>\n"
3900 			      "  TDH                  <%x>\n"
3901 			      "  TDT                  <%x>\n"
3902 			      "  next_to_use          <%x>\n"
3903 			      "  next_to_clean        <%x>\n"
3904 			      "buffer_info[next_to_clean]\n"
3905 			      "  time_stamp           <%lx>\n"
3906 			      "  next_to_watch        <%x>\n"
3907 			      "  jiffies              <%lx>\n"
3908 			      "  next_to_watch.status <%x>\n",
3909 				(unsigned long)(tx_ring - adapter->tx_ring),
3910 				readl(hw->hw_addr + tx_ring->tdh),
3911 				readl(hw->hw_addr + tx_ring->tdt),
3912 				tx_ring->next_to_use,
3913 				tx_ring->next_to_clean,
3914 				tx_ring->buffer_info[eop].time_stamp,
3915 				eop,
3916 				jiffies,
3917 				eop_desc->upper.fields.status);
3918 			e1000_dump(adapter);
3919 			netif_stop_queue(netdev);
3920 		}
3921 	}
3922 	adapter->total_tx_bytes += total_tx_bytes;
3923 	adapter->total_tx_packets += total_tx_packets;
3924 	netdev->stats.tx_bytes += total_tx_bytes;
3925 	netdev->stats.tx_packets += total_tx_packets;
3926 	return count < tx_ring->count;
3927 }
3928 
3929 /**
3930  * e1000_rx_checksum - Receive Checksum Offload for 82543
3931  * @adapter:     board private structure
3932  * @status_err:  receive descriptor status and error fields
3933  * @csum:        receive descriptor csum field
3934  * @sk_buff:     socket buffer with received data
3935  **/
3936 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
3937 			      u32 csum, struct sk_buff *skb)
3938 {
3939 	struct e1000_hw *hw = &adapter->hw;
3940 	u16 status = (u16)status_err;
3941 	u8 errors = (u8)(status_err >> 24);
3942 
3943 	skb_checksum_none_assert(skb);
3944 
3945 	/* 82543 or newer only */
3946 	if (unlikely(hw->mac_type < e1000_82543))
3947 		return;
3948 	/* Ignore Checksum bit is set */
3949 	if (unlikely(status & E1000_RXD_STAT_IXSM))
3950 		return;
3951 	/* TCP/UDP checksum error bit is set */
3952 	if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
3953 		/* let the stack verify checksum errors */
3954 		adapter->hw_csum_err++;
3955 		return;
3956 	}
3957 	/* TCP/UDP Checksum has not been calculated */
3958 	if (!(status & E1000_RXD_STAT_TCPCS))
3959 		return;
3960 
3961 	/* It must be a TCP or UDP packet with a valid checksum */
3962 	if (likely(status & E1000_RXD_STAT_TCPCS)) {
3963 		/* TCP checksum is good */
3964 		skb->ip_summed = CHECKSUM_UNNECESSARY;
3965 	}
3966 	adapter->hw_csum_good++;
3967 }
3968 
3969 /**
3970  * e1000_consume_page - helper function for jumbo Rx path
3971  **/
3972 static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb,
3973 			       u16 length)
3974 {
3975 	bi->rxbuf.page = NULL;
3976 	skb->len += length;
3977 	skb->data_len += length;
3978 	skb->truesize += PAGE_SIZE;
3979 }
3980 
3981 /**
3982  * e1000_receive_skb - helper function to handle rx indications
3983  * @adapter: board private structure
3984  * @status: descriptor status field as written by hardware
3985  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
3986  * @skb: pointer to sk_buff to be indicated to stack
3987  */
3988 static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
3989 			      __le16 vlan, struct sk_buff *skb)
3990 {
3991 	skb->protocol = eth_type_trans(skb, adapter->netdev);
3992 
3993 	if (status & E1000_RXD_STAT_VP) {
3994 		u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
3995 
3996 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
3997 	}
3998 	napi_gro_receive(&adapter->napi, skb);
3999 }
4000 
4001 /**
4002  * e1000_tbi_adjust_stats
4003  * @hw: Struct containing variables accessed by shared code
4004  * @frame_len: The length of the frame in question
4005  * @mac_addr: The Ethernet destination address of the frame in question
4006  *
4007  * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
4008  */
4009 static void e1000_tbi_adjust_stats(struct e1000_hw *hw,
4010 				   struct e1000_hw_stats *stats,
4011 				   u32 frame_len, const u8 *mac_addr)
4012 {
4013 	u64 carry_bit;
4014 
4015 	/* First adjust the frame length. */
4016 	frame_len--;
4017 	/* We need to adjust the statistics counters, since the hardware
4018 	 * counters overcount this packet as a CRC error and undercount
4019 	 * the packet as a good packet
4020 	 */
4021 	/* This packet should not be counted as a CRC error. */
4022 	stats->crcerrs--;
4023 	/* This packet does count as a Good Packet Received. */
4024 	stats->gprc++;
4025 
4026 	/* Adjust the Good Octets received counters */
4027 	carry_bit = 0x80000000 & stats->gorcl;
4028 	stats->gorcl += frame_len;
4029 	/* If the high bit of Gorcl (the low 32 bits of the Good Octets
4030 	 * Received Count) was one before the addition,
4031 	 * AND it is zero after, then we lost the carry out,
4032 	 * need to add one to Gorch (Good Octets Received Count High).
4033 	 * This could be simplified if all environments supported
4034 	 * 64-bit integers.
4035 	 */
4036 	if (carry_bit && ((stats->gorcl & 0x80000000) == 0))
4037 		stats->gorch++;
4038 	/* Is this a broadcast or multicast?  Check broadcast first,
4039 	 * since the test for a multicast frame will test positive on
4040 	 * a broadcast frame.
4041 	 */
4042 	if (is_broadcast_ether_addr(mac_addr))
4043 		stats->bprc++;
4044 	else if (is_multicast_ether_addr(mac_addr))
4045 		stats->mprc++;
4046 
4047 	if (frame_len == hw->max_frame_size) {
4048 		/* In this case, the hardware has overcounted the number of
4049 		 * oversize frames.
4050 		 */
4051 		if (stats->roc > 0)
4052 			stats->roc--;
4053 	}
4054 
4055 	/* Adjust the bin counters when the extra byte put the frame in the
4056 	 * wrong bin. Remember that the frame_len was adjusted above.
4057 	 */
4058 	if (frame_len == 64) {
4059 		stats->prc64++;
4060 		stats->prc127--;
4061 	} else if (frame_len == 127) {
4062 		stats->prc127++;
4063 		stats->prc255--;
4064 	} else if (frame_len == 255) {
4065 		stats->prc255++;
4066 		stats->prc511--;
4067 	} else if (frame_len == 511) {
4068 		stats->prc511++;
4069 		stats->prc1023--;
4070 	} else if (frame_len == 1023) {
4071 		stats->prc1023++;
4072 		stats->prc1522--;
4073 	} else if (frame_len == 1522) {
4074 		stats->prc1522++;
4075 	}
4076 }
4077 
4078 static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
4079 				    u8 status, u8 errors,
4080 				    u32 length, const u8 *data)
4081 {
4082 	struct e1000_hw *hw = &adapter->hw;
4083 	u8 last_byte = *(data + length - 1);
4084 
4085 	if (TBI_ACCEPT(hw, status, errors, length, last_byte)) {
4086 		unsigned long irq_flags;
4087 
4088 		spin_lock_irqsave(&adapter->stats_lock, irq_flags);
4089 		e1000_tbi_adjust_stats(hw, &adapter->stats, length, data);
4090 		spin_unlock_irqrestore(&adapter->stats_lock, irq_flags);
4091 
4092 		return true;
4093 	}
4094 
4095 	return false;
4096 }
4097 
4098 static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
4099 					  unsigned int bufsz)
4100 {
4101 	struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz);
4102 
4103 	if (unlikely(!skb))
4104 		adapter->alloc_rx_buff_failed++;
4105 	return skb;
4106 }
4107 
4108 /**
4109  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
4110  * @adapter: board private structure
4111  * @rx_ring: ring to clean
4112  * @work_done: amount of napi work completed this call
4113  * @work_to_do: max amount of work allowed for this call to do
4114  *
4115  * the return value indicates whether actual cleaning was done, there
4116  * is no guarantee that everything was cleaned
4117  */
4118 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
4119 				     struct e1000_rx_ring *rx_ring,
4120 				     int *work_done, int work_to_do)
4121 {
4122 	struct net_device *netdev = adapter->netdev;
4123 	struct pci_dev *pdev = adapter->pdev;
4124 	struct e1000_rx_desc *rx_desc, *next_rxd;
4125 	struct e1000_rx_buffer *buffer_info, *next_buffer;
4126 	u32 length;
4127 	unsigned int i;
4128 	int cleaned_count = 0;
4129 	bool cleaned = false;
4130 	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4131 
4132 	i = rx_ring->next_to_clean;
4133 	rx_desc = E1000_RX_DESC(*rx_ring, i);
4134 	buffer_info = &rx_ring->buffer_info[i];
4135 
4136 	while (rx_desc->status & E1000_RXD_STAT_DD) {
4137 		struct sk_buff *skb;
4138 		u8 status;
4139 
4140 		if (*work_done >= work_to_do)
4141 			break;
4142 		(*work_done)++;
4143 		dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4144 
4145 		status = rx_desc->status;
4146 
4147 		if (++i == rx_ring->count)
4148 			i = 0;
4149 
4150 		next_rxd = E1000_RX_DESC(*rx_ring, i);
4151 		prefetch(next_rxd);
4152 
4153 		next_buffer = &rx_ring->buffer_info[i];
4154 
4155 		cleaned = true;
4156 		cleaned_count++;
4157 		dma_unmap_page(&pdev->dev, buffer_info->dma,
4158 			       adapter->rx_buffer_len, DMA_FROM_DEVICE);
4159 		buffer_info->dma = 0;
4160 
4161 		length = le16_to_cpu(rx_desc->length);
4162 
4163 		/* errors is only valid for DD + EOP descriptors */
4164 		if (unlikely((status & E1000_RXD_STAT_EOP) &&
4165 		    (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
4166 			u8 *mapped = page_address(buffer_info->rxbuf.page);
4167 
4168 			if (e1000_tbi_should_accept(adapter, status,
4169 						    rx_desc->errors,
4170 						    length, mapped)) {
4171 				length--;
4172 			} else if (netdev->features & NETIF_F_RXALL) {
4173 				goto process_skb;
4174 			} else {
4175 				/* an error means any chain goes out the window
4176 				 * too
4177 				 */
4178 				dev_kfree_skb(rx_ring->rx_skb_top);
4179 				rx_ring->rx_skb_top = NULL;
4180 				goto next_desc;
4181 			}
4182 		}
4183 
4184 #define rxtop rx_ring->rx_skb_top
4185 process_skb:
4186 		if (!(status & E1000_RXD_STAT_EOP)) {
4187 			/* this descriptor is only the beginning (or middle) */
4188 			if (!rxtop) {
4189 				/* this is the beginning of a chain */
4190 				rxtop = napi_get_frags(&adapter->napi);
4191 				if (!rxtop)
4192 					break;
4193 
4194 				skb_fill_page_desc(rxtop, 0,
4195 						   buffer_info->rxbuf.page,
4196 						   0, length);
4197 			} else {
4198 				/* this is the middle of a chain */
4199 				skb_fill_page_desc(rxtop,
4200 				    skb_shinfo(rxtop)->nr_frags,
4201 				    buffer_info->rxbuf.page, 0, length);
4202 			}
4203 			e1000_consume_page(buffer_info, rxtop, length);
4204 			goto next_desc;
4205 		} else {
4206 			if (rxtop) {
4207 				/* end of the chain */
4208 				skb_fill_page_desc(rxtop,
4209 				    skb_shinfo(rxtop)->nr_frags,
4210 				    buffer_info->rxbuf.page, 0, length);
4211 				skb = rxtop;
4212 				rxtop = NULL;
4213 				e1000_consume_page(buffer_info, skb, length);
4214 			} else {
4215 				struct page *p;
4216 				/* no chain, got EOP, this buf is the packet
4217 				 * copybreak to save the put_page/alloc_page
4218 				 */
4219 				p = buffer_info->rxbuf.page;
4220 				if (length <= copybreak) {
4221 					u8 *vaddr;
4222 
4223 					if (likely(!(netdev->features & NETIF_F_RXFCS)))
4224 						length -= 4;
4225 					skb = e1000_alloc_rx_skb(adapter,
4226 								 length);
4227 					if (!skb)
4228 						break;
4229 
4230 					vaddr = kmap_atomic(p);
4231 					memcpy(skb_tail_pointer(skb), vaddr,
4232 					       length);
4233 					kunmap_atomic(vaddr);
4234 					/* re-use the page, so don't erase
4235 					 * buffer_info->rxbuf.page
4236 					 */
4237 					skb_put(skb, length);
4238 					e1000_rx_checksum(adapter,
4239 							  status | rx_desc->errors << 24,
4240 							  le16_to_cpu(rx_desc->csum), skb);
4241 
4242 					total_rx_bytes += skb->len;
4243 					total_rx_packets++;
4244 
4245 					e1000_receive_skb(adapter, status,
4246 							  rx_desc->special, skb);
4247 					goto next_desc;
4248 				} else {
4249 					skb = napi_get_frags(&adapter->napi);
4250 					if (!skb) {
4251 						adapter->alloc_rx_buff_failed++;
4252 						break;
4253 					}
4254 					skb_fill_page_desc(skb, 0, p, 0,
4255 							   length);
4256 					e1000_consume_page(buffer_info, skb,
4257 							   length);
4258 				}
4259 			}
4260 		}
4261 
4262 		/* Receive Checksum Offload XXX recompute due to CRC strip? */
4263 		e1000_rx_checksum(adapter,
4264 				  (u32)(status) |
4265 				  ((u32)(rx_desc->errors) << 24),
4266 				  le16_to_cpu(rx_desc->csum), skb);
4267 
4268 		total_rx_bytes += (skb->len - 4); /* don't count FCS */
4269 		if (likely(!(netdev->features & NETIF_F_RXFCS)))
4270 			pskb_trim(skb, skb->len - 4);
4271 		total_rx_packets++;
4272 
4273 		if (status & E1000_RXD_STAT_VP) {
4274 			__le16 vlan = rx_desc->special;
4275 			u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4276 
4277 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4278 		}
4279 
4280 		napi_gro_frags(&adapter->napi);
4281 
4282 next_desc:
4283 		rx_desc->status = 0;
4284 
4285 		/* return some buffers to hardware, one at a time is too slow */
4286 		if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4287 			adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4288 			cleaned_count = 0;
4289 		}
4290 
4291 		/* use prefetched values */
4292 		rx_desc = next_rxd;
4293 		buffer_info = next_buffer;
4294 	}
4295 	rx_ring->next_to_clean = i;
4296 
4297 	cleaned_count = E1000_DESC_UNUSED(rx_ring);
4298 	if (cleaned_count)
4299 		adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4300 
4301 	adapter->total_rx_packets += total_rx_packets;
4302 	adapter->total_rx_bytes += total_rx_bytes;
4303 	netdev->stats.rx_bytes += total_rx_bytes;
4304 	netdev->stats.rx_packets += total_rx_packets;
4305 	return cleaned;
4306 }
4307 
4308 /* this should improve performance for small packets with large amounts
4309  * of reassembly being done in the stack
4310  */
4311 static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter,
4312 				       struct e1000_rx_buffer *buffer_info,
4313 				       u32 length, const void *data)
4314 {
4315 	struct sk_buff *skb;
4316 
4317 	if (length > copybreak)
4318 		return NULL;
4319 
4320 	skb = e1000_alloc_rx_skb(adapter, length);
4321 	if (!skb)
4322 		return NULL;
4323 
4324 	dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma,
4325 				length, DMA_FROM_DEVICE);
4326 
4327 	skb_put_data(skb, data, length);
4328 
4329 	return skb;
4330 }
4331 
4332 /**
4333  * e1000_clean_rx_irq - Send received data up the network stack; legacy
4334  * @adapter: board private structure
4335  * @rx_ring: ring to clean
4336  * @work_done: amount of napi work completed this call
4337  * @work_to_do: max amount of work allowed for this call to do
4338  */
4339 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
4340 			       struct e1000_rx_ring *rx_ring,
4341 			       int *work_done, int work_to_do)
4342 {
4343 	struct net_device *netdev = adapter->netdev;
4344 	struct pci_dev *pdev = adapter->pdev;
4345 	struct e1000_rx_desc *rx_desc, *next_rxd;
4346 	struct e1000_rx_buffer *buffer_info, *next_buffer;
4347 	u32 length;
4348 	unsigned int i;
4349 	int cleaned_count = 0;
4350 	bool cleaned = false;
4351 	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4352 
4353 	i = rx_ring->next_to_clean;
4354 	rx_desc = E1000_RX_DESC(*rx_ring, i);
4355 	buffer_info = &rx_ring->buffer_info[i];
4356 
4357 	while (rx_desc->status & E1000_RXD_STAT_DD) {
4358 		struct sk_buff *skb;
4359 		u8 *data;
4360 		u8 status;
4361 
4362 		if (*work_done >= work_to_do)
4363 			break;
4364 		(*work_done)++;
4365 		dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4366 
4367 		status = rx_desc->status;
4368 		length = le16_to_cpu(rx_desc->length);
4369 
4370 		data = buffer_info->rxbuf.data;
4371 		prefetch(data);
4372 		skb = e1000_copybreak(adapter, buffer_info, length, data);
4373 		if (!skb) {
4374 			unsigned int frag_len = e1000_frag_len(adapter);
4375 
4376 			skb = build_skb(data - E1000_HEADROOM, frag_len);
4377 			if (!skb) {
4378 				adapter->alloc_rx_buff_failed++;
4379 				break;
4380 			}
4381 
4382 			skb_reserve(skb, E1000_HEADROOM);
4383 			dma_unmap_single(&pdev->dev, buffer_info->dma,
4384 					 adapter->rx_buffer_len,
4385 					 DMA_FROM_DEVICE);
4386 			buffer_info->dma = 0;
4387 			buffer_info->rxbuf.data = NULL;
4388 		}
4389 
4390 		if (++i == rx_ring->count)
4391 			i = 0;
4392 
4393 		next_rxd = E1000_RX_DESC(*rx_ring, i);
4394 		prefetch(next_rxd);
4395 
4396 		next_buffer = &rx_ring->buffer_info[i];
4397 
4398 		cleaned = true;
4399 		cleaned_count++;
4400 
4401 		/* !EOP means multiple descriptors were used to store a single
4402 		 * packet, if thats the case we need to toss it.  In fact, we
4403 		 * to toss every packet with the EOP bit clear and the next
4404 		 * frame that _does_ have the EOP bit set, as it is by
4405 		 * definition only a frame fragment
4406 		 */
4407 		if (unlikely(!(status & E1000_RXD_STAT_EOP)))
4408 			adapter->discarding = true;
4409 
4410 		if (adapter->discarding) {
4411 			/* All receives must fit into a single buffer */
4412 			netdev_dbg(netdev, "Receive packet consumed multiple buffers\n");
4413 			dev_kfree_skb(skb);
4414 			if (status & E1000_RXD_STAT_EOP)
4415 				adapter->discarding = false;
4416 			goto next_desc;
4417 		}
4418 
4419 		if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4420 			if (e1000_tbi_should_accept(adapter, status,
4421 						    rx_desc->errors,
4422 						    length, data)) {
4423 				length--;
4424 			} else if (netdev->features & NETIF_F_RXALL) {
4425 				goto process_skb;
4426 			} else {
4427 				dev_kfree_skb(skb);
4428 				goto next_desc;
4429 			}
4430 		}
4431 
4432 process_skb:
4433 		total_rx_bytes += (length - 4); /* don't count FCS */
4434 		total_rx_packets++;
4435 
4436 		if (likely(!(netdev->features & NETIF_F_RXFCS)))
4437 			/* adjust length to remove Ethernet CRC, this must be
4438 			 * done after the TBI_ACCEPT workaround above
4439 			 */
4440 			length -= 4;
4441 
4442 		if (buffer_info->rxbuf.data == NULL)
4443 			skb_put(skb, length);
4444 		else /* copybreak skb */
4445 			skb_trim(skb, length);
4446 
4447 		/* Receive Checksum Offload */
4448 		e1000_rx_checksum(adapter,
4449 				  (u32)(status) |
4450 				  ((u32)(rx_desc->errors) << 24),
4451 				  le16_to_cpu(rx_desc->csum), skb);
4452 
4453 		e1000_receive_skb(adapter, status, rx_desc->special, skb);
4454 
4455 next_desc:
4456 		rx_desc->status = 0;
4457 
4458 		/* return some buffers to hardware, one at a time is too slow */
4459 		if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4460 			adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4461 			cleaned_count = 0;
4462 		}
4463 
4464 		/* use prefetched values */
4465 		rx_desc = next_rxd;
4466 		buffer_info = next_buffer;
4467 	}
4468 	rx_ring->next_to_clean = i;
4469 
4470 	cleaned_count = E1000_DESC_UNUSED(rx_ring);
4471 	if (cleaned_count)
4472 		adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4473 
4474 	adapter->total_rx_packets += total_rx_packets;
4475 	adapter->total_rx_bytes += total_rx_bytes;
4476 	netdev->stats.rx_bytes += total_rx_bytes;
4477 	netdev->stats.rx_packets += total_rx_packets;
4478 	return cleaned;
4479 }
4480 
4481 /**
4482  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
4483  * @adapter: address of board private structure
4484  * @rx_ring: pointer to receive ring structure
4485  * @cleaned_count: number of buffers to allocate this pass
4486  **/
4487 static void
4488 e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
4489 			     struct e1000_rx_ring *rx_ring, int cleaned_count)
4490 {
4491 	struct pci_dev *pdev = adapter->pdev;
4492 	struct e1000_rx_desc *rx_desc;
4493 	struct e1000_rx_buffer *buffer_info;
4494 	unsigned int i;
4495 
4496 	i = rx_ring->next_to_use;
4497 	buffer_info = &rx_ring->buffer_info[i];
4498 
4499 	while (cleaned_count--) {
4500 		/* allocate a new page if necessary */
4501 		if (!buffer_info->rxbuf.page) {
4502 			buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC);
4503 			if (unlikely(!buffer_info->rxbuf.page)) {
4504 				adapter->alloc_rx_buff_failed++;
4505 				break;
4506 			}
4507 		}
4508 
4509 		if (!buffer_info->dma) {
4510 			buffer_info->dma = dma_map_page(&pdev->dev,
4511 							buffer_info->rxbuf.page, 0,
4512 							adapter->rx_buffer_len,
4513 							DMA_FROM_DEVICE);
4514 			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4515 				put_page(buffer_info->rxbuf.page);
4516 				buffer_info->rxbuf.page = NULL;
4517 				buffer_info->dma = 0;
4518 				adapter->alloc_rx_buff_failed++;
4519 				break;
4520 			}
4521 		}
4522 
4523 		rx_desc = E1000_RX_DESC(*rx_ring, i);
4524 		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4525 
4526 		if (unlikely(++i == rx_ring->count))
4527 			i = 0;
4528 		buffer_info = &rx_ring->buffer_info[i];
4529 	}
4530 
4531 	if (likely(rx_ring->next_to_use != i)) {
4532 		rx_ring->next_to_use = i;
4533 		if (unlikely(i-- == 0))
4534 			i = (rx_ring->count - 1);
4535 
4536 		/* Force memory writes to complete before letting h/w
4537 		 * know there are new descriptors to fetch.  (Only
4538 		 * applicable for weak-ordered memory model archs,
4539 		 * such as IA-64).
4540 		 */
4541 		dma_wmb();
4542 		writel(i, adapter->hw.hw_addr + rx_ring->rdt);
4543 	}
4544 }
4545 
4546 /**
4547  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
4548  * @adapter: address of board private structure
4549  **/
4550 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
4551 				   struct e1000_rx_ring *rx_ring,
4552 				   int cleaned_count)
4553 {
4554 	struct e1000_hw *hw = &adapter->hw;
4555 	struct pci_dev *pdev = adapter->pdev;
4556 	struct e1000_rx_desc *rx_desc;
4557 	struct e1000_rx_buffer *buffer_info;
4558 	unsigned int i;
4559 	unsigned int bufsz = adapter->rx_buffer_len;
4560 
4561 	i = rx_ring->next_to_use;
4562 	buffer_info = &rx_ring->buffer_info[i];
4563 
4564 	while (cleaned_count--) {
4565 		void *data;
4566 
4567 		if (buffer_info->rxbuf.data)
4568 			goto skip;
4569 
4570 		data = e1000_alloc_frag(adapter);
4571 		if (!data) {
4572 			/* Better luck next round */
4573 			adapter->alloc_rx_buff_failed++;
4574 			break;
4575 		}
4576 
4577 		/* Fix for errata 23, can't cross 64kB boundary */
4578 		if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4579 			void *olddata = data;
4580 			e_err(rx_err, "skb align check failed: %u bytes at "
4581 			      "%p\n", bufsz, data);
4582 			/* Try again, without freeing the previous */
4583 			data = e1000_alloc_frag(adapter);
4584 			/* Failed allocation, critical failure */
4585 			if (!data) {
4586 				skb_free_frag(olddata);
4587 				adapter->alloc_rx_buff_failed++;
4588 				break;
4589 			}
4590 
4591 			if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4592 				/* give up */
4593 				skb_free_frag(data);
4594 				skb_free_frag(olddata);
4595 				adapter->alloc_rx_buff_failed++;
4596 				break;
4597 			}
4598 
4599 			/* Use new allocation */
4600 			skb_free_frag(olddata);
4601 		}
4602 		buffer_info->dma = dma_map_single(&pdev->dev,
4603 						  data,
4604 						  adapter->rx_buffer_len,
4605 						  DMA_FROM_DEVICE);
4606 		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4607 			skb_free_frag(data);
4608 			buffer_info->dma = 0;
4609 			adapter->alloc_rx_buff_failed++;
4610 			break;
4611 		}
4612 
4613 		/* XXX if it was allocated cleanly it will never map to a
4614 		 * boundary crossing
4615 		 */
4616 
4617 		/* Fix for errata 23, can't cross 64kB boundary */
4618 		if (!e1000_check_64k_bound(adapter,
4619 					(void *)(unsigned long)buffer_info->dma,
4620 					adapter->rx_buffer_len)) {
4621 			e_err(rx_err, "dma align check failed: %u bytes at "
4622 			      "%p\n", adapter->rx_buffer_len,
4623 			      (void *)(unsigned long)buffer_info->dma);
4624 
4625 			dma_unmap_single(&pdev->dev, buffer_info->dma,
4626 					 adapter->rx_buffer_len,
4627 					 DMA_FROM_DEVICE);
4628 
4629 			skb_free_frag(data);
4630 			buffer_info->rxbuf.data = NULL;
4631 			buffer_info->dma = 0;
4632 
4633 			adapter->alloc_rx_buff_failed++;
4634 			break;
4635 		}
4636 		buffer_info->rxbuf.data = data;
4637  skip:
4638 		rx_desc = E1000_RX_DESC(*rx_ring, i);
4639 		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4640 
4641 		if (unlikely(++i == rx_ring->count))
4642 			i = 0;
4643 		buffer_info = &rx_ring->buffer_info[i];
4644 	}
4645 
4646 	if (likely(rx_ring->next_to_use != i)) {
4647 		rx_ring->next_to_use = i;
4648 		if (unlikely(i-- == 0))
4649 			i = (rx_ring->count - 1);
4650 
4651 		/* Force memory writes to complete before letting h/w
4652 		 * know there are new descriptors to fetch.  (Only
4653 		 * applicable for weak-ordered memory model archs,
4654 		 * such as IA-64).
4655 		 */
4656 		dma_wmb();
4657 		writel(i, hw->hw_addr + rx_ring->rdt);
4658 	}
4659 }
4660 
4661 /**
4662  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
4663  * @adapter:
4664  **/
4665 static void e1000_smartspeed(struct e1000_adapter *adapter)
4666 {
4667 	struct e1000_hw *hw = &adapter->hw;
4668 	u16 phy_status;
4669 	u16 phy_ctrl;
4670 
4671 	if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
4672 	   !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
4673 		return;
4674 
4675 	if (adapter->smartspeed == 0) {
4676 		/* If Master/Slave config fault is asserted twice,
4677 		 * we assume back-to-back
4678 		 */
4679 		e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4680 		if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4681 			return;
4682 		e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4683 		if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4684 			return;
4685 		e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4686 		if (phy_ctrl & CR_1000T_MS_ENABLE) {
4687 			phy_ctrl &= ~CR_1000T_MS_ENABLE;
4688 			e1000_write_phy_reg(hw, PHY_1000T_CTRL,
4689 					    phy_ctrl);
4690 			adapter->smartspeed++;
4691 			if (!e1000_phy_setup_autoneg(hw) &&
4692 			   !e1000_read_phy_reg(hw, PHY_CTRL,
4693 					       &phy_ctrl)) {
4694 				phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4695 					     MII_CR_RESTART_AUTO_NEG);
4696 				e1000_write_phy_reg(hw, PHY_CTRL,
4697 						    phy_ctrl);
4698 			}
4699 		}
4700 		return;
4701 	} else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
4702 		/* If still no link, perhaps using 2/3 pair cable */
4703 		e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4704 		phy_ctrl |= CR_1000T_MS_ENABLE;
4705 		e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
4706 		if (!e1000_phy_setup_autoneg(hw) &&
4707 		   !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
4708 			phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4709 				     MII_CR_RESTART_AUTO_NEG);
4710 			e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
4711 		}
4712 	}
4713 	/* Restart process after E1000_SMARTSPEED_MAX iterations */
4714 	if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
4715 		adapter->smartspeed = 0;
4716 }
4717 
4718 /**
4719  * e1000_ioctl -
4720  * @netdev:
4721  * @ifreq:
4722  * @cmd:
4723  **/
4724 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4725 {
4726 	switch (cmd) {
4727 	case SIOCGMIIPHY:
4728 	case SIOCGMIIREG:
4729 	case SIOCSMIIREG:
4730 		return e1000_mii_ioctl(netdev, ifr, cmd);
4731 	default:
4732 		return -EOPNOTSUPP;
4733 	}
4734 }
4735 
4736 /**
4737  * e1000_mii_ioctl -
4738  * @netdev:
4739  * @ifreq:
4740  * @cmd:
4741  **/
4742 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
4743 			   int cmd)
4744 {
4745 	struct e1000_adapter *adapter = netdev_priv(netdev);
4746 	struct e1000_hw *hw = &adapter->hw;
4747 	struct mii_ioctl_data *data = if_mii(ifr);
4748 	int retval;
4749 	u16 mii_reg;
4750 	unsigned long flags;
4751 
4752 	if (hw->media_type != e1000_media_type_copper)
4753 		return -EOPNOTSUPP;
4754 
4755 	switch (cmd) {
4756 	case SIOCGMIIPHY:
4757 		data->phy_id = hw->phy_addr;
4758 		break;
4759 	case SIOCGMIIREG:
4760 		spin_lock_irqsave(&adapter->stats_lock, flags);
4761 		if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4762 				   &data->val_out)) {
4763 			spin_unlock_irqrestore(&adapter->stats_lock, flags);
4764 			return -EIO;
4765 		}
4766 		spin_unlock_irqrestore(&adapter->stats_lock, flags);
4767 		break;
4768 	case SIOCSMIIREG:
4769 		if (data->reg_num & ~(0x1F))
4770 			return -EFAULT;
4771 		mii_reg = data->val_in;
4772 		spin_lock_irqsave(&adapter->stats_lock, flags);
4773 		if (e1000_write_phy_reg(hw, data->reg_num,
4774 					mii_reg)) {
4775 			spin_unlock_irqrestore(&adapter->stats_lock, flags);
4776 			return -EIO;
4777 		}
4778 		spin_unlock_irqrestore(&adapter->stats_lock, flags);
4779 		if (hw->media_type == e1000_media_type_copper) {
4780 			switch (data->reg_num) {
4781 			case PHY_CTRL:
4782 				if (mii_reg & MII_CR_POWER_DOWN)
4783 					break;
4784 				if (mii_reg & MII_CR_AUTO_NEG_EN) {
4785 					hw->autoneg = 1;
4786 					hw->autoneg_advertised = 0x2F;
4787 				} else {
4788 					u32 speed;
4789 					if (mii_reg & 0x40)
4790 						speed = SPEED_1000;
4791 					else if (mii_reg & 0x2000)
4792 						speed = SPEED_100;
4793 					else
4794 						speed = SPEED_10;
4795 					retval = e1000_set_spd_dplx(
4796 						adapter, speed,
4797 						((mii_reg & 0x100)
4798 						 ? DUPLEX_FULL :
4799 						 DUPLEX_HALF));
4800 					if (retval)
4801 						return retval;
4802 				}
4803 				if (netif_running(adapter->netdev))
4804 					e1000_reinit_locked(adapter);
4805 				else
4806 					e1000_reset(adapter);
4807 				break;
4808 			case M88E1000_PHY_SPEC_CTRL:
4809 			case M88E1000_EXT_PHY_SPEC_CTRL:
4810 				if (e1000_phy_reset(hw))
4811 					return -EIO;
4812 				break;
4813 			}
4814 		} else {
4815 			switch (data->reg_num) {
4816 			case PHY_CTRL:
4817 				if (mii_reg & MII_CR_POWER_DOWN)
4818 					break;
4819 				if (netif_running(adapter->netdev))
4820 					e1000_reinit_locked(adapter);
4821 				else
4822 					e1000_reset(adapter);
4823 				break;
4824 			}
4825 		}
4826 		break;
4827 	default:
4828 		return -EOPNOTSUPP;
4829 	}
4830 	return E1000_SUCCESS;
4831 }
4832 
4833 void e1000_pci_set_mwi(struct e1000_hw *hw)
4834 {
4835 	struct e1000_adapter *adapter = hw->back;
4836 	int ret_val = pci_set_mwi(adapter->pdev);
4837 
4838 	if (ret_val)
4839 		e_err(probe, "Error in setting MWI\n");
4840 }
4841 
4842 void e1000_pci_clear_mwi(struct e1000_hw *hw)
4843 {
4844 	struct e1000_adapter *adapter = hw->back;
4845 
4846 	pci_clear_mwi(adapter->pdev);
4847 }
4848 
4849 int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4850 {
4851 	struct e1000_adapter *adapter = hw->back;
4852 	return pcix_get_mmrbc(adapter->pdev);
4853 }
4854 
4855 void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4856 {
4857 	struct e1000_adapter *adapter = hw->back;
4858 	pcix_set_mmrbc(adapter->pdev, mmrbc);
4859 }
4860 
4861 void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
4862 {
4863 	outl(value, port);
4864 }
4865 
4866 static bool e1000_vlan_used(struct e1000_adapter *adapter)
4867 {
4868 	u16 vid;
4869 
4870 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4871 		return true;
4872 	return false;
4873 }
4874 
4875 static void __e1000_vlan_mode(struct e1000_adapter *adapter,
4876 			      netdev_features_t features)
4877 {
4878 	struct e1000_hw *hw = &adapter->hw;
4879 	u32 ctrl;
4880 
4881 	ctrl = er32(CTRL);
4882 	if (features & NETIF_F_HW_VLAN_CTAG_RX) {
4883 		/* enable VLAN tag insert/strip */
4884 		ctrl |= E1000_CTRL_VME;
4885 	} else {
4886 		/* disable VLAN tag insert/strip */
4887 		ctrl &= ~E1000_CTRL_VME;
4888 	}
4889 	ew32(CTRL, ctrl);
4890 }
4891 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
4892 				     bool filter_on)
4893 {
4894 	struct e1000_hw *hw = &adapter->hw;
4895 	u32 rctl;
4896 
4897 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4898 		e1000_irq_disable(adapter);
4899 
4900 	__e1000_vlan_mode(adapter, adapter->netdev->features);
4901 	if (filter_on) {
4902 		/* enable VLAN receive filtering */
4903 		rctl = er32(RCTL);
4904 		rctl &= ~E1000_RCTL_CFIEN;
4905 		if (!(adapter->netdev->flags & IFF_PROMISC))
4906 			rctl |= E1000_RCTL_VFE;
4907 		ew32(RCTL, rctl);
4908 		e1000_update_mng_vlan(adapter);
4909 	} else {
4910 		/* disable VLAN receive filtering */
4911 		rctl = er32(RCTL);
4912 		rctl &= ~E1000_RCTL_VFE;
4913 		ew32(RCTL, rctl);
4914 	}
4915 
4916 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4917 		e1000_irq_enable(adapter);
4918 }
4919 
4920 static void e1000_vlan_mode(struct net_device *netdev,
4921 			    netdev_features_t features)
4922 {
4923 	struct e1000_adapter *adapter = netdev_priv(netdev);
4924 
4925 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4926 		e1000_irq_disable(adapter);
4927 
4928 	__e1000_vlan_mode(adapter, features);
4929 
4930 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4931 		e1000_irq_enable(adapter);
4932 }
4933 
4934 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
4935 				 __be16 proto, u16 vid)
4936 {
4937 	struct e1000_adapter *adapter = netdev_priv(netdev);
4938 	struct e1000_hw *hw = &adapter->hw;
4939 	u32 vfta, index;
4940 
4941 	if ((hw->mng_cookie.status &
4942 	     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4943 	    (vid == adapter->mng_vlan_id))
4944 		return 0;
4945 
4946 	if (!e1000_vlan_used(adapter))
4947 		e1000_vlan_filter_on_off(adapter, true);
4948 
4949 	/* add VID to filter table */
4950 	index = (vid >> 5) & 0x7F;
4951 	vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4952 	vfta |= (1 << (vid & 0x1F));
4953 	e1000_write_vfta(hw, index, vfta);
4954 
4955 	set_bit(vid, adapter->active_vlans);
4956 
4957 	return 0;
4958 }
4959 
4960 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
4961 				  __be16 proto, u16 vid)
4962 {
4963 	struct e1000_adapter *adapter = netdev_priv(netdev);
4964 	struct e1000_hw *hw = &adapter->hw;
4965 	u32 vfta, index;
4966 
4967 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4968 		e1000_irq_disable(adapter);
4969 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4970 		e1000_irq_enable(adapter);
4971 
4972 	/* remove VID from filter table */
4973 	index = (vid >> 5) & 0x7F;
4974 	vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4975 	vfta &= ~(1 << (vid & 0x1F));
4976 	e1000_write_vfta(hw, index, vfta);
4977 
4978 	clear_bit(vid, adapter->active_vlans);
4979 
4980 	if (!e1000_vlan_used(adapter))
4981 		e1000_vlan_filter_on_off(adapter, false);
4982 
4983 	return 0;
4984 }
4985 
4986 static void e1000_restore_vlan(struct e1000_adapter *adapter)
4987 {
4988 	u16 vid;
4989 
4990 	if (!e1000_vlan_used(adapter))
4991 		return;
4992 
4993 	e1000_vlan_filter_on_off(adapter, true);
4994 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4995 		e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
4996 }
4997 
4998 int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
4999 {
5000 	struct e1000_hw *hw = &adapter->hw;
5001 
5002 	hw->autoneg = 0;
5003 
5004 	/* Make sure dplx is at most 1 bit and lsb of speed is not set
5005 	 * for the switch() below to work
5006 	 */
5007 	if ((spd & 1) || (dplx & ~1))
5008 		goto err_inval;
5009 
5010 	/* Fiber NICs only allow 1000 gbps Full duplex */
5011 	if ((hw->media_type == e1000_media_type_fiber) &&
5012 	    spd != SPEED_1000 &&
5013 	    dplx != DUPLEX_FULL)
5014 		goto err_inval;
5015 
5016 	switch (spd + dplx) {
5017 	case SPEED_10 + DUPLEX_HALF:
5018 		hw->forced_speed_duplex = e1000_10_half;
5019 		break;
5020 	case SPEED_10 + DUPLEX_FULL:
5021 		hw->forced_speed_duplex = e1000_10_full;
5022 		break;
5023 	case SPEED_100 + DUPLEX_HALF:
5024 		hw->forced_speed_duplex = e1000_100_half;
5025 		break;
5026 	case SPEED_100 + DUPLEX_FULL:
5027 		hw->forced_speed_duplex = e1000_100_full;
5028 		break;
5029 	case SPEED_1000 + DUPLEX_FULL:
5030 		hw->autoneg = 1;
5031 		hw->autoneg_advertised = ADVERTISE_1000_FULL;
5032 		break;
5033 	case SPEED_1000 + DUPLEX_HALF: /* not supported */
5034 	default:
5035 		goto err_inval;
5036 	}
5037 
5038 	/* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5039 	hw->mdix = AUTO_ALL_MODES;
5040 
5041 	return 0;
5042 
5043 err_inval:
5044 	e_err(probe, "Unsupported Speed/Duplex configuration\n");
5045 	return -EINVAL;
5046 }
5047 
5048 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
5049 {
5050 	struct net_device *netdev = pci_get_drvdata(pdev);
5051 	struct e1000_adapter *adapter = netdev_priv(netdev);
5052 	struct e1000_hw *hw = &adapter->hw;
5053 	u32 ctrl, ctrl_ext, rctl, status;
5054 	u32 wufc = adapter->wol;
5055 #ifdef CONFIG_PM
5056 	int retval = 0;
5057 #endif
5058 
5059 	netif_device_detach(netdev);
5060 
5061 	if (netif_running(netdev)) {
5062 		int count = E1000_CHECK_RESET_COUNT;
5063 
5064 		while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
5065 			usleep_range(10000, 20000);
5066 
5067 		WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
5068 		e1000_down(adapter);
5069 	}
5070 
5071 #ifdef CONFIG_PM
5072 	retval = pci_save_state(pdev);
5073 	if (retval)
5074 		return retval;
5075 #endif
5076 
5077 	status = er32(STATUS);
5078 	if (status & E1000_STATUS_LU)
5079 		wufc &= ~E1000_WUFC_LNKC;
5080 
5081 	if (wufc) {
5082 		e1000_setup_rctl(adapter);
5083 		e1000_set_rx_mode(netdev);
5084 
5085 		rctl = er32(RCTL);
5086 
5087 		/* turn on all-multi mode if wake on multicast is enabled */
5088 		if (wufc & E1000_WUFC_MC)
5089 			rctl |= E1000_RCTL_MPE;
5090 
5091 		/* enable receives in the hardware */
5092 		ew32(RCTL, rctl | E1000_RCTL_EN);
5093 
5094 		if (hw->mac_type >= e1000_82540) {
5095 			ctrl = er32(CTRL);
5096 			/* advertise wake from D3Cold */
5097 			#define E1000_CTRL_ADVD3WUC 0x00100000
5098 			/* phy power management enable */
5099 			#define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5100 			ctrl |= E1000_CTRL_ADVD3WUC |
5101 				E1000_CTRL_EN_PHY_PWR_MGMT;
5102 			ew32(CTRL, ctrl);
5103 		}
5104 
5105 		if (hw->media_type == e1000_media_type_fiber ||
5106 		    hw->media_type == e1000_media_type_internal_serdes) {
5107 			/* keep the laser running in D3 */
5108 			ctrl_ext = er32(CTRL_EXT);
5109 			ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
5110 			ew32(CTRL_EXT, ctrl_ext);
5111 		}
5112 
5113 		ew32(WUC, E1000_WUC_PME_EN);
5114 		ew32(WUFC, wufc);
5115 	} else {
5116 		ew32(WUC, 0);
5117 		ew32(WUFC, 0);
5118 	}
5119 
5120 	e1000_release_manageability(adapter);
5121 
5122 	*enable_wake = !!wufc;
5123 
5124 	/* make sure adapter isn't asleep if manageability is enabled */
5125 	if (adapter->en_mng_pt)
5126 		*enable_wake = true;
5127 
5128 	if (netif_running(netdev))
5129 		e1000_free_irq(adapter);
5130 
5131 	if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5132 		pci_disable_device(pdev);
5133 
5134 	return 0;
5135 }
5136 
5137 #ifdef CONFIG_PM
5138 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state)
5139 {
5140 	int retval;
5141 	bool wake;
5142 
5143 	retval = __e1000_shutdown(pdev, &wake);
5144 	if (retval)
5145 		return retval;
5146 
5147 	if (wake) {
5148 		pci_prepare_to_sleep(pdev);
5149 	} else {
5150 		pci_wake_from_d3(pdev, false);
5151 		pci_set_power_state(pdev, PCI_D3hot);
5152 	}
5153 
5154 	return 0;
5155 }
5156 
5157 static int e1000_resume(struct pci_dev *pdev)
5158 {
5159 	struct net_device *netdev = pci_get_drvdata(pdev);
5160 	struct e1000_adapter *adapter = netdev_priv(netdev);
5161 	struct e1000_hw *hw = &adapter->hw;
5162 	u32 err;
5163 
5164 	pci_set_power_state(pdev, PCI_D0);
5165 	pci_restore_state(pdev);
5166 	pci_save_state(pdev);
5167 
5168 	if (adapter->need_ioport)
5169 		err = pci_enable_device(pdev);
5170 	else
5171 		err = pci_enable_device_mem(pdev);
5172 	if (err) {
5173 		pr_err("Cannot enable PCI device from suspend\n");
5174 		return err;
5175 	}
5176 
5177 	/* flush memory to make sure state is correct */
5178 	smp_mb__before_atomic();
5179 	clear_bit(__E1000_DISABLED, &adapter->flags);
5180 	pci_set_master(pdev);
5181 
5182 	pci_enable_wake(pdev, PCI_D3hot, 0);
5183 	pci_enable_wake(pdev, PCI_D3cold, 0);
5184 
5185 	if (netif_running(netdev)) {
5186 		err = e1000_request_irq(adapter);
5187 		if (err)
5188 			return err;
5189 	}
5190 
5191 	e1000_power_up_phy(adapter);
5192 	e1000_reset(adapter);
5193 	ew32(WUS, ~0);
5194 
5195 	e1000_init_manageability(adapter);
5196 
5197 	if (netif_running(netdev))
5198 		e1000_up(adapter);
5199 
5200 	netif_device_attach(netdev);
5201 
5202 	return 0;
5203 }
5204 #endif
5205 
5206 static void e1000_shutdown(struct pci_dev *pdev)
5207 {
5208 	bool wake;
5209 
5210 	__e1000_shutdown(pdev, &wake);
5211 
5212 	if (system_state == SYSTEM_POWER_OFF) {
5213 		pci_wake_from_d3(pdev, wake);
5214 		pci_set_power_state(pdev, PCI_D3hot);
5215 	}
5216 }
5217 
5218 #ifdef CONFIG_NET_POLL_CONTROLLER
5219 /* Polling 'interrupt' - used by things like netconsole to send skbs
5220  * without having to re-enable interrupts. It's not called while
5221  * the interrupt routine is executing.
5222  */
5223 static void e1000_netpoll(struct net_device *netdev)
5224 {
5225 	struct e1000_adapter *adapter = netdev_priv(netdev);
5226 
5227 	if (disable_hardirq(adapter->pdev->irq))
5228 		e1000_intr(adapter->pdev->irq, netdev);
5229 	enable_irq(adapter->pdev->irq);
5230 }
5231 #endif
5232 
5233 /**
5234  * e1000_io_error_detected - called when PCI error is detected
5235  * @pdev: Pointer to PCI device
5236  * @state: The current pci connection state
5237  *
5238  * This function is called after a PCI bus error affecting
5239  * this device has been detected.
5240  */
5241 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5242 						pci_channel_state_t state)
5243 {
5244 	struct net_device *netdev = pci_get_drvdata(pdev);
5245 	struct e1000_adapter *adapter = netdev_priv(netdev);
5246 
5247 	netif_device_detach(netdev);
5248 
5249 	if (state == pci_channel_io_perm_failure)
5250 		return PCI_ERS_RESULT_DISCONNECT;
5251 
5252 	if (netif_running(netdev))
5253 		e1000_down(adapter);
5254 
5255 	if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5256 		pci_disable_device(pdev);
5257 
5258 	/* Request a slot slot reset. */
5259 	return PCI_ERS_RESULT_NEED_RESET;
5260 }
5261 
5262 /**
5263  * e1000_io_slot_reset - called after the pci bus has been reset.
5264  * @pdev: Pointer to PCI device
5265  *
5266  * Restart the card from scratch, as if from a cold-boot. Implementation
5267  * resembles the first-half of the e1000_resume routine.
5268  */
5269 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5270 {
5271 	struct net_device *netdev = pci_get_drvdata(pdev);
5272 	struct e1000_adapter *adapter = netdev_priv(netdev);
5273 	struct e1000_hw *hw = &adapter->hw;
5274 	int err;
5275 
5276 	if (adapter->need_ioport)
5277 		err = pci_enable_device(pdev);
5278 	else
5279 		err = pci_enable_device_mem(pdev);
5280 	if (err) {
5281 		pr_err("Cannot re-enable PCI device after reset.\n");
5282 		return PCI_ERS_RESULT_DISCONNECT;
5283 	}
5284 
5285 	/* flush memory to make sure state is correct */
5286 	smp_mb__before_atomic();
5287 	clear_bit(__E1000_DISABLED, &adapter->flags);
5288 	pci_set_master(pdev);
5289 
5290 	pci_enable_wake(pdev, PCI_D3hot, 0);
5291 	pci_enable_wake(pdev, PCI_D3cold, 0);
5292 
5293 	e1000_reset(adapter);
5294 	ew32(WUS, ~0);
5295 
5296 	return PCI_ERS_RESULT_RECOVERED;
5297 }
5298 
5299 /**
5300  * e1000_io_resume - called when traffic can start flowing again.
5301  * @pdev: Pointer to PCI device
5302  *
5303  * This callback is called when the error recovery driver tells us that
5304  * its OK to resume normal operation. Implementation resembles the
5305  * second-half of the e1000_resume routine.
5306  */
5307 static void e1000_io_resume(struct pci_dev *pdev)
5308 {
5309 	struct net_device *netdev = pci_get_drvdata(pdev);
5310 	struct e1000_adapter *adapter = netdev_priv(netdev);
5311 
5312 	e1000_init_manageability(adapter);
5313 
5314 	if (netif_running(netdev)) {
5315 		if (e1000_up(adapter)) {
5316 			pr_info("can't bring device back up after reset\n");
5317 			return;
5318 		}
5319 	}
5320 
5321 	netif_device_attach(netdev);
5322 }
5323 
5324 /* e1000_main.c */
5325