1 /*
2  * tg3.c: Broadcom Tigon3 ethernet driver.
3  *
4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
6  * Copyright (C) 2004 Sun Microsystems Inc.
7  * Copyright (C) 2005-2014 Broadcom Corporation.
8  *
9  * Firmware is:
10  *	Derived from proprietary unpublished source code,
11  *	Copyright (C) 2000-2003 Broadcom Corporation.
12  *
13  *	Permission is hereby granted for the distribution of this firmware
14  *	data in hexadecimal or equivalent format, provided this copyright
15  *	notice is accompanying it.
16  */
17 
18 
19 #include <linux/module.h>
20 #include <linux/moduleparam.h>
21 #include <linux/stringify.h>
22 #include <linux/kernel.h>
23 #include <linux/types.h>
24 #include <linux/compiler.h>
25 #include <linux/slab.h>
26 #include <linux/delay.h>
27 #include <linux/in.h>
28 #include <linux/interrupt.h>
29 #include <linux/ioport.h>
30 #include <linux/pci.h>
31 #include <linux/netdevice.h>
32 #include <linux/etherdevice.h>
33 #include <linux/skbuff.h>
34 #include <linux/ethtool.h>
35 #include <linux/mdio.h>
36 #include <linux/mii.h>
37 #include <linux/phy.h>
38 #include <linux/brcmphy.h>
39 #include <linux/if.h>
40 #include <linux/if_vlan.h>
41 #include <linux/ip.h>
42 #include <linux/tcp.h>
43 #include <linux/workqueue.h>
44 #include <linux/prefetch.h>
45 #include <linux/dma-mapping.h>
46 #include <linux/firmware.h>
47 #include <linux/ssb/ssb_driver_gige.h>
48 #include <linux/hwmon.h>
49 #include <linux/hwmon-sysfs.h>
50 
51 #include <net/checksum.h>
52 #include <net/ip.h>
53 
54 #include <linux/io.h>
55 #include <asm/byteorder.h>
56 #include <linux/uaccess.h>
57 
58 #include <uapi/linux/net_tstamp.h>
59 #include <linux/ptp_clock_kernel.h>
60 
61 #ifdef CONFIG_SPARC
62 #include <asm/idprom.h>
63 #include <asm/prom.h>
64 #endif
65 
66 #define BAR_0	0
67 #define BAR_2	2
68 
69 #include "tg3.h"
70 
71 /* Functions & macros to verify TG3_FLAGS types */
72 
73 static inline int _tg3_flag(enum TG3_FLAGS flag, unsigned long *bits)
74 {
75 	return test_bit(flag, bits);
76 }
77 
78 static inline void _tg3_flag_set(enum TG3_FLAGS flag, unsigned long *bits)
79 {
80 	set_bit(flag, bits);
81 }
82 
83 static inline void _tg3_flag_clear(enum TG3_FLAGS flag, unsigned long *bits)
84 {
85 	clear_bit(flag, bits);
86 }
87 
88 #define tg3_flag(tp, flag)				\
89 	_tg3_flag(TG3_FLAG_##flag, (tp)->tg3_flags)
90 #define tg3_flag_set(tp, flag)				\
91 	_tg3_flag_set(TG3_FLAG_##flag, (tp)->tg3_flags)
92 #define tg3_flag_clear(tp, flag)			\
93 	_tg3_flag_clear(TG3_FLAG_##flag, (tp)->tg3_flags)
94 
95 #define DRV_MODULE_NAME		"tg3"
96 #define TG3_MAJ_NUM			3
97 #define TG3_MIN_NUM			137
98 #define DRV_MODULE_VERSION	\
99 	__stringify(TG3_MAJ_NUM) "." __stringify(TG3_MIN_NUM)
100 #define DRV_MODULE_RELDATE	"May 11, 2014"
101 
102 #define RESET_KIND_SHUTDOWN	0
103 #define RESET_KIND_INIT		1
104 #define RESET_KIND_SUSPEND	2
105 
106 #define TG3_DEF_RX_MODE		0
107 #define TG3_DEF_TX_MODE		0
108 #define TG3_DEF_MSG_ENABLE	  \
109 	(NETIF_MSG_DRV		| \
110 	 NETIF_MSG_PROBE	| \
111 	 NETIF_MSG_LINK		| \
112 	 NETIF_MSG_TIMER	| \
113 	 NETIF_MSG_IFDOWN	| \
114 	 NETIF_MSG_IFUP		| \
115 	 NETIF_MSG_RX_ERR	| \
116 	 NETIF_MSG_TX_ERR)
117 
118 #define TG3_GRC_LCLCTL_PWRSW_DELAY	100
119 
120 /* length of time before we decide the hardware is borked,
121  * and dev->tx_timeout() should be called to fix the problem
122  */
123 
124 #define TG3_TX_TIMEOUT			(5 * HZ)
125 
126 /* hardware minimum and maximum for a single frame's data payload */
127 #define TG3_MIN_MTU			60
128 #define TG3_MAX_MTU(tp)	\
129 	(tg3_flag(tp, JUMBO_CAPABLE) ? 9000 : 1500)
130 
131 /* These numbers seem to be hard coded in the NIC firmware somehow.
132  * You can't change the ring sizes, but you can change where you place
133  * them in the NIC onboard memory.
134  */
135 #define TG3_RX_STD_RING_SIZE(tp) \
136 	(tg3_flag(tp, LRG_PROD_RING_CAP) ? \
137 	 TG3_RX_STD_MAX_SIZE_5717 : TG3_RX_STD_MAX_SIZE_5700)
138 #define TG3_DEF_RX_RING_PENDING		200
139 #define TG3_RX_JMB_RING_SIZE(tp) \
140 	(tg3_flag(tp, LRG_PROD_RING_CAP) ? \
141 	 TG3_RX_JMB_MAX_SIZE_5717 : TG3_RX_JMB_MAX_SIZE_5700)
142 #define TG3_DEF_RX_JUMBO_RING_PENDING	100
143 
144 /* Do not place this n-ring entries value into the tp struct itself,
145  * we really want to expose these constants to GCC so that modulo et
146  * al.  operations are done with shifts and masks instead of with
147  * hw multiply/modulo instructions.  Another solution would be to
148  * replace things like '% foo' with '& (foo - 1)'.
149  */
150 
151 #define TG3_TX_RING_SIZE		512
152 #define TG3_DEF_TX_RING_PENDING		(TG3_TX_RING_SIZE - 1)
153 
154 #define TG3_RX_STD_RING_BYTES(tp) \
155 	(sizeof(struct tg3_rx_buffer_desc) * TG3_RX_STD_RING_SIZE(tp))
156 #define TG3_RX_JMB_RING_BYTES(tp) \
157 	(sizeof(struct tg3_ext_rx_buffer_desc) * TG3_RX_JMB_RING_SIZE(tp))
158 #define TG3_RX_RCB_RING_BYTES(tp) \
159 	(sizeof(struct tg3_rx_buffer_desc) * (tp->rx_ret_ring_mask + 1))
160 #define TG3_TX_RING_BYTES	(sizeof(struct tg3_tx_buffer_desc) * \
161 				 TG3_TX_RING_SIZE)
162 #define NEXT_TX(N)		(((N) + 1) & (TG3_TX_RING_SIZE - 1))
163 
164 #define TG3_DMA_BYTE_ENAB		64
165 
166 #define TG3_RX_STD_DMA_SZ		1536
167 #define TG3_RX_JMB_DMA_SZ		9046
168 
169 #define TG3_RX_DMA_TO_MAP_SZ(x)		((x) + TG3_DMA_BYTE_ENAB)
170 
171 #define TG3_RX_STD_MAP_SZ		TG3_RX_DMA_TO_MAP_SZ(TG3_RX_STD_DMA_SZ)
172 #define TG3_RX_JMB_MAP_SZ		TG3_RX_DMA_TO_MAP_SZ(TG3_RX_JMB_DMA_SZ)
173 
174 #define TG3_RX_STD_BUFF_RING_SIZE(tp) \
175 	(sizeof(struct ring_info) * TG3_RX_STD_RING_SIZE(tp))
176 
177 #define TG3_RX_JMB_BUFF_RING_SIZE(tp) \
178 	(sizeof(struct ring_info) * TG3_RX_JMB_RING_SIZE(tp))
179 
180 /* Due to a hardware bug, the 5701 can only DMA to memory addresses
181  * that are at least dword aligned when used in PCIX mode.  The driver
182  * works around this bug by double copying the packet.  This workaround
183  * is built into the normal double copy length check for efficiency.
184  *
185  * However, the double copy is only necessary on those architectures
186  * where unaligned memory accesses are inefficient.  For those architectures
187  * where unaligned memory accesses incur little penalty, we can reintegrate
188  * the 5701 in the normal rx path.  Doing so saves a device structure
189  * dereference by hardcoding the double copy threshold in place.
190  */
191 #define TG3_RX_COPY_THRESHOLD		256
192 #if NET_IP_ALIGN == 0 || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
193 	#define TG3_RX_COPY_THRESH(tp)	TG3_RX_COPY_THRESHOLD
194 #else
195 	#define TG3_RX_COPY_THRESH(tp)	((tp)->rx_copy_thresh)
196 #endif
197 
198 #if (NET_IP_ALIGN != 0)
199 #define TG3_RX_OFFSET(tp)	((tp)->rx_offset)
200 #else
201 #define TG3_RX_OFFSET(tp)	(NET_SKB_PAD)
202 #endif
203 
204 /* minimum number of free TX descriptors required to wake up TX process */
205 #define TG3_TX_WAKEUP_THRESH(tnapi)		((tnapi)->tx_pending / 4)
206 #define TG3_TX_BD_DMA_MAX_2K		2048
207 #define TG3_TX_BD_DMA_MAX_4K		4096
208 
209 #define TG3_RAW_IP_ALIGN 2
210 
211 #define TG3_MAX_UCAST_ADDR(tp) (tg3_flag((tp), ENABLE_ASF) ? 2 : 3)
212 #define TG3_UCAST_ADDR_IDX(tp) (tg3_flag((tp), ENABLE_ASF) ? 2 : 1)
213 
214 #define TG3_FW_UPDATE_TIMEOUT_SEC	5
215 #define TG3_FW_UPDATE_FREQ_SEC		(TG3_FW_UPDATE_TIMEOUT_SEC / 2)
216 
217 #define FIRMWARE_TG3		"tigon/tg3.bin"
218 #define FIRMWARE_TG357766	"tigon/tg357766.bin"
219 #define FIRMWARE_TG3TSO		"tigon/tg3_tso.bin"
220 #define FIRMWARE_TG3TSO5	"tigon/tg3_tso5.bin"
221 
222 static char version[] =
223 	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")";
224 
225 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
226 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
227 MODULE_LICENSE("GPL");
228 MODULE_VERSION(DRV_MODULE_VERSION);
229 MODULE_FIRMWARE(FIRMWARE_TG3);
230 MODULE_FIRMWARE(FIRMWARE_TG3TSO);
231 MODULE_FIRMWARE(FIRMWARE_TG3TSO5);
232 
233 static int tg3_debug = -1;	/* -1 == use TG3_DEF_MSG_ENABLE as value */
234 module_param(tg3_debug, int, 0);
235 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
236 
237 #define TG3_DRV_DATA_FLAG_10_100_ONLY	0x0001
238 #define TG3_DRV_DATA_FLAG_5705_10_100	0x0002
239 
240 static const struct pci_device_id tg3_pci_tbl[] = {
241 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
242 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
243 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
244 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
245 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
246 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
247 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
248 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
249 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
250 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
251 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
252 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
253 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
254 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
255 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
256 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
257 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
258 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
259 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901),
260 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
261 			TG3_DRV_DATA_FLAG_5705_10_100},
262 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2),
263 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
264 			TG3_DRV_DATA_FLAG_5705_10_100},
265 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
266 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F),
267 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
268 			TG3_DRV_DATA_FLAG_5705_10_100},
269 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
270 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
271 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
272 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
273 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
274 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F),
275 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
276 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
277 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
278 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
279 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
280 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F),
281 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
282 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
283 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
284 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
285 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
286 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
287 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
288 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
289 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5787M,
290 			PCI_VENDOR_ID_LENOVO,
291 			TG3PCI_SUBDEVICE_ID_LENOVO_5787M),
292 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
293 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
294 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F),
295 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
296 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
297 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
298 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
299 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
300 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
301 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
302 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
303 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
304 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
305 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
306 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
307 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
308 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
309 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
310 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761S)},
311 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761SE)},
312 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_G)},
313 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_F)},
314 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780,
315 			PCI_VENDOR_ID_AI, TG3PCI_SUBDEVICE_ID_ACER_57780_A),
316 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
317 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780,
318 			PCI_VENDOR_ID_AI, TG3PCI_SUBDEVICE_ID_ACER_57780_B),
319 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
320 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780)},
321 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)},
322 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790),
323 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
324 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57788)},
325 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717)},
326 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717_C)},
327 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5718)},
328 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57781)},
329 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57785)},
330 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57761)},
331 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57765)},
332 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57791),
333 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
334 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57795),
335 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
336 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5719)},
337 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5720)},
338 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57762)},
339 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57766)},
340 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5762)},
341 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5725)},
342 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5727)},
343 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57764)},
344 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57767)},
345 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57787)},
346 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57782)},
347 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57786)},
348 	{PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
349 	{PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
350 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
351 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
352 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
353 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
354 	{PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
355 	{PCI_DEVICE(0x10cf, 0x11a2)}, /* Fujitsu 1000base-SX with BCM5703SKHB */
356 	{}
357 };
358 
359 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
360 
361 static const struct {
362 	const char string[ETH_GSTRING_LEN];
363 } ethtool_stats_keys[] = {
364 	{ "rx_octets" },
365 	{ "rx_fragments" },
366 	{ "rx_ucast_packets" },
367 	{ "rx_mcast_packets" },
368 	{ "rx_bcast_packets" },
369 	{ "rx_fcs_errors" },
370 	{ "rx_align_errors" },
371 	{ "rx_xon_pause_rcvd" },
372 	{ "rx_xoff_pause_rcvd" },
373 	{ "rx_mac_ctrl_rcvd" },
374 	{ "rx_xoff_entered" },
375 	{ "rx_frame_too_long_errors" },
376 	{ "rx_jabbers" },
377 	{ "rx_undersize_packets" },
378 	{ "rx_in_length_errors" },
379 	{ "rx_out_length_errors" },
380 	{ "rx_64_or_less_octet_packets" },
381 	{ "rx_65_to_127_octet_packets" },
382 	{ "rx_128_to_255_octet_packets" },
383 	{ "rx_256_to_511_octet_packets" },
384 	{ "rx_512_to_1023_octet_packets" },
385 	{ "rx_1024_to_1522_octet_packets" },
386 	{ "rx_1523_to_2047_octet_packets" },
387 	{ "rx_2048_to_4095_octet_packets" },
388 	{ "rx_4096_to_8191_octet_packets" },
389 	{ "rx_8192_to_9022_octet_packets" },
390 
391 	{ "tx_octets" },
392 	{ "tx_collisions" },
393 
394 	{ "tx_xon_sent" },
395 	{ "tx_xoff_sent" },
396 	{ "tx_flow_control" },
397 	{ "tx_mac_errors" },
398 	{ "tx_single_collisions" },
399 	{ "tx_mult_collisions" },
400 	{ "tx_deferred" },
401 	{ "tx_excessive_collisions" },
402 	{ "tx_late_collisions" },
403 	{ "tx_collide_2times" },
404 	{ "tx_collide_3times" },
405 	{ "tx_collide_4times" },
406 	{ "tx_collide_5times" },
407 	{ "tx_collide_6times" },
408 	{ "tx_collide_7times" },
409 	{ "tx_collide_8times" },
410 	{ "tx_collide_9times" },
411 	{ "tx_collide_10times" },
412 	{ "tx_collide_11times" },
413 	{ "tx_collide_12times" },
414 	{ "tx_collide_13times" },
415 	{ "tx_collide_14times" },
416 	{ "tx_collide_15times" },
417 	{ "tx_ucast_packets" },
418 	{ "tx_mcast_packets" },
419 	{ "tx_bcast_packets" },
420 	{ "tx_carrier_sense_errors" },
421 	{ "tx_discards" },
422 	{ "tx_errors" },
423 
424 	{ "dma_writeq_full" },
425 	{ "dma_write_prioq_full" },
426 	{ "rxbds_empty" },
427 	{ "rx_discards" },
428 	{ "rx_errors" },
429 	{ "rx_threshold_hit" },
430 
431 	{ "dma_readq_full" },
432 	{ "dma_read_prioq_full" },
433 	{ "tx_comp_queue_full" },
434 
435 	{ "ring_set_send_prod_index" },
436 	{ "ring_status_update" },
437 	{ "nic_irqs" },
438 	{ "nic_avoided_irqs" },
439 	{ "nic_tx_threshold_hit" },
440 
441 	{ "mbuf_lwm_thresh_hit" },
442 };
443 
444 #define TG3_NUM_STATS	ARRAY_SIZE(ethtool_stats_keys)
445 #define TG3_NVRAM_TEST		0
446 #define TG3_LINK_TEST		1
447 #define TG3_REGISTER_TEST	2
448 #define TG3_MEMORY_TEST		3
449 #define TG3_MAC_LOOPB_TEST	4
450 #define TG3_PHY_LOOPB_TEST	5
451 #define TG3_EXT_LOOPB_TEST	6
452 #define TG3_INTERRUPT_TEST	7
453 
454 
455 static const struct {
456 	const char string[ETH_GSTRING_LEN];
457 } ethtool_test_keys[] = {
458 	[TG3_NVRAM_TEST]	= { "nvram test        (online) " },
459 	[TG3_LINK_TEST]		= { "link test         (online) " },
460 	[TG3_REGISTER_TEST]	= { "register test     (offline)" },
461 	[TG3_MEMORY_TEST]	= { "memory test       (offline)" },
462 	[TG3_MAC_LOOPB_TEST]	= { "mac loopback test (offline)" },
463 	[TG3_PHY_LOOPB_TEST]	= { "phy loopback test (offline)" },
464 	[TG3_EXT_LOOPB_TEST]	= { "ext loopback test (offline)" },
465 	[TG3_INTERRUPT_TEST]	= { "interrupt test    (offline)" },
466 };
467 
468 #define TG3_NUM_TEST	ARRAY_SIZE(ethtool_test_keys)
469 
470 
471 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
472 {
473 	writel(val, tp->regs + off);
474 }
475 
476 static u32 tg3_read32(struct tg3 *tp, u32 off)
477 {
478 	return readl(tp->regs + off);
479 }
480 
481 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
482 {
483 	writel(val, tp->aperegs + off);
484 }
485 
486 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
487 {
488 	return readl(tp->aperegs + off);
489 }
490 
491 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
492 {
493 	unsigned long flags;
494 
495 	spin_lock_irqsave(&tp->indirect_lock, flags);
496 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
497 	pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
498 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
499 }
500 
501 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
502 {
503 	writel(val, tp->regs + off);
504 	readl(tp->regs + off);
505 }
506 
507 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
508 {
509 	unsigned long flags;
510 	u32 val;
511 
512 	spin_lock_irqsave(&tp->indirect_lock, flags);
513 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
514 	pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
515 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
516 	return val;
517 }
518 
519 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
520 {
521 	unsigned long flags;
522 
523 	if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
524 		pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
525 				       TG3_64BIT_REG_LOW, val);
526 		return;
527 	}
528 	if (off == TG3_RX_STD_PROD_IDX_REG) {
529 		pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
530 				       TG3_64BIT_REG_LOW, val);
531 		return;
532 	}
533 
534 	spin_lock_irqsave(&tp->indirect_lock, flags);
535 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
536 	pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
537 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
538 
539 	/* In indirect mode when disabling interrupts, we also need
540 	 * to clear the interrupt bit in the GRC local ctrl register.
541 	 */
542 	if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
543 	    (val == 0x1)) {
544 		pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
545 				       tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
546 	}
547 }
548 
549 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
550 {
551 	unsigned long flags;
552 	u32 val;
553 
554 	spin_lock_irqsave(&tp->indirect_lock, flags);
555 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
556 	pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
557 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
558 	return val;
559 }
560 
561 /* usec_wait specifies the wait time in usec when writing to certain registers
562  * where it is unsafe to read back the register without some delay.
563  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
564  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
565  */
566 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
567 {
568 	if (tg3_flag(tp, PCIX_TARGET_HWBUG) || tg3_flag(tp, ICH_WORKAROUND))
569 		/* Non-posted methods */
570 		tp->write32(tp, off, val);
571 	else {
572 		/* Posted method */
573 		tg3_write32(tp, off, val);
574 		if (usec_wait)
575 			udelay(usec_wait);
576 		tp->read32(tp, off);
577 	}
578 	/* Wait again after the read for the posted method to guarantee that
579 	 * the wait time is met.
580 	 */
581 	if (usec_wait)
582 		udelay(usec_wait);
583 }
584 
585 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
586 {
587 	tp->write32_mbox(tp, off, val);
588 	if (tg3_flag(tp, FLUSH_POSTED_WRITES) ||
589 	    (!tg3_flag(tp, MBOX_WRITE_REORDER) &&
590 	     !tg3_flag(tp, ICH_WORKAROUND)))
591 		tp->read32_mbox(tp, off);
592 }
593 
594 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
595 {
596 	void __iomem *mbox = tp->regs + off;
597 	writel(val, mbox);
598 	if (tg3_flag(tp, TXD_MBOX_HWBUG))
599 		writel(val, mbox);
600 	if (tg3_flag(tp, MBOX_WRITE_REORDER) ||
601 	    tg3_flag(tp, FLUSH_POSTED_WRITES))
602 		readl(mbox);
603 }
604 
605 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
606 {
607 	return readl(tp->regs + off + GRCMBOX_BASE);
608 }
609 
610 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
611 {
612 	writel(val, tp->regs + off + GRCMBOX_BASE);
613 }
614 
615 #define tw32_mailbox(reg, val)		tp->write32_mbox(tp, reg, val)
616 #define tw32_mailbox_f(reg, val)	tw32_mailbox_flush(tp, (reg), (val))
617 #define tw32_rx_mbox(reg, val)		tp->write32_rx_mbox(tp, reg, val)
618 #define tw32_tx_mbox(reg, val)		tp->write32_tx_mbox(tp, reg, val)
619 #define tr32_mailbox(reg)		tp->read32_mbox(tp, reg)
620 
621 #define tw32(reg, val)			tp->write32(tp, reg, val)
622 #define tw32_f(reg, val)		_tw32_flush(tp, (reg), (val), 0)
623 #define tw32_wait_f(reg, val, us)	_tw32_flush(tp, (reg), (val), (us))
624 #define tr32(reg)			tp->read32(tp, reg)
625 
626 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
627 {
628 	unsigned long flags;
629 
630 	if (tg3_asic_rev(tp) == ASIC_REV_5906 &&
631 	    (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
632 		return;
633 
634 	spin_lock_irqsave(&tp->indirect_lock, flags);
635 	if (tg3_flag(tp, SRAM_USE_CONFIG)) {
636 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
637 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
638 
639 		/* Always leave this as zero. */
640 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
641 	} else {
642 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
643 		tw32_f(TG3PCI_MEM_WIN_DATA, val);
644 
645 		/* Always leave this as zero. */
646 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
647 	}
648 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
649 }
650 
651 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
652 {
653 	unsigned long flags;
654 
655 	if (tg3_asic_rev(tp) == ASIC_REV_5906 &&
656 	    (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
657 		*val = 0;
658 		return;
659 	}
660 
661 	spin_lock_irqsave(&tp->indirect_lock, flags);
662 	if (tg3_flag(tp, SRAM_USE_CONFIG)) {
663 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
664 		pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
665 
666 		/* Always leave this as zero. */
667 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
668 	} else {
669 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
670 		*val = tr32(TG3PCI_MEM_WIN_DATA);
671 
672 		/* Always leave this as zero. */
673 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
674 	}
675 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
676 }
677 
678 static void tg3_ape_lock_init(struct tg3 *tp)
679 {
680 	int i;
681 	u32 regbase, bit;
682 
683 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
684 		regbase = TG3_APE_LOCK_GRANT;
685 	else
686 		regbase = TG3_APE_PER_LOCK_GRANT;
687 
688 	/* Make sure the driver hasn't any stale locks. */
689 	for (i = TG3_APE_LOCK_PHY0; i <= TG3_APE_LOCK_GPIO; i++) {
690 		switch (i) {
691 		case TG3_APE_LOCK_PHY0:
692 		case TG3_APE_LOCK_PHY1:
693 		case TG3_APE_LOCK_PHY2:
694 		case TG3_APE_LOCK_PHY3:
695 			bit = APE_LOCK_GRANT_DRIVER;
696 			break;
697 		default:
698 			if (!tp->pci_fn)
699 				bit = APE_LOCK_GRANT_DRIVER;
700 			else
701 				bit = 1 << tp->pci_fn;
702 		}
703 		tg3_ape_write32(tp, regbase + 4 * i, bit);
704 	}
705 
706 }
707 
708 static int tg3_ape_lock(struct tg3 *tp, int locknum)
709 {
710 	int i, off;
711 	int ret = 0;
712 	u32 status, req, gnt, bit;
713 
714 	if (!tg3_flag(tp, ENABLE_APE))
715 		return 0;
716 
717 	switch (locknum) {
718 	case TG3_APE_LOCK_GPIO:
719 		if (tg3_asic_rev(tp) == ASIC_REV_5761)
720 			return 0;
721 	case TG3_APE_LOCK_GRC:
722 	case TG3_APE_LOCK_MEM:
723 		if (!tp->pci_fn)
724 			bit = APE_LOCK_REQ_DRIVER;
725 		else
726 			bit = 1 << tp->pci_fn;
727 		break;
728 	case TG3_APE_LOCK_PHY0:
729 	case TG3_APE_LOCK_PHY1:
730 	case TG3_APE_LOCK_PHY2:
731 	case TG3_APE_LOCK_PHY3:
732 		bit = APE_LOCK_REQ_DRIVER;
733 		break;
734 	default:
735 		return -EINVAL;
736 	}
737 
738 	if (tg3_asic_rev(tp) == ASIC_REV_5761) {
739 		req = TG3_APE_LOCK_REQ;
740 		gnt = TG3_APE_LOCK_GRANT;
741 	} else {
742 		req = TG3_APE_PER_LOCK_REQ;
743 		gnt = TG3_APE_PER_LOCK_GRANT;
744 	}
745 
746 	off = 4 * locknum;
747 
748 	tg3_ape_write32(tp, req + off, bit);
749 
750 	/* Wait for up to 1 millisecond to acquire lock. */
751 	for (i = 0; i < 100; i++) {
752 		status = tg3_ape_read32(tp, gnt + off);
753 		if (status == bit)
754 			break;
755 		if (pci_channel_offline(tp->pdev))
756 			break;
757 
758 		udelay(10);
759 	}
760 
761 	if (status != bit) {
762 		/* Revoke the lock request. */
763 		tg3_ape_write32(tp, gnt + off, bit);
764 		ret = -EBUSY;
765 	}
766 
767 	return ret;
768 }
769 
770 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
771 {
772 	u32 gnt, bit;
773 
774 	if (!tg3_flag(tp, ENABLE_APE))
775 		return;
776 
777 	switch (locknum) {
778 	case TG3_APE_LOCK_GPIO:
779 		if (tg3_asic_rev(tp) == ASIC_REV_5761)
780 			return;
781 	case TG3_APE_LOCK_GRC:
782 	case TG3_APE_LOCK_MEM:
783 		if (!tp->pci_fn)
784 			bit = APE_LOCK_GRANT_DRIVER;
785 		else
786 			bit = 1 << tp->pci_fn;
787 		break;
788 	case TG3_APE_LOCK_PHY0:
789 	case TG3_APE_LOCK_PHY1:
790 	case TG3_APE_LOCK_PHY2:
791 	case TG3_APE_LOCK_PHY3:
792 		bit = APE_LOCK_GRANT_DRIVER;
793 		break;
794 	default:
795 		return;
796 	}
797 
798 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
799 		gnt = TG3_APE_LOCK_GRANT;
800 	else
801 		gnt = TG3_APE_PER_LOCK_GRANT;
802 
803 	tg3_ape_write32(tp, gnt + 4 * locknum, bit);
804 }
805 
806 static int tg3_ape_event_lock(struct tg3 *tp, u32 timeout_us)
807 {
808 	u32 apedata;
809 
810 	while (timeout_us) {
811 		if (tg3_ape_lock(tp, TG3_APE_LOCK_MEM))
812 			return -EBUSY;
813 
814 		apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
815 		if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
816 			break;
817 
818 		tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
819 
820 		udelay(10);
821 		timeout_us -= (timeout_us > 10) ? 10 : timeout_us;
822 	}
823 
824 	return timeout_us ? 0 : -EBUSY;
825 }
826 
827 static int tg3_ape_wait_for_event(struct tg3 *tp, u32 timeout_us)
828 {
829 	u32 i, apedata;
830 
831 	for (i = 0; i < timeout_us / 10; i++) {
832 		apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
833 
834 		if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
835 			break;
836 
837 		udelay(10);
838 	}
839 
840 	return i == timeout_us / 10;
841 }
842 
843 static int tg3_ape_scratchpad_read(struct tg3 *tp, u32 *data, u32 base_off,
844 				   u32 len)
845 {
846 	int err;
847 	u32 i, bufoff, msgoff, maxlen, apedata;
848 
849 	if (!tg3_flag(tp, APE_HAS_NCSI))
850 		return 0;
851 
852 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
853 	if (apedata != APE_SEG_SIG_MAGIC)
854 		return -ENODEV;
855 
856 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
857 	if (!(apedata & APE_FW_STATUS_READY))
858 		return -EAGAIN;
859 
860 	bufoff = tg3_ape_read32(tp, TG3_APE_SEG_MSG_BUF_OFF) +
861 		 TG3_APE_SHMEM_BASE;
862 	msgoff = bufoff + 2 * sizeof(u32);
863 	maxlen = tg3_ape_read32(tp, TG3_APE_SEG_MSG_BUF_LEN);
864 
865 	while (len) {
866 		u32 length;
867 
868 		/* Cap xfer sizes to scratchpad limits. */
869 		length = (len > maxlen) ? maxlen : len;
870 		len -= length;
871 
872 		apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
873 		if (!(apedata & APE_FW_STATUS_READY))
874 			return -EAGAIN;
875 
876 		/* Wait for up to 1 msec for APE to service previous event. */
877 		err = tg3_ape_event_lock(tp, 1000);
878 		if (err)
879 			return err;
880 
881 		apedata = APE_EVENT_STATUS_DRIVER_EVNT |
882 			  APE_EVENT_STATUS_SCRTCHPD_READ |
883 			  APE_EVENT_STATUS_EVENT_PENDING;
884 		tg3_ape_write32(tp, TG3_APE_EVENT_STATUS, apedata);
885 
886 		tg3_ape_write32(tp, bufoff, base_off);
887 		tg3_ape_write32(tp, bufoff + sizeof(u32), length);
888 
889 		tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
890 		tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
891 
892 		base_off += length;
893 
894 		if (tg3_ape_wait_for_event(tp, 30000))
895 			return -EAGAIN;
896 
897 		for (i = 0; length; i += 4, length -= 4) {
898 			u32 val = tg3_ape_read32(tp, msgoff + i);
899 			memcpy(data, &val, sizeof(u32));
900 			data++;
901 		}
902 	}
903 
904 	return 0;
905 }
906 
907 static int tg3_ape_send_event(struct tg3 *tp, u32 event)
908 {
909 	int err;
910 	u32 apedata;
911 
912 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
913 	if (apedata != APE_SEG_SIG_MAGIC)
914 		return -EAGAIN;
915 
916 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
917 	if (!(apedata & APE_FW_STATUS_READY))
918 		return -EAGAIN;
919 
920 	/* Wait for up to 1 millisecond for APE to service previous event. */
921 	err = tg3_ape_event_lock(tp, 1000);
922 	if (err)
923 		return err;
924 
925 	tg3_ape_write32(tp, TG3_APE_EVENT_STATUS,
926 			event | APE_EVENT_STATUS_EVENT_PENDING);
927 
928 	tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
929 	tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
930 
931 	return 0;
932 }
933 
934 static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
935 {
936 	u32 event;
937 	u32 apedata;
938 
939 	if (!tg3_flag(tp, ENABLE_APE))
940 		return;
941 
942 	switch (kind) {
943 	case RESET_KIND_INIT:
944 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
945 				APE_HOST_SEG_SIG_MAGIC);
946 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
947 				APE_HOST_SEG_LEN_MAGIC);
948 		apedata = tg3_ape_read32(tp, TG3_APE_HOST_INIT_COUNT);
949 		tg3_ape_write32(tp, TG3_APE_HOST_INIT_COUNT, ++apedata);
950 		tg3_ape_write32(tp, TG3_APE_HOST_DRIVER_ID,
951 			APE_HOST_DRIVER_ID_MAGIC(TG3_MAJ_NUM, TG3_MIN_NUM));
952 		tg3_ape_write32(tp, TG3_APE_HOST_BEHAVIOR,
953 				APE_HOST_BEHAV_NO_PHYLOCK);
954 		tg3_ape_write32(tp, TG3_APE_HOST_DRVR_STATE,
955 				    TG3_APE_HOST_DRVR_STATE_START);
956 
957 		event = APE_EVENT_STATUS_STATE_START;
958 		break;
959 	case RESET_KIND_SHUTDOWN:
960 		/* With the interface we are currently using,
961 		 * APE does not track driver state.  Wiping
962 		 * out the HOST SEGMENT SIGNATURE forces
963 		 * the APE to assume OS absent status.
964 		 */
965 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG, 0x0);
966 
967 		if (device_may_wakeup(&tp->pdev->dev) &&
968 		    tg3_flag(tp, WOL_ENABLE)) {
969 			tg3_ape_write32(tp, TG3_APE_HOST_WOL_SPEED,
970 					    TG3_APE_HOST_WOL_SPEED_AUTO);
971 			apedata = TG3_APE_HOST_DRVR_STATE_WOL;
972 		} else
973 			apedata = TG3_APE_HOST_DRVR_STATE_UNLOAD;
974 
975 		tg3_ape_write32(tp, TG3_APE_HOST_DRVR_STATE, apedata);
976 
977 		event = APE_EVENT_STATUS_STATE_UNLOAD;
978 		break;
979 	default:
980 		return;
981 	}
982 
983 	event |= APE_EVENT_STATUS_DRIVER_EVNT | APE_EVENT_STATUS_STATE_CHNGE;
984 
985 	tg3_ape_send_event(tp, event);
986 }
987 
988 static void tg3_disable_ints(struct tg3 *tp)
989 {
990 	int i;
991 
992 	tw32(TG3PCI_MISC_HOST_CTRL,
993 	     (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
994 	for (i = 0; i < tp->irq_max; i++)
995 		tw32_mailbox_f(tp->napi[i].int_mbox, 0x00000001);
996 }
997 
998 static void tg3_enable_ints(struct tg3 *tp)
999 {
1000 	int i;
1001 
1002 	tp->irq_sync = 0;
1003 	wmb();
1004 
1005 	tw32(TG3PCI_MISC_HOST_CTRL,
1006 	     (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
1007 
1008 	tp->coal_now = tp->coalesce_mode | HOSTCC_MODE_ENABLE;
1009 	for (i = 0; i < tp->irq_cnt; i++) {
1010 		struct tg3_napi *tnapi = &tp->napi[i];
1011 
1012 		tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
1013 		if (tg3_flag(tp, 1SHOT_MSI))
1014 			tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
1015 
1016 		tp->coal_now |= tnapi->coal_now;
1017 	}
1018 
1019 	/* Force an initial interrupt */
1020 	if (!tg3_flag(tp, TAGGED_STATUS) &&
1021 	    (tp->napi[0].hw_status->status & SD_STATUS_UPDATED))
1022 		tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
1023 	else
1024 		tw32(HOSTCC_MODE, tp->coal_now);
1025 
1026 	tp->coal_now &= ~(tp->napi[0].coal_now | tp->napi[1].coal_now);
1027 }
1028 
1029 static inline unsigned int tg3_has_work(struct tg3_napi *tnapi)
1030 {
1031 	struct tg3 *tp = tnapi->tp;
1032 	struct tg3_hw_status *sblk = tnapi->hw_status;
1033 	unsigned int work_exists = 0;
1034 
1035 	/* check for phy events */
1036 	if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
1037 		if (sblk->status & SD_STATUS_LINK_CHG)
1038 			work_exists = 1;
1039 	}
1040 
1041 	/* check for TX work to do */
1042 	if (sblk->idx[0].tx_consumer != tnapi->tx_cons)
1043 		work_exists = 1;
1044 
1045 	/* check for RX work to do */
1046 	if (tnapi->rx_rcb_prod_idx &&
1047 	    *(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
1048 		work_exists = 1;
1049 
1050 	return work_exists;
1051 }
1052 
1053 /* tg3_int_reenable
1054  *  similar to tg3_enable_ints, but it accurately determines whether there
1055  *  is new work pending and can return without flushing the PIO write
1056  *  which reenables interrupts
1057  */
1058 static void tg3_int_reenable(struct tg3_napi *tnapi)
1059 {
1060 	struct tg3 *tp = tnapi->tp;
1061 
1062 	tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
1063 	mmiowb();
1064 
1065 	/* When doing tagged status, this work check is unnecessary.
1066 	 * The last_tag we write above tells the chip which piece of
1067 	 * work we've completed.
1068 	 */
1069 	if (!tg3_flag(tp, TAGGED_STATUS) && tg3_has_work(tnapi))
1070 		tw32(HOSTCC_MODE, tp->coalesce_mode |
1071 		     HOSTCC_MODE_ENABLE | tnapi->coal_now);
1072 }
1073 
1074 static void tg3_switch_clocks(struct tg3 *tp)
1075 {
1076 	u32 clock_ctrl;
1077 	u32 orig_clock_ctrl;
1078 
1079 	if (tg3_flag(tp, CPMU_PRESENT) || tg3_flag(tp, 5780_CLASS))
1080 		return;
1081 
1082 	clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
1083 
1084 	orig_clock_ctrl = clock_ctrl;
1085 	clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
1086 		       CLOCK_CTRL_CLKRUN_OENABLE |
1087 		       0x1f);
1088 	tp->pci_clock_ctrl = clock_ctrl;
1089 
1090 	if (tg3_flag(tp, 5705_PLUS)) {
1091 		if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
1092 			tw32_wait_f(TG3PCI_CLOCK_CTRL,
1093 				    clock_ctrl | CLOCK_CTRL_625_CORE, 40);
1094 		}
1095 	} else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
1096 		tw32_wait_f(TG3PCI_CLOCK_CTRL,
1097 			    clock_ctrl |
1098 			    (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
1099 			    40);
1100 		tw32_wait_f(TG3PCI_CLOCK_CTRL,
1101 			    clock_ctrl | (CLOCK_CTRL_ALTCLK),
1102 			    40);
1103 	}
1104 	tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
1105 }
1106 
1107 #define PHY_BUSY_LOOPS	5000
1108 
1109 static int __tg3_readphy(struct tg3 *tp, unsigned int phy_addr, int reg,
1110 			 u32 *val)
1111 {
1112 	u32 frame_val;
1113 	unsigned int loops;
1114 	int ret;
1115 
1116 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1117 		tw32_f(MAC_MI_MODE,
1118 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
1119 		udelay(80);
1120 	}
1121 
1122 	tg3_ape_lock(tp, tp->phy_ape_lock);
1123 
1124 	*val = 0x0;
1125 
1126 	frame_val  = ((phy_addr << MI_COM_PHY_ADDR_SHIFT) &
1127 		      MI_COM_PHY_ADDR_MASK);
1128 	frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
1129 		      MI_COM_REG_ADDR_MASK);
1130 	frame_val |= (MI_COM_CMD_READ | MI_COM_START);
1131 
1132 	tw32_f(MAC_MI_COM, frame_val);
1133 
1134 	loops = PHY_BUSY_LOOPS;
1135 	while (loops != 0) {
1136 		udelay(10);
1137 		frame_val = tr32(MAC_MI_COM);
1138 
1139 		if ((frame_val & MI_COM_BUSY) == 0) {
1140 			udelay(5);
1141 			frame_val = tr32(MAC_MI_COM);
1142 			break;
1143 		}
1144 		loops -= 1;
1145 	}
1146 
1147 	ret = -EBUSY;
1148 	if (loops != 0) {
1149 		*val = frame_val & MI_COM_DATA_MASK;
1150 		ret = 0;
1151 	}
1152 
1153 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1154 		tw32_f(MAC_MI_MODE, tp->mi_mode);
1155 		udelay(80);
1156 	}
1157 
1158 	tg3_ape_unlock(tp, tp->phy_ape_lock);
1159 
1160 	return ret;
1161 }
1162 
1163 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
1164 {
1165 	return __tg3_readphy(tp, tp->phy_addr, reg, val);
1166 }
1167 
1168 static int __tg3_writephy(struct tg3 *tp, unsigned int phy_addr, int reg,
1169 			  u32 val)
1170 {
1171 	u32 frame_val;
1172 	unsigned int loops;
1173 	int ret;
1174 
1175 	if ((tp->phy_flags & TG3_PHYFLG_IS_FET) &&
1176 	    (reg == MII_CTRL1000 || reg == MII_TG3_AUX_CTRL))
1177 		return 0;
1178 
1179 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1180 		tw32_f(MAC_MI_MODE,
1181 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
1182 		udelay(80);
1183 	}
1184 
1185 	tg3_ape_lock(tp, tp->phy_ape_lock);
1186 
1187 	frame_val  = ((phy_addr << MI_COM_PHY_ADDR_SHIFT) &
1188 		      MI_COM_PHY_ADDR_MASK);
1189 	frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
1190 		      MI_COM_REG_ADDR_MASK);
1191 	frame_val |= (val & MI_COM_DATA_MASK);
1192 	frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
1193 
1194 	tw32_f(MAC_MI_COM, frame_val);
1195 
1196 	loops = PHY_BUSY_LOOPS;
1197 	while (loops != 0) {
1198 		udelay(10);
1199 		frame_val = tr32(MAC_MI_COM);
1200 		if ((frame_val & MI_COM_BUSY) == 0) {
1201 			udelay(5);
1202 			frame_val = tr32(MAC_MI_COM);
1203 			break;
1204 		}
1205 		loops -= 1;
1206 	}
1207 
1208 	ret = -EBUSY;
1209 	if (loops != 0)
1210 		ret = 0;
1211 
1212 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1213 		tw32_f(MAC_MI_MODE, tp->mi_mode);
1214 		udelay(80);
1215 	}
1216 
1217 	tg3_ape_unlock(tp, tp->phy_ape_lock);
1218 
1219 	return ret;
1220 }
1221 
1222 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
1223 {
1224 	return __tg3_writephy(tp, tp->phy_addr, reg, val);
1225 }
1226 
1227 static int tg3_phy_cl45_write(struct tg3 *tp, u32 devad, u32 addr, u32 val)
1228 {
1229 	int err;
1230 
1231 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
1232 	if (err)
1233 		goto done;
1234 
1235 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
1236 	if (err)
1237 		goto done;
1238 
1239 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
1240 			   MII_TG3_MMD_CTRL_DATA_NOINC | devad);
1241 	if (err)
1242 		goto done;
1243 
1244 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, val);
1245 
1246 done:
1247 	return err;
1248 }
1249 
1250 static int tg3_phy_cl45_read(struct tg3 *tp, u32 devad, u32 addr, u32 *val)
1251 {
1252 	int err;
1253 
1254 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
1255 	if (err)
1256 		goto done;
1257 
1258 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
1259 	if (err)
1260 		goto done;
1261 
1262 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
1263 			   MII_TG3_MMD_CTRL_DATA_NOINC | devad);
1264 	if (err)
1265 		goto done;
1266 
1267 	err = tg3_readphy(tp, MII_TG3_MMD_ADDRESS, val);
1268 
1269 done:
1270 	return err;
1271 }
1272 
1273 static int tg3_phydsp_read(struct tg3 *tp, u32 reg, u32 *val)
1274 {
1275 	int err;
1276 
1277 	err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1278 	if (!err)
1279 		err = tg3_readphy(tp, MII_TG3_DSP_RW_PORT, val);
1280 
1281 	return err;
1282 }
1283 
1284 static int tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
1285 {
1286 	int err;
1287 
1288 	err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1289 	if (!err)
1290 		err = tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
1291 
1292 	return err;
1293 }
1294 
1295 static int tg3_phy_auxctl_read(struct tg3 *tp, int reg, u32 *val)
1296 {
1297 	int err;
1298 
1299 	err = tg3_writephy(tp, MII_TG3_AUX_CTRL,
1300 			   (reg << MII_TG3_AUXCTL_MISC_RDSEL_SHIFT) |
1301 			   MII_TG3_AUXCTL_SHDWSEL_MISC);
1302 	if (!err)
1303 		err = tg3_readphy(tp, MII_TG3_AUX_CTRL, val);
1304 
1305 	return err;
1306 }
1307 
1308 static int tg3_phy_auxctl_write(struct tg3 *tp, int reg, u32 set)
1309 {
1310 	if (reg == MII_TG3_AUXCTL_SHDWSEL_MISC)
1311 		set |= MII_TG3_AUXCTL_MISC_WREN;
1312 
1313 	return tg3_writephy(tp, MII_TG3_AUX_CTRL, set | reg);
1314 }
1315 
1316 static int tg3_phy_toggle_auxctl_smdsp(struct tg3 *tp, bool enable)
1317 {
1318 	u32 val;
1319 	int err;
1320 
1321 	err = tg3_phy_auxctl_read(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
1322 
1323 	if (err)
1324 		return err;
1325 
1326 	if (enable)
1327 		val |= MII_TG3_AUXCTL_ACTL_SMDSP_ENA;
1328 	else
1329 		val &= ~MII_TG3_AUXCTL_ACTL_SMDSP_ENA;
1330 
1331 	err = tg3_phy_auxctl_write((tp), MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
1332 				   val | MII_TG3_AUXCTL_ACTL_TX_6DB);
1333 
1334 	return err;
1335 }
1336 
1337 static int tg3_phy_shdw_write(struct tg3 *tp, int reg, u32 val)
1338 {
1339 	return tg3_writephy(tp, MII_TG3_MISC_SHDW,
1340 			    reg | val | MII_TG3_MISC_SHDW_WREN);
1341 }
1342 
1343 static int tg3_bmcr_reset(struct tg3 *tp)
1344 {
1345 	u32 phy_control;
1346 	int limit, err;
1347 
1348 	/* OK, reset it, and poll the BMCR_RESET bit until it
1349 	 * clears or we time out.
1350 	 */
1351 	phy_control = BMCR_RESET;
1352 	err = tg3_writephy(tp, MII_BMCR, phy_control);
1353 	if (err != 0)
1354 		return -EBUSY;
1355 
1356 	limit = 5000;
1357 	while (limit--) {
1358 		err = tg3_readphy(tp, MII_BMCR, &phy_control);
1359 		if (err != 0)
1360 			return -EBUSY;
1361 
1362 		if ((phy_control & BMCR_RESET) == 0) {
1363 			udelay(40);
1364 			break;
1365 		}
1366 		udelay(10);
1367 	}
1368 	if (limit < 0)
1369 		return -EBUSY;
1370 
1371 	return 0;
1372 }
1373 
1374 static int tg3_mdio_read(struct mii_bus *bp, int mii_id, int reg)
1375 {
1376 	struct tg3 *tp = bp->priv;
1377 	u32 val;
1378 
1379 	spin_lock_bh(&tp->lock);
1380 
1381 	if (__tg3_readphy(tp, mii_id, reg, &val))
1382 		val = -EIO;
1383 
1384 	spin_unlock_bh(&tp->lock);
1385 
1386 	return val;
1387 }
1388 
1389 static int tg3_mdio_write(struct mii_bus *bp, int mii_id, int reg, u16 val)
1390 {
1391 	struct tg3 *tp = bp->priv;
1392 	u32 ret = 0;
1393 
1394 	spin_lock_bh(&tp->lock);
1395 
1396 	if (__tg3_writephy(tp, mii_id, reg, val))
1397 		ret = -EIO;
1398 
1399 	spin_unlock_bh(&tp->lock);
1400 
1401 	return ret;
1402 }
1403 
1404 static void tg3_mdio_config_5785(struct tg3 *tp)
1405 {
1406 	u32 val;
1407 	struct phy_device *phydev;
1408 
1409 	phydev = tp->mdio_bus->phy_map[tp->phy_addr];
1410 	switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1411 	case PHY_ID_BCM50610:
1412 	case PHY_ID_BCM50610M:
1413 		val = MAC_PHYCFG2_50610_LED_MODES;
1414 		break;
1415 	case PHY_ID_BCMAC131:
1416 		val = MAC_PHYCFG2_AC131_LED_MODES;
1417 		break;
1418 	case PHY_ID_RTL8211C:
1419 		val = MAC_PHYCFG2_RTL8211C_LED_MODES;
1420 		break;
1421 	case PHY_ID_RTL8201E:
1422 		val = MAC_PHYCFG2_RTL8201E_LED_MODES;
1423 		break;
1424 	default:
1425 		return;
1426 	}
1427 
1428 	if (phydev->interface != PHY_INTERFACE_MODE_RGMII) {
1429 		tw32(MAC_PHYCFG2, val);
1430 
1431 		val = tr32(MAC_PHYCFG1);
1432 		val &= ~(MAC_PHYCFG1_RGMII_INT |
1433 			 MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK);
1434 		val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT;
1435 		tw32(MAC_PHYCFG1, val);
1436 
1437 		return;
1438 	}
1439 
1440 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE))
1441 		val |= MAC_PHYCFG2_EMODE_MASK_MASK |
1442 		       MAC_PHYCFG2_FMODE_MASK_MASK |
1443 		       MAC_PHYCFG2_GMODE_MASK_MASK |
1444 		       MAC_PHYCFG2_ACT_MASK_MASK   |
1445 		       MAC_PHYCFG2_QUAL_MASK_MASK |
1446 		       MAC_PHYCFG2_INBAND_ENABLE;
1447 
1448 	tw32(MAC_PHYCFG2, val);
1449 
1450 	val = tr32(MAC_PHYCFG1);
1451 	val &= ~(MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK |
1452 		 MAC_PHYCFG1_RGMII_EXT_RX_DEC | MAC_PHYCFG1_RGMII_SND_STAT_EN);
1453 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1454 		if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1455 			val |= MAC_PHYCFG1_RGMII_EXT_RX_DEC;
1456 		if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1457 			val |= MAC_PHYCFG1_RGMII_SND_STAT_EN;
1458 	}
1459 	val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT |
1460 	       MAC_PHYCFG1_RGMII_INT | MAC_PHYCFG1_TXC_DRV;
1461 	tw32(MAC_PHYCFG1, val);
1462 
1463 	val = tr32(MAC_EXT_RGMII_MODE);
1464 	val &= ~(MAC_RGMII_MODE_RX_INT_B |
1465 		 MAC_RGMII_MODE_RX_QUALITY |
1466 		 MAC_RGMII_MODE_RX_ACTIVITY |
1467 		 MAC_RGMII_MODE_RX_ENG_DET |
1468 		 MAC_RGMII_MODE_TX_ENABLE |
1469 		 MAC_RGMII_MODE_TX_LOWPWR |
1470 		 MAC_RGMII_MODE_TX_RESET);
1471 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1472 		if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1473 			val |= MAC_RGMII_MODE_RX_INT_B |
1474 			       MAC_RGMII_MODE_RX_QUALITY |
1475 			       MAC_RGMII_MODE_RX_ACTIVITY |
1476 			       MAC_RGMII_MODE_RX_ENG_DET;
1477 		if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1478 			val |= MAC_RGMII_MODE_TX_ENABLE |
1479 			       MAC_RGMII_MODE_TX_LOWPWR |
1480 			       MAC_RGMII_MODE_TX_RESET;
1481 	}
1482 	tw32(MAC_EXT_RGMII_MODE, val);
1483 }
1484 
1485 static void tg3_mdio_start(struct tg3 *tp)
1486 {
1487 	tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
1488 	tw32_f(MAC_MI_MODE, tp->mi_mode);
1489 	udelay(80);
1490 
1491 	if (tg3_flag(tp, MDIOBUS_INITED) &&
1492 	    tg3_asic_rev(tp) == ASIC_REV_5785)
1493 		tg3_mdio_config_5785(tp);
1494 }
1495 
1496 static int tg3_mdio_init(struct tg3 *tp)
1497 {
1498 	int i;
1499 	u32 reg;
1500 	struct phy_device *phydev;
1501 
1502 	if (tg3_flag(tp, 5717_PLUS)) {
1503 		u32 is_serdes;
1504 
1505 		tp->phy_addr = tp->pci_fn + 1;
1506 
1507 		if (tg3_chip_rev_id(tp) != CHIPREV_ID_5717_A0)
1508 			is_serdes = tr32(SG_DIG_STATUS) & SG_DIG_IS_SERDES;
1509 		else
1510 			is_serdes = tr32(TG3_CPMU_PHY_STRAP) &
1511 				    TG3_CPMU_PHY_STRAP_IS_SERDES;
1512 		if (is_serdes)
1513 			tp->phy_addr += 7;
1514 	} else if (tg3_flag(tp, IS_SSB_CORE) && tg3_flag(tp, ROBOSWITCH)) {
1515 		int addr;
1516 
1517 		addr = ssb_gige_get_phyaddr(tp->pdev);
1518 		if (addr < 0)
1519 			return addr;
1520 		tp->phy_addr = addr;
1521 	} else
1522 		tp->phy_addr = TG3_PHY_MII_ADDR;
1523 
1524 	tg3_mdio_start(tp);
1525 
1526 	if (!tg3_flag(tp, USE_PHYLIB) || tg3_flag(tp, MDIOBUS_INITED))
1527 		return 0;
1528 
1529 	tp->mdio_bus = mdiobus_alloc();
1530 	if (tp->mdio_bus == NULL)
1531 		return -ENOMEM;
1532 
1533 	tp->mdio_bus->name     = "tg3 mdio bus";
1534 	snprintf(tp->mdio_bus->id, MII_BUS_ID_SIZE, "%x",
1535 		 (tp->pdev->bus->number << 8) | tp->pdev->devfn);
1536 	tp->mdio_bus->priv     = tp;
1537 	tp->mdio_bus->parent   = &tp->pdev->dev;
1538 	tp->mdio_bus->read     = &tg3_mdio_read;
1539 	tp->mdio_bus->write    = &tg3_mdio_write;
1540 	tp->mdio_bus->phy_mask = ~(1 << tp->phy_addr);
1541 	tp->mdio_bus->irq      = &tp->mdio_irq[0];
1542 
1543 	for (i = 0; i < PHY_MAX_ADDR; i++)
1544 		tp->mdio_bus->irq[i] = PHY_POLL;
1545 
1546 	/* The bus registration will look for all the PHYs on the mdio bus.
1547 	 * Unfortunately, it does not ensure the PHY is powered up before
1548 	 * accessing the PHY ID registers.  A chip reset is the
1549 	 * quickest way to bring the device back to an operational state..
1550 	 */
1551 	if (tg3_readphy(tp, MII_BMCR, &reg) || (reg & BMCR_PDOWN))
1552 		tg3_bmcr_reset(tp);
1553 
1554 	i = mdiobus_register(tp->mdio_bus);
1555 	if (i) {
1556 		dev_warn(&tp->pdev->dev, "mdiobus_reg failed (0x%x)\n", i);
1557 		mdiobus_free(tp->mdio_bus);
1558 		return i;
1559 	}
1560 
1561 	phydev = tp->mdio_bus->phy_map[tp->phy_addr];
1562 
1563 	if (!phydev || !phydev->drv) {
1564 		dev_warn(&tp->pdev->dev, "No PHY devices\n");
1565 		mdiobus_unregister(tp->mdio_bus);
1566 		mdiobus_free(tp->mdio_bus);
1567 		return -ENODEV;
1568 	}
1569 
1570 	switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1571 	case PHY_ID_BCM57780:
1572 		phydev->interface = PHY_INTERFACE_MODE_GMII;
1573 		phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1574 		break;
1575 	case PHY_ID_BCM50610:
1576 	case PHY_ID_BCM50610M:
1577 		phydev->dev_flags |= PHY_BRCM_CLEAR_RGMII_MODE |
1578 				     PHY_BRCM_RX_REFCLK_UNUSED |
1579 				     PHY_BRCM_DIS_TXCRXC_NOENRGY |
1580 				     PHY_BRCM_AUTO_PWRDWN_ENABLE;
1581 		if (tg3_flag(tp, RGMII_INBAND_DISABLE))
1582 			phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE;
1583 		if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1584 			phydev->dev_flags |= PHY_BRCM_EXT_IBND_RX_ENABLE;
1585 		if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1586 			phydev->dev_flags |= PHY_BRCM_EXT_IBND_TX_ENABLE;
1587 		/* fallthru */
1588 	case PHY_ID_RTL8211C:
1589 		phydev->interface = PHY_INTERFACE_MODE_RGMII;
1590 		break;
1591 	case PHY_ID_RTL8201E:
1592 	case PHY_ID_BCMAC131:
1593 		phydev->interface = PHY_INTERFACE_MODE_MII;
1594 		phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1595 		tp->phy_flags |= TG3_PHYFLG_IS_FET;
1596 		break;
1597 	}
1598 
1599 	tg3_flag_set(tp, MDIOBUS_INITED);
1600 
1601 	if (tg3_asic_rev(tp) == ASIC_REV_5785)
1602 		tg3_mdio_config_5785(tp);
1603 
1604 	return 0;
1605 }
1606 
1607 static void tg3_mdio_fini(struct tg3 *tp)
1608 {
1609 	if (tg3_flag(tp, MDIOBUS_INITED)) {
1610 		tg3_flag_clear(tp, MDIOBUS_INITED);
1611 		mdiobus_unregister(tp->mdio_bus);
1612 		mdiobus_free(tp->mdio_bus);
1613 	}
1614 }
1615 
1616 /* tp->lock is held. */
1617 static inline void tg3_generate_fw_event(struct tg3 *tp)
1618 {
1619 	u32 val;
1620 
1621 	val = tr32(GRC_RX_CPU_EVENT);
1622 	val |= GRC_RX_CPU_DRIVER_EVENT;
1623 	tw32_f(GRC_RX_CPU_EVENT, val);
1624 
1625 	tp->last_event_jiffies = jiffies;
1626 }
1627 
1628 #define TG3_FW_EVENT_TIMEOUT_USEC 2500
1629 
1630 /* tp->lock is held. */
1631 static void tg3_wait_for_event_ack(struct tg3 *tp)
1632 {
1633 	int i;
1634 	unsigned int delay_cnt;
1635 	long time_remain;
1636 
1637 	/* If enough time has passed, no wait is necessary. */
1638 	time_remain = (long)(tp->last_event_jiffies + 1 +
1639 		      usecs_to_jiffies(TG3_FW_EVENT_TIMEOUT_USEC)) -
1640 		      (long)jiffies;
1641 	if (time_remain < 0)
1642 		return;
1643 
1644 	/* Check if we can shorten the wait time. */
1645 	delay_cnt = jiffies_to_usecs(time_remain);
1646 	if (delay_cnt > TG3_FW_EVENT_TIMEOUT_USEC)
1647 		delay_cnt = TG3_FW_EVENT_TIMEOUT_USEC;
1648 	delay_cnt = (delay_cnt >> 3) + 1;
1649 
1650 	for (i = 0; i < delay_cnt; i++) {
1651 		if (!(tr32(GRC_RX_CPU_EVENT) & GRC_RX_CPU_DRIVER_EVENT))
1652 			break;
1653 		if (pci_channel_offline(tp->pdev))
1654 			break;
1655 
1656 		udelay(8);
1657 	}
1658 }
1659 
1660 /* tp->lock is held. */
1661 static void tg3_phy_gather_ump_data(struct tg3 *tp, u32 *data)
1662 {
1663 	u32 reg, val;
1664 
1665 	val = 0;
1666 	if (!tg3_readphy(tp, MII_BMCR, &reg))
1667 		val = reg << 16;
1668 	if (!tg3_readphy(tp, MII_BMSR, &reg))
1669 		val |= (reg & 0xffff);
1670 	*data++ = val;
1671 
1672 	val = 0;
1673 	if (!tg3_readphy(tp, MII_ADVERTISE, &reg))
1674 		val = reg << 16;
1675 	if (!tg3_readphy(tp, MII_LPA, &reg))
1676 		val |= (reg & 0xffff);
1677 	*data++ = val;
1678 
1679 	val = 0;
1680 	if (!(tp->phy_flags & TG3_PHYFLG_MII_SERDES)) {
1681 		if (!tg3_readphy(tp, MII_CTRL1000, &reg))
1682 			val = reg << 16;
1683 		if (!tg3_readphy(tp, MII_STAT1000, &reg))
1684 			val |= (reg & 0xffff);
1685 	}
1686 	*data++ = val;
1687 
1688 	if (!tg3_readphy(tp, MII_PHYADDR, &reg))
1689 		val = reg << 16;
1690 	else
1691 		val = 0;
1692 	*data++ = val;
1693 }
1694 
1695 /* tp->lock is held. */
1696 static void tg3_ump_link_report(struct tg3 *tp)
1697 {
1698 	u32 data[4];
1699 
1700 	if (!tg3_flag(tp, 5780_CLASS) || !tg3_flag(tp, ENABLE_ASF))
1701 		return;
1702 
1703 	tg3_phy_gather_ump_data(tp, data);
1704 
1705 	tg3_wait_for_event_ack(tp);
1706 
1707 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_LINK_UPDATE);
1708 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 14);
1709 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x0, data[0]);
1710 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x4, data[1]);
1711 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x8, data[2]);
1712 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0xc, data[3]);
1713 
1714 	tg3_generate_fw_event(tp);
1715 }
1716 
1717 /* tp->lock is held. */
1718 static void tg3_stop_fw(struct tg3 *tp)
1719 {
1720 	if (tg3_flag(tp, ENABLE_ASF) && !tg3_flag(tp, ENABLE_APE)) {
1721 		/* Wait for RX cpu to ACK the previous event. */
1722 		tg3_wait_for_event_ack(tp);
1723 
1724 		tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
1725 
1726 		tg3_generate_fw_event(tp);
1727 
1728 		/* Wait for RX cpu to ACK this event. */
1729 		tg3_wait_for_event_ack(tp);
1730 	}
1731 }
1732 
1733 /* tp->lock is held. */
1734 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
1735 {
1736 	tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
1737 		      NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
1738 
1739 	if (tg3_flag(tp, ASF_NEW_HANDSHAKE)) {
1740 		switch (kind) {
1741 		case RESET_KIND_INIT:
1742 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1743 				      DRV_STATE_START);
1744 			break;
1745 
1746 		case RESET_KIND_SHUTDOWN:
1747 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1748 				      DRV_STATE_UNLOAD);
1749 			break;
1750 
1751 		case RESET_KIND_SUSPEND:
1752 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1753 				      DRV_STATE_SUSPEND);
1754 			break;
1755 
1756 		default:
1757 			break;
1758 		}
1759 	}
1760 }
1761 
1762 /* tp->lock is held. */
1763 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
1764 {
1765 	if (tg3_flag(tp, ASF_NEW_HANDSHAKE)) {
1766 		switch (kind) {
1767 		case RESET_KIND_INIT:
1768 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1769 				      DRV_STATE_START_DONE);
1770 			break;
1771 
1772 		case RESET_KIND_SHUTDOWN:
1773 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1774 				      DRV_STATE_UNLOAD_DONE);
1775 			break;
1776 
1777 		default:
1778 			break;
1779 		}
1780 	}
1781 }
1782 
1783 /* tp->lock is held. */
1784 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
1785 {
1786 	if (tg3_flag(tp, ENABLE_ASF)) {
1787 		switch (kind) {
1788 		case RESET_KIND_INIT:
1789 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1790 				      DRV_STATE_START);
1791 			break;
1792 
1793 		case RESET_KIND_SHUTDOWN:
1794 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1795 				      DRV_STATE_UNLOAD);
1796 			break;
1797 
1798 		case RESET_KIND_SUSPEND:
1799 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1800 				      DRV_STATE_SUSPEND);
1801 			break;
1802 
1803 		default:
1804 			break;
1805 		}
1806 	}
1807 }
1808 
1809 static int tg3_poll_fw(struct tg3 *tp)
1810 {
1811 	int i;
1812 	u32 val;
1813 
1814 	if (tg3_flag(tp, NO_FWARE_REPORTED))
1815 		return 0;
1816 
1817 	if (tg3_flag(tp, IS_SSB_CORE)) {
1818 		/* We don't use firmware. */
1819 		return 0;
1820 	}
1821 
1822 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
1823 		/* Wait up to 20ms for init done. */
1824 		for (i = 0; i < 200; i++) {
1825 			if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
1826 				return 0;
1827 			if (pci_channel_offline(tp->pdev))
1828 				return -ENODEV;
1829 
1830 			udelay(100);
1831 		}
1832 		return -ENODEV;
1833 	}
1834 
1835 	/* Wait for firmware initialization to complete. */
1836 	for (i = 0; i < 100000; i++) {
1837 		tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
1838 		if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1839 			break;
1840 		if (pci_channel_offline(tp->pdev)) {
1841 			if (!tg3_flag(tp, NO_FWARE_REPORTED)) {
1842 				tg3_flag_set(tp, NO_FWARE_REPORTED);
1843 				netdev_info(tp->dev, "No firmware running\n");
1844 			}
1845 
1846 			break;
1847 		}
1848 
1849 		udelay(10);
1850 	}
1851 
1852 	/* Chip might not be fitted with firmware.  Some Sun onboard
1853 	 * parts are configured like that.  So don't signal the timeout
1854 	 * of the above loop as an error, but do report the lack of
1855 	 * running firmware once.
1856 	 */
1857 	if (i >= 100000 && !tg3_flag(tp, NO_FWARE_REPORTED)) {
1858 		tg3_flag_set(tp, NO_FWARE_REPORTED);
1859 
1860 		netdev_info(tp->dev, "No firmware running\n");
1861 	}
1862 
1863 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0) {
1864 		/* The 57765 A0 needs a little more
1865 		 * time to do some important work.
1866 		 */
1867 		mdelay(10);
1868 	}
1869 
1870 	return 0;
1871 }
1872 
1873 static void tg3_link_report(struct tg3 *tp)
1874 {
1875 	if (!netif_carrier_ok(tp->dev)) {
1876 		netif_info(tp, link, tp->dev, "Link is down\n");
1877 		tg3_ump_link_report(tp);
1878 	} else if (netif_msg_link(tp)) {
1879 		netdev_info(tp->dev, "Link is up at %d Mbps, %s duplex\n",
1880 			    (tp->link_config.active_speed == SPEED_1000 ?
1881 			     1000 :
1882 			     (tp->link_config.active_speed == SPEED_100 ?
1883 			      100 : 10)),
1884 			    (tp->link_config.active_duplex == DUPLEX_FULL ?
1885 			     "full" : "half"));
1886 
1887 		netdev_info(tp->dev, "Flow control is %s for TX and %s for RX\n",
1888 			    (tp->link_config.active_flowctrl & FLOW_CTRL_TX) ?
1889 			    "on" : "off",
1890 			    (tp->link_config.active_flowctrl & FLOW_CTRL_RX) ?
1891 			    "on" : "off");
1892 
1893 		if (tp->phy_flags & TG3_PHYFLG_EEE_CAP)
1894 			netdev_info(tp->dev, "EEE is %s\n",
1895 				    tp->setlpicnt ? "enabled" : "disabled");
1896 
1897 		tg3_ump_link_report(tp);
1898 	}
1899 
1900 	tp->link_up = netif_carrier_ok(tp->dev);
1901 }
1902 
1903 static u32 tg3_decode_flowctrl_1000T(u32 adv)
1904 {
1905 	u32 flowctrl = 0;
1906 
1907 	if (adv & ADVERTISE_PAUSE_CAP) {
1908 		flowctrl |= FLOW_CTRL_RX;
1909 		if (!(adv & ADVERTISE_PAUSE_ASYM))
1910 			flowctrl |= FLOW_CTRL_TX;
1911 	} else if (adv & ADVERTISE_PAUSE_ASYM)
1912 		flowctrl |= FLOW_CTRL_TX;
1913 
1914 	return flowctrl;
1915 }
1916 
1917 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1918 {
1919 	u16 miireg;
1920 
1921 	if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1922 		miireg = ADVERTISE_1000XPAUSE;
1923 	else if (flow_ctrl & FLOW_CTRL_TX)
1924 		miireg = ADVERTISE_1000XPSE_ASYM;
1925 	else if (flow_ctrl & FLOW_CTRL_RX)
1926 		miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1927 	else
1928 		miireg = 0;
1929 
1930 	return miireg;
1931 }
1932 
1933 static u32 tg3_decode_flowctrl_1000X(u32 adv)
1934 {
1935 	u32 flowctrl = 0;
1936 
1937 	if (adv & ADVERTISE_1000XPAUSE) {
1938 		flowctrl |= FLOW_CTRL_RX;
1939 		if (!(adv & ADVERTISE_1000XPSE_ASYM))
1940 			flowctrl |= FLOW_CTRL_TX;
1941 	} else if (adv & ADVERTISE_1000XPSE_ASYM)
1942 		flowctrl |= FLOW_CTRL_TX;
1943 
1944 	return flowctrl;
1945 }
1946 
1947 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1948 {
1949 	u8 cap = 0;
1950 
1951 	if (lcladv & rmtadv & ADVERTISE_1000XPAUSE) {
1952 		cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1953 	} else if (lcladv & rmtadv & ADVERTISE_1000XPSE_ASYM) {
1954 		if (lcladv & ADVERTISE_1000XPAUSE)
1955 			cap = FLOW_CTRL_RX;
1956 		if (rmtadv & ADVERTISE_1000XPAUSE)
1957 			cap = FLOW_CTRL_TX;
1958 	}
1959 
1960 	return cap;
1961 }
1962 
1963 static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv)
1964 {
1965 	u8 autoneg;
1966 	u8 flowctrl = 0;
1967 	u32 old_rx_mode = tp->rx_mode;
1968 	u32 old_tx_mode = tp->tx_mode;
1969 
1970 	if (tg3_flag(tp, USE_PHYLIB))
1971 		autoneg = tp->mdio_bus->phy_map[tp->phy_addr]->autoneg;
1972 	else
1973 		autoneg = tp->link_config.autoneg;
1974 
1975 	if (autoneg == AUTONEG_ENABLE && tg3_flag(tp, PAUSE_AUTONEG)) {
1976 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
1977 			flowctrl = tg3_resolve_flowctrl_1000X(lcladv, rmtadv);
1978 		else
1979 			flowctrl = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
1980 	} else
1981 		flowctrl = tp->link_config.flowctrl;
1982 
1983 	tp->link_config.active_flowctrl = flowctrl;
1984 
1985 	if (flowctrl & FLOW_CTRL_RX)
1986 		tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1987 	else
1988 		tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1989 
1990 	if (old_rx_mode != tp->rx_mode)
1991 		tw32_f(MAC_RX_MODE, tp->rx_mode);
1992 
1993 	if (flowctrl & FLOW_CTRL_TX)
1994 		tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1995 	else
1996 		tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1997 
1998 	if (old_tx_mode != tp->tx_mode)
1999 		tw32_f(MAC_TX_MODE, tp->tx_mode);
2000 }
2001 
2002 static void tg3_adjust_link(struct net_device *dev)
2003 {
2004 	u8 oldflowctrl, linkmesg = 0;
2005 	u32 mac_mode, lcl_adv, rmt_adv;
2006 	struct tg3 *tp = netdev_priv(dev);
2007 	struct phy_device *phydev = tp->mdio_bus->phy_map[tp->phy_addr];
2008 
2009 	spin_lock_bh(&tp->lock);
2010 
2011 	mac_mode = tp->mac_mode & ~(MAC_MODE_PORT_MODE_MASK |
2012 				    MAC_MODE_HALF_DUPLEX);
2013 
2014 	oldflowctrl = tp->link_config.active_flowctrl;
2015 
2016 	if (phydev->link) {
2017 		lcl_adv = 0;
2018 		rmt_adv = 0;
2019 
2020 		if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10)
2021 			mac_mode |= MAC_MODE_PORT_MODE_MII;
2022 		else if (phydev->speed == SPEED_1000 ||
2023 			 tg3_asic_rev(tp) != ASIC_REV_5785)
2024 			mac_mode |= MAC_MODE_PORT_MODE_GMII;
2025 		else
2026 			mac_mode |= MAC_MODE_PORT_MODE_MII;
2027 
2028 		if (phydev->duplex == DUPLEX_HALF)
2029 			mac_mode |= MAC_MODE_HALF_DUPLEX;
2030 		else {
2031 			lcl_adv = mii_advertise_flowctrl(
2032 				  tp->link_config.flowctrl);
2033 
2034 			if (phydev->pause)
2035 				rmt_adv = LPA_PAUSE_CAP;
2036 			if (phydev->asym_pause)
2037 				rmt_adv |= LPA_PAUSE_ASYM;
2038 		}
2039 
2040 		tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
2041 	} else
2042 		mac_mode |= MAC_MODE_PORT_MODE_GMII;
2043 
2044 	if (mac_mode != tp->mac_mode) {
2045 		tp->mac_mode = mac_mode;
2046 		tw32_f(MAC_MODE, tp->mac_mode);
2047 		udelay(40);
2048 	}
2049 
2050 	if (tg3_asic_rev(tp) == ASIC_REV_5785) {
2051 		if (phydev->speed == SPEED_10)
2052 			tw32(MAC_MI_STAT,
2053 			     MAC_MI_STAT_10MBPS_MODE |
2054 			     MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
2055 		else
2056 			tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
2057 	}
2058 
2059 	if (phydev->speed == SPEED_1000 && phydev->duplex == DUPLEX_HALF)
2060 		tw32(MAC_TX_LENGTHS,
2061 		     ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2062 		      (6 << TX_LENGTHS_IPG_SHIFT) |
2063 		      (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
2064 	else
2065 		tw32(MAC_TX_LENGTHS,
2066 		     ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2067 		      (6 << TX_LENGTHS_IPG_SHIFT) |
2068 		      (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
2069 
2070 	if (phydev->link != tp->old_link ||
2071 	    phydev->speed != tp->link_config.active_speed ||
2072 	    phydev->duplex != tp->link_config.active_duplex ||
2073 	    oldflowctrl != tp->link_config.active_flowctrl)
2074 		linkmesg = 1;
2075 
2076 	tp->old_link = phydev->link;
2077 	tp->link_config.active_speed = phydev->speed;
2078 	tp->link_config.active_duplex = phydev->duplex;
2079 
2080 	spin_unlock_bh(&tp->lock);
2081 
2082 	if (linkmesg)
2083 		tg3_link_report(tp);
2084 }
2085 
2086 static int tg3_phy_init(struct tg3 *tp)
2087 {
2088 	struct phy_device *phydev;
2089 
2090 	if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED)
2091 		return 0;
2092 
2093 	/* Bring the PHY back to a known state. */
2094 	tg3_bmcr_reset(tp);
2095 
2096 	phydev = tp->mdio_bus->phy_map[tp->phy_addr];
2097 
2098 	/* Attach the MAC to the PHY. */
2099 	phydev = phy_connect(tp->dev, dev_name(&phydev->dev),
2100 			     tg3_adjust_link, phydev->interface);
2101 	if (IS_ERR(phydev)) {
2102 		dev_err(&tp->pdev->dev, "Could not attach to PHY\n");
2103 		return PTR_ERR(phydev);
2104 	}
2105 
2106 	/* Mask with MAC supported features. */
2107 	switch (phydev->interface) {
2108 	case PHY_INTERFACE_MODE_GMII:
2109 	case PHY_INTERFACE_MODE_RGMII:
2110 		if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
2111 			phydev->supported &= (PHY_GBIT_FEATURES |
2112 					      SUPPORTED_Pause |
2113 					      SUPPORTED_Asym_Pause);
2114 			break;
2115 		}
2116 		/* fallthru */
2117 	case PHY_INTERFACE_MODE_MII:
2118 		phydev->supported &= (PHY_BASIC_FEATURES |
2119 				      SUPPORTED_Pause |
2120 				      SUPPORTED_Asym_Pause);
2121 		break;
2122 	default:
2123 		phy_disconnect(tp->mdio_bus->phy_map[tp->phy_addr]);
2124 		return -EINVAL;
2125 	}
2126 
2127 	tp->phy_flags |= TG3_PHYFLG_IS_CONNECTED;
2128 
2129 	phydev->advertising = phydev->supported;
2130 
2131 	return 0;
2132 }
2133 
2134 static void tg3_phy_start(struct tg3 *tp)
2135 {
2136 	struct phy_device *phydev;
2137 
2138 	if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
2139 		return;
2140 
2141 	phydev = tp->mdio_bus->phy_map[tp->phy_addr];
2142 
2143 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
2144 		tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
2145 		phydev->speed = tp->link_config.speed;
2146 		phydev->duplex = tp->link_config.duplex;
2147 		phydev->autoneg = tp->link_config.autoneg;
2148 		phydev->advertising = tp->link_config.advertising;
2149 	}
2150 
2151 	phy_start(phydev);
2152 
2153 	phy_start_aneg(phydev);
2154 }
2155 
2156 static void tg3_phy_stop(struct tg3 *tp)
2157 {
2158 	if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
2159 		return;
2160 
2161 	phy_stop(tp->mdio_bus->phy_map[tp->phy_addr]);
2162 }
2163 
2164 static void tg3_phy_fini(struct tg3 *tp)
2165 {
2166 	if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
2167 		phy_disconnect(tp->mdio_bus->phy_map[tp->phy_addr]);
2168 		tp->phy_flags &= ~TG3_PHYFLG_IS_CONNECTED;
2169 	}
2170 }
2171 
2172 static int tg3_phy_set_extloopbk(struct tg3 *tp)
2173 {
2174 	int err;
2175 	u32 val;
2176 
2177 	if (tp->phy_flags & TG3_PHYFLG_IS_FET)
2178 		return 0;
2179 
2180 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
2181 		/* Cannot do read-modify-write on 5401 */
2182 		err = tg3_phy_auxctl_write(tp,
2183 					   MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
2184 					   MII_TG3_AUXCTL_ACTL_EXTLOOPBK |
2185 					   0x4c20);
2186 		goto done;
2187 	}
2188 
2189 	err = tg3_phy_auxctl_read(tp,
2190 				  MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
2191 	if (err)
2192 		return err;
2193 
2194 	val |= MII_TG3_AUXCTL_ACTL_EXTLOOPBK;
2195 	err = tg3_phy_auxctl_write(tp,
2196 				   MII_TG3_AUXCTL_SHDWSEL_AUXCTL, val);
2197 
2198 done:
2199 	return err;
2200 }
2201 
2202 static void tg3_phy_fet_toggle_apd(struct tg3 *tp, bool enable)
2203 {
2204 	u32 phytest;
2205 
2206 	if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
2207 		u32 phy;
2208 
2209 		tg3_writephy(tp, MII_TG3_FET_TEST,
2210 			     phytest | MII_TG3_FET_SHADOW_EN);
2211 		if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXSTAT2, &phy)) {
2212 			if (enable)
2213 				phy |= MII_TG3_FET_SHDW_AUXSTAT2_APD;
2214 			else
2215 				phy &= ~MII_TG3_FET_SHDW_AUXSTAT2_APD;
2216 			tg3_writephy(tp, MII_TG3_FET_SHDW_AUXSTAT2, phy);
2217 		}
2218 		tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
2219 	}
2220 }
2221 
2222 static void tg3_phy_toggle_apd(struct tg3 *tp, bool enable)
2223 {
2224 	u32 reg;
2225 
2226 	if (!tg3_flag(tp, 5705_PLUS) ||
2227 	    (tg3_flag(tp, 5717_PLUS) &&
2228 	     (tp->phy_flags & TG3_PHYFLG_MII_SERDES)))
2229 		return;
2230 
2231 	if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2232 		tg3_phy_fet_toggle_apd(tp, enable);
2233 		return;
2234 	}
2235 
2236 	reg = MII_TG3_MISC_SHDW_SCR5_LPED |
2237 	      MII_TG3_MISC_SHDW_SCR5_DLPTLM |
2238 	      MII_TG3_MISC_SHDW_SCR5_SDTL |
2239 	      MII_TG3_MISC_SHDW_SCR5_C125OE;
2240 	if (tg3_asic_rev(tp) != ASIC_REV_5784 || !enable)
2241 		reg |= MII_TG3_MISC_SHDW_SCR5_DLLAPD;
2242 
2243 	tg3_phy_shdw_write(tp, MII_TG3_MISC_SHDW_SCR5_SEL, reg);
2244 
2245 
2246 	reg = MII_TG3_MISC_SHDW_APD_WKTM_84MS;
2247 	if (enable)
2248 		reg |= MII_TG3_MISC_SHDW_APD_ENABLE;
2249 
2250 	tg3_phy_shdw_write(tp, MII_TG3_MISC_SHDW_APD_SEL, reg);
2251 }
2252 
2253 static void tg3_phy_toggle_automdix(struct tg3 *tp, bool enable)
2254 {
2255 	u32 phy;
2256 
2257 	if (!tg3_flag(tp, 5705_PLUS) ||
2258 	    (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
2259 		return;
2260 
2261 	if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2262 		u32 ephy;
2263 
2264 		if (!tg3_readphy(tp, MII_TG3_FET_TEST, &ephy)) {
2265 			u32 reg = MII_TG3_FET_SHDW_MISCCTRL;
2266 
2267 			tg3_writephy(tp, MII_TG3_FET_TEST,
2268 				     ephy | MII_TG3_FET_SHADOW_EN);
2269 			if (!tg3_readphy(tp, reg, &phy)) {
2270 				if (enable)
2271 					phy |= MII_TG3_FET_SHDW_MISCCTRL_MDIX;
2272 				else
2273 					phy &= ~MII_TG3_FET_SHDW_MISCCTRL_MDIX;
2274 				tg3_writephy(tp, reg, phy);
2275 			}
2276 			tg3_writephy(tp, MII_TG3_FET_TEST, ephy);
2277 		}
2278 	} else {
2279 		int ret;
2280 
2281 		ret = tg3_phy_auxctl_read(tp,
2282 					  MII_TG3_AUXCTL_SHDWSEL_MISC, &phy);
2283 		if (!ret) {
2284 			if (enable)
2285 				phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
2286 			else
2287 				phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
2288 			tg3_phy_auxctl_write(tp,
2289 					     MII_TG3_AUXCTL_SHDWSEL_MISC, phy);
2290 		}
2291 	}
2292 }
2293 
2294 static void tg3_phy_set_wirespeed(struct tg3 *tp)
2295 {
2296 	int ret;
2297 	u32 val;
2298 
2299 	if (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED)
2300 		return;
2301 
2302 	ret = tg3_phy_auxctl_read(tp, MII_TG3_AUXCTL_SHDWSEL_MISC, &val);
2303 	if (!ret)
2304 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_MISC,
2305 				     val | MII_TG3_AUXCTL_MISC_WIRESPD_EN);
2306 }
2307 
2308 static void tg3_phy_apply_otp(struct tg3 *tp)
2309 {
2310 	u32 otp, phy;
2311 
2312 	if (!tp->phy_otp)
2313 		return;
2314 
2315 	otp = tp->phy_otp;
2316 
2317 	if (tg3_phy_toggle_auxctl_smdsp(tp, true))
2318 		return;
2319 
2320 	phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
2321 	phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
2322 	tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
2323 
2324 	phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
2325 	      ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
2326 	tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
2327 
2328 	phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
2329 	phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
2330 	tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
2331 
2332 	phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
2333 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
2334 
2335 	phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
2336 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
2337 
2338 	phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
2339 	      ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
2340 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
2341 
2342 	tg3_phy_toggle_auxctl_smdsp(tp, false);
2343 }
2344 
2345 static void tg3_eee_pull_config(struct tg3 *tp, struct ethtool_eee *eee)
2346 {
2347 	u32 val;
2348 	struct ethtool_eee *dest = &tp->eee;
2349 
2350 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
2351 		return;
2352 
2353 	if (eee)
2354 		dest = eee;
2355 
2356 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, TG3_CL45_D7_EEERES_STAT, &val))
2357 		return;
2358 
2359 	/* Pull eee_active */
2360 	if (val == TG3_CL45_D7_EEERES_STAT_LP_1000T ||
2361 	    val == TG3_CL45_D7_EEERES_STAT_LP_100TX) {
2362 		dest->eee_active = 1;
2363 	} else
2364 		dest->eee_active = 0;
2365 
2366 	/* Pull lp advertised settings */
2367 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, MDIO_AN_EEE_LPABLE, &val))
2368 		return;
2369 	dest->lp_advertised = mmd_eee_adv_to_ethtool_adv_t(val);
2370 
2371 	/* Pull advertised and eee_enabled settings */
2372 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, &val))
2373 		return;
2374 	dest->eee_enabled = !!val;
2375 	dest->advertised = mmd_eee_adv_to_ethtool_adv_t(val);
2376 
2377 	/* Pull tx_lpi_enabled */
2378 	val = tr32(TG3_CPMU_EEE_MODE);
2379 	dest->tx_lpi_enabled = !!(val & TG3_CPMU_EEEMD_LPI_IN_TX);
2380 
2381 	/* Pull lpi timer value */
2382 	dest->tx_lpi_timer = tr32(TG3_CPMU_EEE_DBTMR1) & 0xffff;
2383 }
2384 
2385 static void tg3_phy_eee_adjust(struct tg3 *tp, bool current_link_up)
2386 {
2387 	u32 val;
2388 
2389 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
2390 		return;
2391 
2392 	tp->setlpicnt = 0;
2393 
2394 	if (tp->link_config.autoneg == AUTONEG_ENABLE &&
2395 	    current_link_up &&
2396 	    tp->link_config.active_duplex == DUPLEX_FULL &&
2397 	    (tp->link_config.active_speed == SPEED_100 ||
2398 	     tp->link_config.active_speed == SPEED_1000)) {
2399 		u32 eeectl;
2400 
2401 		if (tp->link_config.active_speed == SPEED_1000)
2402 			eeectl = TG3_CPMU_EEE_CTRL_EXIT_16_5_US;
2403 		else
2404 			eeectl = TG3_CPMU_EEE_CTRL_EXIT_36_US;
2405 
2406 		tw32(TG3_CPMU_EEE_CTRL, eeectl);
2407 
2408 		tg3_eee_pull_config(tp, NULL);
2409 		if (tp->eee.eee_active)
2410 			tp->setlpicnt = 2;
2411 	}
2412 
2413 	if (!tp->setlpicnt) {
2414 		if (current_link_up &&
2415 		   !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2416 			tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, 0x0000);
2417 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2418 		}
2419 
2420 		val = tr32(TG3_CPMU_EEE_MODE);
2421 		tw32(TG3_CPMU_EEE_MODE, val & ~TG3_CPMU_EEEMD_LPI_ENABLE);
2422 	}
2423 }
2424 
2425 static void tg3_phy_eee_enable(struct tg3 *tp)
2426 {
2427 	u32 val;
2428 
2429 	if (tp->link_config.active_speed == SPEED_1000 &&
2430 	    (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2431 	     tg3_asic_rev(tp) == ASIC_REV_5719 ||
2432 	     tg3_flag(tp, 57765_CLASS)) &&
2433 	    !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2434 		val = MII_TG3_DSP_TAP26_ALNOKO |
2435 		      MII_TG3_DSP_TAP26_RMRXSTO;
2436 		tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
2437 		tg3_phy_toggle_auxctl_smdsp(tp, false);
2438 	}
2439 
2440 	val = tr32(TG3_CPMU_EEE_MODE);
2441 	tw32(TG3_CPMU_EEE_MODE, val | TG3_CPMU_EEEMD_LPI_ENABLE);
2442 }
2443 
2444 static int tg3_wait_macro_done(struct tg3 *tp)
2445 {
2446 	int limit = 100;
2447 
2448 	while (limit--) {
2449 		u32 tmp32;
2450 
2451 		if (!tg3_readphy(tp, MII_TG3_DSP_CONTROL, &tmp32)) {
2452 			if ((tmp32 & 0x1000) == 0)
2453 				break;
2454 		}
2455 	}
2456 	if (limit < 0)
2457 		return -EBUSY;
2458 
2459 	return 0;
2460 }
2461 
2462 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
2463 {
2464 	static const u32 test_pat[4][6] = {
2465 	{ 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
2466 	{ 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
2467 	{ 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
2468 	{ 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
2469 	};
2470 	int chan;
2471 
2472 	for (chan = 0; chan < 4; chan++) {
2473 		int i;
2474 
2475 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2476 			     (chan * 0x2000) | 0x0200);
2477 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
2478 
2479 		for (i = 0; i < 6; i++)
2480 			tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
2481 				     test_pat[chan][i]);
2482 
2483 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
2484 		if (tg3_wait_macro_done(tp)) {
2485 			*resetp = 1;
2486 			return -EBUSY;
2487 		}
2488 
2489 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2490 			     (chan * 0x2000) | 0x0200);
2491 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0082);
2492 		if (tg3_wait_macro_done(tp)) {
2493 			*resetp = 1;
2494 			return -EBUSY;
2495 		}
2496 
2497 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0802);
2498 		if (tg3_wait_macro_done(tp)) {
2499 			*resetp = 1;
2500 			return -EBUSY;
2501 		}
2502 
2503 		for (i = 0; i < 6; i += 2) {
2504 			u32 low, high;
2505 
2506 			if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
2507 			    tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
2508 			    tg3_wait_macro_done(tp)) {
2509 				*resetp = 1;
2510 				return -EBUSY;
2511 			}
2512 			low &= 0x7fff;
2513 			high &= 0x000f;
2514 			if (low != test_pat[chan][i] ||
2515 			    high != test_pat[chan][i+1]) {
2516 				tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
2517 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
2518 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
2519 
2520 				return -EBUSY;
2521 			}
2522 		}
2523 	}
2524 
2525 	return 0;
2526 }
2527 
2528 static int tg3_phy_reset_chanpat(struct tg3 *tp)
2529 {
2530 	int chan;
2531 
2532 	for (chan = 0; chan < 4; chan++) {
2533 		int i;
2534 
2535 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2536 			     (chan * 0x2000) | 0x0200);
2537 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
2538 		for (i = 0; i < 6; i++)
2539 			tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
2540 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
2541 		if (tg3_wait_macro_done(tp))
2542 			return -EBUSY;
2543 	}
2544 
2545 	return 0;
2546 }
2547 
2548 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
2549 {
2550 	u32 reg32, phy9_orig;
2551 	int retries, do_phy_reset, err;
2552 
2553 	retries = 10;
2554 	do_phy_reset = 1;
2555 	do {
2556 		if (do_phy_reset) {
2557 			err = tg3_bmcr_reset(tp);
2558 			if (err)
2559 				return err;
2560 			do_phy_reset = 0;
2561 		}
2562 
2563 		/* Disable transmitter and interrupt.  */
2564 		if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
2565 			continue;
2566 
2567 		reg32 |= 0x3000;
2568 		tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2569 
2570 		/* Set full-duplex, 1000 mbps.  */
2571 		tg3_writephy(tp, MII_BMCR,
2572 			     BMCR_FULLDPLX | BMCR_SPEED1000);
2573 
2574 		/* Set to master mode.  */
2575 		if (tg3_readphy(tp, MII_CTRL1000, &phy9_orig))
2576 			continue;
2577 
2578 		tg3_writephy(tp, MII_CTRL1000,
2579 			     CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER);
2580 
2581 		err = tg3_phy_toggle_auxctl_smdsp(tp, true);
2582 		if (err)
2583 			return err;
2584 
2585 		/* Block the PHY control access.  */
2586 		tg3_phydsp_write(tp, 0x8005, 0x0800);
2587 
2588 		err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
2589 		if (!err)
2590 			break;
2591 	} while (--retries);
2592 
2593 	err = tg3_phy_reset_chanpat(tp);
2594 	if (err)
2595 		return err;
2596 
2597 	tg3_phydsp_write(tp, 0x8005, 0x0000);
2598 
2599 	tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
2600 	tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0000);
2601 
2602 	tg3_phy_toggle_auxctl_smdsp(tp, false);
2603 
2604 	tg3_writephy(tp, MII_CTRL1000, phy9_orig);
2605 
2606 	err = tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32);
2607 	if (err)
2608 		return err;
2609 
2610 	reg32 &= ~0x3000;
2611 	tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2612 
2613 	return 0;
2614 }
2615 
2616 static void tg3_carrier_off(struct tg3 *tp)
2617 {
2618 	netif_carrier_off(tp->dev);
2619 	tp->link_up = false;
2620 }
2621 
2622 static void tg3_warn_mgmt_link_flap(struct tg3 *tp)
2623 {
2624 	if (tg3_flag(tp, ENABLE_ASF))
2625 		netdev_warn(tp->dev,
2626 			    "Management side-band traffic will be interrupted during phy settings change\n");
2627 }
2628 
2629 /* This will reset the tigon3 PHY if there is no valid
2630  * link unless the FORCE argument is non-zero.
2631  */
2632 static int tg3_phy_reset(struct tg3 *tp)
2633 {
2634 	u32 val, cpmuctrl;
2635 	int err;
2636 
2637 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
2638 		val = tr32(GRC_MISC_CFG);
2639 		tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
2640 		udelay(40);
2641 	}
2642 	err  = tg3_readphy(tp, MII_BMSR, &val);
2643 	err |= tg3_readphy(tp, MII_BMSR, &val);
2644 	if (err != 0)
2645 		return -EBUSY;
2646 
2647 	if (netif_running(tp->dev) && tp->link_up) {
2648 		netif_carrier_off(tp->dev);
2649 		tg3_link_report(tp);
2650 	}
2651 
2652 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
2653 	    tg3_asic_rev(tp) == ASIC_REV_5704 ||
2654 	    tg3_asic_rev(tp) == ASIC_REV_5705) {
2655 		err = tg3_phy_reset_5703_4_5(tp);
2656 		if (err)
2657 			return err;
2658 		goto out;
2659 	}
2660 
2661 	cpmuctrl = 0;
2662 	if (tg3_asic_rev(tp) == ASIC_REV_5784 &&
2663 	    tg3_chip_rev(tp) != CHIPREV_5784_AX) {
2664 		cpmuctrl = tr32(TG3_CPMU_CTRL);
2665 		if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
2666 			tw32(TG3_CPMU_CTRL,
2667 			     cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
2668 	}
2669 
2670 	err = tg3_bmcr_reset(tp);
2671 	if (err)
2672 		return err;
2673 
2674 	if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
2675 		val = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
2676 		tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, val);
2677 
2678 		tw32(TG3_CPMU_CTRL, cpmuctrl);
2679 	}
2680 
2681 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX ||
2682 	    tg3_chip_rev(tp) == CHIPREV_5761_AX) {
2683 		val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2684 		if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
2685 		    CPMU_LSPD_1000MB_MACCLK_12_5) {
2686 			val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2687 			udelay(40);
2688 			tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2689 		}
2690 	}
2691 
2692 	if (tg3_flag(tp, 5717_PLUS) &&
2693 	    (tp->phy_flags & TG3_PHYFLG_MII_SERDES))
2694 		return 0;
2695 
2696 	tg3_phy_apply_otp(tp);
2697 
2698 	if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
2699 		tg3_phy_toggle_apd(tp, true);
2700 	else
2701 		tg3_phy_toggle_apd(tp, false);
2702 
2703 out:
2704 	if ((tp->phy_flags & TG3_PHYFLG_ADC_BUG) &&
2705 	    !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2706 		tg3_phydsp_write(tp, 0x201f, 0x2aaa);
2707 		tg3_phydsp_write(tp, 0x000a, 0x0323);
2708 		tg3_phy_toggle_auxctl_smdsp(tp, false);
2709 	}
2710 
2711 	if (tp->phy_flags & TG3_PHYFLG_5704_A0_BUG) {
2712 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2713 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2714 	}
2715 
2716 	if (tp->phy_flags & TG3_PHYFLG_BER_BUG) {
2717 		if (!tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2718 			tg3_phydsp_write(tp, 0x000a, 0x310b);
2719 			tg3_phydsp_write(tp, 0x201f, 0x9506);
2720 			tg3_phydsp_write(tp, 0x401f, 0x14e2);
2721 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2722 		}
2723 	} else if (tp->phy_flags & TG3_PHYFLG_JITTER_BUG) {
2724 		if (!tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2725 			tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
2726 			if (tp->phy_flags & TG3_PHYFLG_ADJUST_TRIM) {
2727 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
2728 				tg3_writephy(tp, MII_TG3_TEST1,
2729 					     MII_TG3_TEST1_TRIM_EN | 0x4);
2730 			} else
2731 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
2732 
2733 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2734 		}
2735 	}
2736 
2737 	/* Set Extended packet length bit (bit 14) on all chips that */
2738 	/* support jumbo frames */
2739 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
2740 		/* Cannot do read-modify-write on 5401 */
2741 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
2742 	} else if (tg3_flag(tp, JUMBO_CAPABLE)) {
2743 		/* Set bit 14 with read-modify-write to preserve other bits */
2744 		err = tg3_phy_auxctl_read(tp,
2745 					  MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
2746 		if (!err)
2747 			tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
2748 					   val | MII_TG3_AUXCTL_ACTL_EXTPKTLEN);
2749 	}
2750 
2751 	/* Set phy register 0x10 bit 0 to high fifo elasticity to support
2752 	 * jumbo frames transmission.
2753 	 */
2754 	if (tg3_flag(tp, JUMBO_CAPABLE)) {
2755 		if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &val))
2756 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
2757 				     val | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
2758 	}
2759 
2760 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
2761 		/* adjust output voltage */
2762 		tg3_writephy(tp, MII_TG3_FET_PTEST, 0x12);
2763 	}
2764 
2765 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5762_A0)
2766 		tg3_phydsp_write(tp, 0xffb, 0x4000);
2767 
2768 	tg3_phy_toggle_automdix(tp, true);
2769 	tg3_phy_set_wirespeed(tp);
2770 	return 0;
2771 }
2772 
2773 #define TG3_GPIO_MSG_DRVR_PRES		 0x00000001
2774 #define TG3_GPIO_MSG_NEED_VAUX		 0x00000002
2775 #define TG3_GPIO_MSG_MASK		 (TG3_GPIO_MSG_DRVR_PRES | \
2776 					  TG3_GPIO_MSG_NEED_VAUX)
2777 #define TG3_GPIO_MSG_ALL_DRVR_PRES_MASK \
2778 	((TG3_GPIO_MSG_DRVR_PRES << 0) | \
2779 	 (TG3_GPIO_MSG_DRVR_PRES << 4) | \
2780 	 (TG3_GPIO_MSG_DRVR_PRES << 8) | \
2781 	 (TG3_GPIO_MSG_DRVR_PRES << 12))
2782 
2783 #define TG3_GPIO_MSG_ALL_NEED_VAUX_MASK \
2784 	((TG3_GPIO_MSG_NEED_VAUX << 0) | \
2785 	 (TG3_GPIO_MSG_NEED_VAUX << 4) | \
2786 	 (TG3_GPIO_MSG_NEED_VAUX << 8) | \
2787 	 (TG3_GPIO_MSG_NEED_VAUX << 12))
2788 
2789 static inline u32 tg3_set_function_status(struct tg3 *tp, u32 newstat)
2790 {
2791 	u32 status, shift;
2792 
2793 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2794 	    tg3_asic_rev(tp) == ASIC_REV_5719)
2795 		status = tg3_ape_read32(tp, TG3_APE_GPIO_MSG);
2796 	else
2797 		status = tr32(TG3_CPMU_DRV_STATUS);
2798 
2799 	shift = TG3_APE_GPIO_MSG_SHIFT + 4 * tp->pci_fn;
2800 	status &= ~(TG3_GPIO_MSG_MASK << shift);
2801 	status |= (newstat << shift);
2802 
2803 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2804 	    tg3_asic_rev(tp) == ASIC_REV_5719)
2805 		tg3_ape_write32(tp, TG3_APE_GPIO_MSG, status);
2806 	else
2807 		tw32(TG3_CPMU_DRV_STATUS, status);
2808 
2809 	return status >> TG3_APE_GPIO_MSG_SHIFT;
2810 }
2811 
2812 static inline int tg3_pwrsrc_switch_to_vmain(struct tg3 *tp)
2813 {
2814 	if (!tg3_flag(tp, IS_NIC))
2815 		return 0;
2816 
2817 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2818 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
2819 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
2820 		if (tg3_ape_lock(tp, TG3_APE_LOCK_GPIO))
2821 			return -EIO;
2822 
2823 		tg3_set_function_status(tp, TG3_GPIO_MSG_DRVR_PRES);
2824 
2825 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl,
2826 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2827 
2828 		tg3_ape_unlock(tp, TG3_APE_LOCK_GPIO);
2829 	} else {
2830 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl,
2831 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2832 	}
2833 
2834 	return 0;
2835 }
2836 
2837 static void tg3_pwrsrc_die_with_vmain(struct tg3 *tp)
2838 {
2839 	u32 grc_local_ctrl;
2840 
2841 	if (!tg3_flag(tp, IS_NIC) ||
2842 	    tg3_asic_rev(tp) == ASIC_REV_5700 ||
2843 	    tg3_asic_rev(tp) == ASIC_REV_5701)
2844 		return;
2845 
2846 	grc_local_ctrl = tp->grc_local_ctrl | GRC_LCLCTRL_GPIO_OE1;
2847 
2848 	tw32_wait_f(GRC_LOCAL_CTRL,
2849 		    grc_local_ctrl | GRC_LCLCTRL_GPIO_OUTPUT1,
2850 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2851 
2852 	tw32_wait_f(GRC_LOCAL_CTRL,
2853 		    grc_local_ctrl,
2854 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2855 
2856 	tw32_wait_f(GRC_LOCAL_CTRL,
2857 		    grc_local_ctrl | GRC_LCLCTRL_GPIO_OUTPUT1,
2858 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2859 }
2860 
2861 static void tg3_pwrsrc_switch_to_vaux(struct tg3 *tp)
2862 {
2863 	if (!tg3_flag(tp, IS_NIC))
2864 		return;
2865 
2866 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
2867 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
2868 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2869 			    (GRC_LCLCTRL_GPIO_OE0 |
2870 			     GRC_LCLCTRL_GPIO_OE1 |
2871 			     GRC_LCLCTRL_GPIO_OE2 |
2872 			     GRC_LCLCTRL_GPIO_OUTPUT0 |
2873 			     GRC_LCLCTRL_GPIO_OUTPUT1),
2874 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2875 	} else if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
2876 		   tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
2877 		/* The 5761 non-e device swaps GPIO 0 and GPIO 2. */
2878 		u32 grc_local_ctrl = GRC_LCLCTRL_GPIO_OE0 |
2879 				     GRC_LCLCTRL_GPIO_OE1 |
2880 				     GRC_LCLCTRL_GPIO_OE2 |
2881 				     GRC_LCLCTRL_GPIO_OUTPUT0 |
2882 				     GRC_LCLCTRL_GPIO_OUTPUT1 |
2883 				     tp->grc_local_ctrl;
2884 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2885 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2886 
2887 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT2;
2888 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2889 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2890 
2891 		grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT0;
2892 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2893 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2894 	} else {
2895 		u32 no_gpio2;
2896 		u32 grc_local_ctrl = 0;
2897 
2898 		/* Workaround to prevent overdrawing Amps. */
2899 		if (tg3_asic_rev(tp) == ASIC_REV_5714) {
2900 			grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
2901 			tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2902 				    grc_local_ctrl,
2903 				    TG3_GRC_LCLCTL_PWRSW_DELAY);
2904 		}
2905 
2906 		/* On 5753 and variants, GPIO2 cannot be used. */
2907 		no_gpio2 = tp->nic_sram_data_cfg &
2908 			   NIC_SRAM_DATA_CFG_NO_GPIO2;
2909 
2910 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
2911 				  GRC_LCLCTRL_GPIO_OE1 |
2912 				  GRC_LCLCTRL_GPIO_OE2 |
2913 				  GRC_LCLCTRL_GPIO_OUTPUT1 |
2914 				  GRC_LCLCTRL_GPIO_OUTPUT2;
2915 		if (no_gpio2) {
2916 			grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
2917 					    GRC_LCLCTRL_GPIO_OUTPUT2);
2918 		}
2919 		tw32_wait_f(GRC_LOCAL_CTRL,
2920 			    tp->grc_local_ctrl | grc_local_ctrl,
2921 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2922 
2923 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
2924 
2925 		tw32_wait_f(GRC_LOCAL_CTRL,
2926 			    tp->grc_local_ctrl | grc_local_ctrl,
2927 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2928 
2929 		if (!no_gpio2) {
2930 			grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
2931 			tw32_wait_f(GRC_LOCAL_CTRL,
2932 				    tp->grc_local_ctrl | grc_local_ctrl,
2933 				    TG3_GRC_LCLCTL_PWRSW_DELAY);
2934 		}
2935 	}
2936 }
2937 
2938 static void tg3_frob_aux_power_5717(struct tg3 *tp, bool wol_enable)
2939 {
2940 	u32 msg = 0;
2941 
2942 	/* Serialize power state transitions */
2943 	if (tg3_ape_lock(tp, TG3_APE_LOCK_GPIO))
2944 		return;
2945 
2946 	if (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE) || wol_enable)
2947 		msg = TG3_GPIO_MSG_NEED_VAUX;
2948 
2949 	msg = tg3_set_function_status(tp, msg);
2950 
2951 	if (msg & TG3_GPIO_MSG_ALL_DRVR_PRES_MASK)
2952 		goto done;
2953 
2954 	if (msg & TG3_GPIO_MSG_ALL_NEED_VAUX_MASK)
2955 		tg3_pwrsrc_switch_to_vaux(tp);
2956 	else
2957 		tg3_pwrsrc_die_with_vmain(tp);
2958 
2959 done:
2960 	tg3_ape_unlock(tp, TG3_APE_LOCK_GPIO);
2961 }
2962 
2963 static void tg3_frob_aux_power(struct tg3 *tp, bool include_wol)
2964 {
2965 	bool need_vaux = false;
2966 
2967 	/* The GPIOs do something completely different on 57765. */
2968 	if (!tg3_flag(tp, IS_NIC) || tg3_flag(tp, 57765_CLASS))
2969 		return;
2970 
2971 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2972 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
2973 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
2974 		tg3_frob_aux_power_5717(tp, include_wol ?
2975 					tg3_flag(tp, WOL_ENABLE) != 0 : 0);
2976 		return;
2977 	}
2978 
2979 	if (tp->pdev_peer && tp->pdev_peer != tp->pdev) {
2980 		struct net_device *dev_peer;
2981 
2982 		dev_peer = pci_get_drvdata(tp->pdev_peer);
2983 
2984 		/* remove_one() may have been run on the peer. */
2985 		if (dev_peer) {
2986 			struct tg3 *tp_peer = netdev_priv(dev_peer);
2987 
2988 			if (tg3_flag(tp_peer, INIT_COMPLETE))
2989 				return;
2990 
2991 			if ((include_wol && tg3_flag(tp_peer, WOL_ENABLE)) ||
2992 			    tg3_flag(tp_peer, ENABLE_ASF))
2993 				need_vaux = true;
2994 		}
2995 	}
2996 
2997 	if ((include_wol && tg3_flag(tp, WOL_ENABLE)) ||
2998 	    tg3_flag(tp, ENABLE_ASF))
2999 		need_vaux = true;
3000 
3001 	if (need_vaux)
3002 		tg3_pwrsrc_switch_to_vaux(tp);
3003 	else
3004 		tg3_pwrsrc_die_with_vmain(tp);
3005 }
3006 
3007 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
3008 {
3009 	if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
3010 		return 1;
3011 	else if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411) {
3012 		if (speed != SPEED_10)
3013 			return 1;
3014 	} else if (speed == SPEED_10)
3015 		return 1;
3016 
3017 	return 0;
3018 }
3019 
3020 static bool tg3_phy_power_bug(struct tg3 *tp)
3021 {
3022 	switch (tg3_asic_rev(tp)) {
3023 	case ASIC_REV_5700:
3024 	case ASIC_REV_5704:
3025 		return true;
3026 	case ASIC_REV_5780:
3027 		if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
3028 			return true;
3029 		return false;
3030 	case ASIC_REV_5717:
3031 		if (!tp->pci_fn)
3032 			return true;
3033 		return false;
3034 	case ASIC_REV_5719:
3035 	case ASIC_REV_5720:
3036 		if ((tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
3037 		    !tp->pci_fn)
3038 			return true;
3039 		return false;
3040 	}
3041 
3042 	return false;
3043 }
3044 
3045 static bool tg3_phy_led_bug(struct tg3 *tp)
3046 {
3047 	switch (tg3_asic_rev(tp)) {
3048 	case ASIC_REV_5719:
3049 	case ASIC_REV_5720:
3050 		if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
3051 		    !tp->pci_fn)
3052 			return true;
3053 		return false;
3054 	}
3055 
3056 	return false;
3057 }
3058 
3059 static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power)
3060 {
3061 	u32 val;
3062 
3063 	if (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)
3064 		return;
3065 
3066 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
3067 		if (tg3_asic_rev(tp) == ASIC_REV_5704) {
3068 			u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
3069 			u32 serdes_cfg = tr32(MAC_SERDES_CFG);
3070 
3071 			sg_dig_ctrl |=
3072 				SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
3073 			tw32(SG_DIG_CTRL, sg_dig_ctrl);
3074 			tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
3075 		}
3076 		return;
3077 	}
3078 
3079 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
3080 		tg3_bmcr_reset(tp);
3081 		val = tr32(GRC_MISC_CFG);
3082 		tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
3083 		udelay(40);
3084 		return;
3085 	} else if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
3086 		u32 phytest;
3087 		if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
3088 			u32 phy;
3089 
3090 			tg3_writephy(tp, MII_ADVERTISE, 0);
3091 			tg3_writephy(tp, MII_BMCR,
3092 				     BMCR_ANENABLE | BMCR_ANRESTART);
3093 
3094 			tg3_writephy(tp, MII_TG3_FET_TEST,
3095 				     phytest | MII_TG3_FET_SHADOW_EN);
3096 			if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXMODE4, &phy)) {
3097 				phy |= MII_TG3_FET_SHDW_AUXMODE4_SBPD;
3098 				tg3_writephy(tp,
3099 					     MII_TG3_FET_SHDW_AUXMODE4,
3100 					     phy);
3101 			}
3102 			tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
3103 		}
3104 		return;
3105 	} else if (do_low_power) {
3106 		if (!tg3_phy_led_bug(tp))
3107 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
3108 				     MII_TG3_EXT_CTRL_FORCE_LED_OFF);
3109 
3110 		val = MII_TG3_AUXCTL_PCTL_100TX_LPWR |
3111 		      MII_TG3_AUXCTL_PCTL_SPR_ISOLATE |
3112 		      MII_TG3_AUXCTL_PCTL_VREG_11V;
3113 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, val);
3114 	}
3115 
3116 	/* The PHY should not be powered down on some chips because
3117 	 * of bugs.
3118 	 */
3119 	if (tg3_phy_power_bug(tp))
3120 		return;
3121 
3122 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX ||
3123 	    tg3_chip_rev(tp) == CHIPREV_5761_AX) {
3124 		val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
3125 		val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
3126 		val |= CPMU_LSPD_1000MB_MACCLK_12_5;
3127 		tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
3128 	}
3129 
3130 	tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
3131 }
3132 
3133 /* tp->lock is held. */
3134 static int tg3_nvram_lock(struct tg3 *tp)
3135 {
3136 	if (tg3_flag(tp, NVRAM)) {
3137 		int i;
3138 
3139 		if (tp->nvram_lock_cnt == 0) {
3140 			tw32(NVRAM_SWARB, SWARB_REQ_SET1);
3141 			for (i = 0; i < 8000; i++) {
3142 				if (tr32(NVRAM_SWARB) & SWARB_GNT1)
3143 					break;
3144 				udelay(20);
3145 			}
3146 			if (i == 8000) {
3147 				tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
3148 				return -ENODEV;
3149 			}
3150 		}
3151 		tp->nvram_lock_cnt++;
3152 	}
3153 	return 0;
3154 }
3155 
3156 /* tp->lock is held. */
3157 static void tg3_nvram_unlock(struct tg3 *tp)
3158 {
3159 	if (tg3_flag(tp, NVRAM)) {
3160 		if (tp->nvram_lock_cnt > 0)
3161 			tp->nvram_lock_cnt--;
3162 		if (tp->nvram_lock_cnt == 0)
3163 			tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
3164 	}
3165 }
3166 
3167 /* tp->lock is held. */
3168 static void tg3_enable_nvram_access(struct tg3 *tp)
3169 {
3170 	if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
3171 		u32 nvaccess = tr32(NVRAM_ACCESS);
3172 
3173 		tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
3174 	}
3175 }
3176 
3177 /* tp->lock is held. */
3178 static void tg3_disable_nvram_access(struct tg3 *tp)
3179 {
3180 	if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
3181 		u32 nvaccess = tr32(NVRAM_ACCESS);
3182 
3183 		tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
3184 	}
3185 }
3186 
3187 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
3188 					u32 offset, u32 *val)
3189 {
3190 	u32 tmp;
3191 	int i;
3192 
3193 	if (offset > EEPROM_ADDR_ADDR_MASK || (offset % 4) != 0)
3194 		return -EINVAL;
3195 
3196 	tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
3197 					EEPROM_ADDR_DEVID_MASK |
3198 					EEPROM_ADDR_READ);
3199 	tw32(GRC_EEPROM_ADDR,
3200 	     tmp |
3201 	     (0 << EEPROM_ADDR_DEVID_SHIFT) |
3202 	     ((offset << EEPROM_ADDR_ADDR_SHIFT) &
3203 	      EEPROM_ADDR_ADDR_MASK) |
3204 	     EEPROM_ADDR_READ | EEPROM_ADDR_START);
3205 
3206 	for (i = 0; i < 1000; i++) {
3207 		tmp = tr32(GRC_EEPROM_ADDR);
3208 
3209 		if (tmp & EEPROM_ADDR_COMPLETE)
3210 			break;
3211 		msleep(1);
3212 	}
3213 	if (!(tmp & EEPROM_ADDR_COMPLETE))
3214 		return -EBUSY;
3215 
3216 	tmp = tr32(GRC_EEPROM_DATA);
3217 
3218 	/*
3219 	 * The data will always be opposite the native endian
3220 	 * format.  Perform a blind byteswap to compensate.
3221 	 */
3222 	*val = swab32(tmp);
3223 
3224 	return 0;
3225 }
3226 
3227 #define NVRAM_CMD_TIMEOUT 5000
3228 
3229 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
3230 {
3231 	int i;
3232 
3233 	tw32(NVRAM_CMD, nvram_cmd);
3234 	for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
3235 		usleep_range(10, 40);
3236 		if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
3237 			udelay(10);
3238 			break;
3239 		}
3240 	}
3241 
3242 	if (i == NVRAM_CMD_TIMEOUT)
3243 		return -EBUSY;
3244 
3245 	return 0;
3246 }
3247 
3248 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
3249 {
3250 	if (tg3_flag(tp, NVRAM) &&
3251 	    tg3_flag(tp, NVRAM_BUFFERED) &&
3252 	    tg3_flag(tp, FLASH) &&
3253 	    !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
3254 	    (tp->nvram_jedecnum == JEDEC_ATMEL))
3255 
3256 		addr = ((addr / tp->nvram_pagesize) <<
3257 			ATMEL_AT45DB0X1B_PAGE_POS) +
3258 		       (addr % tp->nvram_pagesize);
3259 
3260 	return addr;
3261 }
3262 
3263 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
3264 {
3265 	if (tg3_flag(tp, NVRAM) &&
3266 	    tg3_flag(tp, NVRAM_BUFFERED) &&
3267 	    tg3_flag(tp, FLASH) &&
3268 	    !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
3269 	    (tp->nvram_jedecnum == JEDEC_ATMEL))
3270 
3271 		addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
3272 			tp->nvram_pagesize) +
3273 		       (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
3274 
3275 	return addr;
3276 }
3277 
3278 /* NOTE: Data read in from NVRAM is byteswapped according to
3279  * the byteswapping settings for all other register accesses.
3280  * tg3 devices are BE devices, so on a BE machine, the data
3281  * returned will be exactly as it is seen in NVRAM.  On a LE
3282  * machine, the 32-bit value will be byteswapped.
3283  */
3284 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
3285 {
3286 	int ret;
3287 
3288 	if (!tg3_flag(tp, NVRAM))
3289 		return tg3_nvram_read_using_eeprom(tp, offset, val);
3290 
3291 	offset = tg3_nvram_phys_addr(tp, offset);
3292 
3293 	if (offset > NVRAM_ADDR_MSK)
3294 		return -EINVAL;
3295 
3296 	ret = tg3_nvram_lock(tp);
3297 	if (ret)
3298 		return ret;
3299 
3300 	tg3_enable_nvram_access(tp);
3301 
3302 	tw32(NVRAM_ADDR, offset);
3303 	ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
3304 		NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
3305 
3306 	if (ret == 0)
3307 		*val = tr32(NVRAM_RDDATA);
3308 
3309 	tg3_disable_nvram_access(tp);
3310 
3311 	tg3_nvram_unlock(tp);
3312 
3313 	return ret;
3314 }
3315 
3316 /* Ensures NVRAM data is in bytestream format. */
3317 static int tg3_nvram_read_be32(struct tg3 *tp, u32 offset, __be32 *val)
3318 {
3319 	u32 v;
3320 	int res = tg3_nvram_read(tp, offset, &v);
3321 	if (!res)
3322 		*val = cpu_to_be32(v);
3323 	return res;
3324 }
3325 
3326 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
3327 				    u32 offset, u32 len, u8 *buf)
3328 {
3329 	int i, j, rc = 0;
3330 	u32 val;
3331 
3332 	for (i = 0; i < len; i += 4) {
3333 		u32 addr;
3334 		__be32 data;
3335 
3336 		addr = offset + i;
3337 
3338 		memcpy(&data, buf + i, 4);
3339 
3340 		/*
3341 		 * The SEEPROM interface expects the data to always be opposite
3342 		 * the native endian format.  We accomplish this by reversing
3343 		 * all the operations that would have been performed on the
3344 		 * data from a call to tg3_nvram_read_be32().
3345 		 */
3346 		tw32(GRC_EEPROM_DATA, swab32(be32_to_cpu(data)));
3347 
3348 		val = tr32(GRC_EEPROM_ADDR);
3349 		tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
3350 
3351 		val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
3352 			EEPROM_ADDR_READ);
3353 		tw32(GRC_EEPROM_ADDR, val |
3354 			(0 << EEPROM_ADDR_DEVID_SHIFT) |
3355 			(addr & EEPROM_ADDR_ADDR_MASK) |
3356 			EEPROM_ADDR_START |
3357 			EEPROM_ADDR_WRITE);
3358 
3359 		for (j = 0; j < 1000; j++) {
3360 			val = tr32(GRC_EEPROM_ADDR);
3361 
3362 			if (val & EEPROM_ADDR_COMPLETE)
3363 				break;
3364 			msleep(1);
3365 		}
3366 		if (!(val & EEPROM_ADDR_COMPLETE)) {
3367 			rc = -EBUSY;
3368 			break;
3369 		}
3370 	}
3371 
3372 	return rc;
3373 }
3374 
3375 /* offset and length are dword aligned */
3376 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
3377 		u8 *buf)
3378 {
3379 	int ret = 0;
3380 	u32 pagesize = tp->nvram_pagesize;
3381 	u32 pagemask = pagesize - 1;
3382 	u32 nvram_cmd;
3383 	u8 *tmp;
3384 
3385 	tmp = kmalloc(pagesize, GFP_KERNEL);
3386 	if (tmp == NULL)
3387 		return -ENOMEM;
3388 
3389 	while (len) {
3390 		int j;
3391 		u32 phy_addr, page_off, size;
3392 
3393 		phy_addr = offset & ~pagemask;
3394 
3395 		for (j = 0; j < pagesize; j += 4) {
3396 			ret = tg3_nvram_read_be32(tp, phy_addr + j,
3397 						  (__be32 *) (tmp + j));
3398 			if (ret)
3399 				break;
3400 		}
3401 		if (ret)
3402 			break;
3403 
3404 		page_off = offset & pagemask;
3405 		size = pagesize;
3406 		if (len < size)
3407 			size = len;
3408 
3409 		len -= size;
3410 
3411 		memcpy(tmp + page_off, buf, size);
3412 
3413 		offset = offset + (pagesize - page_off);
3414 
3415 		tg3_enable_nvram_access(tp);
3416 
3417 		/*
3418 		 * Before we can erase the flash page, we need
3419 		 * to issue a special "write enable" command.
3420 		 */
3421 		nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3422 
3423 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3424 			break;
3425 
3426 		/* Erase the target page */
3427 		tw32(NVRAM_ADDR, phy_addr);
3428 
3429 		nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
3430 			NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
3431 
3432 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3433 			break;
3434 
3435 		/* Issue another write enable to start the write. */
3436 		nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3437 
3438 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3439 			break;
3440 
3441 		for (j = 0; j < pagesize; j += 4) {
3442 			__be32 data;
3443 
3444 			data = *((__be32 *) (tmp + j));
3445 
3446 			tw32(NVRAM_WRDATA, be32_to_cpu(data));
3447 
3448 			tw32(NVRAM_ADDR, phy_addr + j);
3449 
3450 			nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
3451 				NVRAM_CMD_WR;
3452 
3453 			if (j == 0)
3454 				nvram_cmd |= NVRAM_CMD_FIRST;
3455 			else if (j == (pagesize - 4))
3456 				nvram_cmd |= NVRAM_CMD_LAST;
3457 
3458 			ret = tg3_nvram_exec_cmd(tp, nvram_cmd);
3459 			if (ret)
3460 				break;
3461 		}
3462 		if (ret)
3463 			break;
3464 	}
3465 
3466 	nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3467 	tg3_nvram_exec_cmd(tp, nvram_cmd);
3468 
3469 	kfree(tmp);
3470 
3471 	return ret;
3472 }
3473 
3474 /* offset and length are dword aligned */
3475 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
3476 		u8 *buf)
3477 {
3478 	int i, ret = 0;
3479 
3480 	for (i = 0; i < len; i += 4, offset += 4) {
3481 		u32 page_off, phy_addr, nvram_cmd;
3482 		__be32 data;
3483 
3484 		memcpy(&data, buf + i, 4);
3485 		tw32(NVRAM_WRDATA, be32_to_cpu(data));
3486 
3487 		page_off = offset % tp->nvram_pagesize;
3488 
3489 		phy_addr = tg3_nvram_phys_addr(tp, offset);
3490 
3491 		nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
3492 
3493 		if (page_off == 0 || i == 0)
3494 			nvram_cmd |= NVRAM_CMD_FIRST;
3495 		if (page_off == (tp->nvram_pagesize - 4))
3496 			nvram_cmd |= NVRAM_CMD_LAST;
3497 
3498 		if (i == (len - 4))
3499 			nvram_cmd |= NVRAM_CMD_LAST;
3500 
3501 		if ((nvram_cmd & NVRAM_CMD_FIRST) ||
3502 		    !tg3_flag(tp, FLASH) ||
3503 		    !tg3_flag(tp, 57765_PLUS))
3504 			tw32(NVRAM_ADDR, phy_addr);
3505 
3506 		if (tg3_asic_rev(tp) != ASIC_REV_5752 &&
3507 		    !tg3_flag(tp, 5755_PLUS) &&
3508 		    (tp->nvram_jedecnum == JEDEC_ST) &&
3509 		    (nvram_cmd & NVRAM_CMD_FIRST)) {
3510 			u32 cmd;
3511 
3512 			cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3513 			ret = tg3_nvram_exec_cmd(tp, cmd);
3514 			if (ret)
3515 				break;
3516 		}
3517 		if (!tg3_flag(tp, FLASH)) {
3518 			/* We always do complete word writes to eeprom. */
3519 			nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
3520 		}
3521 
3522 		ret = tg3_nvram_exec_cmd(tp, nvram_cmd);
3523 		if (ret)
3524 			break;
3525 	}
3526 	return ret;
3527 }
3528 
3529 /* offset and length are dword aligned */
3530 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
3531 {
3532 	int ret;
3533 
3534 	if (tg3_flag(tp, EEPROM_WRITE_PROT)) {
3535 		tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
3536 		       ~GRC_LCLCTRL_GPIO_OUTPUT1);
3537 		udelay(40);
3538 	}
3539 
3540 	if (!tg3_flag(tp, NVRAM)) {
3541 		ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
3542 	} else {
3543 		u32 grc_mode;
3544 
3545 		ret = tg3_nvram_lock(tp);
3546 		if (ret)
3547 			return ret;
3548 
3549 		tg3_enable_nvram_access(tp);
3550 		if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM))
3551 			tw32(NVRAM_WRITE1, 0x406);
3552 
3553 		grc_mode = tr32(GRC_MODE);
3554 		tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
3555 
3556 		if (tg3_flag(tp, NVRAM_BUFFERED) || !tg3_flag(tp, FLASH)) {
3557 			ret = tg3_nvram_write_block_buffered(tp, offset, len,
3558 				buf);
3559 		} else {
3560 			ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
3561 				buf);
3562 		}
3563 
3564 		grc_mode = tr32(GRC_MODE);
3565 		tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
3566 
3567 		tg3_disable_nvram_access(tp);
3568 		tg3_nvram_unlock(tp);
3569 	}
3570 
3571 	if (tg3_flag(tp, EEPROM_WRITE_PROT)) {
3572 		tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
3573 		udelay(40);
3574 	}
3575 
3576 	return ret;
3577 }
3578 
3579 #define RX_CPU_SCRATCH_BASE	0x30000
3580 #define RX_CPU_SCRATCH_SIZE	0x04000
3581 #define TX_CPU_SCRATCH_BASE	0x34000
3582 #define TX_CPU_SCRATCH_SIZE	0x04000
3583 
3584 /* tp->lock is held. */
3585 static int tg3_pause_cpu(struct tg3 *tp, u32 cpu_base)
3586 {
3587 	int i;
3588 	const int iters = 10000;
3589 
3590 	for (i = 0; i < iters; i++) {
3591 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3592 		tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
3593 		if (tr32(cpu_base + CPU_MODE) & CPU_MODE_HALT)
3594 			break;
3595 		if (pci_channel_offline(tp->pdev))
3596 			return -EBUSY;
3597 	}
3598 
3599 	return (i == iters) ? -EBUSY : 0;
3600 }
3601 
3602 /* tp->lock is held. */
3603 static int tg3_rxcpu_pause(struct tg3 *tp)
3604 {
3605 	int rc = tg3_pause_cpu(tp, RX_CPU_BASE);
3606 
3607 	tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
3608 	tw32_f(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
3609 	udelay(10);
3610 
3611 	return rc;
3612 }
3613 
3614 /* tp->lock is held. */
3615 static int tg3_txcpu_pause(struct tg3 *tp)
3616 {
3617 	return tg3_pause_cpu(tp, TX_CPU_BASE);
3618 }
3619 
3620 /* tp->lock is held. */
3621 static void tg3_resume_cpu(struct tg3 *tp, u32 cpu_base)
3622 {
3623 	tw32(cpu_base + CPU_STATE, 0xffffffff);
3624 	tw32_f(cpu_base + CPU_MODE,  0x00000000);
3625 }
3626 
3627 /* tp->lock is held. */
3628 static void tg3_rxcpu_resume(struct tg3 *tp)
3629 {
3630 	tg3_resume_cpu(tp, RX_CPU_BASE);
3631 }
3632 
3633 /* tp->lock is held. */
3634 static int tg3_halt_cpu(struct tg3 *tp, u32 cpu_base)
3635 {
3636 	int rc;
3637 
3638 	BUG_ON(cpu_base == TX_CPU_BASE && tg3_flag(tp, 5705_PLUS));
3639 
3640 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
3641 		u32 val = tr32(GRC_VCPU_EXT_CTRL);
3642 
3643 		tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
3644 		return 0;
3645 	}
3646 	if (cpu_base == RX_CPU_BASE) {
3647 		rc = tg3_rxcpu_pause(tp);
3648 	} else {
3649 		/*
3650 		 * There is only an Rx CPU for the 5750 derivative in the
3651 		 * BCM4785.
3652 		 */
3653 		if (tg3_flag(tp, IS_SSB_CORE))
3654 			return 0;
3655 
3656 		rc = tg3_txcpu_pause(tp);
3657 	}
3658 
3659 	if (rc) {
3660 		netdev_err(tp->dev, "%s timed out, %s CPU\n",
3661 			   __func__, cpu_base == RX_CPU_BASE ? "RX" : "TX");
3662 		return -ENODEV;
3663 	}
3664 
3665 	/* Clear firmware's nvram arbitration. */
3666 	if (tg3_flag(tp, NVRAM))
3667 		tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
3668 	return 0;
3669 }
3670 
3671 static int tg3_fw_data_len(struct tg3 *tp,
3672 			   const struct tg3_firmware_hdr *fw_hdr)
3673 {
3674 	int fw_len;
3675 
3676 	/* Non fragmented firmware have one firmware header followed by a
3677 	 * contiguous chunk of data to be written. The length field in that
3678 	 * header is not the length of data to be written but the complete
3679 	 * length of the bss. The data length is determined based on
3680 	 * tp->fw->size minus headers.
3681 	 *
3682 	 * Fragmented firmware have a main header followed by multiple
3683 	 * fragments. Each fragment is identical to non fragmented firmware
3684 	 * with a firmware header followed by a contiguous chunk of data. In
3685 	 * the main header, the length field is unused and set to 0xffffffff.
3686 	 * In each fragment header the length is the entire size of that
3687 	 * fragment i.e. fragment data + header length. Data length is
3688 	 * therefore length field in the header minus TG3_FW_HDR_LEN.
3689 	 */
3690 	if (tp->fw_len == 0xffffffff)
3691 		fw_len = be32_to_cpu(fw_hdr->len);
3692 	else
3693 		fw_len = tp->fw->size;
3694 
3695 	return (fw_len - TG3_FW_HDR_LEN) / sizeof(u32);
3696 }
3697 
3698 /* tp->lock is held. */
3699 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base,
3700 				 u32 cpu_scratch_base, int cpu_scratch_size,
3701 				 const struct tg3_firmware_hdr *fw_hdr)
3702 {
3703 	int err, i;
3704 	void (*write_op)(struct tg3 *, u32, u32);
3705 	int total_len = tp->fw->size;
3706 
3707 	if (cpu_base == TX_CPU_BASE && tg3_flag(tp, 5705_PLUS)) {
3708 		netdev_err(tp->dev,
3709 			   "%s: Trying to load TX cpu firmware which is 5705\n",
3710 			   __func__);
3711 		return -EINVAL;
3712 	}
3713 
3714 	if (tg3_flag(tp, 5705_PLUS) && tg3_asic_rev(tp) != ASIC_REV_57766)
3715 		write_op = tg3_write_mem;
3716 	else
3717 		write_op = tg3_write_indirect_reg32;
3718 
3719 	if (tg3_asic_rev(tp) != ASIC_REV_57766) {
3720 		/* It is possible that bootcode is still loading at this point.
3721 		 * Get the nvram lock first before halting the cpu.
3722 		 */
3723 		int lock_err = tg3_nvram_lock(tp);
3724 		err = tg3_halt_cpu(tp, cpu_base);
3725 		if (!lock_err)
3726 			tg3_nvram_unlock(tp);
3727 		if (err)
3728 			goto out;
3729 
3730 		for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
3731 			write_op(tp, cpu_scratch_base + i, 0);
3732 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3733 		tw32(cpu_base + CPU_MODE,
3734 		     tr32(cpu_base + CPU_MODE) | CPU_MODE_HALT);
3735 	} else {
3736 		/* Subtract additional main header for fragmented firmware and
3737 		 * advance to the first fragment
3738 		 */
3739 		total_len -= TG3_FW_HDR_LEN;
3740 		fw_hdr++;
3741 	}
3742 
3743 	do {
3744 		u32 *fw_data = (u32 *)(fw_hdr + 1);
3745 		for (i = 0; i < tg3_fw_data_len(tp, fw_hdr); i++)
3746 			write_op(tp, cpu_scratch_base +
3747 				     (be32_to_cpu(fw_hdr->base_addr) & 0xffff) +
3748 				     (i * sizeof(u32)),
3749 				 be32_to_cpu(fw_data[i]));
3750 
3751 		total_len -= be32_to_cpu(fw_hdr->len);
3752 
3753 		/* Advance to next fragment */
3754 		fw_hdr = (struct tg3_firmware_hdr *)
3755 			 ((void *)fw_hdr + be32_to_cpu(fw_hdr->len));
3756 	} while (total_len > 0);
3757 
3758 	err = 0;
3759 
3760 out:
3761 	return err;
3762 }
3763 
3764 /* tp->lock is held. */
3765 static int tg3_pause_cpu_and_set_pc(struct tg3 *tp, u32 cpu_base, u32 pc)
3766 {
3767 	int i;
3768 	const int iters = 5;
3769 
3770 	tw32(cpu_base + CPU_STATE, 0xffffffff);
3771 	tw32_f(cpu_base + CPU_PC, pc);
3772 
3773 	for (i = 0; i < iters; i++) {
3774 		if (tr32(cpu_base + CPU_PC) == pc)
3775 			break;
3776 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3777 		tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
3778 		tw32_f(cpu_base + CPU_PC, pc);
3779 		udelay(1000);
3780 	}
3781 
3782 	return (i == iters) ? -EBUSY : 0;
3783 }
3784 
3785 /* tp->lock is held. */
3786 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
3787 {
3788 	const struct tg3_firmware_hdr *fw_hdr;
3789 	int err;
3790 
3791 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3792 
3793 	/* Firmware blob starts with version numbers, followed by
3794 	   start address and length. We are setting complete length.
3795 	   length = end_address_of_bss - start_address_of_text.
3796 	   Remainder is the blob to be loaded contiguously
3797 	   from start address. */
3798 
3799 	err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
3800 				    RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
3801 				    fw_hdr);
3802 	if (err)
3803 		return err;
3804 
3805 	err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
3806 				    TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
3807 				    fw_hdr);
3808 	if (err)
3809 		return err;
3810 
3811 	/* Now startup only the RX cpu. */
3812 	err = tg3_pause_cpu_and_set_pc(tp, RX_CPU_BASE,
3813 				       be32_to_cpu(fw_hdr->base_addr));
3814 	if (err) {
3815 		netdev_err(tp->dev, "%s fails to set RX CPU PC, is %08x "
3816 			   "should be %08x\n", __func__,
3817 			   tr32(RX_CPU_BASE + CPU_PC),
3818 				be32_to_cpu(fw_hdr->base_addr));
3819 		return -ENODEV;
3820 	}
3821 
3822 	tg3_rxcpu_resume(tp);
3823 
3824 	return 0;
3825 }
3826 
3827 static int tg3_validate_rxcpu_state(struct tg3 *tp)
3828 {
3829 	const int iters = 1000;
3830 	int i;
3831 	u32 val;
3832 
3833 	/* Wait for boot code to complete initialization and enter service
3834 	 * loop. It is then safe to download service patches
3835 	 */
3836 	for (i = 0; i < iters; i++) {
3837 		if (tr32(RX_CPU_HWBKPT) == TG3_SBROM_IN_SERVICE_LOOP)
3838 			break;
3839 
3840 		udelay(10);
3841 	}
3842 
3843 	if (i == iters) {
3844 		netdev_err(tp->dev, "Boot code not ready for service patches\n");
3845 		return -EBUSY;
3846 	}
3847 
3848 	val = tg3_read_indirect_reg32(tp, TG3_57766_FW_HANDSHAKE);
3849 	if (val & 0xff) {
3850 		netdev_warn(tp->dev,
3851 			    "Other patches exist. Not downloading EEE patch\n");
3852 		return -EEXIST;
3853 	}
3854 
3855 	return 0;
3856 }
3857 
3858 /* tp->lock is held. */
3859 static void tg3_load_57766_firmware(struct tg3 *tp)
3860 {
3861 	struct tg3_firmware_hdr *fw_hdr;
3862 
3863 	if (!tg3_flag(tp, NO_NVRAM))
3864 		return;
3865 
3866 	if (tg3_validate_rxcpu_state(tp))
3867 		return;
3868 
3869 	if (!tp->fw)
3870 		return;
3871 
3872 	/* This firmware blob has a different format than older firmware
3873 	 * releases as given below. The main difference is we have fragmented
3874 	 * data to be written to non-contiguous locations.
3875 	 *
3876 	 * In the beginning we have a firmware header identical to other
3877 	 * firmware which consists of version, base addr and length. The length
3878 	 * here is unused and set to 0xffffffff.
3879 	 *
3880 	 * This is followed by a series of firmware fragments which are
3881 	 * individually identical to previous firmware. i.e. they have the
3882 	 * firmware header and followed by data for that fragment. The version
3883 	 * field of the individual fragment header is unused.
3884 	 */
3885 
3886 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3887 	if (be32_to_cpu(fw_hdr->base_addr) != TG3_57766_FW_BASE_ADDR)
3888 		return;
3889 
3890 	if (tg3_rxcpu_pause(tp))
3891 		return;
3892 
3893 	/* tg3_load_firmware_cpu() will always succeed for the 57766 */
3894 	tg3_load_firmware_cpu(tp, 0, TG3_57766_FW_BASE_ADDR, 0, fw_hdr);
3895 
3896 	tg3_rxcpu_resume(tp);
3897 }
3898 
3899 /* tp->lock is held. */
3900 static int tg3_load_tso_firmware(struct tg3 *tp)
3901 {
3902 	const struct tg3_firmware_hdr *fw_hdr;
3903 	unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
3904 	int err;
3905 
3906 	if (!tg3_flag(tp, FW_TSO))
3907 		return 0;
3908 
3909 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3910 
3911 	/* Firmware blob starts with version numbers, followed by
3912 	   start address and length. We are setting complete length.
3913 	   length = end_address_of_bss - start_address_of_text.
3914 	   Remainder is the blob to be loaded contiguously
3915 	   from start address. */
3916 
3917 	cpu_scratch_size = tp->fw_len;
3918 
3919 	if (tg3_asic_rev(tp) == ASIC_REV_5705) {
3920 		cpu_base = RX_CPU_BASE;
3921 		cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
3922 	} else {
3923 		cpu_base = TX_CPU_BASE;
3924 		cpu_scratch_base = TX_CPU_SCRATCH_BASE;
3925 		cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
3926 	}
3927 
3928 	err = tg3_load_firmware_cpu(tp, cpu_base,
3929 				    cpu_scratch_base, cpu_scratch_size,
3930 				    fw_hdr);
3931 	if (err)
3932 		return err;
3933 
3934 	/* Now startup the cpu. */
3935 	err = tg3_pause_cpu_and_set_pc(tp, cpu_base,
3936 				       be32_to_cpu(fw_hdr->base_addr));
3937 	if (err) {
3938 		netdev_err(tp->dev,
3939 			   "%s fails to set CPU PC, is %08x should be %08x\n",
3940 			   __func__, tr32(cpu_base + CPU_PC),
3941 			   be32_to_cpu(fw_hdr->base_addr));
3942 		return -ENODEV;
3943 	}
3944 
3945 	tg3_resume_cpu(tp, cpu_base);
3946 	return 0;
3947 }
3948 
3949 /* tp->lock is held. */
3950 static void __tg3_set_one_mac_addr(struct tg3 *tp, u8 *mac_addr, int index)
3951 {
3952 	u32 addr_high, addr_low;
3953 
3954 	addr_high = ((mac_addr[0] << 8) | mac_addr[1]);
3955 	addr_low = ((mac_addr[2] << 24) | (mac_addr[3] << 16) |
3956 		    (mac_addr[4] <<  8) | mac_addr[5]);
3957 
3958 	if (index < 4) {
3959 		tw32(MAC_ADDR_0_HIGH + (index * 8), addr_high);
3960 		tw32(MAC_ADDR_0_LOW + (index * 8), addr_low);
3961 	} else {
3962 		index -= 4;
3963 		tw32(MAC_EXTADDR_0_HIGH + (index * 8), addr_high);
3964 		tw32(MAC_EXTADDR_0_LOW + (index * 8), addr_low);
3965 	}
3966 }
3967 
3968 /* tp->lock is held. */
3969 static void __tg3_set_mac_addr(struct tg3 *tp, bool skip_mac_1)
3970 {
3971 	u32 addr_high;
3972 	int i;
3973 
3974 	for (i = 0; i < 4; i++) {
3975 		if (i == 1 && skip_mac_1)
3976 			continue;
3977 		__tg3_set_one_mac_addr(tp, tp->dev->dev_addr, i);
3978 	}
3979 
3980 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
3981 	    tg3_asic_rev(tp) == ASIC_REV_5704) {
3982 		for (i = 4; i < 16; i++)
3983 			__tg3_set_one_mac_addr(tp, tp->dev->dev_addr, i);
3984 	}
3985 
3986 	addr_high = (tp->dev->dev_addr[0] +
3987 		     tp->dev->dev_addr[1] +
3988 		     tp->dev->dev_addr[2] +
3989 		     tp->dev->dev_addr[3] +
3990 		     tp->dev->dev_addr[4] +
3991 		     tp->dev->dev_addr[5]) &
3992 		TX_BACKOFF_SEED_MASK;
3993 	tw32(MAC_TX_BACKOFF_SEED, addr_high);
3994 }
3995 
3996 static void tg3_enable_register_access(struct tg3 *tp)
3997 {
3998 	/*
3999 	 * Make sure register accesses (indirect or otherwise) will function
4000 	 * correctly.
4001 	 */
4002 	pci_write_config_dword(tp->pdev,
4003 			       TG3PCI_MISC_HOST_CTRL, tp->misc_host_ctrl);
4004 }
4005 
4006 static int tg3_power_up(struct tg3 *tp)
4007 {
4008 	int err;
4009 
4010 	tg3_enable_register_access(tp);
4011 
4012 	err = pci_set_power_state(tp->pdev, PCI_D0);
4013 	if (!err) {
4014 		/* Switch out of Vaux if it is a NIC */
4015 		tg3_pwrsrc_switch_to_vmain(tp);
4016 	} else {
4017 		netdev_err(tp->dev, "Transition to D0 failed\n");
4018 	}
4019 
4020 	return err;
4021 }
4022 
4023 static int tg3_setup_phy(struct tg3 *, bool);
4024 
4025 static int tg3_power_down_prepare(struct tg3 *tp)
4026 {
4027 	u32 misc_host_ctrl;
4028 	bool device_should_wake, do_low_power;
4029 
4030 	tg3_enable_register_access(tp);
4031 
4032 	/* Restore the CLKREQ setting. */
4033 	if (tg3_flag(tp, CLKREQ_BUG))
4034 		pcie_capability_set_word(tp->pdev, PCI_EXP_LNKCTL,
4035 					 PCI_EXP_LNKCTL_CLKREQ_EN);
4036 
4037 	misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
4038 	tw32(TG3PCI_MISC_HOST_CTRL,
4039 	     misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
4040 
4041 	device_should_wake = device_may_wakeup(&tp->pdev->dev) &&
4042 			     tg3_flag(tp, WOL_ENABLE);
4043 
4044 	if (tg3_flag(tp, USE_PHYLIB)) {
4045 		do_low_power = false;
4046 		if ((tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) &&
4047 		    !(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4048 			struct phy_device *phydev;
4049 			u32 phyid, advertising;
4050 
4051 			phydev = tp->mdio_bus->phy_map[tp->phy_addr];
4052 
4053 			tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
4054 
4055 			tp->link_config.speed = phydev->speed;
4056 			tp->link_config.duplex = phydev->duplex;
4057 			tp->link_config.autoneg = phydev->autoneg;
4058 			tp->link_config.advertising = phydev->advertising;
4059 
4060 			advertising = ADVERTISED_TP |
4061 				      ADVERTISED_Pause |
4062 				      ADVERTISED_Autoneg |
4063 				      ADVERTISED_10baseT_Half;
4064 
4065 			if (tg3_flag(tp, ENABLE_ASF) || device_should_wake) {
4066 				if (tg3_flag(tp, WOL_SPEED_100MB))
4067 					advertising |=
4068 						ADVERTISED_100baseT_Half |
4069 						ADVERTISED_100baseT_Full |
4070 						ADVERTISED_10baseT_Full;
4071 				else
4072 					advertising |= ADVERTISED_10baseT_Full;
4073 			}
4074 
4075 			phydev->advertising = advertising;
4076 
4077 			phy_start_aneg(phydev);
4078 
4079 			phyid = phydev->drv->phy_id & phydev->drv->phy_id_mask;
4080 			if (phyid != PHY_ID_BCMAC131) {
4081 				phyid &= PHY_BCM_OUI_MASK;
4082 				if (phyid == PHY_BCM_OUI_1 ||
4083 				    phyid == PHY_BCM_OUI_2 ||
4084 				    phyid == PHY_BCM_OUI_3)
4085 					do_low_power = true;
4086 			}
4087 		}
4088 	} else {
4089 		do_low_power = true;
4090 
4091 		if (!(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER))
4092 			tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
4093 
4094 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
4095 			tg3_setup_phy(tp, false);
4096 	}
4097 
4098 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
4099 		u32 val;
4100 
4101 		val = tr32(GRC_VCPU_EXT_CTRL);
4102 		tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
4103 	} else if (!tg3_flag(tp, ENABLE_ASF)) {
4104 		int i;
4105 		u32 val;
4106 
4107 		for (i = 0; i < 200; i++) {
4108 			tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
4109 			if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
4110 				break;
4111 			msleep(1);
4112 		}
4113 	}
4114 	if (tg3_flag(tp, WOL_CAP))
4115 		tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
4116 						     WOL_DRV_STATE_SHUTDOWN |
4117 						     WOL_DRV_WOL |
4118 						     WOL_SET_MAGIC_PKT);
4119 
4120 	if (device_should_wake) {
4121 		u32 mac_mode;
4122 
4123 		if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
4124 			if (do_low_power &&
4125 			    !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
4126 				tg3_phy_auxctl_write(tp,
4127 					       MII_TG3_AUXCTL_SHDWSEL_PWRCTL,
4128 					       MII_TG3_AUXCTL_PCTL_WOL_EN |
4129 					       MII_TG3_AUXCTL_PCTL_100TX_LPWR |
4130 					       MII_TG3_AUXCTL_PCTL_CL_AB_TXDAC);
4131 				udelay(40);
4132 			}
4133 
4134 			if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
4135 				mac_mode = MAC_MODE_PORT_MODE_GMII;
4136 			else if (tp->phy_flags &
4137 				 TG3_PHYFLG_KEEP_LINK_ON_PWRDN) {
4138 				if (tp->link_config.active_speed == SPEED_1000)
4139 					mac_mode = MAC_MODE_PORT_MODE_GMII;
4140 				else
4141 					mac_mode = MAC_MODE_PORT_MODE_MII;
4142 			} else
4143 				mac_mode = MAC_MODE_PORT_MODE_MII;
4144 
4145 			mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
4146 			if (tg3_asic_rev(tp) == ASIC_REV_5700) {
4147 				u32 speed = tg3_flag(tp, WOL_SPEED_100MB) ?
4148 					     SPEED_100 : SPEED_10;
4149 				if (tg3_5700_link_polarity(tp, speed))
4150 					mac_mode |= MAC_MODE_LINK_POLARITY;
4151 				else
4152 					mac_mode &= ~MAC_MODE_LINK_POLARITY;
4153 			}
4154 		} else {
4155 			mac_mode = MAC_MODE_PORT_MODE_TBI;
4156 		}
4157 
4158 		if (!tg3_flag(tp, 5750_PLUS))
4159 			tw32(MAC_LED_CTRL, tp->led_ctrl);
4160 
4161 		mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
4162 		if ((tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS)) &&
4163 		    (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)))
4164 			mac_mode |= MAC_MODE_KEEP_FRAME_IN_WOL;
4165 
4166 		if (tg3_flag(tp, ENABLE_APE))
4167 			mac_mode |= MAC_MODE_APE_TX_EN |
4168 				    MAC_MODE_APE_RX_EN |
4169 				    MAC_MODE_TDE_ENABLE;
4170 
4171 		tw32_f(MAC_MODE, mac_mode);
4172 		udelay(100);
4173 
4174 		tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
4175 		udelay(10);
4176 	}
4177 
4178 	if (!tg3_flag(tp, WOL_SPEED_100MB) &&
4179 	    (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4180 	     tg3_asic_rev(tp) == ASIC_REV_5701)) {
4181 		u32 base_val;
4182 
4183 		base_val = tp->pci_clock_ctrl;
4184 		base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
4185 			     CLOCK_CTRL_TXCLK_DISABLE);
4186 
4187 		tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
4188 			    CLOCK_CTRL_PWRDOWN_PLL133, 40);
4189 	} else if (tg3_flag(tp, 5780_CLASS) ||
4190 		   tg3_flag(tp, CPMU_PRESENT) ||
4191 		   tg3_asic_rev(tp) == ASIC_REV_5906) {
4192 		/* do nothing */
4193 	} else if (!(tg3_flag(tp, 5750_PLUS) && tg3_flag(tp, ENABLE_ASF))) {
4194 		u32 newbits1, newbits2;
4195 
4196 		if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4197 		    tg3_asic_rev(tp) == ASIC_REV_5701) {
4198 			newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
4199 				    CLOCK_CTRL_TXCLK_DISABLE |
4200 				    CLOCK_CTRL_ALTCLK);
4201 			newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
4202 		} else if (tg3_flag(tp, 5705_PLUS)) {
4203 			newbits1 = CLOCK_CTRL_625_CORE;
4204 			newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
4205 		} else {
4206 			newbits1 = CLOCK_CTRL_ALTCLK;
4207 			newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
4208 		}
4209 
4210 		tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
4211 			    40);
4212 
4213 		tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
4214 			    40);
4215 
4216 		if (!tg3_flag(tp, 5705_PLUS)) {
4217 			u32 newbits3;
4218 
4219 			if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4220 			    tg3_asic_rev(tp) == ASIC_REV_5701) {
4221 				newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
4222 					    CLOCK_CTRL_TXCLK_DISABLE |
4223 					    CLOCK_CTRL_44MHZ_CORE);
4224 			} else {
4225 				newbits3 = CLOCK_CTRL_44MHZ_CORE;
4226 			}
4227 
4228 			tw32_wait_f(TG3PCI_CLOCK_CTRL,
4229 				    tp->pci_clock_ctrl | newbits3, 40);
4230 		}
4231 	}
4232 
4233 	if (!(device_should_wake) && !tg3_flag(tp, ENABLE_ASF))
4234 		tg3_power_down_phy(tp, do_low_power);
4235 
4236 	tg3_frob_aux_power(tp, true);
4237 
4238 	/* Workaround for unstable PLL clock */
4239 	if ((!tg3_flag(tp, IS_SSB_CORE)) &&
4240 	    ((tg3_chip_rev(tp) == CHIPREV_5750_AX) ||
4241 	     (tg3_chip_rev(tp) == CHIPREV_5750_BX))) {
4242 		u32 val = tr32(0x7d00);
4243 
4244 		val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
4245 		tw32(0x7d00, val);
4246 		if (!tg3_flag(tp, ENABLE_ASF)) {
4247 			int err;
4248 
4249 			err = tg3_nvram_lock(tp);
4250 			tg3_halt_cpu(tp, RX_CPU_BASE);
4251 			if (!err)
4252 				tg3_nvram_unlock(tp);
4253 		}
4254 	}
4255 
4256 	tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
4257 
4258 	tg3_ape_driver_state_change(tp, RESET_KIND_SHUTDOWN);
4259 
4260 	return 0;
4261 }
4262 
4263 static void tg3_power_down(struct tg3 *tp)
4264 {
4265 	pci_wake_from_d3(tp->pdev, tg3_flag(tp, WOL_ENABLE));
4266 	pci_set_power_state(tp->pdev, PCI_D3hot);
4267 }
4268 
4269 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
4270 {
4271 	switch (val & MII_TG3_AUX_STAT_SPDMASK) {
4272 	case MII_TG3_AUX_STAT_10HALF:
4273 		*speed = SPEED_10;
4274 		*duplex = DUPLEX_HALF;
4275 		break;
4276 
4277 	case MII_TG3_AUX_STAT_10FULL:
4278 		*speed = SPEED_10;
4279 		*duplex = DUPLEX_FULL;
4280 		break;
4281 
4282 	case MII_TG3_AUX_STAT_100HALF:
4283 		*speed = SPEED_100;
4284 		*duplex = DUPLEX_HALF;
4285 		break;
4286 
4287 	case MII_TG3_AUX_STAT_100FULL:
4288 		*speed = SPEED_100;
4289 		*duplex = DUPLEX_FULL;
4290 		break;
4291 
4292 	case MII_TG3_AUX_STAT_1000HALF:
4293 		*speed = SPEED_1000;
4294 		*duplex = DUPLEX_HALF;
4295 		break;
4296 
4297 	case MII_TG3_AUX_STAT_1000FULL:
4298 		*speed = SPEED_1000;
4299 		*duplex = DUPLEX_FULL;
4300 		break;
4301 
4302 	default:
4303 		if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
4304 			*speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
4305 				 SPEED_10;
4306 			*duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
4307 				  DUPLEX_HALF;
4308 			break;
4309 		}
4310 		*speed = SPEED_UNKNOWN;
4311 		*duplex = DUPLEX_UNKNOWN;
4312 		break;
4313 	}
4314 }
4315 
4316 static int tg3_phy_autoneg_cfg(struct tg3 *tp, u32 advertise, u32 flowctrl)
4317 {
4318 	int err = 0;
4319 	u32 val, new_adv;
4320 
4321 	new_adv = ADVERTISE_CSMA;
4322 	new_adv |= ethtool_adv_to_mii_adv_t(advertise) & ADVERTISE_ALL;
4323 	new_adv |= mii_advertise_flowctrl(flowctrl);
4324 
4325 	err = tg3_writephy(tp, MII_ADVERTISE, new_adv);
4326 	if (err)
4327 		goto done;
4328 
4329 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4330 		new_adv = ethtool_adv_to_mii_ctrl1000_t(advertise);
4331 
4332 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4333 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0)
4334 			new_adv |= CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER;
4335 
4336 		err = tg3_writephy(tp, MII_CTRL1000, new_adv);
4337 		if (err)
4338 			goto done;
4339 	}
4340 
4341 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
4342 		goto done;
4343 
4344 	tw32(TG3_CPMU_EEE_MODE,
4345 	     tr32(TG3_CPMU_EEE_MODE) & ~TG3_CPMU_EEEMD_LPI_ENABLE);
4346 
4347 	err = tg3_phy_toggle_auxctl_smdsp(tp, true);
4348 	if (!err) {
4349 		u32 err2;
4350 
4351 		val = 0;
4352 		/* Advertise 100-BaseTX EEE ability */
4353 		if (advertise & ADVERTISED_100baseT_Full)
4354 			val |= MDIO_AN_EEE_ADV_100TX;
4355 		/* Advertise 1000-BaseT EEE ability */
4356 		if (advertise & ADVERTISED_1000baseT_Full)
4357 			val |= MDIO_AN_EEE_ADV_1000T;
4358 
4359 		if (!tp->eee.eee_enabled) {
4360 			val = 0;
4361 			tp->eee.advertised = 0;
4362 		} else {
4363 			tp->eee.advertised = advertise &
4364 					     (ADVERTISED_100baseT_Full |
4365 					      ADVERTISED_1000baseT_Full);
4366 		}
4367 
4368 		err = tg3_phy_cl45_write(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, val);
4369 		if (err)
4370 			val = 0;
4371 
4372 		switch (tg3_asic_rev(tp)) {
4373 		case ASIC_REV_5717:
4374 		case ASIC_REV_57765:
4375 		case ASIC_REV_57766:
4376 		case ASIC_REV_5719:
4377 			/* If we advertised any eee advertisements above... */
4378 			if (val)
4379 				val = MII_TG3_DSP_TAP26_ALNOKO |
4380 				      MII_TG3_DSP_TAP26_RMRXSTO |
4381 				      MII_TG3_DSP_TAP26_OPCSINPT;
4382 			tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
4383 			/* Fall through */
4384 		case ASIC_REV_5720:
4385 		case ASIC_REV_5762:
4386 			if (!tg3_phydsp_read(tp, MII_TG3_DSP_CH34TP2, &val))
4387 				tg3_phydsp_write(tp, MII_TG3_DSP_CH34TP2, val |
4388 						 MII_TG3_DSP_CH34TP2_HIBW01);
4389 		}
4390 
4391 		err2 = tg3_phy_toggle_auxctl_smdsp(tp, false);
4392 		if (!err)
4393 			err = err2;
4394 	}
4395 
4396 done:
4397 	return err;
4398 }
4399 
4400 static void tg3_phy_copper_begin(struct tg3 *tp)
4401 {
4402 	if (tp->link_config.autoneg == AUTONEG_ENABLE ||
4403 	    (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4404 		u32 adv, fc;
4405 
4406 		if ((tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) &&
4407 		    !(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)) {
4408 			adv = ADVERTISED_10baseT_Half |
4409 			      ADVERTISED_10baseT_Full;
4410 			if (tg3_flag(tp, WOL_SPEED_100MB))
4411 				adv |= ADVERTISED_100baseT_Half |
4412 				       ADVERTISED_100baseT_Full;
4413 			if (tp->phy_flags & TG3_PHYFLG_1G_ON_VAUX_OK) {
4414 				if (!(tp->phy_flags &
4415 				      TG3_PHYFLG_DISABLE_1G_HD_ADV))
4416 					adv |= ADVERTISED_1000baseT_Half;
4417 				adv |= ADVERTISED_1000baseT_Full;
4418 			}
4419 
4420 			fc = FLOW_CTRL_TX | FLOW_CTRL_RX;
4421 		} else {
4422 			adv = tp->link_config.advertising;
4423 			if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
4424 				adv &= ~(ADVERTISED_1000baseT_Half |
4425 					 ADVERTISED_1000baseT_Full);
4426 
4427 			fc = tp->link_config.flowctrl;
4428 		}
4429 
4430 		tg3_phy_autoneg_cfg(tp, adv, fc);
4431 
4432 		if ((tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) &&
4433 		    (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)) {
4434 			/* Normally during power down we want to autonegotiate
4435 			 * the lowest possible speed for WOL. However, to avoid
4436 			 * link flap, we leave it untouched.
4437 			 */
4438 			return;
4439 		}
4440 
4441 		tg3_writephy(tp, MII_BMCR,
4442 			     BMCR_ANENABLE | BMCR_ANRESTART);
4443 	} else {
4444 		int i;
4445 		u32 bmcr, orig_bmcr;
4446 
4447 		tp->link_config.active_speed = tp->link_config.speed;
4448 		tp->link_config.active_duplex = tp->link_config.duplex;
4449 
4450 		if (tg3_asic_rev(tp) == ASIC_REV_5714) {
4451 			/* With autoneg disabled, 5715 only links up when the
4452 			 * advertisement register has the configured speed
4453 			 * enabled.
4454 			 */
4455 			tg3_writephy(tp, MII_ADVERTISE, ADVERTISE_ALL);
4456 		}
4457 
4458 		bmcr = 0;
4459 		switch (tp->link_config.speed) {
4460 		default:
4461 		case SPEED_10:
4462 			break;
4463 
4464 		case SPEED_100:
4465 			bmcr |= BMCR_SPEED100;
4466 			break;
4467 
4468 		case SPEED_1000:
4469 			bmcr |= BMCR_SPEED1000;
4470 			break;
4471 		}
4472 
4473 		if (tp->link_config.duplex == DUPLEX_FULL)
4474 			bmcr |= BMCR_FULLDPLX;
4475 
4476 		if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
4477 		    (bmcr != orig_bmcr)) {
4478 			tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
4479 			for (i = 0; i < 1500; i++) {
4480 				u32 tmp;
4481 
4482 				udelay(10);
4483 				if (tg3_readphy(tp, MII_BMSR, &tmp) ||
4484 				    tg3_readphy(tp, MII_BMSR, &tmp))
4485 					continue;
4486 				if (!(tmp & BMSR_LSTATUS)) {
4487 					udelay(40);
4488 					break;
4489 				}
4490 			}
4491 			tg3_writephy(tp, MII_BMCR, bmcr);
4492 			udelay(40);
4493 		}
4494 	}
4495 }
4496 
4497 static int tg3_phy_pull_config(struct tg3 *tp)
4498 {
4499 	int err;
4500 	u32 val;
4501 
4502 	err = tg3_readphy(tp, MII_BMCR, &val);
4503 	if (err)
4504 		goto done;
4505 
4506 	if (!(val & BMCR_ANENABLE)) {
4507 		tp->link_config.autoneg = AUTONEG_DISABLE;
4508 		tp->link_config.advertising = 0;
4509 		tg3_flag_clear(tp, PAUSE_AUTONEG);
4510 
4511 		err = -EIO;
4512 
4513 		switch (val & (BMCR_SPEED1000 | BMCR_SPEED100)) {
4514 		case 0:
4515 			if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
4516 				goto done;
4517 
4518 			tp->link_config.speed = SPEED_10;
4519 			break;
4520 		case BMCR_SPEED100:
4521 			if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
4522 				goto done;
4523 
4524 			tp->link_config.speed = SPEED_100;
4525 			break;
4526 		case BMCR_SPEED1000:
4527 			if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4528 				tp->link_config.speed = SPEED_1000;
4529 				break;
4530 			}
4531 			/* Fall through */
4532 		default:
4533 			goto done;
4534 		}
4535 
4536 		if (val & BMCR_FULLDPLX)
4537 			tp->link_config.duplex = DUPLEX_FULL;
4538 		else
4539 			tp->link_config.duplex = DUPLEX_HALF;
4540 
4541 		tp->link_config.flowctrl = FLOW_CTRL_RX | FLOW_CTRL_TX;
4542 
4543 		err = 0;
4544 		goto done;
4545 	}
4546 
4547 	tp->link_config.autoneg = AUTONEG_ENABLE;
4548 	tp->link_config.advertising = ADVERTISED_Autoneg;
4549 	tg3_flag_set(tp, PAUSE_AUTONEG);
4550 
4551 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
4552 		u32 adv;
4553 
4554 		err = tg3_readphy(tp, MII_ADVERTISE, &val);
4555 		if (err)
4556 			goto done;
4557 
4558 		adv = mii_adv_to_ethtool_adv_t(val & ADVERTISE_ALL);
4559 		tp->link_config.advertising |= adv | ADVERTISED_TP;
4560 
4561 		tp->link_config.flowctrl = tg3_decode_flowctrl_1000T(val);
4562 	} else {
4563 		tp->link_config.advertising |= ADVERTISED_FIBRE;
4564 	}
4565 
4566 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4567 		u32 adv;
4568 
4569 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
4570 			err = tg3_readphy(tp, MII_CTRL1000, &val);
4571 			if (err)
4572 				goto done;
4573 
4574 			adv = mii_ctrl1000_to_ethtool_adv_t(val);
4575 		} else {
4576 			err = tg3_readphy(tp, MII_ADVERTISE, &val);
4577 			if (err)
4578 				goto done;
4579 
4580 			adv = tg3_decode_flowctrl_1000X(val);
4581 			tp->link_config.flowctrl = adv;
4582 
4583 			val &= (ADVERTISE_1000XHALF | ADVERTISE_1000XFULL);
4584 			adv = mii_adv_to_ethtool_adv_x(val);
4585 		}
4586 
4587 		tp->link_config.advertising |= adv;
4588 	}
4589 
4590 done:
4591 	return err;
4592 }
4593 
4594 static int tg3_init_5401phy_dsp(struct tg3 *tp)
4595 {
4596 	int err;
4597 
4598 	/* Turn off tap power management. */
4599 	/* Set Extended packet length bit */
4600 	err = tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
4601 
4602 	err |= tg3_phydsp_write(tp, 0x0012, 0x1804);
4603 	err |= tg3_phydsp_write(tp, 0x0013, 0x1204);
4604 	err |= tg3_phydsp_write(tp, 0x8006, 0x0132);
4605 	err |= tg3_phydsp_write(tp, 0x8006, 0x0232);
4606 	err |= tg3_phydsp_write(tp, 0x201f, 0x0a20);
4607 
4608 	udelay(40);
4609 
4610 	return err;
4611 }
4612 
4613 static bool tg3_phy_eee_config_ok(struct tg3 *tp)
4614 {
4615 	struct ethtool_eee eee;
4616 
4617 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
4618 		return true;
4619 
4620 	tg3_eee_pull_config(tp, &eee);
4621 
4622 	if (tp->eee.eee_enabled) {
4623 		if (tp->eee.advertised != eee.advertised ||
4624 		    tp->eee.tx_lpi_timer != eee.tx_lpi_timer ||
4625 		    tp->eee.tx_lpi_enabled != eee.tx_lpi_enabled)
4626 			return false;
4627 	} else {
4628 		/* EEE is disabled but we're advertising */
4629 		if (eee.advertised)
4630 			return false;
4631 	}
4632 
4633 	return true;
4634 }
4635 
4636 static bool tg3_phy_copper_an_config_ok(struct tg3 *tp, u32 *lcladv)
4637 {
4638 	u32 advmsk, tgtadv, advertising;
4639 
4640 	advertising = tp->link_config.advertising;
4641 	tgtadv = ethtool_adv_to_mii_adv_t(advertising) & ADVERTISE_ALL;
4642 
4643 	advmsk = ADVERTISE_ALL;
4644 	if (tp->link_config.active_duplex == DUPLEX_FULL) {
4645 		tgtadv |= mii_advertise_flowctrl(tp->link_config.flowctrl);
4646 		advmsk |= ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
4647 	}
4648 
4649 	if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
4650 		return false;
4651 
4652 	if ((*lcladv & advmsk) != tgtadv)
4653 		return false;
4654 
4655 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4656 		u32 tg3_ctrl;
4657 
4658 		tgtadv = ethtool_adv_to_mii_ctrl1000_t(advertising);
4659 
4660 		if (tg3_readphy(tp, MII_CTRL1000, &tg3_ctrl))
4661 			return false;
4662 
4663 		if (tgtadv &&
4664 		    (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4665 		     tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0)) {
4666 			tgtadv |= CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER;
4667 			tg3_ctrl &= (ADVERTISE_1000HALF | ADVERTISE_1000FULL |
4668 				     CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER);
4669 		} else {
4670 			tg3_ctrl &= (ADVERTISE_1000HALF | ADVERTISE_1000FULL);
4671 		}
4672 
4673 		if (tg3_ctrl != tgtadv)
4674 			return false;
4675 	}
4676 
4677 	return true;
4678 }
4679 
4680 static bool tg3_phy_copper_fetch_rmtadv(struct tg3 *tp, u32 *rmtadv)
4681 {
4682 	u32 lpeth = 0;
4683 
4684 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4685 		u32 val;
4686 
4687 		if (tg3_readphy(tp, MII_STAT1000, &val))
4688 			return false;
4689 
4690 		lpeth = mii_stat1000_to_ethtool_lpa_t(val);
4691 	}
4692 
4693 	if (tg3_readphy(tp, MII_LPA, rmtadv))
4694 		return false;
4695 
4696 	lpeth |= mii_lpa_to_ethtool_lpa_t(*rmtadv);
4697 	tp->link_config.rmt_adv = lpeth;
4698 
4699 	return true;
4700 }
4701 
4702 static bool tg3_test_and_report_link_chg(struct tg3 *tp, bool curr_link_up)
4703 {
4704 	if (curr_link_up != tp->link_up) {
4705 		if (curr_link_up) {
4706 			netif_carrier_on(tp->dev);
4707 		} else {
4708 			netif_carrier_off(tp->dev);
4709 			if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
4710 				tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4711 		}
4712 
4713 		tg3_link_report(tp);
4714 		return true;
4715 	}
4716 
4717 	return false;
4718 }
4719 
4720 static void tg3_clear_mac_status(struct tg3 *tp)
4721 {
4722 	tw32(MAC_EVENT, 0);
4723 
4724 	tw32_f(MAC_STATUS,
4725 	       MAC_STATUS_SYNC_CHANGED |
4726 	       MAC_STATUS_CFG_CHANGED |
4727 	       MAC_STATUS_MI_COMPLETION |
4728 	       MAC_STATUS_LNKSTATE_CHANGED);
4729 	udelay(40);
4730 }
4731 
4732 static void tg3_setup_eee(struct tg3 *tp)
4733 {
4734 	u32 val;
4735 
4736 	val = TG3_CPMU_EEE_LNKIDL_PCIE_NL0 |
4737 	      TG3_CPMU_EEE_LNKIDL_UART_IDL;
4738 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0)
4739 		val |= TG3_CPMU_EEE_LNKIDL_APE_TX_MT;
4740 
4741 	tw32_f(TG3_CPMU_EEE_LNKIDL_CTRL, val);
4742 
4743 	tw32_f(TG3_CPMU_EEE_CTRL,
4744 	       TG3_CPMU_EEE_CTRL_EXIT_20_1_US);
4745 
4746 	val = TG3_CPMU_EEEMD_ERLY_L1_XIT_DET |
4747 	      (tp->eee.tx_lpi_enabled ? TG3_CPMU_EEEMD_LPI_IN_TX : 0) |
4748 	      TG3_CPMU_EEEMD_LPI_IN_RX |
4749 	      TG3_CPMU_EEEMD_EEE_ENABLE;
4750 
4751 	if (tg3_asic_rev(tp) != ASIC_REV_5717)
4752 		val |= TG3_CPMU_EEEMD_SND_IDX_DET_EN;
4753 
4754 	if (tg3_flag(tp, ENABLE_APE))
4755 		val |= TG3_CPMU_EEEMD_APE_TX_DET_EN;
4756 
4757 	tw32_f(TG3_CPMU_EEE_MODE, tp->eee.eee_enabled ? val : 0);
4758 
4759 	tw32_f(TG3_CPMU_EEE_DBTMR1,
4760 	       TG3_CPMU_DBTMR1_PCIEXIT_2047US |
4761 	       (tp->eee.tx_lpi_timer & 0xffff));
4762 
4763 	tw32_f(TG3_CPMU_EEE_DBTMR2,
4764 	       TG3_CPMU_DBTMR2_APE_TX_2047US |
4765 	       TG3_CPMU_DBTMR2_TXIDXEQ_2047US);
4766 }
4767 
4768 static int tg3_setup_copper_phy(struct tg3 *tp, bool force_reset)
4769 {
4770 	bool current_link_up;
4771 	u32 bmsr, val;
4772 	u32 lcl_adv, rmt_adv;
4773 	u16 current_speed;
4774 	u8 current_duplex;
4775 	int i, err;
4776 
4777 	tg3_clear_mac_status(tp);
4778 
4779 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
4780 		tw32_f(MAC_MI_MODE,
4781 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
4782 		udelay(80);
4783 	}
4784 
4785 	tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, 0);
4786 
4787 	/* Some third-party PHYs need to be reset on link going
4788 	 * down.
4789 	 */
4790 	if ((tg3_asic_rev(tp) == ASIC_REV_5703 ||
4791 	     tg3_asic_rev(tp) == ASIC_REV_5704 ||
4792 	     tg3_asic_rev(tp) == ASIC_REV_5705) &&
4793 	    tp->link_up) {
4794 		tg3_readphy(tp, MII_BMSR, &bmsr);
4795 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4796 		    !(bmsr & BMSR_LSTATUS))
4797 			force_reset = true;
4798 	}
4799 	if (force_reset)
4800 		tg3_phy_reset(tp);
4801 
4802 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
4803 		tg3_readphy(tp, MII_BMSR, &bmsr);
4804 		if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
4805 		    !tg3_flag(tp, INIT_COMPLETE))
4806 			bmsr = 0;
4807 
4808 		if (!(bmsr & BMSR_LSTATUS)) {
4809 			err = tg3_init_5401phy_dsp(tp);
4810 			if (err)
4811 				return err;
4812 
4813 			tg3_readphy(tp, MII_BMSR, &bmsr);
4814 			for (i = 0; i < 1000; i++) {
4815 				udelay(10);
4816 				if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4817 				    (bmsr & BMSR_LSTATUS)) {
4818 					udelay(40);
4819 					break;
4820 				}
4821 			}
4822 
4823 			if ((tp->phy_id & TG3_PHY_ID_REV_MASK) ==
4824 			    TG3_PHY_REV_BCM5401_B0 &&
4825 			    !(bmsr & BMSR_LSTATUS) &&
4826 			    tp->link_config.active_speed == SPEED_1000) {
4827 				err = tg3_phy_reset(tp);
4828 				if (!err)
4829 					err = tg3_init_5401phy_dsp(tp);
4830 				if (err)
4831 					return err;
4832 			}
4833 		}
4834 	} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4835 		   tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0) {
4836 		/* 5701 {A0,B0} CRC bug workaround */
4837 		tg3_writephy(tp, 0x15, 0x0a75);
4838 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
4839 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
4840 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
4841 	}
4842 
4843 	/* Clear pending interrupts... */
4844 	tg3_readphy(tp, MII_TG3_ISTAT, &val);
4845 	tg3_readphy(tp, MII_TG3_ISTAT, &val);
4846 
4847 	if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT)
4848 		tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
4849 	else if (!(tp->phy_flags & TG3_PHYFLG_IS_FET))
4850 		tg3_writephy(tp, MII_TG3_IMASK, ~0);
4851 
4852 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4853 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
4854 		if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
4855 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
4856 				     MII_TG3_EXT_CTRL_LNK3_LED_MODE);
4857 		else
4858 			tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
4859 	}
4860 
4861 	current_link_up = false;
4862 	current_speed = SPEED_UNKNOWN;
4863 	current_duplex = DUPLEX_UNKNOWN;
4864 	tp->phy_flags &= ~TG3_PHYFLG_MDIX_STATE;
4865 	tp->link_config.rmt_adv = 0;
4866 
4867 	if (tp->phy_flags & TG3_PHYFLG_CAPACITIVE_COUPLING) {
4868 		err = tg3_phy_auxctl_read(tp,
4869 					  MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
4870 					  &val);
4871 		if (!err && !(val & (1 << 10))) {
4872 			tg3_phy_auxctl_write(tp,
4873 					     MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
4874 					     val | (1 << 10));
4875 			goto relink;
4876 		}
4877 	}
4878 
4879 	bmsr = 0;
4880 	for (i = 0; i < 100; i++) {
4881 		tg3_readphy(tp, MII_BMSR, &bmsr);
4882 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4883 		    (bmsr & BMSR_LSTATUS))
4884 			break;
4885 		udelay(40);
4886 	}
4887 
4888 	if (bmsr & BMSR_LSTATUS) {
4889 		u32 aux_stat, bmcr;
4890 
4891 		tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
4892 		for (i = 0; i < 2000; i++) {
4893 			udelay(10);
4894 			if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
4895 			    aux_stat)
4896 				break;
4897 		}
4898 
4899 		tg3_aux_stat_to_speed_duplex(tp, aux_stat,
4900 					     &current_speed,
4901 					     &current_duplex);
4902 
4903 		bmcr = 0;
4904 		for (i = 0; i < 200; i++) {
4905 			tg3_readphy(tp, MII_BMCR, &bmcr);
4906 			if (tg3_readphy(tp, MII_BMCR, &bmcr))
4907 				continue;
4908 			if (bmcr && bmcr != 0x7fff)
4909 				break;
4910 			udelay(10);
4911 		}
4912 
4913 		lcl_adv = 0;
4914 		rmt_adv = 0;
4915 
4916 		tp->link_config.active_speed = current_speed;
4917 		tp->link_config.active_duplex = current_duplex;
4918 
4919 		if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4920 			bool eee_config_ok = tg3_phy_eee_config_ok(tp);
4921 
4922 			if ((bmcr & BMCR_ANENABLE) &&
4923 			    eee_config_ok &&
4924 			    tg3_phy_copper_an_config_ok(tp, &lcl_adv) &&
4925 			    tg3_phy_copper_fetch_rmtadv(tp, &rmt_adv))
4926 				current_link_up = true;
4927 
4928 			/* EEE settings changes take effect only after a phy
4929 			 * reset.  If we have skipped a reset due to Link Flap
4930 			 * Avoidance being enabled, do it now.
4931 			 */
4932 			if (!eee_config_ok &&
4933 			    (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
4934 			    !force_reset) {
4935 				tg3_setup_eee(tp);
4936 				tg3_phy_reset(tp);
4937 			}
4938 		} else {
4939 			if (!(bmcr & BMCR_ANENABLE) &&
4940 			    tp->link_config.speed == current_speed &&
4941 			    tp->link_config.duplex == current_duplex) {
4942 				current_link_up = true;
4943 			}
4944 		}
4945 
4946 		if (current_link_up &&
4947 		    tp->link_config.active_duplex == DUPLEX_FULL) {
4948 			u32 reg, bit;
4949 
4950 			if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
4951 				reg = MII_TG3_FET_GEN_STAT;
4952 				bit = MII_TG3_FET_GEN_STAT_MDIXSTAT;
4953 			} else {
4954 				reg = MII_TG3_EXT_STAT;
4955 				bit = MII_TG3_EXT_STAT_MDIX;
4956 			}
4957 
4958 			if (!tg3_readphy(tp, reg, &val) && (val & bit))
4959 				tp->phy_flags |= TG3_PHYFLG_MDIX_STATE;
4960 
4961 			tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
4962 		}
4963 	}
4964 
4965 relink:
4966 	if (!current_link_up || (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4967 		tg3_phy_copper_begin(tp);
4968 
4969 		if (tg3_flag(tp, ROBOSWITCH)) {
4970 			current_link_up = true;
4971 			/* FIXME: when BCM5325 switch is used use 100 MBit/s */
4972 			current_speed = SPEED_1000;
4973 			current_duplex = DUPLEX_FULL;
4974 			tp->link_config.active_speed = current_speed;
4975 			tp->link_config.active_duplex = current_duplex;
4976 		}
4977 
4978 		tg3_readphy(tp, MII_BMSR, &bmsr);
4979 		if ((!tg3_readphy(tp, MII_BMSR, &bmsr) && (bmsr & BMSR_LSTATUS)) ||
4980 		    (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
4981 			current_link_up = true;
4982 	}
4983 
4984 	tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
4985 	if (current_link_up) {
4986 		if (tp->link_config.active_speed == SPEED_100 ||
4987 		    tp->link_config.active_speed == SPEED_10)
4988 			tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
4989 		else
4990 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4991 	} else if (tp->phy_flags & TG3_PHYFLG_IS_FET)
4992 		tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
4993 	else
4994 		tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4995 
4996 	/* In order for the 5750 core in BCM4785 chip to work properly
4997 	 * in RGMII mode, the Led Control Register must be set up.
4998 	 */
4999 	if (tg3_flag(tp, RGMII_MODE)) {
5000 		u32 led_ctrl = tr32(MAC_LED_CTRL);
5001 		led_ctrl &= ~(LED_CTRL_1000MBPS_ON | LED_CTRL_100MBPS_ON);
5002 
5003 		if (tp->link_config.active_speed == SPEED_10)
5004 			led_ctrl |= LED_CTRL_LNKLED_OVERRIDE;
5005 		else if (tp->link_config.active_speed == SPEED_100)
5006 			led_ctrl |= (LED_CTRL_LNKLED_OVERRIDE |
5007 				     LED_CTRL_100MBPS_ON);
5008 		else if (tp->link_config.active_speed == SPEED_1000)
5009 			led_ctrl |= (LED_CTRL_LNKLED_OVERRIDE |
5010 				     LED_CTRL_1000MBPS_ON);
5011 
5012 		tw32(MAC_LED_CTRL, led_ctrl);
5013 		udelay(40);
5014 	}
5015 
5016 	tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
5017 	if (tp->link_config.active_duplex == DUPLEX_HALF)
5018 		tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
5019 
5020 	if (tg3_asic_rev(tp) == ASIC_REV_5700) {
5021 		if (current_link_up &&
5022 		    tg3_5700_link_polarity(tp, tp->link_config.active_speed))
5023 			tp->mac_mode |= MAC_MODE_LINK_POLARITY;
5024 		else
5025 			tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
5026 	}
5027 
5028 	/* ??? Without this setting Netgear GA302T PHY does not
5029 	 * ??? send/receive packets...
5030 	 */
5031 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411 &&
5032 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5700_ALTIMA) {
5033 		tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
5034 		tw32_f(MAC_MI_MODE, tp->mi_mode);
5035 		udelay(80);
5036 	}
5037 
5038 	tw32_f(MAC_MODE, tp->mac_mode);
5039 	udelay(40);
5040 
5041 	tg3_phy_eee_adjust(tp, current_link_up);
5042 
5043 	if (tg3_flag(tp, USE_LINKCHG_REG)) {
5044 		/* Polled via timer. */
5045 		tw32_f(MAC_EVENT, 0);
5046 	} else {
5047 		tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5048 	}
5049 	udelay(40);
5050 
5051 	if (tg3_asic_rev(tp) == ASIC_REV_5700 &&
5052 	    current_link_up &&
5053 	    tp->link_config.active_speed == SPEED_1000 &&
5054 	    (tg3_flag(tp, PCIX_MODE) || tg3_flag(tp, PCI_HIGH_SPEED))) {
5055 		udelay(120);
5056 		tw32_f(MAC_STATUS,
5057 		     (MAC_STATUS_SYNC_CHANGED |
5058 		      MAC_STATUS_CFG_CHANGED));
5059 		udelay(40);
5060 		tg3_write_mem(tp,
5061 			      NIC_SRAM_FIRMWARE_MBOX,
5062 			      NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
5063 	}
5064 
5065 	/* Prevent send BD corruption. */
5066 	if (tg3_flag(tp, CLKREQ_BUG)) {
5067 		if (tp->link_config.active_speed == SPEED_100 ||
5068 		    tp->link_config.active_speed == SPEED_10)
5069 			pcie_capability_clear_word(tp->pdev, PCI_EXP_LNKCTL,
5070 						   PCI_EXP_LNKCTL_CLKREQ_EN);
5071 		else
5072 			pcie_capability_set_word(tp->pdev, PCI_EXP_LNKCTL,
5073 						 PCI_EXP_LNKCTL_CLKREQ_EN);
5074 	}
5075 
5076 	tg3_test_and_report_link_chg(tp, current_link_up);
5077 
5078 	return 0;
5079 }
5080 
5081 struct tg3_fiber_aneginfo {
5082 	int state;
5083 #define ANEG_STATE_UNKNOWN		0
5084 #define ANEG_STATE_AN_ENABLE		1
5085 #define ANEG_STATE_RESTART_INIT		2
5086 #define ANEG_STATE_RESTART		3
5087 #define ANEG_STATE_DISABLE_LINK_OK	4
5088 #define ANEG_STATE_ABILITY_DETECT_INIT	5
5089 #define ANEG_STATE_ABILITY_DETECT	6
5090 #define ANEG_STATE_ACK_DETECT_INIT	7
5091 #define ANEG_STATE_ACK_DETECT		8
5092 #define ANEG_STATE_COMPLETE_ACK_INIT	9
5093 #define ANEG_STATE_COMPLETE_ACK		10
5094 #define ANEG_STATE_IDLE_DETECT_INIT	11
5095 #define ANEG_STATE_IDLE_DETECT		12
5096 #define ANEG_STATE_LINK_OK		13
5097 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT	14
5098 #define ANEG_STATE_NEXT_PAGE_WAIT	15
5099 
5100 	u32 flags;
5101 #define MR_AN_ENABLE		0x00000001
5102 #define MR_RESTART_AN		0x00000002
5103 #define MR_AN_COMPLETE		0x00000004
5104 #define MR_PAGE_RX		0x00000008
5105 #define MR_NP_LOADED		0x00000010
5106 #define MR_TOGGLE_TX		0x00000020
5107 #define MR_LP_ADV_FULL_DUPLEX	0x00000040
5108 #define MR_LP_ADV_HALF_DUPLEX	0x00000080
5109 #define MR_LP_ADV_SYM_PAUSE	0x00000100
5110 #define MR_LP_ADV_ASYM_PAUSE	0x00000200
5111 #define MR_LP_ADV_REMOTE_FAULT1	0x00000400
5112 #define MR_LP_ADV_REMOTE_FAULT2	0x00000800
5113 #define MR_LP_ADV_NEXT_PAGE	0x00001000
5114 #define MR_TOGGLE_RX		0x00002000
5115 #define MR_NP_RX		0x00004000
5116 
5117 #define MR_LINK_OK		0x80000000
5118 
5119 	unsigned long link_time, cur_time;
5120 
5121 	u32 ability_match_cfg;
5122 	int ability_match_count;
5123 
5124 	char ability_match, idle_match, ack_match;
5125 
5126 	u32 txconfig, rxconfig;
5127 #define ANEG_CFG_NP		0x00000080
5128 #define ANEG_CFG_ACK		0x00000040
5129 #define ANEG_CFG_RF2		0x00000020
5130 #define ANEG_CFG_RF1		0x00000010
5131 #define ANEG_CFG_PS2		0x00000001
5132 #define ANEG_CFG_PS1		0x00008000
5133 #define ANEG_CFG_HD		0x00004000
5134 #define ANEG_CFG_FD		0x00002000
5135 #define ANEG_CFG_INVAL		0x00001f06
5136 
5137 };
5138 #define ANEG_OK		0
5139 #define ANEG_DONE	1
5140 #define ANEG_TIMER_ENAB	2
5141 #define ANEG_FAILED	-1
5142 
5143 #define ANEG_STATE_SETTLE_TIME	10000
5144 
5145 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
5146 				   struct tg3_fiber_aneginfo *ap)
5147 {
5148 	u16 flowctrl;
5149 	unsigned long delta;
5150 	u32 rx_cfg_reg;
5151 	int ret;
5152 
5153 	if (ap->state == ANEG_STATE_UNKNOWN) {
5154 		ap->rxconfig = 0;
5155 		ap->link_time = 0;
5156 		ap->cur_time = 0;
5157 		ap->ability_match_cfg = 0;
5158 		ap->ability_match_count = 0;
5159 		ap->ability_match = 0;
5160 		ap->idle_match = 0;
5161 		ap->ack_match = 0;
5162 	}
5163 	ap->cur_time++;
5164 
5165 	if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
5166 		rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
5167 
5168 		if (rx_cfg_reg != ap->ability_match_cfg) {
5169 			ap->ability_match_cfg = rx_cfg_reg;
5170 			ap->ability_match = 0;
5171 			ap->ability_match_count = 0;
5172 		} else {
5173 			if (++ap->ability_match_count > 1) {
5174 				ap->ability_match = 1;
5175 				ap->ability_match_cfg = rx_cfg_reg;
5176 			}
5177 		}
5178 		if (rx_cfg_reg & ANEG_CFG_ACK)
5179 			ap->ack_match = 1;
5180 		else
5181 			ap->ack_match = 0;
5182 
5183 		ap->idle_match = 0;
5184 	} else {
5185 		ap->idle_match = 1;
5186 		ap->ability_match_cfg = 0;
5187 		ap->ability_match_count = 0;
5188 		ap->ability_match = 0;
5189 		ap->ack_match = 0;
5190 
5191 		rx_cfg_reg = 0;
5192 	}
5193 
5194 	ap->rxconfig = rx_cfg_reg;
5195 	ret = ANEG_OK;
5196 
5197 	switch (ap->state) {
5198 	case ANEG_STATE_UNKNOWN:
5199 		if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
5200 			ap->state = ANEG_STATE_AN_ENABLE;
5201 
5202 		/* fallthru */
5203 	case ANEG_STATE_AN_ENABLE:
5204 		ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
5205 		if (ap->flags & MR_AN_ENABLE) {
5206 			ap->link_time = 0;
5207 			ap->cur_time = 0;
5208 			ap->ability_match_cfg = 0;
5209 			ap->ability_match_count = 0;
5210 			ap->ability_match = 0;
5211 			ap->idle_match = 0;
5212 			ap->ack_match = 0;
5213 
5214 			ap->state = ANEG_STATE_RESTART_INIT;
5215 		} else {
5216 			ap->state = ANEG_STATE_DISABLE_LINK_OK;
5217 		}
5218 		break;
5219 
5220 	case ANEG_STATE_RESTART_INIT:
5221 		ap->link_time = ap->cur_time;
5222 		ap->flags &= ~(MR_NP_LOADED);
5223 		ap->txconfig = 0;
5224 		tw32(MAC_TX_AUTO_NEG, 0);
5225 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5226 		tw32_f(MAC_MODE, tp->mac_mode);
5227 		udelay(40);
5228 
5229 		ret = ANEG_TIMER_ENAB;
5230 		ap->state = ANEG_STATE_RESTART;
5231 
5232 		/* fallthru */
5233 	case ANEG_STATE_RESTART:
5234 		delta = ap->cur_time - ap->link_time;
5235 		if (delta > ANEG_STATE_SETTLE_TIME)
5236 			ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
5237 		else
5238 			ret = ANEG_TIMER_ENAB;
5239 		break;
5240 
5241 	case ANEG_STATE_DISABLE_LINK_OK:
5242 		ret = ANEG_DONE;
5243 		break;
5244 
5245 	case ANEG_STATE_ABILITY_DETECT_INIT:
5246 		ap->flags &= ~(MR_TOGGLE_TX);
5247 		ap->txconfig = ANEG_CFG_FD;
5248 		flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5249 		if (flowctrl & ADVERTISE_1000XPAUSE)
5250 			ap->txconfig |= ANEG_CFG_PS1;
5251 		if (flowctrl & ADVERTISE_1000XPSE_ASYM)
5252 			ap->txconfig |= ANEG_CFG_PS2;
5253 		tw32(MAC_TX_AUTO_NEG, ap->txconfig);
5254 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5255 		tw32_f(MAC_MODE, tp->mac_mode);
5256 		udelay(40);
5257 
5258 		ap->state = ANEG_STATE_ABILITY_DETECT;
5259 		break;
5260 
5261 	case ANEG_STATE_ABILITY_DETECT:
5262 		if (ap->ability_match != 0 && ap->rxconfig != 0)
5263 			ap->state = ANEG_STATE_ACK_DETECT_INIT;
5264 		break;
5265 
5266 	case ANEG_STATE_ACK_DETECT_INIT:
5267 		ap->txconfig |= ANEG_CFG_ACK;
5268 		tw32(MAC_TX_AUTO_NEG, ap->txconfig);
5269 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5270 		tw32_f(MAC_MODE, tp->mac_mode);
5271 		udelay(40);
5272 
5273 		ap->state = ANEG_STATE_ACK_DETECT;
5274 
5275 		/* fallthru */
5276 	case ANEG_STATE_ACK_DETECT:
5277 		if (ap->ack_match != 0) {
5278 			if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
5279 			    (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
5280 				ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
5281 			} else {
5282 				ap->state = ANEG_STATE_AN_ENABLE;
5283 			}
5284 		} else if (ap->ability_match != 0 &&
5285 			   ap->rxconfig == 0) {
5286 			ap->state = ANEG_STATE_AN_ENABLE;
5287 		}
5288 		break;
5289 
5290 	case ANEG_STATE_COMPLETE_ACK_INIT:
5291 		if (ap->rxconfig & ANEG_CFG_INVAL) {
5292 			ret = ANEG_FAILED;
5293 			break;
5294 		}
5295 		ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
5296 			       MR_LP_ADV_HALF_DUPLEX |
5297 			       MR_LP_ADV_SYM_PAUSE |
5298 			       MR_LP_ADV_ASYM_PAUSE |
5299 			       MR_LP_ADV_REMOTE_FAULT1 |
5300 			       MR_LP_ADV_REMOTE_FAULT2 |
5301 			       MR_LP_ADV_NEXT_PAGE |
5302 			       MR_TOGGLE_RX |
5303 			       MR_NP_RX);
5304 		if (ap->rxconfig & ANEG_CFG_FD)
5305 			ap->flags |= MR_LP_ADV_FULL_DUPLEX;
5306 		if (ap->rxconfig & ANEG_CFG_HD)
5307 			ap->flags |= MR_LP_ADV_HALF_DUPLEX;
5308 		if (ap->rxconfig & ANEG_CFG_PS1)
5309 			ap->flags |= MR_LP_ADV_SYM_PAUSE;
5310 		if (ap->rxconfig & ANEG_CFG_PS2)
5311 			ap->flags |= MR_LP_ADV_ASYM_PAUSE;
5312 		if (ap->rxconfig & ANEG_CFG_RF1)
5313 			ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
5314 		if (ap->rxconfig & ANEG_CFG_RF2)
5315 			ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
5316 		if (ap->rxconfig & ANEG_CFG_NP)
5317 			ap->flags |= MR_LP_ADV_NEXT_PAGE;
5318 
5319 		ap->link_time = ap->cur_time;
5320 
5321 		ap->flags ^= (MR_TOGGLE_TX);
5322 		if (ap->rxconfig & 0x0008)
5323 			ap->flags |= MR_TOGGLE_RX;
5324 		if (ap->rxconfig & ANEG_CFG_NP)
5325 			ap->flags |= MR_NP_RX;
5326 		ap->flags |= MR_PAGE_RX;
5327 
5328 		ap->state = ANEG_STATE_COMPLETE_ACK;
5329 		ret = ANEG_TIMER_ENAB;
5330 		break;
5331 
5332 	case ANEG_STATE_COMPLETE_ACK:
5333 		if (ap->ability_match != 0 &&
5334 		    ap->rxconfig == 0) {
5335 			ap->state = ANEG_STATE_AN_ENABLE;
5336 			break;
5337 		}
5338 		delta = ap->cur_time - ap->link_time;
5339 		if (delta > ANEG_STATE_SETTLE_TIME) {
5340 			if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
5341 				ap->state = ANEG_STATE_IDLE_DETECT_INIT;
5342 			} else {
5343 				if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
5344 				    !(ap->flags & MR_NP_RX)) {
5345 					ap->state = ANEG_STATE_IDLE_DETECT_INIT;
5346 				} else {
5347 					ret = ANEG_FAILED;
5348 				}
5349 			}
5350 		}
5351 		break;
5352 
5353 	case ANEG_STATE_IDLE_DETECT_INIT:
5354 		ap->link_time = ap->cur_time;
5355 		tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
5356 		tw32_f(MAC_MODE, tp->mac_mode);
5357 		udelay(40);
5358 
5359 		ap->state = ANEG_STATE_IDLE_DETECT;
5360 		ret = ANEG_TIMER_ENAB;
5361 		break;
5362 
5363 	case ANEG_STATE_IDLE_DETECT:
5364 		if (ap->ability_match != 0 &&
5365 		    ap->rxconfig == 0) {
5366 			ap->state = ANEG_STATE_AN_ENABLE;
5367 			break;
5368 		}
5369 		delta = ap->cur_time - ap->link_time;
5370 		if (delta > ANEG_STATE_SETTLE_TIME) {
5371 			/* XXX another gem from the Broadcom driver :( */
5372 			ap->state = ANEG_STATE_LINK_OK;
5373 		}
5374 		break;
5375 
5376 	case ANEG_STATE_LINK_OK:
5377 		ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
5378 		ret = ANEG_DONE;
5379 		break;
5380 
5381 	case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
5382 		/* ??? unimplemented */
5383 		break;
5384 
5385 	case ANEG_STATE_NEXT_PAGE_WAIT:
5386 		/* ??? unimplemented */
5387 		break;
5388 
5389 	default:
5390 		ret = ANEG_FAILED;
5391 		break;
5392 	}
5393 
5394 	return ret;
5395 }
5396 
5397 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
5398 {
5399 	int res = 0;
5400 	struct tg3_fiber_aneginfo aninfo;
5401 	int status = ANEG_FAILED;
5402 	unsigned int tick;
5403 	u32 tmp;
5404 
5405 	tw32_f(MAC_TX_AUTO_NEG, 0);
5406 
5407 	tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
5408 	tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
5409 	udelay(40);
5410 
5411 	tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
5412 	udelay(40);
5413 
5414 	memset(&aninfo, 0, sizeof(aninfo));
5415 	aninfo.flags |= MR_AN_ENABLE;
5416 	aninfo.state = ANEG_STATE_UNKNOWN;
5417 	aninfo.cur_time = 0;
5418 	tick = 0;
5419 	while (++tick < 195000) {
5420 		status = tg3_fiber_aneg_smachine(tp, &aninfo);
5421 		if (status == ANEG_DONE || status == ANEG_FAILED)
5422 			break;
5423 
5424 		udelay(1);
5425 	}
5426 
5427 	tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
5428 	tw32_f(MAC_MODE, tp->mac_mode);
5429 	udelay(40);
5430 
5431 	*txflags = aninfo.txconfig;
5432 	*rxflags = aninfo.flags;
5433 
5434 	if (status == ANEG_DONE &&
5435 	    (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
5436 			     MR_LP_ADV_FULL_DUPLEX)))
5437 		res = 1;
5438 
5439 	return res;
5440 }
5441 
5442 static void tg3_init_bcm8002(struct tg3 *tp)
5443 {
5444 	u32 mac_status = tr32(MAC_STATUS);
5445 	int i;
5446 
5447 	/* Reset when initting first time or we have a link. */
5448 	if (tg3_flag(tp, INIT_COMPLETE) &&
5449 	    !(mac_status & MAC_STATUS_PCS_SYNCED))
5450 		return;
5451 
5452 	/* Set PLL lock range. */
5453 	tg3_writephy(tp, 0x16, 0x8007);
5454 
5455 	/* SW reset */
5456 	tg3_writephy(tp, MII_BMCR, BMCR_RESET);
5457 
5458 	/* Wait for reset to complete. */
5459 	/* XXX schedule_timeout() ... */
5460 	for (i = 0; i < 500; i++)
5461 		udelay(10);
5462 
5463 	/* Config mode; select PMA/Ch 1 regs. */
5464 	tg3_writephy(tp, 0x10, 0x8411);
5465 
5466 	/* Enable auto-lock and comdet, select txclk for tx. */
5467 	tg3_writephy(tp, 0x11, 0x0a10);
5468 
5469 	tg3_writephy(tp, 0x18, 0x00a0);
5470 	tg3_writephy(tp, 0x16, 0x41ff);
5471 
5472 	/* Assert and deassert POR. */
5473 	tg3_writephy(tp, 0x13, 0x0400);
5474 	udelay(40);
5475 	tg3_writephy(tp, 0x13, 0x0000);
5476 
5477 	tg3_writephy(tp, 0x11, 0x0a50);
5478 	udelay(40);
5479 	tg3_writephy(tp, 0x11, 0x0a10);
5480 
5481 	/* Wait for signal to stabilize */
5482 	/* XXX schedule_timeout() ... */
5483 	for (i = 0; i < 15000; i++)
5484 		udelay(10);
5485 
5486 	/* Deselect the channel register so we can read the PHYID
5487 	 * later.
5488 	 */
5489 	tg3_writephy(tp, 0x10, 0x8011);
5490 }
5491 
5492 static bool tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
5493 {
5494 	u16 flowctrl;
5495 	bool current_link_up;
5496 	u32 sg_dig_ctrl, sg_dig_status;
5497 	u32 serdes_cfg, expected_sg_dig_ctrl;
5498 	int workaround, port_a;
5499 
5500 	serdes_cfg = 0;
5501 	expected_sg_dig_ctrl = 0;
5502 	workaround = 0;
5503 	port_a = 1;
5504 	current_link_up = false;
5505 
5506 	if (tg3_chip_rev_id(tp) != CHIPREV_ID_5704_A0 &&
5507 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5704_A1) {
5508 		workaround = 1;
5509 		if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
5510 			port_a = 0;
5511 
5512 		/* preserve bits 0-11,13,14 for signal pre-emphasis */
5513 		/* preserve bits 20-23 for voltage regulator */
5514 		serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
5515 	}
5516 
5517 	sg_dig_ctrl = tr32(SG_DIG_CTRL);
5518 
5519 	if (tp->link_config.autoneg != AUTONEG_ENABLE) {
5520 		if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
5521 			if (workaround) {
5522 				u32 val = serdes_cfg;
5523 
5524 				if (port_a)
5525 					val |= 0xc010000;
5526 				else
5527 					val |= 0x4010000;
5528 				tw32_f(MAC_SERDES_CFG, val);
5529 			}
5530 
5531 			tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
5532 		}
5533 		if (mac_status & MAC_STATUS_PCS_SYNCED) {
5534 			tg3_setup_flow_control(tp, 0, 0);
5535 			current_link_up = true;
5536 		}
5537 		goto out;
5538 	}
5539 
5540 	/* Want auto-negotiation.  */
5541 	expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
5542 
5543 	flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5544 	if (flowctrl & ADVERTISE_1000XPAUSE)
5545 		expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
5546 	if (flowctrl & ADVERTISE_1000XPSE_ASYM)
5547 		expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
5548 
5549 	if (sg_dig_ctrl != expected_sg_dig_ctrl) {
5550 		if ((tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT) &&
5551 		    tp->serdes_counter &&
5552 		    ((mac_status & (MAC_STATUS_PCS_SYNCED |
5553 				    MAC_STATUS_RCVD_CFG)) ==
5554 		     MAC_STATUS_PCS_SYNCED)) {
5555 			tp->serdes_counter--;
5556 			current_link_up = true;
5557 			goto out;
5558 		}
5559 restart_autoneg:
5560 		if (workaround)
5561 			tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
5562 		tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
5563 		udelay(5);
5564 		tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
5565 
5566 		tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
5567 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5568 	} else if (mac_status & (MAC_STATUS_PCS_SYNCED |
5569 				 MAC_STATUS_SIGNAL_DET)) {
5570 		sg_dig_status = tr32(SG_DIG_STATUS);
5571 		mac_status = tr32(MAC_STATUS);
5572 
5573 		if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
5574 		    (mac_status & MAC_STATUS_PCS_SYNCED)) {
5575 			u32 local_adv = 0, remote_adv = 0;
5576 
5577 			if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
5578 				local_adv |= ADVERTISE_1000XPAUSE;
5579 			if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
5580 				local_adv |= ADVERTISE_1000XPSE_ASYM;
5581 
5582 			if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
5583 				remote_adv |= LPA_1000XPAUSE;
5584 			if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
5585 				remote_adv |= LPA_1000XPAUSE_ASYM;
5586 
5587 			tp->link_config.rmt_adv =
5588 					   mii_adv_to_ethtool_adv_x(remote_adv);
5589 
5590 			tg3_setup_flow_control(tp, local_adv, remote_adv);
5591 			current_link_up = true;
5592 			tp->serdes_counter = 0;
5593 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5594 		} else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
5595 			if (tp->serdes_counter)
5596 				tp->serdes_counter--;
5597 			else {
5598 				if (workaround) {
5599 					u32 val = serdes_cfg;
5600 
5601 					if (port_a)
5602 						val |= 0xc010000;
5603 					else
5604 						val |= 0x4010000;
5605 
5606 					tw32_f(MAC_SERDES_CFG, val);
5607 				}
5608 
5609 				tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
5610 				udelay(40);
5611 
5612 				/* Link parallel detection - link is up */
5613 				/* only if we have PCS_SYNC and not */
5614 				/* receiving config code words */
5615 				mac_status = tr32(MAC_STATUS);
5616 				if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
5617 				    !(mac_status & MAC_STATUS_RCVD_CFG)) {
5618 					tg3_setup_flow_control(tp, 0, 0);
5619 					current_link_up = true;
5620 					tp->phy_flags |=
5621 						TG3_PHYFLG_PARALLEL_DETECT;
5622 					tp->serdes_counter =
5623 						SERDES_PARALLEL_DET_TIMEOUT;
5624 				} else
5625 					goto restart_autoneg;
5626 			}
5627 		}
5628 	} else {
5629 		tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
5630 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5631 	}
5632 
5633 out:
5634 	return current_link_up;
5635 }
5636 
5637 static bool tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
5638 {
5639 	bool current_link_up = false;
5640 
5641 	if (!(mac_status & MAC_STATUS_PCS_SYNCED))
5642 		goto out;
5643 
5644 	if (tp->link_config.autoneg == AUTONEG_ENABLE) {
5645 		u32 txflags, rxflags;
5646 		int i;
5647 
5648 		if (fiber_autoneg(tp, &txflags, &rxflags)) {
5649 			u32 local_adv = 0, remote_adv = 0;
5650 
5651 			if (txflags & ANEG_CFG_PS1)
5652 				local_adv |= ADVERTISE_1000XPAUSE;
5653 			if (txflags & ANEG_CFG_PS2)
5654 				local_adv |= ADVERTISE_1000XPSE_ASYM;
5655 
5656 			if (rxflags & MR_LP_ADV_SYM_PAUSE)
5657 				remote_adv |= LPA_1000XPAUSE;
5658 			if (rxflags & MR_LP_ADV_ASYM_PAUSE)
5659 				remote_adv |= LPA_1000XPAUSE_ASYM;
5660 
5661 			tp->link_config.rmt_adv =
5662 					   mii_adv_to_ethtool_adv_x(remote_adv);
5663 
5664 			tg3_setup_flow_control(tp, local_adv, remote_adv);
5665 
5666 			current_link_up = true;
5667 		}
5668 		for (i = 0; i < 30; i++) {
5669 			udelay(20);
5670 			tw32_f(MAC_STATUS,
5671 			       (MAC_STATUS_SYNC_CHANGED |
5672 				MAC_STATUS_CFG_CHANGED));
5673 			udelay(40);
5674 			if ((tr32(MAC_STATUS) &
5675 			     (MAC_STATUS_SYNC_CHANGED |
5676 			      MAC_STATUS_CFG_CHANGED)) == 0)
5677 				break;
5678 		}
5679 
5680 		mac_status = tr32(MAC_STATUS);
5681 		if (!current_link_up &&
5682 		    (mac_status & MAC_STATUS_PCS_SYNCED) &&
5683 		    !(mac_status & MAC_STATUS_RCVD_CFG))
5684 			current_link_up = true;
5685 	} else {
5686 		tg3_setup_flow_control(tp, 0, 0);
5687 
5688 		/* Forcing 1000FD link up. */
5689 		current_link_up = true;
5690 
5691 		tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
5692 		udelay(40);
5693 
5694 		tw32_f(MAC_MODE, tp->mac_mode);
5695 		udelay(40);
5696 	}
5697 
5698 out:
5699 	return current_link_up;
5700 }
5701 
5702 static int tg3_setup_fiber_phy(struct tg3 *tp, bool force_reset)
5703 {
5704 	u32 orig_pause_cfg;
5705 	u16 orig_active_speed;
5706 	u8 orig_active_duplex;
5707 	u32 mac_status;
5708 	bool current_link_up;
5709 	int i;
5710 
5711 	orig_pause_cfg = tp->link_config.active_flowctrl;
5712 	orig_active_speed = tp->link_config.active_speed;
5713 	orig_active_duplex = tp->link_config.active_duplex;
5714 
5715 	if (!tg3_flag(tp, HW_AUTONEG) &&
5716 	    tp->link_up &&
5717 	    tg3_flag(tp, INIT_COMPLETE)) {
5718 		mac_status = tr32(MAC_STATUS);
5719 		mac_status &= (MAC_STATUS_PCS_SYNCED |
5720 			       MAC_STATUS_SIGNAL_DET |
5721 			       MAC_STATUS_CFG_CHANGED |
5722 			       MAC_STATUS_RCVD_CFG);
5723 		if (mac_status == (MAC_STATUS_PCS_SYNCED |
5724 				   MAC_STATUS_SIGNAL_DET)) {
5725 			tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
5726 					    MAC_STATUS_CFG_CHANGED));
5727 			return 0;
5728 		}
5729 	}
5730 
5731 	tw32_f(MAC_TX_AUTO_NEG, 0);
5732 
5733 	tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
5734 	tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
5735 	tw32_f(MAC_MODE, tp->mac_mode);
5736 	udelay(40);
5737 
5738 	if (tp->phy_id == TG3_PHY_ID_BCM8002)
5739 		tg3_init_bcm8002(tp);
5740 
5741 	/* Enable link change event even when serdes polling.  */
5742 	tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5743 	udelay(40);
5744 
5745 	current_link_up = false;
5746 	tp->link_config.rmt_adv = 0;
5747 	mac_status = tr32(MAC_STATUS);
5748 
5749 	if (tg3_flag(tp, HW_AUTONEG))
5750 		current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
5751 	else
5752 		current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
5753 
5754 	tp->napi[0].hw_status->status =
5755 		(SD_STATUS_UPDATED |
5756 		 (tp->napi[0].hw_status->status & ~SD_STATUS_LINK_CHG));
5757 
5758 	for (i = 0; i < 100; i++) {
5759 		tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
5760 				    MAC_STATUS_CFG_CHANGED));
5761 		udelay(5);
5762 		if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
5763 					 MAC_STATUS_CFG_CHANGED |
5764 					 MAC_STATUS_LNKSTATE_CHANGED)) == 0)
5765 			break;
5766 	}
5767 
5768 	mac_status = tr32(MAC_STATUS);
5769 	if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
5770 		current_link_up = false;
5771 		if (tp->link_config.autoneg == AUTONEG_ENABLE &&
5772 		    tp->serdes_counter == 0) {
5773 			tw32_f(MAC_MODE, (tp->mac_mode |
5774 					  MAC_MODE_SEND_CONFIGS));
5775 			udelay(1);
5776 			tw32_f(MAC_MODE, tp->mac_mode);
5777 		}
5778 	}
5779 
5780 	if (current_link_up) {
5781 		tp->link_config.active_speed = SPEED_1000;
5782 		tp->link_config.active_duplex = DUPLEX_FULL;
5783 		tw32(MAC_LED_CTRL, (tp->led_ctrl |
5784 				    LED_CTRL_LNKLED_OVERRIDE |
5785 				    LED_CTRL_1000MBPS_ON));
5786 	} else {
5787 		tp->link_config.active_speed = SPEED_UNKNOWN;
5788 		tp->link_config.active_duplex = DUPLEX_UNKNOWN;
5789 		tw32(MAC_LED_CTRL, (tp->led_ctrl |
5790 				    LED_CTRL_LNKLED_OVERRIDE |
5791 				    LED_CTRL_TRAFFIC_OVERRIDE));
5792 	}
5793 
5794 	if (!tg3_test_and_report_link_chg(tp, current_link_up)) {
5795 		u32 now_pause_cfg = tp->link_config.active_flowctrl;
5796 		if (orig_pause_cfg != now_pause_cfg ||
5797 		    orig_active_speed != tp->link_config.active_speed ||
5798 		    orig_active_duplex != tp->link_config.active_duplex)
5799 			tg3_link_report(tp);
5800 	}
5801 
5802 	return 0;
5803 }
5804 
5805 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, bool force_reset)
5806 {
5807 	int err = 0;
5808 	u32 bmsr, bmcr;
5809 	u16 current_speed = SPEED_UNKNOWN;
5810 	u8 current_duplex = DUPLEX_UNKNOWN;
5811 	bool current_link_up = false;
5812 	u32 local_adv, remote_adv, sgsr;
5813 
5814 	if ((tg3_asic_rev(tp) == ASIC_REV_5719 ||
5815 	     tg3_asic_rev(tp) == ASIC_REV_5720) &&
5816 	     !tg3_readphy(tp, SERDES_TG3_1000X_STATUS, &sgsr) &&
5817 	     (sgsr & SERDES_TG3_SGMII_MODE)) {
5818 
5819 		if (force_reset)
5820 			tg3_phy_reset(tp);
5821 
5822 		tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
5823 
5824 		if (!(sgsr & SERDES_TG3_LINK_UP)) {
5825 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5826 		} else {
5827 			current_link_up = true;
5828 			if (sgsr & SERDES_TG3_SPEED_1000) {
5829 				current_speed = SPEED_1000;
5830 				tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5831 			} else if (sgsr & SERDES_TG3_SPEED_100) {
5832 				current_speed = SPEED_100;
5833 				tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
5834 			} else {
5835 				current_speed = SPEED_10;
5836 				tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
5837 			}
5838 
5839 			if (sgsr & SERDES_TG3_FULL_DUPLEX)
5840 				current_duplex = DUPLEX_FULL;
5841 			else
5842 				current_duplex = DUPLEX_HALF;
5843 		}
5844 
5845 		tw32_f(MAC_MODE, tp->mac_mode);
5846 		udelay(40);
5847 
5848 		tg3_clear_mac_status(tp);
5849 
5850 		goto fiber_setup_done;
5851 	}
5852 
5853 	tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5854 	tw32_f(MAC_MODE, tp->mac_mode);
5855 	udelay(40);
5856 
5857 	tg3_clear_mac_status(tp);
5858 
5859 	if (force_reset)
5860 		tg3_phy_reset(tp);
5861 
5862 	tp->link_config.rmt_adv = 0;
5863 
5864 	err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5865 	err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5866 	if (tg3_asic_rev(tp) == ASIC_REV_5714) {
5867 		if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
5868 			bmsr |= BMSR_LSTATUS;
5869 		else
5870 			bmsr &= ~BMSR_LSTATUS;
5871 	}
5872 
5873 	err |= tg3_readphy(tp, MII_BMCR, &bmcr);
5874 
5875 	if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
5876 	    (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
5877 		/* do nothing, just check for link up at the end */
5878 	} else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
5879 		u32 adv, newadv;
5880 
5881 		err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
5882 		newadv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
5883 				 ADVERTISE_1000XPAUSE |
5884 				 ADVERTISE_1000XPSE_ASYM |
5885 				 ADVERTISE_SLCT);
5886 
5887 		newadv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5888 		newadv |= ethtool_adv_to_mii_adv_x(tp->link_config.advertising);
5889 
5890 		if ((newadv != adv) || !(bmcr & BMCR_ANENABLE)) {
5891 			tg3_writephy(tp, MII_ADVERTISE, newadv);
5892 			bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
5893 			tg3_writephy(tp, MII_BMCR, bmcr);
5894 
5895 			tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5896 			tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
5897 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5898 
5899 			return err;
5900 		}
5901 	} else {
5902 		u32 new_bmcr;
5903 
5904 		bmcr &= ~BMCR_SPEED1000;
5905 		new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
5906 
5907 		if (tp->link_config.duplex == DUPLEX_FULL)
5908 			new_bmcr |= BMCR_FULLDPLX;
5909 
5910 		if (new_bmcr != bmcr) {
5911 			/* BMCR_SPEED1000 is a reserved bit that needs
5912 			 * to be set on write.
5913 			 */
5914 			new_bmcr |= BMCR_SPEED1000;
5915 
5916 			/* Force a linkdown */
5917 			if (tp->link_up) {
5918 				u32 adv;
5919 
5920 				err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
5921 				adv &= ~(ADVERTISE_1000XFULL |
5922 					 ADVERTISE_1000XHALF |
5923 					 ADVERTISE_SLCT);
5924 				tg3_writephy(tp, MII_ADVERTISE, adv);
5925 				tg3_writephy(tp, MII_BMCR, bmcr |
5926 							   BMCR_ANRESTART |
5927 							   BMCR_ANENABLE);
5928 				udelay(10);
5929 				tg3_carrier_off(tp);
5930 			}
5931 			tg3_writephy(tp, MII_BMCR, new_bmcr);
5932 			bmcr = new_bmcr;
5933 			err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5934 			err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5935 			if (tg3_asic_rev(tp) == ASIC_REV_5714) {
5936 				if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
5937 					bmsr |= BMSR_LSTATUS;
5938 				else
5939 					bmsr &= ~BMSR_LSTATUS;
5940 			}
5941 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5942 		}
5943 	}
5944 
5945 	if (bmsr & BMSR_LSTATUS) {
5946 		current_speed = SPEED_1000;
5947 		current_link_up = true;
5948 		if (bmcr & BMCR_FULLDPLX)
5949 			current_duplex = DUPLEX_FULL;
5950 		else
5951 			current_duplex = DUPLEX_HALF;
5952 
5953 		local_adv = 0;
5954 		remote_adv = 0;
5955 
5956 		if (bmcr & BMCR_ANENABLE) {
5957 			u32 common;
5958 
5959 			err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
5960 			err |= tg3_readphy(tp, MII_LPA, &remote_adv);
5961 			common = local_adv & remote_adv;
5962 			if (common & (ADVERTISE_1000XHALF |
5963 				      ADVERTISE_1000XFULL)) {
5964 				if (common & ADVERTISE_1000XFULL)
5965 					current_duplex = DUPLEX_FULL;
5966 				else
5967 					current_duplex = DUPLEX_HALF;
5968 
5969 				tp->link_config.rmt_adv =
5970 					   mii_adv_to_ethtool_adv_x(remote_adv);
5971 			} else if (!tg3_flag(tp, 5780_CLASS)) {
5972 				/* Link is up via parallel detect */
5973 			} else {
5974 				current_link_up = false;
5975 			}
5976 		}
5977 	}
5978 
5979 fiber_setup_done:
5980 	if (current_link_up && current_duplex == DUPLEX_FULL)
5981 		tg3_setup_flow_control(tp, local_adv, remote_adv);
5982 
5983 	tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
5984 	if (tp->link_config.active_duplex == DUPLEX_HALF)
5985 		tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
5986 
5987 	tw32_f(MAC_MODE, tp->mac_mode);
5988 	udelay(40);
5989 
5990 	tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5991 
5992 	tp->link_config.active_speed = current_speed;
5993 	tp->link_config.active_duplex = current_duplex;
5994 
5995 	tg3_test_and_report_link_chg(tp, current_link_up);
5996 	return err;
5997 }
5998 
5999 static void tg3_serdes_parallel_detect(struct tg3 *tp)
6000 {
6001 	if (tp->serdes_counter) {
6002 		/* Give autoneg time to complete. */
6003 		tp->serdes_counter--;
6004 		return;
6005 	}
6006 
6007 	if (!tp->link_up &&
6008 	    (tp->link_config.autoneg == AUTONEG_ENABLE)) {
6009 		u32 bmcr;
6010 
6011 		tg3_readphy(tp, MII_BMCR, &bmcr);
6012 		if (bmcr & BMCR_ANENABLE) {
6013 			u32 phy1, phy2;
6014 
6015 			/* Select shadow register 0x1f */
6016 			tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x7c00);
6017 			tg3_readphy(tp, MII_TG3_MISC_SHDW, &phy1);
6018 
6019 			/* Select expansion interrupt status register */
6020 			tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
6021 					 MII_TG3_DSP_EXP1_INT_STAT);
6022 			tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6023 			tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6024 
6025 			if ((phy1 & 0x10) && !(phy2 & 0x20)) {
6026 				/* We have signal detect and not receiving
6027 				 * config code words, link is up by parallel
6028 				 * detection.
6029 				 */
6030 
6031 				bmcr &= ~BMCR_ANENABLE;
6032 				bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
6033 				tg3_writephy(tp, MII_BMCR, bmcr);
6034 				tp->phy_flags |= TG3_PHYFLG_PARALLEL_DETECT;
6035 			}
6036 		}
6037 	} else if (tp->link_up &&
6038 		   (tp->link_config.autoneg == AUTONEG_ENABLE) &&
6039 		   (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
6040 		u32 phy2;
6041 
6042 		/* Select expansion interrupt status register */
6043 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
6044 				 MII_TG3_DSP_EXP1_INT_STAT);
6045 		tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6046 		if (phy2 & 0x20) {
6047 			u32 bmcr;
6048 
6049 			/* Config code words received, turn on autoneg. */
6050 			tg3_readphy(tp, MII_BMCR, &bmcr);
6051 			tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
6052 
6053 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
6054 
6055 		}
6056 	}
6057 }
6058 
6059 static int tg3_setup_phy(struct tg3 *tp, bool force_reset)
6060 {
6061 	u32 val;
6062 	int err;
6063 
6064 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
6065 		err = tg3_setup_fiber_phy(tp, force_reset);
6066 	else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
6067 		err = tg3_setup_fiber_mii_phy(tp, force_reset);
6068 	else
6069 		err = tg3_setup_copper_phy(tp, force_reset);
6070 
6071 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX) {
6072 		u32 scale;
6073 
6074 		val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
6075 		if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
6076 			scale = 65;
6077 		else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
6078 			scale = 6;
6079 		else
6080 			scale = 12;
6081 
6082 		val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
6083 		val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
6084 		tw32(GRC_MISC_CFG, val);
6085 	}
6086 
6087 	val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
6088 	      (6 << TX_LENGTHS_IPG_SHIFT);
6089 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
6090 	    tg3_asic_rev(tp) == ASIC_REV_5762)
6091 		val |= tr32(MAC_TX_LENGTHS) &
6092 		       (TX_LENGTHS_JMB_FRM_LEN_MSK |
6093 			TX_LENGTHS_CNT_DWN_VAL_MSK);
6094 
6095 	if (tp->link_config.active_speed == SPEED_1000 &&
6096 	    tp->link_config.active_duplex == DUPLEX_HALF)
6097 		tw32(MAC_TX_LENGTHS, val |
6098 		     (0xff << TX_LENGTHS_SLOT_TIME_SHIFT));
6099 	else
6100 		tw32(MAC_TX_LENGTHS, val |
6101 		     (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
6102 
6103 	if (!tg3_flag(tp, 5705_PLUS)) {
6104 		if (tp->link_up) {
6105 			tw32(HOSTCC_STAT_COAL_TICKS,
6106 			     tp->coal.stats_block_coalesce_usecs);
6107 		} else {
6108 			tw32(HOSTCC_STAT_COAL_TICKS, 0);
6109 		}
6110 	}
6111 
6112 	if (tg3_flag(tp, ASPM_WORKAROUND)) {
6113 		val = tr32(PCIE_PWR_MGMT_THRESH);
6114 		if (!tp->link_up)
6115 			val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
6116 			      tp->pwrmgmt_thresh;
6117 		else
6118 			val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
6119 		tw32(PCIE_PWR_MGMT_THRESH, val);
6120 	}
6121 
6122 	return err;
6123 }
6124 
6125 /* tp->lock must be held */
6126 static u64 tg3_refclk_read(struct tg3 *tp)
6127 {
6128 	u64 stamp = tr32(TG3_EAV_REF_CLCK_LSB);
6129 	return stamp | (u64)tr32(TG3_EAV_REF_CLCK_MSB) << 32;
6130 }
6131 
6132 /* tp->lock must be held */
6133 static void tg3_refclk_write(struct tg3 *tp, u64 newval)
6134 {
6135 	u32 clock_ctl = tr32(TG3_EAV_REF_CLCK_CTL);
6136 
6137 	tw32(TG3_EAV_REF_CLCK_CTL, clock_ctl | TG3_EAV_REF_CLCK_CTL_STOP);
6138 	tw32(TG3_EAV_REF_CLCK_LSB, newval & 0xffffffff);
6139 	tw32(TG3_EAV_REF_CLCK_MSB, newval >> 32);
6140 	tw32_f(TG3_EAV_REF_CLCK_CTL, clock_ctl | TG3_EAV_REF_CLCK_CTL_RESUME);
6141 }
6142 
6143 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync);
6144 static inline void tg3_full_unlock(struct tg3 *tp);
6145 static int tg3_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info)
6146 {
6147 	struct tg3 *tp = netdev_priv(dev);
6148 
6149 	info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
6150 				SOF_TIMESTAMPING_RX_SOFTWARE |
6151 				SOF_TIMESTAMPING_SOFTWARE;
6152 
6153 	if (tg3_flag(tp, PTP_CAPABLE)) {
6154 		info->so_timestamping |= SOF_TIMESTAMPING_TX_HARDWARE |
6155 					SOF_TIMESTAMPING_RX_HARDWARE |
6156 					SOF_TIMESTAMPING_RAW_HARDWARE;
6157 	}
6158 
6159 	if (tp->ptp_clock)
6160 		info->phc_index = ptp_clock_index(tp->ptp_clock);
6161 	else
6162 		info->phc_index = -1;
6163 
6164 	info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
6165 
6166 	info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
6167 			   (1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
6168 			   (1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
6169 			   (1 << HWTSTAMP_FILTER_PTP_V2_L4_EVENT);
6170 	return 0;
6171 }
6172 
6173 static int tg3_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
6174 {
6175 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6176 	bool neg_adj = false;
6177 	u32 correction = 0;
6178 
6179 	if (ppb < 0) {
6180 		neg_adj = true;
6181 		ppb = -ppb;
6182 	}
6183 
6184 	/* Frequency adjustment is performed using hardware with a 24 bit
6185 	 * accumulator and a programmable correction value. On each clk, the
6186 	 * correction value gets added to the accumulator and when it
6187 	 * overflows, the time counter is incremented/decremented.
6188 	 *
6189 	 * So conversion from ppb to correction value is
6190 	 *		ppb * (1 << 24) / 1000000000
6191 	 */
6192 	correction = div_u64((u64)ppb * (1 << 24), 1000000000ULL) &
6193 		     TG3_EAV_REF_CLK_CORRECT_MASK;
6194 
6195 	tg3_full_lock(tp, 0);
6196 
6197 	if (correction)
6198 		tw32(TG3_EAV_REF_CLK_CORRECT_CTL,
6199 		     TG3_EAV_REF_CLK_CORRECT_EN |
6200 		     (neg_adj ? TG3_EAV_REF_CLK_CORRECT_NEG : 0) | correction);
6201 	else
6202 		tw32(TG3_EAV_REF_CLK_CORRECT_CTL, 0);
6203 
6204 	tg3_full_unlock(tp);
6205 
6206 	return 0;
6207 }
6208 
6209 static int tg3_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
6210 {
6211 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6212 
6213 	tg3_full_lock(tp, 0);
6214 	tp->ptp_adjust += delta;
6215 	tg3_full_unlock(tp);
6216 
6217 	return 0;
6218 }
6219 
6220 static int tg3_ptp_gettime(struct ptp_clock_info *ptp, struct timespec *ts)
6221 {
6222 	u64 ns;
6223 	u32 remainder;
6224 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6225 
6226 	tg3_full_lock(tp, 0);
6227 	ns = tg3_refclk_read(tp);
6228 	ns += tp->ptp_adjust;
6229 	tg3_full_unlock(tp);
6230 
6231 	ts->tv_sec = div_u64_rem(ns, 1000000000, &remainder);
6232 	ts->tv_nsec = remainder;
6233 
6234 	return 0;
6235 }
6236 
6237 static int tg3_ptp_settime(struct ptp_clock_info *ptp,
6238 			   const struct timespec *ts)
6239 {
6240 	u64 ns;
6241 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6242 
6243 	ns = timespec_to_ns(ts);
6244 
6245 	tg3_full_lock(tp, 0);
6246 	tg3_refclk_write(tp, ns);
6247 	tp->ptp_adjust = 0;
6248 	tg3_full_unlock(tp);
6249 
6250 	return 0;
6251 }
6252 
6253 static int tg3_ptp_enable(struct ptp_clock_info *ptp,
6254 			  struct ptp_clock_request *rq, int on)
6255 {
6256 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6257 	u32 clock_ctl;
6258 	int rval = 0;
6259 
6260 	switch (rq->type) {
6261 	case PTP_CLK_REQ_PEROUT:
6262 		if (rq->perout.index != 0)
6263 			return -EINVAL;
6264 
6265 		tg3_full_lock(tp, 0);
6266 		clock_ctl = tr32(TG3_EAV_REF_CLCK_CTL);
6267 		clock_ctl &= ~TG3_EAV_CTL_TSYNC_GPIO_MASK;
6268 
6269 		if (on) {
6270 			u64 nsec;
6271 
6272 			nsec = rq->perout.start.sec * 1000000000ULL +
6273 			       rq->perout.start.nsec;
6274 
6275 			if (rq->perout.period.sec || rq->perout.period.nsec) {
6276 				netdev_warn(tp->dev,
6277 					    "Device supports only a one-shot timesync output, period must be 0\n");
6278 				rval = -EINVAL;
6279 				goto err_out;
6280 			}
6281 
6282 			if (nsec & (1ULL << 63)) {
6283 				netdev_warn(tp->dev,
6284 					    "Start value (nsec) is over limit. Maximum size of start is only 63 bits\n");
6285 				rval = -EINVAL;
6286 				goto err_out;
6287 			}
6288 
6289 			tw32(TG3_EAV_WATCHDOG0_LSB, (nsec & 0xffffffff));
6290 			tw32(TG3_EAV_WATCHDOG0_MSB,
6291 			     TG3_EAV_WATCHDOG0_EN |
6292 			     ((nsec >> 32) & TG3_EAV_WATCHDOG_MSB_MASK));
6293 
6294 			tw32(TG3_EAV_REF_CLCK_CTL,
6295 			     clock_ctl | TG3_EAV_CTL_TSYNC_WDOG0);
6296 		} else {
6297 			tw32(TG3_EAV_WATCHDOG0_MSB, 0);
6298 			tw32(TG3_EAV_REF_CLCK_CTL, clock_ctl);
6299 		}
6300 
6301 err_out:
6302 		tg3_full_unlock(tp);
6303 		return rval;
6304 
6305 	default:
6306 		break;
6307 	}
6308 
6309 	return -EOPNOTSUPP;
6310 }
6311 
6312 static const struct ptp_clock_info tg3_ptp_caps = {
6313 	.owner		= THIS_MODULE,
6314 	.name		= "tg3 clock",
6315 	.max_adj	= 250000000,
6316 	.n_alarm	= 0,
6317 	.n_ext_ts	= 0,
6318 	.n_per_out	= 1,
6319 	.n_pins		= 0,
6320 	.pps		= 0,
6321 	.adjfreq	= tg3_ptp_adjfreq,
6322 	.adjtime	= tg3_ptp_adjtime,
6323 	.gettime	= tg3_ptp_gettime,
6324 	.settime	= tg3_ptp_settime,
6325 	.enable		= tg3_ptp_enable,
6326 };
6327 
6328 static void tg3_hwclock_to_timestamp(struct tg3 *tp, u64 hwclock,
6329 				     struct skb_shared_hwtstamps *timestamp)
6330 {
6331 	memset(timestamp, 0, sizeof(struct skb_shared_hwtstamps));
6332 	timestamp->hwtstamp  = ns_to_ktime((hwclock & TG3_TSTAMP_MASK) +
6333 					   tp->ptp_adjust);
6334 }
6335 
6336 /* tp->lock must be held */
6337 static void tg3_ptp_init(struct tg3 *tp)
6338 {
6339 	if (!tg3_flag(tp, PTP_CAPABLE))
6340 		return;
6341 
6342 	/* Initialize the hardware clock to the system time. */
6343 	tg3_refclk_write(tp, ktime_to_ns(ktime_get_real()));
6344 	tp->ptp_adjust = 0;
6345 	tp->ptp_info = tg3_ptp_caps;
6346 }
6347 
6348 /* tp->lock must be held */
6349 static void tg3_ptp_resume(struct tg3 *tp)
6350 {
6351 	if (!tg3_flag(tp, PTP_CAPABLE))
6352 		return;
6353 
6354 	tg3_refclk_write(tp, ktime_to_ns(ktime_get_real()) + tp->ptp_adjust);
6355 	tp->ptp_adjust = 0;
6356 }
6357 
6358 static void tg3_ptp_fini(struct tg3 *tp)
6359 {
6360 	if (!tg3_flag(tp, PTP_CAPABLE) || !tp->ptp_clock)
6361 		return;
6362 
6363 	ptp_clock_unregister(tp->ptp_clock);
6364 	tp->ptp_clock = NULL;
6365 	tp->ptp_adjust = 0;
6366 }
6367 
6368 static inline int tg3_irq_sync(struct tg3 *tp)
6369 {
6370 	return tp->irq_sync;
6371 }
6372 
6373 static inline void tg3_rd32_loop(struct tg3 *tp, u32 *dst, u32 off, u32 len)
6374 {
6375 	int i;
6376 
6377 	dst = (u32 *)((u8 *)dst + off);
6378 	for (i = 0; i < len; i += sizeof(u32))
6379 		*dst++ = tr32(off + i);
6380 }
6381 
6382 static void tg3_dump_legacy_regs(struct tg3 *tp, u32 *regs)
6383 {
6384 	tg3_rd32_loop(tp, regs, TG3PCI_VENDOR, 0xb0);
6385 	tg3_rd32_loop(tp, regs, MAILBOX_INTERRUPT_0, 0x200);
6386 	tg3_rd32_loop(tp, regs, MAC_MODE, 0x4f0);
6387 	tg3_rd32_loop(tp, regs, SNDDATAI_MODE, 0xe0);
6388 	tg3_rd32_loop(tp, regs, SNDDATAC_MODE, 0x04);
6389 	tg3_rd32_loop(tp, regs, SNDBDS_MODE, 0x80);
6390 	tg3_rd32_loop(tp, regs, SNDBDI_MODE, 0x48);
6391 	tg3_rd32_loop(tp, regs, SNDBDC_MODE, 0x04);
6392 	tg3_rd32_loop(tp, regs, RCVLPC_MODE, 0x20);
6393 	tg3_rd32_loop(tp, regs, RCVLPC_SELLST_BASE, 0x15c);
6394 	tg3_rd32_loop(tp, regs, RCVDBDI_MODE, 0x0c);
6395 	tg3_rd32_loop(tp, regs, RCVDBDI_JUMBO_BD, 0x3c);
6396 	tg3_rd32_loop(tp, regs, RCVDBDI_BD_PROD_IDX_0, 0x44);
6397 	tg3_rd32_loop(tp, regs, RCVDCC_MODE, 0x04);
6398 	tg3_rd32_loop(tp, regs, RCVBDI_MODE, 0x20);
6399 	tg3_rd32_loop(tp, regs, RCVCC_MODE, 0x14);
6400 	tg3_rd32_loop(tp, regs, RCVLSC_MODE, 0x08);
6401 	tg3_rd32_loop(tp, regs, MBFREE_MODE, 0x08);
6402 	tg3_rd32_loop(tp, regs, HOSTCC_MODE, 0x100);
6403 
6404 	if (tg3_flag(tp, SUPPORT_MSIX))
6405 		tg3_rd32_loop(tp, regs, HOSTCC_RXCOL_TICKS_VEC1, 0x180);
6406 
6407 	tg3_rd32_loop(tp, regs, MEMARB_MODE, 0x10);
6408 	tg3_rd32_loop(tp, regs, BUFMGR_MODE, 0x58);
6409 	tg3_rd32_loop(tp, regs, RDMAC_MODE, 0x08);
6410 	tg3_rd32_loop(tp, regs, WDMAC_MODE, 0x08);
6411 	tg3_rd32_loop(tp, regs, RX_CPU_MODE, 0x04);
6412 	tg3_rd32_loop(tp, regs, RX_CPU_STATE, 0x04);
6413 	tg3_rd32_loop(tp, regs, RX_CPU_PGMCTR, 0x04);
6414 	tg3_rd32_loop(tp, regs, RX_CPU_HWBKPT, 0x04);
6415 
6416 	if (!tg3_flag(tp, 5705_PLUS)) {
6417 		tg3_rd32_loop(tp, regs, TX_CPU_MODE, 0x04);
6418 		tg3_rd32_loop(tp, regs, TX_CPU_STATE, 0x04);
6419 		tg3_rd32_loop(tp, regs, TX_CPU_PGMCTR, 0x04);
6420 	}
6421 
6422 	tg3_rd32_loop(tp, regs, GRCMBOX_INTERRUPT_0, 0x110);
6423 	tg3_rd32_loop(tp, regs, FTQ_RESET, 0x120);
6424 	tg3_rd32_loop(tp, regs, MSGINT_MODE, 0x0c);
6425 	tg3_rd32_loop(tp, regs, DMAC_MODE, 0x04);
6426 	tg3_rd32_loop(tp, regs, GRC_MODE, 0x4c);
6427 
6428 	if (tg3_flag(tp, NVRAM))
6429 		tg3_rd32_loop(tp, regs, NVRAM_CMD, 0x24);
6430 }
6431 
6432 static void tg3_dump_state(struct tg3 *tp)
6433 {
6434 	int i;
6435 	u32 *regs;
6436 
6437 	regs = kzalloc(TG3_REG_BLK_SIZE, GFP_ATOMIC);
6438 	if (!regs)
6439 		return;
6440 
6441 	if (tg3_flag(tp, PCI_EXPRESS)) {
6442 		/* Read up to but not including private PCI registers */
6443 		for (i = 0; i < TG3_PCIE_TLDLPL_PORT; i += sizeof(u32))
6444 			regs[i / sizeof(u32)] = tr32(i);
6445 	} else
6446 		tg3_dump_legacy_regs(tp, regs);
6447 
6448 	for (i = 0; i < TG3_REG_BLK_SIZE / sizeof(u32); i += 4) {
6449 		if (!regs[i + 0] && !regs[i + 1] &&
6450 		    !regs[i + 2] && !regs[i + 3])
6451 			continue;
6452 
6453 		netdev_err(tp->dev, "0x%08x: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
6454 			   i * 4,
6455 			   regs[i + 0], regs[i + 1], regs[i + 2], regs[i + 3]);
6456 	}
6457 
6458 	kfree(regs);
6459 
6460 	for (i = 0; i < tp->irq_cnt; i++) {
6461 		struct tg3_napi *tnapi = &tp->napi[i];
6462 
6463 		/* SW status block */
6464 		netdev_err(tp->dev,
6465 			 "%d: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
6466 			   i,
6467 			   tnapi->hw_status->status,
6468 			   tnapi->hw_status->status_tag,
6469 			   tnapi->hw_status->rx_jumbo_consumer,
6470 			   tnapi->hw_status->rx_consumer,
6471 			   tnapi->hw_status->rx_mini_consumer,
6472 			   tnapi->hw_status->idx[0].rx_producer,
6473 			   tnapi->hw_status->idx[0].tx_consumer);
6474 
6475 		netdev_err(tp->dev,
6476 		"%d: NAPI info [%08x:%08x:(%04x:%04x:%04x):%04x:(%04x:%04x:%04x:%04x)]\n",
6477 			   i,
6478 			   tnapi->last_tag, tnapi->last_irq_tag,
6479 			   tnapi->tx_prod, tnapi->tx_cons, tnapi->tx_pending,
6480 			   tnapi->rx_rcb_ptr,
6481 			   tnapi->prodring.rx_std_prod_idx,
6482 			   tnapi->prodring.rx_std_cons_idx,
6483 			   tnapi->prodring.rx_jmb_prod_idx,
6484 			   tnapi->prodring.rx_jmb_cons_idx);
6485 	}
6486 }
6487 
6488 /* This is called whenever we suspect that the system chipset is re-
6489  * ordering the sequence of MMIO to the tx send mailbox. The symptom
6490  * is bogus tx completions. We try to recover by setting the
6491  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
6492  * in the workqueue.
6493  */
6494 static void tg3_tx_recover(struct tg3 *tp)
6495 {
6496 	BUG_ON(tg3_flag(tp, MBOX_WRITE_REORDER) ||
6497 	       tp->write32_tx_mbox == tg3_write_indirect_mbox);
6498 
6499 	netdev_warn(tp->dev,
6500 		    "The system may be re-ordering memory-mapped I/O "
6501 		    "cycles to the network device, attempting to recover. "
6502 		    "Please report the problem to the driver maintainer "
6503 		    "and include system chipset information.\n");
6504 
6505 	tg3_flag_set(tp, TX_RECOVERY_PENDING);
6506 }
6507 
6508 static inline u32 tg3_tx_avail(struct tg3_napi *tnapi)
6509 {
6510 	/* Tell compiler to fetch tx indices from memory. */
6511 	barrier();
6512 	return tnapi->tx_pending -
6513 	       ((tnapi->tx_prod - tnapi->tx_cons) & (TG3_TX_RING_SIZE - 1));
6514 }
6515 
6516 /* Tigon3 never reports partial packet sends.  So we do not
6517  * need special logic to handle SKBs that have not had all
6518  * of their frags sent yet, like SunGEM does.
6519  */
6520 static void tg3_tx(struct tg3_napi *tnapi)
6521 {
6522 	struct tg3 *tp = tnapi->tp;
6523 	u32 hw_idx = tnapi->hw_status->idx[0].tx_consumer;
6524 	u32 sw_idx = tnapi->tx_cons;
6525 	struct netdev_queue *txq;
6526 	int index = tnapi - tp->napi;
6527 	unsigned int pkts_compl = 0, bytes_compl = 0;
6528 
6529 	if (tg3_flag(tp, ENABLE_TSS))
6530 		index--;
6531 
6532 	txq = netdev_get_tx_queue(tp->dev, index);
6533 
6534 	while (sw_idx != hw_idx) {
6535 		struct tg3_tx_ring_info *ri = &tnapi->tx_buffers[sw_idx];
6536 		struct sk_buff *skb = ri->skb;
6537 		int i, tx_bug = 0;
6538 
6539 		if (unlikely(skb == NULL)) {
6540 			tg3_tx_recover(tp);
6541 			return;
6542 		}
6543 
6544 		if (tnapi->tx_ring[sw_idx].len_flags & TXD_FLAG_HWTSTAMP) {
6545 			struct skb_shared_hwtstamps timestamp;
6546 			u64 hwclock = tr32(TG3_TX_TSTAMP_LSB);
6547 			hwclock |= (u64)tr32(TG3_TX_TSTAMP_MSB) << 32;
6548 
6549 			tg3_hwclock_to_timestamp(tp, hwclock, &timestamp);
6550 
6551 			skb_tstamp_tx(skb, &timestamp);
6552 		}
6553 
6554 		pci_unmap_single(tp->pdev,
6555 				 dma_unmap_addr(ri, mapping),
6556 				 skb_headlen(skb),
6557 				 PCI_DMA_TODEVICE);
6558 
6559 		ri->skb = NULL;
6560 
6561 		while (ri->fragmented) {
6562 			ri->fragmented = false;
6563 			sw_idx = NEXT_TX(sw_idx);
6564 			ri = &tnapi->tx_buffers[sw_idx];
6565 		}
6566 
6567 		sw_idx = NEXT_TX(sw_idx);
6568 
6569 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
6570 			ri = &tnapi->tx_buffers[sw_idx];
6571 			if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
6572 				tx_bug = 1;
6573 
6574 			pci_unmap_page(tp->pdev,
6575 				       dma_unmap_addr(ri, mapping),
6576 				       skb_frag_size(&skb_shinfo(skb)->frags[i]),
6577 				       PCI_DMA_TODEVICE);
6578 
6579 			while (ri->fragmented) {
6580 				ri->fragmented = false;
6581 				sw_idx = NEXT_TX(sw_idx);
6582 				ri = &tnapi->tx_buffers[sw_idx];
6583 			}
6584 
6585 			sw_idx = NEXT_TX(sw_idx);
6586 		}
6587 
6588 		pkts_compl++;
6589 		bytes_compl += skb->len;
6590 
6591 		dev_kfree_skb_any(skb);
6592 
6593 		if (unlikely(tx_bug)) {
6594 			tg3_tx_recover(tp);
6595 			return;
6596 		}
6597 	}
6598 
6599 	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
6600 
6601 	tnapi->tx_cons = sw_idx;
6602 
6603 	/* Need to make the tx_cons update visible to tg3_start_xmit()
6604 	 * before checking for netif_queue_stopped().  Without the
6605 	 * memory barrier, there is a small possibility that tg3_start_xmit()
6606 	 * will miss it and cause the queue to be stopped forever.
6607 	 */
6608 	smp_mb();
6609 
6610 	if (unlikely(netif_tx_queue_stopped(txq) &&
6611 		     (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))) {
6612 		__netif_tx_lock(txq, smp_processor_id());
6613 		if (netif_tx_queue_stopped(txq) &&
6614 		    (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))
6615 			netif_tx_wake_queue(txq);
6616 		__netif_tx_unlock(txq);
6617 	}
6618 }
6619 
6620 static void tg3_frag_free(bool is_frag, void *data)
6621 {
6622 	if (is_frag)
6623 		put_page(virt_to_head_page(data));
6624 	else
6625 		kfree(data);
6626 }
6627 
6628 static void tg3_rx_data_free(struct tg3 *tp, struct ring_info *ri, u32 map_sz)
6629 {
6630 	unsigned int skb_size = SKB_DATA_ALIGN(map_sz + TG3_RX_OFFSET(tp)) +
6631 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
6632 
6633 	if (!ri->data)
6634 		return;
6635 
6636 	pci_unmap_single(tp->pdev, dma_unmap_addr(ri, mapping),
6637 			 map_sz, PCI_DMA_FROMDEVICE);
6638 	tg3_frag_free(skb_size <= PAGE_SIZE, ri->data);
6639 	ri->data = NULL;
6640 }
6641 
6642 
6643 /* Returns size of skb allocated or < 0 on error.
6644  *
6645  * We only need to fill in the address because the other members
6646  * of the RX descriptor are invariant, see tg3_init_rings.
6647  *
6648  * Note the purposeful assymetry of cpu vs. chip accesses.  For
6649  * posting buffers we only dirty the first cache line of the RX
6650  * descriptor (containing the address).  Whereas for the RX status
6651  * buffers the cpu only reads the last cacheline of the RX descriptor
6652  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
6653  */
6654 static int tg3_alloc_rx_data(struct tg3 *tp, struct tg3_rx_prodring_set *tpr,
6655 			     u32 opaque_key, u32 dest_idx_unmasked,
6656 			     unsigned int *frag_size)
6657 {
6658 	struct tg3_rx_buffer_desc *desc;
6659 	struct ring_info *map;
6660 	u8 *data;
6661 	dma_addr_t mapping;
6662 	int skb_size, data_size, dest_idx;
6663 
6664 	switch (opaque_key) {
6665 	case RXD_OPAQUE_RING_STD:
6666 		dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
6667 		desc = &tpr->rx_std[dest_idx];
6668 		map = &tpr->rx_std_buffers[dest_idx];
6669 		data_size = tp->rx_pkt_map_sz;
6670 		break;
6671 
6672 	case RXD_OPAQUE_RING_JUMBO:
6673 		dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
6674 		desc = &tpr->rx_jmb[dest_idx].std;
6675 		map = &tpr->rx_jmb_buffers[dest_idx];
6676 		data_size = TG3_RX_JMB_MAP_SZ;
6677 		break;
6678 
6679 	default:
6680 		return -EINVAL;
6681 	}
6682 
6683 	/* Do not overwrite any of the map or rp information
6684 	 * until we are sure we can commit to a new buffer.
6685 	 *
6686 	 * Callers depend upon this behavior and assume that
6687 	 * we leave everything unchanged if we fail.
6688 	 */
6689 	skb_size = SKB_DATA_ALIGN(data_size + TG3_RX_OFFSET(tp)) +
6690 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
6691 	if (skb_size <= PAGE_SIZE) {
6692 		data = netdev_alloc_frag(skb_size);
6693 		*frag_size = skb_size;
6694 	} else {
6695 		data = kmalloc(skb_size, GFP_ATOMIC);
6696 		*frag_size = 0;
6697 	}
6698 	if (!data)
6699 		return -ENOMEM;
6700 
6701 	mapping = pci_map_single(tp->pdev,
6702 				 data + TG3_RX_OFFSET(tp),
6703 				 data_size,
6704 				 PCI_DMA_FROMDEVICE);
6705 	if (unlikely(pci_dma_mapping_error(tp->pdev, mapping))) {
6706 		tg3_frag_free(skb_size <= PAGE_SIZE, data);
6707 		return -EIO;
6708 	}
6709 
6710 	map->data = data;
6711 	dma_unmap_addr_set(map, mapping, mapping);
6712 
6713 	desc->addr_hi = ((u64)mapping >> 32);
6714 	desc->addr_lo = ((u64)mapping & 0xffffffff);
6715 
6716 	return data_size;
6717 }
6718 
6719 /* We only need to move over in the address because the other
6720  * members of the RX descriptor are invariant.  See notes above
6721  * tg3_alloc_rx_data for full details.
6722  */
6723 static void tg3_recycle_rx(struct tg3_napi *tnapi,
6724 			   struct tg3_rx_prodring_set *dpr,
6725 			   u32 opaque_key, int src_idx,
6726 			   u32 dest_idx_unmasked)
6727 {
6728 	struct tg3 *tp = tnapi->tp;
6729 	struct tg3_rx_buffer_desc *src_desc, *dest_desc;
6730 	struct ring_info *src_map, *dest_map;
6731 	struct tg3_rx_prodring_set *spr = &tp->napi[0].prodring;
6732 	int dest_idx;
6733 
6734 	switch (opaque_key) {
6735 	case RXD_OPAQUE_RING_STD:
6736 		dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
6737 		dest_desc = &dpr->rx_std[dest_idx];
6738 		dest_map = &dpr->rx_std_buffers[dest_idx];
6739 		src_desc = &spr->rx_std[src_idx];
6740 		src_map = &spr->rx_std_buffers[src_idx];
6741 		break;
6742 
6743 	case RXD_OPAQUE_RING_JUMBO:
6744 		dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
6745 		dest_desc = &dpr->rx_jmb[dest_idx].std;
6746 		dest_map = &dpr->rx_jmb_buffers[dest_idx];
6747 		src_desc = &spr->rx_jmb[src_idx].std;
6748 		src_map = &spr->rx_jmb_buffers[src_idx];
6749 		break;
6750 
6751 	default:
6752 		return;
6753 	}
6754 
6755 	dest_map->data = src_map->data;
6756 	dma_unmap_addr_set(dest_map, mapping,
6757 			   dma_unmap_addr(src_map, mapping));
6758 	dest_desc->addr_hi = src_desc->addr_hi;
6759 	dest_desc->addr_lo = src_desc->addr_lo;
6760 
6761 	/* Ensure that the update to the skb happens after the physical
6762 	 * addresses have been transferred to the new BD location.
6763 	 */
6764 	smp_wmb();
6765 
6766 	src_map->data = NULL;
6767 }
6768 
6769 /* The RX ring scheme is composed of multiple rings which post fresh
6770  * buffers to the chip, and one special ring the chip uses to report
6771  * status back to the host.
6772  *
6773  * The special ring reports the status of received packets to the
6774  * host.  The chip does not write into the original descriptor the
6775  * RX buffer was obtained from.  The chip simply takes the original
6776  * descriptor as provided by the host, updates the status and length
6777  * field, then writes this into the next status ring entry.
6778  *
6779  * Each ring the host uses to post buffers to the chip is described
6780  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
6781  * it is first placed into the on-chip ram.  When the packet's length
6782  * is known, it walks down the TG3_BDINFO entries to select the ring.
6783  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
6784  * which is within the range of the new packet's length is chosen.
6785  *
6786  * The "separate ring for rx status" scheme may sound queer, but it makes
6787  * sense from a cache coherency perspective.  If only the host writes
6788  * to the buffer post rings, and only the chip writes to the rx status
6789  * rings, then cache lines never move beyond shared-modified state.
6790  * If both the host and chip were to write into the same ring, cache line
6791  * eviction could occur since both entities want it in an exclusive state.
6792  */
6793 static int tg3_rx(struct tg3_napi *tnapi, int budget)
6794 {
6795 	struct tg3 *tp = tnapi->tp;
6796 	u32 work_mask, rx_std_posted = 0;
6797 	u32 std_prod_idx, jmb_prod_idx;
6798 	u32 sw_idx = tnapi->rx_rcb_ptr;
6799 	u16 hw_idx;
6800 	int received;
6801 	struct tg3_rx_prodring_set *tpr = &tnapi->prodring;
6802 
6803 	hw_idx = *(tnapi->rx_rcb_prod_idx);
6804 	/*
6805 	 * We need to order the read of hw_idx and the read of
6806 	 * the opaque cookie.
6807 	 */
6808 	rmb();
6809 	work_mask = 0;
6810 	received = 0;
6811 	std_prod_idx = tpr->rx_std_prod_idx;
6812 	jmb_prod_idx = tpr->rx_jmb_prod_idx;
6813 	while (sw_idx != hw_idx && budget > 0) {
6814 		struct ring_info *ri;
6815 		struct tg3_rx_buffer_desc *desc = &tnapi->rx_rcb[sw_idx];
6816 		unsigned int len;
6817 		struct sk_buff *skb;
6818 		dma_addr_t dma_addr;
6819 		u32 opaque_key, desc_idx, *post_ptr;
6820 		u8 *data;
6821 		u64 tstamp = 0;
6822 
6823 		desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
6824 		opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
6825 		if (opaque_key == RXD_OPAQUE_RING_STD) {
6826 			ri = &tp->napi[0].prodring.rx_std_buffers[desc_idx];
6827 			dma_addr = dma_unmap_addr(ri, mapping);
6828 			data = ri->data;
6829 			post_ptr = &std_prod_idx;
6830 			rx_std_posted++;
6831 		} else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
6832 			ri = &tp->napi[0].prodring.rx_jmb_buffers[desc_idx];
6833 			dma_addr = dma_unmap_addr(ri, mapping);
6834 			data = ri->data;
6835 			post_ptr = &jmb_prod_idx;
6836 		} else
6837 			goto next_pkt_nopost;
6838 
6839 		work_mask |= opaque_key;
6840 
6841 		if (desc->err_vlan & RXD_ERR_MASK) {
6842 		drop_it:
6843 			tg3_recycle_rx(tnapi, tpr, opaque_key,
6844 				       desc_idx, *post_ptr);
6845 		drop_it_no_recycle:
6846 			/* Other statistics kept track of by card. */
6847 			tp->rx_dropped++;
6848 			goto next_pkt;
6849 		}
6850 
6851 		prefetch(data + TG3_RX_OFFSET(tp));
6852 		len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) -
6853 		      ETH_FCS_LEN;
6854 
6855 		if ((desc->type_flags & RXD_FLAG_PTPSTAT_MASK) ==
6856 		     RXD_FLAG_PTPSTAT_PTPV1 ||
6857 		    (desc->type_flags & RXD_FLAG_PTPSTAT_MASK) ==
6858 		     RXD_FLAG_PTPSTAT_PTPV2) {
6859 			tstamp = tr32(TG3_RX_TSTAMP_LSB);
6860 			tstamp |= (u64)tr32(TG3_RX_TSTAMP_MSB) << 32;
6861 		}
6862 
6863 		if (len > TG3_RX_COPY_THRESH(tp)) {
6864 			int skb_size;
6865 			unsigned int frag_size;
6866 
6867 			skb_size = tg3_alloc_rx_data(tp, tpr, opaque_key,
6868 						    *post_ptr, &frag_size);
6869 			if (skb_size < 0)
6870 				goto drop_it;
6871 
6872 			pci_unmap_single(tp->pdev, dma_addr, skb_size,
6873 					 PCI_DMA_FROMDEVICE);
6874 
6875 			/* Ensure that the update to the data happens
6876 			 * after the usage of the old DMA mapping.
6877 			 */
6878 			smp_wmb();
6879 
6880 			ri->data = NULL;
6881 
6882 			skb = build_skb(data, frag_size);
6883 			if (!skb) {
6884 				tg3_frag_free(frag_size != 0, data);
6885 				goto drop_it_no_recycle;
6886 			}
6887 			skb_reserve(skb, TG3_RX_OFFSET(tp));
6888 		} else {
6889 			tg3_recycle_rx(tnapi, tpr, opaque_key,
6890 				       desc_idx, *post_ptr);
6891 
6892 			skb = netdev_alloc_skb(tp->dev,
6893 					       len + TG3_RAW_IP_ALIGN);
6894 			if (skb == NULL)
6895 				goto drop_it_no_recycle;
6896 
6897 			skb_reserve(skb, TG3_RAW_IP_ALIGN);
6898 			pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
6899 			memcpy(skb->data,
6900 			       data + TG3_RX_OFFSET(tp),
6901 			       len);
6902 			pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
6903 		}
6904 
6905 		skb_put(skb, len);
6906 		if (tstamp)
6907 			tg3_hwclock_to_timestamp(tp, tstamp,
6908 						 skb_hwtstamps(skb));
6909 
6910 		if ((tp->dev->features & NETIF_F_RXCSUM) &&
6911 		    (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
6912 		    (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
6913 		      >> RXD_TCPCSUM_SHIFT) == 0xffff))
6914 			skb->ip_summed = CHECKSUM_UNNECESSARY;
6915 		else
6916 			skb_checksum_none_assert(skb);
6917 
6918 		skb->protocol = eth_type_trans(skb, tp->dev);
6919 
6920 		if (len > (tp->dev->mtu + ETH_HLEN) &&
6921 		    skb->protocol != htons(ETH_P_8021Q) &&
6922 		    skb->protocol != htons(ETH_P_8021AD)) {
6923 			dev_kfree_skb_any(skb);
6924 			goto drop_it_no_recycle;
6925 		}
6926 
6927 		if (desc->type_flags & RXD_FLAG_VLAN &&
6928 		    !(tp->rx_mode & RX_MODE_KEEP_VLAN_TAG))
6929 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
6930 					       desc->err_vlan & RXD_VLAN_MASK);
6931 
6932 		napi_gro_receive(&tnapi->napi, skb);
6933 
6934 		received++;
6935 		budget--;
6936 
6937 next_pkt:
6938 		(*post_ptr)++;
6939 
6940 		if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
6941 			tpr->rx_std_prod_idx = std_prod_idx &
6942 					       tp->rx_std_ring_mask;
6943 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
6944 				     tpr->rx_std_prod_idx);
6945 			work_mask &= ~RXD_OPAQUE_RING_STD;
6946 			rx_std_posted = 0;
6947 		}
6948 next_pkt_nopost:
6949 		sw_idx++;
6950 		sw_idx &= tp->rx_ret_ring_mask;
6951 
6952 		/* Refresh hw_idx to see if there is new work */
6953 		if (sw_idx == hw_idx) {
6954 			hw_idx = *(tnapi->rx_rcb_prod_idx);
6955 			rmb();
6956 		}
6957 	}
6958 
6959 	/* ACK the status ring. */
6960 	tnapi->rx_rcb_ptr = sw_idx;
6961 	tw32_rx_mbox(tnapi->consmbox, sw_idx);
6962 
6963 	/* Refill RX ring(s). */
6964 	if (!tg3_flag(tp, ENABLE_RSS)) {
6965 		/* Sync BD data before updating mailbox */
6966 		wmb();
6967 
6968 		if (work_mask & RXD_OPAQUE_RING_STD) {
6969 			tpr->rx_std_prod_idx = std_prod_idx &
6970 					       tp->rx_std_ring_mask;
6971 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
6972 				     tpr->rx_std_prod_idx);
6973 		}
6974 		if (work_mask & RXD_OPAQUE_RING_JUMBO) {
6975 			tpr->rx_jmb_prod_idx = jmb_prod_idx &
6976 					       tp->rx_jmb_ring_mask;
6977 			tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
6978 				     tpr->rx_jmb_prod_idx);
6979 		}
6980 		mmiowb();
6981 	} else if (work_mask) {
6982 		/* rx_std_buffers[] and rx_jmb_buffers[] entries must be
6983 		 * updated before the producer indices can be updated.
6984 		 */
6985 		smp_wmb();
6986 
6987 		tpr->rx_std_prod_idx = std_prod_idx & tp->rx_std_ring_mask;
6988 		tpr->rx_jmb_prod_idx = jmb_prod_idx & tp->rx_jmb_ring_mask;
6989 
6990 		if (tnapi != &tp->napi[1]) {
6991 			tp->rx_refill = true;
6992 			napi_schedule(&tp->napi[1].napi);
6993 		}
6994 	}
6995 
6996 	return received;
6997 }
6998 
6999 static void tg3_poll_link(struct tg3 *tp)
7000 {
7001 	/* handle link change and other phy events */
7002 	if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
7003 		struct tg3_hw_status *sblk = tp->napi[0].hw_status;
7004 
7005 		if (sblk->status & SD_STATUS_LINK_CHG) {
7006 			sblk->status = SD_STATUS_UPDATED |
7007 				       (sblk->status & ~SD_STATUS_LINK_CHG);
7008 			spin_lock(&tp->lock);
7009 			if (tg3_flag(tp, USE_PHYLIB)) {
7010 				tw32_f(MAC_STATUS,
7011 				     (MAC_STATUS_SYNC_CHANGED |
7012 				      MAC_STATUS_CFG_CHANGED |
7013 				      MAC_STATUS_MI_COMPLETION |
7014 				      MAC_STATUS_LNKSTATE_CHANGED));
7015 				udelay(40);
7016 			} else
7017 				tg3_setup_phy(tp, false);
7018 			spin_unlock(&tp->lock);
7019 		}
7020 	}
7021 }
7022 
7023 static int tg3_rx_prodring_xfer(struct tg3 *tp,
7024 				struct tg3_rx_prodring_set *dpr,
7025 				struct tg3_rx_prodring_set *spr)
7026 {
7027 	u32 si, di, cpycnt, src_prod_idx;
7028 	int i, err = 0;
7029 
7030 	while (1) {
7031 		src_prod_idx = spr->rx_std_prod_idx;
7032 
7033 		/* Make sure updates to the rx_std_buffers[] entries and the
7034 		 * standard producer index are seen in the correct order.
7035 		 */
7036 		smp_rmb();
7037 
7038 		if (spr->rx_std_cons_idx == src_prod_idx)
7039 			break;
7040 
7041 		if (spr->rx_std_cons_idx < src_prod_idx)
7042 			cpycnt = src_prod_idx - spr->rx_std_cons_idx;
7043 		else
7044 			cpycnt = tp->rx_std_ring_mask + 1 -
7045 				 spr->rx_std_cons_idx;
7046 
7047 		cpycnt = min(cpycnt,
7048 			     tp->rx_std_ring_mask + 1 - dpr->rx_std_prod_idx);
7049 
7050 		si = spr->rx_std_cons_idx;
7051 		di = dpr->rx_std_prod_idx;
7052 
7053 		for (i = di; i < di + cpycnt; i++) {
7054 			if (dpr->rx_std_buffers[i].data) {
7055 				cpycnt = i - di;
7056 				err = -ENOSPC;
7057 				break;
7058 			}
7059 		}
7060 
7061 		if (!cpycnt)
7062 			break;
7063 
7064 		/* Ensure that updates to the rx_std_buffers ring and the
7065 		 * shadowed hardware producer ring from tg3_recycle_skb() are
7066 		 * ordered correctly WRT the skb check above.
7067 		 */
7068 		smp_rmb();
7069 
7070 		memcpy(&dpr->rx_std_buffers[di],
7071 		       &spr->rx_std_buffers[si],
7072 		       cpycnt * sizeof(struct ring_info));
7073 
7074 		for (i = 0; i < cpycnt; i++, di++, si++) {
7075 			struct tg3_rx_buffer_desc *sbd, *dbd;
7076 			sbd = &spr->rx_std[si];
7077 			dbd = &dpr->rx_std[di];
7078 			dbd->addr_hi = sbd->addr_hi;
7079 			dbd->addr_lo = sbd->addr_lo;
7080 		}
7081 
7082 		spr->rx_std_cons_idx = (spr->rx_std_cons_idx + cpycnt) &
7083 				       tp->rx_std_ring_mask;
7084 		dpr->rx_std_prod_idx = (dpr->rx_std_prod_idx + cpycnt) &
7085 				       tp->rx_std_ring_mask;
7086 	}
7087 
7088 	while (1) {
7089 		src_prod_idx = spr->rx_jmb_prod_idx;
7090 
7091 		/* Make sure updates to the rx_jmb_buffers[] entries and
7092 		 * the jumbo producer index are seen in the correct order.
7093 		 */
7094 		smp_rmb();
7095 
7096 		if (spr->rx_jmb_cons_idx == src_prod_idx)
7097 			break;
7098 
7099 		if (spr->rx_jmb_cons_idx < src_prod_idx)
7100 			cpycnt = src_prod_idx - spr->rx_jmb_cons_idx;
7101 		else
7102 			cpycnt = tp->rx_jmb_ring_mask + 1 -
7103 				 spr->rx_jmb_cons_idx;
7104 
7105 		cpycnt = min(cpycnt,
7106 			     tp->rx_jmb_ring_mask + 1 - dpr->rx_jmb_prod_idx);
7107 
7108 		si = spr->rx_jmb_cons_idx;
7109 		di = dpr->rx_jmb_prod_idx;
7110 
7111 		for (i = di; i < di + cpycnt; i++) {
7112 			if (dpr->rx_jmb_buffers[i].data) {
7113 				cpycnt = i - di;
7114 				err = -ENOSPC;
7115 				break;
7116 			}
7117 		}
7118 
7119 		if (!cpycnt)
7120 			break;
7121 
7122 		/* Ensure that updates to the rx_jmb_buffers ring and the
7123 		 * shadowed hardware producer ring from tg3_recycle_skb() are
7124 		 * ordered correctly WRT the skb check above.
7125 		 */
7126 		smp_rmb();
7127 
7128 		memcpy(&dpr->rx_jmb_buffers[di],
7129 		       &spr->rx_jmb_buffers[si],
7130 		       cpycnt * sizeof(struct ring_info));
7131 
7132 		for (i = 0; i < cpycnt; i++, di++, si++) {
7133 			struct tg3_rx_buffer_desc *sbd, *dbd;
7134 			sbd = &spr->rx_jmb[si].std;
7135 			dbd = &dpr->rx_jmb[di].std;
7136 			dbd->addr_hi = sbd->addr_hi;
7137 			dbd->addr_lo = sbd->addr_lo;
7138 		}
7139 
7140 		spr->rx_jmb_cons_idx = (spr->rx_jmb_cons_idx + cpycnt) &
7141 				       tp->rx_jmb_ring_mask;
7142 		dpr->rx_jmb_prod_idx = (dpr->rx_jmb_prod_idx + cpycnt) &
7143 				       tp->rx_jmb_ring_mask;
7144 	}
7145 
7146 	return err;
7147 }
7148 
7149 static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget)
7150 {
7151 	struct tg3 *tp = tnapi->tp;
7152 
7153 	/* run TX completion thread */
7154 	if (tnapi->hw_status->idx[0].tx_consumer != tnapi->tx_cons) {
7155 		tg3_tx(tnapi);
7156 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7157 			return work_done;
7158 	}
7159 
7160 	if (!tnapi->rx_rcb_prod_idx)
7161 		return work_done;
7162 
7163 	/* run RX thread, within the bounds set by NAPI.
7164 	 * All RX "locking" is done by ensuring outside
7165 	 * code synchronizes with tg3->napi.poll()
7166 	 */
7167 	if (*(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
7168 		work_done += tg3_rx(tnapi, budget - work_done);
7169 
7170 	if (tg3_flag(tp, ENABLE_RSS) && tnapi == &tp->napi[1]) {
7171 		struct tg3_rx_prodring_set *dpr = &tp->napi[0].prodring;
7172 		int i, err = 0;
7173 		u32 std_prod_idx = dpr->rx_std_prod_idx;
7174 		u32 jmb_prod_idx = dpr->rx_jmb_prod_idx;
7175 
7176 		tp->rx_refill = false;
7177 		for (i = 1; i <= tp->rxq_cnt; i++)
7178 			err |= tg3_rx_prodring_xfer(tp, dpr,
7179 						    &tp->napi[i].prodring);
7180 
7181 		wmb();
7182 
7183 		if (std_prod_idx != dpr->rx_std_prod_idx)
7184 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
7185 				     dpr->rx_std_prod_idx);
7186 
7187 		if (jmb_prod_idx != dpr->rx_jmb_prod_idx)
7188 			tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
7189 				     dpr->rx_jmb_prod_idx);
7190 
7191 		mmiowb();
7192 
7193 		if (err)
7194 			tw32_f(HOSTCC_MODE, tp->coal_now);
7195 	}
7196 
7197 	return work_done;
7198 }
7199 
7200 static inline void tg3_reset_task_schedule(struct tg3 *tp)
7201 {
7202 	if (!test_and_set_bit(TG3_FLAG_RESET_TASK_PENDING, tp->tg3_flags))
7203 		schedule_work(&tp->reset_task);
7204 }
7205 
7206 static inline void tg3_reset_task_cancel(struct tg3 *tp)
7207 {
7208 	cancel_work_sync(&tp->reset_task);
7209 	tg3_flag_clear(tp, RESET_TASK_PENDING);
7210 	tg3_flag_clear(tp, TX_RECOVERY_PENDING);
7211 }
7212 
7213 static int tg3_poll_msix(struct napi_struct *napi, int budget)
7214 {
7215 	struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
7216 	struct tg3 *tp = tnapi->tp;
7217 	int work_done = 0;
7218 	struct tg3_hw_status *sblk = tnapi->hw_status;
7219 
7220 	while (1) {
7221 		work_done = tg3_poll_work(tnapi, work_done, budget);
7222 
7223 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7224 			goto tx_recovery;
7225 
7226 		if (unlikely(work_done >= budget))
7227 			break;
7228 
7229 		/* tp->last_tag is used in tg3_int_reenable() below
7230 		 * to tell the hw how much work has been processed,
7231 		 * so we must read it before checking for more work.
7232 		 */
7233 		tnapi->last_tag = sblk->status_tag;
7234 		tnapi->last_irq_tag = tnapi->last_tag;
7235 		rmb();
7236 
7237 		/* check for RX/TX work to do */
7238 		if (likely(sblk->idx[0].tx_consumer == tnapi->tx_cons &&
7239 			   *(tnapi->rx_rcb_prod_idx) == tnapi->rx_rcb_ptr)) {
7240 
7241 			/* This test here is not race free, but will reduce
7242 			 * the number of interrupts by looping again.
7243 			 */
7244 			if (tnapi == &tp->napi[1] && tp->rx_refill)
7245 				continue;
7246 
7247 			napi_complete(napi);
7248 			/* Reenable interrupts. */
7249 			tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
7250 
7251 			/* This test here is synchronized by napi_schedule()
7252 			 * and napi_complete() to close the race condition.
7253 			 */
7254 			if (unlikely(tnapi == &tp->napi[1] && tp->rx_refill)) {
7255 				tw32(HOSTCC_MODE, tp->coalesce_mode |
7256 						  HOSTCC_MODE_ENABLE |
7257 						  tnapi->coal_now);
7258 			}
7259 			mmiowb();
7260 			break;
7261 		}
7262 	}
7263 
7264 	return work_done;
7265 
7266 tx_recovery:
7267 	/* work_done is guaranteed to be less than budget. */
7268 	napi_complete(napi);
7269 	tg3_reset_task_schedule(tp);
7270 	return work_done;
7271 }
7272 
7273 static void tg3_process_error(struct tg3 *tp)
7274 {
7275 	u32 val;
7276 	bool real_error = false;
7277 
7278 	if (tg3_flag(tp, ERROR_PROCESSED))
7279 		return;
7280 
7281 	/* Check Flow Attention register */
7282 	val = tr32(HOSTCC_FLOW_ATTN);
7283 	if (val & ~HOSTCC_FLOW_ATTN_MBUF_LWM) {
7284 		netdev_err(tp->dev, "FLOW Attention error.  Resetting chip.\n");
7285 		real_error = true;
7286 	}
7287 
7288 	if (tr32(MSGINT_STATUS) & ~MSGINT_STATUS_MSI_REQ) {
7289 		netdev_err(tp->dev, "MSI Status error.  Resetting chip.\n");
7290 		real_error = true;
7291 	}
7292 
7293 	if (tr32(RDMAC_STATUS) || tr32(WDMAC_STATUS)) {
7294 		netdev_err(tp->dev, "DMA Status error.  Resetting chip.\n");
7295 		real_error = true;
7296 	}
7297 
7298 	if (!real_error)
7299 		return;
7300 
7301 	tg3_dump_state(tp);
7302 
7303 	tg3_flag_set(tp, ERROR_PROCESSED);
7304 	tg3_reset_task_schedule(tp);
7305 }
7306 
7307 static int tg3_poll(struct napi_struct *napi, int budget)
7308 {
7309 	struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
7310 	struct tg3 *tp = tnapi->tp;
7311 	int work_done = 0;
7312 	struct tg3_hw_status *sblk = tnapi->hw_status;
7313 
7314 	while (1) {
7315 		if (sblk->status & SD_STATUS_ERROR)
7316 			tg3_process_error(tp);
7317 
7318 		tg3_poll_link(tp);
7319 
7320 		work_done = tg3_poll_work(tnapi, work_done, budget);
7321 
7322 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7323 			goto tx_recovery;
7324 
7325 		if (unlikely(work_done >= budget))
7326 			break;
7327 
7328 		if (tg3_flag(tp, TAGGED_STATUS)) {
7329 			/* tp->last_tag is used in tg3_int_reenable() below
7330 			 * to tell the hw how much work has been processed,
7331 			 * so we must read it before checking for more work.
7332 			 */
7333 			tnapi->last_tag = sblk->status_tag;
7334 			tnapi->last_irq_tag = tnapi->last_tag;
7335 			rmb();
7336 		} else
7337 			sblk->status &= ~SD_STATUS_UPDATED;
7338 
7339 		if (likely(!tg3_has_work(tnapi))) {
7340 			napi_complete(napi);
7341 			tg3_int_reenable(tnapi);
7342 			break;
7343 		}
7344 	}
7345 
7346 	return work_done;
7347 
7348 tx_recovery:
7349 	/* work_done is guaranteed to be less than budget. */
7350 	napi_complete(napi);
7351 	tg3_reset_task_schedule(tp);
7352 	return work_done;
7353 }
7354 
7355 static void tg3_napi_disable(struct tg3 *tp)
7356 {
7357 	int i;
7358 
7359 	for (i = tp->irq_cnt - 1; i >= 0; i--)
7360 		napi_disable(&tp->napi[i].napi);
7361 }
7362 
7363 static void tg3_napi_enable(struct tg3 *tp)
7364 {
7365 	int i;
7366 
7367 	for (i = 0; i < tp->irq_cnt; i++)
7368 		napi_enable(&tp->napi[i].napi);
7369 }
7370 
7371 static void tg3_napi_init(struct tg3 *tp)
7372 {
7373 	int i;
7374 
7375 	netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll, 64);
7376 	for (i = 1; i < tp->irq_cnt; i++)
7377 		netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix, 64);
7378 }
7379 
7380 static void tg3_napi_fini(struct tg3 *tp)
7381 {
7382 	int i;
7383 
7384 	for (i = 0; i < tp->irq_cnt; i++)
7385 		netif_napi_del(&tp->napi[i].napi);
7386 }
7387 
7388 static inline void tg3_netif_stop(struct tg3 *tp)
7389 {
7390 	tp->dev->trans_start = jiffies;	/* prevent tx timeout */
7391 	tg3_napi_disable(tp);
7392 	netif_carrier_off(tp->dev);
7393 	netif_tx_disable(tp->dev);
7394 }
7395 
7396 /* tp->lock must be held */
7397 static inline void tg3_netif_start(struct tg3 *tp)
7398 {
7399 	tg3_ptp_resume(tp);
7400 
7401 	/* NOTE: unconditional netif_tx_wake_all_queues is only
7402 	 * appropriate so long as all callers are assured to
7403 	 * have free tx slots (such as after tg3_init_hw)
7404 	 */
7405 	netif_tx_wake_all_queues(tp->dev);
7406 
7407 	if (tp->link_up)
7408 		netif_carrier_on(tp->dev);
7409 
7410 	tg3_napi_enable(tp);
7411 	tp->napi[0].hw_status->status |= SD_STATUS_UPDATED;
7412 	tg3_enable_ints(tp);
7413 }
7414 
7415 static void tg3_irq_quiesce(struct tg3 *tp)
7416 	__releases(tp->lock)
7417 	__acquires(tp->lock)
7418 {
7419 	int i;
7420 
7421 	BUG_ON(tp->irq_sync);
7422 
7423 	tp->irq_sync = 1;
7424 	smp_mb();
7425 
7426 	spin_unlock_bh(&tp->lock);
7427 
7428 	for (i = 0; i < tp->irq_cnt; i++)
7429 		synchronize_irq(tp->napi[i].irq_vec);
7430 
7431 	spin_lock_bh(&tp->lock);
7432 }
7433 
7434 /* Fully shutdown all tg3 driver activity elsewhere in the system.
7435  * If irq_sync is non-zero, then the IRQ handler must be synchronized
7436  * with as well.  Most of the time, this is not necessary except when
7437  * shutting down the device.
7438  */
7439 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
7440 {
7441 	spin_lock_bh(&tp->lock);
7442 	if (irq_sync)
7443 		tg3_irq_quiesce(tp);
7444 }
7445 
7446 static inline void tg3_full_unlock(struct tg3 *tp)
7447 {
7448 	spin_unlock_bh(&tp->lock);
7449 }
7450 
7451 /* One-shot MSI handler - Chip automatically disables interrupt
7452  * after sending MSI so driver doesn't have to do it.
7453  */
7454 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
7455 {
7456 	struct tg3_napi *tnapi = dev_id;
7457 	struct tg3 *tp = tnapi->tp;
7458 
7459 	prefetch(tnapi->hw_status);
7460 	if (tnapi->rx_rcb)
7461 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7462 
7463 	if (likely(!tg3_irq_sync(tp)))
7464 		napi_schedule(&tnapi->napi);
7465 
7466 	return IRQ_HANDLED;
7467 }
7468 
7469 /* MSI ISR - No need to check for interrupt sharing and no need to
7470  * flush status block and interrupt mailbox. PCI ordering rules
7471  * guarantee that MSI will arrive after the status block.
7472  */
7473 static irqreturn_t tg3_msi(int irq, void *dev_id)
7474 {
7475 	struct tg3_napi *tnapi = dev_id;
7476 	struct tg3 *tp = tnapi->tp;
7477 
7478 	prefetch(tnapi->hw_status);
7479 	if (tnapi->rx_rcb)
7480 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7481 	/*
7482 	 * Writing any value to intr-mbox-0 clears PCI INTA# and
7483 	 * chip-internal interrupt pending events.
7484 	 * Writing non-zero to intr-mbox-0 additional tells the
7485 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7486 	 * event coalescing.
7487 	 */
7488 	tw32_mailbox(tnapi->int_mbox, 0x00000001);
7489 	if (likely(!tg3_irq_sync(tp)))
7490 		napi_schedule(&tnapi->napi);
7491 
7492 	return IRQ_RETVAL(1);
7493 }
7494 
7495 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
7496 {
7497 	struct tg3_napi *tnapi = dev_id;
7498 	struct tg3 *tp = tnapi->tp;
7499 	struct tg3_hw_status *sblk = tnapi->hw_status;
7500 	unsigned int handled = 1;
7501 
7502 	/* In INTx mode, it is possible for the interrupt to arrive at
7503 	 * the CPU before the status block posted prior to the interrupt.
7504 	 * Reading the PCI State register will confirm whether the
7505 	 * interrupt is ours and will flush the status block.
7506 	 */
7507 	if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
7508 		if (tg3_flag(tp, CHIP_RESETTING) ||
7509 		    (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7510 			handled = 0;
7511 			goto out;
7512 		}
7513 	}
7514 
7515 	/*
7516 	 * Writing any value to intr-mbox-0 clears PCI INTA# and
7517 	 * chip-internal interrupt pending events.
7518 	 * Writing non-zero to intr-mbox-0 additional tells the
7519 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7520 	 * event coalescing.
7521 	 *
7522 	 * Flush the mailbox to de-assert the IRQ immediately to prevent
7523 	 * spurious interrupts.  The flush impacts performance but
7524 	 * excessive spurious interrupts can be worse in some cases.
7525 	 */
7526 	tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
7527 	if (tg3_irq_sync(tp))
7528 		goto out;
7529 	sblk->status &= ~SD_STATUS_UPDATED;
7530 	if (likely(tg3_has_work(tnapi))) {
7531 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7532 		napi_schedule(&tnapi->napi);
7533 	} else {
7534 		/* No work, shared interrupt perhaps?  re-enable
7535 		 * interrupts, and flush that PCI write
7536 		 */
7537 		tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
7538 			       0x00000000);
7539 	}
7540 out:
7541 	return IRQ_RETVAL(handled);
7542 }
7543 
7544 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
7545 {
7546 	struct tg3_napi *tnapi = dev_id;
7547 	struct tg3 *tp = tnapi->tp;
7548 	struct tg3_hw_status *sblk = tnapi->hw_status;
7549 	unsigned int handled = 1;
7550 
7551 	/* In INTx mode, it is possible for the interrupt to arrive at
7552 	 * the CPU before the status block posted prior to the interrupt.
7553 	 * Reading the PCI State register will confirm whether the
7554 	 * interrupt is ours and will flush the status block.
7555 	 */
7556 	if (unlikely(sblk->status_tag == tnapi->last_irq_tag)) {
7557 		if (tg3_flag(tp, CHIP_RESETTING) ||
7558 		    (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7559 			handled = 0;
7560 			goto out;
7561 		}
7562 	}
7563 
7564 	/*
7565 	 * writing any value to intr-mbox-0 clears PCI INTA# and
7566 	 * chip-internal interrupt pending events.
7567 	 * writing non-zero to intr-mbox-0 additional tells the
7568 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7569 	 * event coalescing.
7570 	 *
7571 	 * Flush the mailbox to de-assert the IRQ immediately to prevent
7572 	 * spurious interrupts.  The flush impacts performance but
7573 	 * excessive spurious interrupts can be worse in some cases.
7574 	 */
7575 	tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
7576 
7577 	/*
7578 	 * In a shared interrupt configuration, sometimes other devices'
7579 	 * interrupts will scream.  We record the current status tag here
7580 	 * so that the above check can report that the screaming interrupts
7581 	 * are unhandled.  Eventually they will be silenced.
7582 	 */
7583 	tnapi->last_irq_tag = sblk->status_tag;
7584 
7585 	if (tg3_irq_sync(tp))
7586 		goto out;
7587 
7588 	prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7589 
7590 	napi_schedule(&tnapi->napi);
7591 
7592 out:
7593 	return IRQ_RETVAL(handled);
7594 }
7595 
7596 /* ISR for interrupt test */
7597 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
7598 {
7599 	struct tg3_napi *tnapi = dev_id;
7600 	struct tg3 *tp = tnapi->tp;
7601 	struct tg3_hw_status *sblk = tnapi->hw_status;
7602 
7603 	if ((sblk->status & SD_STATUS_UPDATED) ||
7604 	    !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7605 		tg3_disable_ints(tp);
7606 		return IRQ_RETVAL(1);
7607 	}
7608 	return IRQ_RETVAL(0);
7609 }
7610 
7611 #ifdef CONFIG_NET_POLL_CONTROLLER
7612 static void tg3_poll_controller(struct net_device *dev)
7613 {
7614 	int i;
7615 	struct tg3 *tp = netdev_priv(dev);
7616 
7617 	if (tg3_irq_sync(tp))
7618 		return;
7619 
7620 	for (i = 0; i < tp->irq_cnt; i++)
7621 		tg3_interrupt(tp->napi[i].irq_vec, &tp->napi[i]);
7622 }
7623 #endif
7624 
7625 static void tg3_tx_timeout(struct net_device *dev)
7626 {
7627 	struct tg3 *tp = netdev_priv(dev);
7628 
7629 	if (netif_msg_tx_err(tp)) {
7630 		netdev_err(dev, "transmit timed out, resetting\n");
7631 		tg3_dump_state(tp);
7632 	}
7633 
7634 	tg3_reset_task_schedule(tp);
7635 }
7636 
7637 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
7638 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
7639 {
7640 	u32 base = (u32) mapping & 0xffffffff;
7641 
7642 	return base + len + 8 < base;
7643 }
7644 
7645 /* Test for TSO DMA buffers that cross into regions which are within MSS bytes
7646  * of any 4GB boundaries: 4G, 8G, etc
7647  */
7648 static inline int tg3_4g_tso_overflow_test(struct tg3 *tp, dma_addr_t mapping,
7649 					   u32 len, u32 mss)
7650 {
7651 	if (tg3_asic_rev(tp) == ASIC_REV_5762 && mss) {
7652 		u32 base = (u32) mapping & 0xffffffff;
7653 
7654 		return ((base + len + (mss & 0x3fff)) < base);
7655 	}
7656 	return 0;
7657 }
7658 
7659 /* Test for DMA addresses > 40-bit */
7660 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
7661 					  int len)
7662 {
7663 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
7664 	if (tg3_flag(tp, 40BIT_DMA_BUG))
7665 		return ((u64) mapping + len) > DMA_BIT_MASK(40);
7666 	return 0;
7667 #else
7668 	return 0;
7669 #endif
7670 }
7671 
7672 static inline void tg3_tx_set_bd(struct tg3_tx_buffer_desc *txbd,
7673 				 dma_addr_t mapping, u32 len, u32 flags,
7674 				 u32 mss, u32 vlan)
7675 {
7676 	txbd->addr_hi = ((u64) mapping >> 32);
7677 	txbd->addr_lo = ((u64) mapping & 0xffffffff);
7678 	txbd->len_flags = (len << TXD_LEN_SHIFT) | (flags & 0x0000ffff);
7679 	txbd->vlan_tag = (mss << TXD_MSS_SHIFT) | (vlan << TXD_VLAN_TAG_SHIFT);
7680 }
7681 
7682 static bool tg3_tx_frag_set(struct tg3_napi *tnapi, u32 *entry, u32 *budget,
7683 			    dma_addr_t map, u32 len, u32 flags,
7684 			    u32 mss, u32 vlan)
7685 {
7686 	struct tg3 *tp = tnapi->tp;
7687 	bool hwbug = false;
7688 
7689 	if (tg3_flag(tp, SHORT_DMA_BUG) && len <= 8)
7690 		hwbug = true;
7691 
7692 	if (tg3_4g_overflow_test(map, len))
7693 		hwbug = true;
7694 
7695 	if (tg3_4g_tso_overflow_test(tp, map, len, mss))
7696 		hwbug = true;
7697 
7698 	if (tg3_40bit_overflow_test(tp, map, len))
7699 		hwbug = true;
7700 
7701 	if (tp->dma_limit) {
7702 		u32 prvidx = *entry;
7703 		u32 tmp_flag = flags & ~TXD_FLAG_END;
7704 		while (len > tp->dma_limit && *budget) {
7705 			u32 frag_len = tp->dma_limit;
7706 			len -= tp->dma_limit;
7707 
7708 			/* Avoid the 8byte DMA problem */
7709 			if (len <= 8) {
7710 				len += tp->dma_limit / 2;
7711 				frag_len = tp->dma_limit / 2;
7712 			}
7713 
7714 			tnapi->tx_buffers[*entry].fragmented = true;
7715 
7716 			tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7717 				      frag_len, tmp_flag, mss, vlan);
7718 			*budget -= 1;
7719 			prvidx = *entry;
7720 			*entry = NEXT_TX(*entry);
7721 
7722 			map += frag_len;
7723 		}
7724 
7725 		if (len) {
7726 			if (*budget) {
7727 				tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7728 					      len, flags, mss, vlan);
7729 				*budget -= 1;
7730 				*entry = NEXT_TX(*entry);
7731 			} else {
7732 				hwbug = true;
7733 				tnapi->tx_buffers[prvidx].fragmented = false;
7734 			}
7735 		}
7736 	} else {
7737 		tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7738 			      len, flags, mss, vlan);
7739 		*entry = NEXT_TX(*entry);
7740 	}
7741 
7742 	return hwbug;
7743 }
7744 
7745 static void tg3_tx_skb_unmap(struct tg3_napi *tnapi, u32 entry, int last)
7746 {
7747 	int i;
7748 	struct sk_buff *skb;
7749 	struct tg3_tx_ring_info *txb = &tnapi->tx_buffers[entry];
7750 
7751 	skb = txb->skb;
7752 	txb->skb = NULL;
7753 
7754 	pci_unmap_single(tnapi->tp->pdev,
7755 			 dma_unmap_addr(txb, mapping),
7756 			 skb_headlen(skb),
7757 			 PCI_DMA_TODEVICE);
7758 
7759 	while (txb->fragmented) {
7760 		txb->fragmented = false;
7761 		entry = NEXT_TX(entry);
7762 		txb = &tnapi->tx_buffers[entry];
7763 	}
7764 
7765 	for (i = 0; i <= last; i++) {
7766 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
7767 
7768 		entry = NEXT_TX(entry);
7769 		txb = &tnapi->tx_buffers[entry];
7770 
7771 		pci_unmap_page(tnapi->tp->pdev,
7772 			       dma_unmap_addr(txb, mapping),
7773 			       skb_frag_size(frag), PCI_DMA_TODEVICE);
7774 
7775 		while (txb->fragmented) {
7776 			txb->fragmented = false;
7777 			entry = NEXT_TX(entry);
7778 			txb = &tnapi->tx_buffers[entry];
7779 		}
7780 	}
7781 }
7782 
7783 /* Workaround 4GB and 40-bit hardware DMA bugs. */
7784 static int tigon3_dma_hwbug_workaround(struct tg3_napi *tnapi,
7785 				       struct sk_buff **pskb,
7786 				       u32 *entry, u32 *budget,
7787 				       u32 base_flags, u32 mss, u32 vlan)
7788 {
7789 	struct tg3 *tp = tnapi->tp;
7790 	struct sk_buff *new_skb, *skb = *pskb;
7791 	dma_addr_t new_addr = 0;
7792 	int ret = 0;
7793 
7794 	if (tg3_asic_rev(tp) != ASIC_REV_5701)
7795 		new_skb = skb_copy(skb, GFP_ATOMIC);
7796 	else {
7797 		int more_headroom = 4 - ((unsigned long)skb->data & 3);
7798 
7799 		new_skb = skb_copy_expand(skb,
7800 					  skb_headroom(skb) + more_headroom,
7801 					  skb_tailroom(skb), GFP_ATOMIC);
7802 	}
7803 
7804 	if (!new_skb) {
7805 		ret = -1;
7806 	} else {
7807 		/* New SKB is guaranteed to be linear. */
7808 		new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
7809 					  PCI_DMA_TODEVICE);
7810 		/* Make sure the mapping succeeded */
7811 		if (pci_dma_mapping_error(tp->pdev, new_addr)) {
7812 			dev_kfree_skb_any(new_skb);
7813 			ret = -1;
7814 		} else {
7815 			u32 save_entry = *entry;
7816 
7817 			base_flags |= TXD_FLAG_END;
7818 
7819 			tnapi->tx_buffers[*entry].skb = new_skb;
7820 			dma_unmap_addr_set(&tnapi->tx_buffers[*entry],
7821 					   mapping, new_addr);
7822 
7823 			if (tg3_tx_frag_set(tnapi, entry, budget, new_addr,
7824 					    new_skb->len, base_flags,
7825 					    mss, vlan)) {
7826 				tg3_tx_skb_unmap(tnapi, save_entry, -1);
7827 				dev_kfree_skb_any(new_skb);
7828 				ret = -1;
7829 			}
7830 		}
7831 	}
7832 
7833 	dev_kfree_skb_any(skb);
7834 	*pskb = new_skb;
7835 	return ret;
7836 }
7837 
7838 static netdev_tx_t tg3_start_xmit(struct sk_buff *, struct net_device *);
7839 
7840 /* Use GSO to workaround all TSO packets that meet HW bug conditions
7841  * indicated in tg3_tx_frag_set()
7842  */
7843 static int tg3_tso_bug(struct tg3 *tp, struct tg3_napi *tnapi,
7844 		       struct netdev_queue *txq, struct sk_buff *skb)
7845 {
7846 	struct sk_buff *segs, *nskb;
7847 	u32 frag_cnt_est = skb_shinfo(skb)->gso_segs * 3;
7848 
7849 	/* Estimate the number of fragments in the worst case */
7850 	if (unlikely(tg3_tx_avail(tnapi) <= frag_cnt_est)) {
7851 		netif_tx_stop_queue(txq);
7852 
7853 		/* netif_tx_stop_queue() must be done before checking
7854 		 * checking tx index in tg3_tx_avail() below, because in
7855 		 * tg3_tx(), we update tx index before checking for
7856 		 * netif_tx_queue_stopped().
7857 		 */
7858 		smp_mb();
7859 		if (tg3_tx_avail(tnapi) <= frag_cnt_est)
7860 			return NETDEV_TX_BUSY;
7861 
7862 		netif_tx_wake_queue(txq);
7863 	}
7864 
7865 	segs = skb_gso_segment(skb, tp->dev->features &
7866 				    ~(NETIF_F_TSO | NETIF_F_TSO6));
7867 	if (IS_ERR(segs) || !segs)
7868 		goto tg3_tso_bug_end;
7869 
7870 	do {
7871 		nskb = segs;
7872 		segs = segs->next;
7873 		nskb->next = NULL;
7874 		tg3_start_xmit(nskb, tp->dev);
7875 	} while (segs);
7876 
7877 tg3_tso_bug_end:
7878 	dev_kfree_skb_any(skb);
7879 
7880 	return NETDEV_TX_OK;
7881 }
7882 
7883 /* hard_start_xmit for all devices */
7884 static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
7885 {
7886 	struct tg3 *tp = netdev_priv(dev);
7887 	u32 len, entry, base_flags, mss, vlan = 0;
7888 	u32 budget;
7889 	int i = -1, would_hit_hwbug;
7890 	dma_addr_t mapping;
7891 	struct tg3_napi *tnapi;
7892 	struct netdev_queue *txq;
7893 	unsigned int last;
7894 	struct iphdr *iph = NULL;
7895 	struct tcphdr *tcph = NULL;
7896 	__sum16 tcp_csum = 0, ip_csum = 0;
7897 	__be16 ip_tot_len = 0;
7898 
7899 	txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
7900 	tnapi = &tp->napi[skb_get_queue_mapping(skb)];
7901 	if (tg3_flag(tp, ENABLE_TSS))
7902 		tnapi++;
7903 
7904 	budget = tg3_tx_avail(tnapi);
7905 
7906 	/* We are running in BH disabled context with netif_tx_lock
7907 	 * and TX reclaim runs via tp->napi.poll inside of a software
7908 	 * interrupt.  Furthermore, IRQ processing runs lockless so we have
7909 	 * no IRQ context deadlocks to worry about either.  Rejoice!
7910 	 */
7911 	if (unlikely(budget <= (skb_shinfo(skb)->nr_frags + 1))) {
7912 		if (!netif_tx_queue_stopped(txq)) {
7913 			netif_tx_stop_queue(txq);
7914 
7915 			/* This is a hard error, log it. */
7916 			netdev_err(dev,
7917 				   "BUG! Tx Ring full when queue awake!\n");
7918 		}
7919 		return NETDEV_TX_BUSY;
7920 	}
7921 
7922 	entry = tnapi->tx_prod;
7923 	base_flags = 0;
7924 
7925 	mss = skb_shinfo(skb)->gso_size;
7926 	if (mss) {
7927 		u32 tcp_opt_len, hdr_len;
7928 
7929 		if (skb_cow_head(skb, 0))
7930 			goto drop;
7931 
7932 		iph = ip_hdr(skb);
7933 		tcp_opt_len = tcp_optlen(skb);
7934 
7935 		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb) - ETH_HLEN;
7936 
7937 		/* HW/FW can not correctly segment packets that have been
7938 		 * vlan encapsulated.
7939 		 */
7940 		if (skb->protocol == htons(ETH_P_8021Q) ||
7941 		    skb->protocol == htons(ETH_P_8021AD))
7942 			return tg3_tso_bug(tp, tnapi, txq, skb);
7943 
7944 		if (!skb_is_gso_v6(skb)) {
7945 			if (unlikely((ETH_HLEN + hdr_len) > 80) &&
7946 			    tg3_flag(tp, TSO_BUG))
7947 				return tg3_tso_bug(tp, tnapi, txq, skb);
7948 
7949 			ip_csum = iph->check;
7950 			ip_tot_len = iph->tot_len;
7951 			iph->check = 0;
7952 			iph->tot_len = htons(mss + hdr_len);
7953 		}
7954 
7955 		base_flags |= (TXD_FLAG_CPU_PRE_DMA |
7956 			       TXD_FLAG_CPU_POST_DMA);
7957 
7958 		tcph = tcp_hdr(skb);
7959 		tcp_csum = tcph->check;
7960 
7961 		if (tg3_flag(tp, HW_TSO_1) ||
7962 		    tg3_flag(tp, HW_TSO_2) ||
7963 		    tg3_flag(tp, HW_TSO_3)) {
7964 			tcph->check = 0;
7965 			base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
7966 		} else {
7967 			tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
7968 							 0, IPPROTO_TCP, 0);
7969 		}
7970 
7971 		if (tg3_flag(tp, HW_TSO_3)) {
7972 			mss |= (hdr_len & 0xc) << 12;
7973 			if (hdr_len & 0x10)
7974 				base_flags |= 0x00000010;
7975 			base_flags |= (hdr_len & 0x3e0) << 5;
7976 		} else if (tg3_flag(tp, HW_TSO_2))
7977 			mss |= hdr_len << 9;
7978 		else if (tg3_flag(tp, HW_TSO_1) ||
7979 			 tg3_asic_rev(tp) == ASIC_REV_5705) {
7980 			if (tcp_opt_len || iph->ihl > 5) {
7981 				int tsflags;
7982 
7983 				tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
7984 				mss |= (tsflags << 11);
7985 			}
7986 		} else {
7987 			if (tcp_opt_len || iph->ihl > 5) {
7988 				int tsflags;
7989 
7990 				tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
7991 				base_flags |= tsflags << 12;
7992 			}
7993 		}
7994 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
7995 		/* HW/FW can not correctly checksum packets that have been
7996 		 * vlan encapsulated.
7997 		 */
7998 		if (skb->protocol == htons(ETH_P_8021Q) ||
7999 		    skb->protocol == htons(ETH_P_8021AD)) {
8000 			if (skb_checksum_help(skb))
8001 				goto drop;
8002 		} else  {
8003 			base_flags |= TXD_FLAG_TCPUDP_CSUM;
8004 		}
8005 	}
8006 
8007 	if (tg3_flag(tp, USE_JUMBO_BDFLAG) &&
8008 	    !mss && skb->len > VLAN_ETH_FRAME_LEN)
8009 		base_flags |= TXD_FLAG_JMB_PKT;
8010 
8011 	if (vlan_tx_tag_present(skb)) {
8012 		base_flags |= TXD_FLAG_VLAN;
8013 		vlan = vlan_tx_tag_get(skb);
8014 	}
8015 
8016 	if ((unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) &&
8017 	    tg3_flag(tp, TX_TSTAMP_EN)) {
8018 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
8019 		base_flags |= TXD_FLAG_HWTSTAMP;
8020 	}
8021 
8022 	len = skb_headlen(skb);
8023 
8024 	mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
8025 	if (pci_dma_mapping_error(tp->pdev, mapping))
8026 		goto drop;
8027 
8028 
8029 	tnapi->tx_buffers[entry].skb = skb;
8030 	dma_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, mapping);
8031 
8032 	would_hit_hwbug = 0;
8033 
8034 	if (tg3_flag(tp, 5701_DMA_BUG))
8035 		would_hit_hwbug = 1;
8036 
8037 	if (tg3_tx_frag_set(tnapi, &entry, &budget, mapping, len, base_flags |
8038 			  ((skb_shinfo(skb)->nr_frags == 0) ? TXD_FLAG_END : 0),
8039 			    mss, vlan)) {
8040 		would_hit_hwbug = 1;
8041 	} else if (skb_shinfo(skb)->nr_frags > 0) {
8042 		u32 tmp_mss = mss;
8043 
8044 		if (!tg3_flag(tp, HW_TSO_1) &&
8045 		    !tg3_flag(tp, HW_TSO_2) &&
8046 		    !tg3_flag(tp, HW_TSO_3))
8047 			tmp_mss = 0;
8048 
8049 		/* Now loop through additional data
8050 		 * fragments, and queue them.
8051 		 */
8052 		last = skb_shinfo(skb)->nr_frags - 1;
8053 		for (i = 0; i <= last; i++) {
8054 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
8055 
8056 			len = skb_frag_size(frag);
8057 			mapping = skb_frag_dma_map(&tp->pdev->dev, frag, 0,
8058 						   len, DMA_TO_DEVICE);
8059 
8060 			tnapi->tx_buffers[entry].skb = NULL;
8061 			dma_unmap_addr_set(&tnapi->tx_buffers[entry], mapping,
8062 					   mapping);
8063 			if (dma_mapping_error(&tp->pdev->dev, mapping))
8064 				goto dma_error;
8065 
8066 			if (!budget ||
8067 			    tg3_tx_frag_set(tnapi, &entry, &budget, mapping,
8068 					    len, base_flags |
8069 					    ((i == last) ? TXD_FLAG_END : 0),
8070 					    tmp_mss, vlan)) {
8071 				would_hit_hwbug = 1;
8072 				break;
8073 			}
8074 		}
8075 	}
8076 
8077 	if (would_hit_hwbug) {
8078 		tg3_tx_skb_unmap(tnapi, tnapi->tx_prod, i);
8079 
8080 		if (mss) {
8081 			/* If it's a TSO packet, do GSO instead of
8082 			 * allocating and copying to a large linear SKB
8083 			 */
8084 			if (ip_tot_len) {
8085 				iph->check = ip_csum;
8086 				iph->tot_len = ip_tot_len;
8087 			}
8088 			tcph->check = tcp_csum;
8089 			return tg3_tso_bug(tp, tnapi, txq, skb);
8090 		}
8091 
8092 		/* If the workaround fails due to memory/mapping
8093 		 * failure, silently drop this packet.
8094 		 */
8095 		entry = tnapi->tx_prod;
8096 		budget = tg3_tx_avail(tnapi);
8097 		if (tigon3_dma_hwbug_workaround(tnapi, &skb, &entry, &budget,
8098 						base_flags, mss, vlan))
8099 			goto drop_nofree;
8100 	}
8101 
8102 	skb_tx_timestamp(skb);
8103 	netdev_tx_sent_queue(txq, skb->len);
8104 
8105 	/* Sync BD data before updating mailbox */
8106 	wmb();
8107 
8108 	tnapi->tx_prod = entry;
8109 	if (unlikely(tg3_tx_avail(tnapi) <= (MAX_SKB_FRAGS + 1))) {
8110 		netif_tx_stop_queue(txq);
8111 
8112 		/* netif_tx_stop_queue() must be done before checking
8113 		 * checking tx index in tg3_tx_avail() below, because in
8114 		 * tg3_tx(), we update tx index before checking for
8115 		 * netif_tx_queue_stopped().
8116 		 */
8117 		smp_mb();
8118 		if (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi))
8119 			netif_tx_wake_queue(txq);
8120 	}
8121 
8122 	if (!skb->xmit_more || netif_xmit_stopped(txq)) {
8123 		/* Packets are ready, update Tx producer idx on card. */
8124 		tw32_tx_mbox(tnapi->prodmbox, entry);
8125 		mmiowb();
8126 	}
8127 
8128 	return NETDEV_TX_OK;
8129 
8130 dma_error:
8131 	tg3_tx_skb_unmap(tnapi, tnapi->tx_prod, --i);
8132 	tnapi->tx_buffers[tnapi->tx_prod].skb = NULL;
8133 drop:
8134 	dev_kfree_skb_any(skb);
8135 drop_nofree:
8136 	tp->tx_dropped++;
8137 	return NETDEV_TX_OK;
8138 }
8139 
8140 static void tg3_mac_loopback(struct tg3 *tp, bool enable)
8141 {
8142 	if (enable) {
8143 		tp->mac_mode &= ~(MAC_MODE_HALF_DUPLEX |
8144 				  MAC_MODE_PORT_MODE_MASK);
8145 
8146 		tp->mac_mode |= MAC_MODE_PORT_INT_LPBACK;
8147 
8148 		if (!tg3_flag(tp, 5705_PLUS))
8149 			tp->mac_mode |= MAC_MODE_LINK_POLARITY;
8150 
8151 		if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
8152 			tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
8153 		else
8154 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
8155 	} else {
8156 		tp->mac_mode &= ~MAC_MODE_PORT_INT_LPBACK;
8157 
8158 		if (tg3_flag(tp, 5705_PLUS) ||
8159 		    (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) ||
8160 		    tg3_asic_rev(tp) == ASIC_REV_5700)
8161 			tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
8162 	}
8163 
8164 	tw32(MAC_MODE, tp->mac_mode);
8165 	udelay(40);
8166 }
8167 
8168 static int tg3_phy_lpbk_set(struct tg3 *tp, u32 speed, bool extlpbk)
8169 {
8170 	u32 val, bmcr, mac_mode, ptest = 0;
8171 
8172 	tg3_phy_toggle_apd(tp, false);
8173 	tg3_phy_toggle_automdix(tp, false);
8174 
8175 	if (extlpbk && tg3_phy_set_extloopbk(tp))
8176 		return -EIO;
8177 
8178 	bmcr = BMCR_FULLDPLX;
8179 	switch (speed) {
8180 	case SPEED_10:
8181 		break;
8182 	case SPEED_100:
8183 		bmcr |= BMCR_SPEED100;
8184 		break;
8185 	case SPEED_1000:
8186 	default:
8187 		if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
8188 			speed = SPEED_100;
8189 			bmcr |= BMCR_SPEED100;
8190 		} else {
8191 			speed = SPEED_1000;
8192 			bmcr |= BMCR_SPEED1000;
8193 		}
8194 	}
8195 
8196 	if (extlpbk) {
8197 		if (!(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
8198 			tg3_readphy(tp, MII_CTRL1000, &val);
8199 			val |= CTL1000_AS_MASTER |
8200 			       CTL1000_ENABLE_MASTER;
8201 			tg3_writephy(tp, MII_CTRL1000, val);
8202 		} else {
8203 			ptest = MII_TG3_FET_PTEST_TRIM_SEL |
8204 				MII_TG3_FET_PTEST_TRIM_2;
8205 			tg3_writephy(tp, MII_TG3_FET_PTEST, ptest);
8206 		}
8207 	} else
8208 		bmcr |= BMCR_LOOPBACK;
8209 
8210 	tg3_writephy(tp, MII_BMCR, bmcr);
8211 
8212 	/* The write needs to be flushed for the FETs */
8213 	if (tp->phy_flags & TG3_PHYFLG_IS_FET)
8214 		tg3_readphy(tp, MII_BMCR, &bmcr);
8215 
8216 	udelay(40);
8217 
8218 	if ((tp->phy_flags & TG3_PHYFLG_IS_FET) &&
8219 	    tg3_asic_rev(tp) == ASIC_REV_5785) {
8220 		tg3_writephy(tp, MII_TG3_FET_PTEST, ptest |
8221 			     MII_TG3_FET_PTEST_FRC_TX_LINK |
8222 			     MII_TG3_FET_PTEST_FRC_TX_LOCK);
8223 
8224 		/* The write needs to be flushed for the AC131 */
8225 		tg3_readphy(tp, MII_TG3_FET_PTEST, &val);
8226 	}
8227 
8228 	/* Reset to prevent losing 1st rx packet intermittently */
8229 	if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
8230 	    tg3_flag(tp, 5780_CLASS)) {
8231 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
8232 		udelay(10);
8233 		tw32_f(MAC_RX_MODE, tp->rx_mode);
8234 	}
8235 
8236 	mac_mode = tp->mac_mode &
8237 		   ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
8238 	if (speed == SPEED_1000)
8239 		mac_mode |= MAC_MODE_PORT_MODE_GMII;
8240 	else
8241 		mac_mode |= MAC_MODE_PORT_MODE_MII;
8242 
8243 	if (tg3_asic_rev(tp) == ASIC_REV_5700) {
8244 		u32 masked_phy_id = tp->phy_id & TG3_PHY_ID_MASK;
8245 
8246 		if (masked_phy_id == TG3_PHY_ID_BCM5401)
8247 			mac_mode &= ~MAC_MODE_LINK_POLARITY;
8248 		else if (masked_phy_id == TG3_PHY_ID_BCM5411)
8249 			mac_mode |= MAC_MODE_LINK_POLARITY;
8250 
8251 		tg3_writephy(tp, MII_TG3_EXT_CTRL,
8252 			     MII_TG3_EXT_CTRL_LNK3_LED_MODE);
8253 	}
8254 
8255 	tw32(MAC_MODE, mac_mode);
8256 	udelay(40);
8257 
8258 	return 0;
8259 }
8260 
8261 static void tg3_set_loopback(struct net_device *dev, netdev_features_t features)
8262 {
8263 	struct tg3 *tp = netdev_priv(dev);
8264 
8265 	if (features & NETIF_F_LOOPBACK) {
8266 		if (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK)
8267 			return;
8268 
8269 		spin_lock_bh(&tp->lock);
8270 		tg3_mac_loopback(tp, true);
8271 		netif_carrier_on(tp->dev);
8272 		spin_unlock_bh(&tp->lock);
8273 		netdev_info(dev, "Internal MAC loopback mode enabled.\n");
8274 	} else {
8275 		if (!(tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
8276 			return;
8277 
8278 		spin_lock_bh(&tp->lock);
8279 		tg3_mac_loopback(tp, false);
8280 		/* Force link status check */
8281 		tg3_setup_phy(tp, true);
8282 		spin_unlock_bh(&tp->lock);
8283 		netdev_info(dev, "Internal MAC loopback mode disabled.\n");
8284 	}
8285 }
8286 
8287 static netdev_features_t tg3_fix_features(struct net_device *dev,
8288 	netdev_features_t features)
8289 {
8290 	struct tg3 *tp = netdev_priv(dev);
8291 
8292 	if (dev->mtu > ETH_DATA_LEN && tg3_flag(tp, 5780_CLASS))
8293 		features &= ~NETIF_F_ALL_TSO;
8294 
8295 	return features;
8296 }
8297 
8298 static int tg3_set_features(struct net_device *dev, netdev_features_t features)
8299 {
8300 	netdev_features_t changed = dev->features ^ features;
8301 
8302 	if ((changed & NETIF_F_LOOPBACK) && netif_running(dev))
8303 		tg3_set_loopback(dev, features);
8304 
8305 	return 0;
8306 }
8307 
8308 static void tg3_rx_prodring_free(struct tg3 *tp,
8309 				 struct tg3_rx_prodring_set *tpr)
8310 {
8311 	int i;
8312 
8313 	if (tpr != &tp->napi[0].prodring) {
8314 		for (i = tpr->rx_std_cons_idx; i != tpr->rx_std_prod_idx;
8315 		     i = (i + 1) & tp->rx_std_ring_mask)
8316 			tg3_rx_data_free(tp, &tpr->rx_std_buffers[i],
8317 					tp->rx_pkt_map_sz);
8318 
8319 		if (tg3_flag(tp, JUMBO_CAPABLE)) {
8320 			for (i = tpr->rx_jmb_cons_idx;
8321 			     i != tpr->rx_jmb_prod_idx;
8322 			     i = (i + 1) & tp->rx_jmb_ring_mask) {
8323 				tg3_rx_data_free(tp, &tpr->rx_jmb_buffers[i],
8324 						TG3_RX_JMB_MAP_SZ);
8325 			}
8326 		}
8327 
8328 		return;
8329 	}
8330 
8331 	for (i = 0; i <= tp->rx_std_ring_mask; i++)
8332 		tg3_rx_data_free(tp, &tpr->rx_std_buffers[i],
8333 				tp->rx_pkt_map_sz);
8334 
8335 	if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS)) {
8336 		for (i = 0; i <= tp->rx_jmb_ring_mask; i++)
8337 			tg3_rx_data_free(tp, &tpr->rx_jmb_buffers[i],
8338 					TG3_RX_JMB_MAP_SZ);
8339 	}
8340 }
8341 
8342 /* Initialize rx rings for packet processing.
8343  *
8344  * The chip has been shut down and the driver detached from
8345  * the networking, so no interrupts or new tx packets will
8346  * end up in the driver.  tp->{tx,}lock are held and thus
8347  * we may not sleep.
8348  */
8349 static int tg3_rx_prodring_alloc(struct tg3 *tp,
8350 				 struct tg3_rx_prodring_set *tpr)
8351 {
8352 	u32 i, rx_pkt_dma_sz;
8353 
8354 	tpr->rx_std_cons_idx = 0;
8355 	tpr->rx_std_prod_idx = 0;
8356 	tpr->rx_jmb_cons_idx = 0;
8357 	tpr->rx_jmb_prod_idx = 0;
8358 
8359 	if (tpr != &tp->napi[0].prodring) {
8360 		memset(&tpr->rx_std_buffers[0], 0,
8361 		       TG3_RX_STD_BUFF_RING_SIZE(tp));
8362 		if (tpr->rx_jmb_buffers)
8363 			memset(&tpr->rx_jmb_buffers[0], 0,
8364 			       TG3_RX_JMB_BUFF_RING_SIZE(tp));
8365 		goto done;
8366 	}
8367 
8368 	/* Zero out all descriptors. */
8369 	memset(tpr->rx_std, 0, TG3_RX_STD_RING_BYTES(tp));
8370 
8371 	rx_pkt_dma_sz = TG3_RX_STD_DMA_SZ;
8372 	if (tg3_flag(tp, 5780_CLASS) &&
8373 	    tp->dev->mtu > ETH_DATA_LEN)
8374 		rx_pkt_dma_sz = TG3_RX_JMB_DMA_SZ;
8375 	tp->rx_pkt_map_sz = TG3_RX_DMA_TO_MAP_SZ(rx_pkt_dma_sz);
8376 
8377 	/* Initialize invariants of the rings, we only set this
8378 	 * stuff once.  This works because the card does not
8379 	 * write into the rx buffer posting rings.
8380 	 */
8381 	for (i = 0; i <= tp->rx_std_ring_mask; i++) {
8382 		struct tg3_rx_buffer_desc *rxd;
8383 
8384 		rxd = &tpr->rx_std[i];
8385 		rxd->idx_len = rx_pkt_dma_sz << RXD_LEN_SHIFT;
8386 		rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
8387 		rxd->opaque = (RXD_OPAQUE_RING_STD |
8388 			       (i << RXD_OPAQUE_INDEX_SHIFT));
8389 	}
8390 
8391 	/* Now allocate fresh SKBs for each rx ring. */
8392 	for (i = 0; i < tp->rx_pending; i++) {
8393 		unsigned int frag_size;
8394 
8395 		if (tg3_alloc_rx_data(tp, tpr, RXD_OPAQUE_RING_STD, i,
8396 				      &frag_size) < 0) {
8397 			netdev_warn(tp->dev,
8398 				    "Using a smaller RX standard ring. Only "
8399 				    "%d out of %d buffers were allocated "
8400 				    "successfully\n", i, tp->rx_pending);
8401 			if (i == 0)
8402 				goto initfail;
8403 			tp->rx_pending = i;
8404 			break;
8405 		}
8406 	}
8407 
8408 	if (!tg3_flag(tp, JUMBO_CAPABLE) || tg3_flag(tp, 5780_CLASS))
8409 		goto done;
8410 
8411 	memset(tpr->rx_jmb, 0, TG3_RX_JMB_RING_BYTES(tp));
8412 
8413 	if (!tg3_flag(tp, JUMBO_RING_ENABLE))
8414 		goto done;
8415 
8416 	for (i = 0; i <= tp->rx_jmb_ring_mask; i++) {
8417 		struct tg3_rx_buffer_desc *rxd;
8418 
8419 		rxd = &tpr->rx_jmb[i].std;
8420 		rxd->idx_len = TG3_RX_JMB_DMA_SZ << RXD_LEN_SHIFT;
8421 		rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
8422 				  RXD_FLAG_JUMBO;
8423 		rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
8424 		       (i << RXD_OPAQUE_INDEX_SHIFT));
8425 	}
8426 
8427 	for (i = 0; i < tp->rx_jumbo_pending; i++) {
8428 		unsigned int frag_size;
8429 
8430 		if (tg3_alloc_rx_data(tp, tpr, RXD_OPAQUE_RING_JUMBO, i,
8431 				      &frag_size) < 0) {
8432 			netdev_warn(tp->dev,
8433 				    "Using a smaller RX jumbo ring. Only %d "
8434 				    "out of %d buffers were allocated "
8435 				    "successfully\n", i, tp->rx_jumbo_pending);
8436 			if (i == 0)
8437 				goto initfail;
8438 			tp->rx_jumbo_pending = i;
8439 			break;
8440 		}
8441 	}
8442 
8443 done:
8444 	return 0;
8445 
8446 initfail:
8447 	tg3_rx_prodring_free(tp, tpr);
8448 	return -ENOMEM;
8449 }
8450 
8451 static void tg3_rx_prodring_fini(struct tg3 *tp,
8452 				 struct tg3_rx_prodring_set *tpr)
8453 {
8454 	kfree(tpr->rx_std_buffers);
8455 	tpr->rx_std_buffers = NULL;
8456 	kfree(tpr->rx_jmb_buffers);
8457 	tpr->rx_jmb_buffers = NULL;
8458 	if (tpr->rx_std) {
8459 		dma_free_coherent(&tp->pdev->dev, TG3_RX_STD_RING_BYTES(tp),
8460 				  tpr->rx_std, tpr->rx_std_mapping);
8461 		tpr->rx_std = NULL;
8462 	}
8463 	if (tpr->rx_jmb) {
8464 		dma_free_coherent(&tp->pdev->dev, TG3_RX_JMB_RING_BYTES(tp),
8465 				  tpr->rx_jmb, tpr->rx_jmb_mapping);
8466 		tpr->rx_jmb = NULL;
8467 	}
8468 }
8469 
8470 static int tg3_rx_prodring_init(struct tg3 *tp,
8471 				struct tg3_rx_prodring_set *tpr)
8472 {
8473 	tpr->rx_std_buffers = kzalloc(TG3_RX_STD_BUFF_RING_SIZE(tp),
8474 				      GFP_KERNEL);
8475 	if (!tpr->rx_std_buffers)
8476 		return -ENOMEM;
8477 
8478 	tpr->rx_std = dma_alloc_coherent(&tp->pdev->dev,
8479 					 TG3_RX_STD_RING_BYTES(tp),
8480 					 &tpr->rx_std_mapping,
8481 					 GFP_KERNEL);
8482 	if (!tpr->rx_std)
8483 		goto err_out;
8484 
8485 	if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS)) {
8486 		tpr->rx_jmb_buffers = kzalloc(TG3_RX_JMB_BUFF_RING_SIZE(tp),
8487 					      GFP_KERNEL);
8488 		if (!tpr->rx_jmb_buffers)
8489 			goto err_out;
8490 
8491 		tpr->rx_jmb = dma_alloc_coherent(&tp->pdev->dev,
8492 						 TG3_RX_JMB_RING_BYTES(tp),
8493 						 &tpr->rx_jmb_mapping,
8494 						 GFP_KERNEL);
8495 		if (!tpr->rx_jmb)
8496 			goto err_out;
8497 	}
8498 
8499 	return 0;
8500 
8501 err_out:
8502 	tg3_rx_prodring_fini(tp, tpr);
8503 	return -ENOMEM;
8504 }
8505 
8506 /* Free up pending packets in all rx/tx rings.
8507  *
8508  * The chip has been shut down and the driver detached from
8509  * the networking, so no interrupts or new tx packets will
8510  * end up in the driver.  tp->{tx,}lock is not held and we are not
8511  * in an interrupt context and thus may sleep.
8512  */
8513 static void tg3_free_rings(struct tg3 *tp)
8514 {
8515 	int i, j;
8516 
8517 	for (j = 0; j < tp->irq_cnt; j++) {
8518 		struct tg3_napi *tnapi = &tp->napi[j];
8519 
8520 		tg3_rx_prodring_free(tp, &tnapi->prodring);
8521 
8522 		if (!tnapi->tx_buffers)
8523 			continue;
8524 
8525 		for (i = 0; i < TG3_TX_RING_SIZE; i++) {
8526 			struct sk_buff *skb = tnapi->tx_buffers[i].skb;
8527 
8528 			if (!skb)
8529 				continue;
8530 
8531 			tg3_tx_skb_unmap(tnapi, i,
8532 					 skb_shinfo(skb)->nr_frags - 1);
8533 
8534 			dev_kfree_skb_any(skb);
8535 		}
8536 		netdev_tx_reset_queue(netdev_get_tx_queue(tp->dev, j));
8537 	}
8538 }
8539 
8540 /* Initialize tx/rx rings for packet processing.
8541  *
8542  * The chip has been shut down and the driver detached from
8543  * the networking, so no interrupts or new tx packets will
8544  * end up in the driver.  tp->{tx,}lock are held and thus
8545  * we may not sleep.
8546  */
8547 static int tg3_init_rings(struct tg3 *tp)
8548 {
8549 	int i;
8550 
8551 	/* Free up all the SKBs. */
8552 	tg3_free_rings(tp);
8553 
8554 	for (i = 0; i < tp->irq_cnt; i++) {
8555 		struct tg3_napi *tnapi = &tp->napi[i];
8556 
8557 		tnapi->last_tag = 0;
8558 		tnapi->last_irq_tag = 0;
8559 		tnapi->hw_status->status = 0;
8560 		tnapi->hw_status->status_tag = 0;
8561 		memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
8562 
8563 		tnapi->tx_prod = 0;
8564 		tnapi->tx_cons = 0;
8565 		if (tnapi->tx_ring)
8566 			memset(tnapi->tx_ring, 0, TG3_TX_RING_BYTES);
8567 
8568 		tnapi->rx_rcb_ptr = 0;
8569 		if (tnapi->rx_rcb)
8570 			memset(tnapi->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
8571 
8572 		if (tnapi->prodring.rx_std &&
8573 		    tg3_rx_prodring_alloc(tp, &tnapi->prodring)) {
8574 			tg3_free_rings(tp);
8575 			return -ENOMEM;
8576 		}
8577 	}
8578 
8579 	return 0;
8580 }
8581 
8582 static void tg3_mem_tx_release(struct tg3 *tp)
8583 {
8584 	int i;
8585 
8586 	for (i = 0; i < tp->irq_max; i++) {
8587 		struct tg3_napi *tnapi = &tp->napi[i];
8588 
8589 		if (tnapi->tx_ring) {
8590 			dma_free_coherent(&tp->pdev->dev, TG3_TX_RING_BYTES,
8591 				tnapi->tx_ring, tnapi->tx_desc_mapping);
8592 			tnapi->tx_ring = NULL;
8593 		}
8594 
8595 		kfree(tnapi->tx_buffers);
8596 		tnapi->tx_buffers = NULL;
8597 	}
8598 }
8599 
8600 static int tg3_mem_tx_acquire(struct tg3 *tp)
8601 {
8602 	int i;
8603 	struct tg3_napi *tnapi = &tp->napi[0];
8604 
8605 	/* If multivector TSS is enabled, vector 0 does not handle
8606 	 * tx interrupts.  Don't allocate any resources for it.
8607 	 */
8608 	if (tg3_flag(tp, ENABLE_TSS))
8609 		tnapi++;
8610 
8611 	for (i = 0; i < tp->txq_cnt; i++, tnapi++) {
8612 		tnapi->tx_buffers = kzalloc(sizeof(struct tg3_tx_ring_info) *
8613 					    TG3_TX_RING_SIZE, GFP_KERNEL);
8614 		if (!tnapi->tx_buffers)
8615 			goto err_out;
8616 
8617 		tnapi->tx_ring = dma_alloc_coherent(&tp->pdev->dev,
8618 						    TG3_TX_RING_BYTES,
8619 						    &tnapi->tx_desc_mapping,
8620 						    GFP_KERNEL);
8621 		if (!tnapi->tx_ring)
8622 			goto err_out;
8623 	}
8624 
8625 	return 0;
8626 
8627 err_out:
8628 	tg3_mem_tx_release(tp);
8629 	return -ENOMEM;
8630 }
8631 
8632 static void tg3_mem_rx_release(struct tg3 *tp)
8633 {
8634 	int i;
8635 
8636 	for (i = 0; i < tp->irq_max; i++) {
8637 		struct tg3_napi *tnapi = &tp->napi[i];
8638 
8639 		tg3_rx_prodring_fini(tp, &tnapi->prodring);
8640 
8641 		if (!tnapi->rx_rcb)
8642 			continue;
8643 
8644 		dma_free_coherent(&tp->pdev->dev,
8645 				  TG3_RX_RCB_RING_BYTES(tp),
8646 				  tnapi->rx_rcb,
8647 				  tnapi->rx_rcb_mapping);
8648 		tnapi->rx_rcb = NULL;
8649 	}
8650 }
8651 
8652 static int tg3_mem_rx_acquire(struct tg3 *tp)
8653 {
8654 	unsigned int i, limit;
8655 
8656 	limit = tp->rxq_cnt;
8657 
8658 	/* If RSS is enabled, we need a (dummy) producer ring
8659 	 * set on vector zero.  This is the true hw prodring.
8660 	 */
8661 	if (tg3_flag(tp, ENABLE_RSS))
8662 		limit++;
8663 
8664 	for (i = 0; i < limit; i++) {
8665 		struct tg3_napi *tnapi = &tp->napi[i];
8666 
8667 		if (tg3_rx_prodring_init(tp, &tnapi->prodring))
8668 			goto err_out;
8669 
8670 		/* If multivector RSS is enabled, vector 0
8671 		 * does not handle rx or tx interrupts.
8672 		 * Don't allocate any resources for it.
8673 		 */
8674 		if (!i && tg3_flag(tp, ENABLE_RSS))
8675 			continue;
8676 
8677 		tnapi->rx_rcb = dma_zalloc_coherent(&tp->pdev->dev,
8678 						    TG3_RX_RCB_RING_BYTES(tp),
8679 						    &tnapi->rx_rcb_mapping,
8680 						    GFP_KERNEL);
8681 		if (!tnapi->rx_rcb)
8682 			goto err_out;
8683 	}
8684 
8685 	return 0;
8686 
8687 err_out:
8688 	tg3_mem_rx_release(tp);
8689 	return -ENOMEM;
8690 }
8691 
8692 /*
8693  * Must not be invoked with interrupt sources disabled and
8694  * the hardware shutdown down.
8695  */
8696 static void tg3_free_consistent(struct tg3 *tp)
8697 {
8698 	int i;
8699 
8700 	for (i = 0; i < tp->irq_cnt; i++) {
8701 		struct tg3_napi *tnapi = &tp->napi[i];
8702 
8703 		if (tnapi->hw_status) {
8704 			dma_free_coherent(&tp->pdev->dev, TG3_HW_STATUS_SIZE,
8705 					  tnapi->hw_status,
8706 					  tnapi->status_mapping);
8707 			tnapi->hw_status = NULL;
8708 		}
8709 	}
8710 
8711 	tg3_mem_rx_release(tp);
8712 	tg3_mem_tx_release(tp);
8713 
8714 	if (tp->hw_stats) {
8715 		dma_free_coherent(&tp->pdev->dev, sizeof(struct tg3_hw_stats),
8716 				  tp->hw_stats, tp->stats_mapping);
8717 		tp->hw_stats = NULL;
8718 	}
8719 }
8720 
8721 /*
8722  * Must not be invoked with interrupt sources disabled and
8723  * the hardware shutdown down.  Can sleep.
8724  */
8725 static int tg3_alloc_consistent(struct tg3 *tp)
8726 {
8727 	int i;
8728 
8729 	tp->hw_stats = dma_zalloc_coherent(&tp->pdev->dev,
8730 					   sizeof(struct tg3_hw_stats),
8731 					   &tp->stats_mapping, GFP_KERNEL);
8732 	if (!tp->hw_stats)
8733 		goto err_out;
8734 
8735 	for (i = 0; i < tp->irq_cnt; i++) {
8736 		struct tg3_napi *tnapi = &tp->napi[i];
8737 		struct tg3_hw_status *sblk;
8738 
8739 		tnapi->hw_status = dma_zalloc_coherent(&tp->pdev->dev,
8740 						       TG3_HW_STATUS_SIZE,
8741 						       &tnapi->status_mapping,
8742 						       GFP_KERNEL);
8743 		if (!tnapi->hw_status)
8744 			goto err_out;
8745 
8746 		sblk = tnapi->hw_status;
8747 
8748 		if (tg3_flag(tp, ENABLE_RSS)) {
8749 			u16 *prodptr = NULL;
8750 
8751 			/*
8752 			 * When RSS is enabled, the status block format changes
8753 			 * slightly.  The "rx_jumbo_consumer", "reserved",
8754 			 * and "rx_mini_consumer" members get mapped to the
8755 			 * other three rx return ring producer indexes.
8756 			 */
8757 			switch (i) {
8758 			case 1:
8759 				prodptr = &sblk->idx[0].rx_producer;
8760 				break;
8761 			case 2:
8762 				prodptr = &sblk->rx_jumbo_consumer;
8763 				break;
8764 			case 3:
8765 				prodptr = &sblk->reserved;
8766 				break;
8767 			case 4:
8768 				prodptr = &sblk->rx_mini_consumer;
8769 				break;
8770 			}
8771 			tnapi->rx_rcb_prod_idx = prodptr;
8772 		} else {
8773 			tnapi->rx_rcb_prod_idx = &sblk->idx[0].rx_producer;
8774 		}
8775 	}
8776 
8777 	if (tg3_mem_tx_acquire(tp) || tg3_mem_rx_acquire(tp))
8778 		goto err_out;
8779 
8780 	return 0;
8781 
8782 err_out:
8783 	tg3_free_consistent(tp);
8784 	return -ENOMEM;
8785 }
8786 
8787 #define MAX_WAIT_CNT 1000
8788 
8789 /* To stop a block, clear the enable bit and poll till it
8790  * clears.  tp->lock is held.
8791  */
8792 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, bool silent)
8793 {
8794 	unsigned int i;
8795 	u32 val;
8796 
8797 	if (tg3_flag(tp, 5705_PLUS)) {
8798 		switch (ofs) {
8799 		case RCVLSC_MODE:
8800 		case DMAC_MODE:
8801 		case MBFREE_MODE:
8802 		case BUFMGR_MODE:
8803 		case MEMARB_MODE:
8804 			/* We can't enable/disable these bits of the
8805 			 * 5705/5750, just say success.
8806 			 */
8807 			return 0;
8808 
8809 		default:
8810 			break;
8811 		}
8812 	}
8813 
8814 	val = tr32(ofs);
8815 	val &= ~enable_bit;
8816 	tw32_f(ofs, val);
8817 
8818 	for (i = 0; i < MAX_WAIT_CNT; i++) {
8819 		if (pci_channel_offline(tp->pdev)) {
8820 			dev_err(&tp->pdev->dev,
8821 				"tg3_stop_block device offline, "
8822 				"ofs=%lx enable_bit=%x\n",
8823 				ofs, enable_bit);
8824 			return -ENODEV;
8825 		}
8826 
8827 		udelay(100);
8828 		val = tr32(ofs);
8829 		if ((val & enable_bit) == 0)
8830 			break;
8831 	}
8832 
8833 	if (i == MAX_WAIT_CNT && !silent) {
8834 		dev_err(&tp->pdev->dev,
8835 			"tg3_stop_block timed out, ofs=%lx enable_bit=%x\n",
8836 			ofs, enable_bit);
8837 		return -ENODEV;
8838 	}
8839 
8840 	return 0;
8841 }
8842 
8843 /* tp->lock is held. */
8844 static int tg3_abort_hw(struct tg3 *tp, bool silent)
8845 {
8846 	int i, err;
8847 
8848 	tg3_disable_ints(tp);
8849 
8850 	if (pci_channel_offline(tp->pdev)) {
8851 		tp->rx_mode &= ~(RX_MODE_ENABLE | TX_MODE_ENABLE);
8852 		tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
8853 		err = -ENODEV;
8854 		goto err_no_dev;
8855 	}
8856 
8857 	tp->rx_mode &= ~RX_MODE_ENABLE;
8858 	tw32_f(MAC_RX_MODE, tp->rx_mode);
8859 	udelay(10);
8860 
8861 	err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
8862 	err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
8863 	err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
8864 	err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
8865 	err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
8866 	err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
8867 
8868 	err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
8869 	err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
8870 	err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
8871 	err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
8872 	err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
8873 	err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
8874 	err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
8875 
8876 	tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
8877 	tw32_f(MAC_MODE, tp->mac_mode);
8878 	udelay(40);
8879 
8880 	tp->tx_mode &= ~TX_MODE_ENABLE;
8881 	tw32_f(MAC_TX_MODE, tp->tx_mode);
8882 
8883 	for (i = 0; i < MAX_WAIT_CNT; i++) {
8884 		udelay(100);
8885 		if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
8886 			break;
8887 	}
8888 	if (i >= MAX_WAIT_CNT) {
8889 		dev_err(&tp->pdev->dev,
8890 			"%s timed out, TX_MODE_ENABLE will not clear "
8891 			"MAC_TX_MODE=%08x\n", __func__, tr32(MAC_TX_MODE));
8892 		err |= -ENODEV;
8893 	}
8894 
8895 	err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
8896 	err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
8897 	err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
8898 
8899 	tw32(FTQ_RESET, 0xffffffff);
8900 	tw32(FTQ_RESET, 0x00000000);
8901 
8902 	err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
8903 	err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
8904 
8905 err_no_dev:
8906 	for (i = 0; i < tp->irq_cnt; i++) {
8907 		struct tg3_napi *tnapi = &tp->napi[i];
8908 		if (tnapi->hw_status)
8909 			memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
8910 	}
8911 
8912 	return err;
8913 }
8914 
8915 /* Save PCI command register before chip reset */
8916 static void tg3_save_pci_state(struct tg3 *tp)
8917 {
8918 	pci_read_config_word(tp->pdev, PCI_COMMAND, &tp->pci_cmd);
8919 }
8920 
8921 /* Restore PCI state after chip reset */
8922 static void tg3_restore_pci_state(struct tg3 *tp)
8923 {
8924 	u32 val;
8925 
8926 	/* Re-enable indirect register accesses. */
8927 	pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
8928 			       tp->misc_host_ctrl);
8929 
8930 	/* Set MAX PCI retry to zero. */
8931 	val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
8932 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0 &&
8933 	    tg3_flag(tp, PCIX_MODE))
8934 		val |= PCISTATE_RETRY_SAME_DMA;
8935 	/* Allow reads and writes to the APE register and memory space. */
8936 	if (tg3_flag(tp, ENABLE_APE))
8937 		val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
8938 		       PCISTATE_ALLOW_APE_SHMEM_WR |
8939 		       PCISTATE_ALLOW_APE_PSPACE_WR;
8940 	pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
8941 
8942 	pci_write_config_word(tp->pdev, PCI_COMMAND, tp->pci_cmd);
8943 
8944 	if (!tg3_flag(tp, PCI_EXPRESS)) {
8945 		pci_write_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
8946 				      tp->pci_cacheline_sz);
8947 		pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
8948 				      tp->pci_lat_timer);
8949 	}
8950 
8951 	/* Make sure PCI-X relaxed ordering bit is clear. */
8952 	if (tg3_flag(tp, PCIX_MODE)) {
8953 		u16 pcix_cmd;
8954 
8955 		pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
8956 				     &pcix_cmd);
8957 		pcix_cmd &= ~PCI_X_CMD_ERO;
8958 		pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
8959 				      pcix_cmd);
8960 	}
8961 
8962 	if (tg3_flag(tp, 5780_CLASS)) {
8963 
8964 		/* Chip reset on 5780 will reset MSI enable bit,
8965 		 * so need to restore it.
8966 		 */
8967 		if (tg3_flag(tp, USING_MSI)) {
8968 			u16 ctrl;
8969 
8970 			pci_read_config_word(tp->pdev,
8971 					     tp->msi_cap + PCI_MSI_FLAGS,
8972 					     &ctrl);
8973 			pci_write_config_word(tp->pdev,
8974 					      tp->msi_cap + PCI_MSI_FLAGS,
8975 					      ctrl | PCI_MSI_FLAGS_ENABLE);
8976 			val = tr32(MSGINT_MODE);
8977 			tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
8978 		}
8979 	}
8980 }
8981 
8982 static void tg3_override_clk(struct tg3 *tp)
8983 {
8984 	u32 val;
8985 
8986 	switch (tg3_asic_rev(tp)) {
8987 	case ASIC_REV_5717:
8988 		val = tr32(TG3_CPMU_CLCK_ORIDE_ENABLE);
8989 		tw32(TG3_CPMU_CLCK_ORIDE_ENABLE, val |
8990 		     TG3_CPMU_MAC_ORIDE_ENABLE);
8991 		break;
8992 
8993 	case ASIC_REV_5719:
8994 	case ASIC_REV_5720:
8995 		tw32(TG3_CPMU_CLCK_ORIDE, CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
8996 		break;
8997 
8998 	default:
8999 		return;
9000 	}
9001 }
9002 
9003 static void tg3_restore_clk(struct tg3 *tp)
9004 {
9005 	u32 val;
9006 
9007 	switch (tg3_asic_rev(tp)) {
9008 	case ASIC_REV_5717:
9009 		val = tr32(TG3_CPMU_CLCK_ORIDE_ENABLE);
9010 		tw32(TG3_CPMU_CLCK_ORIDE_ENABLE,
9011 		     val & ~TG3_CPMU_MAC_ORIDE_ENABLE);
9012 		break;
9013 
9014 	case ASIC_REV_5719:
9015 	case ASIC_REV_5720:
9016 		val = tr32(TG3_CPMU_CLCK_ORIDE);
9017 		tw32(TG3_CPMU_CLCK_ORIDE, val & ~CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
9018 		break;
9019 
9020 	default:
9021 		return;
9022 	}
9023 }
9024 
9025 /* tp->lock is held. */
9026 static int tg3_chip_reset(struct tg3 *tp)
9027 	__releases(tp->lock)
9028 	__acquires(tp->lock)
9029 {
9030 	u32 val;
9031 	void (*write_op)(struct tg3 *, u32, u32);
9032 	int i, err;
9033 
9034 	if (!pci_device_is_present(tp->pdev))
9035 		return -ENODEV;
9036 
9037 	tg3_nvram_lock(tp);
9038 
9039 	tg3_ape_lock(tp, TG3_APE_LOCK_GRC);
9040 
9041 	/* No matching tg3_nvram_unlock() after this because
9042 	 * chip reset below will undo the nvram lock.
9043 	 */
9044 	tp->nvram_lock_cnt = 0;
9045 
9046 	/* GRC_MISC_CFG core clock reset will clear the memory
9047 	 * enable bit in PCI register 4 and the MSI enable bit
9048 	 * on some chips, so we save relevant registers here.
9049 	 */
9050 	tg3_save_pci_state(tp);
9051 
9052 	if (tg3_asic_rev(tp) == ASIC_REV_5752 ||
9053 	    tg3_flag(tp, 5755_PLUS))
9054 		tw32(GRC_FASTBOOT_PC, 0);
9055 
9056 	/*
9057 	 * We must avoid the readl() that normally takes place.
9058 	 * It locks machines, causes machine checks, and other
9059 	 * fun things.  So, temporarily disable the 5701
9060 	 * hardware workaround, while we do the reset.
9061 	 */
9062 	write_op = tp->write32;
9063 	if (write_op == tg3_write_flush_reg32)
9064 		tp->write32 = tg3_write32;
9065 
9066 	/* Prevent the irq handler from reading or writing PCI registers
9067 	 * during chip reset when the memory enable bit in the PCI command
9068 	 * register may be cleared.  The chip does not generate interrupt
9069 	 * at this time, but the irq handler may still be called due to irq
9070 	 * sharing or irqpoll.
9071 	 */
9072 	tg3_flag_set(tp, CHIP_RESETTING);
9073 	for (i = 0; i < tp->irq_cnt; i++) {
9074 		struct tg3_napi *tnapi = &tp->napi[i];
9075 		if (tnapi->hw_status) {
9076 			tnapi->hw_status->status = 0;
9077 			tnapi->hw_status->status_tag = 0;
9078 		}
9079 		tnapi->last_tag = 0;
9080 		tnapi->last_irq_tag = 0;
9081 	}
9082 	smp_mb();
9083 
9084 	tg3_full_unlock(tp);
9085 
9086 	for (i = 0; i < tp->irq_cnt; i++)
9087 		synchronize_irq(tp->napi[i].irq_vec);
9088 
9089 	tg3_full_lock(tp, 0);
9090 
9091 	if (tg3_asic_rev(tp) == ASIC_REV_57780) {
9092 		val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
9093 		tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
9094 	}
9095 
9096 	/* do the reset */
9097 	val = GRC_MISC_CFG_CORECLK_RESET;
9098 
9099 	if (tg3_flag(tp, PCI_EXPRESS)) {
9100 		/* Force PCIe 1.0a mode */
9101 		if (tg3_asic_rev(tp) != ASIC_REV_5785 &&
9102 		    !tg3_flag(tp, 57765_PLUS) &&
9103 		    tr32(TG3_PCIE_PHY_TSTCTL) ==
9104 		    (TG3_PCIE_PHY_TSTCTL_PCIE10 | TG3_PCIE_PHY_TSTCTL_PSCRAM))
9105 			tw32(TG3_PCIE_PHY_TSTCTL, TG3_PCIE_PHY_TSTCTL_PSCRAM);
9106 
9107 		if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0) {
9108 			tw32(GRC_MISC_CFG, (1 << 29));
9109 			val |= (1 << 29);
9110 		}
9111 	}
9112 
9113 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
9114 		tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
9115 		tw32(GRC_VCPU_EXT_CTRL,
9116 		     tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
9117 	}
9118 
9119 	/* Set the clock to the highest frequency to avoid timeouts. With link
9120 	 * aware mode, the clock speed could be slow and bootcode does not
9121 	 * complete within the expected time. Override the clock to allow the
9122 	 * bootcode to finish sooner and then restore it.
9123 	 */
9124 	tg3_override_clk(tp);
9125 
9126 	/* Manage gphy power for all CPMU absent PCIe devices. */
9127 	if (tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, CPMU_PRESENT))
9128 		val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
9129 
9130 	tw32(GRC_MISC_CFG, val);
9131 
9132 	/* restore 5701 hardware bug workaround write method */
9133 	tp->write32 = write_op;
9134 
9135 	/* Unfortunately, we have to delay before the PCI read back.
9136 	 * Some 575X chips even will not respond to a PCI cfg access
9137 	 * when the reset command is given to the chip.
9138 	 *
9139 	 * How do these hardware designers expect things to work
9140 	 * properly if the PCI write is posted for a long period
9141 	 * of time?  It is always necessary to have some method by
9142 	 * which a register read back can occur to push the write
9143 	 * out which does the reset.
9144 	 *
9145 	 * For most tg3 variants the trick below was working.
9146 	 * Ho hum...
9147 	 */
9148 	udelay(120);
9149 
9150 	/* Flush PCI posted writes.  The normal MMIO registers
9151 	 * are inaccessible at this time so this is the only
9152 	 * way to make this reliably (actually, this is no longer
9153 	 * the case, see above).  I tried to use indirect
9154 	 * register read/write but this upset some 5701 variants.
9155 	 */
9156 	pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
9157 
9158 	udelay(120);
9159 
9160 	if (tg3_flag(tp, PCI_EXPRESS) && pci_is_pcie(tp->pdev)) {
9161 		u16 val16;
9162 
9163 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A0) {
9164 			int j;
9165 			u32 cfg_val;
9166 
9167 			/* Wait for link training to complete.  */
9168 			for (j = 0; j < 5000; j++)
9169 				udelay(100);
9170 
9171 			pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
9172 			pci_write_config_dword(tp->pdev, 0xc4,
9173 					       cfg_val | (1 << 15));
9174 		}
9175 
9176 		/* Clear the "no snoop" and "relaxed ordering" bits. */
9177 		val16 = PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN;
9178 		/*
9179 		 * Older PCIe devices only support the 128 byte
9180 		 * MPS setting.  Enforce the restriction.
9181 		 */
9182 		if (!tg3_flag(tp, CPMU_PRESENT))
9183 			val16 |= PCI_EXP_DEVCTL_PAYLOAD;
9184 		pcie_capability_clear_word(tp->pdev, PCI_EXP_DEVCTL, val16);
9185 
9186 		/* Clear error status */
9187 		pcie_capability_write_word(tp->pdev, PCI_EXP_DEVSTA,
9188 				      PCI_EXP_DEVSTA_CED |
9189 				      PCI_EXP_DEVSTA_NFED |
9190 				      PCI_EXP_DEVSTA_FED |
9191 				      PCI_EXP_DEVSTA_URD);
9192 	}
9193 
9194 	tg3_restore_pci_state(tp);
9195 
9196 	tg3_flag_clear(tp, CHIP_RESETTING);
9197 	tg3_flag_clear(tp, ERROR_PROCESSED);
9198 
9199 	val = 0;
9200 	if (tg3_flag(tp, 5780_CLASS))
9201 		val = tr32(MEMARB_MODE);
9202 	tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
9203 
9204 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A3) {
9205 		tg3_stop_fw(tp);
9206 		tw32(0x5000, 0x400);
9207 	}
9208 
9209 	if (tg3_flag(tp, IS_SSB_CORE)) {
9210 		/*
9211 		 * BCM4785: In order to avoid repercussions from using
9212 		 * potentially defective internal ROM, stop the Rx RISC CPU,
9213 		 * which is not required.
9214 		 */
9215 		tg3_stop_fw(tp);
9216 		tg3_halt_cpu(tp, RX_CPU_BASE);
9217 	}
9218 
9219 	err = tg3_poll_fw(tp);
9220 	if (err)
9221 		return err;
9222 
9223 	tw32(GRC_MODE, tp->grc_mode);
9224 
9225 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A0) {
9226 		val = tr32(0xc4);
9227 
9228 		tw32(0xc4, val | (1 << 15));
9229 	}
9230 
9231 	if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
9232 	    tg3_asic_rev(tp) == ASIC_REV_5705) {
9233 		tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
9234 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A0)
9235 			tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
9236 		tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
9237 	}
9238 
9239 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
9240 		tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
9241 		val = tp->mac_mode;
9242 	} else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
9243 		tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
9244 		val = tp->mac_mode;
9245 	} else
9246 		val = 0;
9247 
9248 	tw32_f(MAC_MODE, val);
9249 	udelay(40);
9250 
9251 	tg3_ape_unlock(tp, TG3_APE_LOCK_GRC);
9252 
9253 	tg3_mdio_start(tp);
9254 
9255 	if (tg3_flag(tp, PCI_EXPRESS) &&
9256 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0 &&
9257 	    tg3_asic_rev(tp) != ASIC_REV_5785 &&
9258 	    !tg3_flag(tp, 57765_PLUS)) {
9259 		val = tr32(0x7c00);
9260 
9261 		tw32(0x7c00, val | (1 << 25));
9262 	}
9263 
9264 	tg3_restore_clk(tp);
9265 
9266 	/* Reprobe ASF enable state.  */
9267 	tg3_flag_clear(tp, ENABLE_ASF);
9268 	tp->phy_flags &= ~(TG3_PHYFLG_1G_ON_VAUX_OK |
9269 			   TG3_PHYFLG_KEEP_LINK_ON_PWRDN);
9270 
9271 	tg3_flag_clear(tp, ASF_NEW_HANDSHAKE);
9272 	tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
9273 	if (val == NIC_SRAM_DATA_SIG_MAGIC) {
9274 		u32 nic_cfg;
9275 
9276 		tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
9277 		if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
9278 			tg3_flag_set(tp, ENABLE_ASF);
9279 			tp->last_event_jiffies = jiffies;
9280 			if (tg3_flag(tp, 5750_PLUS))
9281 				tg3_flag_set(tp, ASF_NEW_HANDSHAKE);
9282 
9283 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &nic_cfg);
9284 			if (nic_cfg & NIC_SRAM_1G_ON_VAUX_OK)
9285 				tp->phy_flags |= TG3_PHYFLG_1G_ON_VAUX_OK;
9286 			if (nic_cfg & NIC_SRAM_LNK_FLAP_AVOID)
9287 				tp->phy_flags |= TG3_PHYFLG_KEEP_LINK_ON_PWRDN;
9288 		}
9289 	}
9290 
9291 	return 0;
9292 }
9293 
9294 static void tg3_get_nstats(struct tg3 *, struct rtnl_link_stats64 *);
9295 static void tg3_get_estats(struct tg3 *, struct tg3_ethtool_stats *);
9296 static void __tg3_set_rx_mode(struct net_device *);
9297 
9298 /* tp->lock is held. */
9299 static int tg3_halt(struct tg3 *tp, int kind, bool silent)
9300 {
9301 	int err;
9302 
9303 	tg3_stop_fw(tp);
9304 
9305 	tg3_write_sig_pre_reset(tp, kind);
9306 
9307 	tg3_abort_hw(tp, silent);
9308 	err = tg3_chip_reset(tp);
9309 
9310 	__tg3_set_mac_addr(tp, false);
9311 
9312 	tg3_write_sig_legacy(tp, kind);
9313 	tg3_write_sig_post_reset(tp, kind);
9314 
9315 	if (tp->hw_stats) {
9316 		/* Save the stats across chip resets... */
9317 		tg3_get_nstats(tp, &tp->net_stats_prev);
9318 		tg3_get_estats(tp, &tp->estats_prev);
9319 
9320 		/* And make sure the next sample is new data */
9321 		memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
9322 	}
9323 
9324 	return err;
9325 }
9326 
9327 static int tg3_set_mac_addr(struct net_device *dev, void *p)
9328 {
9329 	struct tg3 *tp = netdev_priv(dev);
9330 	struct sockaddr *addr = p;
9331 	int err = 0;
9332 	bool skip_mac_1 = false;
9333 
9334 	if (!is_valid_ether_addr(addr->sa_data))
9335 		return -EADDRNOTAVAIL;
9336 
9337 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
9338 
9339 	if (!netif_running(dev))
9340 		return 0;
9341 
9342 	if (tg3_flag(tp, ENABLE_ASF)) {
9343 		u32 addr0_high, addr0_low, addr1_high, addr1_low;
9344 
9345 		addr0_high = tr32(MAC_ADDR_0_HIGH);
9346 		addr0_low = tr32(MAC_ADDR_0_LOW);
9347 		addr1_high = tr32(MAC_ADDR_1_HIGH);
9348 		addr1_low = tr32(MAC_ADDR_1_LOW);
9349 
9350 		/* Skip MAC addr 1 if ASF is using it. */
9351 		if ((addr0_high != addr1_high || addr0_low != addr1_low) &&
9352 		    !(addr1_high == 0 && addr1_low == 0))
9353 			skip_mac_1 = true;
9354 	}
9355 	spin_lock_bh(&tp->lock);
9356 	__tg3_set_mac_addr(tp, skip_mac_1);
9357 	__tg3_set_rx_mode(dev);
9358 	spin_unlock_bh(&tp->lock);
9359 
9360 	return err;
9361 }
9362 
9363 /* tp->lock is held. */
9364 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
9365 			   dma_addr_t mapping, u32 maxlen_flags,
9366 			   u32 nic_addr)
9367 {
9368 	tg3_write_mem(tp,
9369 		      (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
9370 		      ((u64) mapping >> 32));
9371 	tg3_write_mem(tp,
9372 		      (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
9373 		      ((u64) mapping & 0xffffffff));
9374 	tg3_write_mem(tp,
9375 		      (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
9376 		       maxlen_flags);
9377 
9378 	if (!tg3_flag(tp, 5705_PLUS))
9379 		tg3_write_mem(tp,
9380 			      (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
9381 			      nic_addr);
9382 }
9383 
9384 
9385 static void tg3_coal_tx_init(struct tg3 *tp, struct ethtool_coalesce *ec)
9386 {
9387 	int i = 0;
9388 
9389 	if (!tg3_flag(tp, ENABLE_TSS)) {
9390 		tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
9391 		tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
9392 		tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
9393 	} else {
9394 		tw32(HOSTCC_TXCOL_TICKS, 0);
9395 		tw32(HOSTCC_TXMAX_FRAMES, 0);
9396 		tw32(HOSTCC_TXCOAL_MAXF_INT, 0);
9397 
9398 		for (; i < tp->txq_cnt; i++) {
9399 			u32 reg;
9400 
9401 			reg = HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18;
9402 			tw32(reg, ec->tx_coalesce_usecs);
9403 			reg = HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18;
9404 			tw32(reg, ec->tx_max_coalesced_frames);
9405 			reg = HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18;
9406 			tw32(reg, ec->tx_max_coalesced_frames_irq);
9407 		}
9408 	}
9409 
9410 	for (; i < tp->irq_max - 1; i++) {
9411 		tw32(HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18, 0);
9412 		tw32(HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18, 0);
9413 		tw32(HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
9414 	}
9415 }
9416 
9417 static void tg3_coal_rx_init(struct tg3 *tp, struct ethtool_coalesce *ec)
9418 {
9419 	int i = 0;
9420 	u32 limit = tp->rxq_cnt;
9421 
9422 	if (!tg3_flag(tp, ENABLE_RSS)) {
9423 		tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
9424 		tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
9425 		tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
9426 		limit--;
9427 	} else {
9428 		tw32(HOSTCC_RXCOL_TICKS, 0);
9429 		tw32(HOSTCC_RXMAX_FRAMES, 0);
9430 		tw32(HOSTCC_RXCOAL_MAXF_INT, 0);
9431 	}
9432 
9433 	for (; i < limit; i++) {
9434 		u32 reg;
9435 
9436 		reg = HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18;
9437 		tw32(reg, ec->rx_coalesce_usecs);
9438 		reg = HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18;
9439 		tw32(reg, ec->rx_max_coalesced_frames);
9440 		reg = HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18;
9441 		tw32(reg, ec->rx_max_coalesced_frames_irq);
9442 	}
9443 
9444 	for (; i < tp->irq_max - 1; i++) {
9445 		tw32(HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18, 0);
9446 		tw32(HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18, 0);
9447 		tw32(HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
9448 	}
9449 }
9450 
9451 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
9452 {
9453 	tg3_coal_tx_init(tp, ec);
9454 	tg3_coal_rx_init(tp, ec);
9455 
9456 	if (!tg3_flag(tp, 5705_PLUS)) {
9457 		u32 val = ec->stats_block_coalesce_usecs;
9458 
9459 		tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
9460 		tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
9461 
9462 		if (!tp->link_up)
9463 			val = 0;
9464 
9465 		tw32(HOSTCC_STAT_COAL_TICKS, val);
9466 	}
9467 }
9468 
9469 /* tp->lock is held. */
9470 static void tg3_tx_rcbs_disable(struct tg3 *tp)
9471 {
9472 	u32 txrcb, limit;
9473 
9474 	/* Disable all transmit rings but the first. */
9475 	if (!tg3_flag(tp, 5705_PLUS))
9476 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 16;
9477 	else if (tg3_flag(tp, 5717_PLUS))
9478 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 4;
9479 	else if (tg3_flag(tp, 57765_CLASS) ||
9480 		 tg3_asic_rev(tp) == ASIC_REV_5762)
9481 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 2;
9482 	else
9483 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
9484 
9485 	for (txrcb = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
9486 	     txrcb < limit; txrcb += TG3_BDINFO_SIZE)
9487 		tg3_write_mem(tp, txrcb + TG3_BDINFO_MAXLEN_FLAGS,
9488 			      BDINFO_FLAGS_DISABLED);
9489 }
9490 
9491 /* tp->lock is held. */
9492 static void tg3_tx_rcbs_init(struct tg3 *tp)
9493 {
9494 	int i = 0;
9495 	u32 txrcb = NIC_SRAM_SEND_RCB;
9496 
9497 	if (tg3_flag(tp, ENABLE_TSS))
9498 		i++;
9499 
9500 	for (; i < tp->irq_max; i++, txrcb += TG3_BDINFO_SIZE) {
9501 		struct tg3_napi *tnapi = &tp->napi[i];
9502 
9503 		if (!tnapi->tx_ring)
9504 			continue;
9505 
9506 		tg3_set_bdinfo(tp, txrcb, tnapi->tx_desc_mapping,
9507 			       (TG3_TX_RING_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT),
9508 			       NIC_SRAM_TX_BUFFER_DESC);
9509 	}
9510 }
9511 
9512 /* tp->lock is held. */
9513 static void tg3_rx_ret_rcbs_disable(struct tg3 *tp)
9514 {
9515 	u32 rxrcb, limit;
9516 
9517 	/* Disable all receive return rings but the first. */
9518 	if (tg3_flag(tp, 5717_PLUS))
9519 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 17;
9520 	else if (!tg3_flag(tp, 5705_PLUS))
9521 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 16;
9522 	else if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
9523 		 tg3_asic_rev(tp) == ASIC_REV_5762 ||
9524 		 tg3_flag(tp, 57765_CLASS))
9525 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 4;
9526 	else
9527 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
9528 
9529 	for (rxrcb = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
9530 	     rxrcb < limit; rxrcb += TG3_BDINFO_SIZE)
9531 		tg3_write_mem(tp, rxrcb + TG3_BDINFO_MAXLEN_FLAGS,
9532 			      BDINFO_FLAGS_DISABLED);
9533 }
9534 
9535 /* tp->lock is held. */
9536 static void tg3_rx_ret_rcbs_init(struct tg3 *tp)
9537 {
9538 	int i = 0;
9539 	u32 rxrcb = NIC_SRAM_RCV_RET_RCB;
9540 
9541 	if (tg3_flag(tp, ENABLE_RSS))
9542 		i++;
9543 
9544 	for (; i < tp->irq_max; i++, rxrcb += TG3_BDINFO_SIZE) {
9545 		struct tg3_napi *tnapi = &tp->napi[i];
9546 
9547 		if (!tnapi->rx_rcb)
9548 			continue;
9549 
9550 		tg3_set_bdinfo(tp, rxrcb, tnapi->rx_rcb_mapping,
9551 			       (tp->rx_ret_ring_mask + 1) <<
9552 				BDINFO_FLAGS_MAXLEN_SHIFT, 0);
9553 	}
9554 }
9555 
9556 /* tp->lock is held. */
9557 static void tg3_rings_reset(struct tg3 *tp)
9558 {
9559 	int i;
9560 	u32 stblk;
9561 	struct tg3_napi *tnapi = &tp->napi[0];
9562 
9563 	tg3_tx_rcbs_disable(tp);
9564 
9565 	tg3_rx_ret_rcbs_disable(tp);
9566 
9567 	/* Disable interrupts */
9568 	tw32_mailbox_f(tp->napi[0].int_mbox, 1);
9569 	tp->napi[0].chk_msi_cnt = 0;
9570 	tp->napi[0].last_rx_cons = 0;
9571 	tp->napi[0].last_tx_cons = 0;
9572 
9573 	/* Zero mailbox registers. */
9574 	if (tg3_flag(tp, SUPPORT_MSIX)) {
9575 		for (i = 1; i < tp->irq_max; i++) {
9576 			tp->napi[i].tx_prod = 0;
9577 			tp->napi[i].tx_cons = 0;
9578 			if (tg3_flag(tp, ENABLE_TSS))
9579 				tw32_mailbox(tp->napi[i].prodmbox, 0);
9580 			tw32_rx_mbox(tp->napi[i].consmbox, 0);
9581 			tw32_mailbox_f(tp->napi[i].int_mbox, 1);
9582 			tp->napi[i].chk_msi_cnt = 0;
9583 			tp->napi[i].last_rx_cons = 0;
9584 			tp->napi[i].last_tx_cons = 0;
9585 		}
9586 		if (!tg3_flag(tp, ENABLE_TSS))
9587 			tw32_mailbox(tp->napi[0].prodmbox, 0);
9588 	} else {
9589 		tp->napi[0].tx_prod = 0;
9590 		tp->napi[0].tx_cons = 0;
9591 		tw32_mailbox(tp->napi[0].prodmbox, 0);
9592 		tw32_rx_mbox(tp->napi[0].consmbox, 0);
9593 	}
9594 
9595 	/* Make sure the NIC-based send BD rings are disabled. */
9596 	if (!tg3_flag(tp, 5705_PLUS)) {
9597 		u32 mbox = MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW;
9598 		for (i = 0; i < 16; i++)
9599 			tw32_tx_mbox(mbox + i * 8, 0);
9600 	}
9601 
9602 	/* Clear status block in ram. */
9603 	memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
9604 
9605 	/* Set status block DMA address */
9606 	tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
9607 	     ((u64) tnapi->status_mapping >> 32));
9608 	tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
9609 	     ((u64) tnapi->status_mapping & 0xffffffff));
9610 
9611 	stblk = HOSTCC_STATBLCK_RING1;
9612 
9613 	for (i = 1, tnapi++; i < tp->irq_cnt; i++, tnapi++) {
9614 		u64 mapping = (u64)tnapi->status_mapping;
9615 		tw32(stblk + TG3_64BIT_REG_HIGH, mapping >> 32);
9616 		tw32(stblk + TG3_64BIT_REG_LOW, mapping & 0xffffffff);
9617 		stblk += 8;
9618 
9619 		/* Clear status block in ram. */
9620 		memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
9621 	}
9622 
9623 	tg3_tx_rcbs_init(tp);
9624 	tg3_rx_ret_rcbs_init(tp);
9625 }
9626 
9627 static void tg3_setup_rxbd_thresholds(struct tg3 *tp)
9628 {
9629 	u32 val, bdcache_maxcnt, host_rep_thresh, nic_rep_thresh;
9630 
9631 	if (!tg3_flag(tp, 5750_PLUS) ||
9632 	    tg3_flag(tp, 5780_CLASS) ||
9633 	    tg3_asic_rev(tp) == ASIC_REV_5750 ||
9634 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
9635 	    tg3_flag(tp, 57765_PLUS))
9636 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5700;
9637 	else if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
9638 		 tg3_asic_rev(tp) == ASIC_REV_5787)
9639 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5755;
9640 	else
9641 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5906;
9642 
9643 	nic_rep_thresh = min(bdcache_maxcnt / 2, tp->rx_std_max_post);
9644 	host_rep_thresh = max_t(u32, tp->rx_pending / 8, 1);
9645 
9646 	val = min(nic_rep_thresh, host_rep_thresh);
9647 	tw32(RCVBDI_STD_THRESH, val);
9648 
9649 	if (tg3_flag(tp, 57765_PLUS))
9650 		tw32(STD_REPLENISH_LWM, bdcache_maxcnt);
9651 
9652 	if (!tg3_flag(tp, JUMBO_CAPABLE) || tg3_flag(tp, 5780_CLASS))
9653 		return;
9654 
9655 	bdcache_maxcnt = TG3_SRAM_RX_JMB_BDCACHE_SIZE_5700;
9656 
9657 	host_rep_thresh = max_t(u32, tp->rx_jumbo_pending / 8, 1);
9658 
9659 	val = min(bdcache_maxcnt / 2, host_rep_thresh);
9660 	tw32(RCVBDI_JUMBO_THRESH, val);
9661 
9662 	if (tg3_flag(tp, 57765_PLUS))
9663 		tw32(JMB_REPLENISH_LWM, bdcache_maxcnt);
9664 }
9665 
9666 static inline u32 calc_crc(unsigned char *buf, int len)
9667 {
9668 	u32 reg;
9669 	u32 tmp;
9670 	int j, k;
9671 
9672 	reg = 0xffffffff;
9673 
9674 	for (j = 0; j < len; j++) {
9675 		reg ^= buf[j];
9676 
9677 		for (k = 0; k < 8; k++) {
9678 			tmp = reg & 0x01;
9679 
9680 			reg >>= 1;
9681 
9682 			if (tmp)
9683 				reg ^= 0xedb88320;
9684 		}
9685 	}
9686 
9687 	return ~reg;
9688 }
9689 
9690 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
9691 {
9692 	/* accept or reject all multicast frames */
9693 	tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
9694 	tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
9695 	tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
9696 	tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
9697 }
9698 
9699 static void __tg3_set_rx_mode(struct net_device *dev)
9700 {
9701 	struct tg3 *tp = netdev_priv(dev);
9702 	u32 rx_mode;
9703 
9704 	rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
9705 				  RX_MODE_KEEP_VLAN_TAG);
9706 
9707 #if !defined(CONFIG_VLAN_8021Q) && !defined(CONFIG_VLAN_8021Q_MODULE)
9708 	/* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
9709 	 * flag clear.
9710 	 */
9711 	if (!tg3_flag(tp, ENABLE_ASF))
9712 		rx_mode |= RX_MODE_KEEP_VLAN_TAG;
9713 #endif
9714 
9715 	if (dev->flags & IFF_PROMISC) {
9716 		/* Promiscuous mode. */
9717 		rx_mode |= RX_MODE_PROMISC;
9718 	} else if (dev->flags & IFF_ALLMULTI) {
9719 		/* Accept all multicast. */
9720 		tg3_set_multi(tp, 1);
9721 	} else if (netdev_mc_empty(dev)) {
9722 		/* Reject all multicast. */
9723 		tg3_set_multi(tp, 0);
9724 	} else {
9725 		/* Accept one or more multicast(s). */
9726 		struct netdev_hw_addr *ha;
9727 		u32 mc_filter[4] = { 0, };
9728 		u32 regidx;
9729 		u32 bit;
9730 		u32 crc;
9731 
9732 		netdev_for_each_mc_addr(ha, dev) {
9733 			crc = calc_crc(ha->addr, ETH_ALEN);
9734 			bit = ~crc & 0x7f;
9735 			regidx = (bit & 0x60) >> 5;
9736 			bit &= 0x1f;
9737 			mc_filter[regidx] |= (1 << bit);
9738 		}
9739 
9740 		tw32(MAC_HASH_REG_0, mc_filter[0]);
9741 		tw32(MAC_HASH_REG_1, mc_filter[1]);
9742 		tw32(MAC_HASH_REG_2, mc_filter[2]);
9743 		tw32(MAC_HASH_REG_3, mc_filter[3]);
9744 	}
9745 
9746 	if (netdev_uc_count(dev) > TG3_MAX_UCAST_ADDR(tp)) {
9747 		rx_mode |= RX_MODE_PROMISC;
9748 	} else if (!(dev->flags & IFF_PROMISC)) {
9749 		/* Add all entries into to the mac addr filter list */
9750 		int i = 0;
9751 		struct netdev_hw_addr *ha;
9752 
9753 		netdev_for_each_uc_addr(ha, dev) {
9754 			__tg3_set_one_mac_addr(tp, ha->addr,
9755 					       i + TG3_UCAST_ADDR_IDX(tp));
9756 			i++;
9757 		}
9758 	}
9759 
9760 	if (rx_mode != tp->rx_mode) {
9761 		tp->rx_mode = rx_mode;
9762 		tw32_f(MAC_RX_MODE, rx_mode);
9763 		udelay(10);
9764 	}
9765 }
9766 
9767 static void tg3_rss_init_dflt_indir_tbl(struct tg3 *tp, u32 qcnt)
9768 {
9769 	int i;
9770 
9771 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
9772 		tp->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, qcnt);
9773 }
9774 
9775 static void tg3_rss_check_indir_tbl(struct tg3 *tp)
9776 {
9777 	int i;
9778 
9779 	if (!tg3_flag(tp, SUPPORT_MSIX))
9780 		return;
9781 
9782 	if (tp->rxq_cnt == 1) {
9783 		memset(&tp->rss_ind_tbl[0], 0, sizeof(tp->rss_ind_tbl));
9784 		return;
9785 	}
9786 
9787 	/* Validate table against current IRQ count */
9788 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++) {
9789 		if (tp->rss_ind_tbl[i] >= tp->rxq_cnt)
9790 			break;
9791 	}
9792 
9793 	if (i != TG3_RSS_INDIR_TBL_SIZE)
9794 		tg3_rss_init_dflt_indir_tbl(tp, tp->rxq_cnt);
9795 }
9796 
9797 static void tg3_rss_write_indir_tbl(struct tg3 *tp)
9798 {
9799 	int i = 0;
9800 	u32 reg = MAC_RSS_INDIR_TBL_0;
9801 
9802 	while (i < TG3_RSS_INDIR_TBL_SIZE) {
9803 		u32 val = tp->rss_ind_tbl[i];
9804 		i++;
9805 		for (; i % 8; i++) {
9806 			val <<= 4;
9807 			val |= tp->rss_ind_tbl[i];
9808 		}
9809 		tw32(reg, val);
9810 		reg += 4;
9811 	}
9812 }
9813 
9814 static inline u32 tg3_lso_rd_dma_workaround_bit(struct tg3 *tp)
9815 {
9816 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
9817 		return TG3_LSO_RD_DMA_TX_LENGTH_WA_5719;
9818 	else
9819 		return TG3_LSO_RD_DMA_TX_LENGTH_WA_5720;
9820 }
9821 
9822 /* tp->lock is held. */
9823 static int tg3_reset_hw(struct tg3 *tp, bool reset_phy)
9824 {
9825 	u32 val, rdmac_mode;
9826 	int i, err, limit;
9827 	struct tg3_rx_prodring_set *tpr = &tp->napi[0].prodring;
9828 
9829 	tg3_disable_ints(tp);
9830 
9831 	tg3_stop_fw(tp);
9832 
9833 	tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
9834 
9835 	if (tg3_flag(tp, INIT_COMPLETE))
9836 		tg3_abort_hw(tp, 1);
9837 
9838 	if ((tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
9839 	    !(tp->phy_flags & TG3_PHYFLG_USER_CONFIGURED)) {
9840 		tg3_phy_pull_config(tp);
9841 		tg3_eee_pull_config(tp, NULL);
9842 		tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
9843 	}
9844 
9845 	/* Enable MAC control of LPI */
9846 	if (tp->phy_flags & TG3_PHYFLG_EEE_CAP)
9847 		tg3_setup_eee(tp);
9848 
9849 	if (reset_phy)
9850 		tg3_phy_reset(tp);
9851 
9852 	err = tg3_chip_reset(tp);
9853 	if (err)
9854 		return err;
9855 
9856 	tg3_write_sig_legacy(tp, RESET_KIND_INIT);
9857 
9858 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX) {
9859 		val = tr32(TG3_CPMU_CTRL);
9860 		val &= ~(CPMU_CTRL_LINK_AWARE_MODE | CPMU_CTRL_LINK_IDLE_MODE);
9861 		tw32(TG3_CPMU_CTRL, val);
9862 
9863 		val = tr32(TG3_CPMU_LSPD_10MB_CLK);
9864 		val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
9865 		val |= CPMU_LSPD_10MB_MACCLK_6_25;
9866 		tw32(TG3_CPMU_LSPD_10MB_CLK, val);
9867 
9868 		val = tr32(TG3_CPMU_LNK_AWARE_PWRMD);
9869 		val &= ~CPMU_LNK_AWARE_MACCLK_MASK;
9870 		val |= CPMU_LNK_AWARE_MACCLK_6_25;
9871 		tw32(TG3_CPMU_LNK_AWARE_PWRMD, val);
9872 
9873 		val = tr32(TG3_CPMU_HST_ACC);
9874 		val &= ~CPMU_HST_ACC_MACCLK_MASK;
9875 		val |= CPMU_HST_ACC_MACCLK_6_25;
9876 		tw32(TG3_CPMU_HST_ACC, val);
9877 	}
9878 
9879 	if (tg3_asic_rev(tp) == ASIC_REV_57780) {
9880 		val = tr32(PCIE_PWR_MGMT_THRESH) & ~PCIE_PWR_MGMT_L1_THRESH_MSK;
9881 		val |= PCIE_PWR_MGMT_EXT_ASPM_TMR_EN |
9882 		       PCIE_PWR_MGMT_L1_THRESH_4MS;
9883 		tw32(PCIE_PWR_MGMT_THRESH, val);
9884 
9885 		val = tr32(TG3_PCIE_EIDLE_DELAY) & ~TG3_PCIE_EIDLE_DELAY_MASK;
9886 		tw32(TG3_PCIE_EIDLE_DELAY, val | TG3_PCIE_EIDLE_DELAY_13_CLKS);
9887 
9888 		tw32(TG3_CORR_ERR_STAT, TG3_CORR_ERR_STAT_CLEAR);
9889 
9890 		val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
9891 		tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
9892 	}
9893 
9894 	if (tg3_flag(tp, L1PLLPD_EN)) {
9895 		u32 grc_mode = tr32(GRC_MODE);
9896 
9897 		/* Access the lower 1K of PL PCIE block registers. */
9898 		val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9899 		tw32(GRC_MODE, val | GRC_MODE_PCIE_PL_SEL);
9900 
9901 		val = tr32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL1);
9902 		tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL1,
9903 		     val | TG3_PCIE_PL_LO_PHYCTL1_L1PLLPD_EN);
9904 
9905 		tw32(GRC_MODE, grc_mode);
9906 	}
9907 
9908 	if (tg3_flag(tp, 57765_CLASS)) {
9909 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0) {
9910 			u32 grc_mode = tr32(GRC_MODE);
9911 
9912 			/* Access the lower 1K of PL PCIE block registers. */
9913 			val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9914 			tw32(GRC_MODE, val | GRC_MODE_PCIE_PL_SEL);
9915 
9916 			val = tr32(TG3_PCIE_TLDLPL_PORT +
9917 				   TG3_PCIE_PL_LO_PHYCTL5);
9918 			tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL5,
9919 			     val | TG3_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ);
9920 
9921 			tw32(GRC_MODE, grc_mode);
9922 		}
9923 
9924 		if (tg3_chip_rev(tp) != CHIPREV_57765_AX) {
9925 			u32 grc_mode;
9926 
9927 			/* Fix transmit hangs */
9928 			val = tr32(TG3_CPMU_PADRNG_CTL);
9929 			val |= TG3_CPMU_PADRNG_CTL_RDIV2;
9930 			tw32(TG3_CPMU_PADRNG_CTL, val);
9931 
9932 			grc_mode = tr32(GRC_MODE);
9933 
9934 			/* Access the lower 1K of DL PCIE block registers. */
9935 			val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9936 			tw32(GRC_MODE, val | GRC_MODE_PCIE_DL_SEL);
9937 
9938 			val = tr32(TG3_PCIE_TLDLPL_PORT +
9939 				   TG3_PCIE_DL_LO_FTSMAX);
9940 			val &= ~TG3_PCIE_DL_LO_FTSMAX_MSK;
9941 			tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_DL_LO_FTSMAX,
9942 			     val | TG3_PCIE_DL_LO_FTSMAX_VAL);
9943 
9944 			tw32(GRC_MODE, grc_mode);
9945 		}
9946 
9947 		val = tr32(TG3_CPMU_LSPD_10MB_CLK);
9948 		val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
9949 		val |= CPMU_LSPD_10MB_MACCLK_6_25;
9950 		tw32(TG3_CPMU_LSPD_10MB_CLK, val);
9951 	}
9952 
9953 	/* This works around an issue with Athlon chipsets on
9954 	 * B3 tigon3 silicon.  This bit has no effect on any
9955 	 * other revision.  But do not set this on PCI Express
9956 	 * chips and don't even touch the clocks if the CPMU is present.
9957 	 */
9958 	if (!tg3_flag(tp, CPMU_PRESENT)) {
9959 		if (!tg3_flag(tp, PCI_EXPRESS))
9960 			tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
9961 		tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
9962 	}
9963 
9964 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0 &&
9965 	    tg3_flag(tp, PCIX_MODE)) {
9966 		val = tr32(TG3PCI_PCISTATE);
9967 		val |= PCISTATE_RETRY_SAME_DMA;
9968 		tw32(TG3PCI_PCISTATE, val);
9969 	}
9970 
9971 	if (tg3_flag(tp, ENABLE_APE)) {
9972 		/* Allow reads and writes to the
9973 		 * APE register and memory space.
9974 		 */
9975 		val = tr32(TG3PCI_PCISTATE);
9976 		val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
9977 		       PCISTATE_ALLOW_APE_SHMEM_WR |
9978 		       PCISTATE_ALLOW_APE_PSPACE_WR;
9979 		tw32(TG3PCI_PCISTATE, val);
9980 	}
9981 
9982 	if (tg3_chip_rev(tp) == CHIPREV_5704_BX) {
9983 		/* Enable some hw fixes.  */
9984 		val = tr32(TG3PCI_MSI_DATA);
9985 		val |= (1 << 26) | (1 << 28) | (1 << 29);
9986 		tw32(TG3PCI_MSI_DATA, val);
9987 	}
9988 
9989 	/* Descriptor ring init may make accesses to the
9990 	 * NIC SRAM area to setup the TX descriptors, so we
9991 	 * can only do this after the hardware has been
9992 	 * successfully reset.
9993 	 */
9994 	err = tg3_init_rings(tp);
9995 	if (err)
9996 		return err;
9997 
9998 	if (tg3_flag(tp, 57765_PLUS)) {
9999 		val = tr32(TG3PCI_DMA_RW_CTRL) &
10000 		      ~DMA_RWCTRL_DIS_CACHE_ALIGNMENT;
10001 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0)
10002 			val &= ~DMA_RWCTRL_CRDRDR_RDMA_MRRS_MSK;
10003 		if (!tg3_flag(tp, 57765_CLASS) &&
10004 		    tg3_asic_rev(tp) != ASIC_REV_5717 &&
10005 		    tg3_asic_rev(tp) != ASIC_REV_5762)
10006 			val |= DMA_RWCTRL_TAGGED_STAT_WA;
10007 		tw32(TG3PCI_DMA_RW_CTRL, val | tp->dma_rwctrl);
10008 	} else if (tg3_asic_rev(tp) != ASIC_REV_5784 &&
10009 		   tg3_asic_rev(tp) != ASIC_REV_5761) {
10010 		/* This value is determined during the probe time DMA
10011 		 * engine test, tg3_test_dma.
10012 		 */
10013 		tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
10014 	}
10015 
10016 	tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
10017 			  GRC_MODE_4X_NIC_SEND_RINGS |
10018 			  GRC_MODE_NO_TX_PHDR_CSUM |
10019 			  GRC_MODE_NO_RX_PHDR_CSUM);
10020 	tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
10021 
10022 	/* Pseudo-header checksum is done by hardware logic and not
10023 	 * the offload processers, so make the chip do the pseudo-
10024 	 * header checksums on receive.  For transmit it is more
10025 	 * convenient to do the pseudo-header checksum in software
10026 	 * as Linux does that on transmit for us in all cases.
10027 	 */
10028 	tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
10029 
10030 	val = GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP;
10031 	if (tp->rxptpctl)
10032 		tw32(TG3_RX_PTP_CTL,
10033 		     tp->rxptpctl | TG3_RX_PTP_CTL_HWTS_INTERLOCK);
10034 
10035 	if (tg3_flag(tp, PTP_CAPABLE))
10036 		val |= GRC_MODE_TIME_SYNC_ENABLE;
10037 
10038 	tw32(GRC_MODE, tp->grc_mode | val);
10039 
10040 	/* Setup the timer prescalar register.  Clock is always 66Mhz. */
10041 	val = tr32(GRC_MISC_CFG);
10042 	val &= ~0xff;
10043 	val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
10044 	tw32(GRC_MISC_CFG, val);
10045 
10046 	/* Initialize MBUF/DESC pool. */
10047 	if (tg3_flag(tp, 5750_PLUS)) {
10048 		/* Do nothing.  */
10049 	} else if (tg3_asic_rev(tp) != ASIC_REV_5705) {
10050 		tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
10051 		if (tg3_asic_rev(tp) == ASIC_REV_5704)
10052 			tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
10053 		else
10054 			tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
10055 		tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
10056 		tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
10057 	} else if (tg3_flag(tp, TSO_CAPABLE)) {
10058 		int fw_len;
10059 
10060 		fw_len = tp->fw_len;
10061 		fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
10062 		tw32(BUFMGR_MB_POOL_ADDR,
10063 		     NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
10064 		tw32(BUFMGR_MB_POOL_SIZE,
10065 		     NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
10066 	}
10067 
10068 	if (tp->dev->mtu <= ETH_DATA_LEN) {
10069 		tw32(BUFMGR_MB_RDMA_LOW_WATER,
10070 		     tp->bufmgr_config.mbuf_read_dma_low_water);
10071 		tw32(BUFMGR_MB_MACRX_LOW_WATER,
10072 		     tp->bufmgr_config.mbuf_mac_rx_low_water);
10073 		tw32(BUFMGR_MB_HIGH_WATER,
10074 		     tp->bufmgr_config.mbuf_high_water);
10075 	} else {
10076 		tw32(BUFMGR_MB_RDMA_LOW_WATER,
10077 		     tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
10078 		tw32(BUFMGR_MB_MACRX_LOW_WATER,
10079 		     tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
10080 		tw32(BUFMGR_MB_HIGH_WATER,
10081 		     tp->bufmgr_config.mbuf_high_water_jumbo);
10082 	}
10083 	tw32(BUFMGR_DMA_LOW_WATER,
10084 	     tp->bufmgr_config.dma_low_water);
10085 	tw32(BUFMGR_DMA_HIGH_WATER,
10086 	     tp->bufmgr_config.dma_high_water);
10087 
10088 	val = BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE;
10089 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
10090 		val |= BUFMGR_MODE_NO_TX_UNDERRUN;
10091 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
10092 	    tg3_asic_rev(tp) == ASIC_REV_5762 ||
10093 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10094 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5720_A0)
10095 		val |= BUFMGR_MODE_MBLOW_ATTN_ENAB;
10096 	tw32(BUFMGR_MODE, val);
10097 	for (i = 0; i < 2000; i++) {
10098 		if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
10099 			break;
10100 		udelay(10);
10101 	}
10102 	if (i >= 2000) {
10103 		netdev_err(tp->dev, "%s cannot enable BUFMGR\n", __func__);
10104 		return -ENODEV;
10105 	}
10106 
10107 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5906_A1)
10108 		tw32(ISO_PKT_TX, (tr32(ISO_PKT_TX) & ~0x3) | 0x2);
10109 
10110 	tg3_setup_rxbd_thresholds(tp);
10111 
10112 	/* Initialize TG3_BDINFO's at:
10113 	 *  RCVDBDI_STD_BD:	standard eth size rx ring
10114 	 *  RCVDBDI_JUMBO_BD:	jumbo frame rx ring
10115 	 *  RCVDBDI_MINI_BD:	small frame rx ring (??? does not work)
10116 	 *
10117 	 * like so:
10118 	 *  TG3_BDINFO_HOST_ADDR:	high/low parts of DMA address of ring
10119 	 *  TG3_BDINFO_MAXLEN_FLAGS:	(rx max buffer size << 16) |
10120 	 *                              ring attribute flags
10121 	 *  TG3_BDINFO_NIC_ADDR:	location of descriptors in nic SRAM
10122 	 *
10123 	 * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
10124 	 * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
10125 	 *
10126 	 * The size of each ring is fixed in the firmware, but the location is
10127 	 * configurable.
10128 	 */
10129 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
10130 	     ((u64) tpr->rx_std_mapping >> 32));
10131 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
10132 	     ((u64) tpr->rx_std_mapping & 0xffffffff));
10133 	if (!tg3_flag(tp, 5717_PLUS))
10134 		tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
10135 		     NIC_SRAM_RX_BUFFER_DESC);
10136 
10137 	/* Disable the mini ring */
10138 	if (!tg3_flag(tp, 5705_PLUS))
10139 		tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
10140 		     BDINFO_FLAGS_DISABLED);
10141 
10142 	/* Program the jumbo buffer descriptor ring control
10143 	 * blocks on those devices that have them.
10144 	 */
10145 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10146 	    (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS))) {
10147 
10148 		if (tg3_flag(tp, JUMBO_RING_ENABLE)) {
10149 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
10150 			     ((u64) tpr->rx_jmb_mapping >> 32));
10151 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
10152 			     ((u64) tpr->rx_jmb_mapping & 0xffffffff));
10153 			val = TG3_RX_JMB_RING_SIZE(tp) <<
10154 			      BDINFO_FLAGS_MAXLEN_SHIFT;
10155 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
10156 			     val | BDINFO_FLAGS_USE_EXT_RECV);
10157 			if (!tg3_flag(tp, USE_JUMBO_BDFLAG) ||
10158 			    tg3_flag(tp, 57765_CLASS) ||
10159 			    tg3_asic_rev(tp) == ASIC_REV_5762)
10160 				tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
10161 				     NIC_SRAM_RX_JUMBO_BUFFER_DESC);
10162 		} else {
10163 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
10164 			     BDINFO_FLAGS_DISABLED);
10165 		}
10166 
10167 		if (tg3_flag(tp, 57765_PLUS)) {
10168 			val = TG3_RX_STD_RING_SIZE(tp);
10169 			val <<= BDINFO_FLAGS_MAXLEN_SHIFT;
10170 			val |= (TG3_RX_STD_DMA_SZ << 2);
10171 		} else
10172 			val = TG3_RX_STD_DMA_SZ << BDINFO_FLAGS_MAXLEN_SHIFT;
10173 	} else
10174 		val = TG3_RX_STD_MAX_SIZE_5700 << BDINFO_FLAGS_MAXLEN_SHIFT;
10175 
10176 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS, val);
10177 
10178 	tpr->rx_std_prod_idx = tp->rx_pending;
10179 	tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG, tpr->rx_std_prod_idx);
10180 
10181 	tpr->rx_jmb_prod_idx =
10182 		tg3_flag(tp, JUMBO_RING_ENABLE) ? tp->rx_jumbo_pending : 0;
10183 	tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG, tpr->rx_jmb_prod_idx);
10184 
10185 	tg3_rings_reset(tp);
10186 
10187 	/* Initialize MAC address and backoff seed. */
10188 	__tg3_set_mac_addr(tp, false);
10189 
10190 	/* MTU + ethernet header + FCS + optional VLAN tag */
10191 	tw32(MAC_RX_MTU_SIZE,
10192 	     tp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
10193 
10194 	/* The slot time is changed by tg3_setup_phy if we
10195 	 * run at gigabit with half duplex.
10196 	 */
10197 	val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
10198 	      (6 << TX_LENGTHS_IPG_SHIFT) |
10199 	      (32 << TX_LENGTHS_SLOT_TIME_SHIFT);
10200 
10201 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10202 	    tg3_asic_rev(tp) == ASIC_REV_5762)
10203 		val |= tr32(MAC_TX_LENGTHS) &
10204 		       (TX_LENGTHS_JMB_FRM_LEN_MSK |
10205 			TX_LENGTHS_CNT_DWN_VAL_MSK);
10206 
10207 	tw32(MAC_TX_LENGTHS, val);
10208 
10209 	/* Receive rules. */
10210 	tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
10211 	tw32(RCVLPC_CONFIG, 0x0181);
10212 
10213 	/* Calculate RDMAC_MODE setting early, we need it to determine
10214 	 * the RCVLPC_STATE_ENABLE mask.
10215 	 */
10216 	rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
10217 		      RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
10218 		      RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
10219 		      RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
10220 		      RDMAC_MODE_LNGREAD_ENAB);
10221 
10222 	if (tg3_asic_rev(tp) == ASIC_REV_5717)
10223 		rdmac_mode |= RDMAC_MODE_MULT_DMA_RD_DIS;
10224 
10225 	if (tg3_asic_rev(tp) == ASIC_REV_5784 ||
10226 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10227 	    tg3_asic_rev(tp) == ASIC_REV_57780)
10228 		rdmac_mode |= RDMAC_MODE_BD_SBD_CRPT_ENAB |
10229 			      RDMAC_MODE_MBUF_RBD_CRPT_ENAB |
10230 			      RDMAC_MODE_MBUF_SBD_CRPT_ENAB;
10231 
10232 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
10233 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
10234 		if (tg3_flag(tp, TSO_CAPABLE) &&
10235 		    tg3_asic_rev(tp) == ASIC_REV_5705) {
10236 			rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
10237 		} else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
10238 			   !tg3_flag(tp, IS_5788)) {
10239 			rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
10240 		}
10241 	}
10242 
10243 	if (tg3_flag(tp, PCI_EXPRESS))
10244 		rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
10245 
10246 	if (tg3_asic_rev(tp) == ASIC_REV_57766) {
10247 		tp->dma_limit = 0;
10248 		if (tp->dev->mtu <= ETH_DATA_LEN) {
10249 			rdmac_mode |= RDMAC_MODE_JMB_2K_MMRR;
10250 			tp->dma_limit = TG3_TX_BD_DMA_MAX_2K;
10251 		}
10252 	}
10253 
10254 	if (tg3_flag(tp, HW_TSO_1) ||
10255 	    tg3_flag(tp, HW_TSO_2) ||
10256 	    tg3_flag(tp, HW_TSO_3))
10257 		rdmac_mode |= RDMAC_MODE_IPV4_LSO_EN;
10258 
10259 	if (tg3_flag(tp, 57765_PLUS) ||
10260 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10261 	    tg3_asic_rev(tp) == ASIC_REV_57780)
10262 		rdmac_mode |= RDMAC_MODE_IPV6_LSO_EN;
10263 
10264 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10265 	    tg3_asic_rev(tp) == ASIC_REV_5762)
10266 		rdmac_mode |= tr32(RDMAC_MODE) & RDMAC_MODE_H2BNC_VLAN_DET;
10267 
10268 	if (tg3_asic_rev(tp) == ASIC_REV_5761 ||
10269 	    tg3_asic_rev(tp) == ASIC_REV_5784 ||
10270 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10271 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
10272 	    tg3_flag(tp, 57765_PLUS)) {
10273 		u32 tgtreg;
10274 
10275 		if (tg3_asic_rev(tp) == ASIC_REV_5762)
10276 			tgtreg = TG3_RDMA_RSRVCTRL_REG2;
10277 		else
10278 			tgtreg = TG3_RDMA_RSRVCTRL_REG;
10279 
10280 		val = tr32(tgtreg);
10281 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10282 		    tg3_asic_rev(tp) == ASIC_REV_5762) {
10283 			val &= ~(TG3_RDMA_RSRVCTRL_TXMRGN_MASK |
10284 				 TG3_RDMA_RSRVCTRL_FIFO_LWM_MASK |
10285 				 TG3_RDMA_RSRVCTRL_FIFO_HWM_MASK);
10286 			val |= TG3_RDMA_RSRVCTRL_TXMRGN_320B |
10287 			       TG3_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
10288 			       TG3_RDMA_RSRVCTRL_FIFO_HWM_1_5K;
10289 		}
10290 		tw32(tgtreg, val | TG3_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
10291 	}
10292 
10293 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
10294 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
10295 	    tg3_asic_rev(tp) == ASIC_REV_5762) {
10296 		u32 tgtreg;
10297 
10298 		if (tg3_asic_rev(tp) == ASIC_REV_5762)
10299 			tgtreg = TG3_LSO_RD_DMA_CRPTEN_CTRL2;
10300 		else
10301 			tgtreg = TG3_LSO_RD_DMA_CRPTEN_CTRL;
10302 
10303 		val = tr32(tgtreg);
10304 		tw32(tgtreg, val |
10305 		     TG3_LSO_RD_DMA_CRPTEN_CTRL_BLEN_BD_4K |
10306 		     TG3_LSO_RD_DMA_CRPTEN_CTRL_BLEN_LSO_4K);
10307 	}
10308 
10309 	/* Receive/send statistics. */
10310 	if (tg3_flag(tp, 5750_PLUS)) {
10311 		val = tr32(RCVLPC_STATS_ENABLE);
10312 		val &= ~RCVLPC_STATSENAB_DACK_FIX;
10313 		tw32(RCVLPC_STATS_ENABLE, val);
10314 	} else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
10315 		   tg3_flag(tp, TSO_CAPABLE)) {
10316 		val = tr32(RCVLPC_STATS_ENABLE);
10317 		val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
10318 		tw32(RCVLPC_STATS_ENABLE, val);
10319 	} else {
10320 		tw32(RCVLPC_STATS_ENABLE, 0xffffff);
10321 	}
10322 	tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
10323 	tw32(SNDDATAI_STATSENAB, 0xffffff);
10324 	tw32(SNDDATAI_STATSCTRL,
10325 	     (SNDDATAI_SCTRL_ENABLE |
10326 	      SNDDATAI_SCTRL_FASTUPD));
10327 
10328 	/* Setup host coalescing engine. */
10329 	tw32(HOSTCC_MODE, 0);
10330 	for (i = 0; i < 2000; i++) {
10331 		if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
10332 			break;
10333 		udelay(10);
10334 	}
10335 
10336 	__tg3_set_coalesce(tp, &tp->coal);
10337 
10338 	if (!tg3_flag(tp, 5705_PLUS)) {
10339 		/* Status/statistics block address.  See tg3_timer,
10340 		 * the tg3_periodic_fetch_stats call there, and
10341 		 * tg3_get_stats to see how this works for 5705/5750 chips.
10342 		 */
10343 		tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
10344 		     ((u64) tp->stats_mapping >> 32));
10345 		tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
10346 		     ((u64) tp->stats_mapping & 0xffffffff));
10347 		tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
10348 
10349 		tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
10350 
10351 		/* Clear statistics and status block memory areas */
10352 		for (i = NIC_SRAM_STATS_BLK;
10353 		     i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
10354 		     i += sizeof(u32)) {
10355 			tg3_write_mem(tp, i, 0);
10356 			udelay(40);
10357 		}
10358 	}
10359 
10360 	tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
10361 
10362 	tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
10363 	tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
10364 	if (!tg3_flag(tp, 5705_PLUS))
10365 		tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
10366 
10367 	if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
10368 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
10369 		/* reset to prevent losing 1st rx packet intermittently */
10370 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
10371 		udelay(10);
10372 	}
10373 
10374 	tp->mac_mode |= MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
10375 			MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE |
10376 			MAC_MODE_FHDE_ENABLE;
10377 	if (tg3_flag(tp, ENABLE_APE))
10378 		tp->mac_mode |= MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
10379 	if (!tg3_flag(tp, 5705_PLUS) &&
10380 	    !(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
10381 	    tg3_asic_rev(tp) != ASIC_REV_5700)
10382 		tp->mac_mode |= MAC_MODE_LINK_POLARITY;
10383 	tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
10384 	udelay(40);
10385 
10386 	/* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
10387 	 * If TG3_FLAG_IS_NIC is zero, we should read the
10388 	 * register to preserve the GPIO settings for LOMs. The GPIOs,
10389 	 * whether used as inputs or outputs, are set by boot code after
10390 	 * reset.
10391 	 */
10392 	if (!tg3_flag(tp, IS_NIC)) {
10393 		u32 gpio_mask;
10394 
10395 		gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE1 |
10396 			    GRC_LCLCTRL_GPIO_OE2 | GRC_LCLCTRL_GPIO_OUTPUT0 |
10397 			    GRC_LCLCTRL_GPIO_OUTPUT1 | GRC_LCLCTRL_GPIO_OUTPUT2;
10398 
10399 		if (tg3_asic_rev(tp) == ASIC_REV_5752)
10400 			gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
10401 				     GRC_LCLCTRL_GPIO_OUTPUT3;
10402 
10403 		if (tg3_asic_rev(tp) == ASIC_REV_5755)
10404 			gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
10405 
10406 		tp->grc_local_ctrl &= ~gpio_mask;
10407 		tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
10408 
10409 		/* GPIO1 must be driven high for eeprom write protect */
10410 		if (tg3_flag(tp, EEPROM_WRITE_PROT))
10411 			tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
10412 					       GRC_LCLCTRL_GPIO_OUTPUT1);
10413 	}
10414 	tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10415 	udelay(100);
10416 
10417 	if (tg3_flag(tp, USING_MSIX)) {
10418 		val = tr32(MSGINT_MODE);
10419 		val |= MSGINT_MODE_ENABLE;
10420 		if (tp->irq_cnt > 1)
10421 			val |= MSGINT_MODE_MULTIVEC_EN;
10422 		if (!tg3_flag(tp, 1SHOT_MSI))
10423 			val |= MSGINT_MODE_ONE_SHOT_DISABLE;
10424 		tw32(MSGINT_MODE, val);
10425 	}
10426 
10427 	if (!tg3_flag(tp, 5705_PLUS)) {
10428 		tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
10429 		udelay(40);
10430 	}
10431 
10432 	val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
10433 	       WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
10434 	       WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
10435 	       WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
10436 	       WDMAC_MODE_LNGREAD_ENAB);
10437 
10438 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
10439 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
10440 		if (tg3_flag(tp, TSO_CAPABLE) &&
10441 		    (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A1 ||
10442 		     tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A2)) {
10443 			/* nothing */
10444 		} else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
10445 			   !tg3_flag(tp, IS_5788)) {
10446 			val |= WDMAC_MODE_RX_ACCEL;
10447 		}
10448 	}
10449 
10450 	/* Enable host coalescing bug fix */
10451 	if (tg3_flag(tp, 5755_PLUS))
10452 		val |= WDMAC_MODE_STATUS_TAG_FIX;
10453 
10454 	if (tg3_asic_rev(tp) == ASIC_REV_5785)
10455 		val |= WDMAC_MODE_BURST_ALL_DATA;
10456 
10457 	tw32_f(WDMAC_MODE, val);
10458 	udelay(40);
10459 
10460 	if (tg3_flag(tp, PCIX_MODE)) {
10461 		u16 pcix_cmd;
10462 
10463 		pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
10464 				     &pcix_cmd);
10465 		if (tg3_asic_rev(tp) == ASIC_REV_5703) {
10466 			pcix_cmd &= ~PCI_X_CMD_MAX_READ;
10467 			pcix_cmd |= PCI_X_CMD_READ_2K;
10468 		} else if (tg3_asic_rev(tp) == ASIC_REV_5704) {
10469 			pcix_cmd &= ~(PCI_X_CMD_MAX_SPLIT | PCI_X_CMD_MAX_READ);
10470 			pcix_cmd |= PCI_X_CMD_READ_2K;
10471 		}
10472 		pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
10473 				      pcix_cmd);
10474 	}
10475 
10476 	tw32_f(RDMAC_MODE, rdmac_mode);
10477 	udelay(40);
10478 
10479 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
10480 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
10481 		for (i = 0; i < TG3_NUM_RDMA_CHANNELS; i++) {
10482 			if (tr32(TG3_RDMA_LENGTH + (i << 2)) > TG3_MAX_MTU(tp))
10483 				break;
10484 		}
10485 		if (i < TG3_NUM_RDMA_CHANNELS) {
10486 			val = tr32(TG3_LSO_RD_DMA_CRPTEN_CTRL);
10487 			val |= tg3_lso_rd_dma_workaround_bit(tp);
10488 			tw32(TG3_LSO_RD_DMA_CRPTEN_CTRL, val);
10489 			tg3_flag_set(tp, 5719_5720_RDMA_BUG);
10490 		}
10491 	}
10492 
10493 	tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
10494 	if (!tg3_flag(tp, 5705_PLUS))
10495 		tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
10496 
10497 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
10498 		tw32(SNDDATAC_MODE,
10499 		     SNDDATAC_MODE_ENABLE | SNDDATAC_MODE_CDELAY);
10500 	else
10501 		tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
10502 
10503 	tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
10504 	tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
10505 	val = RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ;
10506 	if (tg3_flag(tp, LRG_PROD_RING_CAP))
10507 		val |= RCVDBDI_MODE_LRG_RING_SZ;
10508 	tw32(RCVDBDI_MODE, val);
10509 	tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
10510 	if (tg3_flag(tp, HW_TSO_1) ||
10511 	    tg3_flag(tp, HW_TSO_2) ||
10512 	    tg3_flag(tp, HW_TSO_3))
10513 		tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
10514 	val = SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE;
10515 	if (tg3_flag(tp, ENABLE_TSS))
10516 		val |= SNDBDI_MODE_MULTI_TXQ_EN;
10517 	tw32(SNDBDI_MODE, val);
10518 	tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
10519 
10520 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0) {
10521 		err = tg3_load_5701_a0_firmware_fix(tp);
10522 		if (err)
10523 			return err;
10524 	}
10525 
10526 	if (tg3_asic_rev(tp) == ASIC_REV_57766) {
10527 		/* Ignore any errors for the firmware download. If download
10528 		 * fails, the device will operate with EEE disabled
10529 		 */
10530 		tg3_load_57766_firmware(tp);
10531 	}
10532 
10533 	if (tg3_flag(tp, TSO_CAPABLE)) {
10534 		err = tg3_load_tso_firmware(tp);
10535 		if (err)
10536 			return err;
10537 	}
10538 
10539 	tp->tx_mode = TX_MODE_ENABLE;
10540 
10541 	if (tg3_flag(tp, 5755_PLUS) ||
10542 	    tg3_asic_rev(tp) == ASIC_REV_5906)
10543 		tp->tx_mode |= TX_MODE_MBUF_LOCKUP_FIX;
10544 
10545 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10546 	    tg3_asic_rev(tp) == ASIC_REV_5762) {
10547 		val = TX_MODE_JMB_FRM_LEN | TX_MODE_CNT_DN_MODE;
10548 		tp->tx_mode &= ~val;
10549 		tp->tx_mode |= tr32(MAC_TX_MODE) & val;
10550 	}
10551 
10552 	tw32_f(MAC_TX_MODE, tp->tx_mode);
10553 	udelay(100);
10554 
10555 	if (tg3_flag(tp, ENABLE_RSS)) {
10556 		u32 rss_key[10];
10557 
10558 		tg3_rss_write_indir_tbl(tp);
10559 
10560 		netdev_rss_key_fill(rss_key, 10 * sizeof(u32));
10561 
10562 		for (i = 0; i < 10 ; i++)
10563 			tw32(MAC_RSS_HASH_KEY_0 + i*4, rss_key[i]);
10564 	}
10565 
10566 	tp->rx_mode = RX_MODE_ENABLE;
10567 	if (tg3_flag(tp, 5755_PLUS))
10568 		tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
10569 
10570 	if (tg3_asic_rev(tp) == ASIC_REV_5762)
10571 		tp->rx_mode |= RX_MODE_IPV4_FRAG_FIX;
10572 
10573 	if (tg3_flag(tp, ENABLE_RSS))
10574 		tp->rx_mode |= RX_MODE_RSS_ENABLE |
10575 			       RX_MODE_RSS_ITBL_HASH_BITS_7 |
10576 			       RX_MODE_RSS_IPV6_HASH_EN |
10577 			       RX_MODE_RSS_TCP_IPV6_HASH_EN |
10578 			       RX_MODE_RSS_IPV4_HASH_EN |
10579 			       RX_MODE_RSS_TCP_IPV4_HASH_EN;
10580 
10581 	tw32_f(MAC_RX_MODE, tp->rx_mode);
10582 	udelay(10);
10583 
10584 	tw32(MAC_LED_CTRL, tp->led_ctrl);
10585 
10586 	tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
10587 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
10588 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
10589 		udelay(10);
10590 	}
10591 	tw32_f(MAC_RX_MODE, tp->rx_mode);
10592 	udelay(10);
10593 
10594 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
10595 		if ((tg3_asic_rev(tp) == ASIC_REV_5704) &&
10596 		    !(tp->phy_flags & TG3_PHYFLG_SERDES_PREEMPHASIS)) {
10597 			/* Set drive transmission level to 1.2V  */
10598 			/* only if the signal pre-emphasis bit is not set  */
10599 			val = tr32(MAC_SERDES_CFG);
10600 			val &= 0xfffff000;
10601 			val |= 0x880;
10602 			tw32(MAC_SERDES_CFG, val);
10603 		}
10604 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A1)
10605 			tw32(MAC_SERDES_CFG, 0x616000);
10606 	}
10607 
10608 	/* Prevent chip from dropping frames when flow control
10609 	 * is enabled.
10610 	 */
10611 	if (tg3_flag(tp, 57765_CLASS))
10612 		val = 1;
10613 	else
10614 		val = 2;
10615 	tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, val);
10616 
10617 	if (tg3_asic_rev(tp) == ASIC_REV_5704 &&
10618 	    (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
10619 		/* Use hardware link auto-negotiation */
10620 		tg3_flag_set(tp, HW_AUTONEG);
10621 	}
10622 
10623 	if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
10624 	    tg3_asic_rev(tp) == ASIC_REV_5714) {
10625 		u32 tmp;
10626 
10627 		tmp = tr32(SERDES_RX_CTRL);
10628 		tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
10629 		tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
10630 		tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
10631 		tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10632 	}
10633 
10634 	if (!tg3_flag(tp, USE_PHYLIB)) {
10635 		if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
10636 			tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
10637 
10638 		err = tg3_setup_phy(tp, false);
10639 		if (err)
10640 			return err;
10641 
10642 		if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
10643 		    !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
10644 			u32 tmp;
10645 
10646 			/* Clear CRC stats. */
10647 			if (!tg3_readphy(tp, MII_TG3_TEST1, &tmp)) {
10648 				tg3_writephy(tp, MII_TG3_TEST1,
10649 					     tmp | MII_TG3_TEST1_CRC_EN);
10650 				tg3_readphy(tp, MII_TG3_RXR_COUNTERS, &tmp);
10651 			}
10652 		}
10653 	}
10654 
10655 	__tg3_set_rx_mode(tp->dev);
10656 
10657 	/* Initialize receive rules. */
10658 	tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
10659 	tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
10660 	tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
10661 	tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
10662 
10663 	if (tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS))
10664 		limit = 8;
10665 	else
10666 		limit = 16;
10667 	if (tg3_flag(tp, ENABLE_ASF))
10668 		limit -= 4;
10669 	switch (limit) {
10670 	case 16:
10671 		tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
10672 	case 15:
10673 		tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
10674 	case 14:
10675 		tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
10676 	case 13:
10677 		tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
10678 	case 12:
10679 		tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
10680 	case 11:
10681 		tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
10682 	case 10:
10683 		tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
10684 	case 9:
10685 		tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
10686 	case 8:
10687 		tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
10688 	case 7:
10689 		tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
10690 	case 6:
10691 		tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
10692 	case 5:
10693 		tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
10694 	case 4:
10695 		/* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
10696 	case 3:
10697 		/* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
10698 	case 2:
10699 	case 1:
10700 
10701 	default:
10702 		break;
10703 	}
10704 
10705 	if (tg3_flag(tp, ENABLE_APE))
10706 		/* Write our heartbeat update interval to APE. */
10707 		tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
10708 				APE_HOST_HEARTBEAT_INT_DISABLE);
10709 
10710 	tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
10711 
10712 	return 0;
10713 }
10714 
10715 /* Called at device open time to get the chip ready for
10716  * packet processing.  Invoked with tp->lock held.
10717  */
10718 static int tg3_init_hw(struct tg3 *tp, bool reset_phy)
10719 {
10720 	/* Chip may have been just powered on. If so, the boot code may still
10721 	 * be running initialization. Wait for it to finish to avoid races in
10722 	 * accessing the hardware.
10723 	 */
10724 	tg3_enable_register_access(tp);
10725 	tg3_poll_fw(tp);
10726 
10727 	tg3_switch_clocks(tp);
10728 
10729 	tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
10730 
10731 	return tg3_reset_hw(tp, reset_phy);
10732 }
10733 
10734 static void tg3_sd_scan_scratchpad(struct tg3 *tp, struct tg3_ocir *ocir)
10735 {
10736 	int i;
10737 
10738 	for (i = 0; i < TG3_SD_NUM_RECS; i++, ocir++) {
10739 		u32 off = i * TG3_OCIR_LEN, len = TG3_OCIR_LEN;
10740 
10741 		tg3_ape_scratchpad_read(tp, (u32 *) ocir, off, len);
10742 		off += len;
10743 
10744 		if (ocir->signature != TG3_OCIR_SIG_MAGIC ||
10745 		    !(ocir->version_flags & TG3_OCIR_FLAG_ACTIVE))
10746 			memset(ocir, 0, TG3_OCIR_LEN);
10747 	}
10748 }
10749 
10750 /* sysfs attributes for hwmon */
10751 static ssize_t tg3_show_temp(struct device *dev,
10752 			     struct device_attribute *devattr, char *buf)
10753 {
10754 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
10755 	struct tg3 *tp = dev_get_drvdata(dev);
10756 	u32 temperature;
10757 
10758 	spin_lock_bh(&tp->lock);
10759 	tg3_ape_scratchpad_read(tp, &temperature, attr->index,
10760 				sizeof(temperature));
10761 	spin_unlock_bh(&tp->lock);
10762 	return sprintf(buf, "%u\n", temperature);
10763 }
10764 
10765 
10766 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, tg3_show_temp, NULL,
10767 			  TG3_TEMP_SENSOR_OFFSET);
10768 static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, tg3_show_temp, NULL,
10769 			  TG3_TEMP_CAUTION_OFFSET);
10770 static SENSOR_DEVICE_ATTR(temp1_max, S_IRUGO, tg3_show_temp, NULL,
10771 			  TG3_TEMP_MAX_OFFSET);
10772 
10773 static struct attribute *tg3_attrs[] = {
10774 	&sensor_dev_attr_temp1_input.dev_attr.attr,
10775 	&sensor_dev_attr_temp1_crit.dev_attr.attr,
10776 	&sensor_dev_attr_temp1_max.dev_attr.attr,
10777 	NULL
10778 };
10779 ATTRIBUTE_GROUPS(tg3);
10780 
10781 static void tg3_hwmon_close(struct tg3 *tp)
10782 {
10783 	if (tp->hwmon_dev) {
10784 		hwmon_device_unregister(tp->hwmon_dev);
10785 		tp->hwmon_dev = NULL;
10786 	}
10787 }
10788 
10789 static void tg3_hwmon_open(struct tg3 *tp)
10790 {
10791 	int i;
10792 	u32 size = 0;
10793 	struct pci_dev *pdev = tp->pdev;
10794 	struct tg3_ocir ocirs[TG3_SD_NUM_RECS];
10795 
10796 	tg3_sd_scan_scratchpad(tp, ocirs);
10797 
10798 	for (i = 0; i < TG3_SD_NUM_RECS; i++) {
10799 		if (!ocirs[i].src_data_length)
10800 			continue;
10801 
10802 		size += ocirs[i].src_hdr_length;
10803 		size += ocirs[i].src_data_length;
10804 	}
10805 
10806 	if (!size)
10807 		return;
10808 
10809 	tp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, "tg3",
10810 							  tp, tg3_groups);
10811 	if (IS_ERR(tp->hwmon_dev)) {
10812 		tp->hwmon_dev = NULL;
10813 		dev_err(&pdev->dev, "Cannot register hwmon device, aborting\n");
10814 	}
10815 }
10816 
10817 
10818 #define TG3_STAT_ADD32(PSTAT, REG) \
10819 do {	u32 __val = tr32(REG); \
10820 	(PSTAT)->low += __val; \
10821 	if ((PSTAT)->low < __val) \
10822 		(PSTAT)->high += 1; \
10823 } while (0)
10824 
10825 static void tg3_periodic_fetch_stats(struct tg3 *tp)
10826 {
10827 	struct tg3_hw_stats *sp = tp->hw_stats;
10828 
10829 	if (!tp->link_up)
10830 		return;
10831 
10832 	TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
10833 	TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
10834 	TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
10835 	TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
10836 	TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
10837 	TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
10838 	TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
10839 	TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
10840 	TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
10841 	TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
10842 	TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
10843 	TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
10844 	TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
10845 	if (unlikely(tg3_flag(tp, 5719_5720_RDMA_BUG) &&
10846 		     (sp->tx_ucast_packets.low + sp->tx_mcast_packets.low +
10847 		      sp->tx_bcast_packets.low) > TG3_NUM_RDMA_CHANNELS)) {
10848 		u32 val;
10849 
10850 		val = tr32(TG3_LSO_RD_DMA_CRPTEN_CTRL);
10851 		val &= ~tg3_lso_rd_dma_workaround_bit(tp);
10852 		tw32(TG3_LSO_RD_DMA_CRPTEN_CTRL, val);
10853 		tg3_flag_clear(tp, 5719_5720_RDMA_BUG);
10854 	}
10855 
10856 	TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
10857 	TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
10858 	TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
10859 	TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
10860 	TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
10861 	TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
10862 	TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
10863 	TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
10864 	TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
10865 	TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
10866 	TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
10867 	TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
10868 	TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
10869 	TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
10870 
10871 	TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
10872 	if (tg3_asic_rev(tp) != ASIC_REV_5717 &&
10873 	    tg3_asic_rev(tp) != ASIC_REV_5762 &&
10874 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5719_A0 &&
10875 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5720_A0) {
10876 		TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
10877 	} else {
10878 		u32 val = tr32(HOSTCC_FLOW_ATTN);
10879 		val = (val & HOSTCC_FLOW_ATTN_MBUF_LWM) ? 1 : 0;
10880 		if (val) {
10881 			tw32(HOSTCC_FLOW_ATTN, HOSTCC_FLOW_ATTN_MBUF_LWM);
10882 			sp->rx_discards.low += val;
10883 			if (sp->rx_discards.low < val)
10884 				sp->rx_discards.high += 1;
10885 		}
10886 		sp->mbuf_lwm_thresh_hit = sp->rx_discards;
10887 	}
10888 	TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
10889 }
10890 
10891 static void tg3_chk_missed_msi(struct tg3 *tp)
10892 {
10893 	u32 i;
10894 
10895 	for (i = 0; i < tp->irq_cnt; i++) {
10896 		struct tg3_napi *tnapi = &tp->napi[i];
10897 
10898 		if (tg3_has_work(tnapi)) {
10899 			if (tnapi->last_rx_cons == tnapi->rx_rcb_ptr &&
10900 			    tnapi->last_tx_cons == tnapi->tx_cons) {
10901 				if (tnapi->chk_msi_cnt < 1) {
10902 					tnapi->chk_msi_cnt++;
10903 					return;
10904 				}
10905 				tg3_msi(0, tnapi);
10906 			}
10907 		}
10908 		tnapi->chk_msi_cnt = 0;
10909 		tnapi->last_rx_cons = tnapi->rx_rcb_ptr;
10910 		tnapi->last_tx_cons = tnapi->tx_cons;
10911 	}
10912 }
10913 
10914 static void tg3_timer(unsigned long __opaque)
10915 {
10916 	struct tg3 *tp = (struct tg3 *) __opaque;
10917 
10918 	spin_lock(&tp->lock);
10919 
10920 	if (tp->irq_sync || tg3_flag(tp, RESET_TASK_PENDING)) {
10921 		spin_unlock(&tp->lock);
10922 		goto restart_timer;
10923 	}
10924 
10925 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
10926 	    tg3_flag(tp, 57765_CLASS))
10927 		tg3_chk_missed_msi(tp);
10928 
10929 	if (tg3_flag(tp, FLUSH_POSTED_WRITES)) {
10930 		/* BCM4785: Flush posted writes from GbE to host memory. */
10931 		tr32(HOSTCC_MODE);
10932 	}
10933 
10934 	if (!tg3_flag(tp, TAGGED_STATUS)) {
10935 		/* All of this garbage is because when using non-tagged
10936 		 * IRQ status the mailbox/status_block protocol the chip
10937 		 * uses with the cpu is race prone.
10938 		 */
10939 		if (tp->napi[0].hw_status->status & SD_STATUS_UPDATED) {
10940 			tw32(GRC_LOCAL_CTRL,
10941 			     tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
10942 		} else {
10943 			tw32(HOSTCC_MODE, tp->coalesce_mode |
10944 			     HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW);
10945 		}
10946 
10947 		if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
10948 			spin_unlock(&tp->lock);
10949 			tg3_reset_task_schedule(tp);
10950 			goto restart_timer;
10951 		}
10952 	}
10953 
10954 	/* This part only runs once per second. */
10955 	if (!--tp->timer_counter) {
10956 		if (tg3_flag(tp, 5705_PLUS))
10957 			tg3_periodic_fetch_stats(tp);
10958 
10959 		if (tp->setlpicnt && !--tp->setlpicnt)
10960 			tg3_phy_eee_enable(tp);
10961 
10962 		if (tg3_flag(tp, USE_LINKCHG_REG)) {
10963 			u32 mac_stat;
10964 			int phy_event;
10965 
10966 			mac_stat = tr32(MAC_STATUS);
10967 
10968 			phy_event = 0;
10969 			if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT) {
10970 				if (mac_stat & MAC_STATUS_MI_INTERRUPT)
10971 					phy_event = 1;
10972 			} else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
10973 				phy_event = 1;
10974 
10975 			if (phy_event)
10976 				tg3_setup_phy(tp, false);
10977 		} else if (tg3_flag(tp, POLL_SERDES)) {
10978 			u32 mac_stat = tr32(MAC_STATUS);
10979 			int need_setup = 0;
10980 
10981 			if (tp->link_up &&
10982 			    (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
10983 				need_setup = 1;
10984 			}
10985 			if (!tp->link_up &&
10986 			    (mac_stat & (MAC_STATUS_PCS_SYNCED |
10987 					 MAC_STATUS_SIGNAL_DET))) {
10988 				need_setup = 1;
10989 			}
10990 			if (need_setup) {
10991 				if (!tp->serdes_counter) {
10992 					tw32_f(MAC_MODE,
10993 					     (tp->mac_mode &
10994 					      ~MAC_MODE_PORT_MODE_MASK));
10995 					udelay(40);
10996 					tw32_f(MAC_MODE, tp->mac_mode);
10997 					udelay(40);
10998 				}
10999 				tg3_setup_phy(tp, false);
11000 			}
11001 		} else if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
11002 			   tg3_flag(tp, 5780_CLASS)) {
11003 			tg3_serdes_parallel_detect(tp);
11004 		} else if (tg3_flag(tp, POLL_CPMU_LINK)) {
11005 			u32 cpmu = tr32(TG3_CPMU_STATUS);
11006 			bool link_up = !((cpmu & TG3_CPMU_STATUS_LINK_MASK) ==
11007 					 TG3_CPMU_STATUS_LINK_MASK);
11008 
11009 			if (link_up != tp->link_up)
11010 				tg3_setup_phy(tp, false);
11011 		}
11012 
11013 		tp->timer_counter = tp->timer_multiplier;
11014 	}
11015 
11016 	/* Heartbeat is only sent once every 2 seconds.
11017 	 *
11018 	 * The heartbeat is to tell the ASF firmware that the host
11019 	 * driver is still alive.  In the event that the OS crashes,
11020 	 * ASF needs to reset the hardware to free up the FIFO space
11021 	 * that may be filled with rx packets destined for the host.
11022 	 * If the FIFO is full, ASF will no longer function properly.
11023 	 *
11024 	 * Unintended resets have been reported on real time kernels
11025 	 * where the timer doesn't run on time.  Netpoll will also have
11026 	 * same problem.
11027 	 *
11028 	 * The new FWCMD_NICDRV_ALIVE3 command tells the ASF firmware
11029 	 * to check the ring condition when the heartbeat is expiring
11030 	 * before doing the reset.  This will prevent most unintended
11031 	 * resets.
11032 	 */
11033 	if (!--tp->asf_counter) {
11034 		if (tg3_flag(tp, ENABLE_ASF) && !tg3_flag(tp, ENABLE_APE)) {
11035 			tg3_wait_for_event_ack(tp);
11036 
11037 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
11038 				      FWCMD_NICDRV_ALIVE3);
11039 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
11040 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX,
11041 				      TG3_FW_UPDATE_TIMEOUT_SEC);
11042 
11043 			tg3_generate_fw_event(tp);
11044 		}
11045 		tp->asf_counter = tp->asf_multiplier;
11046 	}
11047 
11048 	spin_unlock(&tp->lock);
11049 
11050 restart_timer:
11051 	tp->timer.expires = jiffies + tp->timer_offset;
11052 	add_timer(&tp->timer);
11053 }
11054 
11055 static void tg3_timer_init(struct tg3 *tp)
11056 {
11057 	if (tg3_flag(tp, TAGGED_STATUS) &&
11058 	    tg3_asic_rev(tp) != ASIC_REV_5717 &&
11059 	    !tg3_flag(tp, 57765_CLASS))
11060 		tp->timer_offset = HZ;
11061 	else
11062 		tp->timer_offset = HZ / 10;
11063 
11064 	BUG_ON(tp->timer_offset > HZ);
11065 
11066 	tp->timer_multiplier = (HZ / tp->timer_offset);
11067 	tp->asf_multiplier = (HZ / tp->timer_offset) *
11068 			     TG3_FW_UPDATE_FREQ_SEC;
11069 
11070 	init_timer(&tp->timer);
11071 	tp->timer.data = (unsigned long) tp;
11072 	tp->timer.function = tg3_timer;
11073 }
11074 
11075 static void tg3_timer_start(struct tg3 *tp)
11076 {
11077 	tp->asf_counter   = tp->asf_multiplier;
11078 	tp->timer_counter = tp->timer_multiplier;
11079 
11080 	tp->timer.expires = jiffies + tp->timer_offset;
11081 	add_timer(&tp->timer);
11082 }
11083 
11084 static void tg3_timer_stop(struct tg3 *tp)
11085 {
11086 	del_timer_sync(&tp->timer);
11087 }
11088 
11089 /* Restart hardware after configuration changes, self-test, etc.
11090  * Invoked with tp->lock held.
11091  */
11092 static int tg3_restart_hw(struct tg3 *tp, bool reset_phy)
11093 	__releases(tp->lock)
11094 	__acquires(tp->lock)
11095 {
11096 	int err;
11097 
11098 	err = tg3_init_hw(tp, reset_phy);
11099 	if (err) {
11100 		netdev_err(tp->dev,
11101 			   "Failed to re-initialize device, aborting\n");
11102 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11103 		tg3_full_unlock(tp);
11104 		tg3_timer_stop(tp);
11105 		tp->irq_sync = 0;
11106 		tg3_napi_enable(tp);
11107 		dev_close(tp->dev);
11108 		tg3_full_lock(tp, 0);
11109 	}
11110 	return err;
11111 }
11112 
11113 static void tg3_reset_task(struct work_struct *work)
11114 {
11115 	struct tg3 *tp = container_of(work, struct tg3, reset_task);
11116 	int err;
11117 
11118 	rtnl_lock();
11119 	tg3_full_lock(tp, 0);
11120 
11121 	if (!netif_running(tp->dev)) {
11122 		tg3_flag_clear(tp, RESET_TASK_PENDING);
11123 		tg3_full_unlock(tp);
11124 		rtnl_unlock();
11125 		return;
11126 	}
11127 
11128 	tg3_full_unlock(tp);
11129 
11130 	tg3_phy_stop(tp);
11131 
11132 	tg3_netif_stop(tp);
11133 
11134 	tg3_full_lock(tp, 1);
11135 
11136 	if (tg3_flag(tp, TX_RECOVERY_PENDING)) {
11137 		tp->write32_tx_mbox = tg3_write32_tx_mbox;
11138 		tp->write32_rx_mbox = tg3_write_flush_reg32;
11139 		tg3_flag_set(tp, MBOX_WRITE_REORDER);
11140 		tg3_flag_clear(tp, TX_RECOVERY_PENDING);
11141 	}
11142 
11143 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
11144 	err = tg3_init_hw(tp, true);
11145 	if (err)
11146 		goto out;
11147 
11148 	tg3_netif_start(tp);
11149 
11150 out:
11151 	tg3_full_unlock(tp);
11152 
11153 	if (!err)
11154 		tg3_phy_start(tp);
11155 
11156 	tg3_flag_clear(tp, RESET_TASK_PENDING);
11157 	rtnl_unlock();
11158 }
11159 
11160 static int tg3_request_irq(struct tg3 *tp, int irq_num)
11161 {
11162 	irq_handler_t fn;
11163 	unsigned long flags;
11164 	char *name;
11165 	struct tg3_napi *tnapi = &tp->napi[irq_num];
11166 
11167 	if (tp->irq_cnt == 1)
11168 		name = tp->dev->name;
11169 	else {
11170 		name = &tnapi->irq_lbl[0];
11171 		if (tnapi->tx_buffers && tnapi->rx_rcb)
11172 			snprintf(name, IFNAMSIZ,
11173 				 "%s-txrx-%d", tp->dev->name, irq_num);
11174 		else if (tnapi->tx_buffers)
11175 			snprintf(name, IFNAMSIZ,
11176 				 "%s-tx-%d", tp->dev->name, irq_num);
11177 		else if (tnapi->rx_rcb)
11178 			snprintf(name, IFNAMSIZ,
11179 				 "%s-rx-%d", tp->dev->name, irq_num);
11180 		else
11181 			snprintf(name, IFNAMSIZ,
11182 				 "%s-%d", tp->dev->name, irq_num);
11183 		name[IFNAMSIZ-1] = 0;
11184 	}
11185 
11186 	if (tg3_flag(tp, USING_MSI) || tg3_flag(tp, USING_MSIX)) {
11187 		fn = tg3_msi;
11188 		if (tg3_flag(tp, 1SHOT_MSI))
11189 			fn = tg3_msi_1shot;
11190 		flags = 0;
11191 	} else {
11192 		fn = tg3_interrupt;
11193 		if (tg3_flag(tp, TAGGED_STATUS))
11194 			fn = tg3_interrupt_tagged;
11195 		flags = IRQF_SHARED;
11196 	}
11197 
11198 	return request_irq(tnapi->irq_vec, fn, flags, name, tnapi);
11199 }
11200 
11201 static int tg3_test_interrupt(struct tg3 *tp)
11202 {
11203 	struct tg3_napi *tnapi = &tp->napi[0];
11204 	struct net_device *dev = tp->dev;
11205 	int err, i, intr_ok = 0;
11206 	u32 val;
11207 
11208 	if (!netif_running(dev))
11209 		return -ENODEV;
11210 
11211 	tg3_disable_ints(tp);
11212 
11213 	free_irq(tnapi->irq_vec, tnapi);
11214 
11215 	/*
11216 	 * Turn off MSI one shot mode.  Otherwise this test has no
11217 	 * observable way to know whether the interrupt was delivered.
11218 	 */
11219 	if (tg3_flag(tp, 57765_PLUS)) {
11220 		val = tr32(MSGINT_MODE) | MSGINT_MODE_ONE_SHOT_DISABLE;
11221 		tw32(MSGINT_MODE, val);
11222 	}
11223 
11224 	err = request_irq(tnapi->irq_vec, tg3_test_isr,
11225 			  IRQF_SHARED, dev->name, tnapi);
11226 	if (err)
11227 		return err;
11228 
11229 	tnapi->hw_status->status &= ~SD_STATUS_UPDATED;
11230 	tg3_enable_ints(tp);
11231 
11232 	tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
11233 	       tnapi->coal_now);
11234 
11235 	for (i = 0; i < 5; i++) {
11236 		u32 int_mbox, misc_host_ctrl;
11237 
11238 		int_mbox = tr32_mailbox(tnapi->int_mbox);
11239 		misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
11240 
11241 		if ((int_mbox != 0) ||
11242 		    (misc_host_ctrl & MISC_HOST_CTRL_MASK_PCI_INT)) {
11243 			intr_ok = 1;
11244 			break;
11245 		}
11246 
11247 		if (tg3_flag(tp, 57765_PLUS) &&
11248 		    tnapi->hw_status->status_tag != tnapi->last_tag)
11249 			tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
11250 
11251 		msleep(10);
11252 	}
11253 
11254 	tg3_disable_ints(tp);
11255 
11256 	free_irq(tnapi->irq_vec, tnapi);
11257 
11258 	err = tg3_request_irq(tp, 0);
11259 
11260 	if (err)
11261 		return err;
11262 
11263 	if (intr_ok) {
11264 		/* Reenable MSI one shot mode. */
11265 		if (tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, 1SHOT_MSI)) {
11266 			val = tr32(MSGINT_MODE) & ~MSGINT_MODE_ONE_SHOT_DISABLE;
11267 			tw32(MSGINT_MODE, val);
11268 		}
11269 		return 0;
11270 	}
11271 
11272 	return -EIO;
11273 }
11274 
11275 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
11276  * successfully restored
11277  */
11278 static int tg3_test_msi(struct tg3 *tp)
11279 {
11280 	int err;
11281 	u16 pci_cmd;
11282 
11283 	if (!tg3_flag(tp, USING_MSI))
11284 		return 0;
11285 
11286 	/* Turn off SERR reporting in case MSI terminates with Master
11287 	 * Abort.
11288 	 */
11289 	pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11290 	pci_write_config_word(tp->pdev, PCI_COMMAND,
11291 			      pci_cmd & ~PCI_COMMAND_SERR);
11292 
11293 	err = tg3_test_interrupt(tp);
11294 
11295 	pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11296 
11297 	if (!err)
11298 		return 0;
11299 
11300 	/* other failures */
11301 	if (err != -EIO)
11302 		return err;
11303 
11304 	/* MSI test failed, go back to INTx mode */
11305 	netdev_warn(tp->dev, "No interrupt was generated using MSI. Switching "
11306 		    "to INTx mode. Please report this failure to the PCI "
11307 		    "maintainer and include system chipset information\n");
11308 
11309 	free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
11310 
11311 	pci_disable_msi(tp->pdev);
11312 
11313 	tg3_flag_clear(tp, USING_MSI);
11314 	tp->napi[0].irq_vec = tp->pdev->irq;
11315 
11316 	err = tg3_request_irq(tp, 0);
11317 	if (err)
11318 		return err;
11319 
11320 	/* Need to reset the chip because the MSI cycle may have terminated
11321 	 * with Master Abort.
11322 	 */
11323 	tg3_full_lock(tp, 1);
11324 
11325 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11326 	err = tg3_init_hw(tp, true);
11327 
11328 	tg3_full_unlock(tp);
11329 
11330 	if (err)
11331 		free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
11332 
11333 	return err;
11334 }
11335 
11336 static int tg3_request_firmware(struct tg3 *tp)
11337 {
11338 	const struct tg3_firmware_hdr *fw_hdr;
11339 
11340 	if (request_firmware(&tp->fw, tp->fw_needed, &tp->pdev->dev)) {
11341 		netdev_err(tp->dev, "Failed to load firmware \"%s\"\n",
11342 			   tp->fw_needed);
11343 		return -ENOENT;
11344 	}
11345 
11346 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
11347 
11348 	/* Firmware blob starts with version numbers, followed by
11349 	 * start address and _full_ length including BSS sections
11350 	 * (which must be longer than the actual data, of course
11351 	 */
11352 
11353 	tp->fw_len = be32_to_cpu(fw_hdr->len);	/* includes bss */
11354 	if (tp->fw_len < (tp->fw->size - TG3_FW_HDR_LEN)) {
11355 		netdev_err(tp->dev, "bogus length %d in \"%s\"\n",
11356 			   tp->fw_len, tp->fw_needed);
11357 		release_firmware(tp->fw);
11358 		tp->fw = NULL;
11359 		return -EINVAL;
11360 	}
11361 
11362 	/* We no longer need firmware; we have it. */
11363 	tp->fw_needed = NULL;
11364 	return 0;
11365 }
11366 
11367 static u32 tg3_irq_count(struct tg3 *tp)
11368 {
11369 	u32 irq_cnt = max(tp->rxq_cnt, tp->txq_cnt);
11370 
11371 	if (irq_cnt > 1) {
11372 		/* We want as many rx rings enabled as there are cpus.
11373 		 * In multiqueue MSI-X mode, the first MSI-X vector
11374 		 * only deals with link interrupts, etc, so we add
11375 		 * one to the number of vectors we are requesting.
11376 		 */
11377 		irq_cnt = min_t(unsigned, irq_cnt + 1, tp->irq_max);
11378 	}
11379 
11380 	return irq_cnt;
11381 }
11382 
11383 static bool tg3_enable_msix(struct tg3 *tp)
11384 {
11385 	int i, rc;
11386 	struct msix_entry msix_ent[TG3_IRQ_MAX_VECS];
11387 
11388 	tp->txq_cnt = tp->txq_req;
11389 	tp->rxq_cnt = tp->rxq_req;
11390 	if (!tp->rxq_cnt)
11391 		tp->rxq_cnt = netif_get_num_default_rss_queues();
11392 	if (tp->rxq_cnt > tp->rxq_max)
11393 		tp->rxq_cnt = tp->rxq_max;
11394 
11395 	/* Disable multiple TX rings by default.  Simple round-robin hardware
11396 	 * scheduling of the TX rings can cause starvation of rings with
11397 	 * small packets when other rings have TSO or jumbo packets.
11398 	 */
11399 	if (!tp->txq_req)
11400 		tp->txq_cnt = 1;
11401 
11402 	tp->irq_cnt = tg3_irq_count(tp);
11403 
11404 	for (i = 0; i < tp->irq_max; i++) {
11405 		msix_ent[i].entry  = i;
11406 		msix_ent[i].vector = 0;
11407 	}
11408 
11409 	rc = pci_enable_msix_range(tp->pdev, msix_ent, 1, tp->irq_cnt);
11410 	if (rc < 0) {
11411 		return false;
11412 	} else if (rc < tp->irq_cnt) {
11413 		netdev_notice(tp->dev, "Requested %d MSI-X vectors, received %d\n",
11414 			      tp->irq_cnt, rc);
11415 		tp->irq_cnt = rc;
11416 		tp->rxq_cnt = max(rc - 1, 1);
11417 		if (tp->txq_cnt)
11418 			tp->txq_cnt = min(tp->rxq_cnt, tp->txq_max);
11419 	}
11420 
11421 	for (i = 0; i < tp->irq_max; i++)
11422 		tp->napi[i].irq_vec = msix_ent[i].vector;
11423 
11424 	if (netif_set_real_num_rx_queues(tp->dev, tp->rxq_cnt)) {
11425 		pci_disable_msix(tp->pdev);
11426 		return false;
11427 	}
11428 
11429 	if (tp->irq_cnt == 1)
11430 		return true;
11431 
11432 	tg3_flag_set(tp, ENABLE_RSS);
11433 
11434 	if (tp->txq_cnt > 1)
11435 		tg3_flag_set(tp, ENABLE_TSS);
11436 
11437 	netif_set_real_num_tx_queues(tp->dev, tp->txq_cnt);
11438 
11439 	return true;
11440 }
11441 
11442 static void tg3_ints_init(struct tg3 *tp)
11443 {
11444 	if ((tg3_flag(tp, SUPPORT_MSI) || tg3_flag(tp, SUPPORT_MSIX)) &&
11445 	    !tg3_flag(tp, TAGGED_STATUS)) {
11446 		/* All MSI supporting chips should support tagged
11447 		 * status.  Assert that this is the case.
11448 		 */
11449 		netdev_warn(tp->dev,
11450 			    "MSI without TAGGED_STATUS? Not using MSI\n");
11451 		goto defcfg;
11452 	}
11453 
11454 	if (tg3_flag(tp, SUPPORT_MSIX) && tg3_enable_msix(tp))
11455 		tg3_flag_set(tp, USING_MSIX);
11456 	else if (tg3_flag(tp, SUPPORT_MSI) && pci_enable_msi(tp->pdev) == 0)
11457 		tg3_flag_set(tp, USING_MSI);
11458 
11459 	if (tg3_flag(tp, USING_MSI) || tg3_flag(tp, USING_MSIX)) {
11460 		u32 msi_mode = tr32(MSGINT_MODE);
11461 		if (tg3_flag(tp, USING_MSIX) && tp->irq_cnt > 1)
11462 			msi_mode |= MSGINT_MODE_MULTIVEC_EN;
11463 		if (!tg3_flag(tp, 1SHOT_MSI))
11464 			msi_mode |= MSGINT_MODE_ONE_SHOT_DISABLE;
11465 		tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
11466 	}
11467 defcfg:
11468 	if (!tg3_flag(tp, USING_MSIX)) {
11469 		tp->irq_cnt = 1;
11470 		tp->napi[0].irq_vec = tp->pdev->irq;
11471 	}
11472 
11473 	if (tp->irq_cnt == 1) {
11474 		tp->txq_cnt = 1;
11475 		tp->rxq_cnt = 1;
11476 		netif_set_real_num_tx_queues(tp->dev, 1);
11477 		netif_set_real_num_rx_queues(tp->dev, 1);
11478 	}
11479 }
11480 
11481 static void tg3_ints_fini(struct tg3 *tp)
11482 {
11483 	if (tg3_flag(tp, USING_MSIX))
11484 		pci_disable_msix(tp->pdev);
11485 	else if (tg3_flag(tp, USING_MSI))
11486 		pci_disable_msi(tp->pdev);
11487 	tg3_flag_clear(tp, USING_MSI);
11488 	tg3_flag_clear(tp, USING_MSIX);
11489 	tg3_flag_clear(tp, ENABLE_RSS);
11490 	tg3_flag_clear(tp, ENABLE_TSS);
11491 }
11492 
11493 static int tg3_start(struct tg3 *tp, bool reset_phy, bool test_irq,
11494 		     bool init)
11495 {
11496 	struct net_device *dev = tp->dev;
11497 	int i, err;
11498 
11499 	/*
11500 	 * Setup interrupts first so we know how
11501 	 * many NAPI resources to allocate
11502 	 */
11503 	tg3_ints_init(tp);
11504 
11505 	tg3_rss_check_indir_tbl(tp);
11506 
11507 	/* The placement of this call is tied
11508 	 * to the setup and use of Host TX descriptors.
11509 	 */
11510 	err = tg3_alloc_consistent(tp);
11511 	if (err)
11512 		goto out_ints_fini;
11513 
11514 	tg3_napi_init(tp);
11515 
11516 	tg3_napi_enable(tp);
11517 
11518 	for (i = 0; i < tp->irq_cnt; i++) {
11519 		struct tg3_napi *tnapi = &tp->napi[i];
11520 		err = tg3_request_irq(tp, i);
11521 		if (err) {
11522 			for (i--; i >= 0; i--) {
11523 				tnapi = &tp->napi[i];
11524 				free_irq(tnapi->irq_vec, tnapi);
11525 			}
11526 			goto out_napi_fini;
11527 		}
11528 	}
11529 
11530 	tg3_full_lock(tp, 0);
11531 
11532 	if (init)
11533 		tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
11534 
11535 	err = tg3_init_hw(tp, reset_phy);
11536 	if (err) {
11537 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11538 		tg3_free_rings(tp);
11539 	}
11540 
11541 	tg3_full_unlock(tp);
11542 
11543 	if (err)
11544 		goto out_free_irq;
11545 
11546 	if (test_irq && tg3_flag(tp, USING_MSI)) {
11547 		err = tg3_test_msi(tp);
11548 
11549 		if (err) {
11550 			tg3_full_lock(tp, 0);
11551 			tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11552 			tg3_free_rings(tp);
11553 			tg3_full_unlock(tp);
11554 
11555 			goto out_napi_fini;
11556 		}
11557 
11558 		if (!tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, USING_MSI)) {
11559 			u32 val = tr32(PCIE_TRANSACTION_CFG);
11560 
11561 			tw32(PCIE_TRANSACTION_CFG,
11562 			     val | PCIE_TRANS_CFG_1SHOT_MSI);
11563 		}
11564 	}
11565 
11566 	tg3_phy_start(tp);
11567 
11568 	tg3_hwmon_open(tp);
11569 
11570 	tg3_full_lock(tp, 0);
11571 
11572 	tg3_timer_start(tp);
11573 	tg3_flag_set(tp, INIT_COMPLETE);
11574 	tg3_enable_ints(tp);
11575 
11576 	if (init)
11577 		tg3_ptp_init(tp);
11578 	else
11579 		tg3_ptp_resume(tp);
11580 
11581 
11582 	tg3_full_unlock(tp);
11583 
11584 	netif_tx_start_all_queues(dev);
11585 
11586 	/*
11587 	 * Reset loopback feature if it was turned on while the device was down
11588 	 * make sure that it's installed properly now.
11589 	 */
11590 	if (dev->features & NETIF_F_LOOPBACK)
11591 		tg3_set_loopback(dev, dev->features);
11592 
11593 	return 0;
11594 
11595 out_free_irq:
11596 	for (i = tp->irq_cnt - 1; i >= 0; i--) {
11597 		struct tg3_napi *tnapi = &tp->napi[i];
11598 		free_irq(tnapi->irq_vec, tnapi);
11599 	}
11600 
11601 out_napi_fini:
11602 	tg3_napi_disable(tp);
11603 	tg3_napi_fini(tp);
11604 	tg3_free_consistent(tp);
11605 
11606 out_ints_fini:
11607 	tg3_ints_fini(tp);
11608 
11609 	return err;
11610 }
11611 
11612 static void tg3_stop(struct tg3 *tp)
11613 {
11614 	int i;
11615 
11616 	tg3_reset_task_cancel(tp);
11617 	tg3_netif_stop(tp);
11618 
11619 	tg3_timer_stop(tp);
11620 
11621 	tg3_hwmon_close(tp);
11622 
11623 	tg3_phy_stop(tp);
11624 
11625 	tg3_full_lock(tp, 1);
11626 
11627 	tg3_disable_ints(tp);
11628 
11629 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11630 	tg3_free_rings(tp);
11631 	tg3_flag_clear(tp, INIT_COMPLETE);
11632 
11633 	tg3_full_unlock(tp);
11634 
11635 	for (i = tp->irq_cnt - 1; i >= 0; i--) {
11636 		struct tg3_napi *tnapi = &tp->napi[i];
11637 		free_irq(tnapi->irq_vec, tnapi);
11638 	}
11639 
11640 	tg3_ints_fini(tp);
11641 
11642 	tg3_napi_fini(tp);
11643 
11644 	tg3_free_consistent(tp);
11645 }
11646 
11647 static int tg3_open(struct net_device *dev)
11648 {
11649 	struct tg3 *tp = netdev_priv(dev);
11650 	int err;
11651 
11652 	if (tp->pcierr_recovery) {
11653 		netdev_err(dev, "Failed to open device. PCI error recovery "
11654 			   "in progress\n");
11655 		return -EAGAIN;
11656 	}
11657 
11658 	if (tp->fw_needed) {
11659 		err = tg3_request_firmware(tp);
11660 		if (tg3_asic_rev(tp) == ASIC_REV_57766) {
11661 			if (err) {
11662 				netdev_warn(tp->dev, "EEE capability disabled\n");
11663 				tp->phy_flags &= ~TG3_PHYFLG_EEE_CAP;
11664 			} else if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
11665 				netdev_warn(tp->dev, "EEE capability restored\n");
11666 				tp->phy_flags |= TG3_PHYFLG_EEE_CAP;
11667 			}
11668 		} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0) {
11669 			if (err)
11670 				return err;
11671 		} else if (err) {
11672 			netdev_warn(tp->dev, "TSO capability disabled\n");
11673 			tg3_flag_clear(tp, TSO_CAPABLE);
11674 		} else if (!tg3_flag(tp, TSO_CAPABLE)) {
11675 			netdev_notice(tp->dev, "TSO capability restored\n");
11676 			tg3_flag_set(tp, TSO_CAPABLE);
11677 		}
11678 	}
11679 
11680 	tg3_carrier_off(tp);
11681 
11682 	err = tg3_power_up(tp);
11683 	if (err)
11684 		return err;
11685 
11686 	tg3_full_lock(tp, 0);
11687 
11688 	tg3_disable_ints(tp);
11689 	tg3_flag_clear(tp, INIT_COMPLETE);
11690 
11691 	tg3_full_unlock(tp);
11692 
11693 	err = tg3_start(tp,
11694 			!(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN),
11695 			true, true);
11696 	if (err) {
11697 		tg3_frob_aux_power(tp, false);
11698 		pci_set_power_state(tp->pdev, PCI_D3hot);
11699 	}
11700 
11701 	if (tg3_flag(tp, PTP_CAPABLE)) {
11702 		tp->ptp_clock = ptp_clock_register(&tp->ptp_info,
11703 						   &tp->pdev->dev);
11704 		if (IS_ERR(tp->ptp_clock))
11705 			tp->ptp_clock = NULL;
11706 	}
11707 
11708 	return err;
11709 }
11710 
11711 static int tg3_close(struct net_device *dev)
11712 {
11713 	struct tg3 *tp = netdev_priv(dev);
11714 
11715 	if (tp->pcierr_recovery) {
11716 		netdev_err(dev, "Failed to close device. PCI error recovery "
11717 			   "in progress\n");
11718 		return -EAGAIN;
11719 	}
11720 
11721 	tg3_ptp_fini(tp);
11722 
11723 	tg3_stop(tp);
11724 
11725 	/* Clear stats across close / open calls */
11726 	memset(&tp->net_stats_prev, 0, sizeof(tp->net_stats_prev));
11727 	memset(&tp->estats_prev, 0, sizeof(tp->estats_prev));
11728 
11729 	if (pci_device_is_present(tp->pdev)) {
11730 		tg3_power_down_prepare(tp);
11731 
11732 		tg3_carrier_off(tp);
11733 	}
11734 	return 0;
11735 }
11736 
11737 static inline u64 get_stat64(tg3_stat64_t *val)
11738 {
11739        return ((u64)val->high << 32) | ((u64)val->low);
11740 }
11741 
11742 static u64 tg3_calc_crc_errors(struct tg3 *tp)
11743 {
11744 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11745 
11746 	if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
11747 	    (tg3_asic_rev(tp) == ASIC_REV_5700 ||
11748 	     tg3_asic_rev(tp) == ASIC_REV_5701)) {
11749 		u32 val;
11750 
11751 		if (!tg3_readphy(tp, MII_TG3_TEST1, &val)) {
11752 			tg3_writephy(tp, MII_TG3_TEST1,
11753 				     val | MII_TG3_TEST1_CRC_EN);
11754 			tg3_readphy(tp, MII_TG3_RXR_COUNTERS, &val);
11755 		} else
11756 			val = 0;
11757 
11758 		tp->phy_crc_errors += val;
11759 
11760 		return tp->phy_crc_errors;
11761 	}
11762 
11763 	return get_stat64(&hw_stats->rx_fcs_errors);
11764 }
11765 
11766 #define ESTAT_ADD(member) \
11767 	estats->member =	old_estats->member + \
11768 				get_stat64(&hw_stats->member)
11769 
11770 static void tg3_get_estats(struct tg3 *tp, struct tg3_ethtool_stats *estats)
11771 {
11772 	struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
11773 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11774 
11775 	ESTAT_ADD(rx_octets);
11776 	ESTAT_ADD(rx_fragments);
11777 	ESTAT_ADD(rx_ucast_packets);
11778 	ESTAT_ADD(rx_mcast_packets);
11779 	ESTAT_ADD(rx_bcast_packets);
11780 	ESTAT_ADD(rx_fcs_errors);
11781 	ESTAT_ADD(rx_align_errors);
11782 	ESTAT_ADD(rx_xon_pause_rcvd);
11783 	ESTAT_ADD(rx_xoff_pause_rcvd);
11784 	ESTAT_ADD(rx_mac_ctrl_rcvd);
11785 	ESTAT_ADD(rx_xoff_entered);
11786 	ESTAT_ADD(rx_frame_too_long_errors);
11787 	ESTAT_ADD(rx_jabbers);
11788 	ESTAT_ADD(rx_undersize_packets);
11789 	ESTAT_ADD(rx_in_length_errors);
11790 	ESTAT_ADD(rx_out_length_errors);
11791 	ESTAT_ADD(rx_64_or_less_octet_packets);
11792 	ESTAT_ADD(rx_65_to_127_octet_packets);
11793 	ESTAT_ADD(rx_128_to_255_octet_packets);
11794 	ESTAT_ADD(rx_256_to_511_octet_packets);
11795 	ESTAT_ADD(rx_512_to_1023_octet_packets);
11796 	ESTAT_ADD(rx_1024_to_1522_octet_packets);
11797 	ESTAT_ADD(rx_1523_to_2047_octet_packets);
11798 	ESTAT_ADD(rx_2048_to_4095_octet_packets);
11799 	ESTAT_ADD(rx_4096_to_8191_octet_packets);
11800 	ESTAT_ADD(rx_8192_to_9022_octet_packets);
11801 
11802 	ESTAT_ADD(tx_octets);
11803 	ESTAT_ADD(tx_collisions);
11804 	ESTAT_ADD(tx_xon_sent);
11805 	ESTAT_ADD(tx_xoff_sent);
11806 	ESTAT_ADD(tx_flow_control);
11807 	ESTAT_ADD(tx_mac_errors);
11808 	ESTAT_ADD(tx_single_collisions);
11809 	ESTAT_ADD(tx_mult_collisions);
11810 	ESTAT_ADD(tx_deferred);
11811 	ESTAT_ADD(tx_excessive_collisions);
11812 	ESTAT_ADD(tx_late_collisions);
11813 	ESTAT_ADD(tx_collide_2times);
11814 	ESTAT_ADD(tx_collide_3times);
11815 	ESTAT_ADD(tx_collide_4times);
11816 	ESTAT_ADD(tx_collide_5times);
11817 	ESTAT_ADD(tx_collide_6times);
11818 	ESTAT_ADD(tx_collide_7times);
11819 	ESTAT_ADD(tx_collide_8times);
11820 	ESTAT_ADD(tx_collide_9times);
11821 	ESTAT_ADD(tx_collide_10times);
11822 	ESTAT_ADD(tx_collide_11times);
11823 	ESTAT_ADD(tx_collide_12times);
11824 	ESTAT_ADD(tx_collide_13times);
11825 	ESTAT_ADD(tx_collide_14times);
11826 	ESTAT_ADD(tx_collide_15times);
11827 	ESTAT_ADD(tx_ucast_packets);
11828 	ESTAT_ADD(tx_mcast_packets);
11829 	ESTAT_ADD(tx_bcast_packets);
11830 	ESTAT_ADD(tx_carrier_sense_errors);
11831 	ESTAT_ADD(tx_discards);
11832 	ESTAT_ADD(tx_errors);
11833 
11834 	ESTAT_ADD(dma_writeq_full);
11835 	ESTAT_ADD(dma_write_prioq_full);
11836 	ESTAT_ADD(rxbds_empty);
11837 	ESTAT_ADD(rx_discards);
11838 	ESTAT_ADD(rx_errors);
11839 	ESTAT_ADD(rx_threshold_hit);
11840 
11841 	ESTAT_ADD(dma_readq_full);
11842 	ESTAT_ADD(dma_read_prioq_full);
11843 	ESTAT_ADD(tx_comp_queue_full);
11844 
11845 	ESTAT_ADD(ring_set_send_prod_index);
11846 	ESTAT_ADD(ring_status_update);
11847 	ESTAT_ADD(nic_irqs);
11848 	ESTAT_ADD(nic_avoided_irqs);
11849 	ESTAT_ADD(nic_tx_threshold_hit);
11850 
11851 	ESTAT_ADD(mbuf_lwm_thresh_hit);
11852 }
11853 
11854 static void tg3_get_nstats(struct tg3 *tp, struct rtnl_link_stats64 *stats)
11855 {
11856 	struct rtnl_link_stats64 *old_stats = &tp->net_stats_prev;
11857 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11858 
11859 	stats->rx_packets = old_stats->rx_packets +
11860 		get_stat64(&hw_stats->rx_ucast_packets) +
11861 		get_stat64(&hw_stats->rx_mcast_packets) +
11862 		get_stat64(&hw_stats->rx_bcast_packets);
11863 
11864 	stats->tx_packets = old_stats->tx_packets +
11865 		get_stat64(&hw_stats->tx_ucast_packets) +
11866 		get_stat64(&hw_stats->tx_mcast_packets) +
11867 		get_stat64(&hw_stats->tx_bcast_packets);
11868 
11869 	stats->rx_bytes = old_stats->rx_bytes +
11870 		get_stat64(&hw_stats->rx_octets);
11871 	stats->tx_bytes = old_stats->tx_bytes +
11872 		get_stat64(&hw_stats->tx_octets);
11873 
11874 	stats->rx_errors = old_stats->rx_errors +
11875 		get_stat64(&hw_stats->rx_errors);
11876 	stats->tx_errors = old_stats->tx_errors +
11877 		get_stat64(&hw_stats->tx_errors) +
11878 		get_stat64(&hw_stats->tx_mac_errors) +
11879 		get_stat64(&hw_stats->tx_carrier_sense_errors) +
11880 		get_stat64(&hw_stats->tx_discards);
11881 
11882 	stats->multicast = old_stats->multicast +
11883 		get_stat64(&hw_stats->rx_mcast_packets);
11884 	stats->collisions = old_stats->collisions +
11885 		get_stat64(&hw_stats->tx_collisions);
11886 
11887 	stats->rx_length_errors = old_stats->rx_length_errors +
11888 		get_stat64(&hw_stats->rx_frame_too_long_errors) +
11889 		get_stat64(&hw_stats->rx_undersize_packets);
11890 
11891 	stats->rx_frame_errors = old_stats->rx_frame_errors +
11892 		get_stat64(&hw_stats->rx_align_errors);
11893 	stats->tx_aborted_errors = old_stats->tx_aborted_errors +
11894 		get_stat64(&hw_stats->tx_discards);
11895 	stats->tx_carrier_errors = old_stats->tx_carrier_errors +
11896 		get_stat64(&hw_stats->tx_carrier_sense_errors);
11897 
11898 	stats->rx_crc_errors = old_stats->rx_crc_errors +
11899 		tg3_calc_crc_errors(tp);
11900 
11901 	stats->rx_missed_errors = old_stats->rx_missed_errors +
11902 		get_stat64(&hw_stats->rx_discards);
11903 
11904 	stats->rx_dropped = tp->rx_dropped;
11905 	stats->tx_dropped = tp->tx_dropped;
11906 }
11907 
11908 static int tg3_get_regs_len(struct net_device *dev)
11909 {
11910 	return TG3_REG_BLK_SIZE;
11911 }
11912 
11913 static void tg3_get_regs(struct net_device *dev,
11914 		struct ethtool_regs *regs, void *_p)
11915 {
11916 	struct tg3 *tp = netdev_priv(dev);
11917 
11918 	regs->version = 0;
11919 
11920 	memset(_p, 0, TG3_REG_BLK_SIZE);
11921 
11922 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
11923 		return;
11924 
11925 	tg3_full_lock(tp, 0);
11926 
11927 	tg3_dump_legacy_regs(tp, (u32 *)_p);
11928 
11929 	tg3_full_unlock(tp);
11930 }
11931 
11932 static int tg3_get_eeprom_len(struct net_device *dev)
11933 {
11934 	struct tg3 *tp = netdev_priv(dev);
11935 
11936 	return tp->nvram_size;
11937 }
11938 
11939 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
11940 {
11941 	struct tg3 *tp = netdev_priv(dev);
11942 	int ret, cpmu_restore = 0;
11943 	u8  *pd;
11944 	u32 i, offset, len, b_offset, b_count, cpmu_val = 0;
11945 	__be32 val;
11946 
11947 	if (tg3_flag(tp, NO_NVRAM))
11948 		return -EINVAL;
11949 
11950 	offset = eeprom->offset;
11951 	len = eeprom->len;
11952 	eeprom->len = 0;
11953 
11954 	eeprom->magic = TG3_EEPROM_MAGIC;
11955 
11956 	/* Override clock, link aware and link idle modes */
11957 	if (tg3_flag(tp, CPMU_PRESENT)) {
11958 		cpmu_val = tr32(TG3_CPMU_CTRL);
11959 		if (cpmu_val & (CPMU_CTRL_LINK_AWARE_MODE |
11960 				CPMU_CTRL_LINK_IDLE_MODE)) {
11961 			tw32(TG3_CPMU_CTRL, cpmu_val &
11962 					    ~(CPMU_CTRL_LINK_AWARE_MODE |
11963 					     CPMU_CTRL_LINK_IDLE_MODE));
11964 			cpmu_restore = 1;
11965 		}
11966 	}
11967 	tg3_override_clk(tp);
11968 
11969 	if (offset & 3) {
11970 		/* adjustments to start on required 4 byte boundary */
11971 		b_offset = offset & 3;
11972 		b_count = 4 - b_offset;
11973 		if (b_count > len) {
11974 			/* i.e. offset=1 len=2 */
11975 			b_count = len;
11976 		}
11977 		ret = tg3_nvram_read_be32(tp, offset-b_offset, &val);
11978 		if (ret)
11979 			goto eeprom_done;
11980 		memcpy(data, ((char *)&val) + b_offset, b_count);
11981 		len -= b_count;
11982 		offset += b_count;
11983 		eeprom->len += b_count;
11984 	}
11985 
11986 	/* read bytes up to the last 4 byte boundary */
11987 	pd = &data[eeprom->len];
11988 	for (i = 0; i < (len - (len & 3)); i += 4) {
11989 		ret = tg3_nvram_read_be32(tp, offset + i, &val);
11990 		if (ret) {
11991 			if (i)
11992 				i -= 4;
11993 			eeprom->len += i;
11994 			goto eeprom_done;
11995 		}
11996 		memcpy(pd + i, &val, 4);
11997 		if (need_resched()) {
11998 			if (signal_pending(current)) {
11999 				eeprom->len += i;
12000 				ret = -EINTR;
12001 				goto eeprom_done;
12002 			}
12003 			cond_resched();
12004 		}
12005 	}
12006 	eeprom->len += i;
12007 
12008 	if (len & 3) {
12009 		/* read last bytes not ending on 4 byte boundary */
12010 		pd = &data[eeprom->len];
12011 		b_count = len & 3;
12012 		b_offset = offset + len - b_count;
12013 		ret = tg3_nvram_read_be32(tp, b_offset, &val);
12014 		if (ret)
12015 			goto eeprom_done;
12016 		memcpy(pd, &val, b_count);
12017 		eeprom->len += b_count;
12018 	}
12019 	ret = 0;
12020 
12021 eeprom_done:
12022 	/* Restore clock, link aware and link idle modes */
12023 	tg3_restore_clk(tp);
12024 	if (cpmu_restore)
12025 		tw32(TG3_CPMU_CTRL, cpmu_val);
12026 
12027 	return ret;
12028 }
12029 
12030 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
12031 {
12032 	struct tg3 *tp = netdev_priv(dev);
12033 	int ret;
12034 	u32 offset, len, b_offset, odd_len;
12035 	u8 *buf;
12036 	__be32 start, end;
12037 
12038 	if (tg3_flag(tp, NO_NVRAM) ||
12039 	    eeprom->magic != TG3_EEPROM_MAGIC)
12040 		return -EINVAL;
12041 
12042 	offset = eeprom->offset;
12043 	len = eeprom->len;
12044 
12045 	if ((b_offset = (offset & 3))) {
12046 		/* adjustments to start on required 4 byte boundary */
12047 		ret = tg3_nvram_read_be32(tp, offset-b_offset, &start);
12048 		if (ret)
12049 			return ret;
12050 		len += b_offset;
12051 		offset &= ~3;
12052 		if (len < 4)
12053 			len = 4;
12054 	}
12055 
12056 	odd_len = 0;
12057 	if (len & 3) {
12058 		/* adjustments to end on required 4 byte boundary */
12059 		odd_len = 1;
12060 		len = (len + 3) & ~3;
12061 		ret = tg3_nvram_read_be32(tp, offset+len-4, &end);
12062 		if (ret)
12063 			return ret;
12064 	}
12065 
12066 	buf = data;
12067 	if (b_offset || odd_len) {
12068 		buf = kmalloc(len, GFP_KERNEL);
12069 		if (!buf)
12070 			return -ENOMEM;
12071 		if (b_offset)
12072 			memcpy(buf, &start, 4);
12073 		if (odd_len)
12074 			memcpy(buf+len-4, &end, 4);
12075 		memcpy(buf + b_offset, data, eeprom->len);
12076 	}
12077 
12078 	ret = tg3_nvram_write_block(tp, offset, len, buf);
12079 
12080 	if (buf != data)
12081 		kfree(buf);
12082 
12083 	return ret;
12084 }
12085 
12086 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
12087 {
12088 	struct tg3 *tp = netdev_priv(dev);
12089 
12090 	if (tg3_flag(tp, USE_PHYLIB)) {
12091 		struct phy_device *phydev;
12092 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
12093 			return -EAGAIN;
12094 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
12095 		return phy_ethtool_gset(phydev, cmd);
12096 	}
12097 
12098 	cmd->supported = (SUPPORTED_Autoneg);
12099 
12100 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12101 		cmd->supported |= (SUPPORTED_1000baseT_Half |
12102 				   SUPPORTED_1000baseT_Full);
12103 
12104 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
12105 		cmd->supported |= (SUPPORTED_100baseT_Half |
12106 				  SUPPORTED_100baseT_Full |
12107 				  SUPPORTED_10baseT_Half |
12108 				  SUPPORTED_10baseT_Full |
12109 				  SUPPORTED_TP);
12110 		cmd->port = PORT_TP;
12111 	} else {
12112 		cmd->supported |= SUPPORTED_FIBRE;
12113 		cmd->port = PORT_FIBRE;
12114 	}
12115 
12116 	cmd->advertising = tp->link_config.advertising;
12117 	if (tg3_flag(tp, PAUSE_AUTONEG)) {
12118 		if (tp->link_config.flowctrl & FLOW_CTRL_RX) {
12119 			if (tp->link_config.flowctrl & FLOW_CTRL_TX) {
12120 				cmd->advertising |= ADVERTISED_Pause;
12121 			} else {
12122 				cmd->advertising |= ADVERTISED_Pause |
12123 						    ADVERTISED_Asym_Pause;
12124 			}
12125 		} else if (tp->link_config.flowctrl & FLOW_CTRL_TX) {
12126 			cmd->advertising |= ADVERTISED_Asym_Pause;
12127 		}
12128 	}
12129 	if (netif_running(dev) && tp->link_up) {
12130 		ethtool_cmd_speed_set(cmd, tp->link_config.active_speed);
12131 		cmd->duplex = tp->link_config.active_duplex;
12132 		cmd->lp_advertising = tp->link_config.rmt_adv;
12133 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
12134 			if (tp->phy_flags & TG3_PHYFLG_MDIX_STATE)
12135 				cmd->eth_tp_mdix = ETH_TP_MDI_X;
12136 			else
12137 				cmd->eth_tp_mdix = ETH_TP_MDI;
12138 		}
12139 	} else {
12140 		ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
12141 		cmd->duplex = DUPLEX_UNKNOWN;
12142 		cmd->eth_tp_mdix = ETH_TP_MDI_INVALID;
12143 	}
12144 	cmd->phy_address = tp->phy_addr;
12145 	cmd->transceiver = XCVR_INTERNAL;
12146 	cmd->autoneg = tp->link_config.autoneg;
12147 	cmd->maxtxpkt = 0;
12148 	cmd->maxrxpkt = 0;
12149 	return 0;
12150 }
12151 
12152 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
12153 {
12154 	struct tg3 *tp = netdev_priv(dev);
12155 	u32 speed = ethtool_cmd_speed(cmd);
12156 
12157 	if (tg3_flag(tp, USE_PHYLIB)) {
12158 		struct phy_device *phydev;
12159 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
12160 			return -EAGAIN;
12161 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
12162 		return phy_ethtool_sset(phydev, cmd);
12163 	}
12164 
12165 	if (cmd->autoneg != AUTONEG_ENABLE &&
12166 	    cmd->autoneg != AUTONEG_DISABLE)
12167 		return -EINVAL;
12168 
12169 	if (cmd->autoneg == AUTONEG_DISABLE &&
12170 	    cmd->duplex != DUPLEX_FULL &&
12171 	    cmd->duplex != DUPLEX_HALF)
12172 		return -EINVAL;
12173 
12174 	if (cmd->autoneg == AUTONEG_ENABLE) {
12175 		u32 mask = ADVERTISED_Autoneg |
12176 			   ADVERTISED_Pause |
12177 			   ADVERTISED_Asym_Pause;
12178 
12179 		if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12180 			mask |= ADVERTISED_1000baseT_Half |
12181 				ADVERTISED_1000baseT_Full;
12182 
12183 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
12184 			mask |= ADVERTISED_100baseT_Half |
12185 				ADVERTISED_100baseT_Full |
12186 				ADVERTISED_10baseT_Half |
12187 				ADVERTISED_10baseT_Full |
12188 				ADVERTISED_TP;
12189 		else
12190 			mask |= ADVERTISED_FIBRE;
12191 
12192 		if (cmd->advertising & ~mask)
12193 			return -EINVAL;
12194 
12195 		mask &= (ADVERTISED_1000baseT_Half |
12196 			 ADVERTISED_1000baseT_Full |
12197 			 ADVERTISED_100baseT_Half |
12198 			 ADVERTISED_100baseT_Full |
12199 			 ADVERTISED_10baseT_Half |
12200 			 ADVERTISED_10baseT_Full);
12201 
12202 		cmd->advertising &= mask;
12203 	} else {
12204 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES) {
12205 			if (speed != SPEED_1000)
12206 				return -EINVAL;
12207 
12208 			if (cmd->duplex != DUPLEX_FULL)
12209 				return -EINVAL;
12210 		} else {
12211 			if (speed != SPEED_100 &&
12212 			    speed != SPEED_10)
12213 				return -EINVAL;
12214 		}
12215 	}
12216 
12217 	tg3_full_lock(tp, 0);
12218 
12219 	tp->link_config.autoneg = cmd->autoneg;
12220 	if (cmd->autoneg == AUTONEG_ENABLE) {
12221 		tp->link_config.advertising = (cmd->advertising |
12222 					      ADVERTISED_Autoneg);
12223 		tp->link_config.speed = SPEED_UNKNOWN;
12224 		tp->link_config.duplex = DUPLEX_UNKNOWN;
12225 	} else {
12226 		tp->link_config.advertising = 0;
12227 		tp->link_config.speed = speed;
12228 		tp->link_config.duplex = cmd->duplex;
12229 	}
12230 
12231 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
12232 
12233 	tg3_warn_mgmt_link_flap(tp);
12234 
12235 	if (netif_running(dev))
12236 		tg3_setup_phy(tp, true);
12237 
12238 	tg3_full_unlock(tp);
12239 
12240 	return 0;
12241 }
12242 
12243 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
12244 {
12245 	struct tg3 *tp = netdev_priv(dev);
12246 
12247 	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
12248 	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
12249 	strlcpy(info->fw_version, tp->fw_ver, sizeof(info->fw_version));
12250 	strlcpy(info->bus_info, pci_name(tp->pdev), sizeof(info->bus_info));
12251 }
12252 
12253 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
12254 {
12255 	struct tg3 *tp = netdev_priv(dev);
12256 
12257 	if (tg3_flag(tp, WOL_CAP) && device_can_wakeup(&tp->pdev->dev))
12258 		wol->supported = WAKE_MAGIC;
12259 	else
12260 		wol->supported = 0;
12261 	wol->wolopts = 0;
12262 	if (tg3_flag(tp, WOL_ENABLE) && device_can_wakeup(&tp->pdev->dev))
12263 		wol->wolopts = WAKE_MAGIC;
12264 	memset(&wol->sopass, 0, sizeof(wol->sopass));
12265 }
12266 
12267 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
12268 {
12269 	struct tg3 *tp = netdev_priv(dev);
12270 	struct device *dp = &tp->pdev->dev;
12271 
12272 	if (wol->wolopts & ~WAKE_MAGIC)
12273 		return -EINVAL;
12274 	if ((wol->wolopts & WAKE_MAGIC) &&
12275 	    !(tg3_flag(tp, WOL_CAP) && device_can_wakeup(dp)))
12276 		return -EINVAL;
12277 
12278 	device_set_wakeup_enable(dp, wol->wolopts & WAKE_MAGIC);
12279 
12280 	if (device_may_wakeup(dp))
12281 		tg3_flag_set(tp, WOL_ENABLE);
12282 	else
12283 		tg3_flag_clear(tp, WOL_ENABLE);
12284 
12285 	return 0;
12286 }
12287 
12288 static u32 tg3_get_msglevel(struct net_device *dev)
12289 {
12290 	struct tg3 *tp = netdev_priv(dev);
12291 	return tp->msg_enable;
12292 }
12293 
12294 static void tg3_set_msglevel(struct net_device *dev, u32 value)
12295 {
12296 	struct tg3 *tp = netdev_priv(dev);
12297 	tp->msg_enable = value;
12298 }
12299 
12300 static int tg3_nway_reset(struct net_device *dev)
12301 {
12302 	struct tg3 *tp = netdev_priv(dev);
12303 	int r;
12304 
12305 	if (!netif_running(dev))
12306 		return -EAGAIN;
12307 
12308 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
12309 		return -EINVAL;
12310 
12311 	tg3_warn_mgmt_link_flap(tp);
12312 
12313 	if (tg3_flag(tp, USE_PHYLIB)) {
12314 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
12315 			return -EAGAIN;
12316 		r = phy_start_aneg(tp->mdio_bus->phy_map[tp->phy_addr]);
12317 	} else {
12318 		u32 bmcr;
12319 
12320 		spin_lock_bh(&tp->lock);
12321 		r = -EINVAL;
12322 		tg3_readphy(tp, MII_BMCR, &bmcr);
12323 		if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
12324 		    ((bmcr & BMCR_ANENABLE) ||
12325 		     (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT))) {
12326 			tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
12327 						   BMCR_ANENABLE);
12328 			r = 0;
12329 		}
12330 		spin_unlock_bh(&tp->lock);
12331 	}
12332 
12333 	return r;
12334 }
12335 
12336 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
12337 {
12338 	struct tg3 *tp = netdev_priv(dev);
12339 
12340 	ering->rx_max_pending = tp->rx_std_ring_mask;
12341 	if (tg3_flag(tp, JUMBO_RING_ENABLE))
12342 		ering->rx_jumbo_max_pending = tp->rx_jmb_ring_mask;
12343 	else
12344 		ering->rx_jumbo_max_pending = 0;
12345 
12346 	ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
12347 
12348 	ering->rx_pending = tp->rx_pending;
12349 	if (tg3_flag(tp, JUMBO_RING_ENABLE))
12350 		ering->rx_jumbo_pending = tp->rx_jumbo_pending;
12351 	else
12352 		ering->rx_jumbo_pending = 0;
12353 
12354 	ering->tx_pending = tp->napi[0].tx_pending;
12355 }
12356 
12357 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
12358 {
12359 	struct tg3 *tp = netdev_priv(dev);
12360 	int i, irq_sync = 0, err = 0;
12361 
12362 	if ((ering->rx_pending > tp->rx_std_ring_mask) ||
12363 	    (ering->rx_jumbo_pending > tp->rx_jmb_ring_mask) ||
12364 	    (ering->tx_pending > TG3_TX_RING_SIZE - 1) ||
12365 	    (ering->tx_pending <= MAX_SKB_FRAGS) ||
12366 	    (tg3_flag(tp, TSO_BUG) &&
12367 	     (ering->tx_pending <= (MAX_SKB_FRAGS * 3))))
12368 		return -EINVAL;
12369 
12370 	if (netif_running(dev)) {
12371 		tg3_phy_stop(tp);
12372 		tg3_netif_stop(tp);
12373 		irq_sync = 1;
12374 	}
12375 
12376 	tg3_full_lock(tp, irq_sync);
12377 
12378 	tp->rx_pending = ering->rx_pending;
12379 
12380 	if (tg3_flag(tp, MAX_RXPEND_64) &&
12381 	    tp->rx_pending > 63)
12382 		tp->rx_pending = 63;
12383 
12384 	if (tg3_flag(tp, JUMBO_RING_ENABLE))
12385 		tp->rx_jumbo_pending = ering->rx_jumbo_pending;
12386 
12387 	for (i = 0; i < tp->irq_max; i++)
12388 		tp->napi[i].tx_pending = ering->tx_pending;
12389 
12390 	if (netif_running(dev)) {
12391 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12392 		err = tg3_restart_hw(tp, false);
12393 		if (!err)
12394 			tg3_netif_start(tp);
12395 	}
12396 
12397 	tg3_full_unlock(tp);
12398 
12399 	if (irq_sync && !err)
12400 		tg3_phy_start(tp);
12401 
12402 	return err;
12403 }
12404 
12405 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
12406 {
12407 	struct tg3 *tp = netdev_priv(dev);
12408 
12409 	epause->autoneg = !!tg3_flag(tp, PAUSE_AUTONEG);
12410 
12411 	if (tp->link_config.flowctrl & FLOW_CTRL_RX)
12412 		epause->rx_pause = 1;
12413 	else
12414 		epause->rx_pause = 0;
12415 
12416 	if (tp->link_config.flowctrl & FLOW_CTRL_TX)
12417 		epause->tx_pause = 1;
12418 	else
12419 		epause->tx_pause = 0;
12420 }
12421 
12422 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
12423 {
12424 	struct tg3 *tp = netdev_priv(dev);
12425 	int err = 0;
12426 
12427 	if (tp->link_config.autoneg == AUTONEG_ENABLE)
12428 		tg3_warn_mgmt_link_flap(tp);
12429 
12430 	if (tg3_flag(tp, USE_PHYLIB)) {
12431 		u32 newadv;
12432 		struct phy_device *phydev;
12433 
12434 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
12435 
12436 		if (!(phydev->supported & SUPPORTED_Pause) ||
12437 		    (!(phydev->supported & SUPPORTED_Asym_Pause) &&
12438 		     (epause->rx_pause != epause->tx_pause)))
12439 			return -EINVAL;
12440 
12441 		tp->link_config.flowctrl = 0;
12442 		if (epause->rx_pause) {
12443 			tp->link_config.flowctrl |= FLOW_CTRL_RX;
12444 
12445 			if (epause->tx_pause) {
12446 				tp->link_config.flowctrl |= FLOW_CTRL_TX;
12447 				newadv = ADVERTISED_Pause;
12448 			} else
12449 				newadv = ADVERTISED_Pause |
12450 					 ADVERTISED_Asym_Pause;
12451 		} else if (epause->tx_pause) {
12452 			tp->link_config.flowctrl |= FLOW_CTRL_TX;
12453 			newadv = ADVERTISED_Asym_Pause;
12454 		} else
12455 			newadv = 0;
12456 
12457 		if (epause->autoneg)
12458 			tg3_flag_set(tp, PAUSE_AUTONEG);
12459 		else
12460 			tg3_flag_clear(tp, PAUSE_AUTONEG);
12461 
12462 		if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
12463 			u32 oldadv = phydev->advertising &
12464 				     (ADVERTISED_Pause | ADVERTISED_Asym_Pause);
12465 			if (oldadv != newadv) {
12466 				phydev->advertising &=
12467 					~(ADVERTISED_Pause |
12468 					  ADVERTISED_Asym_Pause);
12469 				phydev->advertising |= newadv;
12470 				if (phydev->autoneg) {
12471 					/*
12472 					 * Always renegotiate the link to
12473 					 * inform our link partner of our
12474 					 * flow control settings, even if the
12475 					 * flow control is forced.  Let
12476 					 * tg3_adjust_link() do the final
12477 					 * flow control setup.
12478 					 */
12479 					return phy_start_aneg(phydev);
12480 				}
12481 			}
12482 
12483 			if (!epause->autoneg)
12484 				tg3_setup_flow_control(tp, 0, 0);
12485 		} else {
12486 			tp->link_config.advertising &=
12487 					~(ADVERTISED_Pause |
12488 					  ADVERTISED_Asym_Pause);
12489 			tp->link_config.advertising |= newadv;
12490 		}
12491 	} else {
12492 		int irq_sync = 0;
12493 
12494 		if (netif_running(dev)) {
12495 			tg3_netif_stop(tp);
12496 			irq_sync = 1;
12497 		}
12498 
12499 		tg3_full_lock(tp, irq_sync);
12500 
12501 		if (epause->autoneg)
12502 			tg3_flag_set(tp, PAUSE_AUTONEG);
12503 		else
12504 			tg3_flag_clear(tp, PAUSE_AUTONEG);
12505 		if (epause->rx_pause)
12506 			tp->link_config.flowctrl |= FLOW_CTRL_RX;
12507 		else
12508 			tp->link_config.flowctrl &= ~FLOW_CTRL_RX;
12509 		if (epause->tx_pause)
12510 			tp->link_config.flowctrl |= FLOW_CTRL_TX;
12511 		else
12512 			tp->link_config.flowctrl &= ~FLOW_CTRL_TX;
12513 
12514 		if (netif_running(dev)) {
12515 			tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12516 			err = tg3_restart_hw(tp, false);
12517 			if (!err)
12518 				tg3_netif_start(tp);
12519 		}
12520 
12521 		tg3_full_unlock(tp);
12522 	}
12523 
12524 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
12525 
12526 	return err;
12527 }
12528 
12529 static int tg3_get_sset_count(struct net_device *dev, int sset)
12530 {
12531 	switch (sset) {
12532 	case ETH_SS_TEST:
12533 		return TG3_NUM_TEST;
12534 	case ETH_SS_STATS:
12535 		return TG3_NUM_STATS;
12536 	default:
12537 		return -EOPNOTSUPP;
12538 	}
12539 }
12540 
12541 static int tg3_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
12542 			 u32 *rules __always_unused)
12543 {
12544 	struct tg3 *tp = netdev_priv(dev);
12545 
12546 	if (!tg3_flag(tp, SUPPORT_MSIX))
12547 		return -EOPNOTSUPP;
12548 
12549 	switch (info->cmd) {
12550 	case ETHTOOL_GRXRINGS:
12551 		if (netif_running(tp->dev))
12552 			info->data = tp->rxq_cnt;
12553 		else {
12554 			info->data = num_online_cpus();
12555 			if (info->data > TG3_RSS_MAX_NUM_QS)
12556 				info->data = TG3_RSS_MAX_NUM_QS;
12557 		}
12558 
12559 		/* The first interrupt vector only
12560 		 * handles link interrupts.
12561 		 */
12562 		info->data -= 1;
12563 		return 0;
12564 
12565 	default:
12566 		return -EOPNOTSUPP;
12567 	}
12568 }
12569 
12570 static u32 tg3_get_rxfh_indir_size(struct net_device *dev)
12571 {
12572 	u32 size = 0;
12573 	struct tg3 *tp = netdev_priv(dev);
12574 
12575 	if (tg3_flag(tp, SUPPORT_MSIX))
12576 		size = TG3_RSS_INDIR_TBL_SIZE;
12577 
12578 	return size;
12579 }
12580 
12581 static int tg3_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc)
12582 {
12583 	struct tg3 *tp = netdev_priv(dev);
12584 	int i;
12585 
12586 	if (hfunc)
12587 		*hfunc = ETH_RSS_HASH_TOP;
12588 	if (!indir)
12589 		return 0;
12590 
12591 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
12592 		indir[i] = tp->rss_ind_tbl[i];
12593 
12594 	return 0;
12595 }
12596 
12597 static int tg3_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key,
12598 			const u8 hfunc)
12599 {
12600 	struct tg3 *tp = netdev_priv(dev);
12601 	size_t i;
12602 
12603 	/* We require at least one supported parameter to be changed and no
12604 	 * change in any of the unsupported parameters
12605 	 */
12606 	if (key ||
12607 	    (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
12608 		return -EOPNOTSUPP;
12609 
12610 	if (!indir)
12611 		return 0;
12612 
12613 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
12614 		tp->rss_ind_tbl[i] = indir[i];
12615 
12616 	if (!netif_running(dev) || !tg3_flag(tp, ENABLE_RSS))
12617 		return 0;
12618 
12619 	/* It is legal to write the indirection
12620 	 * table while the device is running.
12621 	 */
12622 	tg3_full_lock(tp, 0);
12623 	tg3_rss_write_indir_tbl(tp);
12624 	tg3_full_unlock(tp);
12625 
12626 	return 0;
12627 }
12628 
12629 static void tg3_get_channels(struct net_device *dev,
12630 			     struct ethtool_channels *channel)
12631 {
12632 	struct tg3 *tp = netdev_priv(dev);
12633 	u32 deflt_qs = netif_get_num_default_rss_queues();
12634 
12635 	channel->max_rx = tp->rxq_max;
12636 	channel->max_tx = tp->txq_max;
12637 
12638 	if (netif_running(dev)) {
12639 		channel->rx_count = tp->rxq_cnt;
12640 		channel->tx_count = tp->txq_cnt;
12641 	} else {
12642 		if (tp->rxq_req)
12643 			channel->rx_count = tp->rxq_req;
12644 		else
12645 			channel->rx_count = min(deflt_qs, tp->rxq_max);
12646 
12647 		if (tp->txq_req)
12648 			channel->tx_count = tp->txq_req;
12649 		else
12650 			channel->tx_count = min(deflt_qs, tp->txq_max);
12651 	}
12652 }
12653 
12654 static int tg3_set_channels(struct net_device *dev,
12655 			    struct ethtool_channels *channel)
12656 {
12657 	struct tg3 *tp = netdev_priv(dev);
12658 
12659 	if (!tg3_flag(tp, SUPPORT_MSIX))
12660 		return -EOPNOTSUPP;
12661 
12662 	if (channel->rx_count > tp->rxq_max ||
12663 	    channel->tx_count > tp->txq_max)
12664 		return -EINVAL;
12665 
12666 	tp->rxq_req = channel->rx_count;
12667 	tp->txq_req = channel->tx_count;
12668 
12669 	if (!netif_running(dev))
12670 		return 0;
12671 
12672 	tg3_stop(tp);
12673 
12674 	tg3_carrier_off(tp);
12675 
12676 	tg3_start(tp, true, false, false);
12677 
12678 	return 0;
12679 }
12680 
12681 static void tg3_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
12682 {
12683 	switch (stringset) {
12684 	case ETH_SS_STATS:
12685 		memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
12686 		break;
12687 	case ETH_SS_TEST:
12688 		memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
12689 		break;
12690 	default:
12691 		WARN_ON(1);	/* we need a WARN() */
12692 		break;
12693 	}
12694 }
12695 
12696 static int tg3_set_phys_id(struct net_device *dev,
12697 			    enum ethtool_phys_id_state state)
12698 {
12699 	struct tg3 *tp = netdev_priv(dev);
12700 
12701 	if (!netif_running(tp->dev))
12702 		return -EAGAIN;
12703 
12704 	switch (state) {
12705 	case ETHTOOL_ID_ACTIVE:
12706 		return 1;	/* cycle on/off once per second */
12707 
12708 	case ETHTOOL_ID_ON:
12709 		tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
12710 		     LED_CTRL_1000MBPS_ON |
12711 		     LED_CTRL_100MBPS_ON |
12712 		     LED_CTRL_10MBPS_ON |
12713 		     LED_CTRL_TRAFFIC_OVERRIDE |
12714 		     LED_CTRL_TRAFFIC_BLINK |
12715 		     LED_CTRL_TRAFFIC_LED);
12716 		break;
12717 
12718 	case ETHTOOL_ID_OFF:
12719 		tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
12720 		     LED_CTRL_TRAFFIC_OVERRIDE);
12721 		break;
12722 
12723 	case ETHTOOL_ID_INACTIVE:
12724 		tw32(MAC_LED_CTRL, tp->led_ctrl);
12725 		break;
12726 	}
12727 
12728 	return 0;
12729 }
12730 
12731 static void tg3_get_ethtool_stats(struct net_device *dev,
12732 				   struct ethtool_stats *estats, u64 *tmp_stats)
12733 {
12734 	struct tg3 *tp = netdev_priv(dev);
12735 
12736 	if (tp->hw_stats)
12737 		tg3_get_estats(tp, (struct tg3_ethtool_stats *)tmp_stats);
12738 	else
12739 		memset(tmp_stats, 0, sizeof(struct tg3_ethtool_stats));
12740 }
12741 
12742 static __be32 *tg3_vpd_readblock(struct tg3 *tp, u32 *vpdlen)
12743 {
12744 	int i;
12745 	__be32 *buf;
12746 	u32 offset = 0, len = 0;
12747 	u32 magic, val;
12748 
12749 	if (tg3_flag(tp, NO_NVRAM) || tg3_nvram_read(tp, 0, &magic))
12750 		return NULL;
12751 
12752 	if (magic == TG3_EEPROM_MAGIC) {
12753 		for (offset = TG3_NVM_DIR_START;
12754 		     offset < TG3_NVM_DIR_END;
12755 		     offset += TG3_NVM_DIRENT_SIZE) {
12756 			if (tg3_nvram_read(tp, offset, &val))
12757 				return NULL;
12758 
12759 			if ((val >> TG3_NVM_DIRTYPE_SHIFT) ==
12760 			    TG3_NVM_DIRTYPE_EXTVPD)
12761 				break;
12762 		}
12763 
12764 		if (offset != TG3_NVM_DIR_END) {
12765 			len = (val & TG3_NVM_DIRTYPE_LENMSK) * 4;
12766 			if (tg3_nvram_read(tp, offset + 4, &offset))
12767 				return NULL;
12768 
12769 			offset = tg3_nvram_logical_addr(tp, offset);
12770 		}
12771 	}
12772 
12773 	if (!offset || !len) {
12774 		offset = TG3_NVM_VPD_OFF;
12775 		len = TG3_NVM_VPD_LEN;
12776 	}
12777 
12778 	buf = kmalloc(len, GFP_KERNEL);
12779 	if (buf == NULL)
12780 		return NULL;
12781 
12782 	if (magic == TG3_EEPROM_MAGIC) {
12783 		for (i = 0; i < len; i += 4) {
12784 			/* The data is in little-endian format in NVRAM.
12785 			 * Use the big-endian read routines to preserve
12786 			 * the byte order as it exists in NVRAM.
12787 			 */
12788 			if (tg3_nvram_read_be32(tp, offset + i, &buf[i/4]))
12789 				goto error;
12790 		}
12791 	} else {
12792 		u8 *ptr;
12793 		ssize_t cnt;
12794 		unsigned int pos = 0;
12795 
12796 		ptr = (u8 *)&buf[0];
12797 		for (i = 0; pos < len && i < 3; i++, pos += cnt, ptr += cnt) {
12798 			cnt = pci_read_vpd(tp->pdev, pos,
12799 					   len - pos, ptr);
12800 			if (cnt == -ETIMEDOUT || cnt == -EINTR)
12801 				cnt = 0;
12802 			else if (cnt < 0)
12803 				goto error;
12804 		}
12805 		if (pos != len)
12806 			goto error;
12807 	}
12808 
12809 	*vpdlen = len;
12810 
12811 	return buf;
12812 
12813 error:
12814 	kfree(buf);
12815 	return NULL;
12816 }
12817 
12818 #define NVRAM_TEST_SIZE 0x100
12819 #define NVRAM_SELFBOOT_FORMAT1_0_SIZE	0x14
12820 #define NVRAM_SELFBOOT_FORMAT1_2_SIZE	0x18
12821 #define NVRAM_SELFBOOT_FORMAT1_3_SIZE	0x1c
12822 #define NVRAM_SELFBOOT_FORMAT1_4_SIZE	0x20
12823 #define NVRAM_SELFBOOT_FORMAT1_5_SIZE	0x24
12824 #define NVRAM_SELFBOOT_FORMAT1_6_SIZE	0x50
12825 #define NVRAM_SELFBOOT_HW_SIZE 0x20
12826 #define NVRAM_SELFBOOT_DATA_SIZE 0x1c
12827 
12828 static int tg3_test_nvram(struct tg3 *tp)
12829 {
12830 	u32 csum, magic, len;
12831 	__be32 *buf;
12832 	int i, j, k, err = 0, size;
12833 
12834 	if (tg3_flag(tp, NO_NVRAM))
12835 		return 0;
12836 
12837 	if (tg3_nvram_read(tp, 0, &magic) != 0)
12838 		return -EIO;
12839 
12840 	if (magic == TG3_EEPROM_MAGIC)
12841 		size = NVRAM_TEST_SIZE;
12842 	else if ((magic & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW) {
12843 		if ((magic & TG3_EEPROM_SB_FORMAT_MASK) ==
12844 		    TG3_EEPROM_SB_FORMAT_1) {
12845 			switch (magic & TG3_EEPROM_SB_REVISION_MASK) {
12846 			case TG3_EEPROM_SB_REVISION_0:
12847 				size = NVRAM_SELFBOOT_FORMAT1_0_SIZE;
12848 				break;
12849 			case TG3_EEPROM_SB_REVISION_2:
12850 				size = NVRAM_SELFBOOT_FORMAT1_2_SIZE;
12851 				break;
12852 			case TG3_EEPROM_SB_REVISION_3:
12853 				size = NVRAM_SELFBOOT_FORMAT1_3_SIZE;
12854 				break;
12855 			case TG3_EEPROM_SB_REVISION_4:
12856 				size = NVRAM_SELFBOOT_FORMAT1_4_SIZE;
12857 				break;
12858 			case TG3_EEPROM_SB_REVISION_5:
12859 				size = NVRAM_SELFBOOT_FORMAT1_5_SIZE;
12860 				break;
12861 			case TG3_EEPROM_SB_REVISION_6:
12862 				size = NVRAM_SELFBOOT_FORMAT1_6_SIZE;
12863 				break;
12864 			default:
12865 				return -EIO;
12866 			}
12867 		} else
12868 			return 0;
12869 	} else if ((magic & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
12870 		size = NVRAM_SELFBOOT_HW_SIZE;
12871 	else
12872 		return -EIO;
12873 
12874 	buf = kmalloc(size, GFP_KERNEL);
12875 	if (buf == NULL)
12876 		return -ENOMEM;
12877 
12878 	err = -EIO;
12879 	for (i = 0, j = 0; i < size; i += 4, j++) {
12880 		err = tg3_nvram_read_be32(tp, i, &buf[j]);
12881 		if (err)
12882 			break;
12883 	}
12884 	if (i < size)
12885 		goto out;
12886 
12887 	/* Selfboot format */
12888 	magic = be32_to_cpu(buf[0]);
12889 	if ((magic & TG3_EEPROM_MAGIC_FW_MSK) ==
12890 	    TG3_EEPROM_MAGIC_FW) {
12891 		u8 *buf8 = (u8 *) buf, csum8 = 0;
12892 
12893 		if ((magic & TG3_EEPROM_SB_REVISION_MASK) ==
12894 		    TG3_EEPROM_SB_REVISION_2) {
12895 			/* For rev 2, the csum doesn't include the MBA. */
12896 			for (i = 0; i < TG3_EEPROM_SB_F1R2_MBA_OFF; i++)
12897 				csum8 += buf8[i];
12898 			for (i = TG3_EEPROM_SB_F1R2_MBA_OFF + 4; i < size; i++)
12899 				csum8 += buf8[i];
12900 		} else {
12901 			for (i = 0; i < size; i++)
12902 				csum8 += buf8[i];
12903 		}
12904 
12905 		if (csum8 == 0) {
12906 			err = 0;
12907 			goto out;
12908 		}
12909 
12910 		err = -EIO;
12911 		goto out;
12912 	}
12913 
12914 	if ((magic & TG3_EEPROM_MAGIC_HW_MSK) ==
12915 	    TG3_EEPROM_MAGIC_HW) {
12916 		u8 data[NVRAM_SELFBOOT_DATA_SIZE];
12917 		u8 parity[NVRAM_SELFBOOT_DATA_SIZE];
12918 		u8 *buf8 = (u8 *) buf;
12919 
12920 		/* Separate the parity bits and the data bytes.  */
12921 		for (i = 0, j = 0, k = 0; i < NVRAM_SELFBOOT_HW_SIZE; i++) {
12922 			if ((i == 0) || (i == 8)) {
12923 				int l;
12924 				u8 msk;
12925 
12926 				for (l = 0, msk = 0x80; l < 7; l++, msk >>= 1)
12927 					parity[k++] = buf8[i] & msk;
12928 				i++;
12929 			} else if (i == 16) {
12930 				int l;
12931 				u8 msk;
12932 
12933 				for (l = 0, msk = 0x20; l < 6; l++, msk >>= 1)
12934 					parity[k++] = buf8[i] & msk;
12935 				i++;
12936 
12937 				for (l = 0, msk = 0x80; l < 8; l++, msk >>= 1)
12938 					parity[k++] = buf8[i] & msk;
12939 				i++;
12940 			}
12941 			data[j++] = buf8[i];
12942 		}
12943 
12944 		err = -EIO;
12945 		for (i = 0; i < NVRAM_SELFBOOT_DATA_SIZE; i++) {
12946 			u8 hw8 = hweight8(data[i]);
12947 
12948 			if ((hw8 & 0x1) && parity[i])
12949 				goto out;
12950 			else if (!(hw8 & 0x1) && !parity[i])
12951 				goto out;
12952 		}
12953 		err = 0;
12954 		goto out;
12955 	}
12956 
12957 	err = -EIO;
12958 
12959 	/* Bootstrap checksum at offset 0x10 */
12960 	csum = calc_crc((unsigned char *) buf, 0x10);
12961 	if (csum != le32_to_cpu(buf[0x10/4]))
12962 		goto out;
12963 
12964 	/* Manufacturing block starts at offset 0x74, checksum at 0xfc */
12965 	csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
12966 	if (csum != le32_to_cpu(buf[0xfc/4]))
12967 		goto out;
12968 
12969 	kfree(buf);
12970 
12971 	buf = tg3_vpd_readblock(tp, &len);
12972 	if (!buf)
12973 		return -ENOMEM;
12974 
12975 	i = pci_vpd_find_tag((u8 *)buf, 0, len, PCI_VPD_LRDT_RO_DATA);
12976 	if (i > 0) {
12977 		j = pci_vpd_lrdt_size(&((u8 *)buf)[i]);
12978 		if (j < 0)
12979 			goto out;
12980 
12981 		if (i + PCI_VPD_LRDT_TAG_SIZE + j > len)
12982 			goto out;
12983 
12984 		i += PCI_VPD_LRDT_TAG_SIZE;
12985 		j = pci_vpd_find_info_keyword((u8 *)buf, i, j,
12986 					      PCI_VPD_RO_KEYWORD_CHKSUM);
12987 		if (j > 0) {
12988 			u8 csum8 = 0;
12989 
12990 			j += PCI_VPD_INFO_FLD_HDR_SIZE;
12991 
12992 			for (i = 0; i <= j; i++)
12993 				csum8 += ((u8 *)buf)[i];
12994 
12995 			if (csum8)
12996 				goto out;
12997 		}
12998 	}
12999 
13000 	err = 0;
13001 
13002 out:
13003 	kfree(buf);
13004 	return err;
13005 }
13006 
13007 #define TG3_SERDES_TIMEOUT_SEC	2
13008 #define TG3_COPPER_TIMEOUT_SEC	6
13009 
13010 static int tg3_test_link(struct tg3 *tp)
13011 {
13012 	int i, max;
13013 
13014 	if (!netif_running(tp->dev))
13015 		return -ENODEV;
13016 
13017 	if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
13018 		max = TG3_SERDES_TIMEOUT_SEC;
13019 	else
13020 		max = TG3_COPPER_TIMEOUT_SEC;
13021 
13022 	for (i = 0; i < max; i++) {
13023 		if (tp->link_up)
13024 			return 0;
13025 
13026 		if (msleep_interruptible(1000))
13027 			break;
13028 	}
13029 
13030 	return -EIO;
13031 }
13032 
13033 /* Only test the commonly used registers */
13034 static int tg3_test_registers(struct tg3 *tp)
13035 {
13036 	int i, is_5705, is_5750;
13037 	u32 offset, read_mask, write_mask, val, save_val, read_val;
13038 	static struct {
13039 		u16 offset;
13040 		u16 flags;
13041 #define TG3_FL_5705	0x1
13042 #define TG3_FL_NOT_5705	0x2
13043 #define TG3_FL_NOT_5788	0x4
13044 #define TG3_FL_NOT_5750	0x8
13045 		u32 read_mask;
13046 		u32 write_mask;
13047 	} reg_tbl[] = {
13048 		/* MAC Control Registers */
13049 		{ MAC_MODE, TG3_FL_NOT_5705,
13050 			0x00000000, 0x00ef6f8c },
13051 		{ MAC_MODE, TG3_FL_5705,
13052 			0x00000000, 0x01ef6b8c },
13053 		{ MAC_STATUS, TG3_FL_NOT_5705,
13054 			0x03800107, 0x00000000 },
13055 		{ MAC_STATUS, TG3_FL_5705,
13056 			0x03800100, 0x00000000 },
13057 		{ MAC_ADDR_0_HIGH, 0x0000,
13058 			0x00000000, 0x0000ffff },
13059 		{ MAC_ADDR_0_LOW, 0x0000,
13060 			0x00000000, 0xffffffff },
13061 		{ MAC_RX_MTU_SIZE, 0x0000,
13062 			0x00000000, 0x0000ffff },
13063 		{ MAC_TX_MODE, 0x0000,
13064 			0x00000000, 0x00000070 },
13065 		{ MAC_TX_LENGTHS, 0x0000,
13066 			0x00000000, 0x00003fff },
13067 		{ MAC_RX_MODE, TG3_FL_NOT_5705,
13068 			0x00000000, 0x000007fc },
13069 		{ MAC_RX_MODE, TG3_FL_5705,
13070 			0x00000000, 0x000007dc },
13071 		{ MAC_HASH_REG_0, 0x0000,
13072 			0x00000000, 0xffffffff },
13073 		{ MAC_HASH_REG_1, 0x0000,
13074 			0x00000000, 0xffffffff },
13075 		{ MAC_HASH_REG_2, 0x0000,
13076 			0x00000000, 0xffffffff },
13077 		{ MAC_HASH_REG_3, 0x0000,
13078 			0x00000000, 0xffffffff },
13079 
13080 		/* Receive Data and Receive BD Initiator Control Registers. */
13081 		{ RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
13082 			0x00000000, 0xffffffff },
13083 		{ RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
13084 			0x00000000, 0xffffffff },
13085 		{ RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
13086 			0x00000000, 0x00000003 },
13087 		{ RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
13088 			0x00000000, 0xffffffff },
13089 		{ RCVDBDI_STD_BD+0, 0x0000,
13090 			0x00000000, 0xffffffff },
13091 		{ RCVDBDI_STD_BD+4, 0x0000,
13092 			0x00000000, 0xffffffff },
13093 		{ RCVDBDI_STD_BD+8, 0x0000,
13094 			0x00000000, 0xffff0002 },
13095 		{ RCVDBDI_STD_BD+0xc, 0x0000,
13096 			0x00000000, 0xffffffff },
13097 
13098 		/* Receive BD Initiator Control Registers. */
13099 		{ RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
13100 			0x00000000, 0xffffffff },
13101 		{ RCVBDI_STD_THRESH, TG3_FL_5705,
13102 			0x00000000, 0x000003ff },
13103 		{ RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
13104 			0x00000000, 0xffffffff },
13105 
13106 		/* Host Coalescing Control Registers. */
13107 		{ HOSTCC_MODE, TG3_FL_NOT_5705,
13108 			0x00000000, 0x00000004 },
13109 		{ HOSTCC_MODE, TG3_FL_5705,
13110 			0x00000000, 0x000000f6 },
13111 		{ HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
13112 			0x00000000, 0xffffffff },
13113 		{ HOSTCC_RXCOL_TICKS, TG3_FL_5705,
13114 			0x00000000, 0x000003ff },
13115 		{ HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
13116 			0x00000000, 0xffffffff },
13117 		{ HOSTCC_TXCOL_TICKS, TG3_FL_5705,
13118 			0x00000000, 0x000003ff },
13119 		{ HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
13120 			0x00000000, 0xffffffff },
13121 		{ HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
13122 			0x00000000, 0x000000ff },
13123 		{ HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
13124 			0x00000000, 0xffffffff },
13125 		{ HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
13126 			0x00000000, 0x000000ff },
13127 		{ HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
13128 			0x00000000, 0xffffffff },
13129 		{ HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
13130 			0x00000000, 0xffffffff },
13131 		{ HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
13132 			0x00000000, 0xffffffff },
13133 		{ HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
13134 			0x00000000, 0x000000ff },
13135 		{ HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
13136 			0x00000000, 0xffffffff },
13137 		{ HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
13138 			0x00000000, 0x000000ff },
13139 		{ HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
13140 			0x00000000, 0xffffffff },
13141 		{ HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
13142 			0x00000000, 0xffffffff },
13143 		{ HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
13144 			0x00000000, 0xffffffff },
13145 		{ HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
13146 			0x00000000, 0xffffffff },
13147 		{ HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
13148 			0x00000000, 0xffffffff },
13149 		{ HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
13150 			0xffffffff, 0x00000000 },
13151 		{ HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
13152 			0xffffffff, 0x00000000 },
13153 
13154 		/* Buffer Manager Control Registers. */
13155 		{ BUFMGR_MB_POOL_ADDR, TG3_FL_NOT_5750,
13156 			0x00000000, 0x007fff80 },
13157 		{ BUFMGR_MB_POOL_SIZE, TG3_FL_NOT_5750,
13158 			0x00000000, 0x007fffff },
13159 		{ BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
13160 			0x00000000, 0x0000003f },
13161 		{ BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
13162 			0x00000000, 0x000001ff },
13163 		{ BUFMGR_MB_HIGH_WATER, 0x0000,
13164 			0x00000000, 0x000001ff },
13165 		{ BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
13166 			0xffffffff, 0x00000000 },
13167 		{ BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
13168 			0xffffffff, 0x00000000 },
13169 
13170 		/* Mailbox Registers */
13171 		{ GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
13172 			0x00000000, 0x000001ff },
13173 		{ GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
13174 			0x00000000, 0x000001ff },
13175 		{ GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
13176 			0x00000000, 0x000007ff },
13177 		{ GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
13178 			0x00000000, 0x000001ff },
13179 
13180 		{ 0xffff, 0x0000, 0x00000000, 0x00000000 },
13181 	};
13182 
13183 	is_5705 = is_5750 = 0;
13184 	if (tg3_flag(tp, 5705_PLUS)) {
13185 		is_5705 = 1;
13186 		if (tg3_flag(tp, 5750_PLUS))
13187 			is_5750 = 1;
13188 	}
13189 
13190 	for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
13191 		if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
13192 			continue;
13193 
13194 		if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
13195 			continue;
13196 
13197 		if (tg3_flag(tp, IS_5788) &&
13198 		    (reg_tbl[i].flags & TG3_FL_NOT_5788))
13199 			continue;
13200 
13201 		if (is_5750 && (reg_tbl[i].flags & TG3_FL_NOT_5750))
13202 			continue;
13203 
13204 		offset = (u32) reg_tbl[i].offset;
13205 		read_mask = reg_tbl[i].read_mask;
13206 		write_mask = reg_tbl[i].write_mask;
13207 
13208 		/* Save the original register content */
13209 		save_val = tr32(offset);
13210 
13211 		/* Determine the read-only value. */
13212 		read_val = save_val & read_mask;
13213 
13214 		/* Write zero to the register, then make sure the read-only bits
13215 		 * are not changed and the read/write bits are all zeros.
13216 		 */
13217 		tw32(offset, 0);
13218 
13219 		val = tr32(offset);
13220 
13221 		/* Test the read-only and read/write bits. */
13222 		if (((val & read_mask) != read_val) || (val & write_mask))
13223 			goto out;
13224 
13225 		/* Write ones to all the bits defined by RdMask and WrMask, then
13226 		 * make sure the read-only bits are not changed and the
13227 		 * read/write bits are all ones.
13228 		 */
13229 		tw32(offset, read_mask | write_mask);
13230 
13231 		val = tr32(offset);
13232 
13233 		/* Test the read-only bits. */
13234 		if ((val & read_mask) != read_val)
13235 			goto out;
13236 
13237 		/* Test the read/write bits. */
13238 		if ((val & write_mask) != write_mask)
13239 			goto out;
13240 
13241 		tw32(offset, save_val);
13242 	}
13243 
13244 	return 0;
13245 
13246 out:
13247 	if (netif_msg_hw(tp))
13248 		netdev_err(tp->dev,
13249 			   "Register test failed at offset %x\n", offset);
13250 	tw32(offset, save_val);
13251 	return -EIO;
13252 }
13253 
13254 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
13255 {
13256 	static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
13257 	int i;
13258 	u32 j;
13259 
13260 	for (i = 0; i < ARRAY_SIZE(test_pattern); i++) {
13261 		for (j = 0; j < len; j += 4) {
13262 			u32 val;
13263 
13264 			tg3_write_mem(tp, offset + j, test_pattern[i]);
13265 			tg3_read_mem(tp, offset + j, &val);
13266 			if (val != test_pattern[i])
13267 				return -EIO;
13268 		}
13269 	}
13270 	return 0;
13271 }
13272 
13273 static int tg3_test_memory(struct tg3 *tp)
13274 {
13275 	static struct mem_entry {
13276 		u32 offset;
13277 		u32 len;
13278 	} mem_tbl_570x[] = {
13279 		{ 0x00000000, 0x00b50},
13280 		{ 0x00002000, 0x1c000},
13281 		{ 0xffffffff, 0x00000}
13282 	}, mem_tbl_5705[] = {
13283 		{ 0x00000100, 0x0000c},
13284 		{ 0x00000200, 0x00008},
13285 		{ 0x00004000, 0x00800},
13286 		{ 0x00006000, 0x01000},
13287 		{ 0x00008000, 0x02000},
13288 		{ 0x00010000, 0x0e000},
13289 		{ 0xffffffff, 0x00000}
13290 	}, mem_tbl_5755[] = {
13291 		{ 0x00000200, 0x00008},
13292 		{ 0x00004000, 0x00800},
13293 		{ 0x00006000, 0x00800},
13294 		{ 0x00008000, 0x02000},
13295 		{ 0x00010000, 0x0c000},
13296 		{ 0xffffffff, 0x00000}
13297 	}, mem_tbl_5906[] = {
13298 		{ 0x00000200, 0x00008},
13299 		{ 0x00004000, 0x00400},
13300 		{ 0x00006000, 0x00400},
13301 		{ 0x00008000, 0x01000},
13302 		{ 0x00010000, 0x01000},
13303 		{ 0xffffffff, 0x00000}
13304 	}, mem_tbl_5717[] = {
13305 		{ 0x00000200, 0x00008},
13306 		{ 0x00010000, 0x0a000},
13307 		{ 0x00020000, 0x13c00},
13308 		{ 0xffffffff, 0x00000}
13309 	}, mem_tbl_57765[] = {
13310 		{ 0x00000200, 0x00008},
13311 		{ 0x00004000, 0x00800},
13312 		{ 0x00006000, 0x09800},
13313 		{ 0x00010000, 0x0a000},
13314 		{ 0xffffffff, 0x00000}
13315 	};
13316 	struct mem_entry *mem_tbl;
13317 	int err = 0;
13318 	int i;
13319 
13320 	if (tg3_flag(tp, 5717_PLUS))
13321 		mem_tbl = mem_tbl_5717;
13322 	else if (tg3_flag(tp, 57765_CLASS) ||
13323 		 tg3_asic_rev(tp) == ASIC_REV_5762)
13324 		mem_tbl = mem_tbl_57765;
13325 	else if (tg3_flag(tp, 5755_PLUS))
13326 		mem_tbl = mem_tbl_5755;
13327 	else if (tg3_asic_rev(tp) == ASIC_REV_5906)
13328 		mem_tbl = mem_tbl_5906;
13329 	else if (tg3_flag(tp, 5705_PLUS))
13330 		mem_tbl = mem_tbl_5705;
13331 	else
13332 		mem_tbl = mem_tbl_570x;
13333 
13334 	for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
13335 		err = tg3_do_mem_test(tp, mem_tbl[i].offset, mem_tbl[i].len);
13336 		if (err)
13337 			break;
13338 	}
13339 
13340 	return err;
13341 }
13342 
13343 #define TG3_TSO_MSS		500
13344 
13345 #define TG3_TSO_IP_HDR_LEN	20
13346 #define TG3_TSO_TCP_HDR_LEN	20
13347 #define TG3_TSO_TCP_OPT_LEN	12
13348 
13349 static const u8 tg3_tso_header[] = {
13350 0x08, 0x00,
13351 0x45, 0x00, 0x00, 0x00,
13352 0x00, 0x00, 0x40, 0x00,
13353 0x40, 0x06, 0x00, 0x00,
13354 0x0a, 0x00, 0x00, 0x01,
13355 0x0a, 0x00, 0x00, 0x02,
13356 0x0d, 0x00, 0xe0, 0x00,
13357 0x00, 0x00, 0x01, 0x00,
13358 0x00, 0x00, 0x02, 0x00,
13359 0x80, 0x10, 0x10, 0x00,
13360 0x14, 0x09, 0x00, 0x00,
13361 0x01, 0x01, 0x08, 0x0a,
13362 0x11, 0x11, 0x11, 0x11,
13363 0x11, 0x11, 0x11, 0x11,
13364 };
13365 
13366 static int tg3_run_loopback(struct tg3 *tp, u32 pktsz, bool tso_loopback)
13367 {
13368 	u32 rx_start_idx, rx_idx, tx_idx, opaque_key;
13369 	u32 base_flags = 0, mss = 0, desc_idx, coal_now, data_off, val;
13370 	u32 budget;
13371 	struct sk_buff *skb;
13372 	u8 *tx_data, *rx_data;
13373 	dma_addr_t map;
13374 	int num_pkts, tx_len, rx_len, i, err;
13375 	struct tg3_rx_buffer_desc *desc;
13376 	struct tg3_napi *tnapi, *rnapi;
13377 	struct tg3_rx_prodring_set *tpr = &tp->napi[0].prodring;
13378 
13379 	tnapi = &tp->napi[0];
13380 	rnapi = &tp->napi[0];
13381 	if (tp->irq_cnt > 1) {
13382 		if (tg3_flag(tp, ENABLE_RSS))
13383 			rnapi = &tp->napi[1];
13384 		if (tg3_flag(tp, ENABLE_TSS))
13385 			tnapi = &tp->napi[1];
13386 	}
13387 	coal_now = tnapi->coal_now | rnapi->coal_now;
13388 
13389 	err = -EIO;
13390 
13391 	tx_len = pktsz;
13392 	skb = netdev_alloc_skb(tp->dev, tx_len);
13393 	if (!skb)
13394 		return -ENOMEM;
13395 
13396 	tx_data = skb_put(skb, tx_len);
13397 	memcpy(tx_data, tp->dev->dev_addr, ETH_ALEN);
13398 	memset(tx_data + ETH_ALEN, 0x0, 8);
13399 
13400 	tw32(MAC_RX_MTU_SIZE, tx_len + ETH_FCS_LEN);
13401 
13402 	if (tso_loopback) {
13403 		struct iphdr *iph = (struct iphdr *)&tx_data[ETH_HLEN];
13404 
13405 		u32 hdr_len = TG3_TSO_IP_HDR_LEN + TG3_TSO_TCP_HDR_LEN +
13406 			      TG3_TSO_TCP_OPT_LEN;
13407 
13408 		memcpy(tx_data + ETH_ALEN * 2, tg3_tso_header,
13409 		       sizeof(tg3_tso_header));
13410 		mss = TG3_TSO_MSS;
13411 
13412 		val = tx_len - ETH_ALEN * 2 - sizeof(tg3_tso_header);
13413 		num_pkts = DIV_ROUND_UP(val, TG3_TSO_MSS);
13414 
13415 		/* Set the total length field in the IP header */
13416 		iph->tot_len = htons((u16)(mss + hdr_len));
13417 
13418 		base_flags = (TXD_FLAG_CPU_PRE_DMA |
13419 			      TXD_FLAG_CPU_POST_DMA);
13420 
13421 		if (tg3_flag(tp, HW_TSO_1) ||
13422 		    tg3_flag(tp, HW_TSO_2) ||
13423 		    tg3_flag(tp, HW_TSO_3)) {
13424 			struct tcphdr *th;
13425 			val = ETH_HLEN + TG3_TSO_IP_HDR_LEN;
13426 			th = (struct tcphdr *)&tx_data[val];
13427 			th->check = 0;
13428 		} else
13429 			base_flags |= TXD_FLAG_TCPUDP_CSUM;
13430 
13431 		if (tg3_flag(tp, HW_TSO_3)) {
13432 			mss |= (hdr_len & 0xc) << 12;
13433 			if (hdr_len & 0x10)
13434 				base_flags |= 0x00000010;
13435 			base_flags |= (hdr_len & 0x3e0) << 5;
13436 		} else if (tg3_flag(tp, HW_TSO_2))
13437 			mss |= hdr_len << 9;
13438 		else if (tg3_flag(tp, HW_TSO_1) ||
13439 			 tg3_asic_rev(tp) == ASIC_REV_5705) {
13440 			mss |= (TG3_TSO_TCP_OPT_LEN << 9);
13441 		} else {
13442 			base_flags |= (TG3_TSO_TCP_OPT_LEN << 10);
13443 		}
13444 
13445 		data_off = ETH_ALEN * 2 + sizeof(tg3_tso_header);
13446 	} else {
13447 		num_pkts = 1;
13448 		data_off = ETH_HLEN;
13449 
13450 		if (tg3_flag(tp, USE_JUMBO_BDFLAG) &&
13451 		    tx_len > VLAN_ETH_FRAME_LEN)
13452 			base_flags |= TXD_FLAG_JMB_PKT;
13453 	}
13454 
13455 	for (i = data_off; i < tx_len; i++)
13456 		tx_data[i] = (u8) (i & 0xff);
13457 
13458 	map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
13459 	if (pci_dma_mapping_error(tp->pdev, map)) {
13460 		dev_kfree_skb(skb);
13461 		return -EIO;
13462 	}
13463 
13464 	val = tnapi->tx_prod;
13465 	tnapi->tx_buffers[val].skb = skb;
13466 	dma_unmap_addr_set(&tnapi->tx_buffers[val], mapping, map);
13467 
13468 	tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
13469 	       rnapi->coal_now);
13470 
13471 	udelay(10);
13472 
13473 	rx_start_idx = rnapi->hw_status->idx[0].rx_producer;
13474 
13475 	budget = tg3_tx_avail(tnapi);
13476 	if (tg3_tx_frag_set(tnapi, &val, &budget, map, tx_len,
13477 			    base_flags | TXD_FLAG_END, mss, 0)) {
13478 		tnapi->tx_buffers[val].skb = NULL;
13479 		dev_kfree_skb(skb);
13480 		return -EIO;
13481 	}
13482 
13483 	tnapi->tx_prod++;
13484 
13485 	/* Sync BD data before updating mailbox */
13486 	wmb();
13487 
13488 	tw32_tx_mbox(tnapi->prodmbox, tnapi->tx_prod);
13489 	tr32_mailbox(tnapi->prodmbox);
13490 
13491 	udelay(10);
13492 
13493 	/* 350 usec to allow enough time on some 10/100 Mbps devices.  */
13494 	for (i = 0; i < 35; i++) {
13495 		tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
13496 		       coal_now);
13497 
13498 		udelay(10);
13499 
13500 		tx_idx = tnapi->hw_status->idx[0].tx_consumer;
13501 		rx_idx = rnapi->hw_status->idx[0].rx_producer;
13502 		if ((tx_idx == tnapi->tx_prod) &&
13503 		    (rx_idx == (rx_start_idx + num_pkts)))
13504 			break;
13505 	}
13506 
13507 	tg3_tx_skb_unmap(tnapi, tnapi->tx_prod - 1, -1);
13508 	dev_kfree_skb(skb);
13509 
13510 	if (tx_idx != tnapi->tx_prod)
13511 		goto out;
13512 
13513 	if (rx_idx != rx_start_idx + num_pkts)
13514 		goto out;
13515 
13516 	val = data_off;
13517 	while (rx_idx != rx_start_idx) {
13518 		desc = &rnapi->rx_rcb[rx_start_idx++];
13519 		desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
13520 		opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
13521 
13522 		if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
13523 		    (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
13524 			goto out;
13525 
13526 		rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT)
13527 			 - ETH_FCS_LEN;
13528 
13529 		if (!tso_loopback) {
13530 			if (rx_len != tx_len)
13531 				goto out;
13532 
13533 			if (pktsz <= TG3_RX_STD_DMA_SZ - ETH_FCS_LEN) {
13534 				if (opaque_key != RXD_OPAQUE_RING_STD)
13535 					goto out;
13536 			} else {
13537 				if (opaque_key != RXD_OPAQUE_RING_JUMBO)
13538 					goto out;
13539 			}
13540 		} else if ((desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
13541 			   (desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
13542 			    >> RXD_TCPCSUM_SHIFT != 0xffff) {
13543 			goto out;
13544 		}
13545 
13546 		if (opaque_key == RXD_OPAQUE_RING_STD) {
13547 			rx_data = tpr->rx_std_buffers[desc_idx].data;
13548 			map = dma_unmap_addr(&tpr->rx_std_buffers[desc_idx],
13549 					     mapping);
13550 		} else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
13551 			rx_data = tpr->rx_jmb_buffers[desc_idx].data;
13552 			map = dma_unmap_addr(&tpr->rx_jmb_buffers[desc_idx],
13553 					     mapping);
13554 		} else
13555 			goto out;
13556 
13557 		pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len,
13558 					    PCI_DMA_FROMDEVICE);
13559 
13560 		rx_data += TG3_RX_OFFSET(tp);
13561 		for (i = data_off; i < rx_len; i++, val++) {
13562 			if (*(rx_data + i) != (u8) (val & 0xff))
13563 				goto out;
13564 		}
13565 	}
13566 
13567 	err = 0;
13568 
13569 	/* tg3_free_rings will unmap and free the rx_data */
13570 out:
13571 	return err;
13572 }
13573 
13574 #define TG3_STD_LOOPBACK_FAILED		1
13575 #define TG3_JMB_LOOPBACK_FAILED		2
13576 #define TG3_TSO_LOOPBACK_FAILED		4
13577 #define TG3_LOOPBACK_FAILED \
13578 	(TG3_STD_LOOPBACK_FAILED | \
13579 	 TG3_JMB_LOOPBACK_FAILED | \
13580 	 TG3_TSO_LOOPBACK_FAILED)
13581 
13582 static int tg3_test_loopback(struct tg3 *tp, u64 *data, bool do_extlpbk)
13583 {
13584 	int err = -EIO;
13585 	u32 eee_cap;
13586 	u32 jmb_pkt_sz = 9000;
13587 
13588 	if (tp->dma_limit)
13589 		jmb_pkt_sz = tp->dma_limit - ETH_HLEN;
13590 
13591 	eee_cap = tp->phy_flags & TG3_PHYFLG_EEE_CAP;
13592 	tp->phy_flags &= ~TG3_PHYFLG_EEE_CAP;
13593 
13594 	if (!netif_running(tp->dev)) {
13595 		data[TG3_MAC_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13596 		data[TG3_PHY_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13597 		if (do_extlpbk)
13598 			data[TG3_EXT_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13599 		goto done;
13600 	}
13601 
13602 	err = tg3_reset_hw(tp, true);
13603 	if (err) {
13604 		data[TG3_MAC_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13605 		data[TG3_PHY_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13606 		if (do_extlpbk)
13607 			data[TG3_EXT_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13608 		goto done;
13609 	}
13610 
13611 	if (tg3_flag(tp, ENABLE_RSS)) {
13612 		int i;
13613 
13614 		/* Reroute all rx packets to the 1st queue */
13615 		for (i = MAC_RSS_INDIR_TBL_0;
13616 		     i < MAC_RSS_INDIR_TBL_0 + TG3_RSS_INDIR_TBL_SIZE; i += 4)
13617 			tw32(i, 0x0);
13618 	}
13619 
13620 	/* HW errata - mac loopback fails in some cases on 5780.
13621 	 * Normal traffic and PHY loopback are not affected by
13622 	 * errata.  Also, the MAC loopback test is deprecated for
13623 	 * all newer ASIC revisions.
13624 	 */
13625 	if (tg3_asic_rev(tp) != ASIC_REV_5780 &&
13626 	    !tg3_flag(tp, CPMU_PRESENT)) {
13627 		tg3_mac_loopback(tp, true);
13628 
13629 		if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13630 			data[TG3_MAC_LOOPB_TEST] |= TG3_STD_LOOPBACK_FAILED;
13631 
13632 		if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13633 		    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13634 			data[TG3_MAC_LOOPB_TEST] |= TG3_JMB_LOOPBACK_FAILED;
13635 
13636 		tg3_mac_loopback(tp, false);
13637 	}
13638 
13639 	if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
13640 	    !tg3_flag(tp, USE_PHYLIB)) {
13641 		int i;
13642 
13643 		tg3_phy_lpbk_set(tp, 0, false);
13644 
13645 		/* Wait for link */
13646 		for (i = 0; i < 100; i++) {
13647 			if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
13648 				break;
13649 			mdelay(1);
13650 		}
13651 
13652 		if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13653 			data[TG3_PHY_LOOPB_TEST] |= TG3_STD_LOOPBACK_FAILED;
13654 		if (tg3_flag(tp, TSO_CAPABLE) &&
13655 		    tg3_run_loopback(tp, ETH_FRAME_LEN, true))
13656 			data[TG3_PHY_LOOPB_TEST] |= TG3_TSO_LOOPBACK_FAILED;
13657 		if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13658 		    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13659 			data[TG3_PHY_LOOPB_TEST] |= TG3_JMB_LOOPBACK_FAILED;
13660 
13661 		if (do_extlpbk) {
13662 			tg3_phy_lpbk_set(tp, 0, true);
13663 
13664 			/* All link indications report up, but the hardware
13665 			 * isn't really ready for about 20 msec.  Double it
13666 			 * to be sure.
13667 			 */
13668 			mdelay(40);
13669 
13670 			if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13671 				data[TG3_EXT_LOOPB_TEST] |=
13672 							TG3_STD_LOOPBACK_FAILED;
13673 			if (tg3_flag(tp, TSO_CAPABLE) &&
13674 			    tg3_run_loopback(tp, ETH_FRAME_LEN, true))
13675 				data[TG3_EXT_LOOPB_TEST] |=
13676 							TG3_TSO_LOOPBACK_FAILED;
13677 			if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13678 			    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13679 				data[TG3_EXT_LOOPB_TEST] |=
13680 							TG3_JMB_LOOPBACK_FAILED;
13681 		}
13682 
13683 		/* Re-enable gphy autopowerdown. */
13684 		if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
13685 			tg3_phy_toggle_apd(tp, true);
13686 	}
13687 
13688 	err = (data[TG3_MAC_LOOPB_TEST] | data[TG3_PHY_LOOPB_TEST] |
13689 	       data[TG3_EXT_LOOPB_TEST]) ? -EIO : 0;
13690 
13691 done:
13692 	tp->phy_flags |= eee_cap;
13693 
13694 	return err;
13695 }
13696 
13697 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
13698 			  u64 *data)
13699 {
13700 	struct tg3 *tp = netdev_priv(dev);
13701 	bool doextlpbk = etest->flags & ETH_TEST_FL_EXTERNAL_LB;
13702 
13703 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
13704 		if (tg3_power_up(tp)) {
13705 			etest->flags |= ETH_TEST_FL_FAILED;
13706 			memset(data, 1, sizeof(u64) * TG3_NUM_TEST);
13707 			return;
13708 		}
13709 		tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
13710 	}
13711 
13712 	memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
13713 
13714 	if (tg3_test_nvram(tp) != 0) {
13715 		etest->flags |= ETH_TEST_FL_FAILED;
13716 		data[TG3_NVRAM_TEST] = 1;
13717 	}
13718 	if (!doextlpbk && tg3_test_link(tp)) {
13719 		etest->flags |= ETH_TEST_FL_FAILED;
13720 		data[TG3_LINK_TEST] = 1;
13721 	}
13722 	if (etest->flags & ETH_TEST_FL_OFFLINE) {
13723 		int err, err2 = 0, irq_sync = 0;
13724 
13725 		if (netif_running(dev)) {
13726 			tg3_phy_stop(tp);
13727 			tg3_netif_stop(tp);
13728 			irq_sync = 1;
13729 		}
13730 
13731 		tg3_full_lock(tp, irq_sync);
13732 		tg3_halt(tp, RESET_KIND_SUSPEND, 1);
13733 		err = tg3_nvram_lock(tp);
13734 		tg3_halt_cpu(tp, RX_CPU_BASE);
13735 		if (!tg3_flag(tp, 5705_PLUS))
13736 			tg3_halt_cpu(tp, TX_CPU_BASE);
13737 		if (!err)
13738 			tg3_nvram_unlock(tp);
13739 
13740 		if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
13741 			tg3_phy_reset(tp);
13742 
13743 		if (tg3_test_registers(tp) != 0) {
13744 			etest->flags |= ETH_TEST_FL_FAILED;
13745 			data[TG3_REGISTER_TEST] = 1;
13746 		}
13747 
13748 		if (tg3_test_memory(tp) != 0) {
13749 			etest->flags |= ETH_TEST_FL_FAILED;
13750 			data[TG3_MEMORY_TEST] = 1;
13751 		}
13752 
13753 		if (doextlpbk)
13754 			etest->flags |= ETH_TEST_FL_EXTERNAL_LB_DONE;
13755 
13756 		if (tg3_test_loopback(tp, data, doextlpbk))
13757 			etest->flags |= ETH_TEST_FL_FAILED;
13758 
13759 		tg3_full_unlock(tp);
13760 
13761 		if (tg3_test_interrupt(tp) != 0) {
13762 			etest->flags |= ETH_TEST_FL_FAILED;
13763 			data[TG3_INTERRUPT_TEST] = 1;
13764 		}
13765 
13766 		tg3_full_lock(tp, 0);
13767 
13768 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
13769 		if (netif_running(dev)) {
13770 			tg3_flag_set(tp, INIT_COMPLETE);
13771 			err2 = tg3_restart_hw(tp, true);
13772 			if (!err2)
13773 				tg3_netif_start(tp);
13774 		}
13775 
13776 		tg3_full_unlock(tp);
13777 
13778 		if (irq_sync && !err2)
13779 			tg3_phy_start(tp);
13780 	}
13781 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
13782 		tg3_power_down_prepare(tp);
13783 
13784 }
13785 
13786 static int tg3_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
13787 {
13788 	struct tg3 *tp = netdev_priv(dev);
13789 	struct hwtstamp_config stmpconf;
13790 
13791 	if (!tg3_flag(tp, PTP_CAPABLE))
13792 		return -EOPNOTSUPP;
13793 
13794 	if (copy_from_user(&stmpconf, ifr->ifr_data, sizeof(stmpconf)))
13795 		return -EFAULT;
13796 
13797 	if (stmpconf.flags)
13798 		return -EINVAL;
13799 
13800 	if (stmpconf.tx_type != HWTSTAMP_TX_ON &&
13801 	    stmpconf.tx_type != HWTSTAMP_TX_OFF)
13802 		return -ERANGE;
13803 
13804 	switch (stmpconf.rx_filter) {
13805 	case HWTSTAMP_FILTER_NONE:
13806 		tp->rxptpctl = 0;
13807 		break;
13808 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
13809 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13810 			       TG3_RX_PTP_CTL_ALL_V1_EVENTS;
13811 		break;
13812 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
13813 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13814 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13815 		break;
13816 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
13817 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13818 			       TG3_RX_PTP_CTL_DELAY_REQ;
13819 		break;
13820 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
13821 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13822 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13823 		break;
13824 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
13825 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13826 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13827 		break;
13828 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
13829 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13830 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13831 		break;
13832 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
13833 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13834 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13835 		break;
13836 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
13837 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13838 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13839 		break;
13840 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
13841 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13842 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13843 		break;
13844 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
13845 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13846 			       TG3_RX_PTP_CTL_DELAY_REQ;
13847 		break;
13848 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
13849 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13850 			       TG3_RX_PTP_CTL_DELAY_REQ;
13851 		break;
13852 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
13853 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13854 			       TG3_RX_PTP_CTL_DELAY_REQ;
13855 		break;
13856 	default:
13857 		return -ERANGE;
13858 	}
13859 
13860 	if (netif_running(dev) && tp->rxptpctl)
13861 		tw32(TG3_RX_PTP_CTL,
13862 		     tp->rxptpctl | TG3_RX_PTP_CTL_HWTS_INTERLOCK);
13863 
13864 	if (stmpconf.tx_type == HWTSTAMP_TX_ON)
13865 		tg3_flag_set(tp, TX_TSTAMP_EN);
13866 	else
13867 		tg3_flag_clear(tp, TX_TSTAMP_EN);
13868 
13869 	return copy_to_user(ifr->ifr_data, &stmpconf, sizeof(stmpconf)) ?
13870 		-EFAULT : 0;
13871 }
13872 
13873 static int tg3_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
13874 {
13875 	struct tg3 *tp = netdev_priv(dev);
13876 	struct hwtstamp_config stmpconf;
13877 
13878 	if (!tg3_flag(tp, PTP_CAPABLE))
13879 		return -EOPNOTSUPP;
13880 
13881 	stmpconf.flags = 0;
13882 	stmpconf.tx_type = (tg3_flag(tp, TX_TSTAMP_EN) ?
13883 			    HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF);
13884 
13885 	switch (tp->rxptpctl) {
13886 	case 0:
13887 		stmpconf.rx_filter = HWTSTAMP_FILTER_NONE;
13888 		break;
13889 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_ALL_V1_EVENTS:
13890 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
13891 		break;
13892 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13893 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
13894 		break;
13895 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13896 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
13897 		break;
13898 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13899 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
13900 		break;
13901 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13902 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
13903 		break;
13904 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13905 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
13906 		break;
13907 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13908 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
13909 		break;
13910 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13911 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_SYNC;
13912 		break;
13913 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13914 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
13915 		break;
13916 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13917 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
13918 		break;
13919 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13920 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ;
13921 		break;
13922 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13923 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
13924 		break;
13925 	default:
13926 		WARN_ON_ONCE(1);
13927 		return -ERANGE;
13928 	}
13929 
13930 	return copy_to_user(ifr->ifr_data, &stmpconf, sizeof(stmpconf)) ?
13931 		-EFAULT : 0;
13932 }
13933 
13934 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
13935 {
13936 	struct mii_ioctl_data *data = if_mii(ifr);
13937 	struct tg3 *tp = netdev_priv(dev);
13938 	int err;
13939 
13940 	if (tg3_flag(tp, USE_PHYLIB)) {
13941 		struct phy_device *phydev;
13942 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
13943 			return -EAGAIN;
13944 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
13945 		return phy_mii_ioctl(phydev, ifr, cmd);
13946 	}
13947 
13948 	switch (cmd) {
13949 	case SIOCGMIIPHY:
13950 		data->phy_id = tp->phy_addr;
13951 
13952 		/* fallthru */
13953 	case SIOCGMIIREG: {
13954 		u32 mii_regval;
13955 
13956 		if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
13957 			break;			/* We have no PHY */
13958 
13959 		if (!netif_running(dev))
13960 			return -EAGAIN;
13961 
13962 		spin_lock_bh(&tp->lock);
13963 		err = __tg3_readphy(tp, data->phy_id & 0x1f,
13964 				    data->reg_num & 0x1f, &mii_regval);
13965 		spin_unlock_bh(&tp->lock);
13966 
13967 		data->val_out = mii_regval;
13968 
13969 		return err;
13970 	}
13971 
13972 	case SIOCSMIIREG:
13973 		if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
13974 			break;			/* We have no PHY */
13975 
13976 		if (!netif_running(dev))
13977 			return -EAGAIN;
13978 
13979 		spin_lock_bh(&tp->lock);
13980 		err = __tg3_writephy(tp, data->phy_id & 0x1f,
13981 				     data->reg_num & 0x1f, data->val_in);
13982 		spin_unlock_bh(&tp->lock);
13983 
13984 		return err;
13985 
13986 	case SIOCSHWTSTAMP:
13987 		return tg3_hwtstamp_set(dev, ifr);
13988 
13989 	case SIOCGHWTSTAMP:
13990 		return tg3_hwtstamp_get(dev, ifr);
13991 
13992 	default:
13993 		/* do nothing */
13994 		break;
13995 	}
13996 	return -EOPNOTSUPP;
13997 }
13998 
13999 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
14000 {
14001 	struct tg3 *tp = netdev_priv(dev);
14002 
14003 	memcpy(ec, &tp->coal, sizeof(*ec));
14004 	return 0;
14005 }
14006 
14007 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
14008 {
14009 	struct tg3 *tp = netdev_priv(dev);
14010 	u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
14011 	u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
14012 
14013 	if (!tg3_flag(tp, 5705_PLUS)) {
14014 		max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
14015 		max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
14016 		max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
14017 		min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
14018 	}
14019 
14020 	if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
14021 	    (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
14022 	    (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
14023 	    (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
14024 	    (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
14025 	    (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
14026 	    (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
14027 	    (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
14028 	    (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
14029 	    (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
14030 		return -EINVAL;
14031 
14032 	/* No rx interrupts will be generated if both are zero */
14033 	if ((ec->rx_coalesce_usecs == 0) &&
14034 	    (ec->rx_max_coalesced_frames == 0))
14035 		return -EINVAL;
14036 
14037 	/* No tx interrupts will be generated if both are zero */
14038 	if ((ec->tx_coalesce_usecs == 0) &&
14039 	    (ec->tx_max_coalesced_frames == 0))
14040 		return -EINVAL;
14041 
14042 	/* Only copy relevant parameters, ignore all others. */
14043 	tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
14044 	tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
14045 	tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
14046 	tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
14047 	tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
14048 	tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
14049 	tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
14050 	tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
14051 	tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
14052 
14053 	if (netif_running(dev)) {
14054 		tg3_full_lock(tp, 0);
14055 		__tg3_set_coalesce(tp, &tp->coal);
14056 		tg3_full_unlock(tp);
14057 	}
14058 	return 0;
14059 }
14060 
14061 static int tg3_set_eee(struct net_device *dev, struct ethtool_eee *edata)
14062 {
14063 	struct tg3 *tp = netdev_priv(dev);
14064 
14065 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
14066 		netdev_warn(tp->dev, "Board does not support EEE!\n");
14067 		return -EOPNOTSUPP;
14068 	}
14069 
14070 	if (edata->advertised != tp->eee.advertised) {
14071 		netdev_warn(tp->dev,
14072 			    "Direct manipulation of EEE advertisement is not supported\n");
14073 		return -EINVAL;
14074 	}
14075 
14076 	if (edata->tx_lpi_timer > TG3_CPMU_DBTMR1_LNKIDLE_MAX) {
14077 		netdev_warn(tp->dev,
14078 			    "Maximal Tx Lpi timer supported is %#x(u)\n",
14079 			    TG3_CPMU_DBTMR1_LNKIDLE_MAX);
14080 		return -EINVAL;
14081 	}
14082 
14083 	tp->eee = *edata;
14084 
14085 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
14086 	tg3_warn_mgmt_link_flap(tp);
14087 
14088 	if (netif_running(tp->dev)) {
14089 		tg3_full_lock(tp, 0);
14090 		tg3_setup_eee(tp);
14091 		tg3_phy_reset(tp);
14092 		tg3_full_unlock(tp);
14093 	}
14094 
14095 	return 0;
14096 }
14097 
14098 static int tg3_get_eee(struct net_device *dev, struct ethtool_eee *edata)
14099 {
14100 	struct tg3 *tp = netdev_priv(dev);
14101 
14102 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
14103 		netdev_warn(tp->dev,
14104 			    "Board does not support EEE!\n");
14105 		return -EOPNOTSUPP;
14106 	}
14107 
14108 	*edata = tp->eee;
14109 	return 0;
14110 }
14111 
14112 static const struct ethtool_ops tg3_ethtool_ops = {
14113 	.get_settings		= tg3_get_settings,
14114 	.set_settings		= tg3_set_settings,
14115 	.get_drvinfo		= tg3_get_drvinfo,
14116 	.get_regs_len		= tg3_get_regs_len,
14117 	.get_regs		= tg3_get_regs,
14118 	.get_wol		= tg3_get_wol,
14119 	.set_wol		= tg3_set_wol,
14120 	.get_msglevel		= tg3_get_msglevel,
14121 	.set_msglevel		= tg3_set_msglevel,
14122 	.nway_reset		= tg3_nway_reset,
14123 	.get_link		= ethtool_op_get_link,
14124 	.get_eeprom_len		= tg3_get_eeprom_len,
14125 	.get_eeprom		= tg3_get_eeprom,
14126 	.set_eeprom		= tg3_set_eeprom,
14127 	.get_ringparam		= tg3_get_ringparam,
14128 	.set_ringparam		= tg3_set_ringparam,
14129 	.get_pauseparam		= tg3_get_pauseparam,
14130 	.set_pauseparam		= tg3_set_pauseparam,
14131 	.self_test		= tg3_self_test,
14132 	.get_strings		= tg3_get_strings,
14133 	.set_phys_id		= tg3_set_phys_id,
14134 	.get_ethtool_stats	= tg3_get_ethtool_stats,
14135 	.get_coalesce		= tg3_get_coalesce,
14136 	.set_coalesce		= tg3_set_coalesce,
14137 	.get_sset_count		= tg3_get_sset_count,
14138 	.get_rxnfc		= tg3_get_rxnfc,
14139 	.get_rxfh_indir_size    = tg3_get_rxfh_indir_size,
14140 	.get_rxfh		= tg3_get_rxfh,
14141 	.set_rxfh		= tg3_set_rxfh,
14142 	.get_channels		= tg3_get_channels,
14143 	.set_channels		= tg3_set_channels,
14144 	.get_ts_info		= tg3_get_ts_info,
14145 	.get_eee		= tg3_get_eee,
14146 	.set_eee		= tg3_set_eee,
14147 };
14148 
14149 static struct rtnl_link_stats64 *tg3_get_stats64(struct net_device *dev,
14150 						struct rtnl_link_stats64 *stats)
14151 {
14152 	struct tg3 *tp = netdev_priv(dev);
14153 
14154 	spin_lock_bh(&tp->lock);
14155 	if (!tp->hw_stats) {
14156 		*stats = tp->net_stats_prev;
14157 		spin_unlock_bh(&tp->lock);
14158 		return stats;
14159 	}
14160 
14161 	tg3_get_nstats(tp, stats);
14162 	spin_unlock_bh(&tp->lock);
14163 
14164 	return stats;
14165 }
14166 
14167 static void tg3_set_rx_mode(struct net_device *dev)
14168 {
14169 	struct tg3 *tp = netdev_priv(dev);
14170 
14171 	if (!netif_running(dev))
14172 		return;
14173 
14174 	tg3_full_lock(tp, 0);
14175 	__tg3_set_rx_mode(dev);
14176 	tg3_full_unlock(tp);
14177 }
14178 
14179 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
14180 			       int new_mtu)
14181 {
14182 	dev->mtu = new_mtu;
14183 
14184 	if (new_mtu > ETH_DATA_LEN) {
14185 		if (tg3_flag(tp, 5780_CLASS)) {
14186 			netdev_update_features(dev);
14187 			tg3_flag_clear(tp, TSO_CAPABLE);
14188 		} else {
14189 			tg3_flag_set(tp, JUMBO_RING_ENABLE);
14190 		}
14191 	} else {
14192 		if (tg3_flag(tp, 5780_CLASS)) {
14193 			tg3_flag_set(tp, TSO_CAPABLE);
14194 			netdev_update_features(dev);
14195 		}
14196 		tg3_flag_clear(tp, JUMBO_RING_ENABLE);
14197 	}
14198 }
14199 
14200 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
14201 {
14202 	struct tg3 *tp = netdev_priv(dev);
14203 	int err;
14204 	bool reset_phy = false;
14205 
14206 	if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
14207 		return -EINVAL;
14208 
14209 	if (!netif_running(dev)) {
14210 		/* We'll just catch it later when the
14211 		 * device is up'd.
14212 		 */
14213 		tg3_set_mtu(dev, tp, new_mtu);
14214 		return 0;
14215 	}
14216 
14217 	tg3_phy_stop(tp);
14218 
14219 	tg3_netif_stop(tp);
14220 
14221 	tg3_set_mtu(dev, tp, new_mtu);
14222 
14223 	tg3_full_lock(tp, 1);
14224 
14225 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
14226 
14227 	/* Reset PHY, otherwise the read DMA engine will be in a mode that
14228 	 * breaks all requests to 256 bytes.
14229 	 */
14230 	if (tg3_asic_rev(tp) == ASIC_REV_57766)
14231 		reset_phy = true;
14232 
14233 	err = tg3_restart_hw(tp, reset_phy);
14234 
14235 	if (!err)
14236 		tg3_netif_start(tp);
14237 
14238 	tg3_full_unlock(tp);
14239 
14240 	if (!err)
14241 		tg3_phy_start(tp);
14242 
14243 	return err;
14244 }
14245 
14246 static const struct net_device_ops tg3_netdev_ops = {
14247 	.ndo_open		= tg3_open,
14248 	.ndo_stop		= tg3_close,
14249 	.ndo_start_xmit		= tg3_start_xmit,
14250 	.ndo_get_stats64	= tg3_get_stats64,
14251 	.ndo_validate_addr	= eth_validate_addr,
14252 	.ndo_set_rx_mode	= tg3_set_rx_mode,
14253 	.ndo_set_mac_address	= tg3_set_mac_addr,
14254 	.ndo_do_ioctl		= tg3_ioctl,
14255 	.ndo_tx_timeout		= tg3_tx_timeout,
14256 	.ndo_change_mtu		= tg3_change_mtu,
14257 	.ndo_fix_features	= tg3_fix_features,
14258 	.ndo_set_features	= tg3_set_features,
14259 #ifdef CONFIG_NET_POLL_CONTROLLER
14260 	.ndo_poll_controller	= tg3_poll_controller,
14261 #endif
14262 };
14263 
14264 static void tg3_get_eeprom_size(struct tg3 *tp)
14265 {
14266 	u32 cursize, val, magic;
14267 
14268 	tp->nvram_size = EEPROM_CHIP_SIZE;
14269 
14270 	if (tg3_nvram_read(tp, 0, &magic) != 0)
14271 		return;
14272 
14273 	if ((magic != TG3_EEPROM_MAGIC) &&
14274 	    ((magic & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW) &&
14275 	    ((magic & TG3_EEPROM_MAGIC_HW_MSK) != TG3_EEPROM_MAGIC_HW))
14276 		return;
14277 
14278 	/*
14279 	 * Size the chip by reading offsets at increasing powers of two.
14280 	 * When we encounter our validation signature, we know the addressing
14281 	 * has wrapped around, and thus have our chip size.
14282 	 */
14283 	cursize = 0x10;
14284 
14285 	while (cursize < tp->nvram_size) {
14286 		if (tg3_nvram_read(tp, cursize, &val) != 0)
14287 			return;
14288 
14289 		if (val == magic)
14290 			break;
14291 
14292 		cursize <<= 1;
14293 	}
14294 
14295 	tp->nvram_size = cursize;
14296 }
14297 
14298 static void tg3_get_nvram_size(struct tg3 *tp)
14299 {
14300 	u32 val;
14301 
14302 	if (tg3_flag(tp, NO_NVRAM) || tg3_nvram_read(tp, 0, &val) != 0)
14303 		return;
14304 
14305 	/* Selfboot format */
14306 	if (val != TG3_EEPROM_MAGIC) {
14307 		tg3_get_eeprom_size(tp);
14308 		return;
14309 	}
14310 
14311 	if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
14312 		if (val != 0) {
14313 			/* This is confusing.  We want to operate on the
14314 			 * 16-bit value at offset 0xf2.  The tg3_nvram_read()
14315 			 * call will read from NVRAM and byteswap the data
14316 			 * according to the byteswapping settings for all
14317 			 * other register accesses.  This ensures the data we
14318 			 * want will always reside in the lower 16-bits.
14319 			 * However, the data in NVRAM is in LE format, which
14320 			 * means the data from the NVRAM read will always be
14321 			 * opposite the endianness of the CPU.  The 16-bit
14322 			 * byteswap then brings the data to CPU endianness.
14323 			 */
14324 			tp->nvram_size = swab16((u16)(val & 0x0000ffff)) * 1024;
14325 			return;
14326 		}
14327 	}
14328 	tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14329 }
14330 
14331 static void tg3_get_nvram_info(struct tg3 *tp)
14332 {
14333 	u32 nvcfg1;
14334 
14335 	nvcfg1 = tr32(NVRAM_CFG1);
14336 	if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
14337 		tg3_flag_set(tp, FLASH);
14338 	} else {
14339 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14340 		tw32(NVRAM_CFG1, nvcfg1);
14341 	}
14342 
14343 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
14344 	    tg3_flag(tp, 5780_CLASS)) {
14345 		switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
14346 		case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
14347 			tp->nvram_jedecnum = JEDEC_ATMEL;
14348 			tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
14349 			tg3_flag_set(tp, NVRAM_BUFFERED);
14350 			break;
14351 		case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
14352 			tp->nvram_jedecnum = JEDEC_ATMEL;
14353 			tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
14354 			break;
14355 		case FLASH_VENDOR_ATMEL_EEPROM:
14356 			tp->nvram_jedecnum = JEDEC_ATMEL;
14357 			tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14358 			tg3_flag_set(tp, NVRAM_BUFFERED);
14359 			break;
14360 		case FLASH_VENDOR_ST:
14361 			tp->nvram_jedecnum = JEDEC_ST;
14362 			tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
14363 			tg3_flag_set(tp, NVRAM_BUFFERED);
14364 			break;
14365 		case FLASH_VENDOR_SAIFUN:
14366 			tp->nvram_jedecnum = JEDEC_SAIFUN;
14367 			tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
14368 			break;
14369 		case FLASH_VENDOR_SST_SMALL:
14370 		case FLASH_VENDOR_SST_LARGE:
14371 			tp->nvram_jedecnum = JEDEC_SST;
14372 			tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
14373 			break;
14374 		}
14375 	} else {
14376 		tp->nvram_jedecnum = JEDEC_ATMEL;
14377 		tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
14378 		tg3_flag_set(tp, NVRAM_BUFFERED);
14379 	}
14380 }
14381 
14382 static void tg3_nvram_get_pagesize(struct tg3 *tp, u32 nvmcfg1)
14383 {
14384 	switch (nvmcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
14385 	case FLASH_5752PAGE_SIZE_256:
14386 		tp->nvram_pagesize = 256;
14387 		break;
14388 	case FLASH_5752PAGE_SIZE_512:
14389 		tp->nvram_pagesize = 512;
14390 		break;
14391 	case FLASH_5752PAGE_SIZE_1K:
14392 		tp->nvram_pagesize = 1024;
14393 		break;
14394 	case FLASH_5752PAGE_SIZE_2K:
14395 		tp->nvram_pagesize = 2048;
14396 		break;
14397 	case FLASH_5752PAGE_SIZE_4K:
14398 		tp->nvram_pagesize = 4096;
14399 		break;
14400 	case FLASH_5752PAGE_SIZE_264:
14401 		tp->nvram_pagesize = 264;
14402 		break;
14403 	case FLASH_5752PAGE_SIZE_528:
14404 		tp->nvram_pagesize = 528;
14405 		break;
14406 	}
14407 }
14408 
14409 static void tg3_get_5752_nvram_info(struct tg3 *tp)
14410 {
14411 	u32 nvcfg1;
14412 
14413 	nvcfg1 = tr32(NVRAM_CFG1);
14414 
14415 	/* NVRAM protection for TPM */
14416 	if (nvcfg1 & (1 << 27))
14417 		tg3_flag_set(tp, PROTECTED_NVRAM);
14418 
14419 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14420 	case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
14421 	case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
14422 		tp->nvram_jedecnum = JEDEC_ATMEL;
14423 		tg3_flag_set(tp, NVRAM_BUFFERED);
14424 		break;
14425 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14426 		tp->nvram_jedecnum = JEDEC_ATMEL;
14427 		tg3_flag_set(tp, NVRAM_BUFFERED);
14428 		tg3_flag_set(tp, FLASH);
14429 		break;
14430 	case FLASH_5752VENDOR_ST_M45PE10:
14431 	case FLASH_5752VENDOR_ST_M45PE20:
14432 	case FLASH_5752VENDOR_ST_M45PE40:
14433 		tp->nvram_jedecnum = JEDEC_ST;
14434 		tg3_flag_set(tp, NVRAM_BUFFERED);
14435 		tg3_flag_set(tp, FLASH);
14436 		break;
14437 	}
14438 
14439 	if (tg3_flag(tp, FLASH)) {
14440 		tg3_nvram_get_pagesize(tp, nvcfg1);
14441 	} else {
14442 		/* For eeprom, set pagesize to maximum eeprom size */
14443 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14444 
14445 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14446 		tw32(NVRAM_CFG1, nvcfg1);
14447 	}
14448 }
14449 
14450 static void tg3_get_5755_nvram_info(struct tg3 *tp)
14451 {
14452 	u32 nvcfg1, protect = 0;
14453 
14454 	nvcfg1 = tr32(NVRAM_CFG1);
14455 
14456 	/* NVRAM protection for TPM */
14457 	if (nvcfg1 & (1 << 27)) {
14458 		tg3_flag_set(tp, PROTECTED_NVRAM);
14459 		protect = 1;
14460 	}
14461 
14462 	nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
14463 	switch (nvcfg1) {
14464 	case FLASH_5755VENDOR_ATMEL_FLASH_1:
14465 	case FLASH_5755VENDOR_ATMEL_FLASH_2:
14466 	case FLASH_5755VENDOR_ATMEL_FLASH_3:
14467 	case FLASH_5755VENDOR_ATMEL_FLASH_5:
14468 		tp->nvram_jedecnum = JEDEC_ATMEL;
14469 		tg3_flag_set(tp, NVRAM_BUFFERED);
14470 		tg3_flag_set(tp, FLASH);
14471 		tp->nvram_pagesize = 264;
14472 		if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_1 ||
14473 		    nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_5)
14474 			tp->nvram_size = (protect ? 0x3e200 :
14475 					  TG3_NVRAM_SIZE_512KB);
14476 		else if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_2)
14477 			tp->nvram_size = (protect ? 0x1f200 :
14478 					  TG3_NVRAM_SIZE_256KB);
14479 		else
14480 			tp->nvram_size = (protect ? 0x1f200 :
14481 					  TG3_NVRAM_SIZE_128KB);
14482 		break;
14483 	case FLASH_5752VENDOR_ST_M45PE10:
14484 	case FLASH_5752VENDOR_ST_M45PE20:
14485 	case FLASH_5752VENDOR_ST_M45PE40:
14486 		tp->nvram_jedecnum = JEDEC_ST;
14487 		tg3_flag_set(tp, NVRAM_BUFFERED);
14488 		tg3_flag_set(tp, FLASH);
14489 		tp->nvram_pagesize = 256;
14490 		if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE10)
14491 			tp->nvram_size = (protect ?
14492 					  TG3_NVRAM_SIZE_64KB :
14493 					  TG3_NVRAM_SIZE_128KB);
14494 		else if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE20)
14495 			tp->nvram_size = (protect ?
14496 					  TG3_NVRAM_SIZE_64KB :
14497 					  TG3_NVRAM_SIZE_256KB);
14498 		else
14499 			tp->nvram_size = (protect ?
14500 					  TG3_NVRAM_SIZE_128KB :
14501 					  TG3_NVRAM_SIZE_512KB);
14502 		break;
14503 	}
14504 }
14505 
14506 static void tg3_get_5787_nvram_info(struct tg3 *tp)
14507 {
14508 	u32 nvcfg1;
14509 
14510 	nvcfg1 = tr32(NVRAM_CFG1);
14511 
14512 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14513 	case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
14514 	case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
14515 	case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
14516 	case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
14517 		tp->nvram_jedecnum = JEDEC_ATMEL;
14518 		tg3_flag_set(tp, NVRAM_BUFFERED);
14519 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14520 
14521 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14522 		tw32(NVRAM_CFG1, nvcfg1);
14523 		break;
14524 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14525 	case FLASH_5755VENDOR_ATMEL_FLASH_1:
14526 	case FLASH_5755VENDOR_ATMEL_FLASH_2:
14527 	case FLASH_5755VENDOR_ATMEL_FLASH_3:
14528 		tp->nvram_jedecnum = JEDEC_ATMEL;
14529 		tg3_flag_set(tp, NVRAM_BUFFERED);
14530 		tg3_flag_set(tp, FLASH);
14531 		tp->nvram_pagesize = 264;
14532 		break;
14533 	case FLASH_5752VENDOR_ST_M45PE10:
14534 	case FLASH_5752VENDOR_ST_M45PE20:
14535 	case FLASH_5752VENDOR_ST_M45PE40:
14536 		tp->nvram_jedecnum = JEDEC_ST;
14537 		tg3_flag_set(tp, NVRAM_BUFFERED);
14538 		tg3_flag_set(tp, FLASH);
14539 		tp->nvram_pagesize = 256;
14540 		break;
14541 	}
14542 }
14543 
14544 static void tg3_get_5761_nvram_info(struct tg3 *tp)
14545 {
14546 	u32 nvcfg1, protect = 0;
14547 
14548 	nvcfg1 = tr32(NVRAM_CFG1);
14549 
14550 	/* NVRAM protection for TPM */
14551 	if (nvcfg1 & (1 << 27)) {
14552 		tg3_flag_set(tp, PROTECTED_NVRAM);
14553 		protect = 1;
14554 	}
14555 
14556 	nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
14557 	switch (nvcfg1) {
14558 	case FLASH_5761VENDOR_ATMEL_ADB021D:
14559 	case FLASH_5761VENDOR_ATMEL_ADB041D:
14560 	case FLASH_5761VENDOR_ATMEL_ADB081D:
14561 	case FLASH_5761VENDOR_ATMEL_ADB161D:
14562 	case FLASH_5761VENDOR_ATMEL_MDB021D:
14563 	case FLASH_5761VENDOR_ATMEL_MDB041D:
14564 	case FLASH_5761VENDOR_ATMEL_MDB081D:
14565 	case FLASH_5761VENDOR_ATMEL_MDB161D:
14566 		tp->nvram_jedecnum = JEDEC_ATMEL;
14567 		tg3_flag_set(tp, NVRAM_BUFFERED);
14568 		tg3_flag_set(tp, FLASH);
14569 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14570 		tp->nvram_pagesize = 256;
14571 		break;
14572 	case FLASH_5761VENDOR_ST_A_M45PE20:
14573 	case FLASH_5761VENDOR_ST_A_M45PE40:
14574 	case FLASH_5761VENDOR_ST_A_M45PE80:
14575 	case FLASH_5761VENDOR_ST_A_M45PE16:
14576 	case FLASH_5761VENDOR_ST_M_M45PE20:
14577 	case FLASH_5761VENDOR_ST_M_M45PE40:
14578 	case FLASH_5761VENDOR_ST_M_M45PE80:
14579 	case FLASH_5761VENDOR_ST_M_M45PE16:
14580 		tp->nvram_jedecnum = JEDEC_ST;
14581 		tg3_flag_set(tp, NVRAM_BUFFERED);
14582 		tg3_flag_set(tp, FLASH);
14583 		tp->nvram_pagesize = 256;
14584 		break;
14585 	}
14586 
14587 	if (protect) {
14588 		tp->nvram_size = tr32(NVRAM_ADDR_LOCKOUT);
14589 	} else {
14590 		switch (nvcfg1) {
14591 		case FLASH_5761VENDOR_ATMEL_ADB161D:
14592 		case FLASH_5761VENDOR_ATMEL_MDB161D:
14593 		case FLASH_5761VENDOR_ST_A_M45PE16:
14594 		case FLASH_5761VENDOR_ST_M_M45PE16:
14595 			tp->nvram_size = TG3_NVRAM_SIZE_2MB;
14596 			break;
14597 		case FLASH_5761VENDOR_ATMEL_ADB081D:
14598 		case FLASH_5761VENDOR_ATMEL_MDB081D:
14599 		case FLASH_5761VENDOR_ST_A_M45PE80:
14600 		case FLASH_5761VENDOR_ST_M_M45PE80:
14601 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14602 			break;
14603 		case FLASH_5761VENDOR_ATMEL_ADB041D:
14604 		case FLASH_5761VENDOR_ATMEL_MDB041D:
14605 		case FLASH_5761VENDOR_ST_A_M45PE40:
14606 		case FLASH_5761VENDOR_ST_M_M45PE40:
14607 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14608 			break;
14609 		case FLASH_5761VENDOR_ATMEL_ADB021D:
14610 		case FLASH_5761VENDOR_ATMEL_MDB021D:
14611 		case FLASH_5761VENDOR_ST_A_M45PE20:
14612 		case FLASH_5761VENDOR_ST_M_M45PE20:
14613 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14614 			break;
14615 		}
14616 	}
14617 }
14618 
14619 static void tg3_get_5906_nvram_info(struct tg3 *tp)
14620 {
14621 	tp->nvram_jedecnum = JEDEC_ATMEL;
14622 	tg3_flag_set(tp, NVRAM_BUFFERED);
14623 	tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14624 }
14625 
14626 static void tg3_get_57780_nvram_info(struct tg3 *tp)
14627 {
14628 	u32 nvcfg1;
14629 
14630 	nvcfg1 = tr32(NVRAM_CFG1);
14631 
14632 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14633 	case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
14634 	case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
14635 		tp->nvram_jedecnum = JEDEC_ATMEL;
14636 		tg3_flag_set(tp, NVRAM_BUFFERED);
14637 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14638 
14639 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14640 		tw32(NVRAM_CFG1, nvcfg1);
14641 		return;
14642 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14643 	case FLASH_57780VENDOR_ATMEL_AT45DB011D:
14644 	case FLASH_57780VENDOR_ATMEL_AT45DB011B:
14645 	case FLASH_57780VENDOR_ATMEL_AT45DB021D:
14646 	case FLASH_57780VENDOR_ATMEL_AT45DB021B:
14647 	case FLASH_57780VENDOR_ATMEL_AT45DB041D:
14648 	case FLASH_57780VENDOR_ATMEL_AT45DB041B:
14649 		tp->nvram_jedecnum = JEDEC_ATMEL;
14650 		tg3_flag_set(tp, NVRAM_BUFFERED);
14651 		tg3_flag_set(tp, FLASH);
14652 
14653 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14654 		case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14655 		case FLASH_57780VENDOR_ATMEL_AT45DB011D:
14656 		case FLASH_57780VENDOR_ATMEL_AT45DB011B:
14657 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14658 			break;
14659 		case FLASH_57780VENDOR_ATMEL_AT45DB021D:
14660 		case FLASH_57780VENDOR_ATMEL_AT45DB021B:
14661 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14662 			break;
14663 		case FLASH_57780VENDOR_ATMEL_AT45DB041D:
14664 		case FLASH_57780VENDOR_ATMEL_AT45DB041B:
14665 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14666 			break;
14667 		}
14668 		break;
14669 	case FLASH_5752VENDOR_ST_M45PE10:
14670 	case FLASH_5752VENDOR_ST_M45PE20:
14671 	case FLASH_5752VENDOR_ST_M45PE40:
14672 		tp->nvram_jedecnum = JEDEC_ST;
14673 		tg3_flag_set(tp, NVRAM_BUFFERED);
14674 		tg3_flag_set(tp, FLASH);
14675 
14676 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14677 		case FLASH_5752VENDOR_ST_M45PE10:
14678 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14679 			break;
14680 		case FLASH_5752VENDOR_ST_M45PE20:
14681 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14682 			break;
14683 		case FLASH_5752VENDOR_ST_M45PE40:
14684 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14685 			break;
14686 		}
14687 		break;
14688 	default:
14689 		tg3_flag_set(tp, NO_NVRAM);
14690 		return;
14691 	}
14692 
14693 	tg3_nvram_get_pagesize(tp, nvcfg1);
14694 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14695 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14696 }
14697 
14698 
14699 static void tg3_get_5717_nvram_info(struct tg3 *tp)
14700 {
14701 	u32 nvcfg1;
14702 
14703 	nvcfg1 = tr32(NVRAM_CFG1);
14704 
14705 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14706 	case FLASH_5717VENDOR_ATMEL_EEPROM:
14707 	case FLASH_5717VENDOR_MICRO_EEPROM:
14708 		tp->nvram_jedecnum = JEDEC_ATMEL;
14709 		tg3_flag_set(tp, NVRAM_BUFFERED);
14710 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14711 
14712 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14713 		tw32(NVRAM_CFG1, nvcfg1);
14714 		return;
14715 	case FLASH_5717VENDOR_ATMEL_MDB011D:
14716 	case FLASH_5717VENDOR_ATMEL_ADB011B:
14717 	case FLASH_5717VENDOR_ATMEL_ADB011D:
14718 	case FLASH_5717VENDOR_ATMEL_MDB021D:
14719 	case FLASH_5717VENDOR_ATMEL_ADB021B:
14720 	case FLASH_5717VENDOR_ATMEL_ADB021D:
14721 	case FLASH_5717VENDOR_ATMEL_45USPT:
14722 		tp->nvram_jedecnum = JEDEC_ATMEL;
14723 		tg3_flag_set(tp, NVRAM_BUFFERED);
14724 		tg3_flag_set(tp, FLASH);
14725 
14726 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14727 		case FLASH_5717VENDOR_ATMEL_MDB021D:
14728 			/* Detect size with tg3_nvram_get_size() */
14729 			break;
14730 		case FLASH_5717VENDOR_ATMEL_ADB021B:
14731 		case FLASH_5717VENDOR_ATMEL_ADB021D:
14732 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14733 			break;
14734 		default:
14735 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14736 			break;
14737 		}
14738 		break;
14739 	case FLASH_5717VENDOR_ST_M_M25PE10:
14740 	case FLASH_5717VENDOR_ST_A_M25PE10:
14741 	case FLASH_5717VENDOR_ST_M_M45PE10:
14742 	case FLASH_5717VENDOR_ST_A_M45PE10:
14743 	case FLASH_5717VENDOR_ST_M_M25PE20:
14744 	case FLASH_5717VENDOR_ST_A_M25PE20:
14745 	case FLASH_5717VENDOR_ST_M_M45PE20:
14746 	case FLASH_5717VENDOR_ST_A_M45PE20:
14747 	case FLASH_5717VENDOR_ST_25USPT:
14748 	case FLASH_5717VENDOR_ST_45USPT:
14749 		tp->nvram_jedecnum = JEDEC_ST;
14750 		tg3_flag_set(tp, NVRAM_BUFFERED);
14751 		tg3_flag_set(tp, FLASH);
14752 
14753 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14754 		case FLASH_5717VENDOR_ST_M_M25PE20:
14755 		case FLASH_5717VENDOR_ST_M_M45PE20:
14756 			/* Detect size with tg3_nvram_get_size() */
14757 			break;
14758 		case FLASH_5717VENDOR_ST_A_M25PE20:
14759 		case FLASH_5717VENDOR_ST_A_M45PE20:
14760 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14761 			break;
14762 		default:
14763 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14764 			break;
14765 		}
14766 		break;
14767 	default:
14768 		tg3_flag_set(tp, NO_NVRAM);
14769 		return;
14770 	}
14771 
14772 	tg3_nvram_get_pagesize(tp, nvcfg1);
14773 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14774 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14775 }
14776 
14777 static void tg3_get_5720_nvram_info(struct tg3 *tp)
14778 {
14779 	u32 nvcfg1, nvmpinstrp;
14780 
14781 	nvcfg1 = tr32(NVRAM_CFG1);
14782 	nvmpinstrp = nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK;
14783 
14784 	if (tg3_asic_rev(tp) == ASIC_REV_5762) {
14785 		if (!(nvcfg1 & NVRAM_CFG1_5762VENDOR_MASK)) {
14786 			tg3_flag_set(tp, NO_NVRAM);
14787 			return;
14788 		}
14789 
14790 		switch (nvmpinstrp) {
14791 		case FLASH_5762_EEPROM_HD:
14792 			nvmpinstrp = FLASH_5720_EEPROM_HD;
14793 			break;
14794 		case FLASH_5762_EEPROM_LD:
14795 			nvmpinstrp = FLASH_5720_EEPROM_LD;
14796 			break;
14797 		case FLASH_5720VENDOR_M_ST_M45PE20:
14798 			/* This pinstrap supports multiple sizes, so force it
14799 			 * to read the actual size from location 0xf0.
14800 			 */
14801 			nvmpinstrp = FLASH_5720VENDOR_ST_45USPT;
14802 			break;
14803 		}
14804 	}
14805 
14806 	switch (nvmpinstrp) {
14807 	case FLASH_5720_EEPROM_HD:
14808 	case FLASH_5720_EEPROM_LD:
14809 		tp->nvram_jedecnum = JEDEC_ATMEL;
14810 		tg3_flag_set(tp, NVRAM_BUFFERED);
14811 
14812 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14813 		tw32(NVRAM_CFG1, nvcfg1);
14814 		if (nvmpinstrp == FLASH_5720_EEPROM_HD)
14815 			tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14816 		else
14817 			tp->nvram_pagesize = ATMEL_AT24C02_CHIP_SIZE;
14818 		return;
14819 	case FLASH_5720VENDOR_M_ATMEL_DB011D:
14820 	case FLASH_5720VENDOR_A_ATMEL_DB011B:
14821 	case FLASH_5720VENDOR_A_ATMEL_DB011D:
14822 	case FLASH_5720VENDOR_M_ATMEL_DB021D:
14823 	case FLASH_5720VENDOR_A_ATMEL_DB021B:
14824 	case FLASH_5720VENDOR_A_ATMEL_DB021D:
14825 	case FLASH_5720VENDOR_M_ATMEL_DB041D:
14826 	case FLASH_5720VENDOR_A_ATMEL_DB041B:
14827 	case FLASH_5720VENDOR_A_ATMEL_DB041D:
14828 	case FLASH_5720VENDOR_M_ATMEL_DB081D:
14829 	case FLASH_5720VENDOR_A_ATMEL_DB081D:
14830 	case FLASH_5720VENDOR_ATMEL_45USPT:
14831 		tp->nvram_jedecnum = JEDEC_ATMEL;
14832 		tg3_flag_set(tp, NVRAM_BUFFERED);
14833 		tg3_flag_set(tp, FLASH);
14834 
14835 		switch (nvmpinstrp) {
14836 		case FLASH_5720VENDOR_M_ATMEL_DB021D:
14837 		case FLASH_5720VENDOR_A_ATMEL_DB021B:
14838 		case FLASH_5720VENDOR_A_ATMEL_DB021D:
14839 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14840 			break;
14841 		case FLASH_5720VENDOR_M_ATMEL_DB041D:
14842 		case FLASH_5720VENDOR_A_ATMEL_DB041B:
14843 		case FLASH_5720VENDOR_A_ATMEL_DB041D:
14844 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14845 			break;
14846 		case FLASH_5720VENDOR_M_ATMEL_DB081D:
14847 		case FLASH_5720VENDOR_A_ATMEL_DB081D:
14848 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14849 			break;
14850 		default:
14851 			if (tg3_asic_rev(tp) != ASIC_REV_5762)
14852 				tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14853 			break;
14854 		}
14855 		break;
14856 	case FLASH_5720VENDOR_M_ST_M25PE10:
14857 	case FLASH_5720VENDOR_M_ST_M45PE10:
14858 	case FLASH_5720VENDOR_A_ST_M25PE10:
14859 	case FLASH_5720VENDOR_A_ST_M45PE10:
14860 	case FLASH_5720VENDOR_M_ST_M25PE20:
14861 	case FLASH_5720VENDOR_M_ST_M45PE20:
14862 	case FLASH_5720VENDOR_A_ST_M25PE20:
14863 	case FLASH_5720VENDOR_A_ST_M45PE20:
14864 	case FLASH_5720VENDOR_M_ST_M25PE40:
14865 	case FLASH_5720VENDOR_M_ST_M45PE40:
14866 	case FLASH_5720VENDOR_A_ST_M25PE40:
14867 	case FLASH_5720VENDOR_A_ST_M45PE40:
14868 	case FLASH_5720VENDOR_M_ST_M25PE80:
14869 	case FLASH_5720VENDOR_M_ST_M45PE80:
14870 	case FLASH_5720VENDOR_A_ST_M25PE80:
14871 	case FLASH_5720VENDOR_A_ST_M45PE80:
14872 	case FLASH_5720VENDOR_ST_25USPT:
14873 	case FLASH_5720VENDOR_ST_45USPT:
14874 		tp->nvram_jedecnum = JEDEC_ST;
14875 		tg3_flag_set(tp, NVRAM_BUFFERED);
14876 		tg3_flag_set(tp, FLASH);
14877 
14878 		switch (nvmpinstrp) {
14879 		case FLASH_5720VENDOR_M_ST_M25PE20:
14880 		case FLASH_5720VENDOR_M_ST_M45PE20:
14881 		case FLASH_5720VENDOR_A_ST_M25PE20:
14882 		case FLASH_5720VENDOR_A_ST_M45PE20:
14883 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14884 			break;
14885 		case FLASH_5720VENDOR_M_ST_M25PE40:
14886 		case FLASH_5720VENDOR_M_ST_M45PE40:
14887 		case FLASH_5720VENDOR_A_ST_M25PE40:
14888 		case FLASH_5720VENDOR_A_ST_M45PE40:
14889 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14890 			break;
14891 		case FLASH_5720VENDOR_M_ST_M25PE80:
14892 		case FLASH_5720VENDOR_M_ST_M45PE80:
14893 		case FLASH_5720VENDOR_A_ST_M25PE80:
14894 		case FLASH_5720VENDOR_A_ST_M45PE80:
14895 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14896 			break;
14897 		default:
14898 			if (tg3_asic_rev(tp) != ASIC_REV_5762)
14899 				tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14900 			break;
14901 		}
14902 		break;
14903 	default:
14904 		tg3_flag_set(tp, NO_NVRAM);
14905 		return;
14906 	}
14907 
14908 	tg3_nvram_get_pagesize(tp, nvcfg1);
14909 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14910 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14911 
14912 	if (tg3_asic_rev(tp) == ASIC_REV_5762) {
14913 		u32 val;
14914 
14915 		if (tg3_nvram_read(tp, 0, &val))
14916 			return;
14917 
14918 		if (val != TG3_EEPROM_MAGIC &&
14919 		    (val & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW)
14920 			tg3_flag_set(tp, NO_NVRAM);
14921 	}
14922 }
14923 
14924 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
14925 static void tg3_nvram_init(struct tg3 *tp)
14926 {
14927 	if (tg3_flag(tp, IS_SSB_CORE)) {
14928 		/* No NVRAM and EEPROM on the SSB Broadcom GigE core. */
14929 		tg3_flag_clear(tp, NVRAM);
14930 		tg3_flag_clear(tp, NVRAM_BUFFERED);
14931 		tg3_flag_set(tp, NO_NVRAM);
14932 		return;
14933 	}
14934 
14935 	tw32_f(GRC_EEPROM_ADDR,
14936 	     (EEPROM_ADDR_FSM_RESET |
14937 	      (EEPROM_DEFAULT_CLOCK_PERIOD <<
14938 	       EEPROM_ADDR_CLKPERD_SHIFT)));
14939 
14940 	msleep(1);
14941 
14942 	/* Enable seeprom accesses. */
14943 	tw32_f(GRC_LOCAL_CTRL,
14944 	     tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
14945 	udelay(100);
14946 
14947 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
14948 	    tg3_asic_rev(tp) != ASIC_REV_5701) {
14949 		tg3_flag_set(tp, NVRAM);
14950 
14951 		if (tg3_nvram_lock(tp)) {
14952 			netdev_warn(tp->dev,
14953 				    "Cannot get nvram lock, %s failed\n",
14954 				    __func__);
14955 			return;
14956 		}
14957 		tg3_enable_nvram_access(tp);
14958 
14959 		tp->nvram_size = 0;
14960 
14961 		if (tg3_asic_rev(tp) == ASIC_REV_5752)
14962 			tg3_get_5752_nvram_info(tp);
14963 		else if (tg3_asic_rev(tp) == ASIC_REV_5755)
14964 			tg3_get_5755_nvram_info(tp);
14965 		else if (tg3_asic_rev(tp) == ASIC_REV_5787 ||
14966 			 tg3_asic_rev(tp) == ASIC_REV_5784 ||
14967 			 tg3_asic_rev(tp) == ASIC_REV_5785)
14968 			tg3_get_5787_nvram_info(tp);
14969 		else if (tg3_asic_rev(tp) == ASIC_REV_5761)
14970 			tg3_get_5761_nvram_info(tp);
14971 		else if (tg3_asic_rev(tp) == ASIC_REV_5906)
14972 			tg3_get_5906_nvram_info(tp);
14973 		else if (tg3_asic_rev(tp) == ASIC_REV_57780 ||
14974 			 tg3_flag(tp, 57765_CLASS))
14975 			tg3_get_57780_nvram_info(tp);
14976 		else if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
14977 			 tg3_asic_rev(tp) == ASIC_REV_5719)
14978 			tg3_get_5717_nvram_info(tp);
14979 		else if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
14980 			 tg3_asic_rev(tp) == ASIC_REV_5762)
14981 			tg3_get_5720_nvram_info(tp);
14982 		else
14983 			tg3_get_nvram_info(tp);
14984 
14985 		if (tp->nvram_size == 0)
14986 			tg3_get_nvram_size(tp);
14987 
14988 		tg3_disable_nvram_access(tp);
14989 		tg3_nvram_unlock(tp);
14990 
14991 	} else {
14992 		tg3_flag_clear(tp, NVRAM);
14993 		tg3_flag_clear(tp, NVRAM_BUFFERED);
14994 
14995 		tg3_get_eeprom_size(tp);
14996 	}
14997 }
14998 
14999 struct subsys_tbl_ent {
15000 	u16 subsys_vendor, subsys_devid;
15001 	u32 phy_id;
15002 };
15003 
15004 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
15005 	/* Broadcom boards. */
15006 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15007 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700A6, TG3_PHY_ID_BCM5401 },
15008 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15009 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A5, TG3_PHY_ID_BCM5701 },
15010 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15011 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700T6, TG3_PHY_ID_BCM8002 },
15012 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15013 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700A9, 0 },
15014 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15015 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701T1, TG3_PHY_ID_BCM5701 },
15016 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15017 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701T8, TG3_PHY_ID_BCM5701 },
15018 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15019 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A7, 0 },
15020 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15021 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A10, TG3_PHY_ID_BCM5701 },
15022 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15023 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A12, TG3_PHY_ID_BCM5701 },
15024 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15025 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95703AX1, TG3_PHY_ID_BCM5703 },
15026 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15027 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95703AX2, TG3_PHY_ID_BCM5703 },
15028 
15029 	/* 3com boards. */
15030 	{ TG3PCI_SUBVENDOR_ID_3COM,
15031 	  TG3PCI_SUBDEVICE_ID_3COM_3C996T, TG3_PHY_ID_BCM5401 },
15032 	{ TG3PCI_SUBVENDOR_ID_3COM,
15033 	  TG3PCI_SUBDEVICE_ID_3COM_3C996BT, TG3_PHY_ID_BCM5701 },
15034 	{ TG3PCI_SUBVENDOR_ID_3COM,
15035 	  TG3PCI_SUBDEVICE_ID_3COM_3C996SX, 0 },
15036 	{ TG3PCI_SUBVENDOR_ID_3COM,
15037 	  TG3PCI_SUBDEVICE_ID_3COM_3C1000T, TG3_PHY_ID_BCM5701 },
15038 	{ TG3PCI_SUBVENDOR_ID_3COM,
15039 	  TG3PCI_SUBDEVICE_ID_3COM_3C940BR01, TG3_PHY_ID_BCM5701 },
15040 
15041 	/* DELL boards. */
15042 	{ TG3PCI_SUBVENDOR_ID_DELL,
15043 	  TG3PCI_SUBDEVICE_ID_DELL_VIPER, TG3_PHY_ID_BCM5401 },
15044 	{ TG3PCI_SUBVENDOR_ID_DELL,
15045 	  TG3PCI_SUBDEVICE_ID_DELL_JAGUAR, TG3_PHY_ID_BCM5401 },
15046 	{ TG3PCI_SUBVENDOR_ID_DELL,
15047 	  TG3PCI_SUBDEVICE_ID_DELL_MERLOT, TG3_PHY_ID_BCM5411 },
15048 	{ TG3PCI_SUBVENDOR_ID_DELL,
15049 	  TG3PCI_SUBDEVICE_ID_DELL_SLIM_MERLOT, TG3_PHY_ID_BCM5411 },
15050 
15051 	/* Compaq boards. */
15052 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15053 	  TG3PCI_SUBDEVICE_ID_COMPAQ_BANSHEE, TG3_PHY_ID_BCM5701 },
15054 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15055 	  TG3PCI_SUBDEVICE_ID_COMPAQ_BANSHEE_2, TG3_PHY_ID_BCM5701 },
15056 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15057 	  TG3PCI_SUBDEVICE_ID_COMPAQ_CHANGELING, 0 },
15058 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15059 	  TG3PCI_SUBDEVICE_ID_COMPAQ_NC7780, TG3_PHY_ID_BCM5701 },
15060 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15061 	  TG3PCI_SUBDEVICE_ID_COMPAQ_NC7780_2, TG3_PHY_ID_BCM5701 },
15062 
15063 	/* IBM boards. */
15064 	{ TG3PCI_SUBVENDOR_ID_IBM,
15065 	  TG3PCI_SUBDEVICE_ID_IBM_5703SAX2, 0 }
15066 };
15067 
15068 static struct subsys_tbl_ent *tg3_lookup_by_subsys(struct tg3 *tp)
15069 {
15070 	int i;
15071 
15072 	for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
15073 		if ((subsys_id_to_phy_id[i].subsys_vendor ==
15074 		     tp->pdev->subsystem_vendor) &&
15075 		    (subsys_id_to_phy_id[i].subsys_devid ==
15076 		     tp->pdev->subsystem_device))
15077 			return &subsys_id_to_phy_id[i];
15078 	}
15079 	return NULL;
15080 }
15081 
15082 static void tg3_get_eeprom_hw_cfg(struct tg3 *tp)
15083 {
15084 	u32 val;
15085 
15086 	tp->phy_id = TG3_PHY_ID_INVALID;
15087 	tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15088 
15089 	/* Assume an onboard device and WOL capable by default.  */
15090 	tg3_flag_set(tp, EEPROM_WRITE_PROT);
15091 	tg3_flag_set(tp, WOL_CAP);
15092 
15093 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
15094 		if (!(tr32(PCIE_TRANSACTION_CFG) & PCIE_TRANS_CFG_LOM)) {
15095 			tg3_flag_clear(tp, EEPROM_WRITE_PROT);
15096 			tg3_flag_set(tp, IS_NIC);
15097 		}
15098 		val = tr32(VCPU_CFGSHDW);
15099 		if (val & VCPU_CFGSHDW_ASPM_DBNC)
15100 			tg3_flag_set(tp, ASPM_WORKAROUND);
15101 		if ((val & VCPU_CFGSHDW_WOL_ENABLE) &&
15102 		    (val & VCPU_CFGSHDW_WOL_MAGPKT)) {
15103 			tg3_flag_set(tp, WOL_ENABLE);
15104 			device_set_wakeup_enable(&tp->pdev->dev, true);
15105 		}
15106 		goto done;
15107 	}
15108 
15109 	tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
15110 	if (val == NIC_SRAM_DATA_SIG_MAGIC) {
15111 		u32 nic_cfg, led_cfg;
15112 		u32 cfg2 = 0, cfg4 = 0, cfg5 = 0;
15113 		u32 nic_phy_id, ver, eeprom_phy_id;
15114 		int eeprom_phy_serdes = 0;
15115 
15116 		tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
15117 		tp->nic_sram_data_cfg = nic_cfg;
15118 
15119 		tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
15120 		ver >>= NIC_SRAM_DATA_VER_SHIFT;
15121 		if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
15122 		    tg3_asic_rev(tp) != ASIC_REV_5701 &&
15123 		    tg3_asic_rev(tp) != ASIC_REV_5703 &&
15124 		    (ver > 0) && (ver < 0x100))
15125 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
15126 
15127 		if (tg3_asic_rev(tp) == ASIC_REV_5785)
15128 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_4, &cfg4);
15129 
15130 		if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
15131 		    tg3_asic_rev(tp) == ASIC_REV_5719 ||
15132 		    tg3_asic_rev(tp) == ASIC_REV_5720)
15133 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_5, &cfg5);
15134 
15135 		if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
15136 		    NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
15137 			eeprom_phy_serdes = 1;
15138 
15139 		tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
15140 		if (nic_phy_id != 0) {
15141 			u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
15142 			u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
15143 
15144 			eeprom_phy_id  = (id1 >> 16) << 10;
15145 			eeprom_phy_id |= (id2 & 0xfc00) << 16;
15146 			eeprom_phy_id |= (id2 & 0x03ff) <<  0;
15147 		} else
15148 			eeprom_phy_id = 0;
15149 
15150 		tp->phy_id = eeprom_phy_id;
15151 		if (eeprom_phy_serdes) {
15152 			if (!tg3_flag(tp, 5705_PLUS))
15153 				tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15154 			else
15155 				tp->phy_flags |= TG3_PHYFLG_MII_SERDES;
15156 		}
15157 
15158 		if (tg3_flag(tp, 5750_PLUS))
15159 			led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
15160 				    SHASTA_EXT_LED_MODE_MASK);
15161 		else
15162 			led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
15163 
15164 		switch (led_cfg) {
15165 		default:
15166 		case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
15167 			tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15168 			break;
15169 
15170 		case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
15171 			tp->led_ctrl = LED_CTRL_MODE_PHY_2;
15172 			break;
15173 
15174 		case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
15175 			tp->led_ctrl = LED_CTRL_MODE_MAC;
15176 
15177 			/* Default to PHY_1_MODE if 0 (MAC_MODE) is
15178 			 * read on some older 5700/5701 bootcode.
15179 			 */
15180 			if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
15181 			    tg3_asic_rev(tp) == ASIC_REV_5701)
15182 				tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15183 
15184 			break;
15185 
15186 		case SHASTA_EXT_LED_SHARED:
15187 			tp->led_ctrl = LED_CTRL_MODE_SHARED;
15188 			if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0 &&
15189 			    tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A1)
15190 				tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
15191 						 LED_CTRL_MODE_PHY_2);
15192 
15193 			if (tg3_flag(tp, 5717_PLUS) ||
15194 			    tg3_asic_rev(tp) == ASIC_REV_5762)
15195 				tp->led_ctrl |= LED_CTRL_BLINK_RATE_OVERRIDE |
15196 						LED_CTRL_BLINK_RATE_MASK;
15197 
15198 			break;
15199 
15200 		case SHASTA_EXT_LED_MAC:
15201 			tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
15202 			break;
15203 
15204 		case SHASTA_EXT_LED_COMBO:
15205 			tp->led_ctrl = LED_CTRL_MODE_COMBO;
15206 			if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0)
15207 				tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
15208 						 LED_CTRL_MODE_PHY_2);
15209 			break;
15210 
15211 		}
15212 
15213 		if ((tg3_asic_rev(tp) == ASIC_REV_5700 ||
15214 		     tg3_asic_rev(tp) == ASIC_REV_5701) &&
15215 		    tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
15216 			tp->led_ctrl = LED_CTRL_MODE_PHY_2;
15217 
15218 		if (tg3_chip_rev(tp) == CHIPREV_5784_AX)
15219 			tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15220 
15221 		if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP) {
15222 			tg3_flag_set(tp, EEPROM_WRITE_PROT);
15223 			if ((tp->pdev->subsystem_vendor ==
15224 			     PCI_VENDOR_ID_ARIMA) &&
15225 			    (tp->pdev->subsystem_device == 0x205a ||
15226 			     tp->pdev->subsystem_device == 0x2063))
15227 				tg3_flag_clear(tp, EEPROM_WRITE_PROT);
15228 		} else {
15229 			tg3_flag_clear(tp, EEPROM_WRITE_PROT);
15230 			tg3_flag_set(tp, IS_NIC);
15231 		}
15232 
15233 		if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
15234 			tg3_flag_set(tp, ENABLE_ASF);
15235 			if (tg3_flag(tp, 5750_PLUS))
15236 				tg3_flag_set(tp, ASF_NEW_HANDSHAKE);
15237 		}
15238 
15239 		if ((nic_cfg & NIC_SRAM_DATA_CFG_APE_ENABLE) &&
15240 		    tg3_flag(tp, 5750_PLUS))
15241 			tg3_flag_set(tp, ENABLE_APE);
15242 
15243 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES &&
15244 		    !(nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL))
15245 			tg3_flag_clear(tp, WOL_CAP);
15246 
15247 		if (tg3_flag(tp, WOL_CAP) &&
15248 		    (nic_cfg & NIC_SRAM_DATA_CFG_WOL_ENABLE)) {
15249 			tg3_flag_set(tp, WOL_ENABLE);
15250 			device_set_wakeup_enable(&tp->pdev->dev, true);
15251 		}
15252 
15253 		if (cfg2 & (1 << 17))
15254 			tp->phy_flags |= TG3_PHYFLG_CAPACITIVE_COUPLING;
15255 
15256 		/* serdes signal pre-emphasis in register 0x590 set by */
15257 		/* bootcode if bit 18 is set */
15258 		if (cfg2 & (1 << 18))
15259 			tp->phy_flags |= TG3_PHYFLG_SERDES_PREEMPHASIS;
15260 
15261 		if ((tg3_flag(tp, 57765_PLUS) ||
15262 		     (tg3_asic_rev(tp) == ASIC_REV_5784 &&
15263 		      tg3_chip_rev(tp) != CHIPREV_5784_AX)) &&
15264 		    (cfg2 & NIC_SRAM_DATA_CFG_2_APD_EN))
15265 			tp->phy_flags |= TG3_PHYFLG_ENABLE_APD;
15266 
15267 		if (tg3_flag(tp, PCI_EXPRESS)) {
15268 			u32 cfg3;
15269 
15270 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &cfg3);
15271 			if (tg3_asic_rev(tp) != ASIC_REV_5785 &&
15272 			    !tg3_flag(tp, 57765_PLUS) &&
15273 			    (cfg3 & NIC_SRAM_ASPM_DEBOUNCE))
15274 				tg3_flag_set(tp, ASPM_WORKAROUND);
15275 			if (cfg3 & NIC_SRAM_LNK_FLAP_AVOID)
15276 				tp->phy_flags |= TG3_PHYFLG_KEEP_LINK_ON_PWRDN;
15277 			if (cfg3 & NIC_SRAM_1G_ON_VAUX_OK)
15278 				tp->phy_flags |= TG3_PHYFLG_1G_ON_VAUX_OK;
15279 		}
15280 
15281 		if (cfg4 & NIC_SRAM_RGMII_INBAND_DISABLE)
15282 			tg3_flag_set(tp, RGMII_INBAND_DISABLE);
15283 		if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_RX_EN)
15284 			tg3_flag_set(tp, RGMII_EXT_IBND_RX_EN);
15285 		if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_TX_EN)
15286 			tg3_flag_set(tp, RGMII_EXT_IBND_TX_EN);
15287 
15288 		if (cfg5 & NIC_SRAM_DISABLE_1G_HALF_ADV)
15289 			tp->phy_flags |= TG3_PHYFLG_DISABLE_1G_HD_ADV;
15290 	}
15291 done:
15292 	if (tg3_flag(tp, WOL_CAP))
15293 		device_set_wakeup_enable(&tp->pdev->dev,
15294 					 tg3_flag(tp, WOL_ENABLE));
15295 	else
15296 		device_set_wakeup_capable(&tp->pdev->dev, false);
15297 }
15298 
15299 static int tg3_ape_otp_read(struct tg3 *tp, u32 offset, u32 *val)
15300 {
15301 	int i, err;
15302 	u32 val2, off = offset * 8;
15303 
15304 	err = tg3_nvram_lock(tp);
15305 	if (err)
15306 		return err;
15307 
15308 	tg3_ape_write32(tp, TG3_APE_OTP_ADDR, off | APE_OTP_ADDR_CPU_ENABLE);
15309 	tg3_ape_write32(tp, TG3_APE_OTP_CTRL, APE_OTP_CTRL_PROG_EN |
15310 			APE_OTP_CTRL_CMD_RD | APE_OTP_CTRL_START);
15311 	tg3_ape_read32(tp, TG3_APE_OTP_CTRL);
15312 	udelay(10);
15313 
15314 	for (i = 0; i < 100; i++) {
15315 		val2 = tg3_ape_read32(tp, TG3_APE_OTP_STATUS);
15316 		if (val2 & APE_OTP_STATUS_CMD_DONE) {
15317 			*val = tg3_ape_read32(tp, TG3_APE_OTP_RD_DATA);
15318 			break;
15319 		}
15320 		udelay(10);
15321 	}
15322 
15323 	tg3_ape_write32(tp, TG3_APE_OTP_CTRL, 0);
15324 
15325 	tg3_nvram_unlock(tp);
15326 	if (val2 & APE_OTP_STATUS_CMD_DONE)
15327 		return 0;
15328 
15329 	return -EBUSY;
15330 }
15331 
15332 static int tg3_issue_otp_command(struct tg3 *tp, u32 cmd)
15333 {
15334 	int i;
15335 	u32 val;
15336 
15337 	tw32(OTP_CTRL, cmd | OTP_CTRL_OTP_CMD_START);
15338 	tw32(OTP_CTRL, cmd);
15339 
15340 	/* Wait for up to 1 ms for command to execute. */
15341 	for (i = 0; i < 100; i++) {
15342 		val = tr32(OTP_STATUS);
15343 		if (val & OTP_STATUS_CMD_DONE)
15344 			break;
15345 		udelay(10);
15346 	}
15347 
15348 	return (val & OTP_STATUS_CMD_DONE) ? 0 : -EBUSY;
15349 }
15350 
15351 /* Read the gphy configuration from the OTP region of the chip.  The gphy
15352  * configuration is a 32-bit value that straddles the alignment boundary.
15353  * We do two 32-bit reads and then shift and merge the results.
15354  */
15355 static u32 tg3_read_otp_phycfg(struct tg3 *tp)
15356 {
15357 	u32 bhalf_otp, thalf_otp;
15358 
15359 	tw32(OTP_MODE, OTP_MODE_OTP_THRU_GRC);
15360 
15361 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_INIT))
15362 		return 0;
15363 
15364 	tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC1);
15365 
15366 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
15367 		return 0;
15368 
15369 	thalf_otp = tr32(OTP_READ_DATA);
15370 
15371 	tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC2);
15372 
15373 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
15374 		return 0;
15375 
15376 	bhalf_otp = tr32(OTP_READ_DATA);
15377 
15378 	return ((thalf_otp & 0x0000ffff) << 16) | (bhalf_otp >> 16);
15379 }
15380 
15381 static void tg3_phy_init_link_config(struct tg3 *tp)
15382 {
15383 	u32 adv = ADVERTISED_Autoneg;
15384 
15385 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
15386 		if (!(tp->phy_flags & TG3_PHYFLG_DISABLE_1G_HD_ADV))
15387 			adv |= ADVERTISED_1000baseT_Half;
15388 		adv |= ADVERTISED_1000baseT_Full;
15389 	}
15390 
15391 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
15392 		adv |= ADVERTISED_100baseT_Half |
15393 		       ADVERTISED_100baseT_Full |
15394 		       ADVERTISED_10baseT_Half |
15395 		       ADVERTISED_10baseT_Full |
15396 		       ADVERTISED_TP;
15397 	else
15398 		adv |= ADVERTISED_FIBRE;
15399 
15400 	tp->link_config.advertising = adv;
15401 	tp->link_config.speed = SPEED_UNKNOWN;
15402 	tp->link_config.duplex = DUPLEX_UNKNOWN;
15403 	tp->link_config.autoneg = AUTONEG_ENABLE;
15404 	tp->link_config.active_speed = SPEED_UNKNOWN;
15405 	tp->link_config.active_duplex = DUPLEX_UNKNOWN;
15406 
15407 	tp->old_link = -1;
15408 }
15409 
15410 static int tg3_phy_probe(struct tg3 *tp)
15411 {
15412 	u32 hw_phy_id_1, hw_phy_id_2;
15413 	u32 hw_phy_id, hw_phy_id_masked;
15414 	int err;
15415 
15416 	/* flow control autonegotiation is default behavior */
15417 	tg3_flag_set(tp, PAUSE_AUTONEG);
15418 	tp->link_config.flowctrl = FLOW_CTRL_TX | FLOW_CTRL_RX;
15419 
15420 	if (tg3_flag(tp, ENABLE_APE)) {
15421 		switch (tp->pci_fn) {
15422 		case 0:
15423 			tp->phy_ape_lock = TG3_APE_LOCK_PHY0;
15424 			break;
15425 		case 1:
15426 			tp->phy_ape_lock = TG3_APE_LOCK_PHY1;
15427 			break;
15428 		case 2:
15429 			tp->phy_ape_lock = TG3_APE_LOCK_PHY2;
15430 			break;
15431 		case 3:
15432 			tp->phy_ape_lock = TG3_APE_LOCK_PHY3;
15433 			break;
15434 		}
15435 	}
15436 
15437 	if (!tg3_flag(tp, ENABLE_ASF) &&
15438 	    !(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15439 	    !(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
15440 		tp->phy_flags &= ~(TG3_PHYFLG_1G_ON_VAUX_OK |
15441 				   TG3_PHYFLG_KEEP_LINK_ON_PWRDN);
15442 
15443 	if (tg3_flag(tp, USE_PHYLIB))
15444 		return tg3_phy_init(tp);
15445 
15446 	/* Reading the PHY ID register can conflict with ASF
15447 	 * firmware access to the PHY hardware.
15448 	 */
15449 	err = 0;
15450 	if (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)) {
15451 		hw_phy_id = hw_phy_id_masked = TG3_PHY_ID_INVALID;
15452 	} else {
15453 		/* Now read the physical PHY_ID from the chip and verify
15454 		 * that it is sane.  If it doesn't look good, we fall back
15455 		 * to either the hard-coded table based PHY_ID and failing
15456 		 * that the value found in the eeprom area.
15457 		 */
15458 		err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
15459 		err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
15460 
15461 		hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
15462 		hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
15463 		hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
15464 
15465 		hw_phy_id_masked = hw_phy_id & TG3_PHY_ID_MASK;
15466 	}
15467 
15468 	if (!err && TG3_KNOWN_PHY_ID(hw_phy_id_masked)) {
15469 		tp->phy_id = hw_phy_id;
15470 		if (hw_phy_id_masked == TG3_PHY_ID_BCM8002)
15471 			tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15472 		else
15473 			tp->phy_flags &= ~TG3_PHYFLG_PHY_SERDES;
15474 	} else {
15475 		if (tp->phy_id != TG3_PHY_ID_INVALID) {
15476 			/* Do nothing, phy ID already set up in
15477 			 * tg3_get_eeprom_hw_cfg().
15478 			 */
15479 		} else {
15480 			struct subsys_tbl_ent *p;
15481 
15482 			/* No eeprom signature?  Try the hardcoded
15483 			 * subsys device table.
15484 			 */
15485 			p = tg3_lookup_by_subsys(tp);
15486 			if (p) {
15487 				tp->phy_id = p->phy_id;
15488 			} else if (!tg3_flag(tp, IS_SSB_CORE)) {
15489 				/* For now we saw the IDs 0xbc050cd0,
15490 				 * 0xbc050f80 and 0xbc050c30 on devices
15491 				 * connected to an BCM4785 and there are
15492 				 * probably more. Just assume that the phy is
15493 				 * supported when it is connected to a SSB core
15494 				 * for now.
15495 				 */
15496 				return -ENODEV;
15497 			}
15498 
15499 			if (!tp->phy_id ||
15500 			    tp->phy_id == TG3_PHY_ID_BCM8002)
15501 				tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15502 		}
15503 	}
15504 
15505 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15506 	    (tg3_asic_rev(tp) == ASIC_REV_5719 ||
15507 	     tg3_asic_rev(tp) == ASIC_REV_5720 ||
15508 	     tg3_asic_rev(tp) == ASIC_REV_57766 ||
15509 	     tg3_asic_rev(tp) == ASIC_REV_5762 ||
15510 	     (tg3_asic_rev(tp) == ASIC_REV_5717 &&
15511 	      tg3_chip_rev_id(tp) != CHIPREV_ID_5717_A0) ||
15512 	     (tg3_asic_rev(tp) == ASIC_REV_57765 &&
15513 	      tg3_chip_rev_id(tp) != CHIPREV_ID_57765_A0))) {
15514 		tp->phy_flags |= TG3_PHYFLG_EEE_CAP;
15515 
15516 		tp->eee.supported = SUPPORTED_100baseT_Full |
15517 				    SUPPORTED_1000baseT_Full;
15518 		tp->eee.advertised = ADVERTISED_100baseT_Full |
15519 				     ADVERTISED_1000baseT_Full;
15520 		tp->eee.eee_enabled = 1;
15521 		tp->eee.tx_lpi_enabled = 1;
15522 		tp->eee.tx_lpi_timer = TG3_CPMU_DBTMR1_LNKIDLE_2047US;
15523 	}
15524 
15525 	tg3_phy_init_link_config(tp);
15526 
15527 	if (!(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
15528 	    !(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15529 	    !tg3_flag(tp, ENABLE_APE) &&
15530 	    !tg3_flag(tp, ENABLE_ASF)) {
15531 		u32 bmsr, dummy;
15532 
15533 		tg3_readphy(tp, MII_BMSR, &bmsr);
15534 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
15535 		    (bmsr & BMSR_LSTATUS))
15536 			goto skip_phy_reset;
15537 
15538 		err = tg3_phy_reset(tp);
15539 		if (err)
15540 			return err;
15541 
15542 		tg3_phy_set_wirespeed(tp);
15543 
15544 		if (!tg3_phy_copper_an_config_ok(tp, &dummy)) {
15545 			tg3_phy_autoneg_cfg(tp, tp->link_config.advertising,
15546 					    tp->link_config.flowctrl);
15547 
15548 			tg3_writephy(tp, MII_BMCR,
15549 				     BMCR_ANENABLE | BMCR_ANRESTART);
15550 		}
15551 	}
15552 
15553 skip_phy_reset:
15554 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
15555 		err = tg3_init_5401phy_dsp(tp);
15556 		if (err)
15557 			return err;
15558 
15559 		err = tg3_init_5401phy_dsp(tp);
15560 	}
15561 
15562 	return err;
15563 }
15564 
15565 static void tg3_read_vpd(struct tg3 *tp)
15566 {
15567 	u8 *vpd_data;
15568 	unsigned int block_end, rosize, len;
15569 	u32 vpdlen;
15570 	int j, i = 0;
15571 
15572 	vpd_data = (u8 *)tg3_vpd_readblock(tp, &vpdlen);
15573 	if (!vpd_data)
15574 		goto out_no_vpd;
15575 
15576 	i = pci_vpd_find_tag(vpd_data, 0, vpdlen, PCI_VPD_LRDT_RO_DATA);
15577 	if (i < 0)
15578 		goto out_not_found;
15579 
15580 	rosize = pci_vpd_lrdt_size(&vpd_data[i]);
15581 	block_end = i + PCI_VPD_LRDT_TAG_SIZE + rosize;
15582 	i += PCI_VPD_LRDT_TAG_SIZE;
15583 
15584 	if (block_end > vpdlen)
15585 		goto out_not_found;
15586 
15587 	j = pci_vpd_find_info_keyword(vpd_data, i, rosize,
15588 				      PCI_VPD_RO_KEYWORD_MFR_ID);
15589 	if (j > 0) {
15590 		len = pci_vpd_info_field_size(&vpd_data[j]);
15591 
15592 		j += PCI_VPD_INFO_FLD_HDR_SIZE;
15593 		if (j + len > block_end || len != 4 ||
15594 		    memcmp(&vpd_data[j], "1028", 4))
15595 			goto partno;
15596 
15597 		j = pci_vpd_find_info_keyword(vpd_data, i, rosize,
15598 					      PCI_VPD_RO_KEYWORD_VENDOR0);
15599 		if (j < 0)
15600 			goto partno;
15601 
15602 		len = pci_vpd_info_field_size(&vpd_data[j]);
15603 
15604 		j += PCI_VPD_INFO_FLD_HDR_SIZE;
15605 		if (j + len > block_end)
15606 			goto partno;
15607 
15608 		if (len >= sizeof(tp->fw_ver))
15609 			len = sizeof(tp->fw_ver) - 1;
15610 		memset(tp->fw_ver, 0, sizeof(tp->fw_ver));
15611 		snprintf(tp->fw_ver, sizeof(tp->fw_ver), "%.*s bc ", len,
15612 			 &vpd_data[j]);
15613 	}
15614 
15615 partno:
15616 	i = pci_vpd_find_info_keyword(vpd_data, i, rosize,
15617 				      PCI_VPD_RO_KEYWORD_PARTNO);
15618 	if (i < 0)
15619 		goto out_not_found;
15620 
15621 	len = pci_vpd_info_field_size(&vpd_data[i]);
15622 
15623 	i += PCI_VPD_INFO_FLD_HDR_SIZE;
15624 	if (len > TG3_BPN_SIZE ||
15625 	    (len + i) > vpdlen)
15626 		goto out_not_found;
15627 
15628 	memcpy(tp->board_part_number, &vpd_data[i], len);
15629 
15630 out_not_found:
15631 	kfree(vpd_data);
15632 	if (tp->board_part_number[0])
15633 		return;
15634 
15635 out_no_vpd:
15636 	if (tg3_asic_rev(tp) == ASIC_REV_5717) {
15637 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
15638 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C)
15639 			strcpy(tp->board_part_number, "BCM5717");
15640 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718)
15641 			strcpy(tp->board_part_number, "BCM5718");
15642 		else
15643 			goto nomatch;
15644 	} else if (tg3_asic_rev(tp) == ASIC_REV_57780) {
15645 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57780)
15646 			strcpy(tp->board_part_number, "BCM57780");
15647 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57760)
15648 			strcpy(tp->board_part_number, "BCM57760");
15649 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57790)
15650 			strcpy(tp->board_part_number, "BCM57790");
15651 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57788)
15652 			strcpy(tp->board_part_number, "BCM57788");
15653 		else
15654 			goto nomatch;
15655 	} else if (tg3_asic_rev(tp) == ASIC_REV_57765) {
15656 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761)
15657 			strcpy(tp->board_part_number, "BCM57761");
15658 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765)
15659 			strcpy(tp->board_part_number, "BCM57765");
15660 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57781)
15661 			strcpy(tp->board_part_number, "BCM57781");
15662 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57785)
15663 			strcpy(tp->board_part_number, "BCM57785");
15664 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791)
15665 			strcpy(tp->board_part_number, "BCM57791");
15666 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795)
15667 			strcpy(tp->board_part_number, "BCM57795");
15668 		else
15669 			goto nomatch;
15670 	} else if (tg3_asic_rev(tp) == ASIC_REV_57766) {
15671 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57762)
15672 			strcpy(tp->board_part_number, "BCM57762");
15673 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57766)
15674 			strcpy(tp->board_part_number, "BCM57766");
15675 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57782)
15676 			strcpy(tp->board_part_number, "BCM57782");
15677 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57786)
15678 			strcpy(tp->board_part_number, "BCM57786");
15679 		else
15680 			goto nomatch;
15681 	} else if (tg3_asic_rev(tp) == ASIC_REV_5906) {
15682 		strcpy(tp->board_part_number, "BCM95906");
15683 	} else {
15684 nomatch:
15685 		strcpy(tp->board_part_number, "none");
15686 	}
15687 }
15688 
15689 static int tg3_fw_img_is_valid(struct tg3 *tp, u32 offset)
15690 {
15691 	u32 val;
15692 
15693 	if (tg3_nvram_read(tp, offset, &val) ||
15694 	    (val & 0xfc000000) != 0x0c000000 ||
15695 	    tg3_nvram_read(tp, offset + 4, &val) ||
15696 	    val != 0)
15697 		return 0;
15698 
15699 	return 1;
15700 }
15701 
15702 static void tg3_read_bc_ver(struct tg3 *tp)
15703 {
15704 	u32 val, offset, start, ver_offset;
15705 	int i, dst_off;
15706 	bool newver = false;
15707 
15708 	if (tg3_nvram_read(tp, 0xc, &offset) ||
15709 	    tg3_nvram_read(tp, 0x4, &start))
15710 		return;
15711 
15712 	offset = tg3_nvram_logical_addr(tp, offset);
15713 
15714 	if (tg3_nvram_read(tp, offset, &val))
15715 		return;
15716 
15717 	if ((val & 0xfc000000) == 0x0c000000) {
15718 		if (tg3_nvram_read(tp, offset + 4, &val))
15719 			return;
15720 
15721 		if (val == 0)
15722 			newver = true;
15723 	}
15724 
15725 	dst_off = strlen(tp->fw_ver);
15726 
15727 	if (newver) {
15728 		if (TG3_VER_SIZE - dst_off < 16 ||
15729 		    tg3_nvram_read(tp, offset + 8, &ver_offset))
15730 			return;
15731 
15732 		offset = offset + ver_offset - start;
15733 		for (i = 0; i < 16; i += 4) {
15734 			__be32 v;
15735 			if (tg3_nvram_read_be32(tp, offset + i, &v))
15736 				return;
15737 
15738 			memcpy(tp->fw_ver + dst_off + i, &v, sizeof(v));
15739 		}
15740 	} else {
15741 		u32 major, minor;
15742 
15743 		if (tg3_nvram_read(tp, TG3_NVM_PTREV_BCVER, &ver_offset))
15744 			return;
15745 
15746 		major = (ver_offset & TG3_NVM_BCVER_MAJMSK) >>
15747 			TG3_NVM_BCVER_MAJSFT;
15748 		minor = ver_offset & TG3_NVM_BCVER_MINMSK;
15749 		snprintf(&tp->fw_ver[dst_off], TG3_VER_SIZE - dst_off,
15750 			 "v%d.%02d", major, minor);
15751 	}
15752 }
15753 
15754 static void tg3_read_hwsb_ver(struct tg3 *tp)
15755 {
15756 	u32 val, major, minor;
15757 
15758 	/* Use native endian representation */
15759 	if (tg3_nvram_read(tp, TG3_NVM_HWSB_CFG1, &val))
15760 		return;
15761 
15762 	major = (val & TG3_NVM_HWSB_CFG1_MAJMSK) >>
15763 		TG3_NVM_HWSB_CFG1_MAJSFT;
15764 	minor = (val & TG3_NVM_HWSB_CFG1_MINMSK) >>
15765 		TG3_NVM_HWSB_CFG1_MINSFT;
15766 
15767 	snprintf(&tp->fw_ver[0], 32, "sb v%d.%02d", major, minor);
15768 }
15769 
15770 static void tg3_read_sb_ver(struct tg3 *tp, u32 val)
15771 {
15772 	u32 offset, major, minor, build;
15773 
15774 	strncat(tp->fw_ver, "sb", TG3_VER_SIZE - strlen(tp->fw_ver) - 1);
15775 
15776 	if ((val & TG3_EEPROM_SB_FORMAT_MASK) != TG3_EEPROM_SB_FORMAT_1)
15777 		return;
15778 
15779 	switch (val & TG3_EEPROM_SB_REVISION_MASK) {
15780 	case TG3_EEPROM_SB_REVISION_0:
15781 		offset = TG3_EEPROM_SB_F1R0_EDH_OFF;
15782 		break;
15783 	case TG3_EEPROM_SB_REVISION_2:
15784 		offset = TG3_EEPROM_SB_F1R2_EDH_OFF;
15785 		break;
15786 	case TG3_EEPROM_SB_REVISION_3:
15787 		offset = TG3_EEPROM_SB_F1R3_EDH_OFF;
15788 		break;
15789 	case TG3_EEPROM_SB_REVISION_4:
15790 		offset = TG3_EEPROM_SB_F1R4_EDH_OFF;
15791 		break;
15792 	case TG3_EEPROM_SB_REVISION_5:
15793 		offset = TG3_EEPROM_SB_F1R5_EDH_OFF;
15794 		break;
15795 	case TG3_EEPROM_SB_REVISION_6:
15796 		offset = TG3_EEPROM_SB_F1R6_EDH_OFF;
15797 		break;
15798 	default:
15799 		return;
15800 	}
15801 
15802 	if (tg3_nvram_read(tp, offset, &val))
15803 		return;
15804 
15805 	build = (val & TG3_EEPROM_SB_EDH_BLD_MASK) >>
15806 		TG3_EEPROM_SB_EDH_BLD_SHFT;
15807 	major = (val & TG3_EEPROM_SB_EDH_MAJ_MASK) >>
15808 		TG3_EEPROM_SB_EDH_MAJ_SHFT;
15809 	minor =  val & TG3_EEPROM_SB_EDH_MIN_MASK;
15810 
15811 	if (minor > 99 || build > 26)
15812 		return;
15813 
15814 	offset = strlen(tp->fw_ver);
15815 	snprintf(&tp->fw_ver[offset], TG3_VER_SIZE - offset,
15816 		 " v%d.%02d", major, minor);
15817 
15818 	if (build > 0) {
15819 		offset = strlen(tp->fw_ver);
15820 		if (offset < TG3_VER_SIZE - 1)
15821 			tp->fw_ver[offset] = 'a' + build - 1;
15822 	}
15823 }
15824 
15825 static void tg3_read_mgmtfw_ver(struct tg3 *tp)
15826 {
15827 	u32 val, offset, start;
15828 	int i, vlen;
15829 
15830 	for (offset = TG3_NVM_DIR_START;
15831 	     offset < TG3_NVM_DIR_END;
15832 	     offset += TG3_NVM_DIRENT_SIZE) {
15833 		if (tg3_nvram_read(tp, offset, &val))
15834 			return;
15835 
15836 		if ((val >> TG3_NVM_DIRTYPE_SHIFT) == TG3_NVM_DIRTYPE_ASFINI)
15837 			break;
15838 	}
15839 
15840 	if (offset == TG3_NVM_DIR_END)
15841 		return;
15842 
15843 	if (!tg3_flag(tp, 5705_PLUS))
15844 		start = 0x08000000;
15845 	else if (tg3_nvram_read(tp, offset - 4, &start))
15846 		return;
15847 
15848 	if (tg3_nvram_read(tp, offset + 4, &offset) ||
15849 	    !tg3_fw_img_is_valid(tp, offset) ||
15850 	    tg3_nvram_read(tp, offset + 8, &val))
15851 		return;
15852 
15853 	offset += val - start;
15854 
15855 	vlen = strlen(tp->fw_ver);
15856 
15857 	tp->fw_ver[vlen++] = ',';
15858 	tp->fw_ver[vlen++] = ' ';
15859 
15860 	for (i = 0; i < 4; i++) {
15861 		__be32 v;
15862 		if (tg3_nvram_read_be32(tp, offset, &v))
15863 			return;
15864 
15865 		offset += sizeof(v);
15866 
15867 		if (vlen > TG3_VER_SIZE - sizeof(v)) {
15868 			memcpy(&tp->fw_ver[vlen], &v, TG3_VER_SIZE - vlen);
15869 			break;
15870 		}
15871 
15872 		memcpy(&tp->fw_ver[vlen], &v, sizeof(v));
15873 		vlen += sizeof(v);
15874 	}
15875 }
15876 
15877 static void tg3_probe_ncsi(struct tg3 *tp)
15878 {
15879 	u32 apedata;
15880 
15881 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
15882 	if (apedata != APE_SEG_SIG_MAGIC)
15883 		return;
15884 
15885 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
15886 	if (!(apedata & APE_FW_STATUS_READY))
15887 		return;
15888 
15889 	if (tg3_ape_read32(tp, TG3_APE_FW_FEATURES) & TG3_APE_FW_FEATURE_NCSI)
15890 		tg3_flag_set(tp, APE_HAS_NCSI);
15891 }
15892 
15893 static void tg3_read_dash_ver(struct tg3 *tp)
15894 {
15895 	int vlen;
15896 	u32 apedata;
15897 	char *fwtype;
15898 
15899 	apedata = tg3_ape_read32(tp, TG3_APE_FW_VERSION);
15900 
15901 	if (tg3_flag(tp, APE_HAS_NCSI))
15902 		fwtype = "NCSI";
15903 	else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725)
15904 		fwtype = "SMASH";
15905 	else
15906 		fwtype = "DASH";
15907 
15908 	vlen = strlen(tp->fw_ver);
15909 
15910 	snprintf(&tp->fw_ver[vlen], TG3_VER_SIZE - vlen, " %s v%d.%d.%d.%d",
15911 		 fwtype,
15912 		 (apedata & APE_FW_VERSION_MAJMSK) >> APE_FW_VERSION_MAJSFT,
15913 		 (apedata & APE_FW_VERSION_MINMSK) >> APE_FW_VERSION_MINSFT,
15914 		 (apedata & APE_FW_VERSION_REVMSK) >> APE_FW_VERSION_REVSFT,
15915 		 (apedata & APE_FW_VERSION_BLDMSK));
15916 }
15917 
15918 static void tg3_read_otp_ver(struct tg3 *tp)
15919 {
15920 	u32 val, val2;
15921 
15922 	if (tg3_asic_rev(tp) != ASIC_REV_5762)
15923 		return;
15924 
15925 	if (!tg3_ape_otp_read(tp, OTP_ADDRESS_MAGIC0, &val) &&
15926 	    !tg3_ape_otp_read(tp, OTP_ADDRESS_MAGIC0 + 4, &val2) &&
15927 	    TG3_OTP_MAGIC0_VALID(val)) {
15928 		u64 val64 = (u64) val << 32 | val2;
15929 		u32 ver = 0;
15930 		int i, vlen;
15931 
15932 		for (i = 0; i < 7; i++) {
15933 			if ((val64 & 0xff) == 0)
15934 				break;
15935 			ver = val64 & 0xff;
15936 			val64 >>= 8;
15937 		}
15938 		vlen = strlen(tp->fw_ver);
15939 		snprintf(&tp->fw_ver[vlen], TG3_VER_SIZE - vlen, " .%02d", ver);
15940 	}
15941 }
15942 
15943 static void tg3_read_fw_ver(struct tg3 *tp)
15944 {
15945 	u32 val;
15946 	bool vpd_vers = false;
15947 
15948 	if (tp->fw_ver[0] != 0)
15949 		vpd_vers = true;
15950 
15951 	if (tg3_flag(tp, NO_NVRAM)) {
15952 		strcat(tp->fw_ver, "sb");
15953 		tg3_read_otp_ver(tp);
15954 		return;
15955 	}
15956 
15957 	if (tg3_nvram_read(tp, 0, &val))
15958 		return;
15959 
15960 	if (val == TG3_EEPROM_MAGIC)
15961 		tg3_read_bc_ver(tp);
15962 	else if ((val & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW)
15963 		tg3_read_sb_ver(tp, val);
15964 	else if ((val & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
15965 		tg3_read_hwsb_ver(tp);
15966 
15967 	if (tg3_flag(tp, ENABLE_ASF)) {
15968 		if (tg3_flag(tp, ENABLE_APE)) {
15969 			tg3_probe_ncsi(tp);
15970 			if (!vpd_vers)
15971 				tg3_read_dash_ver(tp);
15972 		} else if (!vpd_vers) {
15973 			tg3_read_mgmtfw_ver(tp);
15974 		}
15975 	}
15976 
15977 	tp->fw_ver[TG3_VER_SIZE - 1] = 0;
15978 }
15979 
15980 static inline u32 tg3_rx_ret_ring_size(struct tg3 *tp)
15981 {
15982 	if (tg3_flag(tp, LRG_PROD_RING_CAP))
15983 		return TG3_RX_RET_MAX_SIZE_5717;
15984 	else if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS))
15985 		return TG3_RX_RET_MAX_SIZE_5700;
15986 	else
15987 		return TG3_RX_RET_MAX_SIZE_5705;
15988 }
15989 
15990 static const struct pci_device_id tg3_write_reorder_chipsets[] = {
15991 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_FE_GATE_700C) },
15992 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8131_BRIDGE) },
15993 	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8385_0) },
15994 	{ },
15995 };
15996 
15997 static struct pci_dev *tg3_find_peer(struct tg3 *tp)
15998 {
15999 	struct pci_dev *peer;
16000 	unsigned int func, devnr = tp->pdev->devfn & ~7;
16001 
16002 	for (func = 0; func < 8; func++) {
16003 		peer = pci_get_slot(tp->pdev->bus, devnr | func);
16004 		if (peer && peer != tp->pdev)
16005 			break;
16006 		pci_dev_put(peer);
16007 	}
16008 	/* 5704 can be configured in single-port mode, set peer to
16009 	 * tp->pdev in that case.
16010 	 */
16011 	if (!peer) {
16012 		peer = tp->pdev;
16013 		return peer;
16014 	}
16015 
16016 	/*
16017 	 * We don't need to keep the refcount elevated; there's no way
16018 	 * to remove one half of this device without removing the other
16019 	 */
16020 	pci_dev_put(peer);
16021 
16022 	return peer;
16023 }
16024 
16025 static void tg3_detect_asic_rev(struct tg3 *tp, u32 misc_ctrl_reg)
16026 {
16027 	tp->pci_chip_rev_id = misc_ctrl_reg >> MISC_HOST_CTRL_CHIPREV_SHIFT;
16028 	if (tg3_asic_rev(tp) == ASIC_REV_USE_PROD_ID_REG) {
16029 		u32 reg;
16030 
16031 		/* All devices that use the alternate
16032 		 * ASIC REV location have a CPMU.
16033 		 */
16034 		tg3_flag_set(tp, CPMU_PRESENT);
16035 
16036 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
16037 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C ||
16038 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718 ||
16039 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5719 ||
16040 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5720 ||
16041 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57767 ||
16042 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57764 ||
16043 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5762 ||
16044 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725 ||
16045 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5727 ||
16046 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57787)
16047 			reg = TG3PCI_GEN2_PRODID_ASICREV;
16048 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57781 ||
16049 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57785 ||
16050 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761 ||
16051 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765 ||
16052 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791 ||
16053 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795 ||
16054 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57762 ||
16055 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57766 ||
16056 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57782 ||
16057 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57786)
16058 			reg = TG3PCI_GEN15_PRODID_ASICREV;
16059 		else
16060 			reg = TG3PCI_PRODID_ASICREV;
16061 
16062 		pci_read_config_dword(tp->pdev, reg, &tp->pci_chip_rev_id);
16063 	}
16064 
16065 	/* Wrong chip ID in 5752 A0. This code can be removed later
16066 	 * as A0 is not in production.
16067 	 */
16068 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5752_A0_HW)
16069 		tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
16070 
16071 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5717_C0)
16072 		tp->pci_chip_rev_id = CHIPREV_ID_5720_A0;
16073 
16074 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16075 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
16076 	    tg3_asic_rev(tp) == ASIC_REV_5720)
16077 		tg3_flag_set(tp, 5717_PLUS);
16078 
16079 	if (tg3_asic_rev(tp) == ASIC_REV_57765 ||
16080 	    tg3_asic_rev(tp) == ASIC_REV_57766)
16081 		tg3_flag_set(tp, 57765_CLASS);
16082 
16083 	if (tg3_flag(tp, 57765_CLASS) || tg3_flag(tp, 5717_PLUS) ||
16084 	     tg3_asic_rev(tp) == ASIC_REV_5762)
16085 		tg3_flag_set(tp, 57765_PLUS);
16086 
16087 	/* Intentionally exclude ASIC_REV_5906 */
16088 	if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
16089 	    tg3_asic_rev(tp) == ASIC_REV_5787 ||
16090 	    tg3_asic_rev(tp) == ASIC_REV_5784 ||
16091 	    tg3_asic_rev(tp) == ASIC_REV_5761 ||
16092 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
16093 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
16094 	    tg3_flag(tp, 57765_PLUS))
16095 		tg3_flag_set(tp, 5755_PLUS);
16096 
16097 	if (tg3_asic_rev(tp) == ASIC_REV_5780 ||
16098 	    tg3_asic_rev(tp) == ASIC_REV_5714)
16099 		tg3_flag_set(tp, 5780_CLASS);
16100 
16101 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
16102 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
16103 	    tg3_asic_rev(tp) == ASIC_REV_5906 ||
16104 	    tg3_flag(tp, 5755_PLUS) ||
16105 	    tg3_flag(tp, 5780_CLASS))
16106 		tg3_flag_set(tp, 5750_PLUS);
16107 
16108 	if (tg3_asic_rev(tp) == ASIC_REV_5705 ||
16109 	    tg3_flag(tp, 5750_PLUS))
16110 		tg3_flag_set(tp, 5705_PLUS);
16111 }
16112 
16113 static bool tg3_10_100_only_device(struct tg3 *tp,
16114 				   const struct pci_device_id *ent)
16115 {
16116 	u32 grc_misc_cfg = tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK;
16117 
16118 	if ((tg3_asic_rev(tp) == ASIC_REV_5703 &&
16119 	     (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
16120 	    (tp->phy_flags & TG3_PHYFLG_IS_FET))
16121 		return true;
16122 
16123 	if (ent->driver_data & TG3_DRV_DATA_FLAG_10_100_ONLY) {
16124 		if (tg3_asic_rev(tp) == ASIC_REV_5705) {
16125 			if (ent->driver_data & TG3_DRV_DATA_FLAG_5705_10_100)
16126 				return true;
16127 		} else {
16128 			return true;
16129 		}
16130 	}
16131 
16132 	return false;
16133 }
16134 
16135 static int tg3_get_invariants(struct tg3 *tp, const struct pci_device_id *ent)
16136 {
16137 	u32 misc_ctrl_reg;
16138 	u32 pci_state_reg, grc_misc_cfg;
16139 	u32 val;
16140 	u16 pci_cmd;
16141 	int err;
16142 
16143 	/* Force memory write invalidate off.  If we leave it on,
16144 	 * then on 5700_BX chips we have to enable a workaround.
16145 	 * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
16146 	 * to match the cacheline size.  The Broadcom driver have this
16147 	 * workaround but turns MWI off all the times so never uses
16148 	 * it.  This seems to suggest that the workaround is insufficient.
16149 	 */
16150 	pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16151 	pci_cmd &= ~PCI_COMMAND_INVALIDATE;
16152 	pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16153 
16154 	/* Important! -- Make sure register accesses are byteswapped
16155 	 * correctly.  Also, for those chips that require it, make
16156 	 * sure that indirect register accesses are enabled before
16157 	 * the first operation.
16158 	 */
16159 	pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16160 			      &misc_ctrl_reg);
16161 	tp->misc_host_ctrl |= (misc_ctrl_reg &
16162 			       MISC_HOST_CTRL_CHIPREV);
16163 	pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16164 			       tp->misc_host_ctrl);
16165 
16166 	tg3_detect_asic_rev(tp, misc_ctrl_reg);
16167 
16168 	/* If we have 5702/03 A1 or A2 on certain ICH chipsets,
16169 	 * we need to disable memory and use config. cycles
16170 	 * only to access all registers. The 5702/03 chips
16171 	 * can mistakenly decode the special cycles from the
16172 	 * ICH chipsets as memory write cycles, causing corruption
16173 	 * of register and memory space. Only certain ICH bridges
16174 	 * will drive special cycles with non-zero data during the
16175 	 * address phase which can fall within the 5703's address
16176 	 * range. This is not an ICH bug as the PCI spec allows
16177 	 * non-zero address during special cycles. However, only
16178 	 * these ICH bridges are known to drive non-zero addresses
16179 	 * during special cycles.
16180 	 *
16181 	 * Since special cycles do not cross PCI bridges, we only
16182 	 * enable this workaround if the 5703 is on the secondary
16183 	 * bus of these ICH bridges.
16184 	 */
16185 	if ((tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A1) ||
16186 	    (tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A2)) {
16187 		static struct tg3_dev_id {
16188 			u32	vendor;
16189 			u32	device;
16190 			u32	rev;
16191 		} ich_chipsets[] = {
16192 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
16193 			  PCI_ANY_ID },
16194 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
16195 			  PCI_ANY_ID },
16196 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
16197 			  0xa },
16198 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
16199 			  PCI_ANY_ID },
16200 			{ },
16201 		};
16202 		struct tg3_dev_id *pci_id = &ich_chipsets[0];
16203 		struct pci_dev *bridge = NULL;
16204 
16205 		while (pci_id->vendor != 0) {
16206 			bridge = pci_get_device(pci_id->vendor, pci_id->device,
16207 						bridge);
16208 			if (!bridge) {
16209 				pci_id++;
16210 				continue;
16211 			}
16212 			if (pci_id->rev != PCI_ANY_ID) {
16213 				if (bridge->revision > pci_id->rev)
16214 					continue;
16215 			}
16216 			if (bridge->subordinate &&
16217 			    (bridge->subordinate->number ==
16218 			     tp->pdev->bus->number)) {
16219 				tg3_flag_set(tp, ICH_WORKAROUND);
16220 				pci_dev_put(bridge);
16221 				break;
16222 			}
16223 		}
16224 	}
16225 
16226 	if (tg3_asic_rev(tp) == ASIC_REV_5701) {
16227 		static struct tg3_dev_id {
16228 			u32	vendor;
16229 			u32	device;
16230 		} bridge_chipsets[] = {
16231 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_0 },
16232 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_1 },
16233 			{ },
16234 		};
16235 		struct tg3_dev_id *pci_id = &bridge_chipsets[0];
16236 		struct pci_dev *bridge = NULL;
16237 
16238 		while (pci_id->vendor != 0) {
16239 			bridge = pci_get_device(pci_id->vendor,
16240 						pci_id->device,
16241 						bridge);
16242 			if (!bridge) {
16243 				pci_id++;
16244 				continue;
16245 			}
16246 			if (bridge->subordinate &&
16247 			    (bridge->subordinate->number <=
16248 			     tp->pdev->bus->number) &&
16249 			    (bridge->subordinate->busn_res.end >=
16250 			     tp->pdev->bus->number)) {
16251 				tg3_flag_set(tp, 5701_DMA_BUG);
16252 				pci_dev_put(bridge);
16253 				break;
16254 			}
16255 		}
16256 	}
16257 
16258 	/* The EPB bridge inside 5714, 5715, and 5780 cannot support
16259 	 * DMA addresses > 40-bit. This bridge may have other additional
16260 	 * 57xx devices behind it in some 4-port NIC designs for example.
16261 	 * Any tg3 device found behind the bridge will also need the 40-bit
16262 	 * DMA workaround.
16263 	 */
16264 	if (tg3_flag(tp, 5780_CLASS)) {
16265 		tg3_flag_set(tp, 40BIT_DMA_BUG);
16266 		tp->msi_cap = tp->pdev->msi_cap;
16267 	} else {
16268 		struct pci_dev *bridge = NULL;
16269 
16270 		do {
16271 			bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
16272 						PCI_DEVICE_ID_SERVERWORKS_EPB,
16273 						bridge);
16274 			if (bridge && bridge->subordinate &&
16275 			    (bridge->subordinate->number <=
16276 			     tp->pdev->bus->number) &&
16277 			    (bridge->subordinate->busn_res.end >=
16278 			     tp->pdev->bus->number)) {
16279 				tg3_flag_set(tp, 40BIT_DMA_BUG);
16280 				pci_dev_put(bridge);
16281 				break;
16282 			}
16283 		} while (bridge);
16284 	}
16285 
16286 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16287 	    tg3_asic_rev(tp) == ASIC_REV_5714)
16288 		tp->pdev_peer = tg3_find_peer(tp);
16289 
16290 	/* Determine TSO capabilities */
16291 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0)
16292 		; /* Do nothing. HW bug. */
16293 	else if (tg3_flag(tp, 57765_PLUS))
16294 		tg3_flag_set(tp, HW_TSO_3);
16295 	else if (tg3_flag(tp, 5755_PLUS) ||
16296 		 tg3_asic_rev(tp) == ASIC_REV_5906)
16297 		tg3_flag_set(tp, HW_TSO_2);
16298 	else if (tg3_flag(tp, 5750_PLUS)) {
16299 		tg3_flag_set(tp, HW_TSO_1);
16300 		tg3_flag_set(tp, TSO_BUG);
16301 		if (tg3_asic_rev(tp) == ASIC_REV_5750 &&
16302 		    tg3_chip_rev_id(tp) >= CHIPREV_ID_5750_C2)
16303 			tg3_flag_clear(tp, TSO_BUG);
16304 	} else if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
16305 		   tg3_asic_rev(tp) != ASIC_REV_5701 &&
16306 		   tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
16307 		tg3_flag_set(tp, FW_TSO);
16308 		tg3_flag_set(tp, TSO_BUG);
16309 		if (tg3_asic_rev(tp) == ASIC_REV_5705)
16310 			tp->fw_needed = FIRMWARE_TG3TSO5;
16311 		else
16312 			tp->fw_needed = FIRMWARE_TG3TSO;
16313 	}
16314 
16315 	/* Selectively allow TSO based on operating conditions */
16316 	if (tg3_flag(tp, HW_TSO_1) ||
16317 	    tg3_flag(tp, HW_TSO_2) ||
16318 	    tg3_flag(tp, HW_TSO_3) ||
16319 	    tg3_flag(tp, FW_TSO)) {
16320 		/* For firmware TSO, assume ASF is disabled.
16321 		 * We'll disable TSO later if we discover ASF
16322 		 * is enabled in tg3_get_eeprom_hw_cfg().
16323 		 */
16324 		tg3_flag_set(tp, TSO_CAPABLE);
16325 	} else {
16326 		tg3_flag_clear(tp, TSO_CAPABLE);
16327 		tg3_flag_clear(tp, TSO_BUG);
16328 		tp->fw_needed = NULL;
16329 	}
16330 
16331 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0)
16332 		tp->fw_needed = FIRMWARE_TG3;
16333 
16334 	if (tg3_asic_rev(tp) == ASIC_REV_57766)
16335 		tp->fw_needed = FIRMWARE_TG357766;
16336 
16337 	tp->irq_max = 1;
16338 
16339 	if (tg3_flag(tp, 5750_PLUS)) {
16340 		tg3_flag_set(tp, SUPPORT_MSI);
16341 		if (tg3_chip_rev(tp) == CHIPREV_5750_AX ||
16342 		    tg3_chip_rev(tp) == CHIPREV_5750_BX ||
16343 		    (tg3_asic_rev(tp) == ASIC_REV_5714 &&
16344 		     tg3_chip_rev_id(tp) <= CHIPREV_ID_5714_A2 &&
16345 		     tp->pdev_peer == tp->pdev))
16346 			tg3_flag_clear(tp, SUPPORT_MSI);
16347 
16348 		if (tg3_flag(tp, 5755_PLUS) ||
16349 		    tg3_asic_rev(tp) == ASIC_REV_5906) {
16350 			tg3_flag_set(tp, 1SHOT_MSI);
16351 		}
16352 
16353 		if (tg3_flag(tp, 57765_PLUS)) {
16354 			tg3_flag_set(tp, SUPPORT_MSIX);
16355 			tp->irq_max = TG3_IRQ_MAX_VECS;
16356 		}
16357 	}
16358 
16359 	tp->txq_max = 1;
16360 	tp->rxq_max = 1;
16361 	if (tp->irq_max > 1) {
16362 		tp->rxq_max = TG3_RSS_MAX_NUM_QS;
16363 		tg3_rss_init_dflt_indir_tbl(tp, TG3_RSS_MAX_NUM_QS);
16364 
16365 		if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
16366 		    tg3_asic_rev(tp) == ASIC_REV_5720)
16367 			tp->txq_max = tp->irq_max - 1;
16368 	}
16369 
16370 	if (tg3_flag(tp, 5755_PLUS) ||
16371 	    tg3_asic_rev(tp) == ASIC_REV_5906)
16372 		tg3_flag_set(tp, SHORT_DMA_BUG);
16373 
16374 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
16375 		tp->dma_limit = TG3_TX_BD_DMA_MAX_4K;
16376 
16377 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16378 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
16379 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
16380 	    tg3_asic_rev(tp) == ASIC_REV_5762)
16381 		tg3_flag_set(tp, LRG_PROD_RING_CAP);
16382 
16383 	if (tg3_flag(tp, 57765_PLUS) &&
16384 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5719_A0)
16385 		tg3_flag_set(tp, USE_JUMBO_BDFLAG);
16386 
16387 	if (!tg3_flag(tp, 5705_PLUS) ||
16388 	    tg3_flag(tp, 5780_CLASS) ||
16389 	    tg3_flag(tp, USE_JUMBO_BDFLAG))
16390 		tg3_flag_set(tp, JUMBO_CAPABLE);
16391 
16392 	pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
16393 			      &pci_state_reg);
16394 
16395 	if (pci_is_pcie(tp->pdev)) {
16396 		u16 lnkctl;
16397 
16398 		tg3_flag_set(tp, PCI_EXPRESS);
16399 
16400 		pcie_capability_read_word(tp->pdev, PCI_EXP_LNKCTL, &lnkctl);
16401 		if (lnkctl & PCI_EXP_LNKCTL_CLKREQ_EN) {
16402 			if (tg3_asic_rev(tp) == ASIC_REV_5906) {
16403 				tg3_flag_clear(tp, HW_TSO_2);
16404 				tg3_flag_clear(tp, TSO_CAPABLE);
16405 			}
16406 			if (tg3_asic_rev(tp) == ASIC_REV_5784 ||
16407 			    tg3_asic_rev(tp) == ASIC_REV_5761 ||
16408 			    tg3_chip_rev_id(tp) == CHIPREV_ID_57780_A0 ||
16409 			    tg3_chip_rev_id(tp) == CHIPREV_ID_57780_A1)
16410 				tg3_flag_set(tp, CLKREQ_BUG);
16411 		} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5717_A0) {
16412 			tg3_flag_set(tp, L1PLLPD_EN);
16413 		}
16414 	} else if (tg3_asic_rev(tp) == ASIC_REV_5785) {
16415 		/* BCM5785 devices are effectively PCIe devices, and should
16416 		 * follow PCIe codepaths, but do not have a PCIe capabilities
16417 		 * section.
16418 		 */
16419 		tg3_flag_set(tp, PCI_EXPRESS);
16420 	} else if (!tg3_flag(tp, 5705_PLUS) ||
16421 		   tg3_flag(tp, 5780_CLASS)) {
16422 		tp->pcix_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_PCIX);
16423 		if (!tp->pcix_cap) {
16424 			dev_err(&tp->pdev->dev,
16425 				"Cannot find PCI-X capability, aborting\n");
16426 			return -EIO;
16427 		}
16428 
16429 		if (!(pci_state_reg & PCISTATE_CONV_PCI_MODE))
16430 			tg3_flag_set(tp, PCIX_MODE);
16431 	}
16432 
16433 	/* If we have an AMD 762 or VIA K8T800 chipset, write
16434 	 * reordering to the mailbox registers done by the host
16435 	 * controller can cause major troubles.  We read back from
16436 	 * every mailbox register write to force the writes to be
16437 	 * posted to the chip in order.
16438 	 */
16439 	if (pci_dev_present(tg3_write_reorder_chipsets) &&
16440 	    !tg3_flag(tp, PCI_EXPRESS))
16441 		tg3_flag_set(tp, MBOX_WRITE_REORDER);
16442 
16443 	pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
16444 			     &tp->pci_cacheline_sz);
16445 	pci_read_config_byte(tp->pdev, PCI_LATENCY_TIMER,
16446 			     &tp->pci_lat_timer);
16447 	if (tg3_asic_rev(tp) == ASIC_REV_5703 &&
16448 	    tp->pci_lat_timer < 64) {
16449 		tp->pci_lat_timer = 64;
16450 		pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
16451 				      tp->pci_lat_timer);
16452 	}
16453 
16454 	/* Important! -- It is critical that the PCI-X hw workaround
16455 	 * situation is decided before the first MMIO register access.
16456 	 */
16457 	if (tg3_chip_rev(tp) == CHIPREV_5700_BX) {
16458 		/* 5700 BX chips need to have their TX producer index
16459 		 * mailboxes written twice to workaround a bug.
16460 		 */
16461 		tg3_flag_set(tp, TXD_MBOX_HWBUG);
16462 
16463 		/* If we are in PCI-X mode, enable register write workaround.
16464 		 *
16465 		 * The workaround is to use indirect register accesses
16466 		 * for all chip writes not to mailbox registers.
16467 		 */
16468 		if (tg3_flag(tp, PCIX_MODE)) {
16469 			u32 pm_reg;
16470 
16471 			tg3_flag_set(tp, PCIX_TARGET_HWBUG);
16472 
16473 			/* The chip can have it's power management PCI config
16474 			 * space registers clobbered due to this bug.
16475 			 * So explicitly force the chip into D0 here.
16476 			 */
16477 			pci_read_config_dword(tp->pdev,
16478 					      tp->pdev->pm_cap + PCI_PM_CTRL,
16479 					      &pm_reg);
16480 			pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
16481 			pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
16482 			pci_write_config_dword(tp->pdev,
16483 					       tp->pdev->pm_cap + PCI_PM_CTRL,
16484 					       pm_reg);
16485 
16486 			/* Also, force SERR#/PERR# in PCI command. */
16487 			pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16488 			pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
16489 			pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16490 		}
16491 	}
16492 
16493 	if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
16494 		tg3_flag_set(tp, PCI_HIGH_SPEED);
16495 	if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
16496 		tg3_flag_set(tp, PCI_32BIT);
16497 
16498 	/* Chip-specific fixup from Broadcom driver */
16499 	if ((tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0) &&
16500 	    (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
16501 		pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
16502 		pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
16503 	}
16504 
16505 	/* Default fast path register access methods */
16506 	tp->read32 = tg3_read32;
16507 	tp->write32 = tg3_write32;
16508 	tp->read32_mbox = tg3_read32;
16509 	tp->write32_mbox = tg3_write32;
16510 	tp->write32_tx_mbox = tg3_write32;
16511 	tp->write32_rx_mbox = tg3_write32;
16512 
16513 	/* Various workaround register access methods */
16514 	if (tg3_flag(tp, PCIX_TARGET_HWBUG))
16515 		tp->write32 = tg3_write_indirect_reg32;
16516 	else if (tg3_asic_rev(tp) == ASIC_REV_5701 ||
16517 		 (tg3_flag(tp, PCI_EXPRESS) &&
16518 		  tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A0)) {
16519 		/*
16520 		 * Back to back register writes can cause problems on these
16521 		 * chips, the workaround is to read back all reg writes
16522 		 * except those to mailbox regs.
16523 		 *
16524 		 * See tg3_write_indirect_reg32().
16525 		 */
16526 		tp->write32 = tg3_write_flush_reg32;
16527 	}
16528 
16529 	if (tg3_flag(tp, TXD_MBOX_HWBUG) || tg3_flag(tp, MBOX_WRITE_REORDER)) {
16530 		tp->write32_tx_mbox = tg3_write32_tx_mbox;
16531 		if (tg3_flag(tp, MBOX_WRITE_REORDER))
16532 			tp->write32_rx_mbox = tg3_write_flush_reg32;
16533 	}
16534 
16535 	if (tg3_flag(tp, ICH_WORKAROUND)) {
16536 		tp->read32 = tg3_read_indirect_reg32;
16537 		tp->write32 = tg3_write_indirect_reg32;
16538 		tp->read32_mbox = tg3_read_indirect_mbox;
16539 		tp->write32_mbox = tg3_write_indirect_mbox;
16540 		tp->write32_tx_mbox = tg3_write_indirect_mbox;
16541 		tp->write32_rx_mbox = tg3_write_indirect_mbox;
16542 
16543 		iounmap(tp->regs);
16544 		tp->regs = NULL;
16545 
16546 		pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16547 		pci_cmd &= ~PCI_COMMAND_MEMORY;
16548 		pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16549 	}
16550 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
16551 		tp->read32_mbox = tg3_read32_mbox_5906;
16552 		tp->write32_mbox = tg3_write32_mbox_5906;
16553 		tp->write32_tx_mbox = tg3_write32_mbox_5906;
16554 		tp->write32_rx_mbox = tg3_write32_mbox_5906;
16555 	}
16556 
16557 	if (tp->write32 == tg3_write_indirect_reg32 ||
16558 	    (tg3_flag(tp, PCIX_MODE) &&
16559 	     (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16560 	      tg3_asic_rev(tp) == ASIC_REV_5701)))
16561 		tg3_flag_set(tp, SRAM_USE_CONFIG);
16562 
16563 	/* The memory arbiter has to be enabled in order for SRAM accesses
16564 	 * to succeed.  Normally on powerup the tg3 chip firmware will make
16565 	 * sure it is enabled, but other entities such as system netboot
16566 	 * code might disable it.
16567 	 */
16568 	val = tr32(MEMARB_MODE);
16569 	tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
16570 
16571 	tp->pci_fn = PCI_FUNC(tp->pdev->devfn) & 3;
16572 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16573 	    tg3_flag(tp, 5780_CLASS)) {
16574 		if (tg3_flag(tp, PCIX_MODE)) {
16575 			pci_read_config_dword(tp->pdev,
16576 					      tp->pcix_cap + PCI_X_STATUS,
16577 					      &val);
16578 			tp->pci_fn = val & 0x7;
16579 		}
16580 	} else if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16581 		   tg3_asic_rev(tp) == ASIC_REV_5719 ||
16582 		   tg3_asic_rev(tp) == ASIC_REV_5720) {
16583 		tg3_read_mem(tp, NIC_SRAM_CPMU_STATUS, &val);
16584 		if ((val & NIC_SRAM_CPMUSTAT_SIG_MSK) != NIC_SRAM_CPMUSTAT_SIG)
16585 			val = tr32(TG3_CPMU_STATUS);
16586 
16587 		if (tg3_asic_rev(tp) == ASIC_REV_5717)
16588 			tp->pci_fn = (val & TG3_CPMU_STATUS_FMSK_5717) ? 1 : 0;
16589 		else
16590 			tp->pci_fn = (val & TG3_CPMU_STATUS_FMSK_5719) >>
16591 				     TG3_CPMU_STATUS_FSHFT_5719;
16592 	}
16593 
16594 	if (tg3_flag(tp, FLUSH_POSTED_WRITES)) {
16595 		tp->write32_tx_mbox = tg3_write_flush_reg32;
16596 		tp->write32_rx_mbox = tg3_write_flush_reg32;
16597 	}
16598 
16599 	/* Get eeprom hw config before calling tg3_set_power_state().
16600 	 * In particular, the TG3_FLAG_IS_NIC flag must be
16601 	 * determined before calling tg3_set_power_state() so that
16602 	 * we know whether or not to switch out of Vaux power.
16603 	 * When the flag is set, it means that GPIO1 is used for eeprom
16604 	 * write protect and also implies that it is a LOM where GPIOs
16605 	 * are not used to switch power.
16606 	 */
16607 	tg3_get_eeprom_hw_cfg(tp);
16608 
16609 	if (tg3_flag(tp, FW_TSO) && tg3_flag(tp, ENABLE_ASF)) {
16610 		tg3_flag_clear(tp, TSO_CAPABLE);
16611 		tg3_flag_clear(tp, TSO_BUG);
16612 		tp->fw_needed = NULL;
16613 	}
16614 
16615 	if (tg3_flag(tp, ENABLE_APE)) {
16616 		/* Allow reads and writes to the
16617 		 * APE register and memory space.
16618 		 */
16619 		pci_state_reg |= PCISTATE_ALLOW_APE_CTLSPC_WR |
16620 				 PCISTATE_ALLOW_APE_SHMEM_WR |
16621 				 PCISTATE_ALLOW_APE_PSPACE_WR;
16622 		pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE,
16623 				       pci_state_reg);
16624 
16625 		tg3_ape_lock_init(tp);
16626 	}
16627 
16628 	/* Set up tp->grc_local_ctrl before calling
16629 	 * tg3_pwrsrc_switch_to_vmain().  GPIO1 driven high
16630 	 * will bring 5700's external PHY out of reset.
16631 	 * It is also used as eeprom write protect on LOMs.
16632 	 */
16633 	tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
16634 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16635 	    tg3_flag(tp, EEPROM_WRITE_PROT))
16636 		tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
16637 				       GRC_LCLCTRL_GPIO_OUTPUT1);
16638 	/* Unused GPIO3 must be driven as output on 5752 because there
16639 	 * are no pull-up resistors on unused GPIO pins.
16640 	 */
16641 	else if (tg3_asic_rev(tp) == ASIC_REV_5752)
16642 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
16643 
16644 	if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
16645 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
16646 	    tg3_flag(tp, 57765_CLASS))
16647 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
16648 
16649 	if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
16650 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
16651 		/* Turn off the debug UART. */
16652 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
16653 		if (tg3_flag(tp, IS_NIC))
16654 			/* Keep VMain power. */
16655 			tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
16656 					      GRC_LCLCTRL_GPIO_OUTPUT0;
16657 	}
16658 
16659 	if (tg3_asic_rev(tp) == ASIC_REV_5762)
16660 		tp->grc_local_ctrl |=
16661 			tr32(GRC_LOCAL_CTRL) & GRC_LCLCTRL_GPIO_UART_SEL;
16662 
16663 	/* Switch out of Vaux if it is a NIC */
16664 	tg3_pwrsrc_switch_to_vmain(tp);
16665 
16666 	/* Derive initial jumbo mode from MTU assigned in
16667 	 * ether_setup() via the alloc_etherdev() call
16668 	 */
16669 	if (tp->dev->mtu > ETH_DATA_LEN && !tg3_flag(tp, 5780_CLASS))
16670 		tg3_flag_set(tp, JUMBO_RING_ENABLE);
16671 
16672 	/* Determine WakeOnLan speed to use. */
16673 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16674 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
16675 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0 ||
16676 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B2) {
16677 		tg3_flag_clear(tp, WOL_SPEED_100MB);
16678 	} else {
16679 		tg3_flag_set(tp, WOL_SPEED_100MB);
16680 	}
16681 
16682 	if (tg3_asic_rev(tp) == ASIC_REV_5906)
16683 		tp->phy_flags |= TG3_PHYFLG_IS_FET;
16684 
16685 	/* A few boards don't want Ethernet@WireSpeed phy feature */
16686 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16687 	    (tg3_asic_rev(tp) == ASIC_REV_5705 &&
16688 	     (tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) &&
16689 	     (tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A1)) ||
16690 	    (tp->phy_flags & TG3_PHYFLG_IS_FET) ||
16691 	    (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
16692 		tp->phy_flags |= TG3_PHYFLG_NO_ETH_WIRE_SPEED;
16693 
16694 	if (tg3_chip_rev(tp) == CHIPREV_5703_AX ||
16695 	    tg3_chip_rev(tp) == CHIPREV_5704_AX)
16696 		tp->phy_flags |= TG3_PHYFLG_ADC_BUG;
16697 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0)
16698 		tp->phy_flags |= TG3_PHYFLG_5704_A0_BUG;
16699 
16700 	if (tg3_flag(tp, 5705_PLUS) &&
16701 	    !(tp->phy_flags & TG3_PHYFLG_IS_FET) &&
16702 	    tg3_asic_rev(tp) != ASIC_REV_5785 &&
16703 	    tg3_asic_rev(tp) != ASIC_REV_57780 &&
16704 	    !tg3_flag(tp, 57765_PLUS)) {
16705 		if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
16706 		    tg3_asic_rev(tp) == ASIC_REV_5787 ||
16707 		    tg3_asic_rev(tp) == ASIC_REV_5784 ||
16708 		    tg3_asic_rev(tp) == ASIC_REV_5761) {
16709 			if (tp->pdev->device != PCI_DEVICE_ID_TIGON3_5756 &&
16710 			    tp->pdev->device != PCI_DEVICE_ID_TIGON3_5722)
16711 				tp->phy_flags |= TG3_PHYFLG_JITTER_BUG;
16712 			if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5755M)
16713 				tp->phy_flags |= TG3_PHYFLG_ADJUST_TRIM;
16714 		} else
16715 			tp->phy_flags |= TG3_PHYFLG_BER_BUG;
16716 	}
16717 
16718 	if (tg3_asic_rev(tp) == ASIC_REV_5784 &&
16719 	    tg3_chip_rev(tp) != CHIPREV_5784_AX) {
16720 		tp->phy_otp = tg3_read_otp_phycfg(tp);
16721 		if (tp->phy_otp == 0)
16722 			tp->phy_otp = TG3_OTP_DEFAULT;
16723 	}
16724 
16725 	if (tg3_flag(tp, CPMU_PRESENT))
16726 		tp->mi_mode = MAC_MI_MODE_500KHZ_CONST;
16727 	else
16728 		tp->mi_mode = MAC_MI_MODE_BASE;
16729 
16730 	tp->coalesce_mode = 0;
16731 	if (tg3_chip_rev(tp) != CHIPREV_5700_AX &&
16732 	    tg3_chip_rev(tp) != CHIPREV_5700_BX)
16733 		tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
16734 
16735 	/* Set these bits to enable statistics workaround. */
16736 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16737 	    tg3_asic_rev(tp) == ASIC_REV_5762 ||
16738 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
16739 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5720_A0) {
16740 		tp->coalesce_mode |= HOSTCC_MODE_ATTN;
16741 		tp->grc_mode |= GRC_MODE_IRQ_ON_FLOW_ATTN;
16742 	}
16743 
16744 	if (tg3_asic_rev(tp) == ASIC_REV_5785 ||
16745 	    tg3_asic_rev(tp) == ASIC_REV_57780)
16746 		tg3_flag_set(tp, USE_PHYLIB);
16747 
16748 	err = tg3_mdio_init(tp);
16749 	if (err)
16750 		return err;
16751 
16752 	/* Initialize data/descriptor byte/word swapping. */
16753 	val = tr32(GRC_MODE);
16754 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
16755 	    tg3_asic_rev(tp) == ASIC_REV_5762)
16756 		val &= (GRC_MODE_BYTE_SWAP_B2HRX_DATA |
16757 			GRC_MODE_WORD_SWAP_B2HRX_DATA |
16758 			GRC_MODE_B2HRX_ENABLE |
16759 			GRC_MODE_HTX2B_ENABLE |
16760 			GRC_MODE_HOST_STACKUP);
16761 	else
16762 		val &= GRC_MODE_HOST_STACKUP;
16763 
16764 	tw32(GRC_MODE, val | tp->grc_mode);
16765 
16766 	tg3_switch_clocks(tp);
16767 
16768 	/* Clear this out for sanity. */
16769 	tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
16770 
16771 	/* Clear TG3PCI_REG_BASE_ADDR to prevent hangs. */
16772 	tw32(TG3PCI_REG_BASE_ADDR, 0);
16773 
16774 	pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
16775 			      &pci_state_reg);
16776 	if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
16777 	    !tg3_flag(tp, PCIX_TARGET_HWBUG)) {
16778 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
16779 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0 ||
16780 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B2 ||
16781 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B5) {
16782 			void __iomem *sram_base;
16783 
16784 			/* Write some dummy words into the SRAM status block
16785 			 * area, see if it reads back correctly.  If the return
16786 			 * value is bad, force enable the PCIX workaround.
16787 			 */
16788 			sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
16789 
16790 			writel(0x00000000, sram_base);
16791 			writel(0x00000000, sram_base + 4);
16792 			writel(0xffffffff, sram_base + 4);
16793 			if (readl(sram_base) != 0x00000000)
16794 				tg3_flag_set(tp, PCIX_TARGET_HWBUG);
16795 		}
16796 	}
16797 
16798 	udelay(50);
16799 	tg3_nvram_init(tp);
16800 
16801 	/* If the device has an NVRAM, no need to load patch firmware */
16802 	if (tg3_asic_rev(tp) == ASIC_REV_57766 &&
16803 	    !tg3_flag(tp, NO_NVRAM))
16804 		tp->fw_needed = NULL;
16805 
16806 	grc_misc_cfg = tr32(GRC_MISC_CFG);
16807 	grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
16808 
16809 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
16810 	    (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
16811 	     grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
16812 		tg3_flag_set(tp, IS_5788);
16813 
16814 	if (!tg3_flag(tp, IS_5788) &&
16815 	    tg3_asic_rev(tp) != ASIC_REV_5700)
16816 		tg3_flag_set(tp, TAGGED_STATUS);
16817 	if (tg3_flag(tp, TAGGED_STATUS)) {
16818 		tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
16819 				      HOSTCC_MODE_CLRTICK_TXBD);
16820 
16821 		tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
16822 		pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16823 				       tp->misc_host_ctrl);
16824 	}
16825 
16826 	/* Preserve the APE MAC_MODE bits */
16827 	if (tg3_flag(tp, ENABLE_APE))
16828 		tp->mac_mode = MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
16829 	else
16830 		tp->mac_mode = 0;
16831 
16832 	if (tg3_10_100_only_device(tp, ent))
16833 		tp->phy_flags |= TG3_PHYFLG_10_100_ONLY;
16834 
16835 	err = tg3_phy_probe(tp);
16836 	if (err) {
16837 		dev_err(&tp->pdev->dev, "phy probe failed, err %d\n", err);
16838 		/* ... but do not return immediately ... */
16839 		tg3_mdio_fini(tp);
16840 	}
16841 
16842 	tg3_read_vpd(tp);
16843 	tg3_read_fw_ver(tp);
16844 
16845 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
16846 		tp->phy_flags &= ~TG3_PHYFLG_USE_MI_INTERRUPT;
16847 	} else {
16848 		if (tg3_asic_rev(tp) == ASIC_REV_5700)
16849 			tp->phy_flags |= TG3_PHYFLG_USE_MI_INTERRUPT;
16850 		else
16851 			tp->phy_flags &= ~TG3_PHYFLG_USE_MI_INTERRUPT;
16852 	}
16853 
16854 	/* 5700 {AX,BX} chips have a broken status block link
16855 	 * change bit implementation, so we must use the
16856 	 * status register in those cases.
16857 	 */
16858 	if (tg3_asic_rev(tp) == ASIC_REV_5700)
16859 		tg3_flag_set(tp, USE_LINKCHG_REG);
16860 	else
16861 		tg3_flag_clear(tp, USE_LINKCHG_REG);
16862 
16863 	/* The led_ctrl is set during tg3_phy_probe, here we might
16864 	 * have to force the link status polling mechanism based
16865 	 * upon subsystem IDs.
16866 	 */
16867 	if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
16868 	    tg3_asic_rev(tp) == ASIC_REV_5701 &&
16869 	    !(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
16870 		tp->phy_flags |= TG3_PHYFLG_USE_MI_INTERRUPT;
16871 		tg3_flag_set(tp, USE_LINKCHG_REG);
16872 	}
16873 
16874 	/* For all SERDES we poll the MAC status register. */
16875 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
16876 		tg3_flag_set(tp, POLL_SERDES);
16877 	else
16878 		tg3_flag_clear(tp, POLL_SERDES);
16879 
16880 	if (tg3_flag(tp, ENABLE_APE) && tg3_flag(tp, ENABLE_ASF))
16881 		tg3_flag_set(tp, POLL_CPMU_LINK);
16882 
16883 	tp->rx_offset = NET_SKB_PAD + NET_IP_ALIGN;
16884 	tp->rx_copy_thresh = TG3_RX_COPY_THRESHOLD;
16885 	if (tg3_asic_rev(tp) == ASIC_REV_5701 &&
16886 	    tg3_flag(tp, PCIX_MODE)) {
16887 		tp->rx_offset = NET_SKB_PAD;
16888 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
16889 		tp->rx_copy_thresh = ~(u16)0;
16890 #endif
16891 	}
16892 
16893 	tp->rx_std_ring_mask = TG3_RX_STD_RING_SIZE(tp) - 1;
16894 	tp->rx_jmb_ring_mask = TG3_RX_JMB_RING_SIZE(tp) - 1;
16895 	tp->rx_ret_ring_mask = tg3_rx_ret_ring_size(tp) - 1;
16896 
16897 	tp->rx_std_max_post = tp->rx_std_ring_mask + 1;
16898 
16899 	/* Increment the rx prod index on the rx std ring by at most
16900 	 * 8 for these chips to workaround hw errata.
16901 	 */
16902 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
16903 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
16904 	    tg3_asic_rev(tp) == ASIC_REV_5755)
16905 		tp->rx_std_max_post = 8;
16906 
16907 	if (tg3_flag(tp, ASPM_WORKAROUND))
16908 		tp->pwrmgmt_thresh = tr32(PCIE_PWR_MGMT_THRESH) &
16909 				     PCIE_PWR_MGMT_L1_THRESH_MSK;
16910 
16911 	return err;
16912 }
16913 
16914 #ifdef CONFIG_SPARC
16915 static int tg3_get_macaddr_sparc(struct tg3 *tp)
16916 {
16917 	struct net_device *dev = tp->dev;
16918 	struct pci_dev *pdev = tp->pdev;
16919 	struct device_node *dp = pci_device_to_OF_node(pdev);
16920 	const unsigned char *addr;
16921 	int len;
16922 
16923 	addr = of_get_property(dp, "local-mac-address", &len);
16924 	if (addr && len == ETH_ALEN) {
16925 		memcpy(dev->dev_addr, addr, ETH_ALEN);
16926 		return 0;
16927 	}
16928 	return -ENODEV;
16929 }
16930 
16931 static int tg3_get_default_macaddr_sparc(struct tg3 *tp)
16932 {
16933 	struct net_device *dev = tp->dev;
16934 
16935 	memcpy(dev->dev_addr, idprom->id_ethaddr, ETH_ALEN);
16936 	return 0;
16937 }
16938 #endif
16939 
16940 static int tg3_get_device_address(struct tg3 *tp)
16941 {
16942 	struct net_device *dev = tp->dev;
16943 	u32 hi, lo, mac_offset;
16944 	int addr_ok = 0;
16945 	int err;
16946 
16947 #ifdef CONFIG_SPARC
16948 	if (!tg3_get_macaddr_sparc(tp))
16949 		return 0;
16950 #endif
16951 
16952 	if (tg3_flag(tp, IS_SSB_CORE)) {
16953 		err = ssb_gige_get_macaddr(tp->pdev, &dev->dev_addr[0]);
16954 		if (!err && is_valid_ether_addr(&dev->dev_addr[0]))
16955 			return 0;
16956 	}
16957 
16958 	mac_offset = 0x7c;
16959 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16960 	    tg3_flag(tp, 5780_CLASS)) {
16961 		if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
16962 			mac_offset = 0xcc;
16963 		if (tg3_nvram_lock(tp))
16964 			tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
16965 		else
16966 			tg3_nvram_unlock(tp);
16967 	} else if (tg3_flag(tp, 5717_PLUS)) {
16968 		if (tp->pci_fn & 1)
16969 			mac_offset = 0xcc;
16970 		if (tp->pci_fn > 1)
16971 			mac_offset += 0x18c;
16972 	} else if (tg3_asic_rev(tp) == ASIC_REV_5906)
16973 		mac_offset = 0x10;
16974 
16975 	/* First try to get it from MAC address mailbox. */
16976 	tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
16977 	if ((hi >> 16) == 0x484b) {
16978 		dev->dev_addr[0] = (hi >>  8) & 0xff;
16979 		dev->dev_addr[1] = (hi >>  0) & 0xff;
16980 
16981 		tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
16982 		dev->dev_addr[2] = (lo >> 24) & 0xff;
16983 		dev->dev_addr[3] = (lo >> 16) & 0xff;
16984 		dev->dev_addr[4] = (lo >>  8) & 0xff;
16985 		dev->dev_addr[5] = (lo >>  0) & 0xff;
16986 
16987 		/* Some old bootcode may report a 0 MAC address in SRAM */
16988 		addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
16989 	}
16990 	if (!addr_ok) {
16991 		/* Next, try NVRAM. */
16992 		if (!tg3_flag(tp, NO_NVRAM) &&
16993 		    !tg3_nvram_read_be32(tp, mac_offset + 0, &hi) &&
16994 		    !tg3_nvram_read_be32(tp, mac_offset + 4, &lo)) {
16995 			memcpy(&dev->dev_addr[0], ((char *)&hi) + 2, 2);
16996 			memcpy(&dev->dev_addr[2], (char *)&lo, sizeof(lo));
16997 		}
16998 		/* Finally just fetch it out of the MAC control regs. */
16999 		else {
17000 			hi = tr32(MAC_ADDR_0_HIGH);
17001 			lo = tr32(MAC_ADDR_0_LOW);
17002 
17003 			dev->dev_addr[5] = lo & 0xff;
17004 			dev->dev_addr[4] = (lo >> 8) & 0xff;
17005 			dev->dev_addr[3] = (lo >> 16) & 0xff;
17006 			dev->dev_addr[2] = (lo >> 24) & 0xff;
17007 			dev->dev_addr[1] = hi & 0xff;
17008 			dev->dev_addr[0] = (hi >> 8) & 0xff;
17009 		}
17010 	}
17011 
17012 	if (!is_valid_ether_addr(&dev->dev_addr[0])) {
17013 #ifdef CONFIG_SPARC
17014 		if (!tg3_get_default_macaddr_sparc(tp))
17015 			return 0;
17016 #endif
17017 		return -EINVAL;
17018 	}
17019 	return 0;
17020 }
17021 
17022 #define BOUNDARY_SINGLE_CACHELINE	1
17023 #define BOUNDARY_MULTI_CACHELINE	2
17024 
17025 static u32 tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
17026 {
17027 	int cacheline_size;
17028 	u8 byte;
17029 	int goal;
17030 
17031 	pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
17032 	if (byte == 0)
17033 		cacheline_size = 1024;
17034 	else
17035 		cacheline_size = (int) byte * 4;
17036 
17037 	/* On 5703 and later chips, the boundary bits have no
17038 	 * effect.
17039 	 */
17040 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
17041 	    tg3_asic_rev(tp) != ASIC_REV_5701 &&
17042 	    !tg3_flag(tp, PCI_EXPRESS))
17043 		goto out;
17044 
17045 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
17046 	goal = BOUNDARY_MULTI_CACHELINE;
17047 #else
17048 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
17049 	goal = BOUNDARY_SINGLE_CACHELINE;
17050 #else
17051 	goal = 0;
17052 #endif
17053 #endif
17054 
17055 	if (tg3_flag(tp, 57765_PLUS)) {
17056 		val = goal ? 0 : DMA_RWCTRL_DIS_CACHE_ALIGNMENT;
17057 		goto out;
17058 	}
17059 
17060 	if (!goal)
17061 		goto out;
17062 
17063 	/* PCI controllers on most RISC systems tend to disconnect
17064 	 * when a device tries to burst across a cache-line boundary.
17065 	 * Therefore, letting tg3 do so just wastes PCI bandwidth.
17066 	 *
17067 	 * Unfortunately, for PCI-E there are only limited
17068 	 * write-side controls for this, and thus for reads
17069 	 * we will still get the disconnects.  We'll also waste
17070 	 * these PCI cycles for both read and write for chips
17071 	 * other than 5700 and 5701 which do not implement the
17072 	 * boundary bits.
17073 	 */
17074 	if (tg3_flag(tp, PCIX_MODE) && !tg3_flag(tp, PCI_EXPRESS)) {
17075 		switch (cacheline_size) {
17076 		case 16:
17077 		case 32:
17078 		case 64:
17079 		case 128:
17080 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17081 				val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
17082 					DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
17083 			} else {
17084 				val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
17085 					DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
17086 			}
17087 			break;
17088 
17089 		case 256:
17090 			val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
17091 				DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
17092 			break;
17093 
17094 		default:
17095 			val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
17096 				DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
17097 			break;
17098 		}
17099 	} else if (tg3_flag(tp, PCI_EXPRESS)) {
17100 		switch (cacheline_size) {
17101 		case 16:
17102 		case 32:
17103 		case 64:
17104 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17105 				val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
17106 				val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
17107 				break;
17108 			}
17109 			/* fallthrough */
17110 		case 128:
17111 		default:
17112 			val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
17113 			val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
17114 			break;
17115 		}
17116 	} else {
17117 		switch (cacheline_size) {
17118 		case 16:
17119 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17120 				val |= (DMA_RWCTRL_READ_BNDRY_16 |
17121 					DMA_RWCTRL_WRITE_BNDRY_16);
17122 				break;
17123 			}
17124 			/* fallthrough */
17125 		case 32:
17126 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17127 				val |= (DMA_RWCTRL_READ_BNDRY_32 |
17128 					DMA_RWCTRL_WRITE_BNDRY_32);
17129 				break;
17130 			}
17131 			/* fallthrough */
17132 		case 64:
17133 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17134 				val |= (DMA_RWCTRL_READ_BNDRY_64 |
17135 					DMA_RWCTRL_WRITE_BNDRY_64);
17136 				break;
17137 			}
17138 			/* fallthrough */
17139 		case 128:
17140 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17141 				val |= (DMA_RWCTRL_READ_BNDRY_128 |
17142 					DMA_RWCTRL_WRITE_BNDRY_128);
17143 				break;
17144 			}
17145 			/* fallthrough */
17146 		case 256:
17147 			val |= (DMA_RWCTRL_READ_BNDRY_256 |
17148 				DMA_RWCTRL_WRITE_BNDRY_256);
17149 			break;
17150 		case 512:
17151 			val |= (DMA_RWCTRL_READ_BNDRY_512 |
17152 				DMA_RWCTRL_WRITE_BNDRY_512);
17153 			break;
17154 		case 1024:
17155 		default:
17156 			val |= (DMA_RWCTRL_READ_BNDRY_1024 |
17157 				DMA_RWCTRL_WRITE_BNDRY_1024);
17158 			break;
17159 		}
17160 	}
17161 
17162 out:
17163 	return val;
17164 }
17165 
17166 static int tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma,
17167 			   int size, bool to_device)
17168 {
17169 	struct tg3_internal_buffer_desc test_desc;
17170 	u32 sram_dma_descs;
17171 	int i, ret;
17172 
17173 	sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
17174 
17175 	tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
17176 	tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
17177 	tw32(RDMAC_STATUS, 0);
17178 	tw32(WDMAC_STATUS, 0);
17179 
17180 	tw32(BUFMGR_MODE, 0);
17181 	tw32(FTQ_RESET, 0);
17182 
17183 	test_desc.addr_hi = ((u64) buf_dma) >> 32;
17184 	test_desc.addr_lo = buf_dma & 0xffffffff;
17185 	test_desc.nic_mbuf = 0x00002100;
17186 	test_desc.len = size;
17187 
17188 	/*
17189 	 * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
17190 	 * the *second* time the tg3 driver was getting loaded after an
17191 	 * initial scan.
17192 	 *
17193 	 * Broadcom tells me:
17194 	 *   ...the DMA engine is connected to the GRC block and a DMA
17195 	 *   reset may affect the GRC block in some unpredictable way...
17196 	 *   The behavior of resets to individual blocks has not been tested.
17197 	 *
17198 	 * Broadcom noted the GRC reset will also reset all sub-components.
17199 	 */
17200 	if (to_device) {
17201 		test_desc.cqid_sqid = (13 << 8) | 2;
17202 
17203 		tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
17204 		udelay(40);
17205 	} else {
17206 		test_desc.cqid_sqid = (16 << 8) | 7;
17207 
17208 		tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
17209 		udelay(40);
17210 	}
17211 	test_desc.flags = 0x00000005;
17212 
17213 	for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
17214 		u32 val;
17215 
17216 		val = *(((u32 *)&test_desc) + i);
17217 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
17218 				       sram_dma_descs + (i * sizeof(u32)));
17219 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
17220 	}
17221 	pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
17222 
17223 	if (to_device)
17224 		tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
17225 	else
17226 		tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
17227 
17228 	ret = -ENODEV;
17229 	for (i = 0; i < 40; i++) {
17230 		u32 val;
17231 
17232 		if (to_device)
17233 			val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
17234 		else
17235 			val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
17236 		if ((val & 0xffff) == sram_dma_descs) {
17237 			ret = 0;
17238 			break;
17239 		}
17240 
17241 		udelay(100);
17242 	}
17243 
17244 	return ret;
17245 }
17246 
17247 #define TEST_BUFFER_SIZE	0x2000
17248 
17249 static const struct pci_device_id tg3_dma_wait_state_chipsets[] = {
17250 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
17251 	{ },
17252 };
17253 
17254 static int tg3_test_dma(struct tg3 *tp)
17255 {
17256 	dma_addr_t buf_dma;
17257 	u32 *buf, saved_dma_rwctrl;
17258 	int ret = 0;
17259 
17260 	buf = dma_alloc_coherent(&tp->pdev->dev, TEST_BUFFER_SIZE,
17261 				 &buf_dma, GFP_KERNEL);
17262 	if (!buf) {
17263 		ret = -ENOMEM;
17264 		goto out_nofree;
17265 	}
17266 
17267 	tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
17268 			  (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
17269 
17270 	tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
17271 
17272 	if (tg3_flag(tp, 57765_PLUS))
17273 		goto out;
17274 
17275 	if (tg3_flag(tp, PCI_EXPRESS)) {
17276 		/* DMA read watermark not used on PCIE */
17277 		tp->dma_rwctrl |= 0x00180000;
17278 	} else if (!tg3_flag(tp, PCIX_MODE)) {
17279 		if (tg3_asic_rev(tp) == ASIC_REV_5705 ||
17280 		    tg3_asic_rev(tp) == ASIC_REV_5750)
17281 			tp->dma_rwctrl |= 0x003f0000;
17282 		else
17283 			tp->dma_rwctrl |= 0x003f000f;
17284 	} else {
17285 		if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
17286 		    tg3_asic_rev(tp) == ASIC_REV_5704) {
17287 			u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
17288 			u32 read_water = 0x7;
17289 
17290 			/* If the 5704 is behind the EPB bridge, we can
17291 			 * do the less restrictive ONE_DMA workaround for
17292 			 * better performance.
17293 			 */
17294 			if (tg3_flag(tp, 40BIT_DMA_BUG) &&
17295 			    tg3_asic_rev(tp) == ASIC_REV_5704)
17296 				tp->dma_rwctrl |= 0x8000;
17297 			else if (ccval == 0x6 || ccval == 0x7)
17298 				tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
17299 
17300 			if (tg3_asic_rev(tp) == ASIC_REV_5703)
17301 				read_water = 4;
17302 			/* Set bit 23 to enable PCIX hw bug fix */
17303 			tp->dma_rwctrl |=
17304 				(read_water << DMA_RWCTRL_READ_WATER_SHIFT) |
17305 				(0x3 << DMA_RWCTRL_WRITE_WATER_SHIFT) |
17306 				(1 << 23);
17307 		} else if (tg3_asic_rev(tp) == ASIC_REV_5780) {
17308 			/* 5780 always in PCIX mode */
17309 			tp->dma_rwctrl |= 0x00144000;
17310 		} else if (tg3_asic_rev(tp) == ASIC_REV_5714) {
17311 			/* 5714 always in PCIX mode */
17312 			tp->dma_rwctrl |= 0x00148000;
17313 		} else {
17314 			tp->dma_rwctrl |= 0x001b000f;
17315 		}
17316 	}
17317 	if (tg3_flag(tp, ONE_DMA_AT_ONCE))
17318 		tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
17319 
17320 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
17321 	    tg3_asic_rev(tp) == ASIC_REV_5704)
17322 		tp->dma_rwctrl &= 0xfffffff0;
17323 
17324 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
17325 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
17326 		/* Remove this if it causes problems for some boards. */
17327 		tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
17328 
17329 		/* On 5700/5701 chips, we need to set this bit.
17330 		 * Otherwise the chip will issue cacheline transactions
17331 		 * to streamable DMA memory with not all the byte
17332 		 * enables turned on.  This is an error on several
17333 		 * RISC PCI controllers, in particular sparc64.
17334 		 *
17335 		 * On 5703/5704 chips, this bit has been reassigned
17336 		 * a different meaning.  In particular, it is used
17337 		 * on those chips to enable a PCI-X workaround.
17338 		 */
17339 		tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
17340 	}
17341 
17342 	tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17343 
17344 
17345 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
17346 	    tg3_asic_rev(tp) != ASIC_REV_5701)
17347 		goto out;
17348 
17349 	/* It is best to perform DMA test with maximum write burst size
17350 	 * to expose the 5700/5701 write DMA bug.
17351 	 */
17352 	saved_dma_rwctrl = tp->dma_rwctrl;
17353 	tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17354 	tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17355 
17356 	while (1) {
17357 		u32 *p = buf, i;
17358 
17359 		for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
17360 			p[i] = i;
17361 
17362 		/* Send the buffer to the chip. */
17363 		ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, true);
17364 		if (ret) {
17365 			dev_err(&tp->pdev->dev,
17366 				"%s: Buffer write failed. err = %d\n",
17367 				__func__, ret);
17368 			break;
17369 		}
17370 
17371 		/* Now read it back. */
17372 		ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, false);
17373 		if (ret) {
17374 			dev_err(&tp->pdev->dev, "%s: Buffer read failed. "
17375 				"err = %d\n", __func__, ret);
17376 			break;
17377 		}
17378 
17379 		/* Verify it. */
17380 		for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
17381 			if (p[i] == i)
17382 				continue;
17383 
17384 			if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
17385 			    DMA_RWCTRL_WRITE_BNDRY_16) {
17386 				tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17387 				tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
17388 				tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17389 				break;
17390 			} else {
17391 				dev_err(&tp->pdev->dev,
17392 					"%s: Buffer corrupted on read back! "
17393 					"(%d != %d)\n", __func__, p[i], i);
17394 				ret = -ENODEV;
17395 				goto out;
17396 			}
17397 		}
17398 
17399 		if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
17400 			/* Success. */
17401 			ret = 0;
17402 			break;
17403 		}
17404 	}
17405 	if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
17406 	    DMA_RWCTRL_WRITE_BNDRY_16) {
17407 		/* DMA test passed without adjusting DMA boundary,
17408 		 * now look for chipsets that are known to expose the
17409 		 * DMA bug without failing the test.
17410 		 */
17411 		if (pci_dev_present(tg3_dma_wait_state_chipsets)) {
17412 			tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17413 			tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
17414 		} else {
17415 			/* Safe to use the calculated DMA boundary. */
17416 			tp->dma_rwctrl = saved_dma_rwctrl;
17417 		}
17418 
17419 		tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17420 	}
17421 
17422 out:
17423 	dma_free_coherent(&tp->pdev->dev, TEST_BUFFER_SIZE, buf, buf_dma);
17424 out_nofree:
17425 	return ret;
17426 }
17427 
17428 static void tg3_init_bufmgr_config(struct tg3 *tp)
17429 {
17430 	if (tg3_flag(tp, 57765_PLUS)) {
17431 		tp->bufmgr_config.mbuf_read_dma_low_water =
17432 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17433 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17434 			DEFAULT_MB_MACRX_LOW_WATER_57765;
17435 		tp->bufmgr_config.mbuf_high_water =
17436 			DEFAULT_MB_HIGH_WATER_57765;
17437 
17438 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17439 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17440 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17441 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO_57765;
17442 		tp->bufmgr_config.mbuf_high_water_jumbo =
17443 			DEFAULT_MB_HIGH_WATER_JUMBO_57765;
17444 	} else if (tg3_flag(tp, 5705_PLUS)) {
17445 		tp->bufmgr_config.mbuf_read_dma_low_water =
17446 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17447 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17448 			DEFAULT_MB_MACRX_LOW_WATER_5705;
17449 		tp->bufmgr_config.mbuf_high_water =
17450 			DEFAULT_MB_HIGH_WATER_5705;
17451 		if (tg3_asic_rev(tp) == ASIC_REV_5906) {
17452 			tp->bufmgr_config.mbuf_mac_rx_low_water =
17453 				DEFAULT_MB_MACRX_LOW_WATER_5906;
17454 			tp->bufmgr_config.mbuf_high_water =
17455 				DEFAULT_MB_HIGH_WATER_5906;
17456 		}
17457 
17458 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17459 			DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
17460 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17461 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
17462 		tp->bufmgr_config.mbuf_high_water_jumbo =
17463 			DEFAULT_MB_HIGH_WATER_JUMBO_5780;
17464 	} else {
17465 		tp->bufmgr_config.mbuf_read_dma_low_water =
17466 			DEFAULT_MB_RDMA_LOW_WATER;
17467 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17468 			DEFAULT_MB_MACRX_LOW_WATER;
17469 		tp->bufmgr_config.mbuf_high_water =
17470 			DEFAULT_MB_HIGH_WATER;
17471 
17472 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17473 			DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
17474 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17475 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
17476 		tp->bufmgr_config.mbuf_high_water_jumbo =
17477 			DEFAULT_MB_HIGH_WATER_JUMBO;
17478 	}
17479 
17480 	tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
17481 	tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
17482 }
17483 
17484 static char *tg3_phy_string(struct tg3 *tp)
17485 {
17486 	switch (tp->phy_id & TG3_PHY_ID_MASK) {
17487 	case TG3_PHY_ID_BCM5400:	return "5400";
17488 	case TG3_PHY_ID_BCM5401:	return "5401";
17489 	case TG3_PHY_ID_BCM5411:	return "5411";
17490 	case TG3_PHY_ID_BCM5701:	return "5701";
17491 	case TG3_PHY_ID_BCM5703:	return "5703";
17492 	case TG3_PHY_ID_BCM5704:	return "5704";
17493 	case TG3_PHY_ID_BCM5705:	return "5705";
17494 	case TG3_PHY_ID_BCM5750:	return "5750";
17495 	case TG3_PHY_ID_BCM5752:	return "5752";
17496 	case TG3_PHY_ID_BCM5714:	return "5714";
17497 	case TG3_PHY_ID_BCM5780:	return "5780";
17498 	case TG3_PHY_ID_BCM5755:	return "5755";
17499 	case TG3_PHY_ID_BCM5787:	return "5787";
17500 	case TG3_PHY_ID_BCM5784:	return "5784";
17501 	case TG3_PHY_ID_BCM5756:	return "5722/5756";
17502 	case TG3_PHY_ID_BCM5906:	return "5906";
17503 	case TG3_PHY_ID_BCM5761:	return "5761";
17504 	case TG3_PHY_ID_BCM5718C:	return "5718C";
17505 	case TG3_PHY_ID_BCM5718S:	return "5718S";
17506 	case TG3_PHY_ID_BCM57765:	return "57765";
17507 	case TG3_PHY_ID_BCM5719C:	return "5719C";
17508 	case TG3_PHY_ID_BCM5720C:	return "5720C";
17509 	case TG3_PHY_ID_BCM5762:	return "5762C";
17510 	case TG3_PHY_ID_BCM8002:	return "8002/serdes";
17511 	case 0:			return "serdes";
17512 	default:		return "unknown";
17513 	}
17514 }
17515 
17516 static char *tg3_bus_string(struct tg3 *tp, char *str)
17517 {
17518 	if (tg3_flag(tp, PCI_EXPRESS)) {
17519 		strcpy(str, "PCI Express");
17520 		return str;
17521 	} else if (tg3_flag(tp, PCIX_MODE)) {
17522 		u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
17523 
17524 		strcpy(str, "PCIX:");
17525 
17526 		if ((clock_ctrl == 7) ||
17527 		    ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
17528 		     GRC_MISC_CFG_BOARD_ID_5704CIOBE))
17529 			strcat(str, "133MHz");
17530 		else if (clock_ctrl == 0)
17531 			strcat(str, "33MHz");
17532 		else if (clock_ctrl == 2)
17533 			strcat(str, "50MHz");
17534 		else if (clock_ctrl == 4)
17535 			strcat(str, "66MHz");
17536 		else if (clock_ctrl == 6)
17537 			strcat(str, "100MHz");
17538 	} else {
17539 		strcpy(str, "PCI:");
17540 		if (tg3_flag(tp, PCI_HIGH_SPEED))
17541 			strcat(str, "66MHz");
17542 		else
17543 			strcat(str, "33MHz");
17544 	}
17545 	if (tg3_flag(tp, PCI_32BIT))
17546 		strcat(str, ":32-bit");
17547 	else
17548 		strcat(str, ":64-bit");
17549 	return str;
17550 }
17551 
17552 static void tg3_init_coal(struct tg3 *tp)
17553 {
17554 	struct ethtool_coalesce *ec = &tp->coal;
17555 
17556 	memset(ec, 0, sizeof(*ec));
17557 	ec->cmd = ETHTOOL_GCOALESCE;
17558 	ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
17559 	ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
17560 	ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
17561 	ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
17562 	ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
17563 	ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
17564 	ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
17565 	ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
17566 	ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
17567 
17568 	if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
17569 				 HOSTCC_MODE_CLRTICK_TXBD)) {
17570 		ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
17571 		ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
17572 		ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
17573 		ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
17574 	}
17575 
17576 	if (tg3_flag(tp, 5705_PLUS)) {
17577 		ec->rx_coalesce_usecs_irq = 0;
17578 		ec->tx_coalesce_usecs_irq = 0;
17579 		ec->stats_block_coalesce_usecs = 0;
17580 	}
17581 }
17582 
17583 static int tg3_init_one(struct pci_dev *pdev,
17584 				  const struct pci_device_id *ent)
17585 {
17586 	struct net_device *dev;
17587 	struct tg3 *tp;
17588 	int i, err;
17589 	u32 sndmbx, rcvmbx, intmbx;
17590 	char str[40];
17591 	u64 dma_mask, persist_dma_mask;
17592 	netdev_features_t features = 0;
17593 
17594 	printk_once(KERN_INFO "%s\n", version);
17595 
17596 	err = pci_enable_device(pdev);
17597 	if (err) {
17598 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
17599 		return err;
17600 	}
17601 
17602 	err = pci_request_regions(pdev, DRV_MODULE_NAME);
17603 	if (err) {
17604 		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
17605 		goto err_out_disable_pdev;
17606 	}
17607 
17608 	pci_set_master(pdev);
17609 
17610 	dev = alloc_etherdev_mq(sizeof(*tp), TG3_IRQ_MAX_VECS);
17611 	if (!dev) {
17612 		err = -ENOMEM;
17613 		goto err_out_free_res;
17614 	}
17615 
17616 	SET_NETDEV_DEV(dev, &pdev->dev);
17617 
17618 	tp = netdev_priv(dev);
17619 	tp->pdev = pdev;
17620 	tp->dev = dev;
17621 	tp->rx_mode = TG3_DEF_RX_MODE;
17622 	tp->tx_mode = TG3_DEF_TX_MODE;
17623 	tp->irq_sync = 1;
17624 	tp->pcierr_recovery = false;
17625 
17626 	if (tg3_debug > 0)
17627 		tp->msg_enable = tg3_debug;
17628 	else
17629 		tp->msg_enable = TG3_DEF_MSG_ENABLE;
17630 
17631 	if (pdev_is_ssb_gige_core(pdev)) {
17632 		tg3_flag_set(tp, IS_SSB_CORE);
17633 		if (ssb_gige_must_flush_posted_writes(pdev))
17634 			tg3_flag_set(tp, FLUSH_POSTED_WRITES);
17635 		if (ssb_gige_one_dma_at_once(pdev))
17636 			tg3_flag_set(tp, ONE_DMA_AT_ONCE);
17637 		if (ssb_gige_have_roboswitch(pdev)) {
17638 			tg3_flag_set(tp, USE_PHYLIB);
17639 			tg3_flag_set(tp, ROBOSWITCH);
17640 		}
17641 		if (ssb_gige_is_rgmii(pdev))
17642 			tg3_flag_set(tp, RGMII_MODE);
17643 	}
17644 
17645 	/* The word/byte swap controls here control register access byte
17646 	 * swapping.  DMA data byte swapping is controlled in the GRC_MODE
17647 	 * setting below.
17648 	 */
17649 	tp->misc_host_ctrl =
17650 		MISC_HOST_CTRL_MASK_PCI_INT |
17651 		MISC_HOST_CTRL_WORD_SWAP |
17652 		MISC_HOST_CTRL_INDIR_ACCESS |
17653 		MISC_HOST_CTRL_PCISTATE_RW;
17654 
17655 	/* The NONFRM (non-frame) byte/word swap controls take effect
17656 	 * on descriptor entries, anything which isn't packet data.
17657 	 *
17658 	 * The StrongARM chips on the board (one for tx, one for rx)
17659 	 * are running in big-endian mode.
17660 	 */
17661 	tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
17662 			GRC_MODE_WSWAP_NONFRM_DATA);
17663 #ifdef __BIG_ENDIAN
17664 	tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
17665 #endif
17666 	spin_lock_init(&tp->lock);
17667 	spin_lock_init(&tp->indirect_lock);
17668 	INIT_WORK(&tp->reset_task, tg3_reset_task);
17669 
17670 	tp->regs = pci_ioremap_bar(pdev, BAR_0);
17671 	if (!tp->regs) {
17672 		dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
17673 		err = -ENOMEM;
17674 		goto err_out_free_dev;
17675 	}
17676 
17677 	if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
17678 	    tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761E ||
17679 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S ||
17680 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761SE ||
17681 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
17682 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C ||
17683 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718 ||
17684 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5719 ||
17685 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5720 ||
17686 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57767 ||
17687 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57764 ||
17688 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5762 ||
17689 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725 ||
17690 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5727 ||
17691 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57787) {
17692 		tg3_flag_set(tp, ENABLE_APE);
17693 		tp->aperegs = pci_ioremap_bar(pdev, BAR_2);
17694 		if (!tp->aperegs) {
17695 			dev_err(&pdev->dev,
17696 				"Cannot map APE registers, aborting\n");
17697 			err = -ENOMEM;
17698 			goto err_out_iounmap;
17699 		}
17700 	}
17701 
17702 	tp->rx_pending = TG3_DEF_RX_RING_PENDING;
17703 	tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
17704 
17705 	dev->ethtool_ops = &tg3_ethtool_ops;
17706 	dev->watchdog_timeo = TG3_TX_TIMEOUT;
17707 	dev->netdev_ops = &tg3_netdev_ops;
17708 	dev->irq = pdev->irq;
17709 
17710 	err = tg3_get_invariants(tp, ent);
17711 	if (err) {
17712 		dev_err(&pdev->dev,
17713 			"Problem fetching invariants of chip, aborting\n");
17714 		goto err_out_apeunmap;
17715 	}
17716 
17717 	/* The EPB bridge inside 5714, 5715, and 5780 and any
17718 	 * device behind the EPB cannot support DMA addresses > 40-bit.
17719 	 * On 64-bit systems with IOMMU, use 40-bit dma_mask.
17720 	 * On 64-bit systems without IOMMU, use 64-bit dma_mask and
17721 	 * do DMA address check in tg3_start_xmit().
17722 	 */
17723 	if (tg3_flag(tp, IS_5788))
17724 		persist_dma_mask = dma_mask = DMA_BIT_MASK(32);
17725 	else if (tg3_flag(tp, 40BIT_DMA_BUG)) {
17726 		persist_dma_mask = dma_mask = DMA_BIT_MASK(40);
17727 #ifdef CONFIG_HIGHMEM
17728 		dma_mask = DMA_BIT_MASK(64);
17729 #endif
17730 	} else
17731 		persist_dma_mask = dma_mask = DMA_BIT_MASK(64);
17732 
17733 	/* Configure DMA attributes. */
17734 	if (dma_mask > DMA_BIT_MASK(32)) {
17735 		err = pci_set_dma_mask(pdev, dma_mask);
17736 		if (!err) {
17737 			features |= NETIF_F_HIGHDMA;
17738 			err = pci_set_consistent_dma_mask(pdev,
17739 							  persist_dma_mask);
17740 			if (err < 0) {
17741 				dev_err(&pdev->dev, "Unable to obtain 64 bit "
17742 					"DMA for consistent allocations\n");
17743 				goto err_out_apeunmap;
17744 			}
17745 		}
17746 	}
17747 	if (err || dma_mask == DMA_BIT_MASK(32)) {
17748 		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
17749 		if (err) {
17750 			dev_err(&pdev->dev,
17751 				"No usable DMA configuration, aborting\n");
17752 			goto err_out_apeunmap;
17753 		}
17754 	}
17755 
17756 	tg3_init_bufmgr_config(tp);
17757 
17758 	/* 5700 B0 chips do not support checksumming correctly due
17759 	 * to hardware bugs.
17760 	 */
17761 	if (tg3_chip_rev_id(tp) != CHIPREV_ID_5700_B0) {
17762 		features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
17763 
17764 		if (tg3_flag(tp, 5755_PLUS))
17765 			features |= NETIF_F_IPV6_CSUM;
17766 	}
17767 
17768 	/* TSO is on by default on chips that support hardware TSO.
17769 	 * Firmware TSO on older chips gives lower performance, so it
17770 	 * is off by default, but can be enabled using ethtool.
17771 	 */
17772 	if ((tg3_flag(tp, HW_TSO_1) ||
17773 	     tg3_flag(tp, HW_TSO_2) ||
17774 	     tg3_flag(tp, HW_TSO_3)) &&
17775 	    (features & NETIF_F_IP_CSUM))
17776 		features |= NETIF_F_TSO;
17777 	if (tg3_flag(tp, HW_TSO_2) || tg3_flag(tp, HW_TSO_3)) {
17778 		if (features & NETIF_F_IPV6_CSUM)
17779 			features |= NETIF_F_TSO6;
17780 		if (tg3_flag(tp, HW_TSO_3) ||
17781 		    tg3_asic_rev(tp) == ASIC_REV_5761 ||
17782 		    (tg3_asic_rev(tp) == ASIC_REV_5784 &&
17783 		     tg3_chip_rev(tp) != CHIPREV_5784_AX) ||
17784 		    tg3_asic_rev(tp) == ASIC_REV_5785 ||
17785 		    tg3_asic_rev(tp) == ASIC_REV_57780)
17786 			features |= NETIF_F_TSO_ECN;
17787 	}
17788 
17789 	dev->features |= features | NETIF_F_HW_VLAN_CTAG_TX |
17790 			 NETIF_F_HW_VLAN_CTAG_RX;
17791 	dev->vlan_features |= features;
17792 
17793 	/*
17794 	 * Add loopback capability only for a subset of devices that support
17795 	 * MAC-LOOPBACK. Eventually this need to be enhanced to allow INT-PHY
17796 	 * loopback for the remaining devices.
17797 	 */
17798 	if (tg3_asic_rev(tp) != ASIC_REV_5780 &&
17799 	    !tg3_flag(tp, CPMU_PRESENT))
17800 		/* Add the loopback capability */
17801 		features |= NETIF_F_LOOPBACK;
17802 
17803 	dev->hw_features |= features;
17804 	dev->priv_flags |= IFF_UNICAST_FLT;
17805 
17806 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A1 &&
17807 	    !tg3_flag(tp, TSO_CAPABLE) &&
17808 	    !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
17809 		tg3_flag_set(tp, MAX_RXPEND_64);
17810 		tp->rx_pending = 63;
17811 	}
17812 
17813 	err = tg3_get_device_address(tp);
17814 	if (err) {
17815 		dev_err(&pdev->dev,
17816 			"Could not obtain valid ethernet address, aborting\n");
17817 		goto err_out_apeunmap;
17818 	}
17819 
17820 	intmbx = MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW;
17821 	rcvmbx = MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW;
17822 	sndmbx = MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW;
17823 	for (i = 0; i < tp->irq_max; i++) {
17824 		struct tg3_napi *tnapi = &tp->napi[i];
17825 
17826 		tnapi->tp = tp;
17827 		tnapi->tx_pending = TG3_DEF_TX_RING_PENDING;
17828 
17829 		tnapi->int_mbox = intmbx;
17830 		if (i <= 4)
17831 			intmbx += 0x8;
17832 		else
17833 			intmbx += 0x4;
17834 
17835 		tnapi->consmbox = rcvmbx;
17836 		tnapi->prodmbox = sndmbx;
17837 
17838 		if (i)
17839 			tnapi->coal_now = HOSTCC_MODE_COAL_VEC1_NOW << (i - 1);
17840 		else
17841 			tnapi->coal_now = HOSTCC_MODE_NOW;
17842 
17843 		if (!tg3_flag(tp, SUPPORT_MSIX))
17844 			break;
17845 
17846 		/*
17847 		 * If we support MSIX, we'll be using RSS.  If we're using
17848 		 * RSS, the first vector only handles link interrupts and the
17849 		 * remaining vectors handle rx and tx interrupts.  Reuse the
17850 		 * mailbox values for the next iteration.  The values we setup
17851 		 * above are still useful for the single vectored mode.
17852 		 */
17853 		if (!i)
17854 			continue;
17855 
17856 		rcvmbx += 0x8;
17857 
17858 		if (sndmbx & 0x4)
17859 			sndmbx -= 0x4;
17860 		else
17861 			sndmbx += 0xc;
17862 	}
17863 
17864 	/*
17865 	 * Reset chip in case UNDI or EFI driver did not shutdown
17866 	 * DMA self test will enable WDMAC and we'll see (spurious)
17867 	 * pending DMA on the PCI bus at that point.
17868 	 */
17869 	if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
17870 	    (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
17871 		tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
17872 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
17873 	}
17874 
17875 	err = tg3_test_dma(tp);
17876 	if (err) {
17877 		dev_err(&pdev->dev, "DMA engine test failed, aborting\n");
17878 		goto err_out_apeunmap;
17879 	}
17880 
17881 	tg3_init_coal(tp);
17882 
17883 	pci_set_drvdata(pdev, dev);
17884 
17885 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
17886 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
17887 	    tg3_asic_rev(tp) == ASIC_REV_5762)
17888 		tg3_flag_set(tp, PTP_CAPABLE);
17889 
17890 	tg3_timer_init(tp);
17891 
17892 	tg3_carrier_off(tp);
17893 
17894 	err = register_netdev(dev);
17895 	if (err) {
17896 		dev_err(&pdev->dev, "Cannot register net device, aborting\n");
17897 		goto err_out_apeunmap;
17898 	}
17899 
17900 	netdev_info(dev, "Tigon3 [partno(%s) rev %04x] (%s) MAC address %pM\n",
17901 		    tp->board_part_number,
17902 		    tg3_chip_rev_id(tp),
17903 		    tg3_bus_string(tp, str),
17904 		    dev->dev_addr);
17905 
17906 	if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
17907 		struct phy_device *phydev;
17908 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
17909 		netdev_info(dev,
17910 			    "attached PHY driver [%s] (mii_bus:phy_addr=%s)\n",
17911 			    phydev->drv->name, dev_name(&phydev->dev));
17912 	} else {
17913 		char *ethtype;
17914 
17915 		if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
17916 			ethtype = "10/100Base-TX";
17917 		else if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
17918 			ethtype = "1000Base-SX";
17919 		else
17920 			ethtype = "10/100/1000Base-T";
17921 
17922 		netdev_info(dev, "attached PHY is %s (%s Ethernet) "
17923 			    "(WireSpeed[%d], EEE[%d])\n",
17924 			    tg3_phy_string(tp), ethtype,
17925 			    (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED) == 0,
17926 			    (tp->phy_flags & TG3_PHYFLG_EEE_CAP) != 0);
17927 	}
17928 
17929 	netdev_info(dev, "RXcsums[%d] LinkChgREG[%d] MIirq[%d] ASF[%d] TSOcap[%d]\n",
17930 		    (dev->features & NETIF_F_RXCSUM) != 0,
17931 		    tg3_flag(tp, USE_LINKCHG_REG) != 0,
17932 		    (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT) != 0,
17933 		    tg3_flag(tp, ENABLE_ASF) != 0,
17934 		    tg3_flag(tp, TSO_CAPABLE) != 0);
17935 	netdev_info(dev, "dma_rwctrl[%08x] dma_mask[%d-bit]\n",
17936 		    tp->dma_rwctrl,
17937 		    pdev->dma_mask == DMA_BIT_MASK(32) ? 32 :
17938 		    ((u64)pdev->dma_mask) == DMA_BIT_MASK(40) ? 40 : 64);
17939 
17940 	pci_save_state(pdev);
17941 
17942 	return 0;
17943 
17944 err_out_apeunmap:
17945 	if (tp->aperegs) {
17946 		iounmap(tp->aperegs);
17947 		tp->aperegs = NULL;
17948 	}
17949 
17950 err_out_iounmap:
17951 	if (tp->regs) {
17952 		iounmap(tp->regs);
17953 		tp->regs = NULL;
17954 	}
17955 
17956 err_out_free_dev:
17957 	free_netdev(dev);
17958 
17959 err_out_free_res:
17960 	pci_release_regions(pdev);
17961 
17962 err_out_disable_pdev:
17963 	if (pci_is_enabled(pdev))
17964 		pci_disable_device(pdev);
17965 	return err;
17966 }
17967 
17968 static void tg3_remove_one(struct pci_dev *pdev)
17969 {
17970 	struct net_device *dev = pci_get_drvdata(pdev);
17971 
17972 	if (dev) {
17973 		struct tg3 *tp = netdev_priv(dev);
17974 
17975 		release_firmware(tp->fw);
17976 
17977 		tg3_reset_task_cancel(tp);
17978 
17979 		if (tg3_flag(tp, USE_PHYLIB)) {
17980 			tg3_phy_fini(tp);
17981 			tg3_mdio_fini(tp);
17982 		}
17983 
17984 		unregister_netdev(dev);
17985 		if (tp->aperegs) {
17986 			iounmap(tp->aperegs);
17987 			tp->aperegs = NULL;
17988 		}
17989 		if (tp->regs) {
17990 			iounmap(tp->regs);
17991 			tp->regs = NULL;
17992 		}
17993 		free_netdev(dev);
17994 		pci_release_regions(pdev);
17995 		pci_disable_device(pdev);
17996 	}
17997 }
17998 
17999 #ifdef CONFIG_PM_SLEEP
18000 static int tg3_suspend(struct device *device)
18001 {
18002 	struct pci_dev *pdev = to_pci_dev(device);
18003 	struct net_device *dev = pci_get_drvdata(pdev);
18004 	struct tg3 *tp = netdev_priv(dev);
18005 	int err = 0;
18006 
18007 	rtnl_lock();
18008 
18009 	if (!netif_running(dev))
18010 		goto unlock;
18011 
18012 	tg3_reset_task_cancel(tp);
18013 	tg3_phy_stop(tp);
18014 	tg3_netif_stop(tp);
18015 
18016 	tg3_timer_stop(tp);
18017 
18018 	tg3_full_lock(tp, 1);
18019 	tg3_disable_ints(tp);
18020 	tg3_full_unlock(tp);
18021 
18022 	netif_device_detach(dev);
18023 
18024 	tg3_full_lock(tp, 0);
18025 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
18026 	tg3_flag_clear(tp, INIT_COMPLETE);
18027 	tg3_full_unlock(tp);
18028 
18029 	err = tg3_power_down_prepare(tp);
18030 	if (err) {
18031 		int err2;
18032 
18033 		tg3_full_lock(tp, 0);
18034 
18035 		tg3_flag_set(tp, INIT_COMPLETE);
18036 		err2 = tg3_restart_hw(tp, true);
18037 		if (err2)
18038 			goto out;
18039 
18040 		tg3_timer_start(tp);
18041 
18042 		netif_device_attach(dev);
18043 		tg3_netif_start(tp);
18044 
18045 out:
18046 		tg3_full_unlock(tp);
18047 
18048 		if (!err2)
18049 			tg3_phy_start(tp);
18050 	}
18051 
18052 unlock:
18053 	rtnl_unlock();
18054 	return err;
18055 }
18056 
18057 static int tg3_resume(struct device *device)
18058 {
18059 	struct pci_dev *pdev = to_pci_dev(device);
18060 	struct net_device *dev = pci_get_drvdata(pdev);
18061 	struct tg3 *tp = netdev_priv(dev);
18062 	int err = 0;
18063 
18064 	rtnl_lock();
18065 
18066 	if (!netif_running(dev))
18067 		goto unlock;
18068 
18069 	netif_device_attach(dev);
18070 
18071 	tg3_full_lock(tp, 0);
18072 
18073 	tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
18074 
18075 	tg3_flag_set(tp, INIT_COMPLETE);
18076 	err = tg3_restart_hw(tp,
18077 			     !(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN));
18078 	if (err)
18079 		goto out;
18080 
18081 	tg3_timer_start(tp);
18082 
18083 	tg3_netif_start(tp);
18084 
18085 out:
18086 	tg3_full_unlock(tp);
18087 
18088 	if (!err)
18089 		tg3_phy_start(tp);
18090 
18091 unlock:
18092 	rtnl_unlock();
18093 	return err;
18094 }
18095 #endif /* CONFIG_PM_SLEEP */
18096 
18097 static SIMPLE_DEV_PM_OPS(tg3_pm_ops, tg3_suspend, tg3_resume);
18098 
18099 static void tg3_shutdown(struct pci_dev *pdev)
18100 {
18101 	struct net_device *dev = pci_get_drvdata(pdev);
18102 	struct tg3 *tp = netdev_priv(dev);
18103 
18104 	rtnl_lock();
18105 	netif_device_detach(dev);
18106 
18107 	if (netif_running(dev))
18108 		dev_close(dev);
18109 
18110 	if (system_state == SYSTEM_POWER_OFF)
18111 		tg3_power_down(tp);
18112 
18113 	rtnl_unlock();
18114 }
18115 
18116 /**
18117  * tg3_io_error_detected - called when PCI error is detected
18118  * @pdev: Pointer to PCI device
18119  * @state: The current pci connection state
18120  *
18121  * This function is called after a PCI bus error affecting
18122  * this device has been detected.
18123  */
18124 static pci_ers_result_t tg3_io_error_detected(struct pci_dev *pdev,
18125 					      pci_channel_state_t state)
18126 {
18127 	struct net_device *netdev = pci_get_drvdata(pdev);
18128 	struct tg3 *tp = netdev_priv(netdev);
18129 	pci_ers_result_t err = PCI_ERS_RESULT_NEED_RESET;
18130 
18131 	netdev_info(netdev, "PCI I/O error detected\n");
18132 
18133 	rtnl_lock();
18134 
18135 	tp->pcierr_recovery = true;
18136 
18137 	/* We probably don't have netdev yet */
18138 	if (!netdev || !netif_running(netdev))
18139 		goto done;
18140 
18141 	tg3_phy_stop(tp);
18142 
18143 	tg3_netif_stop(tp);
18144 
18145 	tg3_timer_stop(tp);
18146 
18147 	/* Want to make sure that the reset task doesn't run */
18148 	tg3_reset_task_cancel(tp);
18149 
18150 	netif_device_detach(netdev);
18151 
18152 	/* Clean up software state, even if MMIO is blocked */
18153 	tg3_full_lock(tp, 0);
18154 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
18155 	tg3_full_unlock(tp);
18156 
18157 done:
18158 	if (state == pci_channel_io_perm_failure) {
18159 		if (netdev) {
18160 			tg3_napi_enable(tp);
18161 			dev_close(netdev);
18162 		}
18163 		err = PCI_ERS_RESULT_DISCONNECT;
18164 	} else {
18165 		pci_disable_device(pdev);
18166 	}
18167 
18168 	rtnl_unlock();
18169 
18170 	return err;
18171 }
18172 
18173 /**
18174  * tg3_io_slot_reset - called after the pci bus has been reset.
18175  * @pdev: Pointer to PCI device
18176  *
18177  * Restart the card from scratch, as if from a cold-boot.
18178  * At this point, the card has exprienced a hard reset,
18179  * followed by fixups by BIOS, and has its config space
18180  * set up identically to what it was at cold boot.
18181  */
18182 static pci_ers_result_t tg3_io_slot_reset(struct pci_dev *pdev)
18183 {
18184 	struct net_device *netdev = pci_get_drvdata(pdev);
18185 	struct tg3 *tp = netdev_priv(netdev);
18186 	pci_ers_result_t rc = PCI_ERS_RESULT_DISCONNECT;
18187 	int err;
18188 
18189 	rtnl_lock();
18190 
18191 	if (pci_enable_device(pdev)) {
18192 		dev_err(&pdev->dev,
18193 			"Cannot re-enable PCI device after reset.\n");
18194 		goto done;
18195 	}
18196 
18197 	pci_set_master(pdev);
18198 	pci_restore_state(pdev);
18199 	pci_save_state(pdev);
18200 
18201 	if (!netdev || !netif_running(netdev)) {
18202 		rc = PCI_ERS_RESULT_RECOVERED;
18203 		goto done;
18204 	}
18205 
18206 	err = tg3_power_up(tp);
18207 	if (err)
18208 		goto done;
18209 
18210 	rc = PCI_ERS_RESULT_RECOVERED;
18211 
18212 done:
18213 	if (rc != PCI_ERS_RESULT_RECOVERED && netdev && netif_running(netdev)) {
18214 		tg3_napi_enable(tp);
18215 		dev_close(netdev);
18216 	}
18217 	rtnl_unlock();
18218 
18219 	return rc;
18220 }
18221 
18222 /**
18223  * tg3_io_resume - called when traffic can start flowing again.
18224  * @pdev: Pointer to PCI device
18225  *
18226  * This callback is called when the error recovery driver tells
18227  * us that its OK to resume normal operation.
18228  */
18229 static void tg3_io_resume(struct pci_dev *pdev)
18230 {
18231 	struct net_device *netdev = pci_get_drvdata(pdev);
18232 	struct tg3 *tp = netdev_priv(netdev);
18233 	int err;
18234 
18235 	rtnl_lock();
18236 
18237 	if (!netif_running(netdev))
18238 		goto done;
18239 
18240 	tg3_full_lock(tp, 0);
18241 	tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
18242 	tg3_flag_set(tp, INIT_COMPLETE);
18243 	err = tg3_restart_hw(tp, true);
18244 	if (err) {
18245 		tg3_full_unlock(tp);
18246 		netdev_err(netdev, "Cannot restart hardware after reset.\n");
18247 		goto done;
18248 	}
18249 
18250 	netif_device_attach(netdev);
18251 
18252 	tg3_timer_start(tp);
18253 
18254 	tg3_netif_start(tp);
18255 
18256 	tg3_full_unlock(tp);
18257 
18258 	tg3_phy_start(tp);
18259 
18260 done:
18261 	tp->pcierr_recovery = false;
18262 	rtnl_unlock();
18263 }
18264 
18265 static const struct pci_error_handlers tg3_err_handler = {
18266 	.error_detected	= tg3_io_error_detected,
18267 	.slot_reset	= tg3_io_slot_reset,
18268 	.resume		= tg3_io_resume
18269 };
18270 
18271 static struct pci_driver tg3_driver = {
18272 	.name		= DRV_MODULE_NAME,
18273 	.id_table	= tg3_pci_tbl,
18274 	.probe		= tg3_init_one,
18275 	.remove		= tg3_remove_one,
18276 	.err_handler	= &tg3_err_handler,
18277 	.driver.pm	= &tg3_pm_ops,
18278 	.shutdown	= tg3_shutdown,
18279 };
18280 
18281 module_pci_driver(tg3_driver);
18282