1 /*
2  * tg3.c: Broadcom Tigon3 ethernet driver.
3  *
4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
6  * Copyright (C) 2004 Sun Microsystems Inc.
7  * Copyright (C) 2005-2013 Broadcom Corporation.
8  *
9  * Firmware is:
10  *	Derived from proprietary unpublished source code,
11  *	Copyright (C) 2000-2003 Broadcom Corporation.
12  *
13  *	Permission is hereby granted for the distribution of this firmware
14  *	data in hexadecimal or equivalent format, provided this copyright
15  *	notice is accompanying it.
16  */
17 
18 
19 #include <linux/module.h>
20 #include <linux/moduleparam.h>
21 #include <linux/stringify.h>
22 #include <linux/kernel.h>
23 #include <linux/types.h>
24 #include <linux/compiler.h>
25 #include <linux/slab.h>
26 #include <linux/delay.h>
27 #include <linux/in.h>
28 #include <linux/interrupt.h>
29 #include <linux/ioport.h>
30 #include <linux/pci.h>
31 #include <linux/netdevice.h>
32 #include <linux/etherdevice.h>
33 #include <linux/skbuff.h>
34 #include <linux/ethtool.h>
35 #include <linux/mdio.h>
36 #include <linux/mii.h>
37 #include <linux/phy.h>
38 #include <linux/brcmphy.h>
39 #include <linux/if.h>
40 #include <linux/if_vlan.h>
41 #include <linux/ip.h>
42 #include <linux/tcp.h>
43 #include <linux/workqueue.h>
44 #include <linux/prefetch.h>
45 #include <linux/dma-mapping.h>
46 #include <linux/firmware.h>
47 #include <linux/ssb/ssb_driver_gige.h>
48 #include <linux/hwmon.h>
49 #include <linux/hwmon-sysfs.h>
50 
51 #include <net/checksum.h>
52 #include <net/ip.h>
53 
54 #include <linux/io.h>
55 #include <asm/byteorder.h>
56 #include <linux/uaccess.h>
57 
58 #include <uapi/linux/net_tstamp.h>
59 #include <linux/ptp_clock_kernel.h>
60 
61 #ifdef CONFIG_SPARC
62 #include <asm/idprom.h>
63 #include <asm/prom.h>
64 #endif
65 
66 #define BAR_0	0
67 #define BAR_2	2
68 
69 #include "tg3.h"
70 
71 /* Functions & macros to verify TG3_FLAGS types */
72 
73 static inline int _tg3_flag(enum TG3_FLAGS flag, unsigned long *bits)
74 {
75 	return test_bit(flag, bits);
76 }
77 
78 static inline void _tg3_flag_set(enum TG3_FLAGS flag, unsigned long *bits)
79 {
80 	set_bit(flag, bits);
81 }
82 
83 static inline void _tg3_flag_clear(enum TG3_FLAGS flag, unsigned long *bits)
84 {
85 	clear_bit(flag, bits);
86 }
87 
88 #define tg3_flag(tp, flag)				\
89 	_tg3_flag(TG3_FLAG_##flag, (tp)->tg3_flags)
90 #define tg3_flag_set(tp, flag)				\
91 	_tg3_flag_set(TG3_FLAG_##flag, (tp)->tg3_flags)
92 #define tg3_flag_clear(tp, flag)			\
93 	_tg3_flag_clear(TG3_FLAG_##flag, (tp)->tg3_flags)
94 
95 #define DRV_MODULE_NAME		"tg3"
96 #define TG3_MAJ_NUM			3
97 #define TG3_MIN_NUM			136
98 #define DRV_MODULE_VERSION	\
99 	__stringify(TG3_MAJ_NUM) "." __stringify(TG3_MIN_NUM)
100 #define DRV_MODULE_RELDATE	"Jan 03, 2014"
101 
102 #define RESET_KIND_SHUTDOWN	0
103 #define RESET_KIND_INIT		1
104 #define RESET_KIND_SUSPEND	2
105 
106 #define TG3_DEF_RX_MODE		0
107 #define TG3_DEF_TX_MODE		0
108 #define TG3_DEF_MSG_ENABLE	  \
109 	(NETIF_MSG_DRV		| \
110 	 NETIF_MSG_PROBE	| \
111 	 NETIF_MSG_LINK		| \
112 	 NETIF_MSG_TIMER	| \
113 	 NETIF_MSG_IFDOWN	| \
114 	 NETIF_MSG_IFUP		| \
115 	 NETIF_MSG_RX_ERR	| \
116 	 NETIF_MSG_TX_ERR)
117 
118 #define TG3_GRC_LCLCTL_PWRSW_DELAY	100
119 
120 /* length of time before we decide the hardware is borked,
121  * and dev->tx_timeout() should be called to fix the problem
122  */
123 
124 #define TG3_TX_TIMEOUT			(5 * HZ)
125 
126 /* hardware minimum and maximum for a single frame's data payload */
127 #define TG3_MIN_MTU			60
128 #define TG3_MAX_MTU(tp)	\
129 	(tg3_flag(tp, JUMBO_CAPABLE) ? 9000 : 1500)
130 
131 /* These numbers seem to be hard coded in the NIC firmware somehow.
132  * You can't change the ring sizes, but you can change where you place
133  * them in the NIC onboard memory.
134  */
135 #define TG3_RX_STD_RING_SIZE(tp) \
136 	(tg3_flag(tp, LRG_PROD_RING_CAP) ? \
137 	 TG3_RX_STD_MAX_SIZE_5717 : TG3_RX_STD_MAX_SIZE_5700)
138 #define TG3_DEF_RX_RING_PENDING		200
139 #define TG3_RX_JMB_RING_SIZE(tp) \
140 	(tg3_flag(tp, LRG_PROD_RING_CAP) ? \
141 	 TG3_RX_JMB_MAX_SIZE_5717 : TG3_RX_JMB_MAX_SIZE_5700)
142 #define TG3_DEF_RX_JUMBO_RING_PENDING	100
143 
144 /* Do not place this n-ring entries value into the tp struct itself,
145  * we really want to expose these constants to GCC so that modulo et
146  * al.  operations are done with shifts and masks instead of with
147  * hw multiply/modulo instructions.  Another solution would be to
148  * replace things like '% foo' with '& (foo - 1)'.
149  */
150 
151 #define TG3_TX_RING_SIZE		512
152 #define TG3_DEF_TX_RING_PENDING		(TG3_TX_RING_SIZE - 1)
153 
154 #define TG3_RX_STD_RING_BYTES(tp) \
155 	(sizeof(struct tg3_rx_buffer_desc) * TG3_RX_STD_RING_SIZE(tp))
156 #define TG3_RX_JMB_RING_BYTES(tp) \
157 	(sizeof(struct tg3_ext_rx_buffer_desc) * TG3_RX_JMB_RING_SIZE(tp))
158 #define TG3_RX_RCB_RING_BYTES(tp) \
159 	(sizeof(struct tg3_rx_buffer_desc) * (tp->rx_ret_ring_mask + 1))
160 #define TG3_TX_RING_BYTES	(sizeof(struct tg3_tx_buffer_desc) * \
161 				 TG3_TX_RING_SIZE)
162 #define NEXT_TX(N)		(((N) + 1) & (TG3_TX_RING_SIZE - 1))
163 
164 #define TG3_DMA_BYTE_ENAB		64
165 
166 #define TG3_RX_STD_DMA_SZ		1536
167 #define TG3_RX_JMB_DMA_SZ		9046
168 
169 #define TG3_RX_DMA_TO_MAP_SZ(x)		((x) + TG3_DMA_BYTE_ENAB)
170 
171 #define TG3_RX_STD_MAP_SZ		TG3_RX_DMA_TO_MAP_SZ(TG3_RX_STD_DMA_SZ)
172 #define TG3_RX_JMB_MAP_SZ		TG3_RX_DMA_TO_MAP_SZ(TG3_RX_JMB_DMA_SZ)
173 
174 #define TG3_RX_STD_BUFF_RING_SIZE(tp) \
175 	(sizeof(struct ring_info) * TG3_RX_STD_RING_SIZE(tp))
176 
177 #define TG3_RX_JMB_BUFF_RING_SIZE(tp) \
178 	(sizeof(struct ring_info) * TG3_RX_JMB_RING_SIZE(tp))
179 
180 /* Due to a hardware bug, the 5701 can only DMA to memory addresses
181  * that are at least dword aligned when used in PCIX mode.  The driver
182  * works around this bug by double copying the packet.  This workaround
183  * is built into the normal double copy length check for efficiency.
184  *
185  * However, the double copy is only necessary on those architectures
186  * where unaligned memory accesses are inefficient.  For those architectures
187  * where unaligned memory accesses incur little penalty, we can reintegrate
188  * the 5701 in the normal rx path.  Doing so saves a device structure
189  * dereference by hardcoding the double copy threshold in place.
190  */
191 #define TG3_RX_COPY_THRESHOLD		256
192 #if NET_IP_ALIGN == 0 || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
193 	#define TG3_RX_COPY_THRESH(tp)	TG3_RX_COPY_THRESHOLD
194 #else
195 	#define TG3_RX_COPY_THRESH(tp)	((tp)->rx_copy_thresh)
196 #endif
197 
198 #if (NET_IP_ALIGN != 0)
199 #define TG3_RX_OFFSET(tp)	((tp)->rx_offset)
200 #else
201 #define TG3_RX_OFFSET(tp)	(NET_SKB_PAD)
202 #endif
203 
204 /* minimum number of free TX descriptors required to wake up TX process */
205 #define TG3_TX_WAKEUP_THRESH(tnapi)		((tnapi)->tx_pending / 4)
206 #define TG3_TX_BD_DMA_MAX_2K		2048
207 #define TG3_TX_BD_DMA_MAX_4K		4096
208 
209 #define TG3_RAW_IP_ALIGN 2
210 
211 #define TG3_MAX_UCAST_ADDR(tp) (tg3_flag((tp), ENABLE_ASF) ? 2 : 3)
212 #define TG3_UCAST_ADDR_IDX(tp) (tg3_flag((tp), ENABLE_ASF) ? 2 : 1)
213 
214 #define TG3_FW_UPDATE_TIMEOUT_SEC	5
215 #define TG3_FW_UPDATE_FREQ_SEC		(TG3_FW_UPDATE_TIMEOUT_SEC / 2)
216 
217 #define FIRMWARE_TG3		"tigon/tg3.bin"
218 #define FIRMWARE_TG357766	"tigon/tg357766.bin"
219 #define FIRMWARE_TG3TSO		"tigon/tg3_tso.bin"
220 #define FIRMWARE_TG3TSO5	"tigon/tg3_tso5.bin"
221 
222 static char version[] =
223 	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")";
224 
225 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
226 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
227 MODULE_LICENSE("GPL");
228 MODULE_VERSION(DRV_MODULE_VERSION);
229 MODULE_FIRMWARE(FIRMWARE_TG3);
230 MODULE_FIRMWARE(FIRMWARE_TG3TSO);
231 MODULE_FIRMWARE(FIRMWARE_TG3TSO5);
232 
233 static int tg3_debug = -1;	/* -1 == use TG3_DEF_MSG_ENABLE as value */
234 module_param(tg3_debug, int, 0);
235 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
236 
237 #define TG3_DRV_DATA_FLAG_10_100_ONLY	0x0001
238 #define TG3_DRV_DATA_FLAG_5705_10_100	0x0002
239 
240 static DEFINE_PCI_DEVICE_TABLE(tg3_pci_tbl) = {
241 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
242 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
243 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
244 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
245 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
246 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
247 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
248 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
249 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
250 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
251 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
252 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
253 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
254 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
255 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
256 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
257 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
258 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
259 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901),
260 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
261 			TG3_DRV_DATA_FLAG_5705_10_100},
262 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2),
263 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
264 			TG3_DRV_DATA_FLAG_5705_10_100},
265 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
266 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F),
267 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
268 			TG3_DRV_DATA_FLAG_5705_10_100},
269 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
270 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
271 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
272 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
273 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
274 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F),
275 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
276 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
277 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
278 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
279 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
280 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F),
281 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
282 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
283 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
284 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
285 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
286 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
287 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
288 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
289 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5787M,
290 			PCI_VENDOR_ID_LENOVO,
291 			TG3PCI_SUBDEVICE_ID_LENOVO_5787M),
292 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
293 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
294 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F),
295 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
296 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
297 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
298 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
299 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
300 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
301 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
302 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
303 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
304 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
305 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
306 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
307 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
308 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
309 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
310 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761S)},
311 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761SE)},
312 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_G)},
313 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_F)},
314 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780,
315 			PCI_VENDOR_ID_AI, TG3PCI_SUBDEVICE_ID_ACER_57780_A),
316 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
317 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780,
318 			PCI_VENDOR_ID_AI, TG3PCI_SUBDEVICE_ID_ACER_57780_B),
319 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
320 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780)},
321 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)},
322 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790),
323 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
324 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57788)},
325 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717)},
326 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717_C)},
327 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5718)},
328 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57781)},
329 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57785)},
330 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57761)},
331 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57765)},
332 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57791),
333 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
334 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57795),
335 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
336 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5719)},
337 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5720)},
338 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57762)},
339 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57766)},
340 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5762)},
341 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5725)},
342 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5727)},
343 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57764)},
344 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57767)},
345 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57787)},
346 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57782)},
347 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57786)},
348 	{PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
349 	{PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
350 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
351 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
352 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
353 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
354 	{PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
355 	{PCI_DEVICE(0x10cf, 0x11a2)}, /* Fujitsu 1000base-SX with BCM5703SKHB */
356 	{}
357 };
358 
359 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
360 
361 static const struct {
362 	const char string[ETH_GSTRING_LEN];
363 } ethtool_stats_keys[] = {
364 	{ "rx_octets" },
365 	{ "rx_fragments" },
366 	{ "rx_ucast_packets" },
367 	{ "rx_mcast_packets" },
368 	{ "rx_bcast_packets" },
369 	{ "rx_fcs_errors" },
370 	{ "rx_align_errors" },
371 	{ "rx_xon_pause_rcvd" },
372 	{ "rx_xoff_pause_rcvd" },
373 	{ "rx_mac_ctrl_rcvd" },
374 	{ "rx_xoff_entered" },
375 	{ "rx_frame_too_long_errors" },
376 	{ "rx_jabbers" },
377 	{ "rx_undersize_packets" },
378 	{ "rx_in_length_errors" },
379 	{ "rx_out_length_errors" },
380 	{ "rx_64_or_less_octet_packets" },
381 	{ "rx_65_to_127_octet_packets" },
382 	{ "rx_128_to_255_octet_packets" },
383 	{ "rx_256_to_511_octet_packets" },
384 	{ "rx_512_to_1023_octet_packets" },
385 	{ "rx_1024_to_1522_octet_packets" },
386 	{ "rx_1523_to_2047_octet_packets" },
387 	{ "rx_2048_to_4095_octet_packets" },
388 	{ "rx_4096_to_8191_octet_packets" },
389 	{ "rx_8192_to_9022_octet_packets" },
390 
391 	{ "tx_octets" },
392 	{ "tx_collisions" },
393 
394 	{ "tx_xon_sent" },
395 	{ "tx_xoff_sent" },
396 	{ "tx_flow_control" },
397 	{ "tx_mac_errors" },
398 	{ "tx_single_collisions" },
399 	{ "tx_mult_collisions" },
400 	{ "tx_deferred" },
401 	{ "tx_excessive_collisions" },
402 	{ "tx_late_collisions" },
403 	{ "tx_collide_2times" },
404 	{ "tx_collide_3times" },
405 	{ "tx_collide_4times" },
406 	{ "tx_collide_5times" },
407 	{ "tx_collide_6times" },
408 	{ "tx_collide_7times" },
409 	{ "tx_collide_8times" },
410 	{ "tx_collide_9times" },
411 	{ "tx_collide_10times" },
412 	{ "tx_collide_11times" },
413 	{ "tx_collide_12times" },
414 	{ "tx_collide_13times" },
415 	{ "tx_collide_14times" },
416 	{ "tx_collide_15times" },
417 	{ "tx_ucast_packets" },
418 	{ "tx_mcast_packets" },
419 	{ "tx_bcast_packets" },
420 	{ "tx_carrier_sense_errors" },
421 	{ "tx_discards" },
422 	{ "tx_errors" },
423 
424 	{ "dma_writeq_full" },
425 	{ "dma_write_prioq_full" },
426 	{ "rxbds_empty" },
427 	{ "rx_discards" },
428 	{ "rx_errors" },
429 	{ "rx_threshold_hit" },
430 
431 	{ "dma_readq_full" },
432 	{ "dma_read_prioq_full" },
433 	{ "tx_comp_queue_full" },
434 
435 	{ "ring_set_send_prod_index" },
436 	{ "ring_status_update" },
437 	{ "nic_irqs" },
438 	{ "nic_avoided_irqs" },
439 	{ "nic_tx_threshold_hit" },
440 
441 	{ "mbuf_lwm_thresh_hit" },
442 };
443 
444 #define TG3_NUM_STATS	ARRAY_SIZE(ethtool_stats_keys)
445 #define TG3_NVRAM_TEST		0
446 #define TG3_LINK_TEST		1
447 #define TG3_REGISTER_TEST	2
448 #define TG3_MEMORY_TEST		3
449 #define TG3_MAC_LOOPB_TEST	4
450 #define TG3_PHY_LOOPB_TEST	5
451 #define TG3_EXT_LOOPB_TEST	6
452 #define TG3_INTERRUPT_TEST	7
453 
454 
455 static const struct {
456 	const char string[ETH_GSTRING_LEN];
457 } ethtool_test_keys[] = {
458 	[TG3_NVRAM_TEST]	= { "nvram test        (online) " },
459 	[TG3_LINK_TEST]		= { "link test         (online) " },
460 	[TG3_REGISTER_TEST]	= { "register test     (offline)" },
461 	[TG3_MEMORY_TEST]	= { "memory test       (offline)" },
462 	[TG3_MAC_LOOPB_TEST]	= { "mac loopback test (offline)" },
463 	[TG3_PHY_LOOPB_TEST]	= { "phy loopback test (offline)" },
464 	[TG3_EXT_LOOPB_TEST]	= { "ext loopback test (offline)" },
465 	[TG3_INTERRUPT_TEST]	= { "interrupt test    (offline)" },
466 };
467 
468 #define TG3_NUM_TEST	ARRAY_SIZE(ethtool_test_keys)
469 
470 
471 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
472 {
473 	writel(val, tp->regs + off);
474 }
475 
476 static u32 tg3_read32(struct tg3 *tp, u32 off)
477 {
478 	return readl(tp->regs + off);
479 }
480 
481 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
482 {
483 	writel(val, tp->aperegs + off);
484 }
485 
486 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
487 {
488 	return readl(tp->aperegs + off);
489 }
490 
491 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
492 {
493 	unsigned long flags;
494 
495 	spin_lock_irqsave(&tp->indirect_lock, flags);
496 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
497 	pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
498 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
499 }
500 
501 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
502 {
503 	writel(val, tp->regs + off);
504 	readl(tp->regs + off);
505 }
506 
507 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
508 {
509 	unsigned long flags;
510 	u32 val;
511 
512 	spin_lock_irqsave(&tp->indirect_lock, flags);
513 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
514 	pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
515 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
516 	return val;
517 }
518 
519 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
520 {
521 	unsigned long flags;
522 
523 	if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
524 		pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
525 				       TG3_64BIT_REG_LOW, val);
526 		return;
527 	}
528 	if (off == TG3_RX_STD_PROD_IDX_REG) {
529 		pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
530 				       TG3_64BIT_REG_LOW, val);
531 		return;
532 	}
533 
534 	spin_lock_irqsave(&tp->indirect_lock, flags);
535 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
536 	pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
537 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
538 
539 	/* In indirect mode when disabling interrupts, we also need
540 	 * to clear the interrupt bit in the GRC local ctrl register.
541 	 */
542 	if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
543 	    (val == 0x1)) {
544 		pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
545 				       tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
546 	}
547 }
548 
549 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
550 {
551 	unsigned long flags;
552 	u32 val;
553 
554 	spin_lock_irqsave(&tp->indirect_lock, flags);
555 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
556 	pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
557 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
558 	return val;
559 }
560 
561 /* usec_wait specifies the wait time in usec when writing to certain registers
562  * where it is unsafe to read back the register without some delay.
563  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
564  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
565  */
566 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
567 {
568 	if (tg3_flag(tp, PCIX_TARGET_HWBUG) || tg3_flag(tp, ICH_WORKAROUND))
569 		/* Non-posted methods */
570 		tp->write32(tp, off, val);
571 	else {
572 		/* Posted method */
573 		tg3_write32(tp, off, val);
574 		if (usec_wait)
575 			udelay(usec_wait);
576 		tp->read32(tp, off);
577 	}
578 	/* Wait again after the read for the posted method to guarantee that
579 	 * the wait time is met.
580 	 */
581 	if (usec_wait)
582 		udelay(usec_wait);
583 }
584 
585 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
586 {
587 	tp->write32_mbox(tp, off, val);
588 	if (tg3_flag(tp, FLUSH_POSTED_WRITES) ||
589 	    (!tg3_flag(tp, MBOX_WRITE_REORDER) &&
590 	     !tg3_flag(tp, ICH_WORKAROUND)))
591 		tp->read32_mbox(tp, off);
592 }
593 
594 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
595 {
596 	void __iomem *mbox = tp->regs + off;
597 	writel(val, mbox);
598 	if (tg3_flag(tp, TXD_MBOX_HWBUG))
599 		writel(val, mbox);
600 	if (tg3_flag(tp, MBOX_WRITE_REORDER) ||
601 	    tg3_flag(tp, FLUSH_POSTED_WRITES))
602 		readl(mbox);
603 }
604 
605 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
606 {
607 	return readl(tp->regs + off + GRCMBOX_BASE);
608 }
609 
610 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
611 {
612 	writel(val, tp->regs + off + GRCMBOX_BASE);
613 }
614 
615 #define tw32_mailbox(reg, val)		tp->write32_mbox(tp, reg, val)
616 #define tw32_mailbox_f(reg, val)	tw32_mailbox_flush(tp, (reg), (val))
617 #define tw32_rx_mbox(reg, val)		tp->write32_rx_mbox(tp, reg, val)
618 #define tw32_tx_mbox(reg, val)		tp->write32_tx_mbox(tp, reg, val)
619 #define tr32_mailbox(reg)		tp->read32_mbox(tp, reg)
620 
621 #define tw32(reg, val)			tp->write32(tp, reg, val)
622 #define tw32_f(reg, val)		_tw32_flush(tp, (reg), (val), 0)
623 #define tw32_wait_f(reg, val, us)	_tw32_flush(tp, (reg), (val), (us))
624 #define tr32(reg)			tp->read32(tp, reg)
625 
626 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
627 {
628 	unsigned long flags;
629 
630 	if (tg3_asic_rev(tp) == ASIC_REV_5906 &&
631 	    (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
632 		return;
633 
634 	spin_lock_irqsave(&tp->indirect_lock, flags);
635 	if (tg3_flag(tp, SRAM_USE_CONFIG)) {
636 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
637 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
638 
639 		/* Always leave this as zero. */
640 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
641 	} else {
642 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
643 		tw32_f(TG3PCI_MEM_WIN_DATA, val);
644 
645 		/* Always leave this as zero. */
646 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
647 	}
648 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
649 }
650 
651 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
652 {
653 	unsigned long flags;
654 
655 	if (tg3_asic_rev(tp) == ASIC_REV_5906 &&
656 	    (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
657 		*val = 0;
658 		return;
659 	}
660 
661 	spin_lock_irqsave(&tp->indirect_lock, flags);
662 	if (tg3_flag(tp, SRAM_USE_CONFIG)) {
663 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
664 		pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
665 
666 		/* Always leave this as zero. */
667 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
668 	} else {
669 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
670 		*val = tr32(TG3PCI_MEM_WIN_DATA);
671 
672 		/* Always leave this as zero. */
673 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
674 	}
675 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
676 }
677 
678 static void tg3_ape_lock_init(struct tg3 *tp)
679 {
680 	int i;
681 	u32 regbase, bit;
682 
683 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
684 		regbase = TG3_APE_LOCK_GRANT;
685 	else
686 		regbase = TG3_APE_PER_LOCK_GRANT;
687 
688 	/* Make sure the driver hasn't any stale locks. */
689 	for (i = TG3_APE_LOCK_PHY0; i <= TG3_APE_LOCK_GPIO; i++) {
690 		switch (i) {
691 		case TG3_APE_LOCK_PHY0:
692 		case TG3_APE_LOCK_PHY1:
693 		case TG3_APE_LOCK_PHY2:
694 		case TG3_APE_LOCK_PHY3:
695 			bit = APE_LOCK_GRANT_DRIVER;
696 			break;
697 		default:
698 			if (!tp->pci_fn)
699 				bit = APE_LOCK_GRANT_DRIVER;
700 			else
701 				bit = 1 << tp->pci_fn;
702 		}
703 		tg3_ape_write32(tp, regbase + 4 * i, bit);
704 	}
705 
706 }
707 
708 static int tg3_ape_lock(struct tg3 *tp, int locknum)
709 {
710 	int i, off;
711 	int ret = 0;
712 	u32 status, req, gnt, bit;
713 
714 	if (!tg3_flag(tp, ENABLE_APE))
715 		return 0;
716 
717 	switch (locknum) {
718 	case TG3_APE_LOCK_GPIO:
719 		if (tg3_asic_rev(tp) == ASIC_REV_5761)
720 			return 0;
721 	case TG3_APE_LOCK_GRC:
722 	case TG3_APE_LOCK_MEM:
723 		if (!tp->pci_fn)
724 			bit = APE_LOCK_REQ_DRIVER;
725 		else
726 			bit = 1 << tp->pci_fn;
727 		break;
728 	case TG3_APE_LOCK_PHY0:
729 	case TG3_APE_LOCK_PHY1:
730 	case TG3_APE_LOCK_PHY2:
731 	case TG3_APE_LOCK_PHY3:
732 		bit = APE_LOCK_REQ_DRIVER;
733 		break;
734 	default:
735 		return -EINVAL;
736 	}
737 
738 	if (tg3_asic_rev(tp) == ASIC_REV_5761) {
739 		req = TG3_APE_LOCK_REQ;
740 		gnt = TG3_APE_LOCK_GRANT;
741 	} else {
742 		req = TG3_APE_PER_LOCK_REQ;
743 		gnt = TG3_APE_PER_LOCK_GRANT;
744 	}
745 
746 	off = 4 * locknum;
747 
748 	tg3_ape_write32(tp, req + off, bit);
749 
750 	/* Wait for up to 1 millisecond to acquire lock. */
751 	for (i = 0; i < 100; i++) {
752 		status = tg3_ape_read32(tp, gnt + off);
753 		if (status == bit)
754 			break;
755 		if (pci_channel_offline(tp->pdev))
756 			break;
757 
758 		udelay(10);
759 	}
760 
761 	if (status != bit) {
762 		/* Revoke the lock request. */
763 		tg3_ape_write32(tp, gnt + off, bit);
764 		ret = -EBUSY;
765 	}
766 
767 	return ret;
768 }
769 
770 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
771 {
772 	u32 gnt, bit;
773 
774 	if (!tg3_flag(tp, ENABLE_APE))
775 		return;
776 
777 	switch (locknum) {
778 	case TG3_APE_LOCK_GPIO:
779 		if (tg3_asic_rev(tp) == ASIC_REV_5761)
780 			return;
781 	case TG3_APE_LOCK_GRC:
782 	case TG3_APE_LOCK_MEM:
783 		if (!tp->pci_fn)
784 			bit = APE_LOCK_GRANT_DRIVER;
785 		else
786 			bit = 1 << tp->pci_fn;
787 		break;
788 	case TG3_APE_LOCK_PHY0:
789 	case TG3_APE_LOCK_PHY1:
790 	case TG3_APE_LOCK_PHY2:
791 	case TG3_APE_LOCK_PHY3:
792 		bit = APE_LOCK_GRANT_DRIVER;
793 		break;
794 	default:
795 		return;
796 	}
797 
798 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
799 		gnt = TG3_APE_LOCK_GRANT;
800 	else
801 		gnt = TG3_APE_PER_LOCK_GRANT;
802 
803 	tg3_ape_write32(tp, gnt + 4 * locknum, bit);
804 }
805 
806 static int tg3_ape_event_lock(struct tg3 *tp, u32 timeout_us)
807 {
808 	u32 apedata;
809 
810 	while (timeout_us) {
811 		if (tg3_ape_lock(tp, TG3_APE_LOCK_MEM))
812 			return -EBUSY;
813 
814 		apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
815 		if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
816 			break;
817 
818 		tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
819 
820 		udelay(10);
821 		timeout_us -= (timeout_us > 10) ? 10 : timeout_us;
822 	}
823 
824 	return timeout_us ? 0 : -EBUSY;
825 }
826 
827 static int tg3_ape_wait_for_event(struct tg3 *tp, u32 timeout_us)
828 {
829 	u32 i, apedata;
830 
831 	for (i = 0; i < timeout_us / 10; i++) {
832 		apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
833 
834 		if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
835 			break;
836 
837 		udelay(10);
838 	}
839 
840 	return i == timeout_us / 10;
841 }
842 
843 static int tg3_ape_scratchpad_read(struct tg3 *tp, u32 *data, u32 base_off,
844 				   u32 len)
845 {
846 	int err;
847 	u32 i, bufoff, msgoff, maxlen, apedata;
848 
849 	if (!tg3_flag(tp, APE_HAS_NCSI))
850 		return 0;
851 
852 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
853 	if (apedata != APE_SEG_SIG_MAGIC)
854 		return -ENODEV;
855 
856 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
857 	if (!(apedata & APE_FW_STATUS_READY))
858 		return -EAGAIN;
859 
860 	bufoff = tg3_ape_read32(tp, TG3_APE_SEG_MSG_BUF_OFF) +
861 		 TG3_APE_SHMEM_BASE;
862 	msgoff = bufoff + 2 * sizeof(u32);
863 	maxlen = tg3_ape_read32(tp, TG3_APE_SEG_MSG_BUF_LEN);
864 
865 	while (len) {
866 		u32 length;
867 
868 		/* Cap xfer sizes to scratchpad limits. */
869 		length = (len > maxlen) ? maxlen : len;
870 		len -= length;
871 
872 		apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
873 		if (!(apedata & APE_FW_STATUS_READY))
874 			return -EAGAIN;
875 
876 		/* Wait for up to 1 msec for APE to service previous event. */
877 		err = tg3_ape_event_lock(tp, 1000);
878 		if (err)
879 			return err;
880 
881 		apedata = APE_EVENT_STATUS_DRIVER_EVNT |
882 			  APE_EVENT_STATUS_SCRTCHPD_READ |
883 			  APE_EVENT_STATUS_EVENT_PENDING;
884 		tg3_ape_write32(tp, TG3_APE_EVENT_STATUS, apedata);
885 
886 		tg3_ape_write32(tp, bufoff, base_off);
887 		tg3_ape_write32(tp, bufoff + sizeof(u32), length);
888 
889 		tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
890 		tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
891 
892 		base_off += length;
893 
894 		if (tg3_ape_wait_for_event(tp, 30000))
895 			return -EAGAIN;
896 
897 		for (i = 0; length; i += 4, length -= 4) {
898 			u32 val = tg3_ape_read32(tp, msgoff + i);
899 			memcpy(data, &val, sizeof(u32));
900 			data++;
901 		}
902 	}
903 
904 	return 0;
905 }
906 
907 static int tg3_ape_send_event(struct tg3 *tp, u32 event)
908 {
909 	int err;
910 	u32 apedata;
911 
912 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
913 	if (apedata != APE_SEG_SIG_MAGIC)
914 		return -EAGAIN;
915 
916 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
917 	if (!(apedata & APE_FW_STATUS_READY))
918 		return -EAGAIN;
919 
920 	/* Wait for up to 1 millisecond for APE to service previous event. */
921 	err = tg3_ape_event_lock(tp, 1000);
922 	if (err)
923 		return err;
924 
925 	tg3_ape_write32(tp, TG3_APE_EVENT_STATUS,
926 			event | APE_EVENT_STATUS_EVENT_PENDING);
927 
928 	tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
929 	tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
930 
931 	return 0;
932 }
933 
934 static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
935 {
936 	u32 event;
937 	u32 apedata;
938 
939 	if (!tg3_flag(tp, ENABLE_APE))
940 		return;
941 
942 	switch (kind) {
943 	case RESET_KIND_INIT:
944 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
945 				APE_HOST_SEG_SIG_MAGIC);
946 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
947 				APE_HOST_SEG_LEN_MAGIC);
948 		apedata = tg3_ape_read32(tp, TG3_APE_HOST_INIT_COUNT);
949 		tg3_ape_write32(tp, TG3_APE_HOST_INIT_COUNT, ++apedata);
950 		tg3_ape_write32(tp, TG3_APE_HOST_DRIVER_ID,
951 			APE_HOST_DRIVER_ID_MAGIC(TG3_MAJ_NUM, TG3_MIN_NUM));
952 		tg3_ape_write32(tp, TG3_APE_HOST_BEHAVIOR,
953 				APE_HOST_BEHAV_NO_PHYLOCK);
954 		tg3_ape_write32(tp, TG3_APE_HOST_DRVR_STATE,
955 				    TG3_APE_HOST_DRVR_STATE_START);
956 
957 		event = APE_EVENT_STATUS_STATE_START;
958 		break;
959 	case RESET_KIND_SHUTDOWN:
960 		/* With the interface we are currently using,
961 		 * APE does not track driver state.  Wiping
962 		 * out the HOST SEGMENT SIGNATURE forces
963 		 * the APE to assume OS absent status.
964 		 */
965 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG, 0x0);
966 
967 		if (device_may_wakeup(&tp->pdev->dev) &&
968 		    tg3_flag(tp, WOL_ENABLE)) {
969 			tg3_ape_write32(tp, TG3_APE_HOST_WOL_SPEED,
970 					    TG3_APE_HOST_WOL_SPEED_AUTO);
971 			apedata = TG3_APE_HOST_DRVR_STATE_WOL;
972 		} else
973 			apedata = TG3_APE_HOST_DRVR_STATE_UNLOAD;
974 
975 		tg3_ape_write32(tp, TG3_APE_HOST_DRVR_STATE, apedata);
976 
977 		event = APE_EVENT_STATUS_STATE_UNLOAD;
978 		break;
979 	default:
980 		return;
981 	}
982 
983 	event |= APE_EVENT_STATUS_DRIVER_EVNT | APE_EVENT_STATUS_STATE_CHNGE;
984 
985 	tg3_ape_send_event(tp, event);
986 }
987 
988 static void tg3_disable_ints(struct tg3 *tp)
989 {
990 	int i;
991 
992 	tw32(TG3PCI_MISC_HOST_CTRL,
993 	     (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
994 	for (i = 0; i < tp->irq_max; i++)
995 		tw32_mailbox_f(tp->napi[i].int_mbox, 0x00000001);
996 }
997 
998 static void tg3_enable_ints(struct tg3 *tp)
999 {
1000 	int i;
1001 
1002 	tp->irq_sync = 0;
1003 	wmb();
1004 
1005 	tw32(TG3PCI_MISC_HOST_CTRL,
1006 	     (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
1007 
1008 	tp->coal_now = tp->coalesce_mode | HOSTCC_MODE_ENABLE;
1009 	for (i = 0; i < tp->irq_cnt; i++) {
1010 		struct tg3_napi *tnapi = &tp->napi[i];
1011 
1012 		tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
1013 		if (tg3_flag(tp, 1SHOT_MSI))
1014 			tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
1015 
1016 		tp->coal_now |= tnapi->coal_now;
1017 	}
1018 
1019 	/* Force an initial interrupt */
1020 	if (!tg3_flag(tp, TAGGED_STATUS) &&
1021 	    (tp->napi[0].hw_status->status & SD_STATUS_UPDATED))
1022 		tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
1023 	else
1024 		tw32(HOSTCC_MODE, tp->coal_now);
1025 
1026 	tp->coal_now &= ~(tp->napi[0].coal_now | tp->napi[1].coal_now);
1027 }
1028 
1029 static inline unsigned int tg3_has_work(struct tg3_napi *tnapi)
1030 {
1031 	struct tg3 *tp = tnapi->tp;
1032 	struct tg3_hw_status *sblk = tnapi->hw_status;
1033 	unsigned int work_exists = 0;
1034 
1035 	/* check for phy events */
1036 	if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
1037 		if (sblk->status & SD_STATUS_LINK_CHG)
1038 			work_exists = 1;
1039 	}
1040 
1041 	/* check for TX work to do */
1042 	if (sblk->idx[0].tx_consumer != tnapi->tx_cons)
1043 		work_exists = 1;
1044 
1045 	/* check for RX work to do */
1046 	if (tnapi->rx_rcb_prod_idx &&
1047 	    *(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
1048 		work_exists = 1;
1049 
1050 	return work_exists;
1051 }
1052 
1053 /* tg3_int_reenable
1054  *  similar to tg3_enable_ints, but it accurately determines whether there
1055  *  is new work pending and can return without flushing the PIO write
1056  *  which reenables interrupts
1057  */
1058 static void tg3_int_reenable(struct tg3_napi *tnapi)
1059 {
1060 	struct tg3 *tp = tnapi->tp;
1061 
1062 	tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
1063 	mmiowb();
1064 
1065 	/* When doing tagged status, this work check is unnecessary.
1066 	 * The last_tag we write above tells the chip which piece of
1067 	 * work we've completed.
1068 	 */
1069 	if (!tg3_flag(tp, TAGGED_STATUS) && tg3_has_work(tnapi))
1070 		tw32(HOSTCC_MODE, tp->coalesce_mode |
1071 		     HOSTCC_MODE_ENABLE | tnapi->coal_now);
1072 }
1073 
1074 static void tg3_switch_clocks(struct tg3 *tp)
1075 {
1076 	u32 clock_ctrl;
1077 	u32 orig_clock_ctrl;
1078 
1079 	if (tg3_flag(tp, CPMU_PRESENT) || tg3_flag(tp, 5780_CLASS))
1080 		return;
1081 
1082 	clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
1083 
1084 	orig_clock_ctrl = clock_ctrl;
1085 	clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
1086 		       CLOCK_CTRL_CLKRUN_OENABLE |
1087 		       0x1f);
1088 	tp->pci_clock_ctrl = clock_ctrl;
1089 
1090 	if (tg3_flag(tp, 5705_PLUS)) {
1091 		if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
1092 			tw32_wait_f(TG3PCI_CLOCK_CTRL,
1093 				    clock_ctrl | CLOCK_CTRL_625_CORE, 40);
1094 		}
1095 	} else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
1096 		tw32_wait_f(TG3PCI_CLOCK_CTRL,
1097 			    clock_ctrl |
1098 			    (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
1099 			    40);
1100 		tw32_wait_f(TG3PCI_CLOCK_CTRL,
1101 			    clock_ctrl | (CLOCK_CTRL_ALTCLK),
1102 			    40);
1103 	}
1104 	tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
1105 }
1106 
1107 #define PHY_BUSY_LOOPS	5000
1108 
1109 static int __tg3_readphy(struct tg3 *tp, unsigned int phy_addr, int reg,
1110 			 u32 *val)
1111 {
1112 	u32 frame_val;
1113 	unsigned int loops;
1114 	int ret;
1115 
1116 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1117 		tw32_f(MAC_MI_MODE,
1118 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
1119 		udelay(80);
1120 	}
1121 
1122 	tg3_ape_lock(tp, tp->phy_ape_lock);
1123 
1124 	*val = 0x0;
1125 
1126 	frame_val  = ((phy_addr << MI_COM_PHY_ADDR_SHIFT) &
1127 		      MI_COM_PHY_ADDR_MASK);
1128 	frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
1129 		      MI_COM_REG_ADDR_MASK);
1130 	frame_val |= (MI_COM_CMD_READ | MI_COM_START);
1131 
1132 	tw32_f(MAC_MI_COM, frame_val);
1133 
1134 	loops = PHY_BUSY_LOOPS;
1135 	while (loops != 0) {
1136 		udelay(10);
1137 		frame_val = tr32(MAC_MI_COM);
1138 
1139 		if ((frame_val & MI_COM_BUSY) == 0) {
1140 			udelay(5);
1141 			frame_val = tr32(MAC_MI_COM);
1142 			break;
1143 		}
1144 		loops -= 1;
1145 	}
1146 
1147 	ret = -EBUSY;
1148 	if (loops != 0) {
1149 		*val = frame_val & MI_COM_DATA_MASK;
1150 		ret = 0;
1151 	}
1152 
1153 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1154 		tw32_f(MAC_MI_MODE, tp->mi_mode);
1155 		udelay(80);
1156 	}
1157 
1158 	tg3_ape_unlock(tp, tp->phy_ape_lock);
1159 
1160 	return ret;
1161 }
1162 
1163 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
1164 {
1165 	return __tg3_readphy(tp, tp->phy_addr, reg, val);
1166 }
1167 
1168 static int __tg3_writephy(struct tg3 *tp, unsigned int phy_addr, int reg,
1169 			  u32 val)
1170 {
1171 	u32 frame_val;
1172 	unsigned int loops;
1173 	int ret;
1174 
1175 	if ((tp->phy_flags & TG3_PHYFLG_IS_FET) &&
1176 	    (reg == MII_CTRL1000 || reg == MII_TG3_AUX_CTRL))
1177 		return 0;
1178 
1179 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1180 		tw32_f(MAC_MI_MODE,
1181 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
1182 		udelay(80);
1183 	}
1184 
1185 	tg3_ape_lock(tp, tp->phy_ape_lock);
1186 
1187 	frame_val  = ((phy_addr << MI_COM_PHY_ADDR_SHIFT) &
1188 		      MI_COM_PHY_ADDR_MASK);
1189 	frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
1190 		      MI_COM_REG_ADDR_MASK);
1191 	frame_val |= (val & MI_COM_DATA_MASK);
1192 	frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
1193 
1194 	tw32_f(MAC_MI_COM, frame_val);
1195 
1196 	loops = PHY_BUSY_LOOPS;
1197 	while (loops != 0) {
1198 		udelay(10);
1199 		frame_val = tr32(MAC_MI_COM);
1200 		if ((frame_val & MI_COM_BUSY) == 0) {
1201 			udelay(5);
1202 			frame_val = tr32(MAC_MI_COM);
1203 			break;
1204 		}
1205 		loops -= 1;
1206 	}
1207 
1208 	ret = -EBUSY;
1209 	if (loops != 0)
1210 		ret = 0;
1211 
1212 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1213 		tw32_f(MAC_MI_MODE, tp->mi_mode);
1214 		udelay(80);
1215 	}
1216 
1217 	tg3_ape_unlock(tp, tp->phy_ape_lock);
1218 
1219 	return ret;
1220 }
1221 
1222 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
1223 {
1224 	return __tg3_writephy(tp, tp->phy_addr, reg, val);
1225 }
1226 
1227 static int tg3_phy_cl45_write(struct tg3 *tp, u32 devad, u32 addr, u32 val)
1228 {
1229 	int err;
1230 
1231 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
1232 	if (err)
1233 		goto done;
1234 
1235 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
1236 	if (err)
1237 		goto done;
1238 
1239 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
1240 			   MII_TG3_MMD_CTRL_DATA_NOINC | devad);
1241 	if (err)
1242 		goto done;
1243 
1244 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, val);
1245 
1246 done:
1247 	return err;
1248 }
1249 
1250 static int tg3_phy_cl45_read(struct tg3 *tp, u32 devad, u32 addr, u32 *val)
1251 {
1252 	int err;
1253 
1254 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
1255 	if (err)
1256 		goto done;
1257 
1258 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
1259 	if (err)
1260 		goto done;
1261 
1262 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
1263 			   MII_TG3_MMD_CTRL_DATA_NOINC | devad);
1264 	if (err)
1265 		goto done;
1266 
1267 	err = tg3_readphy(tp, MII_TG3_MMD_ADDRESS, val);
1268 
1269 done:
1270 	return err;
1271 }
1272 
1273 static int tg3_phydsp_read(struct tg3 *tp, u32 reg, u32 *val)
1274 {
1275 	int err;
1276 
1277 	err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1278 	if (!err)
1279 		err = tg3_readphy(tp, MII_TG3_DSP_RW_PORT, val);
1280 
1281 	return err;
1282 }
1283 
1284 static int tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
1285 {
1286 	int err;
1287 
1288 	err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1289 	if (!err)
1290 		err = tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
1291 
1292 	return err;
1293 }
1294 
1295 static int tg3_phy_auxctl_read(struct tg3 *tp, int reg, u32 *val)
1296 {
1297 	int err;
1298 
1299 	err = tg3_writephy(tp, MII_TG3_AUX_CTRL,
1300 			   (reg << MII_TG3_AUXCTL_MISC_RDSEL_SHIFT) |
1301 			   MII_TG3_AUXCTL_SHDWSEL_MISC);
1302 	if (!err)
1303 		err = tg3_readphy(tp, MII_TG3_AUX_CTRL, val);
1304 
1305 	return err;
1306 }
1307 
1308 static int tg3_phy_auxctl_write(struct tg3 *tp, int reg, u32 set)
1309 {
1310 	if (reg == MII_TG3_AUXCTL_SHDWSEL_MISC)
1311 		set |= MII_TG3_AUXCTL_MISC_WREN;
1312 
1313 	return tg3_writephy(tp, MII_TG3_AUX_CTRL, set | reg);
1314 }
1315 
1316 static int tg3_phy_toggle_auxctl_smdsp(struct tg3 *tp, bool enable)
1317 {
1318 	u32 val;
1319 	int err;
1320 
1321 	err = tg3_phy_auxctl_read(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
1322 
1323 	if (err)
1324 		return err;
1325 
1326 	if (enable)
1327 		val |= MII_TG3_AUXCTL_ACTL_SMDSP_ENA;
1328 	else
1329 		val &= ~MII_TG3_AUXCTL_ACTL_SMDSP_ENA;
1330 
1331 	err = tg3_phy_auxctl_write((tp), MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
1332 				   val | MII_TG3_AUXCTL_ACTL_TX_6DB);
1333 
1334 	return err;
1335 }
1336 
1337 static int tg3_phy_shdw_write(struct tg3 *tp, int reg, u32 val)
1338 {
1339 	return tg3_writephy(tp, MII_TG3_MISC_SHDW,
1340 			    reg | val | MII_TG3_MISC_SHDW_WREN);
1341 }
1342 
1343 static int tg3_bmcr_reset(struct tg3 *tp)
1344 {
1345 	u32 phy_control;
1346 	int limit, err;
1347 
1348 	/* OK, reset it, and poll the BMCR_RESET bit until it
1349 	 * clears or we time out.
1350 	 */
1351 	phy_control = BMCR_RESET;
1352 	err = tg3_writephy(tp, MII_BMCR, phy_control);
1353 	if (err != 0)
1354 		return -EBUSY;
1355 
1356 	limit = 5000;
1357 	while (limit--) {
1358 		err = tg3_readphy(tp, MII_BMCR, &phy_control);
1359 		if (err != 0)
1360 			return -EBUSY;
1361 
1362 		if ((phy_control & BMCR_RESET) == 0) {
1363 			udelay(40);
1364 			break;
1365 		}
1366 		udelay(10);
1367 	}
1368 	if (limit < 0)
1369 		return -EBUSY;
1370 
1371 	return 0;
1372 }
1373 
1374 static int tg3_mdio_read(struct mii_bus *bp, int mii_id, int reg)
1375 {
1376 	struct tg3 *tp = bp->priv;
1377 	u32 val;
1378 
1379 	spin_lock_bh(&tp->lock);
1380 
1381 	if (__tg3_readphy(tp, mii_id, reg, &val))
1382 		val = -EIO;
1383 
1384 	spin_unlock_bh(&tp->lock);
1385 
1386 	return val;
1387 }
1388 
1389 static int tg3_mdio_write(struct mii_bus *bp, int mii_id, int reg, u16 val)
1390 {
1391 	struct tg3 *tp = bp->priv;
1392 	u32 ret = 0;
1393 
1394 	spin_lock_bh(&tp->lock);
1395 
1396 	if (__tg3_writephy(tp, mii_id, reg, val))
1397 		ret = -EIO;
1398 
1399 	spin_unlock_bh(&tp->lock);
1400 
1401 	return ret;
1402 }
1403 
1404 static void tg3_mdio_config_5785(struct tg3 *tp)
1405 {
1406 	u32 val;
1407 	struct phy_device *phydev;
1408 
1409 	phydev = tp->mdio_bus->phy_map[tp->phy_addr];
1410 	switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1411 	case PHY_ID_BCM50610:
1412 	case PHY_ID_BCM50610M:
1413 		val = MAC_PHYCFG2_50610_LED_MODES;
1414 		break;
1415 	case PHY_ID_BCMAC131:
1416 		val = MAC_PHYCFG2_AC131_LED_MODES;
1417 		break;
1418 	case PHY_ID_RTL8211C:
1419 		val = MAC_PHYCFG2_RTL8211C_LED_MODES;
1420 		break;
1421 	case PHY_ID_RTL8201E:
1422 		val = MAC_PHYCFG2_RTL8201E_LED_MODES;
1423 		break;
1424 	default:
1425 		return;
1426 	}
1427 
1428 	if (phydev->interface != PHY_INTERFACE_MODE_RGMII) {
1429 		tw32(MAC_PHYCFG2, val);
1430 
1431 		val = tr32(MAC_PHYCFG1);
1432 		val &= ~(MAC_PHYCFG1_RGMII_INT |
1433 			 MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK);
1434 		val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT;
1435 		tw32(MAC_PHYCFG1, val);
1436 
1437 		return;
1438 	}
1439 
1440 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE))
1441 		val |= MAC_PHYCFG2_EMODE_MASK_MASK |
1442 		       MAC_PHYCFG2_FMODE_MASK_MASK |
1443 		       MAC_PHYCFG2_GMODE_MASK_MASK |
1444 		       MAC_PHYCFG2_ACT_MASK_MASK   |
1445 		       MAC_PHYCFG2_QUAL_MASK_MASK |
1446 		       MAC_PHYCFG2_INBAND_ENABLE;
1447 
1448 	tw32(MAC_PHYCFG2, val);
1449 
1450 	val = tr32(MAC_PHYCFG1);
1451 	val &= ~(MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK |
1452 		 MAC_PHYCFG1_RGMII_EXT_RX_DEC | MAC_PHYCFG1_RGMII_SND_STAT_EN);
1453 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1454 		if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1455 			val |= MAC_PHYCFG1_RGMII_EXT_RX_DEC;
1456 		if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1457 			val |= MAC_PHYCFG1_RGMII_SND_STAT_EN;
1458 	}
1459 	val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT |
1460 	       MAC_PHYCFG1_RGMII_INT | MAC_PHYCFG1_TXC_DRV;
1461 	tw32(MAC_PHYCFG1, val);
1462 
1463 	val = tr32(MAC_EXT_RGMII_MODE);
1464 	val &= ~(MAC_RGMII_MODE_RX_INT_B |
1465 		 MAC_RGMII_MODE_RX_QUALITY |
1466 		 MAC_RGMII_MODE_RX_ACTIVITY |
1467 		 MAC_RGMII_MODE_RX_ENG_DET |
1468 		 MAC_RGMII_MODE_TX_ENABLE |
1469 		 MAC_RGMII_MODE_TX_LOWPWR |
1470 		 MAC_RGMII_MODE_TX_RESET);
1471 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1472 		if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1473 			val |= MAC_RGMII_MODE_RX_INT_B |
1474 			       MAC_RGMII_MODE_RX_QUALITY |
1475 			       MAC_RGMII_MODE_RX_ACTIVITY |
1476 			       MAC_RGMII_MODE_RX_ENG_DET;
1477 		if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1478 			val |= MAC_RGMII_MODE_TX_ENABLE |
1479 			       MAC_RGMII_MODE_TX_LOWPWR |
1480 			       MAC_RGMII_MODE_TX_RESET;
1481 	}
1482 	tw32(MAC_EXT_RGMII_MODE, val);
1483 }
1484 
1485 static void tg3_mdio_start(struct tg3 *tp)
1486 {
1487 	tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
1488 	tw32_f(MAC_MI_MODE, tp->mi_mode);
1489 	udelay(80);
1490 
1491 	if (tg3_flag(tp, MDIOBUS_INITED) &&
1492 	    tg3_asic_rev(tp) == ASIC_REV_5785)
1493 		tg3_mdio_config_5785(tp);
1494 }
1495 
1496 static int tg3_mdio_init(struct tg3 *tp)
1497 {
1498 	int i;
1499 	u32 reg;
1500 	struct phy_device *phydev;
1501 
1502 	if (tg3_flag(tp, 5717_PLUS)) {
1503 		u32 is_serdes;
1504 
1505 		tp->phy_addr = tp->pci_fn + 1;
1506 
1507 		if (tg3_chip_rev_id(tp) != CHIPREV_ID_5717_A0)
1508 			is_serdes = tr32(SG_DIG_STATUS) & SG_DIG_IS_SERDES;
1509 		else
1510 			is_serdes = tr32(TG3_CPMU_PHY_STRAP) &
1511 				    TG3_CPMU_PHY_STRAP_IS_SERDES;
1512 		if (is_serdes)
1513 			tp->phy_addr += 7;
1514 	} else if (tg3_flag(tp, IS_SSB_CORE) && tg3_flag(tp, ROBOSWITCH)) {
1515 		int addr;
1516 
1517 		addr = ssb_gige_get_phyaddr(tp->pdev);
1518 		if (addr < 0)
1519 			return addr;
1520 		tp->phy_addr = addr;
1521 	} else
1522 		tp->phy_addr = TG3_PHY_MII_ADDR;
1523 
1524 	tg3_mdio_start(tp);
1525 
1526 	if (!tg3_flag(tp, USE_PHYLIB) || tg3_flag(tp, MDIOBUS_INITED))
1527 		return 0;
1528 
1529 	tp->mdio_bus = mdiobus_alloc();
1530 	if (tp->mdio_bus == NULL)
1531 		return -ENOMEM;
1532 
1533 	tp->mdio_bus->name     = "tg3 mdio bus";
1534 	snprintf(tp->mdio_bus->id, MII_BUS_ID_SIZE, "%x",
1535 		 (tp->pdev->bus->number << 8) | tp->pdev->devfn);
1536 	tp->mdio_bus->priv     = tp;
1537 	tp->mdio_bus->parent   = &tp->pdev->dev;
1538 	tp->mdio_bus->read     = &tg3_mdio_read;
1539 	tp->mdio_bus->write    = &tg3_mdio_write;
1540 	tp->mdio_bus->phy_mask = ~(1 << tp->phy_addr);
1541 	tp->mdio_bus->irq      = &tp->mdio_irq[0];
1542 
1543 	for (i = 0; i < PHY_MAX_ADDR; i++)
1544 		tp->mdio_bus->irq[i] = PHY_POLL;
1545 
1546 	/* The bus registration will look for all the PHYs on the mdio bus.
1547 	 * Unfortunately, it does not ensure the PHY is powered up before
1548 	 * accessing the PHY ID registers.  A chip reset is the
1549 	 * quickest way to bring the device back to an operational state..
1550 	 */
1551 	if (tg3_readphy(tp, MII_BMCR, &reg) || (reg & BMCR_PDOWN))
1552 		tg3_bmcr_reset(tp);
1553 
1554 	i = mdiobus_register(tp->mdio_bus);
1555 	if (i) {
1556 		dev_warn(&tp->pdev->dev, "mdiobus_reg failed (0x%x)\n", i);
1557 		mdiobus_free(tp->mdio_bus);
1558 		return i;
1559 	}
1560 
1561 	phydev = tp->mdio_bus->phy_map[tp->phy_addr];
1562 
1563 	if (!phydev || !phydev->drv) {
1564 		dev_warn(&tp->pdev->dev, "No PHY devices\n");
1565 		mdiobus_unregister(tp->mdio_bus);
1566 		mdiobus_free(tp->mdio_bus);
1567 		return -ENODEV;
1568 	}
1569 
1570 	switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1571 	case PHY_ID_BCM57780:
1572 		phydev->interface = PHY_INTERFACE_MODE_GMII;
1573 		phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1574 		break;
1575 	case PHY_ID_BCM50610:
1576 	case PHY_ID_BCM50610M:
1577 		phydev->dev_flags |= PHY_BRCM_CLEAR_RGMII_MODE |
1578 				     PHY_BRCM_RX_REFCLK_UNUSED |
1579 				     PHY_BRCM_DIS_TXCRXC_NOENRGY |
1580 				     PHY_BRCM_AUTO_PWRDWN_ENABLE;
1581 		if (tg3_flag(tp, RGMII_INBAND_DISABLE))
1582 			phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE;
1583 		if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1584 			phydev->dev_flags |= PHY_BRCM_EXT_IBND_RX_ENABLE;
1585 		if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1586 			phydev->dev_flags |= PHY_BRCM_EXT_IBND_TX_ENABLE;
1587 		/* fallthru */
1588 	case PHY_ID_RTL8211C:
1589 		phydev->interface = PHY_INTERFACE_MODE_RGMII;
1590 		break;
1591 	case PHY_ID_RTL8201E:
1592 	case PHY_ID_BCMAC131:
1593 		phydev->interface = PHY_INTERFACE_MODE_MII;
1594 		phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1595 		tp->phy_flags |= TG3_PHYFLG_IS_FET;
1596 		break;
1597 	}
1598 
1599 	tg3_flag_set(tp, MDIOBUS_INITED);
1600 
1601 	if (tg3_asic_rev(tp) == ASIC_REV_5785)
1602 		tg3_mdio_config_5785(tp);
1603 
1604 	return 0;
1605 }
1606 
1607 static void tg3_mdio_fini(struct tg3 *tp)
1608 {
1609 	if (tg3_flag(tp, MDIOBUS_INITED)) {
1610 		tg3_flag_clear(tp, MDIOBUS_INITED);
1611 		mdiobus_unregister(tp->mdio_bus);
1612 		mdiobus_free(tp->mdio_bus);
1613 	}
1614 }
1615 
1616 /* tp->lock is held. */
1617 static inline void tg3_generate_fw_event(struct tg3 *tp)
1618 {
1619 	u32 val;
1620 
1621 	val = tr32(GRC_RX_CPU_EVENT);
1622 	val |= GRC_RX_CPU_DRIVER_EVENT;
1623 	tw32_f(GRC_RX_CPU_EVENT, val);
1624 
1625 	tp->last_event_jiffies = jiffies;
1626 }
1627 
1628 #define TG3_FW_EVENT_TIMEOUT_USEC 2500
1629 
1630 /* tp->lock is held. */
1631 static void tg3_wait_for_event_ack(struct tg3 *tp)
1632 {
1633 	int i;
1634 	unsigned int delay_cnt;
1635 	long time_remain;
1636 
1637 	/* If enough time has passed, no wait is necessary. */
1638 	time_remain = (long)(tp->last_event_jiffies + 1 +
1639 		      usecs_to_jiffies(TG3_FW_EVENT_TIMEOUT_USEC)) -
1640 		      (long)jiffies;
1641 	if (time_remain < 0)
1642 		return;
1643 
1644 	/* Check if we can shorten the wait time. */
1645 	delay_cnt = jiffies_to_usecs(time_remain);
1646 	if (delay_cnt > TG3_FW_EVENT_TIMEOUT_USEC)
1647 		delay_cnt = TG3_FW_EVENT_TIMEOUT_USEC;
1648 	delay_cnt = (delay_cnt >> 3) + 1;
1649 
1650 	for (i = 0; i < delay_cnt; i++) {
1651 		if (!(tr32(GRC_RX_CPU_EVENT) & GRC_RX_CPU_DRIVER_EVENT))
1652 			break;
1653 		if (pci_channel_offline(tp->pdev))
1654 			break;
1655 
1656 		udelay(8);
1657 	}
1658 }
1659 
1660 /* tp->lock is held. */
1661 static void tg3_phy_gather_ump_data(struct tg3 *tp, u32 *data)
1662 {
1663 	u32 reg, val;
1664 
1665 	val = 0;
1666 	if (!tg3_readphy(tp, MII_BMCR, &reg))
1667 		val = reg << 16;
1668 	if (!tg3_readphy(tp, MII_BMSR, &reg))
1669 		val |= (reg & 0xffff);
1670 	*data++ = val;
1671 
1672 	val = 0;
1673 	if (!tg3_readphy(tp, MII_ADVERTISE, &reg))
1674 		val = reg << 16;
1675 	if (!tg3_readphy(tp, MII_LPA, &reg))
1676 		val |= (reg & 0xffff);
1677 	*data++ = val;
1678 
1679 	val = 0;
1680 	if (!(tp->phy_flags & TG3_PHYFLG_MII_SERDES)) {
1681 		if (!tg3_readphy(tp, MII_CTRL1000, &reg))
1682 			val = reg << 16;
1683 		if (!tg3_readphy(tp, MII_STAT1000, &reg))
1684 			val |= (reg & 0xffff);
1685 	}
1686 	*data++ = val;
1687 
1688 	if (!tg3_readphy(tp, MII_PHYADDR, &reg))
1689 		val = reg << 16;
1690 	else
1691 		val = 0;
1692 	*data++ = val;
1693 }
1694 
1695 /* tp->lock is held. */
1696 static void tg3_ump_link_report(struct tg3 *tp)
1697 {
1698 	u32 data[4];
1699 
1700 	if (!tg3_flag(tp, 5780_CLASS) || !tg3_flag(tp, ENABLE_ASF))
1701 		return;
1702 
1703 	tg3_phy_gather_ump_data(tp, data);
1704 
1705 	tg3_wait_for_event_ack(tp);
1706 
1707 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_LINK_UPDATE);
1708 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 14);
1709 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x0, data[0]);
1710 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x4, data[1]);
1711 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x8, data[2]);
1712 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0xc, data[3]);
1713 
1714 	tg3_generate_fw_event(tp);
1715 }
1716 
1717 /* tp->lock is held. */
1718 static void tg3_stop_fw(struct tg3 *tp)
1719 {
1720 	if (tg3_flag(tp, ENABLE_ASF) && !tg3_flag(tp, ENABLE_APE)) {
1721 		/* Wait for RX cpu to ACK the previous event. */
1722 		tg3_wait_for_event_ack(tp);
1723 
1724 		tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
1725 
1726 		tg3_generate_fw_event(tp);
1727 
1728 		/* Wait for RX cpu to ACK this event. */
1729 		tg3_wait_for_event_ack(tp);
1730 	}
1731 }
1732 
1733 /* tp->lock is held. */
1734 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
1735 {
1736 	tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
1737 		      NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
1738 
1739 	if (tg3_flag(tp, ASF_NEW_HANDSHAKE)) {
1740 		switch (kind) {
1741 		case RESET_KIND_INIT:
1742 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1743 				      DRV_STATE_START);
1744 			break;
1745 
1746 		case RESET_KIND_SHUTDOWN:
1747 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1748 				      DRV_STATE_UNLOAD);
1749 			break;
1750 
1751 		case RESET_KIND_SUSPEND:
1752 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1753 				      DRV_STATE_SUSPEND);
1754 			break;
1755 
1756 		default:
1757 			break;
1758 		}
1759 	}
1760 }
1761 
1762 /* tp->lock is held. */
1763 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
1764 {
1765 	if (tg3_flag(tp, ASF_NEW_HANDSHAKE)) {
1766 		switch (kind) {
1767 		case RESET_KIND_INIT:
1768 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1769 				      DRV_STATE_START_DONE);
1770 			break;
1771 
1772 		case RESET_KIND_SHUTDOWN:
1773 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1774 				      DRV_STATE_UNLOAD_DONE);
1775 			break;
1776 
1777 		default:
1778 			break;
1779 		}
1780 	}
1781 }
1782 
1783 /* tp->lock is held. */
1784 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
1785 {
1786 	if (tg3_flag(tp, ENABLE_ASF)) {
1787 		switch (kind) {
1788 		case RESET_KIND_INIT:
1789 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1790 				      DRV_STATE_START);
1791 			break;
1792 
1793 		case RESET_KIND_SHUTDOWN:
1794 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1795 				      DRV_STATE_UNLOAD);
1796 			break;
1797 
1798 		case RESET_KIND_SUSPEND:
1799 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1800 				      DRV_STATE_SUSPEND);
1801 			break;
1802 
1803 		default:
1804 			break;
1805 		}
1806 	}
1807 }
1808 
1809 static int tg3_poll_fw(struct tg3 *tp)
1810 {
1811 	int i;
1812 	u32 val;
1813 
1814 	if (tg3_flag(tp, NO_FWARE_REPORTED))
1815 		return 0;
1816 
1817 	if (tg3_flag(tp, IS_SSB_CORE)) {
1818 		/* We don't use firmware. */
1819 		return 0;
1820 	}
1821 
1822 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
1823 		/* Wait up to 20ms for init done. */
1824 		for (i = 0; i < 200; i++) {
1825 			if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
1826 				return 0;
1827 			if (pci_channel_offline(tp->pdev))
1828 				return -ENODEV;
1829 
1830 			udelay(100);
1831 		}
1832 		return -ENODEV;
1833 	}
1834 
1835 	/* Wait for firmware initialization to complete. */
1836 	for (i = 0; i < 100000; i++) {
1837 		tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
1838 		if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1839 			break;
1840 		if (pci_channel_offline(tp->pdev)) {
1841 			if (!tg3_flag(tp, NO_FWARE_REPORTED)) {
1842 				tg3_flag_set(tp, NO_FWARE_REPORTED);
1843 				netdev_info(tp->dev, "No firmware running\n");
1844 			}
1845 
1846 			break;
1847 		}
1848 
1849 		udelay(10);
1850 	}
1851 
1852 	/* Chip might not be fitted with firmware.  Some Sun onboard
1853 	 * parts are configured like that.  So don't signal the timeout
1854 	 * of the above loop as an error, but do report the lack of
1855 	 * running firmware once.
1856 	 */
1857 	if (i >= 100000 && !tg3_flag(tp, NO_FWARE_REPORTED)) {
1858 		tg3_flag_set(tp, NO_FWARE_REPORTED);
1859 
1860 		netdev_info(tp->dev, "No firmware running\n");
1861 	}
1862 
1863 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0) {
1864 		/* The 57765 A0 needs a little more
1865 		 * time to do some important work.
1866 		 */
1867 		mdelay(10);
1868 	}
1869 
1870 	return 0;
1871 }
1872 
1873 static void tg3_link_report(struct tg3 *tp)
1874 {
1875 	if (!netif_carrier_ok(tp->dev)) {
1876 		netif_info(tp, link, tp->dev, "Link is down\n");
1877 		tg3_ump_link_report(tp);
1878 	} else if (netif_msg_link(tp)) {
1879 		netdev_info(tp->dev, "Link is up at %d Mbps, %s duplex\n",
1880 			    (tp->link_config.active_speed == SPEED_1000 ?
1881 			     1000 :
1882 			     (tp->link_config.active_speed == SPEED_100 ?
1883 			      100 : 10)),
1884 			    (tp->link_config.active_duplex == DUPLEX_FULL ?
1885 			     "full" : "half"));
1886 
1887 		netdev_info(tp->dev, "Flow control is %s for TX and %s for RX\n",
1888 			    (tp->link_config.active_flowctrl & FLOW_CTRL_TX) ?
1889 			    "on" : "off",
1890 			    (tp->link_config.active_flowctrl & FLOW_CTRL_RX) ?
1891 			    "on" : "off");
1892 
1893 		if (tp->phy_flags & TG3_PHYFLG_EEE_CAP)
1894 			netdev_info(tp->dev, "EEE is %s\n",
1895 				    tp->setlpicnt ? "enabled" : "disabled");
1896 
1897 		tg3_ump_link_report(tp);
1898 	}
1899 
1900 	tp->link_up = netif_carrier_ok(tp->dev);
1901 }
1902 
1903 static u32 tg3_decode_flowctrl_1000T(u32 adv)
1904 {
1905 	u32 flowctrl = 0;
1906 
1907 	if (adv & ADVERTISE_PAUSE_CAP) {
1908 		flowctrl |= FLOW_CTRL_RX;
1909 		if (!(adv & ADVERTISE_PAUSE_ASYM))
1910 			flowctrl |= FLOW_CTRL_TX;
1911 	} else if (adv & ADVERTISE_PAUSE_ASYM)
1912 		flowctrl |= FLOW_CTRL_TX;
1913 
1914 	return flowctrl;
1915 }
1916 
1917 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1918 {
1919 	u16 miireg;
1920 
1921 	if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1922 		miireg = ADVERTISE_1000XPAUSE;
1923 	else if (flow_ctrl & FLOW_CTRL_TX)
1924 		miireg = ADVERTISE_1000XPSE_ASYM;
1925 	else if (flow_ctrl & FLOW_CTRL_RX)
1926 		miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1927 	else
1928 		miireg = 0;
1929 
1930 	return miireg;
1931 }
1932 
1933 static u32 tg3_decode_flowctrl_1000X(u32 adv)
1934 {
1935 	u32 flowctrl = 0;
1936 
1937 	if (adv & ADVERTISE_1000XPAUSE) {
1938 		flowctrl |= FLOW_CTRL_RX;
1939 		if (!(adv & ADVERTISE_1000XPSE_ASYM))
1940 			flowctrl |= FLOW_CTRL_TX;
1941 	} else if (adv & ADVERTISE_1000XPSE_ASYM)
1942 		flowctrl |= FLOW_CTRL_TX;
1943 
1944 	return flowctrl;
1945 }
1946 
1947 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1948 {
1949 	u8 cap = 0;
1950 
1951 	if (lcladv & rmtadv & ADVERTISE_1000XPAUSE) {
1952 		cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1953 	} else if (lcladv & rmtadv & ADVERTISE_1000XPSE_ASYM) {
1954 		if (lcladv & ADVERTISE_1000XPAUSE)
1955 			cap = FLOW_CTRL_RX;
1956 		if (rmtadv & ADVERTISE_1000XPAUSE)
1957 			cap = FLOW_CTRL_TX;
1958 	}
1959 
1960 	return cap;
1961 }
1962 
1963 static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv)
1964 {
1965 	u8 autoneg;
1966 	u8 flowctrl = 0;
1967 	u32 old_rx_mode = tp->rx_mode;
1968 	u32 old_tx_mode = tp->tx_mode;
1969 
1970 	if (tg3_flag(tp, USE_PHYLIB))
1971 		autoneg = tp->mdio_bus->phy_map[tp->phy_addr]->autoneg;
1972 	else
1973 		autoneg = tp->link_config.autoneg;
1974 
1975 	if (autoneg == AUTONEG_ENABLE && tg3_flag(tp, PAUSE_AUTONEG)) {
1976 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
1977 			flowctrl = tg3_resolve_flowctrl_1000X(lcladv, rmtadv);
1978 		else
1979 			flowctrl = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
1980 	} else
1981 		flowctrl = tp->link_config.flowctrl;
1982 
1983 	tp->link_config.active_flowctrl = flowctrl;
1984 
1985 	if (flowctrl & FLOW_CTRL_RX)
1986 		tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1987 	else
1988 		tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1989 
1990 	if (old_rx_mode != tp->rx_mode)
1991 		tw32_f(MAC_RX_MODE, tp->rx_mode);
1992 
1993 	if (flowctrl & FLOW_CTRL_TX)
1994 		tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1995 	else
1996 		tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1997 
1998 	if (old_tx_mode != tp->tx_mode)
1999 		tw32_f(MAC_TX_MODE, tp->tx_mode);
2000 }
2001 
2002 static void tg3_adjust_link(struct net_device *dev)
2003 {
2004 	u8 oldflowctrl, linkmesg = 0;
2005 	u32 mac_mode, lcl_adv, rmt_adv;
2006 	struct tg3 *tp = netdev_priv(dev);
2007 	struct phy_device *phydev = tp->mdio_bus->phy_map[tp->phy_addr];
2008 
2009 	spin_lock_bh(&tp->lock);
2010 
2011 	mac_mode = tp->mac_mode & ~(MAC_MODE_PORT_MODE_MASK |
2012 				    MAC_MODE_HALF_DUPLEX);
2013 
2014 	oldflowctrl = tp->link_config.active_flowctrl;
2015 
2016 	if (phydev->link) {
2017 		lcl_adv = 0;
2018 		rmt_adv = 0;
2019 
2020 		if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10)
2021 			mac_mode |= MAC_MODE_PORT_MODE_MII;
2022 		else if (phydev->speed == SPEED_1000 ||
2023 			 tg3_asic_rev(tp) != ASIC_REV_5785)
2024 			mac_mode |= MAC_MODE_PORT_MODE_GMII;
2025 		else
2026 			mac_mode |= MAC_MODE_PORT_MODE_MII;
2027 
2028 		if (phydev->duplex == DUPLEX_HALF)
2029 			mac_mode |= MAC_MODE_HALF_DUPLEX;
2030 		else {
2031 			lcl_adv = mii_advertise_flowctrl(
2032 				  tp->link_config.flowctrl);
2033 
2034 			if (phydev->pause)
2035 				rmt_adv = LPA_PAUSE_CAP;
2036 			if (phydev->asym_pause)
2037 				rmt_adv |= LPA_PAUSE_ASYM;
2038 		}
2039 
2040 		tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
2041 	} else
2042 		mac_mode |= MAC_MODE_PORT_MODE_GMII;
2043 
2044 	if (mac_mode != tp->mac_mode) {
2045 		tp->mac_mode = mac_mode;
2046 		tw32_f(MAC_MODE, tp->mac_mode);
2047 		udelay(40);
2048 	}
2049 
2050 	if (tg3_asic_rev(tp) == ASIC_REV_5785) {
2051 		if (phydev->speed == SPEED_10)
2052 			tw32(MAC_MI_STAT,
2053 			     MAC_MI_STAT_10MBPS_MODE |
2054 			     MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
2055 		else
2056 			tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
2057 	}
2058 
2059 	if (phydev->speed == SPEED_1000 && phydev->duplex == DUPLEX_HALF)
2060 		tw32(MAC_TX_LENGTHS,
2061 		     ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2062 		      (6 << TX_LENGTHS_IPG_SHIFT) |
2063 		      (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
2064 	else
2065 		tw32(MAC_TX_LENGTHS,
2066 		     ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2067 		      (6 << TX_LENGTHS_IPG_SHIFT) |
2068 		      (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
2069 
2070 	if (phydev->link != tp->old_link ||
2071 	    phydev->speed != tp->link_config.active_speed ||
2072 	    phydev->duplex != tp->link_config.active_duplex ||
2073 	    oldflowctrl != tp->link_config.active_flowctrl)
2074 		linkmesg = 1;
2075 
2076 	tp->old_link = phydev->link;
2077 	tp->link_config.active_speed = phydev->speed;
2078 	tp->link_config.active_duplex = phydev->duplex;
2079 
2080 	spin_unlock_bh(&tp->lock);
2081 
2082 	if (linkmesg)
2083 		tg3_link_report(tp);
2084 }
2085 
2086 static int tg3_phy_init(struct tg3 *tp)
2087 {
2088 	struct phy_device *phydev;
2089 
2090 	if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED)
2091 		return 0;
2092 
2093 	/* Bring the PHY back to a known state. */
2094 	tg3_bmcr_reset(tp);
2095 
2096 	phydev = tp->mdio_bus->phy_map[tp->phy_addr];
2097 
2098 	/* Attach the MAC to the PHY. */
2099 	phydev = phy_connect(tp->dev, dev_name(&phydev->dev),
2100 			     tg3_adjust_link, phydev->interface);
2101 	if (IS_ERR(phydev)) {
2102 		dev_err(&tp->pdev->dev, "Could not attach to PHY\n");
2103 		return PTR_ERR(phydev);
2104 	}
2105 
2106 	/* Mask with MAC supported features. */
2107 	switch (phydev->interface) {
2108 	case PHY_INTERFACE_MODE_GMII:
2109 	case PHY_INTERFACE_MODE_RGMII:
2110 		if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
2111 			phydev->supported &= (PHY_GBIT_FEATURES |
2112 					      SUPPORTED_Pause |
2113 					      SUPPORTED_Asym_Pause);
2114 			break;
2115 		}
2116 		/* fallthru */
2117 	case PHY_INTERFACE_MODE_MII:
2118 		phydev->supported &= (PHY_BASIC_FEATURES |
2119 				      SUPPORTED_Pause |
2120 				      SUPPORTED_Asym_Pause);
2121 		break;
2122 	default:
2123 		phy_disconnect(tp->mdio_bus->phy_map[tp->phy_addr]);
2124 		return -EINVAL;
2125 	}
2126 
2127 	tp->phy_flags |= TG3_PHYFLG_IS_CONNECTED;
2128 
2129 	phydev->advertising = phydev->supported;
2130 
2131 	return 0;
2132 }
2133 
2134 static void tg3_phy_start(struct tg3 *tp)
2135 {
2136 	struct phy_device *phydev;
2137 
2138 	if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
2139 		return;
2140 
2141 	phydev = tp->mdio_bus->phy_map[tp->phy_addr];
2142 
2143 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
2144 		tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
2145 		phydev->speed = tp->link_config.speed;
2146 		phydev->duplex = tp->link_config.duplex;
2147 		phydev->autoneg = tp->link_config.autoneg;
2148 		phydev->advertising = tp->link_config.advertising;
2149 	}
2150 
2151 	phy_start(phydev);
2152 
2153 	phy_start_aneg(phydev);
2154 }
2155 
2156 static void tg3_phy_stop(struct tg3 *tp)
2157 {
2158 	if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
2159 		return;
2160 
2161 	phy_stop(tp->mdio_bus->phy_map[tp->phy_addr]);
2162 }
2163 
2164 static void tg3_phy_fini(struct tg3 *tp)
2165 {
2166 	if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
2167 		phy_disconnect(tp->mdio_bus->phy_map[tp->phy_addr]);
2168 		tp->phy_flags &= ~TG3_PHYFLG_IS_CONNECTED;
2169 	}
2170 }
2171 
2172 static int tg3_phy_set_extloopbk(struct tg3 *tp)
2173 {
2174 	int err;
2175 	u32 val;
2176 
2177 	if (tp->phy_flags & TG3_PHYFLG_IS_FET)
2178 		return 0;
2179 
2180 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
2181 		/* Cannot do read-modify-write on 5401 */
2182 		err = tg3_phy_auxctl_write(tp,
2183 					   MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
2184 					   MII_TG3_AUXCTL_ACTL_EXTLOOPBK |
2185 					   0x4c20);
2186 		goto done;
2187 	}
2188 
2189 	err = tg3_phy_auxctl_read(tp,
2190 				  MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
2191 	if (err)
2192 		return err;
2193 
2194 	val |= MII_TG3_AUXCTL_ACTL_EXTLOOPBK;
2195 	err = tg3_phy_auxctl_write(tp,
2196 				   MII_TG3_AUXCTL_SHDWSEL_AUXCTL, val);
2197 
2198 done:
2199 	return err;
2200 }
2201 
2202 static void tg3_phy_fet_toggle_apd(struct tg3 *tp, bool enable)
2203 {
2204 	u32 phytest;
2205 
2206 	if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
2207 		u32 phy;
2208 
2209 		tg3_writephy(tp, MII_TG3_FET_TEST,
2210 			     phytest | MII_TG3_FET_SHADOW_EN);
2211 		if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXSTAT2, &phy)) {
2212 			if (enable)
2213 				phy |= MII_TG3_FET_SHDW_AUXSTAT2_APD;
2214 			else
2215 				phy &= ~MII_TG3_FET_SHDW_AUXSTAT2_APD;
2216 			tg3_writephy(tp, MII_TG3_FET_SHDW_AUXSTAT2, phy);
2217 		}
2218 		tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
2219 	}
2220 }
2221 
2222 static void tg3_phy_toggle_apd(struct tg3 *tp, bool enable)
2223 {
2224 	u32 reg;
2225 
2226 	if (!tg3_flag(tp, 5705_PLUS) ||
2227 	    (tg3_flag(tp, 5717_PLUS) &&
2228 	     (tp->phy_flags & TG3_PHYFLG_MII_SERDES)))
2229 		return;
2230 
2231 	if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2232 		tg3_phy_fet_toggle_apd(tp, enable);
2233 		return;
2234 	}
2235 
2236 	reg = MII_TG3_MISC_SHDW_SCR5_LPED |
2237 	      MII_TG3_MISC_SHDW_SCR5_DLPTLM |
2238 	      MII_TG3_MISC_SHDW_SCR5_SDTL |
2239 	      MII_TG3_MISC_SHDW_SCR5_C125OE;
2240 	if (tg3_asic_rev(tp) != ASIC_REV_5784 || !enable)
2241 		reg |= MII_TG3_MISC_SHDW_SCR5_DLLAPD;
2242 
2243 	tg3_phy_shdw_write(tp, MII_TG3_MISC_SHDW_SCR5_SEL, reg);
2244 
2245 
2246 	reg = MII_TG3_MISC_SHDW_APD_WKTM_84MS;
2247 	if (enable)
2248 		reg |= MII_TG3_MISC_SHDW_APD_ENABLE;
2249 
2250 	tg3_phy_shdw_write(tp, MII_TG3_MISC_SHDW_APD_SEL, reg);
2251 }
2252 
2253 static void tg3_phy_toggle_automdix(struct tg3 *tp, bool enable)
2254 {
2255 	u32 phy;
2256 
2257 	if (!tg3_flag(tp, 5705_PLUS) ||
2258 	    (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
2259 		return;
2260 
2261 	if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2262 		u32 ephy;
2263 
2264 		if (!tg3_readphy(tp, MII_TG3_FET_TEST, &ephy)) {
2265 			u32 reg = MII_TG3_FET_SHDW_MISCCTRL;
2266 
2267 			tg3_writephy(tp, MII_TG3_FET_TEST,
2268 				     ephy | MII_TG3_FET_SHADOW_EN);
2269 			if (!tg3_readphy(tp, reg, &phy)) {
2270 				if (enable)
2271 					phy |= MII_TG3_FET_SHDW_MISCCTRL_MDIX;
2272 				else
2273 					phy &= ~MII_TG3_FET_SHDW_MISCCTRL_MDIX;
2274 				tg3_writephy(tp, reg, phy);
2275 			}
2276 			tg3_writephy(tp, MII_TG3_FET_TEST, ephy);
2277 		}
2278 	} else {
2279 		int ret;
2280 
2281 		ret = tg3_phy_auxctl_read(tp,
2282 					  MII_TG3_AUXCTL_SHDWSEL_MISC, &phy);
2283 		if (!ret) {
2284 			if (enable)
2285 				phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
2286 			else
2287 				phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
2288 			tg3_phy_auxctl_write(tp,
2289 					     MII_TG3_AUXCTL_SHDWSEL_MISC, phy);
2290 		}
2291 	}
2292 }
2293 
2294 static void tg3_phy_set_wirespeed(struct tg3 *tp)
2295 {
2296 	int ret;
2297 	u32 val;
2298 
2299 	if (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED)
2300 		return;
2301 
2302 	ret = tg3_phy_auxctl_read(tp, MII_TG3_AUXCTL_SHDWSEL_MISC, &val);
2303 	if (!ret)
2304 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_MISC,
2305 				     val | MII_TG3_AUXCTL_MISC_WIRESPD_EN);
2306 }
2307 
2308 static void tg3_phy_apply_otp(struct tg3 *tp)
2309 {
2310 	u32 otp, phy;
2311 
2312 	if (!tp->phy_otp)
2313 		return;
2314 
2315 	otp = tp->phy_otp;
2316 
2317 	if (tg3_phy_toggle_auxctl_smdsp(tp, true))
2318 		return;
2319 
2320 	phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
2321 	phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
2322 	tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
2323 
2324 	phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
2325 	      ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
2326 	tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
2327 
2328 	phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
2329 	phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
2330 	tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
2331 
2332 	phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
2333 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
2334 
2335 	phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
2336 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
2337 
2338 	phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
2339 	      ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
2340 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
2341 
2342 	tg3_phy_toggle_auxctl_smdsp(tp, false);
2343 }
2344 
2345 static void tg3_eee_pull_config(struct tg3 *tp, struct ethtool_eee *eee)
2346 {
2347 	u32 val;
2348 	struct ethtool_eee *dest = &tp->eee;
2349 
2350 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
2351 		return;
2352 
2353 	if (eee)
2354 		dest = eee;
2355 
2356 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, TG3_CL45_D7_EEERES_STAT, &val))
2357 		return;
2358 
2359 	/* Pull eee_active */
2360 	if (val == TG3_CL45_D7_EEERES_STAT_LP_1000T ||
2361 	    val == TG3_CL45_D7_EEERES_STAT_LP_100TX) {
2362 		dest->eee_active = 1;
2363 	} else
2364 		dest->eee_active = 0;
2365 
2366 	/* Pull lp advertised settings */
2367 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, MDIO_AN_EEE_LPABLE, &val))
2368 		return;
2369 	dest->lp_advertised = mmd_eee_adv_to_ethtool_adv_t(val);
2370 
2371 	/* Pull advertised and eee_enabled settings */
2372 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, &val))
2373 		return;
2374 	dest->eee_enabled = !!val;
2375 	dest->advertised = mmd_eee_adv_to_ethtool_adv_t(val);
2376 
2377 	/* Pull tx_lpi_enabled */
2378 	val = tr32(TG3_CPMU_EEE_MODE);
2379 	dest->tx_lpi_enabled = !!(val & TG3_CPMU_EEEMD_LPI_IN_TX);
2380 
2381 	/* Pull lpi timer value */
2382 	dest->tx_lpi_timer = tr32(TG3_CPMU_EEE_DBTMR1) & 0xffff;
2383 }
2384 
2385 static void tg3_phy_eee_adjust(struct tg3 *tp, bool current_link_up)
2386 {
2387 	u32 val;
2388 
2389 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
2390 		return;
2391 
2392 	tp->setlpicnt = 0;
2393 
2394 	if (tp->link_config.autoneg == AUTONEG_ENABLE &&
2395 	    current_link_up &&
2396 	    tp->link_config.active_duplex == DUPLEX_FULL &&
2397 	    (tp->link_config.active_speed == SPEED_100 ||
2398 	     tp->link_config.active_speed == SPEED_1000)) {
2399 		u32 eeectl;
2400 
2401 		if (tp->link_config.active_speed == SPEED_1000)
2402 			eeectl = TG3_CPMU_EEE_CTRL_EXIT_16_5_US;
2403 		else
2404 			eeectl = TG3_CPMU_EEE_CTRL_EXIT_36_US;
2405 
2406 		tw32(TG3_CPMU_EEE_CTRL, eeectl);
2407 
2408 		tg3_eee_pull_config(tp, NULL);
2409 		if (tp->eee.eee_active)
2410 			tp->setlpicnt = 2;
2411 	}
2412 
2413 	if (!tp->setlpicnt) {
2414 		if (current_link_up &&
2415 		   !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2416 			tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, 0x0000);
2417 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2418 		}
2419 
2420 		val = tr32(TG3_CPMU_EEE_MODE);
2421 		tw32(TG3_CPMU_EEE_MODE, val & ~TG3_CPMU_EEEMD_LPI_ENABLE);
2422 	}
2423 }
2424 
2425 static void tg3_phy_eee_enable(struct tg3 *tp)
2426 {
2427 	u32 val;
2428 
2429 	if (tp->link_config.active_speed == SPEED_1000 &&
2430 	    (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2431 	     tg3_asic_rev(tp) == ASIC_REV_5719 ||
2432 	     tg3_flag(tp, 57765_CLASS)) &&
2433 	    !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2434 		val = MII_TG3_DSP_TAP26_ALNOKO |
2435 		      MII_TG3_DSP_TAP26_RMRXSTO;
2436 		tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
2437 		tg3_phy_toggle_auxctl_smdsp(tp, false);
2438 	}
2439 
2440 	val = tr32(TG3_CPMU_EEE_MODE);
2441 	tw32(TG3_CPMU_EEE_MODE, val | TG3_CPMU_EEEMD_LPI_ENABLE);
2442 }
2443 
2444 static int tg3_wait_macro_done(struct tg3 *tp)
2445 {
2446 	int limit = 100;
2447 
2448 	while (limit--) {
2449 		u32 tmp32;
2450 
2451 		if (!tg3_readphy(tp, MII_TG3_DSP_CONTROL, &tmp32)) {
2452 			if ((tmp32 & 0x1000) == 0)
2453 				break;
2454 		}
2455 	}
2456 	if (limit < 0)
2457 		return -EBUSY;
2458 
2459 	return 0;
2460 }
2461 
2462 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
2463 {
2464 	static const u32 test_pat[4][6] = {
2465 	{ 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
2466 	{ 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
2467 	{ 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
2468 	{ 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
2469 	};
2470 	int chan;
2471 
2472 	for (chan = 0; chan < 4; chan++) {
2473 		int i;
2474 
2475 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2476 			     (chan * 0x2000) | 0x0200);
2477 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
2478 
2479 		for (i = 0; i < 6; i++)
2480 			tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
2481 				     test_pat[chan][i]);
2482 
2483 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
2484 		if (tg3_wait_macro_done(tp)) {
2485 			*resetp = 1;
2486 			return -EBUSY;
2487 		}
2488 
2489 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2490 			     (chan * 0x2000) | 0x0200);
2491 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0082);
2492 		if (tg3_wait_macro_done(tp)) {
2493 			*resetp = 1;
2494 			return -EBUSY;
2495 		}
2496 
2497 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0802);
2498 		if (tg3_wait_macro_done(tp)) {
2499 			*resetp = 1;
2500 			return -EBUSY;
2501 		}
2502 
2503 		for (i = 0; i < 6; i += 2) {
2504 			u32 low, high;
2505 
2506 			if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
2507 			    tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
2508 			    tg3_wait_macro_done(tp)) {
2509 				*resetp = 1;
2510 				return -EBUSY;
2511 			}
2512 			low &= 0x7fff;
2513 			high &= 0x000f;
2514 			if (low != test_pat[chan][i] ||
2515 			    high != test_pat[chan][i+1]) {
2516 				tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
2517 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
2518 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
2519 
2520 				return -EBUSY;
2521 			}
2522 		}
2523 	}
2524 
2525 	return 0;
2526 }
2527 
2528 static int tg3_phy_reset_chanpat(struct tg3 *tp)
2529 {
2530 	int chan;
2531 
2532 	for (chan = 0; chan < 4; chan++) {
2533 		int i;
2534 
2535 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2536 			     (chan * 0x2000) | 0x0200);
2537 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
2538 		for (i = 0; i < 6; i++)
2539 			tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
2540 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
2541 		if (tg3_wait_macro_done(tp))
2542 			return -EBUSY;
2543 	}
2544 
2545 	return 0;
2546 }
2547 
2548 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
2549 {
2550 	u32 reg32, phy9_orig;
2551 	int retries, do_phy_reset, err;
2552 
2553 	retries = 10;
2554 	do_phy_reset = 1;
2555 	do {
2556 		if (do_phy_reset) {
2557 			err = tg3_bmcr_reset(tp);
2558 			if (err)
2559 				return err;
2560 			do_phy_reset = 0;
2561 		}
2562 
2563 		/* Disable transmitter and interrupt.  */
2564 		if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
2565 			continue;
2566 
2567 		reg32 |= 0x3000;
2568 		tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2569 
2570 		/* Set full-duplex, 1000 mbps.  */
2571 		tg3_writephy(tp, MII_BMCR,
2572 			     BMCR_FULLDPLX | BMCR_SPEED1000);
2573 
2574 		/* Set to master mode.  */
2575 		if (tg3_readphy(tp, MII_CTRL1000, &phy9_orig))
2576 			continue;
2577 
2578 		tg3_writephy(tp, MII_CTRL1000,
2579 			     CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER);
2580 
2581 		err = tg3_phy_toggle_auxctl_smdsp(tp, true);
2582 		if (err)
2583 			return err;
2584 
2585 		/* Block the PHY control access.  */
2586 		tg3_phydsp_write(tp, 0x8005, 0x0800);
2587 
2588 		err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
2589 		if (!err)
2590 			break;
2591 	} while (--retries);
2592 
2593 	err = tg3_phy_reset_chanpat(tp);
2594 	if (err)
2595 		return err;
2596 
2597 	tg3_phydsp_write(tp, 0x8005, 0x0000);
2598 
2599 	tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
2600 	tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0000);
2601 
2602 	tg3_phy_toggle_auxctl_smdsp(tp, false);
2603 
2604 	tg3_writephy(tp, MII_CTRL1000, phy9_orig);
2605 
2606 	err = tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32);
2607 	if (err)
2608 		return err;
2609 
2610 	reg32 &= ~0x3000;
2611 	tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2612 
2613 	return 0;
2614 }
2615 
2616 static void tg3_carrier_off(struct tg3 *tp)
2617 {
2618 	netif_carrier_off(tp->dev);
2619 	tp->link_up = false;
2620 }
2621 
2622 static void tg3_warn_mgmt_link_flap(struct tg3 *tp)
2623 {
2624 	if (tg3_flag(tp, ENABLE_ASF))
2625 		netdev_warn(tp->dev,
2626 			    "Management side-band traffic will be interrupted during phy settings change\n");
2627 }
2628 
2629 /* This will reset the tigon3 PHY if there is no valid
2630  * link unless the FORCE argument is non-zero.
2631  */
2632 static int tg3_phy_reset(struct tg3 *tp)
2633 {
2634 	u32 val, cpmuctrl;
2635 	int err;
2636 
2637 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
2638 		val = tr32(GRC_MISC_CFG);
2639 		tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
2640 		udelay(40);
2641 	}
2642 	err  = tg3_readphy(tp, MII_BMSR, &val);
2643 	err |= tg3_readphy(tp, MII_BMSR, &val);
2644 	if (err != 0)
2645 		return -EBUSY;
2646 
2647 	if (netif_running(tp->dev) && tp->link_up) {
2648 		netif_carrier_off(tp->dev);
2649 		tg3_link_report(tp);
2650 	}
2651 
2652 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
2653 	    tg3_asic_rev(tp) == ASIC_REV_5704 ||
2654 	    tg3_asic_rev(tp) == ASIC_REV_5705) {
2655 		err = tg3_phy_reset_5703_4_5(tp);
2656 		if (err)
2657 			return err;
2658 		goto out;
2659 	}
2660 
2661 	cpmuctrl = 0;
2662 	if (tg3_asic_rev(tp) == ASIC_REV_5784 &&
2663 	    tg3_chip_rev(tp) != CHIPREV_5784_AX) {
2664 		cpmuctrl = tr32(TG3_CPMU_CTRL);
2665 		if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
2666 			tw32(TG3_CPMU_CTRL,
2667 			     cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
2668 	}
2669 
2670 	err = tg3_bmcr_reset(tp);
2671 	if (err)
2672 		return err;
2673 
2674 	if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
2675 		val = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
2676 		tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, val);
2677 
2678 		tw32(TG3_CPMU_CTRL, cpmuctrl);
2679 	}
2680 
2681 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX ||
2682 	    tg3_chip_rev(tp) == CHIPREV_5761_AX) {
2683 		val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2684 		if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
2685 		    CPMU_LSPD_1000MB_MACCLK_12_5) {
2686 			val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2687 			udelay(40);
2688 			tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2689 		}
2690 	}
2691 
2692 	if (tg3_flag(tp, 5717_PLUS) &&
2693 	    (tp->phy_flags & TG3_PHYFLG_MII_SERDES))
2694 		return 0;
2695 
2696 	tg3_phy_apply_otp(tp);
2697 
2698 	if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
2699 		tg3_phy_toggle_apd(tp, true);
2700 	else
2701 		tg3_phy_toggle_apd(tp, false);
2702 
2703 out:
2704 	if ((tp->phy_flags & TG3_PHYFLG_ADC_BUG) &&
2705 	    !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2706 		tg3_phydsp_write(tp, 0x201f, 0x2aaa);
2707 		tg3_phydsp_write(tp, 0x000a, 0x0323);
2708 		tg3_phy_toggle_auxctl_smdsp(tp, false);
2709 	}
2710 
2711 	if (tp->phy_flags & TG3_PHYFLG_5704_A0_BUG) {
2712 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2713 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2714 	}
2715 
2716 	if (tp->phy_flags & TG3_PHYFLG_BER_BUG) {
2717 		if (!tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2718 			tg3_phydsp_write(tp, 0x000a, 0x310b);
2719 			tg3_phydsp_write(tp, 0x201f, 0x9506);
2720 			tg3_phydsp_write(tp, 0x401f, 0x14e2);
2721 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2722 		}
2723 	} else if (tp->phy_flags & TG3_PHYFLG_JITTER_BUG) {
2724 		if (!tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2725 			tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
2726 			if (tp->phy_flags & TG3_PHYFLG_ADJUST_TRIM) {
2727 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
2728 				tg3_writephy(tp, MII_TG3_TEST1,
2729 					     MII_TG3_TEST1_TRIM_EN | 0x4);
2730 			} else
2731 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
2732 
2733 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2734 		}
2735 	}
2736 
2737 	/* Set Extended packet length bit (bit 14) on all chips that */
2738 	/* support jumbo frames */
2739 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
2740 		/* Cannot do read-modify-write on 5401 */
2741 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
2742 	} else if (tg3_flag(tp, JUMBO_CAPABLE)) {
2743 		/* Set bit 14 with read-modify-write to preserve other bits */
2744 		err = tg3_phy_auxctl_read(tp,
2745 					  MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
2746 		if (!err)
2747 			tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
2748 					   val | MII_TG3_AUXCTL_ACTL_EXTPKTLEN);
2749 	}
2750 
2751 	/* Set phy register 0x10 bit 0 to high fifo elasticity to support
2752 	 * jumbo frames transmission.
2753 	 */
2754 	if (tg3_flag(tp, JUMBO_CAPABLE)) {
2755 		if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &val))
2756 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
2757 				     val | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
2758 	}
2759 
2760 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
2761 		/* adjust output voltage */
2762 		tg3_writephy(tp, MII_TG3_FET_PTEST, 0x12);
2763 	}
2764 
2765 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5762_A0)
2766 		tg3_phydsp_write(tp, 0xffb, 0x4000);
2767 
2768 	tg3_phy_toggle_automdix(tp, true);
2769 	tg3_phy_set_wirespeed(tp);
2770 	return 0;
2771 }
2772 
2773 #define TG3_GPIO_MSG_DRVR_PRES		 0x00000001
2774 #define TG3_GPIO_MSG_NEED_VAUX		 0x00000002
2775 #define TG3_GPIO_MSG_MASK		 (TG3_GPIO_MSG_DRVR_PRES | \
2776 					  TG3_GPIO_MSG_NEED_VAUX)
2777 #define TG3_GPIO_MSG_ALL_DRVR_PRES_MASK \
2778 	((TG3_GPIO_MSG_DRVR_PRES << 0) | \
2779 	 (TG3_GPIO_MSG_DRVR_PRES << 4) | \
2780 	 (TG3_GPIO_MSG_DRVR_PRES << 8) | \
2781 	 (TG3_GPIO_MSG_DRVR_PRES << 12))
2782 
2783 #define TG3_GPIO_MSG_ALL_NEED_VAUX_MASK \
2784 	((TG3_GPIO_MSG_NEED_VAUX << 0) | \
2785 	 (TG3_GPIO_MSG_NEED_VAUX << 4) | \
2786 	 (TG3_GPIO_MSG_NEED_VAUX << 8) | \
2787 	 (TG3_GPIO_MSG_NEED_VAUX << 12))
2788 
2789 static inline u32 tg3_set_function_status(struct tg3 *tp, u32 newstat)
2790 {
2791 	u32 status, shift;
2792 
2793 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2794 	    tg3_asic_rev(tp) == ASIC_REV_5719)
2795 		status = tg3_ape_read32(tp, TG3_APE_GPIO_MSG);
2796 	else
2797 		status = tr32(TG3_CPMU_DRV_STATUS);
2798 
2799 	shift = TG3_APE_GPIO_MSG_SHIFT + 4 * tp->pci_fn;
2800 	status &= ~(TG3_GPIO_MSG_MASK << shift);
2801 	status |= (newstat << shift);
2802 
2803 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2804 	    tg3_asic_rev(tp) == ASIC_REV_5719)
2805 		tg3_ape_write32(tp, TG3_APE_GPIO_MSG, status);
2806 	else
2807 		tw32(TG3_CPMU_DRV_STATUS, status);
2808 
2809 	return status >> TG3_APE_GPIO_MSG_SHIFT;
2810 }
2811 
2812 static inline int tg3_pwrsrc_switch_to_vmain(struct tg3 *tp)
2813 {
2814 	if (!tg3_flag(tp, IS_NIC))
2815 		return 0;
2816 
2817 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2818 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
2819 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
2820 		if (tg3_ape_lock(tp, TG3_APE_LOCK_GPIO))
2821 			return -EIO;
2822 
2823 		tg3_set_function_status(tp, TG3_GPIO_MSG_DRVR_PRES);
2824 
2825 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl,
2826 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2827 
2828 		tg3_ape_unlock(tp, TG3_APE_LOCK_GPIO);
2829 	} else {
2830 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl,
2831 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2832 	}
2833 
2834 	return 0;
2835 }
2836 
2837 static void tg3_pwrsrc_die_with_vmain(struct tg3 *tp)
2838 {
2839 	u32 grc_local_ctrl;
2840 
2841 	if (!tg3_flag(tp, IS_NIC) ||
2842 	    tg3_asic_rev(tp) == ASIC_REV_5700 ||
2843 	    tg3_asic_rev(tp) == ASIC_REV_5701)
2844 		return;
2845 
2846 	grc_local_ctrl = tp->grc_local_ctrl | GRC_LCLCTRL_GPIO_OE1;
2847 
2848 	tw32_wait_f(GRC_LOCAL_CTRL,
2849 		    grc_local_ctrl | GRC_LCLCTRL_GPIO_OUTPUT1,
2850 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2851 
2852 	tw32_wait_f(GRC_LOCAL_CTRL,
2853 		    grc_local_ctrl,
2854 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2855 
2856 	tw32_wait_f(GRC_LOCAL_CTRL,
2857 		    grc_local_ctrl | GRC_LCLCTRL_GPIO_OUTPUT1,
2858 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2859 }
2860 
2861 static void tg3_pwrsrc_switch_to_vaux(struct tg3 *tp)
2862 {
2863 	if (!tg3_flag(tp, IS_NIC))
2864 		return;
2865 
2866 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
2867 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
2868 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2869 			    (GRC_LCLCTRL_GPIO_OE0 |
2870 			     GRC_LCLCTRL_GPIO_OE1 |
2871 			     GRC_LCLCTRL_GPIO_OE2 |
2872 			     GRC_LCLCTRL_GPIO_OUTPUT0 |
2873 			     GRC_LCLCTRL_GPIO_OUTPUT1),
2874 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2875 	} else if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
2876 		   tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
2877 		/* The 5761 non-e device swaps GPIO 0 and GPIO 2. */
2878 		u32 grc_local_ctrl = GRC_LCLCTRL_GPIO_OE0 |
2879 				     GRC_LCLCTRL_GPIO_OE1 |
2880 				     GRC_LCLCTRL_GPIO_OE2 |
2881 				     GRC_LCLCTRL_GPIO_OUTPUT0 |
2882 				     GRC_LCLCTRL_GPIO_OUTPUT1 |
2883 				     tp->grc_local_ctrl;
2884 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2885 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2886 
2887 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT2;
2888 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2889 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2890 
2891 		grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT0;
2892 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2893 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2894 	} else {
2895 		u32 no_gpio2;
2896 		u32 grc_local_ctrl = 0;
2897 
2898 		/* Workaround to prevent overdrawing Amps. */
2899 		if (tg3_asic_rev(tp) == ASIC_REV_5714) {
2900 			grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
2901 			tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2902 				    grc_local_ctrl,
2903 				    TG3_GRC_LCLCTL_PWRSW_DELAY);
2904 		}
2905 
2906 		/* On 5753 and variants, GPIO2 cannot be used. */
2907 		no_gpio2 = tp->nic_sram_data_cfg &
2908 			   NIC_SRAM_DATA_CFG_NO_GPIO2;
2909 
2910 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
2911 				  GRC_LCLCTRL_GPIO_OE1 |
2912 				  GRC_LCLCTRL_GPIO_OE2 |
2913 				  GRC_LCLCTRL_GPIO_OUTPUT1 |
2914 				  GRC_LCLCTRL_GPIO_OUTPUT2;
2915 		if (no_gpio2) {
2916 			grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
2917 					    GRC_LCLCTRL_GPIO_OUTPUT2);
2918 		}
2919 		tw32_wait_f(GRC_LOCAL_CTRL,
2920 			    tp->grc_local_ctrl | grc_local_ctrl,
2921 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2922 
2923 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
2924 
2925 		tw32_wait_f(GRC_LOCAL_CTRL,
2926 			    tp->grc_local_ctrl | grc_local_ctrl,
2927 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2928 
2929 		if (!no_gpio2) {
2930 			grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
2931 			tw32_wait_f(GRC_LOCAL_CTRL,
2932 				    tp->grc_local_ctrl | grc_local_ctrl,
2933 				    TG3_GRC_LCLCTL_PWRSW_DELAY);
2934 		}
2935 	}
2936 }
2937 
2938 static void tg3_frob_aux_power_5717(struct tg3 *tp, bool wol_enable)
2939 {
2940 	u32 msg = 0;
2941 
2942 	/* Serialize power state transitions */
2943 	if (tg3_ape_lock(tp, TG3_APE_LOCK_GPIO))
2944 		return;
2945 
2946 	if (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE) || wol_enable)
2947 		msg = TG3_GPIO_MSG_NEED_VAUX;
2948 
2949 	msg = tg3_set_function_status(tp, msg);
2950 
2951 	if (msg & TG3_GPIO_MSG_ALL_DRVR_PRES_MASK)
2952 		goto done;
2953 
2954 	if (msg & TG3_GPIO_MSG_ALL_NEED_VAUX_MASK)
2955 		tg3_pwrsrc_switch_to_vaux(tp);
2956 	else
2957 		tg3_pwrsrc_die_with_vmain(tp);
2958 
2959 done:
2960 	tg3_ape_unlock(tp, TG3_APE_LOCK_GPIO);
2961 }
2962 
2963 static void tg3_frob_aux_power(struct tg3 *tp, bool include_wol)
2964 {
2965 	bool need_vaux = false;
2966 
2967 	/* The GPIOs do something completely different on 57765. */
2968 	if (!tg3_flag(tp, IS_NIC) || tg3_flag(tp, 57765_CLASS))
2969 		return;
2970 
2971 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2972 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
2973 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
2974 		tg3_frob_aux_power_5717(tp, include_wol ?
2975 					tg3_flag(tp, WOL_ENABLE) != 0 : 0);
2976 		return;
2977 	}
2978 
2979 	if (tp->pdev_peer && tp->pdev_peer != tp->pdev) {
2980 		struct net_device *dev_peer;
2981 
2982 		dev_peer = pci_get_drvdata(tp->pdev_peer);
2983 
2984 		/* remove_one() may have been run on the peer. */
2985 		if (dev_peer) {
2986 			struct tg3 *tp_peer = netdev_priv(dev_peer);
2987 
2988 			if (tg3_flag(tp_peer, INIT_COMPLETE))
2989 				return;
2990 
2991 			if ((include_wol && tg3_flag(tp_peer, WOL_ENABLE)) ||
2992 			    tg3_flag(tp_peer, ENABLE_ASF))
2993 				need_vaux = true;
2994 		}
2995 	}
2996 
2997 	if ((include_wol && tg3_flag(tp, WOL_ENABLE)) ||
2998 	    tg3_flag(tp, ENABLE_ASF))
2999 		need_vaux = true;
3000 
3001 	if (need_vaux)
3002 		tg3_pwrsrc_switch_to_vaux(tp);
3003 	else
3004 		tg3_pwrsrc_die_with_vmain(tp);
3005 }
3006 
3007 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
3008 {
3009 	if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
3010 		return 1;
3011 	else if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411) {
3012 		if (speed != SPEED_10)
3013 			return 1;
3014 	} else if (speed == SPEED_10)
3015 		return 1;
3016 
3017 	return 0;
3018 }
3019 
3020 static bool tg3_phy_power_bug(struct tg3 *tp)
3021 {
3022 	switch (tg3_asic_rev(tp)) {
3023 	case ASIC_REV_5700:
3024 	case ASIC_REV_5704:
3025 		return true;
3026 	case ASIC_REV_5780:
3027 		if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
3028 			return true;
3029 		return false;
3030 	case ASIC_REV_5717:
3031 		if (!tp->pci_fn)
3032 			return true;
3033 		return false;
3034 	case ASIC_REV_5719:
3035 	case ASIC_REV_5720:
3036 		if ((tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
3037 		    !tp->pci_fn)
3038 			return true;
3039 		return false;
3040 	}
3041 
3042 	return false;
3043 }
3044 
3045 static bool tg3_phy_led_bug(struct tg3 *tp)
3046 {
3047 	switch (tg3_asic_rev(tp)) {
3048 	case ASIC_REV_5719:
3049 	case ASIC_REV_5720:
3050 		if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
3051 		    !tp->pci_fn)
3052 			return true;
3053 		return false;
3054 	}
3055 
3056 	return false;
3057 }
3058 
3059 static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power)
3060 {
3061 	u32 val;
3062 
3063 	if (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)
3064 		return;
3065 
3066 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
3067 		if (tg3_asic_rev(tp) == ASIC_REV_5704) {
3068 			u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
3069 			u32 serdes_cfg = tr32(MAC_SERDES_CFG);
3070 
3071 			sg_dig_ctrl |=
3072 				SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
3073 			tw32(SG_DIG_CTRL, sg_dig_ctrl);
3074 			tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
3075 		}
3076 		return;
3077 	}
3078 
3079 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
3080 		tg3_bmcr_reset(tp);
3081 		val = tr32(GRC_MISC_CFG);
3082 		tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
3083 		udelay(40);
3084 		return;
3085 	} else if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
3086 		u32 phytest;
3087 		if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
3088 			u32 phy;
3089 
3090 			tg3_writephy(tp, MII_ADVERTISE, 0);
3091 			tg3_writephy(tp, MII_BMCR,
3092 				     BMCR_ANENABLE | BMCR_ANRESTART);
3093 
3094 			tg3_writephy(tp, MII_TG3_FET_TEST,
3095 				     phytest | MII_TG3_FET_SHADOW_EN);
3096 			if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXMODE4, &phy)) {
3097 				phy |= MII_TG3_FET_SHDW_AUXMODE4_SBPD;
3098 				tg3_writephy(tp,
3099 					     MII_TG3_FET_SHDW_AUXMODE4,
3100 					     phy);
3101 			}
3102 			tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
3103 		}
3104 		return;
3105 	} else if (do_low_power) {
3106 		if (!tg3_phy_led_bug(tp))
3107 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
3108 				     MII_TG3_EXT_CTRL_FORCE_LED_OFF);
3109 
3110 		val = MII_TG3_AUXCTL_PCTL_100TX_LPWR |
3111 		      MII_TG3_AUXCTL_PCTL_SPR_ISOLATE |
3112 		      MII_TG3_AUXCTL_PCTL_VREG_11V;
3113 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, val);
3114 	}
3115 
3116 	/* The PHY should not be powered down on some chips because
3117 	 * of bugs.
3118 	 */
3119 	if (tg3_phy_power_bug(tp))
3120 		return;
3121 
3122 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX ||
3123 	    tg3_chip_rev(tp) == CHIPREV_5761_AX) {
3124 		val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
3125 		val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
3126 		val |= CPMU_LSPD_1000MB_MACCLK_12_5;
3127 		tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
3128 	}
3129 
3130 	tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
3131 }
3132 
3133 /* tp->lock is held. */
3134 static int tg3_nvram_lock(struct tg3 *tp)
3135 {
3136 	if (tg3_flag(tp, NVRAM)) {
3137 		int i;
3138 
3139 		if (tp->nvram_lock_cnt == 0) {
3140 			tw32(NVRAM_SWARB, SWARB_REQ_SET1);
3141 			for (i = 0; i < 8000; i++) {
3142 				if (tr32(NVRAM_SWARB) & SWARB_GNT1)
3143 					break;
3144 				udelay(20);
3145 			}
3146 			if (i == 8000) {
3147 				tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
3148 				return -ENODEV;
3149 			}
3150 		}
3151 		tp->nvram_lock_cnt++;
3152 	}
3153 	return 0;
3154 }
3155 
3156 /* tp->lock is held. */
3157 static void tg3_nvram_unlock(struct tg3 *tp)
3158 {
3159 	if (tg3_flag(tp, NVRAM)) {
3160 		if (tp->nvram_lock_cnt > 0)
3161 			tp->nvram_lock_cnt--;
3162 		if (tp->nvram_lock_cnt == 0)
3163 			tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
3164 	}
3165 }
3166 
3167 /* tp->lock is held. */
3168 static void tg3_enable_nvram_access(struct tg3 *tp)
3169 {
3170 	if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
3171 		u32 nvaccess = tr32(NVRAM_ACCESS);
3172 
3173 		tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
3174 	}
3175 }
3176 
3177 /* tp->lock is held. */
3178 static void tg3_disable_nvram_access(struct tg3 *tp)
3179 {
3180 	if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
3181 		u32 nvaccess = tr32(NVRAM_ACCESS);
3182 
3183 		tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
3184 	}
3185 }
3186 
3187 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
3188 					u32 offset, u32 *val)
3189 {
3190 	u32 tmp;
3191 	int i;
3192 
3193 	if (offset > EEPROM_ADDR_ADDR_MASK || (offset % 4) != 0)
3194 		return -EINVAL;
3195 
3196 	tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
3197 					EEPROM_ADDR_DEVID_MASK |
3198 					EEPROM_ADDR_READ);
3199 	tw32(GRC_EEPROM_ADDR,
3200 	     tmp |
3201 	     (0 << EEPROM_ADDR_DEVID_SHIFT) |
3202 	     ((offset << EEPROM_ADDR_ADDR_SHIFT) &
3203 	      EEPROM_ADDR_ADDR_MASK) |
3204 	     EEPROM_ADDR_READ | EEPROM_ADDR_START);
3205 
3206 	for (i = 0; i < 1000; i++) {
3207 		tmp = tr32(GRC_EEPROM_ADDR);
3208 
3209 		if (tmp & EEPROM_ADDR_COMPLETE)
3210 			break;
3211 		msleep(1);
3212 	}
3213 	if (!(tmp & EEPROM_ADDR_COMPLETE))
3214 		return -EBUSY;
3215 
3216 	tmp = tr32(GRC_EEPROM_DATA);
3217 
3218 	/*
3219 	 * The data will always be opposite the native endian
3220 	 * format.  Perform a blind byteswap to compensate.
3221 	 */
3222 	*val = swab32(tmp);
3223 
3224 	return 0;
3225 }
3226 
3227 #define NVRAM_CMD_TIMEOUT 10000
3228 
3229 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
3230 {
3231 	int i;
3232 
3233 	tw32(NVRAM_CMD, nvram_cmd);
3234 	for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
3235 		udelay(10);
3236 		if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
3237 			udelay(10);
3238 			break;
3239 		}
3240 	}
3241 
3242 	if (i == NVRAM_CMD_TIMEOUT)
3243 		return -EBUSY;
3244 
3245 	return 0;
3246 }
3247 
3248 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
3249 {
3250 	if (tg3_flag(tp, NVRAM) &&
3251 	    tg3_flag(tp, NVRAM_BUFFERED) &&
3252 	    tg3_flag(tp, FLASH) &&
3253 	    !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
3254 	    (tp->nvram_jedecnum == JEDEC_ATMEL))
3255 
3256 		addr = ((addr / tp->nvram_pagesize) <<
3257 			ATMEL_AT45DB0X1B_PAGE_POS) +
3258 		       (addr % tp->nvram_pagesize);
3259 
3260 	return addr;
3261 }
3262 
3263 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
3264 {
3265 	if (tg3_flag(tp, NVRAM) &&
3266 	    tg3_flag(tp, NVRAM_BUFFERED) &&
3267 	    tg3_flag(tp, FLASH) &&
3268 	    !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
3269 	    (tp->nvram_jedecnum == JEDEC_ATMEL))
3270 
3271 		addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
3272 			tp->nvram_pagesize) +
3273 		       (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
3274 
3275 	return addr;
3276 }
3277 
3278 /* NOTE: Data read in from NVRAM is byteswapped according to
3279  * the byteswapping settings for all other register accesses.
3280  * tg3 devices are BE devices, so on a BE machine, the data
3281  * returned will be exactly as it is seen in NVRAM.  On a LE
3282  * machine, the 32-bit value will be byteswapped.
3283  */
3284 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
3285 {
3286 	int ret;
3287 
3288 	if (!tg3_flag(tp, NVRAM))
3289 		return tg3_nvram_read_using_eeprom(tp, offset, val);
3290 
3291 	offset = tg3_nvram_phys_addr(tp, offset);
3292 
3293 	if (offset > NVRAM_ADDR_MSK)
3294 		return -EINVAL;
3295 
3296 	ret = tg3_nvram_lock(tp);
3297 	if (ret)
3298 		return ret;
3299 
3300 	tg3_enable_nvram_access(tp);
3301 
3302 	tw32(NVRAM_ADDR, offset);
3303 	ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
3304 		NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
3305 
3306 	if (ret == 0)
3307 		*val = tr32(NVRAM_RDDATA);
3308 
3309 	tg3_disable_nvram_access(tp);
3310 
3311 	tg3_nvram_unlock(tp);
3312 
3313 	return ret;
3314 }
3315 
3316 /* Ensures NVRAM data is in bytestream format. */
3317 static int tg3_nvram_read_be32(struct tg3 *tp, u32 offset, __be32 *val)
3318 {
3319 	u32 v;
3320 	int res = tg3_nvram_read(tp, offset, &v);
3321 	if (!res)
3322 		*val = cpu_to_be32(v);
3323 	return res;
3324 }
3325 
3326 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
3327 				    u32 offset, u32 len, u8 *buf)
3328 {
3329 	int i, j, rc = 0;
3330 	u32 val;
3331 
3332 	for (i = 0; i < len; i += 4) {
3333 		u32 addr;
3334 		__be32 data;
3335 
3336 		addr = offset + i;
3337 
3338 		memcpy(&data, buf + i, 4);
3339 
3340 		/*
3341 		 * The SEEPROM interface expects the data to always be opposite
3342 		 * the native endian format.  We accomplish this by reversing
3343 		 * all the operations that would have been performed on the
3344 		 * data from a call to tg3_nvram_read_be32().
3345 		 */
3346 		tw32(GRC_EEPROM_DATA, swab32(be32_to_cpu(data)));
3347 
3348 		val = tr32(GRC_EEPROM_ADDR);
3349 		tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
3350 
3351 		val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
3352 			EEPROM_ADDR_READ);
3353 		tw32(GRC_EEPROM_ADDR, val |
3354 			(0 << EEPROM_ADDR_DEVID_SHIFT) |
3355 			(addr & EEPROM_ADDR_ADDR_MASK) |
3356 			EEPROM_ADDR_START |
3357 			EEPROM_ADDR_WRITE);
3358 
3359 		for (j = 0; j < 1000; j++) {
3360 			val = tr32(GRC_EEPROM_ADDR);
3361 
3362 			if (val & EEPROM_ADDR_COMPLETE)
3363 				break;
3364 			msleep(1);
3365 		}
3366 		if (!(val & EEPROM_ADDR_COMPLETE)) {
3367 			rc = -EBUSY;
3368 			break;
3369 		}
3370 	}
3371 
3372 	return rc;
3373 }
3374 
3375 /* offset and length are dword aligned */
3376 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
3377 		u8 *buf)
3378 {
3379 	int ret = 0;
3380 	u32 pagesize = tp->nvram_pagesize;
3381 	u32 pagemask = pagesize - 1;
3382 	u32 nvram_cmd;
3383 	u8 *tmp;
3384 
3385 	tmp = kmalloc(pagesize, GFP_KERNEL);
3386 	if (tmp == NULL)
3387 		return -ENOMEM;
3388 
3389 	while (len) {
3390 		int j;
3391 		u32 phy_addr, page_off, size;
3392 
3393 		phy_addr = offset & ~pagemask;
3394 
3395 		for (j = 0; j < pagesize; j += 4) {
3396 			ret = tg3_nvram_read_be32(tp, phy_addr + j,
3397 						  (__be32 *) (tmp + j));
3398 			if (ret)
3399 				break;
3400 		}
3401 		if (ret)
3402 			break;
3403 
3404 		page_off = offset & pagemask;
3405 		size = pagesize;
3406 		if (len < size)
3407 			size = len;
3408 
3409 		len -= size;
3410 
3411 		memcpy(tmp + page_off, buf, size);
3412 
3413 		offset = offset + (pagesize - page_off);
3414 
3415 		tg3_enable_nvram_access(tp);
3416 
3417 		/*
3418 		 * Before we can erase the flash page, we need
3419 		 * to issue a special "write enable" command.
3420 		 */
3421 		nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3422 
3423 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3424 			break;
3425 
3426 		/* Erase the target page */
3427 		tw32(NVRAM_ADDR, phy_addr);
3428 
3429 		nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
3430 			NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
3431 
3432 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3433 			break;
3434 
3435 		/* Issue another write enable to start the write. */
3436 		nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3437 
3438 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3439 			break;
3440 
3441 		for (j = 0; j < pagesize; j += 4) {
3442 			__be32 data;
3443 
3444 			data = *((__be32 *) (tmp + j));
3445 
3446 			tw32(NVRAM_WRDATA, be32_to_cpu(data));
3447 
3448 			tw32(NVRAM_ADDR, phy_addr + j);
3449 
3450 			nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
3451 				NVRAM_CMD_WR;
3452 
3453 			if (j == 0)
3454 				nvram_cmd |= NVRAM_CMD_FIRST;
3455 			else if (j == (pagesize - 4))
3456 				nvram_cmd |= NVRAM_CMD_LAST;
3457 
3458 			ret = tg3_nvram_exec_cmd(tp, nvram_cmd);
3459 			if (ret)
3460 				break;
3461 		}
3462 		if (ret)
3463 			break;
3464 	}
3465 
3466 	nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3467 	tg3_nvram_exec_cmd(tp, nvram_cmd);
3468 
3469 	kfree(tmp);
3470 
3471 	return ret;
3472 }
3473 
3474 /* offset and length are dword aligned */
3475 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
3476 		u8 *buf)
3477 {
3478 	int i, ret = 0;
3479 
3480 	for (i = 0; i < len; i += 4, offset += 4) {
3481 		u32 page_off, phy_addr, nvram_cmd;
3482 		__be32 data;
3483 
3484 		memcpy(&data, buf + i, 4);
3485 		tw32(NVRAM_WRDATA, be32_to_cpu(data));
3486 
3487 		page_off = offset % tp->nvram_pagesize;
3488 
3489 		phy_addr = tg3_nvram_phys_addr(tp, offset);
3490 
3491 		nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
3492 
3493 		if (page_off == 0 || i == 0)
3494 			nvram_cmd |= NVRAM_CMD_FIRST;
3495 		if (page_off == (tp->nvram_pagesize - 4))
3496 			nvram_cmd |= NVRAM_CMD_LAST;
3497 
3498 		if (i == (len - 4))
3499 			nvram_cmd |= NVRAM_CMD_LAST;
3500 
3501 		if ((nvram_cmd & NVRAM_CMD_FIRST) ||
3502 		    !tg3_flag(tp, FLASH) ||
3503 		    !tg3_flag(tp, 57765_PLUS))
3504 			tw32(NVRAM_ADDR, phy_addr);
3505 
3506 		if (tg3_asic_rev(tp) != ASIC_REV_5752 &&
3507 		    !tg3_flag(tp, 5755_PLUS) &&
3508 		    (tp->nvram_jedecnum == JEDEC_ST) &&
3509 		    (nvram_cmd & NVRAM_CMD_FIRST)) {
3510 			u32 cmd;
3511 
3512 			cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3513 			ret = tg3_nvram_exec_cmd(tp, cmd);
3514 			if (ret)
3515 				break;
3516 		}
3517 		if (!tg3_flag(tp, FLASH)) {
3518 			/* We always do complete word writes to eeprom. */
3519 			nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
3520 		}
3521 
3522 		ret = tg3_nvram_exec_cmd(tp, nvram_cmd);
3523 		if (ret)
3524 			break;
3525 	}
3526 	return ret;
3527 }
3528 
3529 /* offset and length are dword aligned */
3530 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
3531 {
3532 	int ret;
3533 
3534 	if (tg3_flag(tp, EEPROM_WRITE_PROT)) {
3535 		tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
3536 		       ~GRC_LCLCTRL_GPIO_OUTPUT1);
3537 		udelay(40);
3538 	}
3539 
3540 	if (!tg3_flag(tp, NVRAM)) {
3541 		ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
3542 	} else {
3543 		u32 grc_mode;
3544 
3545 		ret = tg3_nvram_lock(tp);
3546 		if (ret)
3547 			return ret;
3548 
3549 		tg3_enable_nvram_access(tp);
3550 		if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM))
3551 			tw32(NVRAM_WRITE1, 0x406);
3552 
3553 		grc_mode = tr32(GRC_MODE);
3554 		tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
3555 
3556 		if (tg3_flag(tp, NVRAM_BUFFERED) || !tg3_flag(tp, FLASH)) {
3557 			ret = tg3_nvram_write_block_buffered(tp, offset, len,
3558 				buf);
3559 		} else {
3560 			ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
3561 				buf);
3562 		}
3563 
3564 		grc_mode = tr32(GRC_MODE);
3565 		tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
3566 
3567 		tg3_disable_nvram_access(tp);
3568 		tg3_nvram_unlock(tp);
3569 	}
3570 
3571 	if (tg3_flag(tp, EEPROM_WRITE_PROT)) {
3572 		tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
3573 		udelay(40);
3574 	}
3575 
3576 	return ret;
3577 }
3578 
3579 #define RX_CPU_SCRATCH_BASE	0x30000
3580 #define RX_CPU_SCRATCH_SIZE	0x04000
3581 #define TX_CPU_SCRATCH_BASE	0x34000
3582 #define TX_CPU_SCRATCH_SIZE	0x04000
3583 
3584 /* tp->lock is held. */
3585 static int tg3_pause_cpu(struct tg3 *tp, u32 cpu_base)
3586 {
3587 	int i;
3588 	const int iters = 10000;
3589 
3590 	for (i = 0; i < iters; i++) {
3591 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3592 		tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
3593 		if (tr32(cpu_base + CPU_MODE) & CPU_MODE_HALT)
3594 			break;
3595 		if (pci_channel_offline(tp->pdev))
3596 			return -EBUSY;
3597 	}
3598 
3599 	return (i == iters) ? -EBUSY : 0;
3600 }
3601 
3602 /* tp->lock is held. */
3603 static int tg3_rxcpu_pause(struct tg3 *tp)
3604 {
3605 	int rc = tg3_pause_cpu(tp, RX_CPU_BASE);
3606 
3607 	tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
3608 	tw32_f(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
3609 	udelay(10);
3610 
3611 	return rc;
3612 }
3613 
3614 /* tp->lock is held. */
3615 static int tg3_txcpu_pause(struct tg3 *tp)
3616 {
3617 	return tg3_pause_cpu(tp, TX_CPU_BASE);
3618 }
3619 
3620 /* tp->lock is held. */
3621 static void tg3_resume_cpu(struct tg3 *tp, u32 cpu_base)
3622 {
3623 	tw32(cpu_base + CPU_STATE, 0xffffffff);
3624 	tw32_f(cpu_base + CPU_MODE,  0x00000000);
3625 }
3626 
3627 /* tp->lock is held. */
3628 static void tg3_rxcpu_resume(struct tg3 *tp)
3629 {
3630 	tg3_resume_cpu(tp, RX_CPU_BASE);
3631 }
3632 
3633 /* tp->lock is held. */
3634 static int tg3_halt_cpu(struct tg3 *tp, u32 cpu_base)
3635 {
3636 	int rc;
3637 
3638 	BUG_ON(cpu_base == TX_CPU_BASE && tg3_flag(tp, 5705_PLUS));
3639 
3640 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
3641 		u32 val = tr32(GRC_VCPU_EXT_CTRL);
3642 
3643 		tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
3644 		return 0;
3645 	}
3646 	if (cpu_base == RX_CPU_BASE) {
3647 		rc = tg3_rxcpu_pause(tp);
3648 	} else {
3649 		/*
3650 		 * There is only an Rx CPU for the 5750 derivative in the
3651 		 * BCM4785.
3652 		 */
3653 		if (tg3_flag(tp, IS_SSB_CORE))
3654 			return 0;
3655 
3656 		rc = tg3_txcpu_pause(tp);
3657 	}
3658 
3659 	if (rc) {
3660 		netdev_err(tp->dev, "%s timed out, %s CPU\n",
3661 			   __func__, cpu_base == RX_CPU_BASE ? "RX" : "TX");
3662 		return -ENODEV;
3663 	}
3664 
3665 	/* Clear firmware's nvram arbitration. */
3666 	if (tg3_flag(tp, NVRAM))
3667 		tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
3668 	return 0;
3669 }
3670 
3671 static int tg3_fw_data_len(struct tg3 *tp,
3672 			   const struct tg3_firmware_hdr *fw_hdr)
3673 {
3674 	int fw_len;
3675 
3676 	/* Non fragmented firmware have one firmware header followed by a
3677 	 * contiguous chunk of data to be written. The length field in that
3678 	 * header is not the length of data to be written but the complete
3679 	 * length of the bss. The data length is determined based on
3680 	 * tp->fw->size minus headers.
3681 	 *
3682 	 * Fragmented firmware have a main header followed by multiple
3683 	 * fragments. Each fragment is identical to non fragmented firmware
3684 	 * with a firmware header followed by a contiguous chunk of data. In
3685 	 * the main header, the length field is unused and set to 0xffffffff.
3686 	 * In each fragment header the length is the entire size of that
3687 	 * fragment i.e. fragment data + header length. Data length is
3688 	 * therefore length field in the header minus TG3_FW_HDR_LEN.
3689 	 */
3690 	if (tp->fw_len == 0xffffffff)
3691 		fw_len = be32_to_cpu(fw_hdr->len);
3692 	else
3693 		fw_len = tp->fw->size;
3694 
3695 	return (fw_len - TG3_FW_HDR_LEN) / sizeof(u32);
3696 }
3697 
3698 /* tp->lock is held. */
3699 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base,
3700 				 u32 cpu_scratch_base, int cpu_scratch_size,
3701 				 const struct tg3_firmware_hdr *fw_hdr)
3702 {
3703 	int err, i;
3704 	void (*write_op)(struct tg3 *, u32, u32);
3705 	int total_len = tp->fw->size;
3706 
3707 	if (cpu_base == TX_CPU_BASE && tg3_flag(tp, 5705_PLUS)) {
3708 		netdev_err(tp->dev,
3709 			   "%s: Trying to load TX cpu firmware which is 5705\n",
3710 			   __func__);
3711 		return -EINVAL;
3712 	}
3713 
3714 	if (tg3_flag(tp, 5705_PLUS) && tg3_asic_rev(tp) != ASIC_REV_57766)
3715 		write_op = tg3_write_mem;
3716 	else
3717 		write_op = tg3_write_indirect_reg32;
3718 
3719 	if (tg3_asic_rev(tp) != ASIC_REV_57766) {
3720 		/* It is possible that bootcode is still loading at this point.
3721 		 * Get the nvram lock first before halting the cpu.
3722 		 */
3723 		int lock_err = tg3_nvram_lock(tp);
3724 		err = tg3_halt_cpu(tp, cpu_base);
3725 		if (!lock_err)
3726 			tg3_nvram_unlock(tp);
3727 		if (err)
3728 			goto out;
3729 
3730 		for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
3731 			write_op(tp, cpu_scratch_base + i, 0);
3732 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3733 		tw32(cpu_base + CPU_MODE,
3734 		     tr32(cpu_base + CPU_MODE) | CPU_MODE_HALT);
3735 	} else {
3736 		/* Subtract additional main header for fragmented firmware and
3737 		 * advance to the first fragment
3738 		 */
3739 		total_len -= TG3_FW_HDR_LEN;
3740 		fw_hdr++;
3741 	}
3742 
3743 	do {
3744 		u32 *fw_data = (u32 *)(fw_hdr + 1);
3745 		for (i = 0; i < tg3_fw_data_len(tp, fw_hdr); i++)
3746 			write_op(tp, cpu_scratch_base +
3747 				     (be32_to_cpu(fw_hdr->base_addr) & 0xffff) +
3748 				     (i * sizeof(u32)),
3749 				 be32_to_cpu(fw_data[i]));
3750 
3751 		total_len -= be32_to_cpu(fw_hdr->len);
3752 
3753 		/* Advance to next fragment */
3754 		fw_hdr = (struct tg3_firmware_hdr *)
3755 			 ((void *)fw_hdr + be32_to_cpu(fw_hdr->len));
3756 	} while (total_len > 0);
3757 
3758 	err = 0;
3759 
3760 out:
3761 	return err;
3762 }
3763 
3764 /* tp->lock is held. */
3765 static int tg3_pause_cpu_and_set_pc(struct tg3 *tp, u32 cpu_base, u32 pc)
3766 {
3767 	int i;
3768 	const int iters = 5;
3769 
3770 	tw32(cpu_base + CPU_STATE, 0xffffffff);
3771 	tw32_f(cpu_base + CPU_PC, pc);
3772 
3773 	for (i = 0; i < iters; i++) {
3774 		if (tr32(cpu_base + CPU_PC) == pc)
3775 			break;
3776 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3777 		tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
3778 		tw32_f(cpu_base + CPU_PC, pc);
3779 		udelay(1000);
3780 	}
3781 
3782 	return (i == iters) ? -EBUSY : 0;
3783 }
3784 
3785 /* tp->lock is held. */
3786 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
3787 {
3788 	const struct tg3_firmware_hdr *fw_hdr;
3789 	int err;
3790 
3791 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3792 
3793 	/* Firmware blob starts with version numbers, followed by
3794 	   start address and length. We are setting complete length.
3795 	   length = end_address_of_bss - start_address_of_text.
3796 	   Remainder is the blob to be loaded contiguously
3797 	   from start address. */
3798 
3799 	err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
3800 				    RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
3801 				    fw_hdr);
3802 	if (err)
3803 		return err;
3804 
3805 	err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
3806 				    TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
3807 				    fw_hdr);
3808 	if (err)
3809 		return err;
3810 
3811 	/* Now startup only the RX cpu. */
3812 	err = tg3_pause_cpu_and_set_pc(tp, RX_CPU_BASE,
3813 				       be32_to_cpu(fw_hdr->base_addr));
3814 	if (err) {
3815 		netdev_err(tp->dev, "%s fails to set RX CPU PC, is %08x "
3816 			   "should be %08x\n", __func__,
3817 			   tr32(RX_CPU_BASE + CPU_PC),
3818 				be32_to_cpu(fw_hdr->base_addr));
3819 		return -ENODEV;
3820 	}
3821 
3822 	tg3_rxcpu_resume(tp);
3823 
3824 	return 0;
3825 }
3826 
3827 static int tg3_validate_rxcpu_state(struct tg3 *tp)
3828 {
3829 	const int iters = 1000;
3830 	int i;
3831 	u32 val;
3832 
3833 	/* Wait for boot code to complete initialization and enter service
3834 	 * loop. It is then safe to download service patches
3835 	 */
3836 	for (i = 0; i < iters; i++) {
3837 		if (tr32(RX_CPU_HWBKPT) == TG3_SBROM_IN_SERVICE_LOOP)
3838 			break;
3839 
3840 		udelay(10);
3841 	}
3842 
3843 	if (i == iters) {
3844 		netdev_err(tp->dev, "Boot code not ready for service patches\n");
3845 		return -EBUSY;
3846 	}
3847 
3848 	val = tg3_read_indirect_reg32(tp, TG3_57766_FW_HANDSHAKE);
3849 	if (val & 0xff) {
3850 		netdev_warn(tp->dev,
3851 			    "Other patches exist. Not downloading EEE patch\n");
3852 		return -EEXIST;
3853 	}
3854 
3855 	return 0;
3856 }
3857 
3858 /* tp->lock is held. */
3859 static void tg3_load_57766_firmware(struct tg3 *tp)
3860 {
3861 	struct tg3_firmware_hdr *fw_hdr;
3862 
3863 	if (!tg3_flag(tp, NO_NVRAM))
3864 		return;
3865 
3866 	if (tg3_validate_rxcpu_state(tp))
3867 		return;
3868 
3869 	if (!tp->fw)
3870 		return;
3871 
3872 	/* This firmware blob has a different format than older firmware
3873 	 * releases as given below. The main difference is we have fragmented
3874 	 * data to be written to non-contiguous locations.
3875 	 *
3876 	 * In the beginning we have a firmware header identical to other
3877 	 * firmware which consists of version, base addr and length. The length
3878 	 * here is unused and set to 0xffffffff.
3879 	 *
3880 	 * This is followed by a series of firmware fragments which are
3881 	 * individually identical to previous firmware. i.e. they have the
3882 	 * firmware header and followed by data for that fragment. The version
3883 	 * field of the individual fragment header is unused.
3884 	 */
3885 
3886 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3887 	if (be32_to_cpu(fw_hdr->base_addr) != TG3_57766_FW_BASE_ADDR)
3888 		return;
3889 
3890 	if (tg3_rxcpu_pause(tp))
3891 		return;
3892 
3893 	/* tg3_load_firmware_cpu() will always succeed for the 57766 */
3894 	tg3_load_firmware_cpu(tp, 0, TG3_57766_FW_BASE_ADDR, 0, fw_hdr);
3895 
3896 	tg3_rxcpu_resume(tp);
3897 }
3898 
3899 /* tp->lock is held. */
3900 static int tg3_load_tso_firmware(struct tg3 *tp)
3901 {
3902 	const struct tg3_firmware_hdr *fw_hdr;
3903 	unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
3904 	int err;
3905 
3906 	if (!tg3_flag(tp, FW_TSO))
3907 		return 0;
3908 
3909 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3910 
3911 	/* Firmware blob starts with version numbers, followed by
3912 	   start address and length. We are setting complete length.
3913 	   length = end_address_of_bss - start_address_of_text.
3914 	   Remainder is the blob to be loaded contiguously
3915 	   from start address. */
3916 
3917 	cpu_scratch_size = tp->fw_len;
3918 
3919 	if (tg3_asic_rev(tp) == ASIC_REV_5705) {
3920 		cpu_base = RX_CPU_BASE;
3921 		cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
3922 	} else {
3923 		cpu_base = TX_CPU_BASE;
3924 		cpu_scratch_base = TX_CPU_SCRATCH_BASE;
3925 		cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
3926 	}
3927 
3928 	err = tg3_load_firmware_cpu(tp, cpu_base,
3929 				    cpu_scratch_base, cpu_scratch_size,
3930 				    fw_hdr);
3931 	if (err)
3932 		return err;
3933 
3934 	/* Now startup the cpu. */
3935 	err = tg3_pause_cpu_and_set_pc(tp, cpu_base,
3936 				       be32_to_cpu(fw_hdr->base_addr));
3937 	if (err) {
3938 		netdev_err(tp->dev,
3939 			   "%s fails to set CPU PC, is %08x should be %08x\n",
3940 			   __func__, tr32(cpu_base + CPU_PC),
3941 			   be32_to_cpu(fw_hdr->base_addr));
3942 		return -ENODEV;
3943 	}
3944 
3945 	tg3_resume_cpu(tp, cpu_base);
3946 	return 0;
3947 }
3948 
3949 /* tp->lock is held. */
3950 static void __tg3_set_one_mac_addr(struct tg3 *tp, u8 *mac_addr, int index)
3951 {
3952 	u32 addr_high, addr_low;
3953 
3954 	addr_high = ((mac_addr[0] << 8) | mac_addr[1]);
3955 	addr_low = ((mac_addr[2] << 24) | (mac_addr[3] << 16) |
3956 		    (mac_addr[4] <<  8) | mac_addr[5]);
3957 
3958 	if (index < 4) {
3959 		tw32(MAC_ADDR_0_HIGH + (index * 8), addr_high);
3960 		tw32(MAC_ADDR_0_LOW + (index * 8), addr_low);
3961 	} else {
3962 		index -= 4;
3963 		tw32(MAC_EXTADDR_0_HIGH + (index * 8), addr_high);
3964 		tw32(MAC_EXTADDR_0_LOW + (index * 8), addr_low);
3965 	}
3966 }
3967 
3968 /* tp->lock is held. */
3969 static void __tg3_set_mac_addr(struct tg3 *tp, bool skip_mac_1)
3970 {
3971 	u32 addr_high;
3972 	int i;
3973 
3974 	for (i = 0; i < 4; i++) {
3975 		if (i == 1 && skip_mac_1)
3976 			continue;
3977 		__tg3_set_one_mac_addr(tp, tp->dev->dev_addr, i);
3978 	}
3979 
3980 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
3981 	    tg3_asic_rev(tp) == ASIC_REV_5704) {
3982 		for (i = 4; i < 16; i++)
3983 			__tg3_set_one_mac_addr(tp, tp->dev->dev_addr, i);
3984 	}
3985 
3986 	addr_high = (tp->dev->dev_addr[0] +
3987 		     tp->dev->dev_addr[1] +
3988 		     tp->dev->dev_addr[2] +
3989 		     tp->dev->dev_addr[3] +
3990 		     tp->dev->dev_addr[4] +
3991 		     tp->dev->dev_addr[5]) &
3992 		TX_BACKOFF_SEED_MASK;
3993 	tw32(MAC_TX_BACKOFF_SEED, addr_high);
3994 }
3995 
3996 static void tg3_enable_register_access(struct tg3 *tp)
3997 {
3998 	/*
3999 	 * Make sure register accesses (indirect or otherwise) will function
4000 	 * correctly.
4001 	 */
4002 	pci_write_config_dword(tp->pdev,
4003 			       TG3PCI_MISC_HOST_CTRL, tp->misc_host_ctrl);
4004 }
4005 
4006 static int tg3_power_up(struct tg3 *tp)
4007 {
4008 	int err;
4009 
4010 	tg3_enable_register_access(tp);
4011 
4012 	err = pci_set_power_state(tp->pdev, PCI_D0);
4013 	if (!err) {
4014 		/* Switch out of Vaux if it is a NIC */
4015 		tg3_pwrsrc_switch_to_vmain(tp);
4016 	} else {
4017 		netdev_err(tp->dev, "Transition to D0 failed\n");
4018 	}
4019 
4020 	return err;
4021 }
4022 
4023 static int tg3_setup_phy(struct tg3 *, bool);
4024 
4025 static int tg3_power_down_prepare(struct tg3 *tp)
4026 {
4027 	u32 misc_host_ctrl;
4028 	bool device_should_wake, do_low_power;
4029 
4030 	tg3_enable_register_access(tp);
4031 
4032 	/* Restore the CLKREQ setting. */
4033 	if (tg3_flag(tp, CLKREQ_BUG))
4034 		pcie_capability_set_word(tp->pdev, PCI_EXP_LNKCTL,
4035 					 PCI_EXP_LNKCTL_CLKREQ_EN);
4036 
4037 	misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
4038 	tw32(TG3PCI_MISC_HOST_CTRL,
4039 	     misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
4040 
4041 	device_should_wake = device_may_wakeup(&tp->pdev->dev) &&
4042 			     tg3_flag(tp, WOL_ENABLE);
4043 
4044 	if (tg3_flag(tp, USE_PHYLIB)) {
4045 		do_low_power = false;
4046 		if ((tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) &&
4047 		    !(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4048 			struct phy_device *phydev;
4049 			u32 phyid, advertising;
4050 
4051 			phydev = tp->mdio_bus->phy_map[tp->phy_addr];
4052 
4053 			tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
4054 
4055 			tp->link_config.speed = phydev->speed;
4056 			tp->link_config.duplex = phydev->duplex;
4057 			tp->link_config.autoneg = phydev->autoneg;
4058 			tp->link_config.advertising = phydev->advertising;
4059 
4060 			advertising = ADVERTISED_TP |
4061 				      ADVERTISED_Pause |
4062 				      ADVERTISED_Autoneg |
4063 				      ADVERTISED_10baseT_Half;
4064 
4065 			if (tg3_flag(tp, ENABLE_ASF) || device_should_wake) {
4066 				if (tg3_flag(tp, WOL_SPEED_100MB))
4067 					advertising |=
4068 						ADVERTISED_100baseT_Half |
4069 						ADVERTISED_100baseT_Full |
4070 						ADVERTISED_10baseT_Full;
4071 				else
4072 					advertising |= ADVERTISED_10baseT_Full;
4073 			}
4074 
4075 			phydev->advertising = advertising;
4076 
4077 			phy_start_aneg(phydev);
4078 
4079 			phyid = phydev->drv->phy_id & phydev->drv->phy_id_mask;
4080 			if (phyid != PHY_ID_BCMAC131) {
4081 				phyid &= PHY_BCM_OUI_MASK;
4082 				if (phyid == PHY_BCM_OUI_1 ||
4083 				    phyid == PHY_BCM_OUI_2 ||
4084 				    phyid == PHY_BCM_OUI_3)
4085 					do_low_power = true;
4086 			}
4087 		}
4088 	} else {
4089 		do_low_power = true;
4090 
4091 		if (!(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER))
4092 			tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
4093 
4094 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
4095 			tg3_setup_phy(tp, false);
4096 	}
4097 
4098 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
4099 		u32 val;
4100 
4101 		val = tr32(GRC_VCPU_EXT_CTRL);
4102 		tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
4103 	} else if (!tg3_flag(tp, ENABLE_ASF)) {
4104 		int i;
4105 		u32 val;
4106 
4107 		for (i = 0; i < 200; i++) {
4108 			tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
4109 			if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
4110 				break;
4111 			msleep(1);
4112 		}
4113 	}
4114 	if (tg3_flag(tp, WOL_CAP))
4115 		tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
4116 						     WOL_DRV_STATE_SHUTDOWN |
4117 						     WOL_DRV_WOL |
4118 						     WOL_SET_MAGIC_PKT);
4119 
4120 	if (device_should_wake) {
4121 		u32 mac_mode;
4122 
4123 		if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
4124 			if (do_low_power &&
4125 			    !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
4126 				tg3_phy_auxctl_write(tp,
4127 					       MII_TG3_AUXCTL_SHDWSEL_PWRCTL,
4128 					       MII_TG3_AUXCTL_PCTL_WOL_EN |
4129 					       MII_TG3_AUXCTL_PCTL_100TX_LPWR |
4130 					       MII_TG3_AUXCTL_PCTL_CL_AB_TXDAC);
4131 				udelay(40);
4132 			}
4133 
4134 			if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
4135 				mac_mode = MAC_MODE_PORT_MODE_GMII;
4136 			else if (tp->phy_flags &
4137 				 TG3_PHYFLG_KEEP_LINK_ON_PWRDN) {
4138 				if (tp->link_config.active_speed == SPEED_1000)
4139 					mac_mode = MAC_MODE_PORT_MODE_GMII;
4140 				else
4141 					mac_mode = MAC_MODE_PORT_MODE_MII;
4142 			} else
4143 				mac_mode = MAC_MODE_PORT_MODE_MII;
4144 
4145 			mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
4146 			if (tg3_asic_rev(tp) == ASIC_REV_5700) {
4147 				u32 speed = tg3_flag(tp, WOL_SPEED_100MB) ?
4148 					     SPEED_100 : SPEED_10;
4149 				if (tg3_5700_link_polarity(tp, speed))
4150 					mac_mode |= MAC_MODE_LINK_POLARITY;
4151 				else
4152 					mac_mode &= ~MAC_MODE_LINK_POLARITY;
4153 			}
4154 		} else {
4155 			mac_mode = MAC_MODE_PORT_MODE_TBI;
4156 		}
4157 
4158 		if (!tg3_flag(tp, 5750_PLUS))
4159 			tw32(MAC_LED_CTRL, tp->led_ctrl);
4160 
4161 		mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
4162 		if ((tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS)) &&
4163 		    (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)))
4164 			mac_mode |= MAC_MODE_KEEP_FRAME_IN_WOL;
4165 
4166 		if (tg3_flag(tp, ENABLE_APE))
4167 			mac_mode |= MAC_MODE_APE_TX_EN |
4168 				    MAC_MODE_APE_RX_EN |
4169 				    MAC_MODE_TDE_ENABLE;
4170 
4171 		tw32_f(MAC_MODE, mac_mode);
4172 		udelay(100);
4173 
4174 		tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
4175 		udelay(10);
4176 	}
4177 
4178 	if (!tg3_flag(tp, WOL_SPEED_100MB) &&
4179 	    (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4180 	     tg3_asic_rev(tp) == ASIC_REV_5701)) {
4181 		u32 base_val;
4182 
4183 		base_val = tp->pci_clock_ctrl;
4184 		base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
4185 			     CLOCK_CTRL_TXCLK_DISABLE);
4186 
4187 		tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
4188 			    CLOCK_CTRL_PWRDOWN_PLL133, 40);
4189 	} else if (tg3_flag(tp, 5780_CLASS) ||
4190 		   tg3_flag(tp, CPMU_PRESENT) ||
4191 		   tg3_asic_rev(tp) == ASIC_REV_5906) {
4192 		/* do nothing */
4193 	} else if (!(tg3_flag(tp, 5750_PLUS) && tg3_flag(tp, ENABLE_ASF))) {
4194 		u32 newbits1, newbits2;
4195 
4196 		if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4197 		    tg3_asic_rev(tp) == ASIC_REV_5701) {
4198 			newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
4199 				    CLOCK_CTRL_TXCLK_DISABLE |
4200 				    CLOCK_CTRL_ALTCLK);
4201 			newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
4202 		} else if (tg3_flag(tp, 5705_PLUS)) {
4203 			newbits1 = CLOCK_CTRL_625_CORE;
4204 			newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
4205 		} else {
4206 			newbits1 = CLOCK_CTRL_ALTCLK;
4207 			newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
4208 		}
4209 
4210 		tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
4211 			    40);
4212 
4213 		tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
4214 			    40);
4215 
4216 		if (!tg3_flag(tp, 5705_PLUS)) {
4217 			u32 newbits3;
4218 
4219 			if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4220 			    tg3_asic_rev(tp) == ASIC_REV_5701) {
4221 				newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
4222 					    CLOCK_CTRL_TXCLK_DISABLE |
4223 					    CLOCK_CTRL_44MHZ_CORE);
4224 			} else {
4225 				newbits3 = CLOCK_CTRL_44MHZ_CORE;
4226 			}
4227 
4228 			tw32_wait_f(TG3PCI_CLOCK_CTRL,
4229 				    tp->pci_clock_ctrl | newbits3, 40);
4230 		}
4231 	}
4232 
4233 	if (!(device_should_wake) && !tg3_flag(tp, ENABLE_ASF))
4234 		tg3_power_down_phy(tp, do_low_power);
4235 
4236 	tg3_frob_aux_power(tp, true);
4237 
4238 	/* Workaround for unstable PLL clock */
4239 	if ((!tg3_flag(tp, IS_SSB_CORE)) &&
4240 	    ((tg3_chip_rev(tp) == CHIPREV_5750_AX) ||
4241 	     (tg3_chip_rev(tp) == CHIPREV_5750_BX))) {
4242 		u32 val = tr32(0x7d00);
4243 
4244 		val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
4245 		tw32(0x7d00, val);
4246 		if (!tg3_flag(tp, ENABLE_ASF)) {
4247 			int err;
4248 
4249 			err = tg3_nvram_lock(tp);
4250 			tg3_halt_cpu(tp, RX_CPU_BASE);
4251 			if (!err)
4252 				tg3_nvram_unlock(tp);
4253 		}
4254 	}
4255 
4256 	tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
4257 
4258 	tg3_ape_driver_state_change(tp, RESET_KIND_SHUTDOWN);
4259 
4260 	return 0;
4261 }
4262 
4263 static void tg3_power_down(struct tg3 *tp)
4264 {
4265 	pci_wake_from_d3(tp->pdev, tg3_flag(tp, WOL_ENABLE));
4266 	pci_set_power_state(tp->pdev, PCI_D3hot);
4267 }
4268 
4269 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
4270 {
4271 	switch (val & MII_TG3_AUX_STAT_SPDMASK) {
4272 	case MII_TG3_AUX_STAT_10HALF:
4273 		*speed = SPEED_10;
4274 		*duplex = DUPLEX_HALF;
4275 		break;
4276 
4277 	case MII_TG3_AUX_STAT_10FULL:
4278 		*speed = SPEED_10;
4279 		*duplex = DUPLEX_FULL;
4280 		break;
4281 
4282 	case MII_TG3_AUX_STAT_100HALF:
4283 		*speed = SPEED_100;
4284 		*duplex = DUPLEX_HALF;
4285 		break;
4286 
4287 	case MII_TG3_AUX_STAT_100FULL:
4288 		*speed = SPEED_100;
4289 		*duplex = DUPLEX_FULL;
4290 		break;
4291 
4292 	case MII_TG3_AUX_STAT_1000HALF:
4293 		*speed = SPEED_1000;
4294 		*duplex = DUPLEX_HALF;
4295 		break;
4296 
4297 	case MII_TG3_AUX_STAT_1000FULL:
4298 		*speed = SPEED_1000;
4299 		*duplex = DUPLEX_FULL;
4300 		break;
4301 
4302 	default:
4303 		if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
4304 			*speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
4305 				 SPEED_10;
4306 			*duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
4307 				  DUPLEX_HALF;
4308 			break;
4309 		}
4310 		*speed = SPEED_UNKNOWN;
4311 		*duplex = DUPLEX_UNKNOWN;
4312 		break;
4313 	}
4314 }
4315 
4316 static int tg3_phy_autoneg_cfg(struct tg3 *tp, u32 advertise, u32 flowctrl)
4317 {
4318 	int err = 0;
4319 	u32 val, new_adv;
4320 
4321 	new_adv = ADVERTISE_CSMA;
4322 	new_adv |= ethtool_adv_to_mii_adv_t(advertise) & ADVERTISE_ALL;
4323 	new_adv |= mii_advertise_flowctrl(flowctrl);
4324 
4325 	err = tg3_writephy(tp, MII_ADVERTISE, new_adv);
4326 	if (err)
4327 		goto done;
4328 
4329 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4330 		new_adv = ethtool_adv_to_mii_ctrl1000_t(advertise);
4331 
4332 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4333 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0)
4334 			new_adv |= CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER;
4335 
4336 		err = tg3_writephy(tp, MII_CTRL1000, new_adv);
4337 		if (err)
4338 			goto done;
4339 	}
4340 
4341 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
4342 		goto done;
4343 
4344 	tw32(TG3_CPMU_EEE_MODE,
4345 	     tr32(TG3_CPMU_EEE_MODE) & ~TG3_CPMU_EEEMD_LPI_ENABLE);
4346 
4347 	err = tg3_phy_toggle_auxctl_smdsp(tp, true);
4348 	if (!err) {
4349 		u32 err2;
4350 
4351 		val = 0;
4352 		/* Advertise 100-BaseTX EEE ability */
4353 		if (advertise & ADVERTISED_100baseT_Full)
4354 			val |= MDIO_AN_EEE_ADV_100TX;
4355 		/* Advertise 1000-BaseT EEE ability */
4356 		if (advertise & ADVERTISED_1000baseT_Full)
4357 			val |= MDIO_AN_EEE_ADV_1000T;
4358 
4359 		if (!tp->eee.eee_enabled) {
4360 			val = 0;
4361 			tp->eee.advertised = 0;
4362 		} else {
4363 			tp->eee.advertised = advertise &
4364 					     (ADVERTISED_100baseT_Full |
4365 					      ADVERTISED_1000baseT_Full);
4366 		}
4367 
4368 		err = tg3_phy_cl45_write(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, val);
4369 		if (err)
4370 			val = 0;
4371 
4372 		switch (tg3_asic_rev(tp)) {
4373 		case ASIC_REV_5717:
4374 		case ASIC_REV_57765:
4375 		case ASIC_REV_57766:
4376 		case ASIC_REV_5719:
4377 			/* If we advertised any eee advertisements above... */
4378 			if (val)
4379 				val = MII_TG3_DSP_TAP26_ALNOKO |
4380 				      MII_TG3_DSP_TAP26_RMRXSTO |
4381 				      MII_TG3_DSP_TAP26_OPCSINPT;
4382 			tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
4383 			/* Fall through */
4384 		case ASIC_REV_5720:
4385 		case ASIC_REV_5762:
4386 			if (!tg3_phydsp_read(tp, MII_TG3_DSP_CH34TP2, &val))
4387 				tg3_phydsp_write(tp, MII_TG3_DSP_CH34TP2, val |
4388 						 MII_TG3_DSP_CH34TP2_HIBW01);
4389 		}
4390 
4391 		err2 = tg3_phy_toggle_auxctl_smdsp(tp, false);
4392 		if (!err)
4393 			err = err2;
4394 	}
4395 
4396 done:
4397 	return err;
4398 }
4399 
4400 static void tg3_phy_copper_begin(struct tg3 *tp)
4401 {
4402 	if (tp->link_config.autoneg == AUTONEG_ENABLE ||
4403 	    (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4404 		u32 adv, fc;
4405 
4406 		if ((tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) &&
4407 		    !(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)) {
4408 			adv = ADVERTISED_10baseT_Half |
4409 			      ADVERTISED_10baseT_Full;
4410 			if (tg3_flag(tp, WOL_SPEED_100MB))
4411 				adv |= ADVERTISED_100baseT_Half |
4412 				       ADVERTISED_100baseT_Full;
4413 			if (tp->phy_flags & TG3_PHYFLG_1G_ON_VAUX_OK) {
4414 				if (!(tp->phy_flags &
4415 				      TG3_PHYFLG_DISABLE_1G_HD_ADV))
4416 					adv |= ADVERTISED_1000baseT_Half;
4417 				adv |= ADVERTISED_1000baseT_Full;
4418 			}
4419 
4420 			fc = FLOW_CTRL_TX | FLOW_CTRL_RX;
4421 		} else {
4422 			adv = tp->link_config.advertising;
4423 			if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
4424 				adv &= ~(ADVERTISED_1000baseT_Half |
4425 					 ADVERTISED_1000baseT_Full);
4426 
4427 			fc = tp->link_config.flowctrl;
4428 		}
4429 
4430 		tg3_phy_autoneg_cfg(tp, adv, fc);
4431 
4432 		if ((tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) &&
4433 		    (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)) {
4434 			/* Normally during power down we want to autonegotiate
4435 			 * the lowest possible speed for WOL. However, to avoid
4436 			 * link flap, we leave it untouched.
4437 			 */
4438 			return;
4439 		}
4440 
4441 		tg3_writephy(tp, MII_BMCR,
4442 			     BMCR_ANENABLE | BMCR_ANRESTART);
4443 	} else {
4444 		int i;
4445 		u32 bmcr, orig_bmcr;
4446 
4447 		tp->link_config.active_speed = tp->link_config.speed;
4448 		tp->link_config.active_duplex = tp->link_config.duplex;
4449 
4450 		if (tg3_asic_rev(tp) == ASIC_REV_5714) {
4451 			/* With autoneg disabled, 5715 only links up when the
4452 			 * advertisement register has the configured speed
4453 			 * enabled.
4454 			 */
4455 			tg3_writephy(tp, MII_ADVERTISE, ADVERTISE_ALL);
4456 		}
4457 
4458 		bmcr = 0;
4459 		switch (tp->link_config.speed) {
4460 		default:
4461 		case SPEED_10:
4462 			break;
4463 
4464 		case SPEED_100:
4465 			bmcr |= BMCR_SPEED100;
4466 			break;
4467 
4468 		case SPEED_1000:
4469 			bmcr |= BMCR_SPEED1000;
4470 			break;
4471 		}
4472 
4473 		if (tp->link_config.duplex == DUPLEX_FULL)
4474 			bmcr |= BMCR_FULLDPLX;
4475 
4476 		if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
4477 		    (bmcr != orig_bmcr)) {
4478 			tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
4479 			for (i = 0; i < 1500; i++) {
4480 				u32 tmp;
4481 
4482 				udelay(10);
4483 				if (tg3_readphy(tp, MII_BMSR, &tmp) ||
4484 				    tg3_readphy(tp, MII_BMSR, &tmp))
4485 					continue;
4486 				if (!(tmp & BMSR_LSTATUS)) {
4487 					udelay(40);
4488 					break;
4489 				}
4490 			}
4491 			tg3_writephy(tp, MII_BMCR, bmcr);
4492 			udelay(40);
4493 		}
4494 	}
4495 }
4496 
4497 static int tg3_phy_pull_config(struct tg3 *tp)
4498 {
4499 	int err;
4500 	u32 val;
4501 
4502 	err = tg3_readphy(tp, MII_BMCR, &val);
4503 	if (err)
4504 		goto done;
4505 
4506 	if (!(val & BMCR_ANENABLE)) {
4507 		tp->link_config.autoneg = AUTONEG_DISABLE;
4508 		tp->link_config.advertising = 0;
4509 		tg3_flag_clear(tp, PAUSE_AUTONEG);
4510 
4511 		err = -EIO;
4512 
4513 		switch (val & (BMCR_SPEED1000 | BMCR_SPEED100)) {
4514 		case 0:
4515 			if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
4516 				goto done;
4517 
4518 			tp->link_config.speed = SPEED_10;
4519 			break;
4520 		case BMCR_SPEED100:
4521 			if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
4522 				goto done;
4523 
4524 			tp->link_config.speed = SPEED_100;
4525 			break;
4526 		case BMCR_SPEED1000:
4527 			if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4528 				tp->link_config.speed = SPEED_1000;
4529 				break;
4530 			}
4531 			/* Fall through */
4532 		default:
4533 			goto done;
4534 		}
4535 
4536 		if (val & BMCR_FULLDPLX)
4537 			tp->link_config.duplex = DUPLEX_FULL;
4538 		else
4539 			tp->link_config.duplex = DUPLEX_HALF;
4540 
4541 		tp->link_config.flowctrl = FLOW_CTRL_RX | FLOW_CTRL_TX;
4542 
4543 		err = 0;
4544 		goto done;
4545 	}
4546 
4547 	tp->link_config.autoneg = AUTONEG_ENABLE;
4548 	tp->link_config.advertising = ADVERTISED_Autoneg;
4549 	tg3_flag_set(tp, PAUSE_AUTONEG);
4550 
4551 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
4552 		u32 adv;
4553 
4554 		err = tg3_readphy(tp, MII_ADVERTISE, &val);
4555 		if (err)
4556 			goto done;
4557 
4558 		adv = mii_adv_to_ethtool_adv_t(val & ADVERTISE_ALL);
4559 		tp->link_config.advertising |= adv | ADVERTISED_TP;
4560 
4561 		tp->link_config.flowctrl = tg3_decode_flowctrl_1000T(val);
4562 	} else {
4563 		tp->link_config.advertising |= ADVERTISED_FIBRE;
4564 	}
4565 
4566 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4567 		u32 adv;
4568 
4569 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
4570 			err = tg3_readphy(tp, MII_CTRL1000, &val);
4571 			if (err)
4572 				goto done;
4573 
4574 			adv = mii_ctrl1000_to_ethtool_adv_t(val);
4575 		} else {
4576 			err = tg3_readphy(tp, MII_ADVERTISE, &val);
4577 			if (err)
4578 				goto done;
4579 
4580 			adv = tg3_decode_flowctrl_1000X(val);
4581 			tp->link_config.flowctrl = adv;
4582 
4583 			val &= (ADVERTISE_1000XHALF | ADVERTISE_1000XFULL);
4584 			adv = mii_adv_to_ethtool_adv_x(val);
4585 		}
4586 
4587 		tp->link_config.advertising |= adv;
4588 	}
4589 
4590 done:
4591 	return err;
4592 }
4593 
4594 static int tg3_init_5401phy_dsp(struct tg3 *tp)
4595 {
4596 	int err;
4597 
4598 	/* Turn off tap power management. */
4599 	/* Set Extended packet length bit */
4600 	err = tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
4601 
4602 	err |= tg3_phydsp_write(tp, 0x0012, 0x1804);
4603 	err |= tg3_phydsp_write(tp, 0x0013, 0x1204);
4604 	err |= tg3_phydsp_write(tp, 0x8006, 0x0132);
4605 	err |= tg3_phydsp_write(tp, 0x8006, 0x0232);
4606 	err |= tg3_phydsp_write(tp, 0x201f, 0x0a20);
4607 
4608 	udelay(40);
4609 
4610 	return err;
4611 }
4612 
4613 static bool tg3_phy_eee_config_ok(struct tg3 *tp)
4614 {
4615 	struct ethtool_eee eee;
4616 
4617 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
4618 		return true;
4619 
4620 	tg3_eee_pull_config(tp, &eee);
4621 
4622 	if (tp->eee.eee_enabled) {
4623 		if (tp->eee.advertised != eee.advertised ||
4624 		    tp->eee.tx_lpi_timer != eee.tx_lpi_timer ||
4625 		    tp->eee.tx_lpi_enabled != eee.tx_lpi_enabled)
4626 			return false;
4627 	} else {
4628 		/* EEE is disabled but we're advertising */
4629 		if (eee.advertised)
4630 			return false;
4631 	}
4632 
4633 	return true;
4634 }
4635 
4636 static bool tg3_phy_copper_an_config_ok(struct tg3 *tp, u32 *lcladv)
4637 {
4638 	u32 advmsk, tgtadv, advertising;
4639 
4640 	advertising = tp->link_config.advertising;
4641 	tgtadv = ethtool_adv_to_mii_adv_t(advertising) & ADVERTISE_ALL;
4642 
4643 	advmsk = ADVERTISE_ALL;
4644 	if (tp->link_config.active_duplex == DUPLEX_FULL) {
4645 		tgtadv |= mii_advertise_flowctrl(tp->link_config.flowctrl);
4646 		advmsk |= ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
4647 	}
4648 
4649 	if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
4650 		return false;
4651 
4652 	if ((*lcladv & advmsk) != tgtadv)
4653 		return false;
4654 
4655 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4656 		u32 tg3_ctrl;
4657 
4658 		tgtadv = ethtool_adv_to_mii_ctrl1000_t(advertising);
4659 
4660 		if (tg3_readphy(tp, MII_CTRL1000, &tg3_ctrl))
4661 			return false;
4662 
4663 		if (tgtadv &&
4664 		    (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4665 		     tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0)) {
4666 			tgtadv |= CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER;
4667 			tg3_ctrl &= (ADVERTISE_1000HALF | ADVERTISE_1000FULL |
4668 				     CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER);
4669 		} else {
4670 			tg3_ctrl &= (ADVERTISE_1000HALF | ADVERTISE_1000FULL);
4671 		}
4672 
4673 		if (tg3_ctrl != tgtadv)
4674 			return false;
4675 	}
4676 
4677 	return true;
4678 }
4679 
4680 static bool tg3_phy_copper_fetch_rmtadv(struct tg3 *tp, u32 *rmtadv)
4681 {
4682 	u32 lpeth = 0;
4683 
4684 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4685 		u32 val;
4686 
4687 		if (tg3_readphy(tp, MII_STAT1000, &val))
4688 			return false;
4689 
4690 		lpeth = mii_stat1000_to_ethtool_lpa_t(val);
4691 	}
4692 
4693 	if (tg3_readphy(tp, MII_LPA, rmtadv))
4694 		return false;
4695 
4696 	lpeth |= mii_lpa_to_ethtool_lpa_t(*rmtadv);
4697 	tp->link_config.rmt_adv = lpeth;
4698 
4699 	return true;
4700 }
4701 
4702 static bool tg3_test_and_report_link_chg(struct tg3 *tp, bool curr_link_up)
4703 {
4704 	if (curr_link_up != tp->link_up) {
4705 		if (curr_link_up) {
4706 			netif_carrier_on(tp->dev);
4707 		} else {
4708 			netif_carrier_off(tp->dev);
4709 			if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
4710 				tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4711 		}
4712 
4713 		tg3_link_report(tp);
4714 		return true;
4715 	}
4716 
4717 	return false;
4718 }
4719 
4720 static void tg3_clear_mac_status(struct tg3 *tp)
4721 {
4722 	tw32(MAC_EVENT, 0);
4723 
4724 	tw32_f(MAC_STATUS,
4725 	       MAC_STATUS_SYNC_CHANGED |
4726 	       MAC_STATUS_CFG_CHANGED |
4727 	       MAC_STATUS_MI_COMPLETION |
4728 	       MAC_STATUS_LNKSTATE_CHANGED);
4729 	udelay(40);
4730 }
4731 
4732 static void tg3_setup_eee(struct tg3 *tp)
4733 {
4734 	u32 val;
4735 
4736 	val = TG3_CPMU_EEE_LNKIDL_PCIE_NL0 |
4737 	      TG3_CPMU_EEE_LNKIDL_UART_IDL;
4738 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0)
4739 		val |= TG3_CPMU_EEE_LNKIDL_APE_TX_MT;
4740 
4741 	tw32_f(TG3_CPMU_EEE_LNKIDL_CTRL, val);
4742 
4743 	tw32_f(TG3_CPMU_EEE_CTRL,
4744 	       TG3_CPMU_EEE_CTRL_EXIT_20_1_US);
4745 
4746 	val = TG3_CPMU_EEEMD_ERLY_L1_XIT_DET |
4747 	      (tp->eee.tx_lpi_enabled ? TG3_CPMU_EEEMD_LPI_IN_TX : 0) |
4748 	      TG3_CPMU_EEEMD_LPI_IN_RX |
4749 	      TG3_CPMU_EEEMD_EEE_ENABLE;
4750 
4751 	if (tg3_asic_rev(tp) != ASIC_REV_5717)
4752 		val |= TG3_CPMU_EEEMD_SND_IDX_DET_EN;
4753 
4754 	if (tg3_flag(tp, ENABLE_APE))
4755 		val |= TG3_CPMU_EEEMD_APE_TX_DET_EN;
4756 
4757 	tw32_f(TG3_CPMU_EEE_MODE, tp->eee.eee_enabled ? val : 0);
4758 
4759 	tw32_f(TG3_CPMU_EEE_DBTMR1,
4760 	       TG3_CPMU_DBTMR1_PCIEXIT_2047US |
4761 	       (tp->eee.tx_lpi_timer & 0xffff));
4762 
4763 	tw32_f(TG3_CPMU_EEE_DBTMR2,
4764 	       TG3_CPMU_DBTMR2_APE_TX_2047US |
4765 	       TG3_CPMU_DBTMR2_TXIDXEQ_2047US);
4766 }
4767 
4768 static int tg3_setup_copper_phy(struct tg3 *tp, bool force_reset)
4769 {
4770 	bool current_link_up;
4771 	u32 bmsr, val;
4772 	u32 lcl_adv, rmt_adv;
4773 	u16 current_speed;
4774 	u8 current_duplex;
4775 	int i, err;
4776 
4777 	tg3_clear_mac_status(tp);
4778 
4779 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
4780 		tw32_f(MAC_MI_MODE,
4781 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
4782 		udelay(80);
4783 	}
4784 
4785 	tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, 0);
4786 
4787 	/* Some third-party PHYs need to be reset on link going
4788 	 * down.
4789 	 */
4790 	if ((tg3_asic_rev(tp) == ASIC_REV_5703 ||
4791 	     tg3_asic_rev(tp) == ASIC_REV_5704 ||
4792 	     tg3_asic_rev(tp) == ASIC_REV_5705) &&
4793 	    tp->link_up) {
4794 		tg3_readphy(tp, MII_BMSR, &bmsr);
4795 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4796 		    !(bmsr & BMSR_LSTATUS))
4797 			force_reset = true;
4798 	}
4799 	if (force_reset)
4800 		tg3_phy_reset(tp);
4801 
4802 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
4803 		tg3_readphy(tp, MII_BMSR, &bmsr);
4804 		if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
4805 		    !tg3_flag(tp, INIT_COMPLETE))
4806 			bmsr = 0;
4807 
4808 		if (!(bmsr & BMSR_LSTATUS)) {
4809 			err = tg3_init_5401phy_dsp(tp);
4810 			if (err)
4811 				return err;
4812 
4813 			tg3_readphy(tp, MII_BMSR, &bmsr);
4814 			for (i = 0; i < 1000; i++) {
4815 				udelay(10);
4816 				if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4817 				    (bmsr & BMSR_LSTATUS)) {
4818 					udelay(40);
4819 					break;
4820 				}
4821 			}
4822 
4823 			if ((tp->phy_id & TG3_PHY_ID_REV_MASK) ==
4824 			    TG3_PHY_REV_BCM5401_B0 &&
4825 			    !(bmsr & BMSR_LSTATUS) &&
4826 			    tp->link_config.active_speed == SPEED_1000) {
4827 				err = tg3_phy_reset(tp);
4828 				if (!err)
4829 					err = tg3_init_5401phy_dsp(tp);
4830 				if (err)
4831 					return err;
4832 			}
4833 		}
4834 	} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4835 		   tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0) {
4836 		/* 5701 {A0,B0} CRC bug workaround */
4837 		tg3_writephy(tp, 0x15, 0x0a75);
4838 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
4839 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
4840 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
4841 	}
4842 
4843 	/* Clear pending interrupts... */
4844 	tg3_readphy(tp, MII_TG3_ISTAT, &val);
4845 	tg3_readphy(tp, MII_TG3_ISTAT, &val);
4846 
4847 	if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT)
4848 		tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
4849 	else if (!(tp->phy_flags & TG3_PHYFLG_IS_FET))
4850 		tg3_writephy(tp, MII_TG3_IMASK, ~0);
4851 
4852 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4853 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
4854 		if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
4855 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
4856 				     MII_TG3_EXT_CTRL_LNK3_LED_MODE);
4857 		else
4858 			tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
4859 	}
4860 
4861 	current_link_up = false;
4862 	current_speed = SPEED_UNKNOWN;
4863 	current_duplex = DUPLEX_UNKNOWN;
4864 	tp->phy_flags &= ~TG3_PHYFLG_MDIX_STATE;
4865 	tp->link_config.rmt_adv = 0;
4866 
4867 	if (tp->phy_flags & TG3_PHYFLG_CAPACITIVE_COUPLING) {
4868 		err = tg3_phy_auxctl_read(tp,
4869 					  MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
4870 					  &val);
4871 		if (!err && !(val & (1 << 10))) {
4872 			tg3_phy_auxctl_write(tp,
4873 					     MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
4874 					     val | (1 << 10));
4875 			goto relink;
4876 		}
4877 	}
4878 
4879 	bmsr = 0;
4880 	for (i = 0; i < 100; i++) {
4881 		tg3_readphy(tp, MII_BMSR, &bmsr);
4882 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4883 		    (bmsr & BMSR_LSTATUS))
4884 			break;
4885 		udelay(40);
4886 	}
4887 
4888 	if (bmsr & BMSR_LSTATUS) {
4889 		u32 aux_stat, bmcr;
4890 
4891 		tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
4892 		for (i = 0; i < 2000; i++) {
4893 			udelay(10);
4894 			if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
4895 			    aux_stat)
4896 				break;
4897 		}
4898 
4899 		tg3_aux_stat_to_speed_duplex(tp, aux_stat,
4900 					     &current_speed,
4901 					     &current_duplex);
4902 
4903 		bmcr = 0;
4904 		for (i = 0; i < 200; i++) {
4905 			tg3_readphy(tp, MII_BMCR, &bmcr);
4906 			if (tg3_readphy(tp, MII_BMCR, &bmcr))
4907 				continue;
4908 			if (bmcr && bmcr != 0x7fff)
4909 				break;
4910 			udelay(10);
4911 		}
4912 
4913 		lcl_adv = 0;
4914 		rmt_adv = 0;
4915 
4916 		tp->link_config.active_speed = current_speed;
4917 		tp->link_config.active_duplex = current_duplex;
4918 
4919 		if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4920 			bool eee_config_ok = tg3_phy_eee_config_ok(tp);
4921 
4922 			if ((bmcr & BMCR_ANENABLE) &&
4923 			    eee_config_ok &&
4924 			    tg3_phy_copper_an_config_ok(tp, &lcl_adv) &&
4925 			    tg3_phy_copper_fetch_rmtadv(tp, &rmt_adv))
4926 				current_link_up = true;
4927 
4928 			/* EEE settings changes take effect only after a phy
4929 			 * reset.  If we have skipped a reset due to Link Flap
4930 			 * Avoidance being enabled, do it now.
4931 			 */
4932 			if (!eee_config_ok &&
4933 			    (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
4934 			    !force_reset) {
4935 				tg3_setup_eee(tp);
4936 				tg3_phy_reset(tp);
4937 			}
4938 		} else {
4939 			if (!(bmcr & BMCR_ANENABLE) &&
4940 			    tp->link_config.speed == current_speed &&
4941 			    tp->link_config.duplex == current_duplex) {
4942 				current_link_up = true;
4943 			}
4944 		}
4945 
4946 		if (current_link_up &&
4947 		    tp->link_config.active_duplex == DUPLEX_FULL) {
4948 			u32 reg, bit;
4949 
4950 			if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
4951 				reg = MII_TG3_FET_GEN_STAT;
4952 				bit = MII_TG3_FET_GEN_STAT_MDIXSTAT;
4953 			} else {
4954 				reg = MII_TG3_EXT_STAT;
4955 				bit = MII_TG3_EXT_STAT_MDIX;
4956 			}
4957 
4958 			if (!tg3_readphy(tp, reg, &val) && (val & bit))
4959 				tp->phy_flags |= TG3_PHYFLG_MDIX_STATE;
4960 
4961 			tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
4962 		}
4963 	}
4964 
4965 relink:
4966 	if (!current_link_up || (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4967 		tg3_phy_copper_begin(tp);
4968 
4969 		if (tg3_flag(tp, ROBOSWITCH)) {
4970 			current_link_up = true;
4971 			/* FIXME: when BCM5325 switch is used use 100 MBit/s */
4972 			current_speed = SPEED_1000;
4973 			current_duplex = DUPLEX_FULL;
4974 			tp->link_config.active_speed = current_speed;
4975 			tp->link_config.active_duplex = current_duplex;
4976 		}
4977 
4978 		tg3_readphy(tp, MII_BMSR, &bmsr);
4979 		if ((!tg3_readphy(tp, MII_BMSR, &bmsr) && (bmsr & BMSR_LSTATUS)) ||
4980 		    (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
4981 			current_link_up = true;
4982 	}
4983 
4984 	tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
4985 	if (current_link_up) {
4986 		if (tp->link_config.active_speed == SPEED_100 ||
4987 		    tp->link_config.active_speed == SPEED_10)
4988 			tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
4989 		else
4990 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4991 	} else if (tp->phy_flags & TG3_PHYFLG_IS_FET)
4992 		tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
4993 	else
4994 		tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4995 
4996 	/* In order for the 5750 core in BCM4785 chip to work properly
4997 	 * in RGMII mode, the Led Control Register must be set up.
4998 	 */
4999 	if (tg3_flag(tp, RGMII_MODE)) {
5000 		u32 led_ctrl = tr32(MAC_LED_CTRL);
5001 		led_ctrl &= ~(LED_CTRL_1000MBPS_ON | LED_CTRL_100MBPS_ON);
5002 
5003 		if (tp->link_config.active_speed == SPEED_10)
5004 			led_ctrl |= LED_CTRL_LNKLED_OVERRIDE;
5005 		else if (tp->link_config.active_speed == SPEED_100)
5006 			led_ctrl |= (LED_CTRL_LNKLED_OVERRIDE |
5007 				     LED_CTRL_100MBPS_ON);
5008 		else if (tp->link_config.active_speed == SPEED_1000)
5009 			led_ctrl |= (LED_CTRL_LNKLED_OVERRIDE |
5010 				     LED_CTRL_1000MBPS_ON);
5011 
5012 		tw32(MAC_LED_CTRL, led_ctrl);
5013 		udelay(40);
5014 	}
5015 
5016 	tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
5017 	if (tp->link_config.active_duplex == DUPLEX_HALF)
5018 		tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
5019 
5020 	if (tg3_asic_rev(tp) == ASIC_REV_5700) {
5021 		if (current_link_up &&
5022 		    tg3_5700_link_polarity(tp, tp->link_config.active_speed))
5023 			tp->mac_mode |= MAC_MODE_LINK_POLARITY;
5024 		else
5025 			tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
5026 	}
5027 
5028 	/* ??? Without this setting Netgear GA302T PHY does not
5029 	 * ??? send/receive packets...
5030 	 */
5031 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411 &&
5032 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5700_ALTIMA) {
5033 		tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
5034 		tw32_f(MAC_MI_MODE, tp->mi_mode);
5035 		udelay(80);
5036 	}
5037 
5038 	tw32_f(MAC_MODE, tp->mac_mode);
5039 	udelay(40);
5040 
5041 	tg3_phy_eee_adjust(tp, current_link_up);
5042 
5043 	if (tg3_flag(tp, USE_LINKCHG_REG)) {
5044 		/* Polled via timer. */
5045 		tw32_f(MAC_EVENT, 0);
5046 	} else {
5047 		tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5048 	}
5049 	udelay(40);
5050 
5051 	if (tg3_asic_rev(tp) == ASIC_REV_5700 &&
5052 	    current_link_up &&
5053 	    tp->link_config.active_speed == SPEED_1000 &&
5054 	    (tg3_flag(tp, PCIX_MODE) || tg3_flag(tp, PCI_HIGH_SPEED))) {
5055 		udelay(120);
5056 		tw32_f(MAC_STATUS,
5057 		     (MAC_STATUS_SYNC_CHANGED |
5058 		      MAC_STATUS_CFG_CHANGED));
5059 		udelay(40);
5060 		tg3_write_mem(tp,
5061 			      NIC_SRAM_FIRMWARE_MBOX,
5062 			      NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
5063 	}
5064 
5065 	/* Prevent send BD corruption. */
5066 	if (tg3_flag(tp, CLKREQ_BUG)) {
5067 		if (tp->link_config.active_speed == SPEED_100 ||
5068 		    tp->link_config.active_speed == SPEED_10)
5069 			pcie_capability_clear_word(tp->pdev, PCI_EXP_LNKCTL,
5070 						   PCI_EXP_LNKCTL_CLKREQ_EN);
5071 		else
5072 			pcie_capability_set_word(tp->pdev, PCI_EXP_LNKCTL,
5073 						 PCI_EXP_LNKCTL_CLKREQ_EN);
5074 	}
5075 
5076 	tg3_test_and_report_link_chg(tp, current_link_up);
5077 
5078 	return 0;
5079 }
5080 
5081 struct tg3_fiber_aneginfo {
5082 	int state;
5083 #define ANEG_STATE_UNKNOWN		0
5084 #define ANEG_STATE_AN_ENABLE		1
5085 #define ANEG_STATE_RESTART_INIT		2
5086 #define ANEG_STATE_RESTART		3
5087 #define ANEG_STATE_DISABLE_LINK_OK	4
5088 #define ANEG_STATE_ABILITY_DETECT_INIT	5
5089 #define ANEG_STATE_ABILITY_DETECT	6
5090 #define ANEG_STATE_ACK_DETECT_INIT	7
5091 #define ANEG_STATE_ACK_DETECT		8
5092 #define ANEG_STATE_COMPLETE_ACK_INIT	9
5093 #define ANEG_STATE_COMPLETE_ACK		10
5094 #define ANEG_STATE_IDLE_DETECT_INIT	11
5095 #define ANEG_STATE_IDLE_DETECT		12
5096 #define ANEG_STATE_LINK_OK		13
5097 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT	14
5098 #define ANEG_STATE_NEXT_PAGE_WAIT	15
5099 
5100 	u32 flags;
5101 #define MR_AN_ENABLE		0x00000001
5102 #define MR_RESTART_AN		0x00000002
5103 #define MR_AN_COMPLETE		0x00000004
5104 #define MR_PAGE_RX		0x00000008
5105 #define MR_NP_LOADED		0x00000010
5106 #define MR_TOGGLE_TX		0x00000020
5107 #define MR_LP_ADV_FULL_DUPLEX	0x00000040
5108 #define MR_LP_ADV_HALF_DUPLEX	0x00000080
5109 #define MR_LP_ADV_SYM_PAUSE	0x00000100
5110 #define MR_LP_ADV_ASYM_PAUSE	0x00000200
5111 #define MR_LP_ADV_REMOTE_FAULT1	0x00000400
5112 #define MR_LP_ADV_REMOTE_FAULT2	0x00000800
5113 #define MR_LP_ADV_NEXT_PAGE	0x00001000
5114 #define MR_TOGGLE_RX		0x00002000
5115 #define MR_NP_RX		0x00004000
5116 
5117 #define MR_LINK_OK		0x80000000
5118 
5119 	unsigned long link_time, cur_time;
5120 
5121 	u32 ability_match_cfg;
5122 	int ability_match_count;
5123 
5124 	char ability_match, idle_match, ack_match;
5125 
5126 	u32 txconfig, rxconfig;
5127 #define ANEG_CFG_NP		0x00000080
5128 #define ANEG_CFG_ACK		0x00000040
5129 #define ANEG_CFG_RF2		0x00000020
5130 #define ANEG_CFG_RF1		0x00000010
5131 #define ANEG_CFG_PS2		0x00000001
5132 #define ANEG_CFG_PS1		0x00008000
5133 #define ANEG_CFG_HD		0x00004000
5134 #define ANEG_CFG_FD		0x00002000
5135 #define ANEG_CFG_INVAL		0x00001f06
5136 
5137 };
5138 #define ANEG_OK		0
5139 #define ANEG_DONE	1
5140 #define ANEG_TIMER_ENAB	2
5141 #define ANEG_FAILED	-1
5142 
5143 #define ANEG_STATE_SETTLE_TIME	10000
5144 
5145 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
5146 				   struct tg3_fiber_aneginfo *ap)
5147 {
5148 	u16 flowctrl;
5149 	unsigned long delta;
5150 	u32 rx_cfg_reg;
5151 	int ret;
5152 
5153 	if (ap->state == ANEG_STATE_UNKNOWN) {
5154 		ap->rxconfig = 0;
5155 		ap->link_time = 0;
5156 		ap->cur_time = 0;
5157 		ap->ability_match_cfg = 0;
5158 		ap->ability_match_count = 0;
5159 		ap->ability_match = 0;
5160 		ap->idle_match = 0;
5161 		ap->ack_match = 0;
5162 	}
5163 	ap->cur_time++;
5164 
5165 	if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
5166 		rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
5167 
5168 		if (rx_cfg_reg != ap->ability_match_cfg) {
5169 			ap->ability_match_cfg = rx_cfg_reg;
5170 			ap->ability_match = 0;
5171 			ap->ability_match_count = 0;
5172 		} else {
5173 			if (++ap->ability_match_count > 1) {
5174 				ap->ability_match = 1;
5175 				ap->ability_match_cfg = rx_cfg_reg;
5176 			}
5177 		}
5178 		if (rx_cfg_reg & ANEG_CFG_ACK)
5179 			ap->ack_match = 1;
5180 		else
5181 			ap->ack_match = 0;
5182 
5183 		ap->idle_match = 0;
5184 	} else {
5185 		ap->idle_match = 1;
5186 		ap->ability_match_cfg = 0;
5187 		ap->ability_match_count = 0;
5188 		ap->ability_match = 0;
5189 		ap->ack_match = 0;
5190 
5191 		rx_cfg_reg = 0;
5192 	}
5193 
5194 	ap->rxconfig = rx_cfg_reg;
5195 	ret = ANEG_OK;
5196 
5197 	switch (ap->state) {
5198 	case ANEG_STATE_UNKNOWN:
5199 		if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
5200 			ap->state = ANEG_STATE_AN_ENABLE;
5201 
5202 		/* fallthru */
5203 	case ANEG_STATE_AN_ENABLE:
5204 		ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
5205 		if (ap->flags & MR_AN_ENABLE) {
5206 			ap->link_time = 0;
5207 			ap->cur_time = 0;
5208 			ap->ability_match_cfg = 0;
5209 			ap->ability_match_count = 0;
5210 			ap->ability_match = 0;
5211 			ap->idle_match = 0;
5212 			ap->ack_match = 0;
5213 
5214 			ap->state = ANEG_STATE_RESTART_INIT;
5215 		} else {
5216 			ap->state = ANEG_STATE_DISABLE_LINK_OK;
5217 		}
5218 		break;
5219 
5220 	case ANEG_STATE_RESTART_INIT:
5221 		ap->link_time = ap->cur_time;
5222 		ap->flags &= ~(MR_NP_LOADED);
5223 		ap->txconfig = 0;
5224 		tw32(MAC_TX_AUTO_NEG, 0);
5225 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5226 		tw32_f(MAC_MODE, tp->mac_mode);
5227 		udelay(40);
5228 
5229 		ret = ANEG_TIMER_ENAB;
5230 		ap->state = ANEG_STATE_RESTART;
5231 
5232 		/* fallthru */
5233 	case ANEG_STATE_RESTART:
5234 		delta = ap->cur_time - ap->link_time;
5235 		if (delta > ANEG_STATE_SETTLE_TIME)
5236 			ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
5237 		else
5238 			ret = ANEG_TIMER_ENAB;
5239 		break;
5240 
5241 	case ANEG_STATE_DISABLE_LINK_OK:
5242 		ret = ANEG_DONE;
5243 		break;
5244 
5245 	case ANEG_STATE_ABILITY_DETECT_INIT:
5246 		ap->flags &= ~(MR_TOGGLE_TX);
5247 		ap->txconfig = ANEG_CFG_FD;
5248 		flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5249 		if (flowctrl & ADVERTISE_1000XPAUSE)
5250 			ap->txconfig |= ANEG_CFG_PS1;
5251 		if (flowctrl & ADVERTISE_1000XPSE_ASYM)
5252 			ap->txconfig |= ANEG_CFG_PS2;
5253 		tw32(MAC_TX_AUTO_NEG, ap->txconfig);
5254 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5255 		tw32_f(MAC_MODE, tp->mac_mode);
5256 		udelay(40);
5257 
5258 		ap->state = ANEG_STATE_ABILITY_DETECT;
5259 		break;
5260 
5261 	case ANEG_STATE_ABILITY_DETECT:
5262 		if (ap->ability_match != 0 && ap->rxconfig != 0)
5263 			ap->state = ANEG_STATE_ACK_DETECT_INIT;
5264 		break;
5265 
5266 	case ANEG_STATE_ACK_DETECT_INIT:
5267 		ap->txconfig |= ANEG_CFG_ACK;
5268 		tw32(MAC_TX_AUTO_NEG, ap->txconfig);
5269 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5270 		tw32_f(MAC_MODE, tp->mac_mode);
5271 		udelay(40);
5272 
5273 		ap->state = ANEG_STATE_ACK_DETECT;
5274 
5275 		/* fallthru */
5276 	case ANEG_STATE_ACK_DETECT:
5277 		if (ap->ack_match != 0) {
5278 			if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
5279 			    (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
5280 				ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
5281 			} else {
5282 				ap->state = ANEG_STATE_AN_ENABLE;
5283 			}
5284 		} else if (ap->ability_match != 0 &&
5285 			   ap->rxconfig == 0) {
5286 			ap->state = ANEG_STATE_AN_ENABLE;
5287 		}
5288 		break;
5289 
5290 	case ANEG_STATE_COMPLETE_ACK_INIT:
5291 		if (ap->rxconfig & ANEG_CFG_INVAL) {
5292 			ret = ANEG_FAILED;
5293 			break;
5294 		}
5295 		ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
5296 			       MR_LP_ADV_HALF_DUPLEX |
5297 			       MR_LP_ADV_SYM_PAUSE |
5298 			       MR_LP_ADV_ASYM_PAUSE |
5299 			       MR_LP_ADV_REMOTE_FAULT1 |
5300 			       MR_LP_ADV_REMOTE_FAULT2 |
5301 			       MR_LP_ADV_NEXT_PAGE |
5302 			       MR_TOGGLE_RX |
5303 			       MR_NP_RX);
5304 		if (ap->rxconfig & ANEG_CFG_FD)
5305 			ap->flags |= MR_LP_ADV_FULL_DUPLEX;
5306 		if (ap->rxconfig & ANEG_CFG_HD)
5307 			ap->flags |= MR_LP_ADV_HALF_DUPLEX;
5308 		if (ap->rxconfig & ANEG_CFG_PS1)
5309 			ap->flags |= MR_LP_ADV_SYM_PAUSE;
5310 		if (ap->rxconfig & ANEG_CFG_PS2)
5311 			ap->flags |= MR_LP_ADV_ASYM_PAUSE;
5312 		if (ap->rxconfig & ANEG_CFG_RF1)
5313 			ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
5314 		if (ap->rxconfig & ANEG_CFG_RF2)
5315 			ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
5316 		if (ap->rxconfig & ANEG_CFG_NP)
5317 			ap->flags |= MR_LP_ADV_NEXT_PAGE;
5318 
5319 		ap->link_time = ap->cur_time;
5320 
5321 		ap->flags ^= (MR_TOGGLE_TX);
5322 		if (ap->rxconfig & 0x0008)
5323 			ap->flags |= MR_TOGGLE_RX;
5324 		if (ap->rxconfig & ANEG_CFG_NP)
5325 			ap->flags |= MR_NP_RX;
5326 		ap->flags |= MR_PAGE_RX;
5327 
5328 		ap->state = ANEG_STATE_COMPLETE_ACK;
5329 		ret = ANEG_TIMER_ENAB;
5330 		break;
5331 
5332 	case ANEG_STATE_COMPLETE_ACK:
5333 		if (ap->ability_match != 0 &&
5334 		    ap->rxconfig == 0) {
5335 			ap->state = ANEG_STATE_AN_ENABLE;
5336 			break;
5337 		}
5338 		delta = ap->cur_time - ap->link_time;
5339 		if (delta > ANEG_STATE_SETTLE_TIME) {
5340 			if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
5341 				ap->state = ANEG_STATE_IDLE_DETECT_INIT;
5342 			} else {
5343 				if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
5344 				    !(ap->flags & MR_NP_RX)) {
5345 					ap->state = ANEG_STATE_IDLE_DETECT_INIT;
5346 				} else {
5347 					ret = ANEG_FAILED;
5348 				}
5349 			}
5350 		}
5351 		break;
5352 
5353 	case ANEG_STATE_IDLE_DETECT_INIT:
5354 		ap->link_time = ap->cur_time;
5355 		tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
5356 		tw32_f(MAC_MODE, tp->mac_mode);
5357 		udelay(40);
5358 
5359 		ap->state = ANEG_STATE_IDLE_DETECT;
5360 		ret = ANEG_TIMER_ENAB;
5361 		break;
5362 
5363 	case ANEG_STATE_IDLE_DETECT:
5364 		if (ap->ability_match != 0 &&
5365 		    ap->rxconfig == 0) {
5366 			ap->state = ANEG_STATE_AN_ENABLE;
5367 			break;
5368 		}
5369 		delta = ap->cur_time - ap->link_time;
5370 		if (delta > ANEG_STATE_SETTLE_TIME) {
5371 			/* XXX another gem from the Broadcom driver :( */
5372 			ap->state = ANEG_STATE_LINK_OK;
5373 		}
5374 		break;
5375 
5376 	case ANEG_STATE_LINK_OK:
5377 		ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
5378 		ret = ANEG_DONE;
5379 		break;
5380 
5381 	case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
5382 		/* ??? unimplemented */
5383 		break;
5384 
5385 	case ANEG_STATE_NEXT_PAGE_WAIT:
5386 		/* ??? unimplemented */
5387 		break;
5388 
5389 	default:
5390 		ret = ANEG_FAILED;
5391 		break;
5392 	}
5393 
5394 	return ret;
5395 }
5396 
5397 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
5398 {
5399 	int res = 0;
5400 	struct tg3_fiber_aneginfo aninfo;
5401 	int status = ANEG_FAILED;
5402 	unsigned int tick;
5403 	u32 tmp;
5404 
5405 	tw32_f(MAC_TX_AUTO_NEG, 0);
5406 
5407 	tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
5408 	tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
5409 	udelay(40);
5410 
5411 	tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
5412 	udelay(40);
5413 
5414 	memset(&aninfo, 0, sizeof(aninfo));
5415 	aninfo.flags |= MR_AN_ENABLE;
5416 	aninfo.state = ANEG_STATE_UNKNOWN;
5417 	aninfo.cur_time = 0;
5418 	tick = 0;
5419 	while (++tick < 195000) {
5420 		status = tg3_fiber_aneg_smachine(tp, &aninfo);
5421 		if (status == ANEG_DONE || status == ANEG_FAILED)
5422 			break;
5423 
5424 		udelay(1);
5425 	}
5426 
5427 	tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
5428 	tw32_f(MAC_MODE, tp->mac_mode);
5429 	udelay(40);
5430 
5431 	*txflags = aninfo.txconfig;
5432 	*rxflags = aninfo.flags;
5433 
5434 	if (status == ANEG_DONE &&
5435 	    (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
5436 			     MR_LP_ADV_FULL_DUPLEX)))
5437 		res = 1;
5438 
5439 	return res;
5440 }
5441 
5442 static void tg3_init_bcm8002(struct tg3 *tp)
5443 {
5444 	u32 mac_status = tr32(MAC_STATUS);
5445 	int i;
5446 
5447 	/* Reset when initting first time or we have a link. */
5448 	if (tg3_flag(tp, INIT_COMPLETE) &&
5449 	    !(mac_status & MAC_STATUS_PCS_SYNCED))
5450 		return;
5451 
5452 	/* Set PLL lock range. */
5453 	tg3_writephy(tp, 0x16, 0x8007);
5454 
5455 	/* SW reset */
5456 	tg3_writephy(tp, MII_BMCR, BMCR_RESET);
5457 
5458 	/* Wait for reset to complete. */
5459 	/* XXX schedule_timeout() ... */
5460 	for (i = 0; i < 500; i++)
5461 		udelay(10);
5462 
5463 	/* Config mode; select PMA/Ch 1 regs. */
5464 	tg3_writephy(tp, 0x10, 0x8411);
5465 
5466 	/* Enable auto-lock and comdet, select txclk for tx. */
5467 	tg3_writephy(tp, 0x11, 0x0a10);
5468 
5469 	tg3_writephy(tp, 0x18, 0x00a0);
5470 	tg3_writephy(tp, 0x16, 0x41ff);
5471 
5472 	/* Assert and deassert POR. */
5473 	tg3_writephy(tp, 0x13, 0x0400);
5474 	udelay(40);
5475 	tg3_writephy(tp, 0x13, 0x0000);
5476 
5477 	tg3_writephy(tp, 0x11, 0x0a50);
5478 	udelay(40);
5479 	tg3_writephy(tp, 0x11, 0x0a10);
5480 
5481 	/* Wait for signal to stabilize */
5482 	/* XXX schedule_timeout() ... */
5483 	for (i = 0; i < 15000; i++)
5484 		udelay(10);
5485 
5486 	/* Deselect the channel register so we can read the PHYID
5487 	 * later.
5488 	 */
5489 	tg3_writephy(tp, 0x10, 0x8011);
5490 }
5491 
5492 static bool tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
5493 {
5494 	u16 flowctrl;
5495 	bool current_link_up;
5496 	u32 sg_dig_ctrl, sg_dig_status;
5497 	u32 serdes_cfg, expected_sg_dig_ctrl;
5498 	int workaround, port_a;
5499 
5500 	serdes_cfg = 0;
5501 	expected_sg_dig_ctrl = 0;
5502 	workaround = 0;
5503 	port_a = 1;
5504 	current_link_up = false;
5505 
5506 	if (tg3_chip_rev_id(tp) != CHIPREV_ID_5704_A0 &&
5507 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5704_A1) {
5508 		workaround = 1;
5509 		if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
5510 			port_a = 0;
5511 
5512 		/* preserve bits 0-11,13,14 for signal pre-emphasis */
5513 		/* preserve bits 20-23 for voltage regulator */
5514 		serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
5515 	}
5516 
5517 	sg_dig_ctrl = tr32(SG_DIG_CTRL);
5518 
5519 	if (tp->link_config.autoneg != AUTONEG_ENABLE) {
5520 		if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
5521 			if (workaround) {
5522 				u32 val = serdes_cfg;
5523 
5524 				if (port_a)
5525 					val |= 0xc010000;
5526 				else
5527 					val |= 0x4010000;
5528 				tw32_f(MAC_SERDES_CFG, val);
5529 			}
5530 
5531 			tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
5532 		}
5533 		if (mac_status & MAC_STATUS_PCS_SYNCED) {
5534 			tg3_setup_flow_control(tp, 0, 0);
5535 			current_link_up = true;
5536 		}
5537 		goto out;
5538 	}
5539 
5540 	/* Want auto-negotiation.  */
5541 	expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
5542 
5543 	flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5544 	if (flowctrl & ADVERTISE_1000XPAUSE)
5545 		expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
5546 	if (flowctrl & ADVERTISE_1000XPSE_ASYM)
5547 		expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
5548 
5549 	if (sg_dig_ctrl != expected_sg_dig_ctrl) {
5550 		if ((tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT) &&
5551 		    tp->serdes_counter &&
5552 		    ((mac_status & (MAC_STATUS_PCS_SYNCED |
5553 				    MAC_STATUS_RCVD_CFG)) ==
5554 		     MAC_STATUS_PCS_SYNCED)) {
5555 			tp->serdes_counter--;
5556 			current_link_up = true;
5557 			goto out;
5558 		}
5559 restart_autoneg:
5560 		if (workaround)
5561 			tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
5562 		tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
5563 		udelay(5);
5564 		tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
5565 
5566 		tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
5567 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5568 	} else if (mac_status & (MAC_STATUS_PCS_SYNCED |
5569 				 MAC_STATUS_SIGNAL_DET)) {
5570 		sg_dig_status = tr32(SG_DIG_STATUS);
5571 		mac_status = tr32(MAC_STATUS);
5572 
5573 		if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
5574 		    (mac_status & MAC_STATUS_PCS_SYNCED)) {
5575 			u32 local_adv = 0, remote_adv = 0;
5576 
5577 			if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
5578 				local_adv |= ADVERTISE_1000XPAUSE;
5579 			if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
5580 				local_adv |= ADVERTISE_1000XPSE_ASYM;
5581 
5582 			if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
5583 				remote_adv |= LPA_1000XPAUSE;
5584 			if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
5585 				remote_adv |= LPA_1000XPAUSE_ASYM;
5586 
5587 			tp->link_config.rmt_adv =
5588 					   mii_adv_to_ethtool_adv_x(remote_adv);
5589 
5590 			tg3_setup_flow_control(tp, local_adv, remote_adv);
5591 			current_link_up = true;
5592 			tp->serdes_counter = 0;
5593 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5594 		} else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
5595 			if (tp->serdes_counter)
5596 				tp->serdes_counter--;
5597 			else {
5598 				if (workaround) {
5599 					u32 val = serdes_cfg;
5600 
5601 					if (port_a)
5602 						val |= 0xc010000;
5603 					else
5604 						val |= 0x4010000;
5605 
5606 					tw32_f(MAC_SERDES_CFG, val);
5607 				}
5608 
5609 				tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
5610 				udelay(40);
5611 
5612 				/* Link parallel detection - link is up */
5613 				/* only if we have PCS_SYNC and not */
5614 				/* receiving config code words */
5615 				mac_status = tr32(MAC_STATUS);
5616 				if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
5617 				    !(mac_status & MAC_STATUS_RCVD_CFG)) {
5618 					tg3_setup_flow_control(tp, 0, 0);
5619 					current_link_up = true;
5620 					tp->phy_flags |=
5621 						TG3_PHYFLG_PARALLEL_DETECT;
5622 					tp->serdes_counter =
5623 						SERDES_PARALLEL_DET_TIMEOUT;
5624 				} else
5625 					goto restart_autoneg;
5626 			}
5627 		}
5628 	} else {
5629 		tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
5630 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5631 	}
5632 
5633 out:
5634 	return current_link_up;
5635 }
5636 
5637 static bool tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
5638 {
5639 	bool current_link_up = false;
5640 
5641 	if (!(mac_status & MAC_STATUS_PCS_SYNCED))
5642 		goto out;
5643 
5644 	if (tp->link_config.autoneg == AUTONEG_ENABLE) {
5645 		u32 txflags, rxflags;
5646 		int i;
5647 
5648 		if (fiber_autoneg(tp, &txflags, &rxflags)) {
5649 			u32 local_adv = 0, remote_adv = 0;
5650 
5651 			if (txflags & ANEG_CFG_PS1)
5652 				local_adv |= ADVERTISE_1000XPAUSE;
5653 			if (txflags & ANEG_CFG_PS2)
5654 				local_adv |= ADVERTISE_1000XPSE_ASYM;
5655 
5656 			if (rxflags & MR_LP_ADV_SYM_PAUSE)
5657 				remote_adv |= LPA_1000XPAUSE;
5658 			if (rxflags & MR_LP_ADV_ASYM_PAUSE)
5659 				remote_adv |= LPA_1000XPAUSE_ASYM;
5660 
5661 			tp->link_config.rmt_adv =
5662 					   mii_adv_to_ethtool_adv_x(remote_adv);
5663 
5664 			tg3_setup_flow_control(tp, local_adv, remote_adv);
5665 
5666 			current_link_up = true;
5667 		}
5668 		for (i = 0; i < 30; i++) {
5669 			udelay(20);
5670 			tw32_f(MAC_STATUS,
5671 			       (MAC_STATUS_SYNC_CHANGED |
5672 				MAC_STATUS_CFG_CHANGED));
5673 			udelay(40);
5674 			if ((tr32(MAC_STATUS) &
5675 			     (MAC_STATUS_SYNC_CHANGED |
5676 			      MAC_STATUS_CFG_CHANGED)) == 0)
5677 				break;
5678 		}
5679 
5680 		mac_status = tr32(MAC_STATUS);
5681 		if (!current_link_up &&
5682 		    (mac_status & MAC_STATUS_PCS_SYNCED) &&
5683 		    !(mac_status & MAC_STATUS_RCVD_CFG))
5684 			current_link_up = true;
5685 	} else {
5686 		tg3_setup_flow_control(tp, 0, 0);
5687 
5688 		/* Forcing 1000FD link up. */
5689 		current_link_up = true;
5690 
5691 		tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
5692 		udelay(40);
5693 
5694 		tw32_f(MAC_MODE, tp->mac_mode);
5695 		udelay(40);
5696 	}
5697 
5698 out:
5699 	return current_link_up;
5700 }
5701 
5702 static int tg3_setup_fiber_phy(struct tg3 *tp, bool force_reset)
5703 {
5704 	u32 orig_pause_cfg;
5705 	u16 orig_active_speed;
5706 	u8 orig_active_duplex;
5707 	u32 mac_status;
5708 	bool current_link_up;
5709 	int i;
5710 
5711 	orig_pause_cfg = tp->link_config.active_flowctrl;
5712 	orig_active_speed = tp->link_config.active_speed;
5713 	orig_active_duplex = tp->link_config.active_duplex;
5714 
5715 	if (!tg3_flag(tp, HW_AUTONEG) &&
5716 	    tp->link_up &&
5717 	    tg3_flag(tp, INIT_COMPLETE)) {
5718 		mac_status = tr32(MAC_STATUS);
5719 		mac_status &= (MAC_STATUS_PCS_SYNCED |
5720 			       MAC_STATUS_SIGNAL_DET |
5721 			       MAC_STATUS_CFG_CHANGED |
5722 			       MAC_STATUS_RCVD_CFG);
5723 		if (mac_status == (MAC_STATUS_PCS_SYNCED |
5724 				   MAC_STATUS_SIGNAL_DET)) {
5725 			tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
5726 					    MAC_STATUS_CFG_CHANGED));
5727 			return 0;
5728 		}
5729 	}
5730 
5731 	tw32_f(MAC_TX_AUTO_NEG, 0);
5732 
5733 	tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
5734 	tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
5735 	tw32_f(MAC_MODE, tp->mac_mode);
5736 	udelay(40);
5737 
5738 	if (tp->phy_id == TG3_PHY_ID_BCM8002)
5739 		tg3_init_bcm8002(tp);
5740 
5741 	/* Enable link change event even when serdes polling.  */
5742 	tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5743 	udelay(40);
5744 
5745 	current_link_up = false;
5746 	tp->link_config.rmt_adv = 0;
5747 	mac_status = tr32(MAC_STATUS);
5748 
5749 	if (tg3_flag(tp, HW_AUTONEG))
5750 		current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
5751 	else
5752 		current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
5753 
5754 	tp->napi[0].hw_status->status =
5755 		(SD_STATUS_UPDATED |
5756 		 (tp->napi[0].hw_status->status & ~SD_STATUS_LINK_CHG));
5757 
5758 	for (i = 0; i < 100; i++) {
5759 		tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
5760 				    MAC_STATUS_CFG_CHANGED));
5761 		udelay(5);
5762 		if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
5763 					 MAC_STATUS_CFG_CHANGED |
5764 					 MAC_STATUS_LNKSTATE_CHANGED)) == 0)
5765 			break;
5766 	}
5767 
5768 	mac_status = tr32(MAC_STATUS);
5769 	if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
5770 		current_link_up = false;
5771 		if (tp->link_config.autoneg == AUTONEG_ENABLE &&
5772 		    tp->serdes_counter == 0) {
5773 			tw32_f(MAC_MODE, (tp->mac_mode |
5774 					  MAC_MODE_SEND_CONFIGS));
5775 			udelay(1);
5776 			tw32_f(MAC_MODE, tp->mac_mode);
5777 		}
5778 	}
5779 
5780 	if (current_link_up) {
5781 		tp->link_config.active_speed = SPEED_1000;
5782 		tp->link_config.active_duplex = DUPLEX_FULL;
5783 		tw32(MAC_LED_CTRL, (tp->led_ctrl |
5784 				    LED_CTRL_LNKLED_OVERRIDE |
5785 				    LED_CTRL_1000MBPS_ON));
5786 	} else {
5787 		tp->link_config.active_speed = SPEED_UNKNOWN;
5788 		tp->link_config.active_duplex = DUPLEX_UNKNOWN;
5789 		tw32(MAC_LED_CTRL, (tp->led_ctrl |
5790 				    LED_CTRL_LNKLED_OVERRIDE |
5791 				    LED_CTRL_TRAFFIC_OVERRIDE));
5792 	}
5793 
5794 	if (!tg3_test_and_report_link_chg(tp, current_link_up)) {
5795 		u32 now_pause_cfg = tp->link_config.active_flowctrl;
5796 		if (orig_pause_cfg != now_pause_cfg ||
5797 		    orig_active_speed != tp->link_config.active_speed ||
5798 		    orig_active_duplex != tp->link_config.active_duplex)
5799 			tg3_link_report(tp);
5800 	}
5801 
5802 	return 0;
5803 }
5804 
5805 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, bool force_reset)
5806 {
5807 	int err = 0;
5808 	u32 bmsr, bmcr;
5809 	u16 current_speed = SPEED_UNKNOWN;
5810 	u8 current_duplex = DUPLEX_UNKNOWN;
5811 	bool current_link_up = false;
5812 	u32 local_adv, remote_adv, sgsr;
5813 
5814 	if ((tg3_asic_rev(tp) == ASIC_REV_5719 ||
5815 	     tg3_asic_rev(tp) == ASIC_REV_5720) &&
5816 	     !tg3_readphy(tp, SERDES_TG3_1000X_STATUS, &sgsr) &&
5817 	     (sgsr & SERDES_TG3_SGMII_MODE)) {
5818 
5819 		if (force_reset)
5820 			tg3_phy_reset(tp);
5821 
5822 		tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
5823 
5824 		if (!(sgsr & SERDES_TG3_LINK_UP)) {
5825 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5826 		} else {
5827 			current_link_up = true;
5828 			if (sgsr & SERDES_TG3_SPEED_1000) {
5829 				current_speed = SPEED_1000;
5830 				tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5831 			} else if (sgsr & SERDES_TG3_SPEED_100) {
5832 				current_speed = SPEED_100;
5833 				tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
5834 			} else {
5835 				current_speed = SPEED_10;
5836 				tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
5837 			}
5838 
5839 			if (sgsr & SERDES_TG3_FULL_DUPLEX)
5840 				current_duplex = DUPLEX_FULL;
5841 			else
5842 				current_duplex = DUPLEX_HALF;
5843 		}
5844 
5845 		tw32_f(MAC_MODE, tp->mac_mode);
5846 		udelay(40);
5847 
5848 		tg3_clear_mac_status(tp);
5849 
5850 		goto fiber_setup_done;
5851 	}
5852 
5853 	tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5854 	tw32_f(MAC_MODE, tp->mac_mode);
5855 	udelay(40);
5856 
5857 	tg3_clear_mac_status(tp);
5858 
5859 	if (force_reset)
5860 		tg3_phy_reset(tp);
5861 
5862 	tp->link_config.rmt_adv = 0;
5863 
5864 	err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5865 	err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5866 	if (tg3_asic_rev(tp) == ASIC_REV_5714) {
5867 		if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
5868 			bmsr |= BMSR_LSTATUS;
5869 		else
5870 			bmsr &= ~BMSR_LSTATUS;
5871 	}
5872 
5873 	err |= tg3_readphy(tp, MII_BMCR, &bmcr);
5874 
5875 	if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
5876 	    (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
5877 		/* do nothing, just check for link up at the end */
5878 	} else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
5879 		u32 adv, newadv;
5880 
5881 		err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
5882 		newadv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
5883 				 ADVERTISE_1000XPAUSE |
5884 				 ADVERTISE_1000XPSE_ASYM |
5885 				 ADVERTISE_SLCT);
5886 
5887 		newadv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5888 		newadv |= ethtool_adv_to_mii_adv_x(tp->link_config.advertising);
5889 
5890 		if ((newadv != adv) || !(bmcr & BMCR_ANENABLE)) {
5891 			tg3_writephy(tp, MII_ADVERTISE, newadv);
5892 			bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
5893 			tg3_writephy(tp, MII_BMCR, bmcr);
5894 
5895 			tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5896 			tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
5897 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5898 
5899 			return err;
5900 		}
5901 	} else {
5902 		u32 new_bmcr;
5903 
5904 		bmcr &= ~BMCR_SPEED1000;
5905 		new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
5906 
5907 		if (tp->link_config.duplex == DUPLEX_FULL)
5908 			new_bmcr |= BMCR_FULLDPLX;
5909 
5910 		if (new_bmcr != bmcr) {
5911 			/* BMCR_SPEED1000 is a reserved bit that needs
5912 			 * to be set on write.
5913 			 */
5914 			new_bmcr |= BMCR_SPEED1000;
5915 
5916 			/* Force a linkdown */
5917 			if (tp->link_up) {
5918 				u32 adv;
5919 
5920 				err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
5921 				adv &= ~(ADVERTISE_1000XFULL |
5922 					 ADVERTISE_1000XHALF |
5923 					 ADVERTISE_SLCT);
5924 				tg3_writephy(tp, MII_ADVERTISE, adv);
5925 				tg3_writephy(tp, MII_BMCR, bmcr |
5926 							   BMCR_ANRESTART |
5927 							   BMCR_ANENABLE);
5928 				udelay(10);
5929 				tg3_carrier_off(tp);
5930 			}
5931 			tg3_writephy(tp, MII_BMCR, new_bmcr);
5932 			bmcr = new_bmcr;
5933 			err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5934 			err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5935 			if (tg3_asic_rev(tp) == ASIC_REV_5714) {
5936 				if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
5937 					bmsr |= BMSR_LSTATUS;
5938 				else
5939 					bmsr &= ~BMSR_LSTATUS;
5940 			}
5941 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5942 		}
5943 	}
5944 
5945 	if (bmsr & BMSR_LSTATUS) {
5946 		current_speed = SPEED_1000;
5947 		current_link_up = true;
5948 		if (bmcr & BMCR_FULLDPLX)
5949 			current_duplex = DUPLEX_FULL;
5950 		else
5951 			current_duplex = DUPLEX_HALF;
5952 
5953 		local_adv = 0;
5954 		remote_adv = 0;
5955 
5956 		if (bmcr & BMCR_ANENABLE) {
5957 			u32 common;
5958 
5959 			err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
5960 			err |= tg3_readphy(tp, MII_LPA, &remote_adv);
5961 			common = local_adv & remote_adv;
5962 			if (common & (ADVERTISE_1000XHALF |
5963 				      ADVERTISE_1000XFULL)) {
5964 				if (common & ADVERTISE_1000XFULL)
5965 					current_duplex = DUPLEX_FULL;
5966 				else
5967 					current_duplex = DUPLEX_HALF;
5968 
5969 				tp->link_config.rmt_adv =
5970 					   mii_adv_to_ethtool_adv_x(remote_adv);
5971 			} else if (!tg3_flag(tp, 5780_CLASS)) {
5972 				/* Link is up via parallel detect */
5973 			} else {
5974 				current_link_up = false;
5975 			}
5976 		}
5977 	}
5978 
5979 fiber_setup_done:
5980 	if (current_link_up && current_duplex == DUPLEX_FULL)
5981 		tg3_setup_flow_control(tp, local_adv, remote_adv);
5982 
5983 	tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
5984 	if (tp->link_config.active_duplex == DUPLEX_HALF)
5985 		tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
5986 
5987 	tw32_f(MAC_MODE, tp->mac_mode);
5988 	udelay(40);
5989 
5990 	tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5991 
5992 	tp->link_config.active_speed = current_speed;
5993 	tp->link_config.active_duplex = current_duplex;
5994 
5995 	tg3_test_and_report_link_chg(tp, current_link_up);
5996 	return err;
5997 }
5998 
5999 static void tg3_serdes_parallel_detect(struct tg3 *tp)
6000 {
6001 	if (tp->serdes_counter) {
6002 		/* Give autoneg time to complete. */
6003 		tp->serdes_counter--;
6004 		return;
6005 	}
6006 
6007 	if (!tp->link_up &&
6008 	    (tp->link_config.autoneg == AUTONEG_ENABLE)) {
6009 		u32 bmcr;
6010 
6011 		tg3_readphy(tp, MII_BMCR, &bmcr);
6012 		if (bmcr & BMCR_ANENABLE) {
6013 			u32 phy1, phy2;
6014 
6015 			/* Select shadow register 0x1f */
6016 			tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x7c00);
6017 			tg3_readphy(tp, MII_TG3_MISC_SHDW, &phy1);
6018 
6019 			/* Select expansion interrupt status register */
6020 			tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
6021 					 MII_TG3_DSP_EXP1_INT_STAT);
6022 			tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6023 			tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6024 
6025 			if ((phy1 & 0x10) && !(phy2 & 0x20)) {
6026 				/* We have signal detect and not receiving
6027 				 * config code words, link is up by parallel
6028 				 * detection.
6029 				 */
6030 
6031 				bmcr &= ~BMCR_ANENABLE;
6032 				bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
6033 				tg3_writephy(tp, MII_BMCR, bmcr);
6034 				tp->phy_flags |= TG3_PHYFLG_PARALLEL_DETECT;
6035 			}
6036 		}
6037 	} else if (tp->link_up &&
6038 		   (tp->link_config.autoneg == AUTONEG_ENABLE) &&
6039 		   (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
6040 		u32 phy2;
6041 
6042 		/* Select expansion interrupt status register */
6043 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
6044 				 MII_TG3_DSP_EXP1_INT_STAT);
6045 		tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6046 		if (phy2 & 0x20) {
6047 			u32 bmcr;
6048 
6049 			/* Config code words received, turn on autoneg. */
6050 			tg3_readphy(tp, MII_BMCR, &bmcr);
6051 			tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
6052 
6053 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
6054 
6055 		}
6056 	}
6057 }
6058 
6059 static int tg3_setup_phy(struct tg3 *tp, bool force_reset)
6060 {
6061 	u32 val;
6062 	int err;
6063 
6064 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
6065 		err = tg3_setup_fiber_phy(tp, force_reset);
6066 	else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
6067 		err = tg3_setup_fiber_mii_phy(tp, force_reset);
6068 	else
6069 		err = tg3_setup_copper_phy(tp, force_reset);
6070 
6071 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX) {
6072 		u32 scale;
6073 
6074 		val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
6075 		if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
6076 			scale = 65;
6077 		else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
6078 			scale = 6;
6079 		else
6080 			scale = 12;
6081 
6082 		val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
6083 		val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
6084 		tw32(GRC_MISC_CFG, val);
6085 	}
6086 
6087 	val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
6088 	      (6 << TX_LENGTHS_IPG_SHIFT);
6089 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
6090 	    tg3_asic_rev(tp) == ASIC_REV_5762)
6091 		val |= tr32(MAC_TX_LENGTHS) &
6092 		       (TX_LENGTHS_JMB_FRM_LEN_MSK |
6093 			TX_LENGTHS_CNT_DWN_VAL_MSK);
6094 
6095 	if (tp->link_config.active_speed == SPEED_1000 &&
6096 	    tp->link_config.active_duplex == DUPLEX_HALF)
6097 		tw32(MAC_TX_LENGTHS, val |
6098 		     (0xff << TX_LENGTHS_SLOT_TIME_SHIFT));
6099 	else
6100 		tw32(MAC_TX_LENGTHS, val |
6101 		     (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
6102 
6103 	if (!tg3_flag(tp, 5705_PLUS)) {
6104 		if (tp->link_up) {
6105 			tw32(HOSTCC_STAT_COAL_TICKS,
6106 			     tp->coal.stats_block_coalesce_usecs);
6107 		} else {
6108 			tw32(HOSTCC_STAT_COAL_TICKS, 0);
6109 		}
6110 	}
6111 
6112 	if (tg3_flag(tp, ASPM_WORKAROUND)) {
6113 		val = tr32(PCIE_PWR_MGMT_THRESH);
6114 		if (!tp->link_up)
6115 			val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
6116 			      tp->pwrmgmt_thresh;
6117 		else
6118 			val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
6119 		tw32(PCIE_PWR_MGMT_THRESH, val);
6120 	}
6121 
6122 	return err;
6123 }
6124 
6125 /* tp->lock must be held */
6126 static u64 tg3_refclk_read(struct tg3 *tp)
6127 {
6128 	u64 stamp = tr32(TG3_EAV_REF_CLCK_LSB);
6129 	return stamp | (u64)tr32(TG3_EAV_REF_CLCK_MSB) << 32;
6130 }
6131 
6132 /* tp->lock must be held */
6133 static void tg3_refclk_write(struct tg3 *tp, u64 newval)
6134 {
6135 	u32 clock_ctl = tr32(TG3_EAV_REF_CLCK_CTL);
6136 
6137 	tw32(TG3_EAV_REF_CLCK_CTL, clock_ctl | TG3_EAV_REF_CLCK_CTL_STOP);
6138 	tw32(TG3_EAV_REF_CLCK_LSB, newval & 0xffffffff);
6139 	tw32(TG3_EAV_REF_CLCK_MSB, newval >> 32);
6140 	tw32_f(TG3_EAV_REF_CLCK_CTL, clock_ctl | TG3_EAV_REF_CLCK_CTL_RESUME);
6141 }
6142 
6143 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync);
6144 static inline void tg3_full_unlock(struct tg3 *tp);
6145 static int tg3_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info)
6146 {
6147 	struct tg3 *tp = netdev_priv(dev);
6148 
6149 	info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
6150 				SOF_TIMESTAMPING_RX_SOFTWARE |
6151 				SOF_TIMESTAMPING_SOFTWARE;
6152 
6153 	if (tg3_flag(tp, PTP_CAPABLE)) {
6154 		info->so_timestamping |= SOF_TIMESTAMPING_TX_HARDWARE |
6155 					SOF_TIMESTAMPING_RX_HARDWARE |
6156 					SOF_TIMESTAMPING_RAW_HARDWARE;
6157 	}
6158 
6159 	if (tp->ptp_clock)
6160 		info->phc_index = ptp_clock_index(tp->ptp_clock);
6161 	else
6162 		info->phc_index = -1;
6163 
6164 	info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
6165 
6166 	info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
6167 			   (1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
6168 			   (1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
6169 			   (1 << HWTSTAMP_FILTER_PTP_V2_L4_EVENT);
6170 	return 0;
6171 }
6172 
6173 static int tg3_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
6174 {
6175 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6176 	bool neg_adj = false;
6177 	u32 correction = 0;
6178 
6179 	if (ppb < 0) {
6180 		neg_adj = true;
6181 		ppb = -ppb;
6182 	}
6183 
6184 	/* Frequency adjustment is performed using hardware with a 24 bit
6185 	 * accumulator and a programmable correction value. On each clk, the
6186 	 * correction value gets added to the accumulator and when it
6187 	 * overflows, the time counter is incremented/decremented.
6188 	 *
6189 	 * So conversion from ppb to correction value is
6190 	 *		ppb * (1 << 24) / 1000000000
6191 	 */
6192 	correction = div_u64((u64)ppb * (1 << 24), 1000000000ULL) &
6193 		     TG3_EAV_REF_CLK_CORRECT_MASK;
6194 
6195 	tg3_full_lock(tp, 0);
6196 
6197 	if (correction)
6198 		tw32(TG3_EAV_REF_CLK_CORRECT_CTL,
6199 		     TG3_EAV_REF_CLK_CORRECT_EN |
6200 		     (neg_adj ? TG3_EAV_REF_CLK_CORRECT_NEG : 0) | correction);
6201 	else
6202 		tw32(TG3_EAV_REF_CLK_CORRECT_CTL, 0);
6203 
6204 	tg3_full_unlock(tp);
6205 
6206 	return 0;
6207 }
6208 
6209 static int tg3_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
6210 {
6211 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6212 
6213 	tg3_full_lock(tp, 0);
6214 	tp->ptp_adjust += delta;
6215 	tg3_full_unlock(tp);
6216 
6217 	return 0;
6218 }
6219 
6220 static int tg3_ptp_gettime(struct ptp_clock_info *ptp, struct timespec *ts)
6221 {
6222 	u64 ns;
6223 	u32 remainder;
6224 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6225 
6226 	tg3_full_lock(tp, 0);
6227 	ns = tg3_refclk_read(tp);
6228 	ns += tp->ptp_adjust;
6229 	tg3_full_unlock(tp);
6230 
6231 	ts->tv_sec = div_u64_rem(ns, 1000000000, &remainder);
6232 	ts->tv_nsec = remainder;
6233 
6234 	return 0;
6235 }
6236 
6237 static int tg3_ptp_settime(struct ptp_clock_info *ptp,
6238 			   const struct timespec *ts)
6239 {
6240 	u64 ns;
6241 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6242 
6243 	ns = timespec_to_ns(ts);
6244 
6245 	tg3_full_lock(tp, 0);
6246 	tg3_refclk_write(tp, ns);
6247 	tp->ptp_adjust = 0;
6248 	tg3_full_unlock(tp);
6249 
6250 	return 0;
6251 }
6252 
6253 static int tg3_ptp_enable(struct ptp_clock_info *ptp,
6254 			  struct ptp_clock_request *rq, int on)
6255 {
6256 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6257 	u32 clock_ctl;
6258 	int rval = 0;
6259 
6260 	switch (rq->type) {
6261 	case PTP_CLK_REQ_PEROUT:
6262 		if (rq->perout.index != 0)
6263 			return -EINVAL;
6264 
6265 		tg3_full_lock(tp, 0);
6266 		clock_ctl = tr32(TG3_EAV_REF_CLCK_CTL);
6267 		clock_ctl &= ~TG3_EAV_CTL_TSYNC_GPIO_MASK;
6268 
6269 		if (on) {
6270 			u64 nsec;
6271 
6272 			nsec = rq->perout.start.sec * 1000000000ULL +
6273 			       rq->perout.start.nsec;
6274 
6275 			if (rq->perout.period.sec || rq->perout.period.nsec) {
6276 				netdev_warn(tp->dev,
6277 					    "Device supports only a one-shot timesync output, period must be 0\n");
6278 				rval = -EINVAL;
6279 				goto err_out;
6280 			}
6281 
6282 			if (nsec & (1ULL << 63)) {
6283 				netdev_warn(tp->dev,
6284 					    "Start value (nsec) is over limit. Maximum size of start is only 63 bits\n");
6285 				rval = -EINVAL;
6286 				goto err_out;
6287 			}
6288 
6289 			tw32(TG3_EAV_WATCHDOG0_LSB, (nsec & 0xffffffff));
6290 			tw32(TG3_EAV_WATCHDOG0_MSB,
6291 			     TG3_EAV_WATCHDOG0_EN |
6292 			     ((nsec >> 32) & TG3_EAV_WATCHDOG_MSB_MASK));
6293 
6294 			tw32(TG3_EAV_REF_CLCK_CTL,
6295 			     clock_ctl | TG3_EAV_CTL_TSYNC_WDOG0);
6296 		} else {
6297 			tw32(TG3_EAV_WATCHDOG0_MSB, 0);
6298 			tw32(TG3_EAV_REF_CLCK_CTL, clock_ctl);
6299 		}
6300 
6301 err_out:
6302 		tg3_full_unlock(tp);
6303 		return rval;
6304 
6305 	default:
6306 		break;
6307 	}
6308 
6309 	return -EOPNOTSUPP;
6310 }
6311 
6312 static const struct ptp_clock_info tg3_ptp_caps = {
6313 	.owner		= THIS_MODULE,
6314 	.name		= "tg3 clock",
6315 	.max_adj	= 250000000,
6316 	.n_alarm	= 0,
6317 	.n_ext_ts	= 0,
6318 	.n_per_out	= 1,
6319 	.n_pins		= 0,
6320 	.pps		= 0,
6321 	.adjfreq	= tg3_ptp_adjfreq,
6322 	.adjtime	= tg3_ptp_adjtime,
6323 	.gettime	= tg3_ptp_gettime,
6324 	.settime	= tg3_ptp_settime,
6325 	.enable		= tg3_ptp_enable,
6326 };
6327 
6328 static void tg3_hwclock_to_timestamp(struct tg3 *tp, u64 hwclock,
6329 				     struct skb_shared_hwtstamps *timestamp)
6330 {
6331 	memset(timestamp, 0, sizeof(struct skb_shared_hwtstamps));
6332 	timestamp->hwtstamp  = ns_to_ktime((hwclock & TG3_TSTAMP_MASK) +
6333 					   tp->ptp_adjust);
6334 }
6335 
6336 /* tp->lock must be held */
6337 static void tg3_ptp_init(struct tg3 *tp)
6338 {
6339 	if (!tg3_flag(tp, PTP_CAPABLE))
6340 		return;
6341 
6342 	/* Initialize the hardware clock to the system time. */
6343 	tg3_refclk_write(tp, ktime_to_ns(ktime_get_real()));
6344 	tp->ptp_adjust = 0;
6345 	tp->ptp_info = tg3_ptp_caps;
6346 }
6347 
6348 /* tp->lock must be held */
6349 static void tg3_ptp_resume(struct tg3 *tp)
6350 {
6351 	if (!tg3_flag(tp, PTP_CAPABLE))
6352 		return;
6353 
6354 	tg3_refclk_write(tp, ktime_to_ns(ktime_get_real()) + tp->ptp_adjust);
6355 	tp->ptp_adjust = 0;
6356 }
6357 
6358 static void tg3_ptp_fini(struct tg3 *tp)
6359 {
6360 	if (!tg3_flag(tp, PTP_CAPABLE) || !tp->ptp_clock)
6361 		return;
6362 
6363 	ptp_clock_unregister(tp->ptp_clock);
6364 	tp->ptp_clock = NULL;
6365 	tp->ptp_adjust = 0;
6366 }
6367 
6368 static inline int tg3_irq_sync(struct tg3 *tp)
6369 {
6370 	return tp->irq_sync;
6371 }
6372 
6373 static inline void tg3_rd32_loop(struct tg3 *tp, u32 *dst, u32 off, u32 len)
6374 {
6375 	int i;
6376 
6377 	dst = (u32 *)((u8 *)dst + off);
6378 	for (i = 0; i < len; i += sizeof(u32))
6379 		*dst++ = tr32(off + i);
6380 }
6381 
6382 static void tg3_dump_legacy_regs(struct tg3 *tp, u32 *regs)
6383 {
6384 	tg3_rd32_loop(tp, regs, TG3PCI_VENDOR, 0xb0);
6385 	tg3_rd32_loop(tp, regs, MAILBOX_INTERRUPT_0, 0x200);
6386 	tg3_rd32_loop(tp, regs, MAC_MODE, 0x4f0);
6387 	tg3_rd32_loop(tp, regs, SNDDATAI_MODE, 0xe0);
6388 	tg3_rd32_loop(tp, regs, SNDDATAC_MODE, 0x04);
6389 	tg3_rd32_loop(tp, regs, SNDBDS_MODE, 0x80);
6390 	tg3_rd32_loop(tp, regs, SNDBDI_MODE, 0x48);
6391 	tg3_rd32_loop(tp, regs, SNDBDC_MODE, 0x04);
6392 	tg3_rd32_loop(tp, regs, RCVLPC_MODE, 0x20);
6393 	tg3_rd32_loop(tp, regs, RCVLPC_SELLST_BASE, 0x15c);
6394 	tg3_rd32_loop(tp, regs, RCVDBDI_MODE, 0x0c);
6395 	tg3_rd32_loop(tp, regs, RCVDBDI_JUMBO_BD, 0x3c);
6396 	tg3_rd32_loop(tp, regs, RCVDBDI_BD_PROD_IDX_0, 0x44);
6397 	tg3_rd32_loop(tp, regs, RCVDCC_MODE, 0x04);
6398 	tg3_rd32_loop(tp, regs, RCVBDI_MODE, 0x20);
6399 	tg3_rd32_loop(tp, regs, RCVCC_MODE, 0x14);
6400 	tg3_rd32_loop(tp, regs, RCVLSC_MODE, 0x08);
6401 	tg3_rd32_loop(tp, regs, MBFREE_MODE, 0x08);
6402 	tg3_rd32_loop(tp, regs, HOSTCC_MODE, 0x100);
6403 
6404 	if (tg3_flag(tp, SUPPORT_MSIX))
6405 		tg3_rd32_loop(tp, regs, HOSTCC_RXCOL_TICKS_VEC1, 0x180);
6406 
6407 	tg3_rd32_loop(tp, regs, MEMARB_MODE, 0x10);
6408 	tg3_rd32_loop(tp, regs, BUFMGR_MODE, 0x58);
6409 	tg3_rd32_loop(tp, regs, RDMAC_MODE, 0x08);
6410 	tg3_rd32_loop(tp, regs, WDMAC_MODE, 0x08);
6411 	tg3_rd32_loop(tp, regs, RX_CPU_MODE, 0x04);
6412 	tg3_rd32_loop(tp, regs, RX_CPU_STATE, 0x04);
6413 	tg3_rd32_loop(tp, regs, RX_CPU_PGMCTR, 0x04);
6414 	tg3_rd32_loop(tp, regs, RX_CPU_HWBKPT, 0x04);
6415 
6416 	if (!tg3_flag(tp, 5705_PLUS)) {
6417 		tg3_rd32_loop(tp, regs, TX_CPU_MODE, 0x04);
6418 		tg3_rd32_loop(tp, regs, TX_CPU_STATE, 0x04);
6419 		tg3_rd32_loop(tp, regs, TX_CPU_PGMCTR, 0x04);
6420 	}
6421 
6422 	tg3_rd32_loop(tp, regs, GRCMBOX_INTERRUPT_0, 0x110);
6423 	tg3_rd32_loop(tp, regs, FTQ_RESET, 0x120);
6424 	tg3_rd32_loop(tp, regs, MSGINT_MODE, 0x0c);
6425 	tg3_rd32_loop(tp, regs, DMAC_MODE, 0x04);
6426 	tg3_rd32_loop(tp, regs, GRC_MODE, 0x4c);
6427 
6428 	if (tg3_flag(tp, NVRAM))
6429 		tg3_rd32_loop(tp, regs, NVRAM_CMD, 0x24);
6430 }
6431 
6432 static void tg3_dump_state(struct tg3 *tp)
6433 {
6434 	int i;
6435 	u32 *regs;
6436 
6437 	regs = kzalloc(TG3_REG_BLK_SIZE, GFP_ATOMIC);
6438 	if (!regs)
6439 		return;
6440 
6441 	if (tg3_flag(tp, PCI_EXPRESS)) {
6442 		/* Read up to but not including private PCI registers */
6443 		for (i = 0; i < TG3_PCIE_TLDLPL_PORT; i += sizeof(u32))
6444 			regs[i / sizeof(u32)] = tr32(i);
6445 	} else
6446 		tg3_dump_legacy_regs(tp, regs);
6447 
6448 	for (i = 0; i < TG3_REG_BLK_SIZE / sizeof(u32); i += 4) {
6449 		if (!regs[i + 0] && !regs[i + 1] &&
6450 		    !regs[i + 2] && !regs[i + 3])
6451 			continue;
6452 
6453 		netdev_err(tp->dev, "0x%08x: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
6454 			   i * 4,
6455 			   regs[i + 0], regs[i + 1], regs[i + 2], regs[i + 3]);
6456 	}
6457 
6458 	kfree(regs);
6459 
6460 	for (i = 0; i < tp->irq_cnt; i++) {
6461 		struct tg3_napi *tnapi = &tp->napi[i];
6462 
6463 		/* SW status block */
6464 		netdev_err(tp->dev,
6465 			 "%d: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
6466 			   i,
6467 			   tnapi->hw_status->status,
6468 			   tnapi->hw_status->status_tag,
6469 			   tnapi->hw_status->rx_jumbo_consumer,
6470 			   tnapi->hw_status->rx_consumer,
6471 			   tnapi->hw_status->rx_mini_consumer,
6472 			   tnapi->hw_status->idx[0].rx_producer,
6473 			   tnapi->hw_status->idx[0].tx_consumer);
6474 
6475 		netdev_err(tp->dev,
6476 		"%d: NAPI info [%08x:%08x:(%04x:%04x:%04x):%04x:(%04x:%04x:%04x:%04x)]\n",
6477 			   i,
6478 			   tnapi->last_tag, tnapi->last_irq_tag,
6479 			   tnapi->tx_prod, tnapi->tx_cons, tnapi->tx_pending,
6480 			   tnapi->rx_rcb_ptr,
6481 			   tnapi->prodring.rx_std_prod_idx,
6482 			   tnapi->prodring.rx_std_cons_idx,
6483 			   tnapi->prodring.rx_jmb_prod_idx,
6484 			   tnapi->prodring.rx_jmb_cons_idx);
6485 	}
6486 }
6487 
6488 /* This is called whenever we suspect that the system chipset is re-
6489  * ordering the sequence of MMIO to the tx send mailbox. The symptom
6490  * is bogus tx completions. We try to recover by setting the
6491  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
6492  * in the workqueue.
6493  */
6494 static void tg3_tx_recover(struct tg3 *tp)
6495 {
6496 	BUG_ON(tg3_flag(tp, MBOX_WRITE_REORDER) ||
6497 	       tp->write32_tx_mbox == tg3_write_indirect_mbox);
6498 
6499 	netdev_warn(tp->dev,
6500 		    "The system may be re-ordering memory-mapped I/O "
6501 		    "cycles to the network device, attempting to recover. "
6502 		    "Please report the problem to the driver maintainer "
6503 		    "and include system chipset information.\n");
6504 
6505 	tg3_flag_set(tp, TX_RECOVERY_PENDING);
6506 }
6507 
6508 static inline u32 tg3_tx_avail(struct tg3_napi *tnapi)
6509 {
6510 	/* Tell compiler to fetch tx indices from memory. */
6511 	barrier();
6512 	return tnapi->tx_pending -
6513 	       ((tnapi->tx_prod - tnapi->tx_cons) & (TG3_TX_RING_SIZE - 1));
6514 }
6515 
6516 /* Tigon3 never reports partial packet sends.  So we do not
6517  * need special logic to handle SKBs that have not had all
6518  * of their frags sent yet, like SunGEM does.
6519  */
6520 static void tg3_tx(struct tg3_napi *tnapi)
6521 {
6522 	struct tg3 *tp = tnapi->tp;
6523 	u32 hw_idx = tnapi->hw_status->idx[0].tx_consumer;
6524 	u32 sw_idx = tnapi->tx_cons;
6525 	struct netdev_queue *txq;
6526 	int index = tnapi - tp->napi;
6527 	unsigned int pkts_compl = 0, bytes_compl = 0;
6528 
6529 	if (tg3_flag(tp, ENABLE_TSS))
6530 		index--;
6531 
6532 	txq = netdev_get_tx_queue(tp->dev, index);
6533 
6534 	while (sw_idx != hw_idx) {
6535 		struct tg3_tx_ring_info *ri = &tnapi->tx_buffers[sw_idx];
6536 		struct sk_buff *skb = ri->skb;
6537 		int i, tx_bug = 0;
6538 
6539 		if (unlikely(skb == NULL)) {
6540 			tg3_tx_recover(tp);
6541 			return;
6542 		}
6543 
6544 		if (tnapi->tx_ring[sw_idx].len_flags & TXD_FLAG_HWTSTAMP) {
6545 			struct skb_shared_hwtstamps timestamp;
6546 			u64 hwclock = tr32(TG3_TX_TSTAMP_LSB);
6547 			hwclock |= (u64)tr32(TG3_TX_TSTAMP_MSB) << 32;
6548 
6549 			tg3_hwclock_to_timestamp(tp, hwclock, &timestamp);
6550 
6551 			skb_tstamp_tx(skb, &timestamp);
6552 		}
6553 
6554 		pci_unmap_single(tp->pdev,
6555 				 dma_unmap_addr(ri, mapping),
6556 				 skb_headlen(skb),
6557 				 PCI_DMA_TODEVICE);
6558 
6559 		ri->skb = NULL;
6560 
6561 		while (ri->fragmented) {
6562 			ri->fragmented = false;
6563 			sw_idx = NEXT_TX(sw_idx);
6564 			ri = &tnapi->tx_buffers[sw_idx];
6565 		}
6566 
6567 		sw_idx = NEXT_TX(sw_idx);
6568 
6569 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
6570 			ri = &tnapi->tx_buffers[sw_idx];
6571 			if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
6572 				tx_bug = 1;
6573 
6574 			pci_unmap_page(tp->pdev,
6575 				       dma_unmap_addr(ri, mapping),
6576 				       skb_frag_size(&skb_shinfo(skb)->frags[i]),
6577 				       PCI_DMA_TODEVICE);
6578 
6579 			while (ri->fragmented) {
6580 				ri->fragmented = false;
6581 				sw_idx = NEXT_TX(sw_idx);
6582 				ri = &tnapi->tx_buffers[sw_idx];
6583 			}
6584 
6585 			sw_idx = NEXT_TX(sw_idx);
6586 		}
6587 
6588 		pkts_compl++;
6589 		bytes_compl += skb->len;
6590 
6591 		dev_kfree_skb_any(skb);
6592 
6593 		if (unlikely(tx_bug)) {
6594 			tg3_tx_recover(tp);
6595 			return;
6596 		}
6597 	}
6598 
6599 	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
6600 
6601 	tnapi->tx_cons = sw_idx;
6602 
6603 	/* Need to make the tx_cons update visible to tg3_start_xmit()
6604 	 * before checking for netif_queue_stopped().  Without the
6605 	 * memory barrier, there is a small possibility that tg3_start_xmit()
6606 	 * will miss it and cause the queue to be stopped forever.
6607 	 */
6608 	smp_mb();
6609 
6610 	if (unlikely(netif_tx_queue_stopped(txq) &&
6611 		     (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))) {
6612 		__netif_tx_lock(txq, smp_processor_id());
6613 		if (netif_tx_queue_stopped(txq) &&
6614 		    (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))
6615 			netif_tx_wake_queue(txq);
6616 		__netif_tx_unlock(txq);
6617 	}
6618 }
6619 
6620 static void tg3_frag_free(bool is_frag, void *data)
6621 {
6622 	if (is_frag)
6623 		put_page(virt_to_head_page(data));
6624 	else
6625 		kfree(data);
6626 }
6627 
6628 static void tg3_rx_data_free(struct tg3 *tp, struct ring_info *ri, u32 map_sz)
6629 {
6630 	unsigned int skb_size = SKB_DATA_ALIGN(map_sz + TG3_RX_OFFSET(tp)) +
6631 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
6632 
6633 	if (!ri->data)
6634 		return;
6635 
6636 	pci_unmap_single(tp->pdev, dma_unmap_addr(ri, mapping),
6637 			 map_sz, PCI_DMA_FROMDEVICE);
6638 	tg3_frag_free(skb_size <= PAGE_SIZE, ri->data);
6639 	ri->data = NULL;
6640 }
6641 
6642 
6643 /* Returns size of skb allocated or < 0 on error.
6644  *
6645  * We only need to fill in the address because the other members
6646  * of the RX descriptor are invariant, see tg3_init_rings.
6647  *
6648  * Note the purposeful assymetry of cpu vs. chip accesses.  For
6649  * posting buffers we only dirty the first cache line of the RX
6650  * descriptor (containing the address).  Whereas for the RX status
6651  * buffers the cpu only reads the last cacheline of the RX descriptor
6652  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
6653  */
6654 static int tg3_alloc_rx_data(struct tg3 *tp, struct tg3_rx_prodring_set *tpr,
6655 			     u32 opaque_key, u32 dest_idx_unmasked,
6656 			     unsigned int *frag_size)
6657 {
6658 	struct tg3_rx_buffer_desc *desc;
6659 	struct ring_info *map;
6660 	u8 *data;
6661 	dma_addr_t mapping;
6662 	int skb_size, data_size, dest_idx;
6663 
6664 	switch (opaque_key) {
6665 	case RXD_OPAQUE_RING_STD:
6666 		dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
6667 		desc = &tpr->rx_std[dest_idx];
6668 		map = &tpr->rx_std_buffers[dest_idx];
6669 		data_size = tp->rx_pkt_map_sz;
6670 		break;
6671 
6672 	case RXD_OPAQUE_RING_JUMBO:
6673 		dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
6674 		desc = &tpr->rx_jmb[dest_idx].std;
6675 		map = &tpr->rx_jmb_buffers[dest_idx];
6676 		data_size = TG3_RX_JMB_MAP_SZ;
6677 		break;
6678 
6679 	default:
6680 		return -EINVAL;
6681 	}
6682 
6683 	/* Do not overwrite any of the map or rp information
6684 	 * until we are sure we can commit to a new buffer.
6685 	 *
6686 	 * Callers depend upon this behavior and assume that
6687 	 * we leave everything unchanged if we fail.
6688 	 */
6689 	skb_size = SKB_DATA_ALIGN(data_size + TG3_RX_OFFSET(tp)) +
6690 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
6691 	if (skb_size <= PAGE_SIZE) {
6692 		data = netdev_alloc_frag(skb_size);
6693 		*frag_size = skb_size;
6694 	} else {
6695 		data = kmalloc(skb_size, GFP_ATOMIC);
6696 		*frag_size = 0;
6697 	}
6698 	if (!data)
6699 		return -ENOMEM;
6700 
6701 	mapping = pci_map_single(tp->pdev,
6702 				 data + TG3_RX_OFFSET(tp),
6703 				 data_size,
6704 				 PCI_DMA_FROMDEVICE);
6705 	if (unlikely(pci_dma_mapping_error(tp->pdev, mapping))) {
6706 		tg3_frag_free(skb_size <= PAGE_SIZE, data);
6707 		return -EIO;
6708 	}
6709 
6710 	map->data = data;
6711 	dma_unmap_addr_set(map, mapping, mapping);
6712 
6713 	desc->addr_hi = ((u64)mapping >> 32);
6714 	desc->addr_lo = ((u64)mapping & 0xffffffff);
6715 
6716 	return data_size;
6717 }
6718 
6719 /* We only need to move over in the address because the other
6720  * members of the RX descriptor are invariant.  See notes above
6721  * tg3_alloc_rx_data for full details.
6722  */
6723 static void tg3_recycle_rx(struct tg3_napi *tnapi,
6724 			   struct tg3_rx_prodring_set *dpr,
6725 			   u32 opaque_key, int src_idx,
6726 			   u32 dest_idx_unmasked)
6727 {
6728 	struct tg3 *tp = tnapi->tp;
6729 	struct tg3_rx_buffer_desc *src_desc, *dest_desc;
6730 	struct ring_info *src_map, *dest_map;
6731 	struct tg3_rx_prodring_set *spr = &tp->napi[0].prodring;
6732 	int dest_idx;
6733 
6734 	switch (opaque_key) {
6735 	case RXD_OPAQUE_RING_STD:
6736 		dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
6737 		dest_desc = &dpr->rx_std[dest_idx];
6738 		dest_map = &dpr->rx_std_buffers[dest_idx];
6739 		src_desc = &spr->rx_std[src_idx];
6740 		src_map = &spr->rx_std_buffers[src_idx];
6741 		break;
6742 
6743 	case RXD_OPAQUE_RING_JUMBO:
6744 		dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
6745 		dest_desc = &dpr->rx_jmb[dest_idx].std;
6746 		dest_map = &dpr->rx_jmb_buffers[dest_idx];
6747 		src_desc = &spr->rx_jmb[src_idx].std;
6748 		src_map = &spr->rx_jmb_buffers[src_idx];
6749 		break;
6750 
6751 	default:
6752 		return;
6753 	}
6754 
6755 	dest_map->data = src_map->data;
6756 	dma_unmap_addr_set(dest_map, mapping,
6757 			   dma_unmap_addr(src_map, mapping));
6758 	dest_desc->addr_hi = src_desc->addr_hi;
6759 	dest_desc->addr_lo = src_desc->addr_lo;
6760 
6761 	/* Ensure that the update to the skb happens after the physical
6762 	 * addresses have been transferred to the new BD location.
6763 	 */
6764 	smp_wmb();
6765 
6766 	src_map->data = NULL;
6767 }
6768 
6769 /* The RX ring scheme is composed of multiple rings which post fresh
6770  * buffers to the chip, and one special ring the chip uses to report
6771  * status back to the host.
6772  *
6773  * The special ring reports the status of received packets to the
6774  * host.  The chip does not write into the original descriptor the
6775  * RX buffer was obtained from.  The chip simply takes the original
6776  * descriptor as provided by the host, updates the status and length
6777  * field, then writes this into the next status ring entry.
6778  *
6779  * Each ring the host uses to post buffers to the chip is described
6780  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
6781  * it is first placed into the on-chip ram.  When the packet's length
6782  * is known, it walks down the TG3_BDINFO entries to select the ring.
6783  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
6784  * which is within the range of the new packet's length is chosen.
6785  *
6786  * The "separate ring for rx status" scheme may sound queer, but it makes
6787  * sense from a cache coherency perspective.  If only the host writes
6788  * to the buffer post rings, and only the chip writes to the rx status
6789  * rings, then cache lines never move beyond shared-modified state.
6790  * If both the host and chip were to write into the same ring, cache line
6791  * eviction could occur since both entities want it in an exclusive state.
6792  */
6793 static int tg3_rx(struct tg3_napi *tnapi, int budget)
6794 {
6795 	struct tg3 *tp = tnapi->tp;
6796 	u32 work_mask, rx_std_posted = 0;
6797 	u32 std_prod_idx, jmb_prod_idx;
6798 	u32 sw_idx = tnapi->rx_rcb_ptr;
6799 	u16 hw_idx;
6800 	int received;
6801 	struct tg3_rx_prodring_set *tpr = &tnapi->prodring;
6802 
6803 	hw_idx = *(tnapi->rx_rcb_prod_idx);
6804 	/*
6805 	 * We need to order the read of hw_idx and the read of
6806 	 * the opaque cookie.
6807 	 */
6808 	rmb();
6809 	work_mask = 0;
6810 	received = 0;
6811 	std_prod_idx = tpr->rx_std_prod_idx;
6812 	jmb_prod_idx = tpr->rx_jmb_prod_idx;
6813 	while (sw_idx != hw_idx && budget > 0) {
6814 		struct ring_info *ri;
6815 		struct tg3_rx_buffer_desc *desc = &tnapi->rx_rcb[sw_idx];
6816 		unsigned int len;
6817 		struct sk_buff *skb;
6818 		dma_addr_t dma_addr;
6819 		u32 opaque_key, desc_idx, *post_ptr;
6820 		u8 *data;
6821 		u64 tstamp = 0;
6822 
6823 		desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
6824 		opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
6825 		if (opaque_key == RXD_OPAQUE_RING_STD) {
6826 			ri = &tp->napi[0].prodring.rx_std_buffers[desc_idx];
6827 			dma_addr = dma_unmap_addr(ri, mapping);
6828 			data = ri->data;
6829 			post_ptr = &std_prod_idx;
6830 			rx_std_posted++;
6831 		} else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
6832 			ri = &tp->napi[0].prodring.rx_jmb_buffers[desc_idx];
6833 			dma_addr = dma_unmap_addr(ri, mapping);
6834 			data = ri->data;
6835 			post_ptr = &jmb_prod_idx;
6836 		} else
6837 			goto next_pkt_nopost;
6838 
6839 		work_mask |= opaque_key;
6840 
6841 		if (desc->err_vlan & RXD_ERR_MASK) {
6842 		drop_it:
6843 			tg3_recycle_rx(tnapi, tpr, opaque_key,
6844 				       desc_idx, *post_ptr);
6845 		drop_it_no_recycle:
6846 			/* Other statistics kept track of by card. */
6847 			tp->rx_dropped++;
6848 			goto next_pkt;
6849 		}
6850 
6851 		prefetch(data + TG3_RX_OFFSET(tp));
6852 		len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) -
6853 		      ETH_FCS_LEN;
6854 
6855 		if ((desc->type_flags & RXD_FLAG_PTPSTAT_MASK) ==
6856 		     RXD_FLAG_PTPSTAT_PTPV1 ||
6857 		    (desc->type_flags & RXD_FLAG_PTPSTAT_MASK) ==
6858 		     RXD_FLAG_PTPSTAT_PTPV2) {
6859 			tstamp = tr32(TG3_RX_TSTAMP_LSB);
6860 			tstamp |= (u64)tr32(TG3_RX_TSTAMP_MSB) << 32;
6861 		}
6862 
6863 		if (len > TG3_RX_COPY_THRESH(tp)) {
6864 			int skb_size;
6865 			unsigned int frag_size;
6866 
6867 			skb_size = tg3_alloc_rx_data(tp, tpr, opaque_key,
6868 						    *post_ptr, &frag_size);
6869 			if (skb_size < 0)
6870 				goto drop_it;
6871 
6872 			pci_unmap_single(tp->pdev, dma_addr, skb_size,
6873 					 PCI_DMA_FROMDEVICE);
6874 
6875 			/* Ensure that the update to the data happens
6876 			 * after the usage of the old DMA mapping.
6877 			 */
6878 			smp_wmb();
6879 
6880 			ri->data = NULL;
6881 
6882 			skb = build_skb(data, frag_size);
6883 			if (!skb) {
6884 				tg3_frag_free(frag_size != 0, data);
6885 				goto drop_it_no_recycle;
6886 			}
6887 			skb_reserve(skb, TG3_RX_OFFSET(tp));
6888 		} else {
6889 			tg3_recycle_rx(tnapi, tpr, opaque_key,
6890 				       desc_idx, *post_ptr);
6891 
6892 			skb = netdev_alloc_skb(tp->dev,
6893 					       len + TG3_RAW_IP_ALIGN);
6894 			if (skb == NULL)
6895 				goto drop_it_no_recycle;
6896 
6897 			skb_reserve(skb, TG3_RAW_IP_ALIGN);
6898 			pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
6899 			memcpy(skb->data,
6900 			       data + TG3_RX_OFFSET(tp),
6901 			       len);
6902 			pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
6903 		}
6904 
6905 		skb_put(skb, len);
6906 		if (tstamp)
6907 			tg3_hwclock_to_timestamp(tp, tstamp,
6908 						 skb_hwtstamps(skb));
6909 
6910 		if ((tp->dev->features & NETIF_F_RXCSUM) &&
6911 		    (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
6912 		    (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
6913 		      >> RXD_TCPCSUM_SHIFT) == 0xffff))
6914 			skb->ip_summed = CHECKSUM_UNNECESSARY;
6915 		else
6916 			skb_checksum_none_assert(skb);
6917 
6918 		skb->protocol = eth_type_trans(skb, tp->dev);
6919 
6920 		if (len > (tp->dev->mtu + ETH_HLEN) &&
6921 		    skb->protocol != htons(ETH_P_8021Q)) {
6922 			dev_kfree_skb_any(skb);
6923 			goto drop_it_no_recycle;
6924 		}
6925 
6926 		if (desc->type_flags & RXD_FLAG_VLAN &&
6927 		    !(tp->rx_mode & RX_MODE_KEEP_VLAN_TAG))
6928 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
6929 					       desc->err_vlan & RXD_VLAN_MASK);
6930 
6931 		napi_gro_receive(&tnapi->napi, skb);
6932 
6933 		received++;
6934 		budget--;
6935 
6936 next_pkt:
6937 		(*post_ptr)++;
6938 
6939 		if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
6940 			tpr->rx_std_prod_idx = std_prod_idx &
6941 					       tp->rx_std_ring_mask;
6942 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
6943 				     tpr->rx_std_prod_idx);
6944 			work_mask &= ~RXD_OPAQUE_RING_STD;
6945 			rx_std_posted = 0;
6946 		}
6947 next_pkt_nopost:
6948 		sw_idx++;
6949 		sw_idx &= tp->rx_ret_ring_mask;
6950 
6951 		/* Refresh hw_idx to see if there is new work */
6952 		if (sw_idx == hw_idx) {
6953 			hw_idx = *(tnapi->rx_rcb_prod_idx);
6954 			rmb();
6955 		}
6956 	}
6957 
6958 	/* ACK the status ring. */
6959 	tnapi->rx_rcb_ptr = sw_idx;
6960 	tw32_rx_mbox(tnapi->consmbox, sw_idx);
6961 
6962 	/* Refill RX ring(s). */
6963 	if (!tg3_flag(tp, ENABLE_RSS)) {
6964 		/* Sync BD data before updating mailbox */
6965 		wmb();
6966 
6967 		if (work_mask & RXD_OPAQUE_RING_STD) {
6968 			tpr->rx_std_prod_idx = std_prod_idx &
6969 					       tp->rx_std_ring_mask;
6970 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
6971 				     tpr->rx_std_prod_idx);
6972 		}
6973 		if (work_mask & RXD_OPAQUE_RING_JUMBO) {
6974 			tpr->rx_jmb_prod_idx = jmb_prod_idx &
6975 					       tp->rx_jmb_ring_mask;
6976 			tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
6977 				     tpr->rx_jmb_prod_idx);
6978 		}
6979 		mmiowb();
6980 	} else if (work_mask) {
6981 		/* rx_std_buffers[] and rx_jmb_buffers[] entries must be
6982 		 * updated before the producer indices can be updated.
6983 		 */
6984 		smp_wmb();
6985 
6986 		tpr->rx_std_prod_idx = std_prod_idx & tp->rx_std_ring_mask;
6987 		tpr->rx_jmb_prod_idx = jmb_prod_idx & tp->rx_jmb_ring_mask;
6988 
6989 		if (tnapi != &tp->napi[1]) {
6990 			tp->rx_refill = true;
6991 			napi_schedule(&tp->napi[1].napi);
6992 		}
6993 	}
6994 
6995 	return received;
6996 }
6997 
6998 static void tg3_poll_link(struct tg3 *tp)
6999 {
7000 	/* handle link change and other phy events */
7001 	if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
7002 		struct tg3_hw_status *sblk = tp->napi[0].hw_status;
7003 
7004 		if (sblk->status & SD_STATUS_LINK_CHG) {
7005 			sblk->status = SD_STATUS_UPDATED |
7006 				       (sblk->status & ~SD_STATUS_LINK_CHG);
7007 			spin_lock(&tp->lock);
7008 			if (tg3_flag(tp, USE_PHYLIB)) {
7009 				tw32_f(MAC_STATUS,
7010 				     (MAC_STATUS_SYNC_CHANGED |
7011 				      MAC_STATUS_CFG_CHANGED |
7012 				      MAC_STATUS_MI_COMPLETION |
7013 				      MAC_STATUS_LNKSTATE_CHANGED));
7014 				udelay(40);
7015 			} else
7016 				tg3_setup_phy(tp, false);
7017 			spin_unlock(&tp->lock);
7018 		}
7019 	}
7020 }
7021 
7022 static int tg3_rx_prodring_xfer(struct tg3 *tp,
7023 				struct tg3_rx_prodring_set *dpr,
7024 				struct tg3_rx_prodring_set *spr)
7025 {
7026 	u32 si, di, cpycnt, src_prod_idx;
7027 	int i, err = 0;
7028 
7029 	while (1) {
7030 		src_prod_idx = spr->rx_std_prod_idx;
7031 
7032 		/* Make sure updates to the rx_std_buffers[] entries and the
7033 		 * standard producer index are seen in the correct order.
7034 		 */
7035 		smp_rmb();
7036 
7037 		if (spr->rx_std_cons_idx == src_prod_idx)
7038 			break;
7039 
7040 		if (spr->rx_std_cons_idx < src_prod_idx)
7041 			cpycnt = src_prod_idx - spr->rx_std_cons_idx;
7042 		else
7043 			cpycnt = tp->rx_std_ring_mask + 1 -
7044 				 spr->rx_std_cons_idx;
7045 
7046 		cpycnt = min(cpycnt,
7047 			     tp->rx_std_ring_mask + 1 - dpr->rx_std_prod_idx);
7048 
7049 		si = spr->rx_std_cons_idx;
7050 		di = dpr->rx_std_prod_idx;
7051 
7052 		for (i = di; i < di + cpycnt; i++) {
7053 			if (dpr->rx_std_buffers[i].data) {
7054 				cpycnt = i - di;
7055 				err = -ENOSPC;
7056 				break;
7057 			}
7058 		}
7059 
7060 		if (!cpycnt)
7061 			break;
7062 
7063 		/* Ensure that updates to the rx_std_buffers ring and the
7064 		 * shadowed hardware producer ring from tg3_recycle_skb() are
7065 		 * ordered correctly WRT the skb check above.
7066 		 */
7067 		smp_rmb();
7068 
7069 		memcpy(&dpr->rx_std_buffers[di],
7070 		       &spr->rx_std_buffers[si],
7071 		       cpycnt * sizeof(struct ring_info));
7072 
7073 		for (i = 0; i < cpycnt; i++, di++, si++) {
7074 			struct tg3_rx_buffer_desc *sbd, *dbd;
7075 			sbd = &spr->rx_std[si];
7076 			dbd = &dpr->rx_std[di];
7077 			dbd->addr_hi = sbd->addr_hi;
7078 			dbd->addr_lo = sbd->addr_lo;
7079 		}
7080 
7081 		spr->rx_std_cons_idx = (spr->rx_std_cons_idx + cpycnt) &
7082 				       tp->rx_std_ring_mask;
7083 		dpr->rx_std_prod_idx = (dpr->rx_std_prod_idx + cpycnt) &
7084 				       tp->rx_std_ring_mask;
7085 	}
7086 
7087 	while (1) {
7088 		src_prod_idx = spr->rx_jmb_prod_idx;
7089 
7090 		/* Make sure updates to the rx_jmb_buffers[] entries and
7091 		 * the jumbo producer index are seen in the correct order.
7092 		 */
7093 		smp_rmb();
7094 
7095 		if (spr->rx_jmb_cons_idx == src_prod_idx)
7096 			break;
7097 
7098 		if (spr->rx_jmb_cons_idx < src_prod_idx)
7099 			cpycnt = src_prod_idx - spr->rx_jmb_cons_idx;
7100 		else
7101 			cpycnt = tp->rx_jmb_ring_mask + 1 -
7102 				 spr->rx_jmb_cons_idx;
7103 
7104 		cpycnt = min(cpycnt,
7105 			     tp->rx_jmb_ring_mask + 1 - dpr->rx_jmb_prod_idx);
7106 
7107 		si = spr->rx_jmb_cons_idx;
7108 		di = dpr->rx_jmb_prod_idx;
7109 
7110 		for (i = di; i < di + cpycnt; i++) {
7111 			if (dpr->rx_jmb_buffers[i].data) {
7112 				cpycnt = i - di;
7113 				err = -ENOSPC;
7114 				break;
7115 			}
7116 		}
7117 
7118 		if (!cpycnt)
7119 			break;
7120 
7121 		/* Ensure that updates to the rx_jmb_buffers ring and the
7122 		 * shadowed hardware producer ring from tg3_recycle_skb() are
7123 		 * ordered correctly WRT the skb check above.
7124 		 */
7125 		smp_rmb();
7126 
7127 		memcpy(&dpr->rx_jmb_buffers[di],
7128 		       &spr->rx_jmb_buffers[si],
7129 		       cpycnt * sizeof(struct ring_info));
7130 
7131 		for (i = 0; i < cpycnt; i++, di++, si++) {
7132 			struct tg3_rx_buffer_desc *sbd, *dbd;
7133 			sbd = &spr->rx_jmb[si].std;
7134 			dbd = &dpr->rx_jmb[di].std;
7135 			dbd->addr_hi = sbd->addr_hi;
7136 			dbd->addr_lo = sbd->addr_lo;
7137 		}
7138 
7139 		spr->rx_jmb_cons_idx = (spr->rx_jmb_cons_idx + cpycnt) &
7140 				       tp->rx_jmb_ring_mask;
7141 		dpr->rx_jmb_prod_idx = (dpr->rx_jmb_prod_idx + cpycnt) &
7142 				       tp->rx_jmb_ring_mask;
7143 	}
7144 
7145 	return err;
7146 }
7147 
7148 static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget)
7149 {
7150 	struct tg3 *tp = tnapi->tp;
7151 
7152 	/* run TX completion thread */
7153 	if (tnapi->hw_status->idx[0].tx_consumer != tnapi->tx_cons) {
7154 		tg3_tx(tnapi);
7155 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7156 			return work_done;
7157 	}
7158 
7159 	if (!tnapi->rx_rcb_prod_idx)
7160 		return work_done;
7161 
7162 	/* run RX thread, within the bounds set by NAPI.
7163 	 * All RX "locking" is done by ensuring outside
7164 	 * code synchronizes with tg3->napi.poll()
7165 	 */
7166 	if (*(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
7167 		work_done += tg3_rx(tnapi, budget - work_done);
7168 
7169 	if (tg3_flag(tp, ENABLE_RSS) && tnapi == &tp->napi[1]) {
7170 		struct tg3_rx_prodring_set *dpr = &tp->napi[0].prodring;
7171 		int i, err = 0;
7172 		u32 std_prod_idx = dpr->rx_std_prod_idx;
7173 		u32 jmb_prod_idx = dpr->rx_jmb_prod_idx;
7174 
7175 		tp->rx_refill = false;
7176 		for (i = 1; i <= tp->rxq_cnt; i++)
7177 			err |= tg3_rx_prodring_xfer(tp, dpr,
7178 						    &tp->napi[i].prodring);
7179 
7180 		wmb();
7181 
7182 		if (std_prod_idx != dpr->rx_std_prod_idx)
7183 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
7184 				     dpr->rx_std_prod_idx);
7185 
7186 		if (jmb_prod_idx != dpr->rx_jmb_prod_idx)
7187 			tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
7188 				     dpr->rx_jmb_prod_idx);
7189 
7190 		mmiowb();
7191 
7192 		if (err)
7193 			tw32_f(HOSTCC_MODE, tp->coal_now);
7194 	}
7195 
7196 	return work_done;
7197 }
7198 
7199 static inline void tg3_reset_task_schedule(struct tg3 *tp)
7200 {
7201 	if (!test_and_set_bit(TG3_FLAG_RESET_TASK_PENDING, tp->tg3_flags))
7202 		schedule_work(&tp->reset_task);
7203 }
7204 
7205 static inline void tg3_reset_task_cancel(struct tg3 *tp)
7206 {
7207 	cancel_work_sync(&tp->reset_task);
7208 	tg3_flag_clear(tp, RESET_TASK_PENDING);
7209 	tg3_flag_clear(tp, TX_RECOVERY_PENDING);
7210 }
7211 
7212 static int tg3_poll_msix(struct napi_struct *napi, int budget)
7213 {
7214 	struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
7215 	struct tg3 *tp = tnapi->tp;
7216 	int work_done = 0;
7217 	struct tg3_hw_status *sblk = tnapi->hw_status;
7218 
7219 	while (1) {
7220 		work_done = tg3_poll_work(tnapi, work_done, budget);
7221 
7222 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7223 			goto tx_recovery;
7224 
7225 		if (unlikely(work_done >= budget))
7226 			break;
7227 
7228 		/* tp->last_tag is used in tg3_int_reenable() below
7229 		 * to tell the hw how much work has been processed,
7230 		 * so we must read it before checking for more work.
7231 		 */
7232 		tnapi->last_tag = sblk->status_tag;
7233 		tnapi->last_irq_tag = tnapi->last_tag;
7234 		rmb();
7235 
7236 		/* check for RX/TX work to do */
7237 		if (likely(sblk->idx[0].tx_consumer == tnapi->tx_cons &&
7238 			   *(tnapi->rx_rcb_prod_idx) == tnapi->rx_rcb_ptr)) {
7239 
7240 			/* This test here is not race free, but will reduce
7241 			 * the number of interrupts by looping again.
7242 			 */
7243 			if (tnapi == &tp->napi[1] && tp->rx_refill)
7244 				continue;
7245 
7246 			napi_complete(napi);
7247 			/* Reenable interrupts. */
7248 			tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
7249 
7250 			/* This test here is synchronized by napi_schedule()
7251 			 * and napi_complete() to close the race condition.
7252 			 */
7253 			if (unlikely(tnapi == &tp->napi[1] && tp->rx_refill)) {
7254 				tw32(HOSTCC_MODE, tp->coalesce_mode |
7255 						  HOSTCC_MODE_ENABLE |
7256 						  tnapi->coal_now);
7257 			}
7258 			mmiowb();
7259 			break;
7260 		}
7261 	}
7262 
7263 	return work_done;
7264 
7265 tx_recovery:
7266 	/* work_done is guaranteed to be less than budget. */
7267 	napi_complete(napi);
7268 	tg3_reset_task_schedule(tp);
7269 	return work_done;
7270 }
7271 
7272 static void tg3_process_error(struct tg3 *tp)
7273 {
7274 	u32 val;
7275 	bool real_error = false;
7276 
7277 	if (tg3_flag(tp, ERROR_PROCESSED))
7278 		return;
7279 
7280 	/* Check Flow Attention register */
7281 	val = tr32(HOSTCC_FLOW_ATTN);
7282 	if (val & ~HOSTCC_FLOW_ATTN_MBUF_LWM) {
7283 		netdev_err(tp->dev, "FLOW Attention error.  Resetting chip.\n");
7284 		real_error = true;
7285 	}
7286 
7287 	if (tr32(MSGINT_STATUS) & ~MSGINT_STATUS_MSI_REQ) {
7288 		netdev_err(tp->dev, "MSI Status error.  Resetting chip.\n");
7289 		real_error = true;
7290 	}
7291 
7292 	if (tr32(RDMAC_STATUS) || tr32(WDMAC_STATUS)) {
7293 		netdev_err(tp->dev, "DMA Status error.  Resetting chip.\n");
7294 		real_error = true;
7295 	}
7296 
7297 	if (!real_error)
7298 		return;
7299 
7300 	tg3_dump_state(tp);
7301 
7302 	tg3_flag_set(tp, ERROR_PROCESSED);
7303 	tg3_reset_task_schedule(tp);
7304 }
7305 
7306 static int tg3_poll(struct napi_struct *napi, int budget)
7307 {
7308 	struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
7309 	struct tg3 *tp = tnapi->tp;
7310 	int work_done = 0;
7311 	struct tg3_hw_status *sblk = tnapi->hw_status;
7312 
7313 	while (1) {
7314 		if (sblk->status & SD_STATUS_ERROR)
7315 			tg3_process_error(tp);
7316 
7317 		tg3_poll_link(tp);
7318 
7319 		work_done = tg3_poll_work(tnapi, work_done, budget);
7320 
7321 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7322 			goto tx_recovery;
7323 
7324 		if (unlikely(work_done >= budget))
7325 			break;
7326 
7327 		if (tg3_flag(tp, TAGGED_STATUS)) {
7328 			/* tp->last_tag is used in tg3_int_reenable() below
7329 			 * to tell the hw how much work has been processed,
7330 			 * so we must read it before checking for more work.
7331 			 */
7332 			tnapi->last_tag = sblk->status_tag;
7333 			tnapi->last_irq_tag = tnapi->last_tag;
7334 			rmb();
7335 		} else
7336 			sblk->status &= ~SD_STATUS_UPDATED;
7337 
7338 		if (likely(!tg3_has_work(tnapi))) {
7339 			napi_complete(napi);
7340 			tg3_int_reenable(tnapi);
7341 			break;
7342 		}
7343 	}
7344 
7345 	return work_done;
7346 
7347 tx_recovery:
7348 	/* work_done is guaranteed to be less than budget. */
7349 	napi_complete(napi);
7350 	tg3_reset_task_schedule(tp);
7351 	return work_done;
7352 }
7353 
7354 static void tg3_napi_disable(struct tg3 *tp)
7355 {
7356 	int i;
7357 
7358 	for (i = tp->irq_cnt - 1; i >= 0; i--)
7359 		napi_disable(&tp->napi[i].napi);
7360 }
7361 
7362 static void tg3_napi_enable(struct tg3 *tp)
7363 {
7364 	int i;
7365 
7366 	for (i = 0; i < tp->irq_cnt; i++)
7367 		napi_enable(&tp->napi[i].napi);
7368 }
7369 
7370 static void tg3_napi_init(struct tg3 *tp)
7371 {
7372 	int i;
7373 
7374 	netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll, 64);
7375 	for (i = 1; i < tp->irq_cnt; i++)
7376 		netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix, 64);
7377 }
7378 
7379 static void tg3_napi_fini(struct tg3 *tp)
7380 {
7381 	int i;
7382 
7383 	for (i = 0; i < tp->irq_cnt; i++)
7384 		netif_napi_del(&tp->napi[i].napi);
7385 }
7386 
7387 static inline void tg3_netif_stop(struct tg3 *tp)
7388 {
7389 	tp->dev->trans_start = jiffies;	/* prevent tx timeout */
7390 	tg3_napi_disable(tp);
7391 	netif_carrier_off(tp->dev);
7392 	netif_tx_disable(tp->dev);
7393 }
7394 
7395 /* tp->lock must be held */
7396 static inline void tg3_netif_start(struct tg3 *tp)
7397 {
7398 	tg3_ptp_resume(tp);
7399 
7400 	/* NOTE: unconditional netif_tx_wake_all_queues is only
7401 	 * appropriate so long as all callers are assured to
7402 	 * have free tx slots (such as after tg3_init_hw)
7403 	 */
7404 	netif_tx_wake_all_queues(tp->dev);
7405 
7406 	if (tp->link_up)
7407 		netif_carrier_on(tp->dev);
7408 
7409 	tg3_napi_enable(tp);
7410 	tp->napi[0].hw_status->status |= SD_STATUS_UPDATED;
7411 	tg3_enable_ints(tp);
7412 }
7413 
7414 static void tg3_irq_quiesce(struct tg3 *tp)
7415 {
7416 	int i;
7417 
7418 	BUG_ON(tp->irq_sync);
7419 
7420 	tp->irq_sync = 1;
7421 	smp_mb();
7422 
7423 	for (i = 0; i < tp->irq_cnt; i++)
7424 		synchronize_irq(tp->napi[i].irq_vec);
7425 }
7426 
7427 /* Fully shutdown all tg3 driver activity elsewhere in the system.
7428  * If irq_sync is non-zero, then the IRQ handler must be synchronized
7429  * with as well.  Most of the time, this is not necessary except when
7430  * shutting down the device.
7431  */
7432 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
7433 {
7434 	spin_lock_bh(&tp->lock);
7435 	if (irq_sync)
7436 		tg3_irq_quiesce(tp);
7437 }
7438 
7439 static inline void tg3_full_unlock(struct tg3 *tp)
7440 {
7441 	spin_unlock_bh(&tp->lock);
7442 }
7443 
7444 /* One-shot MSI handler - Chip automatically disables interrupt
7445  * after sending MSI so driver doesn't have to do it.
7446  */
7447 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
7448 {
7449 	struct tg3_napi *tnapi = dev_id;
7450 	struct tg3 *tp = tnapi->tp;
7451 
7452 	prefetch(tnapi->hw_status);
7453 	if (tnapi->rx_rcb)
7454 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7455 
7456 	if (likely(!tg3_irq_sync(tp)))
7457 		napi_schedule(&tnapi->napi);
7458 
7459 	return IRQ_HANDLED;
7460 }
7461 
7462 /* MSI ISR - No need to check for interrupt sharing and no need to
7463  * flush status block and interrupt mailbox. PCI ordering rules
7464  * guarantee that MSI will arrive after the status block.
7465  */
7466 static irqreturn_t tg3_msi(int irq, void *dev_id)
7467 {
7468 	struct tg3_napi *tnapi = dev_id;
7469 	struct tg3 *tp = tnapi->tp;
7470 
7471 	prefetch(tnapi->hw_status);
7472 	if (tnapi->rx_rcb)
7473 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7474 	/*
7475 	 * Writing any value to intr-mbox-0 clears PCI INTA# and
7476 	 * chip-internal interrupt pending events.
7477 	 * Writing non-zero to intr-mbox-0 additional tells the
7478 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7479 	 * event coalescing.
7480 	 */
7481 	tw32_mailbox(tnapi->int_mbox, 0x00000001);
7482 	if (likely(!tg3_irq_sync(tp)))
7483 		napi_schedule(&tnapi->napi);
7484 
7485 	return IRQ_RETVAL(1);
7486 }
7487 
7488 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
7489 {
7490 	struct tg3_napi *tnapi = dev_id;
7491 	struct tg3 *tp = tnapi->tp;
7492 	struct tg3_hw_status *sblk = tnapi->hw_status;
7493 	unsigned int handled = 1;
7494 
7495 	/* In INTx mode, it is possible for the interrupt to arrive at
7496 	 * the CPU before the status block posted prior to the interrupt.
7497 	 * Reading the PCI State register will confirm whether the
7498 	 * interrupt is ours and will flush the status block.
7499 	 */
7500 	if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
7501 		if (tg3_flag(tp, CHIP_RESETTING) ||
7502 		    (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7503 			handled = 0;
7504 			goto out;
7505 		}
7506 	}
7507 
7508 	/*
7509 	 * Writing any value to intr-mbox-0 clears PCI INTA# and
7510 	 * chip-internal interrupt pending events.
7511 	 * Writing non-zero to intr-mbox-0 additional tells the
7512 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7513 	 * event coalescing.
7514 	 *
7515 	 * Flush the mailbox to de-assert the IRQ immediately to prevent
7516 	 * spurious interrupts.  The flush impacts performance but
7517 	 * excessive spurious interrupts can be worse in some cases.
7518 	 */
7519 	tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
7520 	if (tg3_irq_sync(tp))
7521 		goto out;
7522 	sblk->status &= ~SD_STATUS_UPDATED;
7523 	if (likely(tg3_has_work(tnapi))) {
7524 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7525 		napi_schedule(&tnapi->napi);
7526 	} else {
7527 		/* No work, shared interrupt perhaps?  re-enable
7528 		 * interrupts, and flush that PCI write
7529 		 */
7530 		tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
7531 			       0x00000000);
7532 	}
7533 out:
7534 	return IRQ_RETVAL(handled);
7535 }
7536 
7537 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
7538 {
7539 	struct tg3_napi *tnapi = dev_id;
7540 	struct tg3 *tp = tnapi->tp;
7541 	struct tg3_hw_status *sblk = tnapi->hw_status;
7542 	unsigned int handled = 1;
7543 
7544 	/* In INTx mode, it is possible for the interrupt to arrive at
7545 	 * the CPU before the status block posted prior to the interrupt.
7546 	 * Reading the PCI State register will confirm whether the
7547 	 * interrupt is ours and will flush the status block.
7548 	 */
7549 	if (unlikely(sblk->status_tag == tnapi->last_irq_tag)) {
7550 		if (tg3_flag(tp, CHIP_RESETTING) ||
7551 		    (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7552 			handled = 0;
7553 			goto out;
7554 		}
7555 	}
7556 
7557 	/*
7558 	 * writing any value to intr-mbox-0 clears PCI INTA# and
7559 	 * chip-internal interrupt pending events.
7560 	 * writing non-zero to intr-mbox-0 additional tells the
7561 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7562 	 * event coalescing.
7563 	 *
7564 	 * Flush the mailbox to de-assert the IRQ immediately to prevent
7565 	 * spurious interrupts.  The flush impacts performance but
7566 	 * excessive spurious interrupts can be worse in some cases.
7567 	 */
7568 	tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
7569 
7570 	/*
7571 	 * In a shared interrupt configuration, sometimes other devices'
7572 	 * interrupts will scream.  We record the current status tag here
7573 	 * so that the above check can report that the screaming interrupts
7574 	 * are unhandled.  Eventually they will be silenced.
7575 	 */
7576 	tnapi->last_irq_tag = sblk->status_tag;
7577 
7578 	if (tg3_irq_sync(tp))
7579 		goto out;
7580 
7581 	prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7582 
7583 	napi_schedule(&tnapi->napi);
7584 
7585 out:
7586 	return IRQ_RETVAL(handled);
7587 }
7588 
7589 /* ISR for interrupt test */
7590 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
7591 {
7592 	struct tg3_napi *tnapi = dev_id;
7593 	struct tg3 *tp = tnapi->tp;
7594 	struct tg3_hw_status *sblk = tnapi->hw_status;
7595 
7596 	if ((sblk->status & SD_STATUS_UPDATED) ||
7597 	    !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7598 		tg3_disable_ints(tp);
7599 		return IRQ_RETVAL(1);
7600 	}
7601 	return IRQ_RETVAL(0);
7602 }
7603 
7604 #ifdef CONFIG_NET_POLL_CONTROLLER
7605 static void tg3_poll_controller(struct net_device *dev)
7606 {
7607 	int i;
7608 	struct tg3 *tp = netdev_priv(dev);
7609 
7610 	if (tg3_irq_sync(tp))
7611 		return;
7612 
7613 	for (i = 0; i < tp->irq_cnt; i++)
7614 		tg3_interrupt(tp->napi[i].irq_vec, &tp->napi[i]);
7615 }
7616 #endif
7617 
7618 static void tg3_tx_timeout(struct net_device *dev)
7619 {
7620 	struct tg3 *tp = netdev_priv(dev);
7621 
7622 	if (netif_msg_tx_err(tp)) {
7623 		netdev_err(dev, "transmit timed out, resetting\n");
7624 		tg3_dump_state(tp);
7625 	}
7626 
7627 	tg3_reset_task_schedule(tp);
7628 }
7629 
7630 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
7631 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
7632 {
7633 	u32 base = (u32) mapping & 0xffffffff;
7634 
7635 	return base + len + 8 < base;
7636 }
7637 
7638 /* Test for TSO DMA buffers that cross into regions which are within MSS bytes
7639  * of any 4GB boundaries: 4G, 8G, etc
7640  */
7641 static inline int tg3_4g_tso_overflow_test(struct tg3 *tp, dma_addr_t mapping,
7642 					   u32 len, u32 mss)
7643 {
7644 	if (tg3_asic_rev(tp) == ASIC_REV_5762 && mss) {
7645 		u32 base = (u32) mapping & 0xffffffff;
7646 
7647 		return ((base + len + (mss & 0x3fff)) < base);
7648 	}
7649 	return 0;
7650 }
7651 
7652 /* Test for DMA addresses > 40-bit */
7653 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
7654 					  int len)
7655 {
7656 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
7657 	if (tg3_flag(tp, 40BIT_DMA_BUG))
7658 		return ((u64) mapping + len) > DMA_BIT_MASK(40);
7659 	return 0;
7660 #else
7661 	return 0;
7662 #endif
7663 }
7664 
7665 static inline void tg3_tx_set_bd(struct tg3_tx_buffer_desc *txbd,
7666 				 dma_addr_t mapping, u32 len, u32 flags,
7667 				 u32 mss, u32 vlan)
7668 {
7669 	txbd->addr_hi = ((u64) mapping >> 32);
7670 	txbd->addr_lo = ((u64) mapping & 0xffffffff);
7671 	txbd->len_flags = (len << TXD_LEN_SHIFT) | (flags & 0x0000ffff);
7672 	txbd->vlan_tag = (mss << TXD_MSS_SHIFT) | (vlan << TXD_VLAN_TAG_SHIFT);
7673 }
7674 
7675 static bool tg3_tx_frag_set(struct tg3_napi *tnapi, u32 *entry, u32 *budget,
7676 			    dma_addr_t map, u32 len, u32 flags,
7677 			    u32 mss, u32 vlan)
7678 {
7679 	struct tg3 *tp = tnapi->tp;
7680 	bool hwbug = false;
7681 
7682 	if (tg3_flag(tp, SHORT_DMA_BUG) && len <= 8)
7683 		hwbug = true;
7684 
7685 	if (tg3_4g_overflow_test(map, len))
7686 		hwbug = true;
7687 
7688 	if (tg3_4g_tso_overflow_test(tp, map, len, mss))
7689 		hwbug = true;
7690 
7691 	if (tg3_40bit_overflow_test(tp, map, len))
7692 		hwbug = true;
7693 
7694 	if (tp->dma_limit) {
7695 		u32 prvidx = *entry;
7696 		u32 tmp_flag = flags & ~TXD_FLAG_END;
7697 		while (len > tp->dma_limit && *budget) {
7698 			u32 frag_len = tp->dma_limit;
7699 			len -= tp->dma_limit;
7700 
7701 			/* Avoid the 8byte DMA problem */
7702 			if (len <= 8) {
7703 				len += tp->dma_limit / 2;
7704 				frag_len = tp->dma_limit / 2;
7705 			}
7706 
7707 			tnapi->tx_buffers[*entry].fragmented = true;
7708 
7709 			tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7710 				      frag_len, tmp_flag, mss, vlan);
7711 			*budget -= 1;
7712 			prvidx = *entry;
7713 			*entry = NEXT_TX(*entry);
7714 
7715 			map += frag_len;
7716 		}
7717 
7718 		if (len) {
7719 			if (*budget) {
7720 				tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7721 					      len, flags, mss, vlan);
7722 				*budget -= 1;
7723 				*entry = NEXT_TX(*entry);
7724 			} else {
7725 				hwbug = true;
7726 				tnapi->tx_buffers[prvidx].fragmented = false;
7727 			}
7728 		}
7729 	} else {
7730 		tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7731 			      len, flags, mss, vlan);
7732 		*entry = NEXT_TX(*entry);
7733 	}
7734 
7735 	return hwbug;
7736 }
7737 
7738 static void tg3_tx_skb_unmap(struct tg3_napi *tnapi, u32 entry, int last)
7739 {
7740 	int i;
7741 	struct sk_buff *skb;
7742 	struct tg3_tx_ring_info *txb = &tnapi->tx_buffers[entry];
7743 
7744 	skb = txb->skb;
7745 	txb->skb = NULL;
7746 
7747 	pci_unmap_single(tnapi->tp->pdev,
7748 			 dma_unmap_addr(txb, mapping),
7749 			 skb_headlen(skb),
7750 			 PCI_DMA_TODEVICE);
7751 
7752 	while (txb->fragmented) {
7753 		txb->fragmented = false;
7754 		entry = NEXT_TX(entry);
7755 		txb = &tnapi->tx_buffers[entry];
7756 	}
7757 
7758 	for (i = 0; i <= last; i++) {
7759 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
7760 
7761 		entry = NEXT_TX(entry);
7762 		txb = &tnapi->tx_buffers[entry];
7763 
7764 		pci_unmap_page(tnapi->tp->pdev,
7765 			       dma_unmap_addr(txb, mapping),
7766 			       skb_frag_size(frag), PCI_DMA_TODEVICE);
7767 
7768 		while (txb->fragmented) {
7769 			txb->fragmented = false;
7770 			entry = NEXT_TX(entry);
7771 			txb = &tnapi->tx_buffers[entry];
7772 		}
7773 	}
7774 }
7775 
7776 /* Workaround 4GB and 40-bit hardware DMA bugs. */
7777 static int tigon3_dma_hwbug_workaround(struct tg3_napi *tnapi,
7778 				       struct sk_buff **pskb,
7779 				       u32 *entry, u32 *budget,
7780 				       u32 base_flags, u32 mss, u32 vlan)
7781 {
7782 	struct tg3 *tp = tnapi->tp;
7783 	struct sk_buff *new_skb, *skb = *pskb;
7784 	dma_addr_t new_addr = 0;
7785 	int ret = 0;
7786 
7787 	if (tg3_asic_rev(tp) != ASIC_REV_5701)
7788 		new_skb = skb_copy(skb, GFP_ATOMIC);
7789 	else {
7790 		int more_headroom = 4 - ((unsigned long)skb->data & 3);
7791 
7792 		new_skb = skb_copy_expand(skb,
7793 					  skb_headroom(skb) + more_headroom,
7794 					  skb_tailroom(skb), GFP_ATOMIC);
7795 	}
7796 
7797 	if (!new_skb) {
7798 		ret = -1;
7799 	} else {
7800 		/* New SKB is guaranteed to be linear. */
7801 		new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
7802 					  PCI_DMA_TODEVICE);
7803 		/* Make sure the mapping succeeded */
7804 		if (pci_dma_mapping_error(tp->pdev, new_addr)) {
7805 			dev_kfree_skb_any(new_skb);
7806 			ret = -1;
7807 		} else {
7808 			u32 save_entry = *entry;
7809 
7810 			base_flags |= TXD_FLAG_END;
7811 
7812 			tnapi->tx_buffers[*entry].skb = new_skb;
7813 			dma_unmap_addr_set(&tnapi->tx_buffers[*entry],
7814 					   mapping, new_addr);
7815 
7816 			if (tg3_tx_frag_set(tnapi, entry, budget, new_addr,
7817 					    new_skb->len, base_flags,
7818 					    mss, vlan)) {
7819 				tg3_tx_skb_unmap(tnapi, save_entry, -1);
7820 				dev_kfree_skb_any(new_skb);
7821 				ret = -1;
7822 			}
7823 		}
7824 	}
7825 
7826 	dev_kfree_skb_any(skb);
7827 	*pskb = new_skb;
7828 	return ret;
7829 }
7830 
7831 static netdev_tx_t tg3_start_xmit(struct sk_buff *, struct net_device *);
7832 
7833 /* Use GSO to workaround a rare TSO bug that may be triggered when the
7834  * TSO header is greater than 80 bytes.
7835  */
7836 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
7837 {
7838 	struct sk_buff *segs, *nskb;
7839 	u32 frag_cnt_est = skb_shinfo(skb)->gso_segs * 3;
7840 
7841 	/* Estimate the number of fragments in the worst case */
7842 	if (unlikely(tg3_tx_avail(&tp->napi[0]) <= frag_cnt_est)) {
7843 		netif_stop_queue(tp->dev);
7844 
7845 		/* netif_tx_stop_queue() must be done before checking
7846 		 * checking tx index in tg3_tx_avail() below, because in
7847 		 * tg3_tx(), we update tx index before checking for
7848 		 * netif_tx_queue_stopped().
7849 		 */
7850 		smp_mb();
7851 		if (tg3_tx_avail(&tp->napi[0]) <= frag_cnt_est)
7852 			return NETDEV_TX_BUSY;
7853 
7854 		netif_wake_queue(tp->dev);
7855 	}
7856 
7857 	segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
7858 	if (IS_ERR(segs))
7859 		goto tg3_tso_bug_end;
7860 
7861 	do {
7862 		nskb = segs;
7863 		segs = segs->next;
7864 		nskb->next = NULL;
7865 		tg3_start_xmit(nskb, tp->dev);
7866 	} while (segs);
7867 
7868 tg3_tso_bug_end:
7869 	dev_kfree_skb_any(skb);
7870 
7871 	return NETDEV_TX_OK;
7872 }
7873 
7874 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
7875  * support TG3_FLAG_HW_TSO_1 or firmware TSO only.
7876  */
7877 static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
7878 {
7879 	struct tg3 *tp = netdev_priv(dev);
7880 	u32 len, entry, base_flags, mss, vlan = 0;
7881 	u32 budget;
7882 	int i = -1, would_hit_hwbug;
7883 	dma_addr_t mapping;
7884 	struct tg3_napi *tnapi;
7885 	struct netdev_queue *txq;
7886 	unsigned int last;
7887 
7888 	txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
7889 	tnapi = &tp->napi[skb_get_queue_mapping(skb)];
7890 	if (tg3_flag(tp, ENABLE_TSS))
7891 		tnapi++;
7892 
7893 	budget = tg3_tx_avail(tnapi);
7894 
7895 	/* We are running in BH disabled context with netif_tx_lock
7896 	 * and TX reclaim runs via tp->napi.poll inside of a software
7897 	 * interrupt.  Furthermore, IRQ processing runs lockless so we have
7898 	 * no IRQ context deadlocks to worry about either.  Rejoice!
7899 	 */
7900 	if (unlikely(budget <= (skb_shinfo(skb)->nr_frags + 1))) {
7901 		if (!netif_tx_queue_stopped(txq)) {
7902 			netif_tx_stop_queue(txq);
7903 
7904 			/* This is a hard error, log it. */
7905 			netdev_err(dev,
7906 				   "BUG! Tx Ring full when queue awake!\n");
7907 		}
7908 		return NETDEV_TX_BUSY;
7909 	}
7910 
7911 	entry = tnapi->tx_prod;
7912 	base_flags = 0;
7913 	if (skb->ip_summed == CHECKSUM_PARTIAL)
7914 		base_flags |= TXD_FLAG_TCPUDP_CSUM;
7915 
7916 	mss = skb_shinfo(skb)->gso_size;
7917 	if (mss) {
7918 		struct iphdr *iph;
7919 		u32 tcp_opt_len, hdr_len;
7920 
7921 		if (skb_cow_head(skb, 0))
7922 			goto drop;
7923 
7924 		iph = ip_hdr(skb);
7925 		tcp_opt_len = tcp_optlen(skb);
7926 
7927 		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb) - ETH_HLEN;
7928 
7929 		if (!skb_is_gso_v6(skb)) {
7930 			iph->check = 0;
7931 			iph->tot_len = htons(mss + hdr_len);
7932 		}
7933 
7934 		if (unlikely((ETH_HLEN + hdr_len) > 80) &&
7935 		    tg3_flag(tp, TSO_BUG))
7936 			return tg3_tso_bug(tp, skb);
7937 
7938 		base_flags |= (TXD_FLAG_CPU_PRE_DMA |
7939 			       TXD_FLAG_CPU_POST_DMA);
7940 
7941 		if (tg3_flag(tp, HW_TSO_1) ||
7942 		    tg3_flag(tp, HW_TSO_2) ||
7943 		    tg3_flag(tp, HW_TSO_3)) {
7944 			tcp_hdr(skb)->check = 0;
7945 			base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
7946 		} else
7947 			tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
7948 								 iph->daddr, 0,
7949 								 IPPROTO_TCP,
7950 								 0);
7951 
7952 		if (tg3_flag(tp, HW_TSO_3)) {
7953 			mss |= (hdr_len & 0xc) << 12;
7954 			if (hdr_len & 0x10)
7955 				base_flags |= 0x00000010;
7956 			base_flags |= (hdr_len & 0x3e0) << 5;
7957 		} else if (tg3_flag(tp, HW_TSO_2))
7958 			mss |= hdr_len << 9;
7959 		else if (tg3_flag(tp, HW_TSO_1) ||
7960 			 tg3_asic_rev(tp) == ASIC_REV_5705) {
7961 			if (tcp_opt_len || iph->ihl > 5) {
7962 				int tsflags;
7963 
7964 				tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
7965 				mss |= (tsflags << 11);
7966 			}
7967 		} else {
7968 			if (tcp_opt_len || iph->ihl > 5) {
7969 				int tsflags;
7970 
7971 				tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
7972 				base_flags |= tsflags << 12;
7973 			}
7974 		}
7975 	}
7976 
7977 	if (tg3_flag(tp, USE_JUMBO_BDFLAG) &&
7978 	    !mss && skb->len > VLAN_ETH_FRAME_LEN)
7979 		base_flags |= TXD_FLAG_JMB_PKT;
7980 
7981 	if (vlan_tx_tag_present(skb)) {
7982 		base_flags |= TXD_FLAG_VLAN;
7983 		vlan = vlan_tx_tag_get(skb);
7984 	}
7985 
7986 	if ((unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) &&
7987 	    tg3_flag(tp, TX_TSTAMP_EN)) {
7988 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
7989 		base_flags |= TXD_FLAG_HWTSTAMP;
7990 	}
7991 
7992 	len = skb_headlen(skb);
7993 
7994 	mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
7995 	if (pci_dma_mapping_error(tp->pdev, mapping))
7996 		goto drop;
7997 
7998 
7999 	tnapi->tx_buffers[entry].skb = skb;
8000 	dma_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, mapping);
8001 
8002 	would_hit_hwbug = 0;
8003 
8004 	if (tg3_flag(tp, 5701_DMA_BUG))
8005 		would_hit_hwbug = 1;
8006 
8007 	if (tg3_tx_frag_set(tnapi, &entry, &budget, mapping, len, base_flags |
8008 			  ((skb_shinfo(skb)->nr_frags == 0) ? TXD_FLAG_END : 0),
8009 			    mss, vlan)) {
8010 		would_hit_hwbug = 1;
8011 	} else if (skb_shinfo(skb)->nr_frags > 0) {
8012 		u32 tmp_mss = mss;
8013 
8014 		if (!tg3_flag(tp, HW_TSO_1) &&
8015 		    !tg3_flag(tp, HW_TSO_2) &&
8016 		    !tg3_flag(tp, HW_TSO_3))
8017 			tmp_mss = 0;
8018 
8019 		/* Now loop through additional data
8020 		 * fragments, and queue them.
8021 		 */
8022 		last = skb_shinfo(skb)->nr_frags - 1;
8023 		for (i = 0; i <= last; i++) {
8024 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
8025 
8026 			len = skb_frag_size(frag);
8027 			mapping = skb_frag_dma_map(&tp->pdev->dev, frag, 0,
8028 						   len, DMA_TO_DEVICE);
8029 
8030 			tnapi->tx_buffers[entry].skb = NULL;
8031 			dma_unmap_addr_set(&tnapi->tx_buffers[entry], mapping,
8032 					   mapping);
8033 			if (dma_mapping_error(&tp->pdev->dev, mapping))
8034 				goto dma_error;
8035 
8036 			if (!budget ||
8037 			    tg3_tx_frag_set(tnapi, &entry, &budget, mapping,
8038 					    len, base_flags |
8039 					    ((i == last) ? TXD_FLAG_END : 0),
8040 					    tmp_mss, vlan)) {
8041 				would_hit_hwbug = 1;
8042 				break;
8043 			}
8044 		}
8045 	}
8046 
8047 	if (would_hit_hwbug) {
8048 		tg3_tx_skb_unmap(tnapi, tnapi->tx_prod, i);
8049 
8050 		/* If the workaround fails due to memory/mapping
8051 		 * failure, silently drop this packet.
8052 		 */
8053 		entry = tnapi->tx_prod;
8054 		budget = tg3_tx_avail(tnapi);
8055 		if (tigon3_dma_hwbug_workaround(tnapi, &skb, &entry, &budget,
8056 						base_flags, mss, vlan))
8057 			goto drop_nofree;
8058 	}
8059 
8060 	skb_tx_timestamp(skb);
8061 	netdev_tx_sent_queue(txq, skb->len);
8062 
8063 	/* Sync BD data before updating mailbox */
8064 	wmb();
8065 
8066 	/* Packets are ready, update Tx producer idx local and on card. */
8067 	tw32_tx_mbox(tnapi->prodmbox, entry);
8068 
8069 	tnapi->tx_prod = entry;
8070 	if (unlikely(tg3_tx_avail(tnapi) <= (MAX_SKB_FRAGS + 1))) {
8071 		netif_tx_stop_queue(txq);
8072 
8073 		/* netif_tx_stop_queue() must be done before checking
8074 		 * checking tx index in tg3_tx_avail() below, because in
8075 		 * tg3_tx(), we update tx index before checking for
8076 		 * netif_tx_queue_stopped().
8077 		 */
8078 		smp_mb();
8079 		if (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi))
8080 			netif_tx_wake_queue(txq);
8081 	}
8082 
8083 	mmiowb();
8084 	return NETDEV_TX_OK;
8085 
8086 dma_error:
8087 	tg3_tx_skb_unmap(tnapi, tnapi->tx_prod, --i);
8088 	tnapi->tx_buffers[tnapi->tx_prod].skb = NULL;
8089 drop:
8090 	dev_kfree_skb_any(skb);
8091 drop_nofree:
8092 	tp->tx_dropped++;
8093 	return NETDEV_TX_OK;
8094 }
8095 
8096 static void tg3_mac_loopback(struct tg3 *tp, bool enable)
8097 {
8098 	if (enable) {
8099 		tp->mac_mode &= ~(MAC_MODE_HALF_DUPLEX |
8100 				  MAC_MODE_PORT_MODE_MASK);
8101 
8102 		tp->mac_mode |= MAC_MODE_PORT_INT_LPBACK;
8103 
8104 		if (!tg3_flag(tp, 5705_PLUS))
8105 			tp->mac_mode |= MAC_MODE_LINK_POLARITY;
8106 
8107 		if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
8108 			tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
8109 		else
8110 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
8111 	} else {
8112 		tp->mac_mode &= ~MAC_MODE_PORT_INT_LPBACK;
8113 
8114 		if (tg3_flag(tp, 5705_PLUS) ||
8115 		    (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) ||
8116 		    tg3_asic_rev(tp) == ASIC_REV_5700)
8117 			tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
8118 	}
8119 
8120 	tw32(MAC_MODE, tp->mac_mode);
8121 	udelay(40);
8122 }
8123 
8124 static int tg3_phy_lpbk_set(struct tg3 *tp, u32 speed, bool extlpbk)
8125 {
8126 	u32 val, bmcr, mac_mode, ptest = 0;
8127 
8128 	tg3_phy_toggle_apd(tp, false);
8129 	tg3_phy_toggle_automdix(tp, false);
8130 
8131 	if (extlpbk && tg3_phy_set_extloopbk(tp))
8132 		return -EIO;
8133 
8134 	bmcr = BMCR_FULLDPLX;
8135 	switch (speed) {
8136 	case SPEED_10:
8137 		break;
8138 	case SPEED_100:
8139 		bmcr |= BMCR_SPEED100;
8140 		break;
8141 	case SPEED_1000:
8142 	default:
8143 		if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
8144 			speed = SPEED_100;
8145 			bmcr |= BMCR_SPEED100;
8146 		} else {
8147 			speed = SPEED_1000;
8148 			bmcr |= BMCR_SPEED1000;
8149 		}
8150 	}
8151 
8152 	if (extlpbk) {
8153 		if (!(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
8154 			tg3_readphy(tp, MII_CTRL1000, &val);
8155 			val |= CTL1000_AS_MASTER |
8156 			       CTL1000_ENABLE_MASTER;
8157 			tg3_writephy(tp, MII_CTRL1000, val);
8158 		} else {
8159 			ptest = MII_TG3_FET_PTEST_TRIM_SEL |
8160 				MII_TG3_FET_PTEST_TRIM_2;
8161 			tg3_writephy(tp, MII_TG3_FET_PTEST, ptest);
8162 		}
8163 	} else
8164 		bmcr |= BMCR_LOOPBACK;
8165 
8166 	tg3_writephy(tp, MII_BMCR, bmcr);
8167 
8168 	/* The write needs to be flushed for the FETs */
8169 	if (tp->phy_flags & TG3_PHYFLG_IS_FET)
8170 		tg3_readphy(tp, MII_BMCR, &bmcr);
8171 
8172 	udelay(40);
8173 
8174 	if ((tp->phy_flags & TG3_PHYFLG_IS_FET) &&
8175 	    tg3_asic_rev(tp) == ASIC_REV_5785) {
8176 		tg3_writephy(tp, MII_TG3_FET_PTEST, ptest |
8177 			     MII_TG3_FET_PTEST_FRC_TX_LINK |
8178 			     MII_TG3_FET_PTEST_FRC_TX_LOCK);
8179 
8180 		/* The write needs to be flushed for the AC131 */
8181 		tg3_readphy(tp, MII_TG3_FET_PTEST, &val);
8182 	}
8183 
8184 	/* Reset to prevent losing 1st rx packet intermittently */
8185 	if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
8186 	    tg3_flag(tp, 5780_CLASS)) {
8187 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
8188 		udelay(10);
8189 		tw32_f(MAC_RX_MODE, tp->rx_mode);
8190 	}
8191 
8192 	mac_mode = tp->mac_mode &
8193 		   ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
8194 	if (speed == SPEED_1000)
8195 		mac_mode |= MAC_MODE_PORT_MODE_GMII;
8196 	else
8197 		mac_mode |= MAC_MODE_PORT_MODE_MII;
8198 
8199 	if (tg3_asic_rev(tp) == ASIC_REV_5700) {
8200 		u32 masked_phy_id = tp->phy_id & TG3_PHY_ID_MASK;
8201 
8202 		if (masked_phy_id == TG3_PHY_ID_BCM5401)
8203 			mac_mode &= ~MAC_MODE_LINK_POLARITY;
8204 		else if (masked_phy_id == TG3_PHY_ID_BCM5411)
8205 			mac_mode |= MAC_MODE_LINK_POLARITY;
8206 
8207 		tg3_writephy(tp, MII_TG3_EXT_CTRL,
8208 			     MII_TG3_EXT_CTRL_LNK3_LED_MODE);
8209 	}
8210 
8211 	tw32(MAC_MODE, mac_mode);
8212 	udelay(40);
8213 
8214 	return 0;
8215 }
8216 
8217 static void tg3_set_loopback(struct net_device *dev, netdev_features_t features)
8218 {
8219 	struct tg3 *tp = netdev_priv(dev);
8220 
8221 	if (features & NETIF_F_LOOPBACK) {
8222 		if (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK)
8223 			return;
8224 
8225 		spin_lock_bh(&tp->lock);
8226 		tg3_mac_loopback(tp, true);
8227 		netif_carrier_on(tp->dev);
8228 		spin_unlock_bh(&tp->lock);
8229 		netdev_info(dev, "Internal MAC loopback mode enabled.\n");
8230 	} else {
8231 		if (!(tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
8232 			return;
8233 
8234 		spin_lock_bh(&tp->lock);
8235 		tg3_mac_loopback(tp, false);
8236 		/* Force link status check */
8237 		tg3_setup_phy(tp, true);
8238 		spin_unlock_bh(&tp->lock);
8239 		netdev_info(dev, "Internal MAC loopback mode disabled.\n");
8240 	}
8241 }
8242 
8243 static netdev_features_t tg3_fix_features(struct net_device *dev,
8244 	netdev_features_t features)
8245 {
8246 	struct tg3 *tp = netdev_priv(dev);
8247 
8248 	if (dev->mtu > ETH_DATA_LEN && tg3_flag(tp, 5780_CLASS))
8249 		features &= ~NETIF_F_ALL_TSO;
8250 
8251 	return features;
8252 }
8253 
8254 static int tg3_set_features(struct net_device *dev, netdev_features_t features)
8255 {
8256 	netdev_features_t changed = dev->features ^ features;
8257 
8258 	if ((changed & NETIF_F_LOOPBACK) && netif_running(dev))
8259 		tg3_set_loopback(dev, features);
8260 
8261 	return 0;
8262 }
8263 
8264 static void tg3_rx_prodring_free(struct tg3 *tp,
8265 				 struct tg3_rx_prodring_set *tpr)
8266 {
8267 	int i;
8268 
8269 	if (tpr != &tp->napi[0].prodring) {
8270 		for (i = tpr->rx_std_cons_idx; i != tpr->rx_std_prod_idx;
8271 		     i = (i + 1) & tp->rx_std_ring_mask)
8272 			tg3_rx_data_free(tp, &tpr->rx_std_buffers[i],
8273 					tp->rx_pkt_map_sz);
8274 
8275 		if (tg3_flag(tp, JUMBO_CAPABLE)) {
8276 			for (i = tpr->rx_jmb_cons_idx;
8277 			     i != tpr->rx_jmb_prod_idx;
8278 			     i = (i + 1) & tp->rx_jmb_ring_mask) {
8279 				tg3_rx_data_free(tp, &tpr->rx_jmb_buffers[i],
8280 						TG3_RX_JMB_MAP_SZ);
8281 			}
8282 		}
8283 
8284 		return;
8285 	}
8286 
8287 	for (i = 0; i <= tp->rx_std_ring_mask; i++)
8288 		tg3_rx_data_free(tp, &tpr->rx_std_buffers[i],
8289 				tp->rx_pkt_map_sz);
8290 
8291 	if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS)) {
8292 		for (i = 0; i <= tp->rx_jmb_ring_mask; i++)
8293 			tg3_rx_data_free(tp, &tpr->rx_jmb_buffers[i],
8294 					TG3_RX_JMB_MAP_SZ);
8295 	}
8296 }
8297 
8298 /* Initialize rx rings for packet processing.
8299  *
8300  * The chip has been shut down and the driver detached from
8301  * the networking, so no interrupts or new tx packets will
8302  * end up in the driver.  tp->{tx,}lock are held and thus
8303  * we may not sleep.
8304  */
8305 static int tg3_rx_prodring_alloc(struct tg3 *tp,
8306 				 struct tg3_rx_prodring_set *tpr)
8307 {
8308 	u32 i, rx_pkt_dma_sz;
8309 
8310 	tpr->rx_std_cons_idx = 0;
8311 	tpr->rx_std_prod_idx = 0;
8312 	tpr->rx_jmb_cons_idx = 0;
8313 	tpr->rx_jmb_prod_idx = 0;
8314 
8315 	if (tpr != &tp->napi[0].prodring) {
8316 		memset(&tpr->rx_std_buffers[0], 0,
8317 		       TG3_RX_STD_BUFF_RING_SIZE(tp));
8318 		if (tpr->rx_jmb_buffers)
8319 			memset(&tpr->rx_jmb_buffers[0], 0,
8320 			       TG3_RX_JMB_BUFF_RING_SIZE(tp));
8321 		goto done;
8322 	}
8323 
8324 	/* Zero out all descriptors. */
8325 	memset(tpr->rx_std, 0, TG3_RX_STD_RING_BYTES(tp));
8326 
8327 	rx_pkt_dma_sz = TG3_RX_STD_DMA_SZ;
8328 	if (tg3_flag(tp, 5780_CLASS) &&
8329 	    tp->dev->mtu > ETH_DATA_LEN)
8330 		rx_pkt_dma_sz = TG3_RX_JMB_DMA_SZ;
8331 	tp->rx_pkt_map_sz = TG3_RX_DMA_TO_MAP_SZ(rx_pkt_dma_sz);
8332 
8333 	/* Initialize invariants of the rings, we only set this
8334 	 * stuff once.  This works because the card does not
8335 	 * write into the rx buffer posting rings.
8336 	 */
8337 	for (i = 0; i <= tp->rx_std_ring_mask; i++) {
8338 		struct tg3_rx_buffer_desc *rxd;
8339 
8340 		rxd = &tpr->rx_std[i];
8341 		rxd->idx_len = rx_pkt_dma_sz << RXD_LEN_SHIFT;
8342 		rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
8343 		rxd->opaque = (RXD_OPAQUE_RING_STD |
8344 			       (i << RXD_OPAQUE_INDEX_SHIFT));
8345 	}
8346 
8347 	/* Now allocate fresh SKBs for each rx ring. */
8348 	for (i = 0; i < tp->rx_pending; i++) {
8349 		unsigned int frag_size;
8350 
8351 		if (tg3_alloc_rx_data(tp, tpr, RXD_OPAQUE_RING_STD, i,
8352 				      &frag_size) < 0) {
8353 			netdev_warn(tp->dev,
8354 				    "Using a smaller RX standard ring. Only "
8355 				    "%d out of %d buffers were allocated "
8356 				    "successfully\n", i, tp->rx_pending);
8357 			if (i == 0)
8358 				goto initfail;
8359 			tp->rx_pending = i;
8360 			break;
8361 		}
8362 	}
8363 
8364 	if (!tg3_flag(tp, JUMBO_CAPABLE) || tg3_flag(tp, 5780_CLASS))
8365 		goto done;
8366 
8367 	memset(tpr->rx_jmb, 0, TG3_RX_JMB_RING_BYTES(tp));
8368 
8369 	if (!tg3_flag(tp, JUMBO_RING_ENABLE))
8370 		goto done;
8371 
8372 	for (i = 0; i <= tp->rx_jmb_ring_mask; i++) {
8373 		struct tg3_rx_buffer_desc *rxd;
8374 
8375 		rxd = &tpr->rx_jmb[i].std;
8376 		rxd->idx_len = TG3_RX_JMB_DMA_SZ << RXD_LEN_SHIFT;
8377 		rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
8378 				  RXD_FLAG_JUMBO;
8379 		rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
8380 		       (i << RXD_OPAQUE_INDEX_SHIFT));
8381 	}
8382 
8383 	for (i = 0; i < tp->rx_jumbo_pending; i++) {
8384 		unsigned int frag_size;
8385 
8386 		if (tg3_alloc_rx_data(tp, tpr, RXD_OPAQUE_RING_JUMBO, i,
8387 				      &frag_size) < 0) {
8388 			netdev_warn(tp->dev,
8389 				    "Using a smaller RX jumbo ring. Only %d "
8390 				    "out of %d buffers were allocated "
8391 				    "successfully\n", i, tp->rx_jumbo_pending);
8392 			if (i == 0)
8393 				goto initfail;
8394 			tp->rx_jumbo_pending = i;
8395 			break;
8396 		}
8397 	}
8398 
8399 done:
8400 	return 0;
8401 
8402 initfail:
8403 	tg3_rx_prodring_free(tp, tpr);
8404 	return -ENOMEM;
8405 }
8406 
8407 static void tg3_rx_prodring_fini(struct tg3 *tp,
8408 				 struct tg3_rx_prodring_set *tpr)
8409 {
8410 	kfree(tpr->rx_std_buffers);
8411 	tpr->rx_std_buffers = NULL;
8412 	kfree(tpr->rx_jmb_buffers);
8413 	tpr->rx_jmb_buffers = NULL;
8414 	if (tpr->rx_std) {
8415 		dma_free_coherent(&tp->pdev->dev, TG3_RX_STD_RING_BYTES(tp),
8416 				  tpr->rx_std, tpr->rx_std_mapping);
8417 		tpr->rx_std = NULL;
8418 	}
8419 	if (tpr->rx_jmb) {
8420 		dma_free_coherent(&tp->pdev->dev, TG3_RX_JMB_RING_BYTES(tp),
8421 				  tpr->rx_jmb, tpr->rx_jmb_mapping);
8422 		tpr->rx_jmb = NULL;
8423 	}
8424 }
8425 
8426 static int tg3_rx_prodring_init(struct tg3 *tp,
8427 				struct tg3_rx_prodring_set *tpr)
8428 {
8429 	tpr->rx_std_buffers = kzalloc(TG3_RX_STD_BUFF_RING_SIZE(tp),
8430 				      GFP_KERNEL);
8431 	if (!tpr->rx_std_buffers)
8432 		return -ENOMEM;
8433 
8434 	tpr->rx_std = dma_alloc_coherent(&tp->pdev->dev,
8435 					 TG3_RX_STD_RING_BYTES(tp),
8436 					 &tpr->rx_std_mapping,
8437 					 GFP_KERNEL);
8438 	if (!tpr->rx_std)
8439 		goto err_out;
8440 
8441 	if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS)) {
8442 		tpr->rx_jmb_buffers = kzalloc(TG3_RX_JMB_BUFF_RING_SIZE(tp),
8443 					      GFP_KERNEL);
8444 		if (!tpr->rx_jmb_buffers)
8445 			goto err_out;
8446 
8447 		tpr->rx_jmb = dma_alloc_coherent(&tp->pdev->dev,
8448 						 TG3_RX_JMB_RING_BYTES(tp),
8449 						 &tpr->rx_jmb_mapping,
8450 						 GFP_KERNEL);
8451 		if (!tpr->rx_jmb)
8452 			goto err_out;
8453 	}
8454 
8455 	return 0;
8456 
8457 err_out:
8458 	tg3_rx_prodring_fini(tp, tpr);
8459 	return -ENOMEM;
8460 }
8461 
8462 /* Free up pending packets in all rx/tx rings.
8463  *
8464  * The chip has been shut down and the driver detached from
8465  * the networking, so no interrupts or new tx packets will
8466  * end up in the driver.  tp->{tx,}lock is not held and we are not
8467  * in an interrupt context and thus may sleep.
8468  */
8469 static void tg3_free_rings(struct tg3 *tp)
8470 {
8471 	int i, j;
8472 
8473 	for (j = 0; j < tp->irq_cnt; j++) {
8474 		struct tg3_napi *tnapi = &tp->napi[j];
8475 
8476 		tg3_rx_prodring_free(tp, &tnapi->prodring);
8477 
8478 		if (!tnapi->tx_buffers)
8479 			continue;
8480 
8481 		for (i = 0; i < TG3_TX_RING_SIZE; i++) {
8482 			struct sk_buff *skb = tnapi->tx_buffers[i].skb;
8483 
8484 			if (!skb)
8485 				continue;
8486 
8487 			tg3_tx_skb_unmap(tnapi, i,
8488 					 skb_shinfo(skb)->nr_frags - 1);
8489 
8490 			dev_kfree_skb_any(skb);
8491 		}
8492 		netdev_tx_reset_queue(netdev_get_tx_queue(tp->dev, j));
8493 	}
8494 }
8495 
8496 /* Initialize tx/rx rings for packet processing.
8497  *
8498  * The chip has been shut down and the driver detached from
8499  * the networking, so no interrupts or new tx packets will
8500  * end up in the driver.  tp->{tx,}lock are held and thus
8501  * we may not sleep.
8502  */
8503 static int tg3_init_rings(struct tg3 *tp)
8504 {
8505 	int i;
8506 
8507 	/* Free up all the SKBs. */
8508 	tg3_free_rings(tp);
8509 
8510 	for (i = 0; i < tp->irq_cnt; i++) {
8511 		struct tg3_napi *tnapi = &tp->napi[i];
8512 
8513 		tnapi->last_tag = 0;
8514 		tnapi->last_irq_tag = 0;
8515 		tnapi->hw_status->status = 0;
8516 		tnapi->hw_status->status_tag = 0;
8517 		memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
8518 
8519 		tnapi->tx_prod = 0;
8520 		tnapi->tx_cons = 0;
8521 		if (tnapi->tx_ring)
8522 			memset(tnapi->tx_ring, 0, TG3_TX_RING_BYTES);
8523 
8524 		tnapi->rx_rcb_ptr = 0;
8525 		if (tnapi->rx_rcb)
8526 			memset(tnapi->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
8527 
8528 		if (tg3_rx_prodring_alloc(tp, &tnapi->prodring)) {
8529 			tg3_free_rings(tp);
8530 			return -ENOMEM;
8531 		}
8532 	}
8533 
8534 	return 0;
8535 }
8536 
8537 static void tg3_mem_tx_release(struct tg3 *tp)
8538 {
8539 	int i;
8540 
8541 	for (i = 0; i < tp->irq_max; i++) {
8542 		struct tg3_napi *tnapi = &tp->napi[i];
8543 
8544 		if (tnapi->tx_ring) {
8545 			dma_free_coherent(&tp->pdev->dev, TG3_TX_RING_BYTES,
8546 				tnapi->tx_ring, tnapi->tx_desc_mapping);
8547 			tnapi->tx_ring = NULL;
8548 		}
8549 
8550 		kfree(tnapi->tx_buffers);
8551 		tnapi->tx_buffers = NULL;
8552 	}
8553 }
8554 
8555 static int tg3_mem_tx_acquire(struct tg3 *tp)
8556 {
8557 	int i;
8558 	struct tg3_napi *tnapi = &tp->napi[0];
8559 
8560 	/* If multivector TSS is enabled, vector 0 does not handle
8561 	 * tx interrupts.  Don't allocate any resources for it.
8562 	 */
8563 	if (tg3_flag(tp, ENABLE_TSS))
8564 		tnapi++;
8565 
8566 	for (i = 0; i < tp->txq_cnt; i++, tnapi++) {
8567 		tnapi->tx_buffers = kzalloc(sizeof(struct tg3_tx_ring_info) *
8568 					    TG3_TX_RING_SIZE, GFP_KERNEL);
8569 		if (!tnapi->tx_buffers)
8570 			goto err_out;
8571 
8572 		tnapi->tx_ring = dma_alloc_coherent(&tp->pdev->dev,
8573 						    TG3_TX_RING_BYTES,
8574 						    &tnapi->tx_desc_mapping,
8575 						    GFP_KERNEL);
8576 		if (!tnapi->tx_ring)
8577 			goto err_out;
8578 	}
8579 
8580 	return 0;
8581 
8582 err_out:
8583 	tg3_mem_tx_release(tp);
8584 	return -ENOMEM;
8585 }
8586 
8587 static void tg3_mem_rx_release(struct tg3 *tp)
8588 {
8589 	int i;
8590 
8591 	for (i = 0; i < tp->irq_max; i++) {
8592 		struct tg3_napi *tnapi = &tp->napi[i];
8593 
8594 		tg3_rx_prodring_fini(tp, &tnapi->prodring);
8595 
8596 		if (!tnapi->rx_rcb)
8597 			continue;
8598 
8599 		dma_free_coherent(&tp->pdev->dev,
8600 				  TG3_RX_RCB_RING_BYTES(tp),
8601 				  tnapi->rx_rcb,
8602 				  tnapi->rx_rcb_mapping);
8603 		tnapi->rx_rcb = NULL;
8604 	}
8605 }
8606 
8607 static int tg3_mem_rx_acquire(struct tg3 *tp)
8608 {
8609 	unsigned int i, limit;
8610 
8611 	limit = tp->rxq_cnt;
8612 
8613 	/* If RSS is enabled, we need a (dummy) producer ring
8614 	 * set on vector zero.  This is the true hw prodring.
8615 	 */
8616 	if (tg3_flag(tp, ENABLE_RSS))
8617 		limit++;
8618 
8619 	for (i = 0; i < limit; i++) {
8620 		struct tg3_napi *tnapi = &tp->napi[i];
8621 
8622 		if (tg3_rx_prodring_init(tp, &tnapi->prodring))
8623 			goto err_out;
8624 
8625 		/* If multivector RSS is enabled, vector 0
8626 		 * does not handle rx or tx interrupts.
8627 		 * Don't allocate any resources for it.
8628 		 */
8629 		if (!i && tg3_flag(tp, ENABLE_RSS))
8630 			continue;
8631 
8632 		tnapi->rx_rcb = dma_zalloc_coherent(&tp->pdev->dev,
8633 						    TG3_RX_RCB_RING_BYTES(tp),
8634 						    &tnapi->rx_rcb_mapping,
8635 						    GFP_KERNEL);
8636 		if (!tnapi->rx_rcb)
8637 			goto err_out;
8638 	}
8639 
8640 	return 0;
8641 
8642 err_out:
8643 	tg3_mem_rx_release(tp);
8644 	return -ENOMEM;
8645 }
8646 
8647 /*
8648  * Must not be invoked with interrupt sources disabled and
8649  * the hardware shutdown down.
8650  */
8651 static void tg3_free_consistent(struct tg3 *tp)
8652 {
8653 	int i;
8654 
8655 	for (i = 0; i < tp->irq_cnt; i++) {
8656 		struct tg3_napi *tnapi = &tp->napi[i];
8657 
8658 		if (tnapi->hw_status) {
8659 			dma_free_coherent(&tp->pdev->dev, TG3_HW_STATUS_SIZE,
8660 					  tnapi->hw_status,
8661 					  tnapi->status_mapping);
8662 			tnapi->hw_status = NULL;
8663 		}
8664 	}
8665 
8666 	tg3_mem_rx_release(tp);
8667 	tg3_mem_tx_release(tp);
8668 
8669 	if (tp->hw_stats) {
8670 		dma_free_coherent(&tp->pdev->dev, sizeof(struct tg3_hw_stats),
8671 				  tp->hw_stats, tp->stats_mapping);
8672 		tp->hw_stats = NULL;
8673 	}
8674 }
8675 
8676 /*
8677  * Must not be invoked with interrupt sources disabled and
8678  * the hardware shutdown down.  Can sleep.
8679  */
8680 static int tg3_alloc_consistent(struct tg3 *tp)
8681 {
8682 	int i;
8683 
8684 	tp->hw_stats = dma_zalloc_coherent(&tp->pdev->dev,
8685 					   sizeof(struct tg3_hw_stats),
8686 					   &tp->stats_mapping, GFP_KERNEL);
8687 	if (!tp->hw_stats)
8688 		goto err_out;
8689 
8690 	for (i = 0; i < tp->irq_cnt; i++) {
8691 		struct tg3_napi *tnapi = &tp->napi[i];
8692 		struct tg3_hw_status *sblk;
8693 
8694 		tnapi->hw_status = dma_zalloc_coherent(&tp->pdev->dev,
8695 						       TG3_HW_STATUS_SIZE,
8696 						       &tnapi->status_mapping,
8697 						       GFP_KERNEL);
8698 		if (!tnapi->hw_status)
8699 			goto err_out;
8700 
8701 		sblk = tnapi->hw_status;
8702 
8703 		if (tg3_flag(tp, ENABLE_RSS)) {
8704 			u16 *prodptr = NULL;
8705 
8706 			/*
8707 			 * When RSS is enabled, the status block format changes
8708 			 * slightly.  The "rx_jumbo_consumer", "reserved",
8709 			 * and "rx_mini_consumer" members get mapped to the
8710 			 * other three rx return ring producer indexes.
8711 			 */
8712 			switch (i) {
8713 			case 1:
8714 				prodptr = &sblk->idx[0].rx_producer;
8715 				break;
8716 			case 2:
8717 				prodptr = &sblk->rx_jumbo_consumer;
8718 				break;
8719 			case 3:
8720 				prodptr = &sblk->reserved;
8721 				break;
8722 			case 4:
8723 				prodptr = &sblk->rx_mini_consumer;
8724 				break;
8725 			}
8726 			tnapi->rx_rcb_prod_idx = prodptr;
8727 		} else {
8728 			tnapi->rx_rcb_prod_idx = &sblk->idx[0].rx_producer;
8729 		}
8730 	}
8731 
8732 	if (tg3_mem_tx_acquire(tp) || tg3_mem_rx_acquire(tp))
8733 		goto err_out;
8734 
8735 	return 0;
8736 
8737 err_out:
8738 	tg3_free_consistent(tp);
8739 	return -ENOMEM;
8740 }
8741 
8742 #define MAX_WAIT_CNT 1000
8743 
8744 /* To stop a block, clear the enable bit and poll till it
8745  * clears.  tp->lock is held.
8746  */
8747 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, bool silent)
8748 {
8749 	unsigned int i;
8750 	u32 val;
8751 
8752 	if (tg3_flag(tp, 5705_PLUS)) {
8753 		switch (ofs) {
8754 		case RCVLSC_MODE:
8755 		case DMAC_MODE:
8756 		case MBFREE_MODE:
8757 		case BUFMGR_MODE:
8758 		case MEMARB_MODE:
8759 			/* We can't enable/disable these bits of the
8760 			 * 5705/5750, just say success.
8761 			 */
8762 			return 0;
8763 
8764 		default:
8765 			break;
8766 		}
8767 	}
8768 
8769 	val = tr32(ofs);
8770 	val &= ~enable_bit;
8771 	tw32_f(ofs, val);
8772 
8773 	for (i = 0; i < MAX_WAIT_CNT; i++) {
8774 		if (pci_channel_offline(tp->pdev)) {
8775 			dev_err(&tp->pdev->dev,
8776 				"tg3_stop_block device offline, "
8777 				"ofs=%lx enable_bit=%x\n",
8778 				ofs, enable_bit);
8779 			return -ENODEV;
8780 		}
8781 
8782 		udelay(100);
8783 		val = tr32(ofs);
8784 		if ((val & enable_bit) == 0)
8785 			break;
8786 	}
8787 
8788 	if (i == MAX_WAIT_CNT && !silent) {
8789 		dev_err(&tp->pdev->dev,
8790 			"tg3_stop_block timed out, ofs=%lx enable_bit=%x\n",
8791 			ofs, enable_bit);
8792 		return -ENODEV;
8793 	}
8794 
8795 	return 0;
8796 }
8797 
8798 /* tp->lock is held. */
8799 static int tg3_abort_hw(struct tg3 *tp, bool silent)
8800 {
8801 	int i, err;
8802 
8803 	tg3_disable_ints(tp);
8804 
8805 	if (pci_channel_offline(tp->pdev)) {
8806 		tp->rx_mode &= ~(RX_MODE_ENABLE | TX_MODE_ENABLE);
8807 		tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
8808 		err = -ENODEV;
8809 		goto err_no_dev;
8810 	}
8811 
8812 	tp->rx_mode &= ~RX_MODE_ENABLE;
8813 	tw32_f(MAC_RX_MODE, tp->rx_mode);
8814 	udelay(10);
8815 
8816 	err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
8817 	err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
8818 	err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
8819 	err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
8820 	err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
8821 	err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
8822 
8823 	err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
8824 	err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
8825 	err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
8826 	err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
8827 	err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
8828 	err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
8829 	err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
8830 
8831 	tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
8832 	tw32_f(MAC_MODE, tp->mac_mode);
8833 	udelay(40);
8834 
8835 	tp->tx_mode &= ~TX_MODE_ENABLE;
8836 	tw32_f(MAC_TX_MODE, tp->tx_mode);
8837 
8838 	for (i = 0; i < MAX_WAIT_CNT; i++) {
8839 		udelay(100);
8840 		if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
8841 			break;
8842 	}
8843 	if (i >= MAX_WAIT_CNT) {
8844 		dev_err(&tp->pdev->dev,
8845 			"%s timed out, TX_MODE_ENABLE will not clear "
8846 			"MAC_TX_MODE=%08x\n", __func__, tr32(MAC_TX_MODE));
8847 		err |= -ENODEV;
8848 	}
8849 
8850 	err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
8851 	err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
8852 	err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
8853 
8854 	tw32(FTQ_RESET, 0xffffffff);
8855 	tw32(FTQ_RESET, 0x00000000);
8856 
8857 	err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
8858 	err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
8859 
8860 err_no_dev:
8861 	for (i = 0; i < tp->irq_cnt; i++) {
8862 		struct tg3_napi *tnapi = &tp->napi[i];
8863 		if (tnapi->hw_status)
8864 			memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
8865 	}
8866 
8867 	return err;
8868 }
8869 
8870 /* Save PCI command register before chip reset */
8871 static void tg3_save_pci_state(struct tg3 *tp)
8872 {
8873 	pci_read_config_word(tp->pdev, PCI_COMMAND, &tp->pci_cmd);
8874 }
8875 
8876 /* Restore PCI state after chip reset */
8877 static void tg3_restore_pci_state(struct tg3 *tp)
8878 {
8879 	u32 val;
8880 
8881 	/* Re-enable indirect register accesses. */
8882 	pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
8883 			       tp->misc_host_ctrl);
8884 
8885 	/* Set MAX PCI retry to zero. */
8886 	val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
8887 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0 &&
8888 	    tg3_flag(tp, PCIX_MODE))
8889 		val |= PCISTATE_RETRY_SAME_DMA;
8890 	/* Allow reads and writes to the APE register and memory space. */
8891 	if (tg3_flag(tp, ENABLE_APE))
8892 		val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
8893 		       PCISTATE_ALLOW_APE_SHMEM_WR |
8894 		       PCISTATE_ALLOW_APE_PSPACE_WR;
8895 	pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
8896 
8897 	pci_write_config_word(tp->pdev, PCI_COMMAND, tp->pci_cmd);
8898 
8899 	if (!tg3_flag(tp, PCI_EXPRESS)) {
8900 		pci_write_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
8901 				      tp->pci_cacheline_sz);
8902 		pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
8903 				      tp->pci_lat_timer);
8904 	}
8905 
8906 	/* Make sure PCI-X relaxed ordering bit is clear. */
8907 	if (tg3_flag(tp, PCIX_MODE)) {
8908 		u16 pcix_cmd;
8909 
8910 		pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
8911 				     &pcix_cmd);
8912 		pcix_cmd &= ~PCI_X_CMD_ERO;
8913 		pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
8914 				      pcix_cmd);
8915 	}
8916 
8917 	if (tg3_flag(tp, 5780_CLASS)) {
8918 
8919 		/* Chip reset on 5780 will reset MSI enable bit,
8920 		 * so need to restore it.
8921 		 */
8922 		if (tg3_flag(tp, USING_MSI)) {
8923 			u16 ctrl;
8924 
8925 			pci_read_config_word(tp->pdev,
8926 					     tp->msi_cap + PCI_MSI_FLAGS,
8927 					     &ctrl);
8928 			pci_write_config_word(tp->pdev,
8929 					      tp->msi_cap + PCI_MSI_FLAGS,
8930 					      ctrl | PCI_MSI_FLAGS_ENABLE);
8931 			val = tr32(MSGINT_MODE);
8932 			tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
8933 		}
8934 	}
8935 }
8936 
8937 static void tg3_override_clk(struct tg3 *tp)
8938 {
8939 	u32 val;
8940 
8941 	switch (tg3_asic_rev(tp)) {
8942 	case ASIC_REV_5717:
8943 		val = tr32(TG3_CPMU_CLCK_ORIDE_ENABLE);
8944 		tw32(TG3_CPMU_CLCK_ORIDE_ENABLE, val |
8945 		     TG3_CPMU_MAC_ORIDE_ENABLE);
8946 		break;
8947 
8948 	case ASIC_REV_5719:
8949 	case ASIC_REV_5720:
8950 		tw32(TG3_CPMU_CLCK_ORIDE, CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
8951 		break;
8952 
8953 	default:
8954 		return;
8955 	}
8956 }
8957 
8958 static void tg3_restore_clk(struct tg3 *tp)
8959 {
8960 	u32 val;
8961 
8962 	switch (tg3_asic_rev(tp)) {
8963 	case ASIC_REV_5717:
8964 		val = tr32(TG3_CPMU_CLCK_ORIDE_ENABLE);
8965 		tw32(TG3_CPMU_CLCK_ORIDE_ENABLE,
8966 		     val & ~TG3_CPMU_MAC_ORIDE_ENABLE);
8967 		break;
8968 
8969 	case ASIC_REV_5719:
8970 	case ASIC_REV_5720:
8971 		val = tr32(TG3_CPMU_CLCK_ORIDE);
8972 		tw32(TG3_CPMU_CLCK_ORIDE, val & ~CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
8973 		break;
8974 
8975 	default:
8976 		return;
8977 	}
8978 }
8979 
8980 /* tp->lock is held. */
8981 static int tg3_chip_reset(struct tg3 *tp)
8982 {
8983 	u32 val;
8984 	void (*write_op)(struct tg3 *, u32, u32);
8985 	int i, err;
8986 
8987 	if (!pci_device_is_present(tp->pdev))
8988 		return -ENODEV;
8989 
8990 	tg3_nvram_lock(tp);
8991 
8992 	tg3_ape_lock(tp, TG3_APE_LOCK_GRC);
8993 
8994 	/* No matching tg3_nvram_unlock() after this because
8995 	 * chip reset below will undo the nvram lock.
8996 	 */
8997 	tp->nvram_lock_cnt = 0;
8998 
8999 	/* GRC_MISC_CFG core clock reset will clear the memory
9000 	 * enable bit in PCI register 4 and the MSI enable bit
9001 	 * on some chips, so we save relevant registers here.
9002 	 */
9003 	tg3_save_pci_state(tp);
9004 
9005 	if (tg3_asic_rev(tp) == ASIC_REV_5752 ||
9006 	    tg3_flag(tp, 5755_PLUS))
9007 		tw32(GRC_FASTBOOT_PC, 0);
9008 
9009 	/*
9010 	 * We must avoid the readl() that normally takes place.
9011 	 * It locks machines, causes machine checks, and other
9012 	 * fun things.  So, temporarily disable the 5701
9013 	 * hardware workaround, while we do the reset.
9014 	 */
9015 	write_op = tp->write32;
9016 	if (write_op == tg3_write_flush_reg32)
9017 		tp->write32 = tg3_write32;
9018 
9019 	/* Prevent the irq handler from reading or writing PCI registers
9020 	 * during chip reset when the memory enable bit in the PCI command
9021 	 * register may be cleared.  The chip does not generate interrupt
9022 	 * at this time, but the irq handler may still be called due to irq
9023 	 * sharing or irqpoll.
9024 	 */
9025 	tg3_flag_set(tp, CHIP_RESETTING);
9026 	for (i = 0; i < tp->irq_cnt; i++) {
9027 		struct tg3_napi *tnapi = &tp->napi[i];
9028 		if (tnapi->hw_status) {
9029 			tnapi->hw_status->status = 0;
9030 			tnapi->hw_status->status_tag = 0;
9031 		}
9032 		tnapi->last_tag = 0;
9033 		tnapi->last_irq_tag = 0;
9034 	}
9035 	smp_mb();
9036 
9037 	for (i = 0; i < tp->irq_cnt; i++)
9038 		synchronize_irq(tp->napi[i].irq_vec);
9039 
9040 	if (tg3_asic_rev(tp) == ASIC_REV_57780) {
9041 		val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
9042 		tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
9043 	}
9044 
9045 	/* do the reset */
9046 	val = GRC_MISC_CFG_CORECLK_RESET;
9047 
9048 	if (tg3_flag(tp, PCI_EXPRESS)) {
9049 		/* Force PCIe 1.0a mode */
9050 		if (tg3_asic_rev(tp) != ASIC_REV_5785 &&
9051 		    !tg3_flag(tp, 57765_PLUS) &&
9052 		    tr32(TG3_PCIE_PHY_TSTCTL) ==
9053 		    (TG3_PCIE_PHY_TSTCTL_PCIE10 | TG3_PCIE_PHY_TSTCTL_PSCRAM))
9054 			tw32(TG3_PCIE_PHY_TSTCTL, TG3_PCIE_PHY_TSTCTL_PSCRAM);
9055 
9056 		if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0) {
9057 			tw32(GRC_MISC_CFG, (1 << 29));
9058 			val |= (1 << 29);
9059 		}
9060 	}
9061 
9062 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
9063 		tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
9064 		tw32(GRC_VCPU_EXT_CTRL,
9065 		     tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
9066 	}
9067 
9068 	/* Set the clock to the highest frequency to avoid timeouts. With link
9069 	 * aware mode, the clock speed could be slow and bootcode does not
9070 	 * complete within the expected time. Override the clock to allow the
9071 	 * bootcode to finish sooner and then restore it.
9072 	 */
9073 	tg3_override_clk(tp);
9074 
9075 	/* Manage gphy power for all CPMU absent PCIe devices. */
9076 	if (tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, CPMU_PRESENT))
9077 		val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
9078 
9079 	tw32(GRC_MISC_CFG, val);
9080 
9081 	/* restore 5701 hardware bug workaround write method */
9082 	tp->write32 = write_op;
9083 
9084 	/* Unfortunately, we have to delay before the PCI read back.
9085 	 * Some 575X chips even will not respond to a PCI cfg access
9086 	 * when the reset command is given to the chip.
9087 	 *
9088 	 * How do these hardware designers expect things to work
9089 	 * properly if the PCI write is posted for a long period
9090 	 * of time?  It is always necessary to have some method by
9091 	 * which a register read back can occur to push the write
9092 	 * out which does the reset.
9093 	 *
9094 	 * For most tg3 variants the trick below was working.
9095 	 * Ho hum...
9096 	 */
9097 	udelay(120);
9098 
9099 	/* Flush PCI posted writes.  The normal MMIO registers
9100 	 * are inaccessible at this time so this is the only
9101 	 * way to make this reliably (actually, this is no longer
9102 	 * the case, see above).  I tried to use indirect
9103 	 * register read/write but this upset some 5701 variants.
9104 	 */
9105 	pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
9106 
9107 	udelay(120);
9108 
9109 	if (tg3_flag(tp, PCI_EXPRESS) && pci_is_pcie(tp->pdev)) {
9110 		u16 val16;
9111 
9112 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A0) {
9113 			int j;
9114 			u32 cfg_val;
9115 
9116 			/* Wait for link training to complete.  */
9117 			for (j = 0; j < 5000; j++)
9118 				udelay(100);
9119 
9120 			pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
9121 			pci_write_config_dword(tp->pdev, 0xc4,
9122 					       cfg_val | (1 << 15));
9123 		}
9124 
9125 		/* Clear the "no snoop" and "relaxed ordering" bits. */
9126 		val16 = PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN;
9127 		/*
9128 		 * Older PCIe devices only support the 128 byte
9129 		 * MPS setting.  Enforce the restriction.
9130 		 */
9131 		if (!tg3_flag(tp, CPMU_PRESENT))
9132 			val16 |= PCI_EXP_DEVCTL_PAYLOAD;
9133 		pcie_capability_clear_word(tp->pdev, PCI_EXP_DEVCTL, val16);
9134 
9135 		/* Clear error status */
9136 		pcie_capability_write_word(tp->pdev, PCI_EXP_DEVSTA,
9137 				      PCI_EXP_DEVSTA_CED |
9138 				      PCI_EXP_DEVSTA_NFED |
9139 				      PCI_EXP_DEVSTA_FED |
9140 				      PCI_EXP_DEVSTA_URD);
9141 	}
9142 
9143 	tg3_restore_pci_state(tp);
9144 
9145 	tg3_flag_clear(tp, CHIP_RESETTING);
9146 	tg3_flag_clear(tp, ERROR_PROCESSED);
9147 
9148 	val = 0;
9149 	if (tg3_flag(tp, 5780_CLASS))
9150 		val = tr32(MEMARB_MODE);
9151 	tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
9152 
9153 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A3) {
9154 		tg3_stop_fw(tp);
9155 		tw32(0x5000, 0x400);
9156 	}
9157 
9158 	if (tg3_flag(tp, IS_SSB_CORE)) {
9159 		/*
9160 		 * BCM4785: In order to avoid repercussions from using
9161 		 * potentially defective internal ROM, stop the Rx RISC CPU,
9162 		 * which is not required.
9163 		 */
9164 		tg3_stop_fw(tp);
9165 		tg3_halt_cpu(tp, RX_CPU_BASE);
9166 	}
9167 
9168 	err = tg3_poll_fw(tp);
9169 	if (err)
9170 		return err;
9171 
9172 	tw32(GRC_MODE, tp->grc_mode);
9173 
9174 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A0) {
9175 		val = tr32(0xc4);
9176 
9177 		tw32(0xc4, val | (1 << 15));
9178 	}
9179 
9180 	if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
9181 	    tg3_asic_rev(tp) == ASIC_REV_5705) {
9182 		tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
9183 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A0)
9184 			tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
9185 		tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
9186 	}
9187 
9188 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
9189 		tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
9190 		val = tp->mac_mode;
9191 	} else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
9192 		tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
9193 		val = tp->mac_mode;
9194 	} else
9195 		val = 0;
9196 
9197 	tw32_f(MAC_MODE, val);
9198 	udelay(40);
9199 
9200 	tg3_ape_unlock(tp, TG3_APE_LOCK_GRC);
9201 
9202 	tg3_mdio_start(tp);
9203 
9204 	if (tg3_flag(tp, PCI_EXPRESS) &&
9205 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0 &&
9206 	    tg3_asic_rev(tp) != ASIC_REV_5785 &&
9207 	    !tg3_flag(tp, 57765_PLUS)) {
9208 		val = tr32(0x7c00);
9209 
9210 		tw32(0x7c00, val | (1 << 25));
9211 	}
9212 
9213 	tg3_restore_clk(tp);
9214 
9215 	/* Reprobe ASF enable state.  */
9216 	tg3_flag_clear(tp, ENABLE_ASF);
9217 	tp->phy_flags &= ~(TG3_PHYFLG_1G_ON_VAUX_OK |
9218 			   TG3_PHYFLG_KEEP_LINK_ON_PWRDN);
9219 
9220 	tg3_flag_clear(tp, ASF_NEW_HANDSHAKE);
9221 	tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
9222 	if (val == NIC_SRAM_DATA_SIG_MAGIC) {
9223 		u32 nic_cfg;
9224 
9225 		tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
9226 		if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
9227 			tg3_flag_set(tp, ENABLE_ASF);
9228 			tp->last_event_jiffies = jiffies;
9229 			if (tg3_flag(tp, 5750_PLUS))
9230 				tg3_flag_set(tp, ASF_NEW_HANDSHAKE);
9231 
9232 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &nic_cfg);
9233 			if (nic_cfg & NIC_SRAM_1G_ON_VAUX_OK)
9234 				tp->phy_flags |= TG3_PHYFLG_1G_ON_VAUX_OK;
9235 			if (nic_cfg & NIC_SRAM_LNK_FLAP_AVOID)
9236 				tp->phy_flags |= TG3_PHYFLG_KEEP_LINK_ON_PWRDN;
9237 		}
9238 	}
9239 
9240 	return 0;
9241 }
9242 
9243 static void tg3_get_nstats(struct tg3 *, struct rtnl_link_stats64 *);
9244 static void tg3_get_estats(struct tg3 *, struct tg3_ethtool_stats *);
9245 static void __tg3_set_rx_mode(struct net_device *);
9246 
9247 /* tp->lock is held. */
9248 static int tg3_halt(struct tg3 *tp, int kind, bool silent)
9249 {
9250 	int err;
9251 
9252 	tg3_stop_fw(tp);
9253 
9254 	tg3_write_sig_pre_reset(tp, kind);
9255 
9256 	tg3_abort_hw(tp, silent);
9257 	err = tg3_chip_reset(tp);
9258 
9259 	__tg3_set_mac_addr(tp, false);
9260 
9261 	tg3_write_sig_legacy(tp, kind);
9262 	tg3_write_sig_post_reset(tp, kind);
9263 
9264 	if (tp->hw_stats) {
9265 		/* Save the stats across chip resets... */
9266 		tg3_get_nstats(tp, &tp->net_stats_prev);
9267 		tg3_get_estats(tp, &tp->estats_prev);
9268 
9269 		/* And make sure the next sample is new data */
9270 		memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
9271 	}
9272 
9273 	return err;
9274 }
9275 
9276 static int tg3_set_mac_addr(struct net_device *dev, void *p)
9277 {
9278 	struct tg3 *tp = netdev_priv(dev);
9279 	struct sockaddr *addr = p;
9280 	int err = 0;
9281 	bool skip_mac_1 = false;
9282 
9283 	if (!is_valid_ether_addr(addr->sa_data))
9284 		return -EADDRNOTAVAIL;
9285 
9286 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
9287 
9288 	if (!netif_running(dev))
9289 		return 0;
9290 
9291 	if (tg3_flag(tp, ENABLE_ASF)) {
9292 		u32 addr0_high, addr0_low, addr1_high, addr1_low;
9293 
9294 		addr0_high = tr32(MAC_ADDR_0_HIGH);
9295 		addr0_low = tr32(MAC_ADDR_0_LOW);
9296 		addr1_high = tr32(MAC_ADDR_1_HIGH);
9297 		addr1_low = tr32(MAC_ADDR_1_LOW);
9298 
9299 		/* Skip MAC addr 1 if ASF is using it. */
9300 		if ((addr0_high != addr1_high || addr0_low != addr1_low) &&
9301 		    !(addr1_high == 0 && addr1_low == 0))
9302 			skip_mac_1 = true;
9303 	}
9304 	spin_lock_bh(&tp->lock);
9305 	__tg3_set_mac_addr(tp, skip_mac_1);
9306 	__tg3_set_rx_mode(dev);
9307 	spin_unlock_bh(&tp->lock);
9308 
9309 	return err;
9310 }
9311 
9312 /* tp->lock is held. */
9313 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
9314 			   dma_addr_t mapping, u32 maxlen_flags,
9315 			   u32 nic_addr)
9316 {
9317 	tg3_write_mem(tp,
9318 		      (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
9319 		      ((u64) mapping >> 32));
9320 	tg3_write_mem(tp,
9321 		      (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
9322 		      ((u64) mapping & 0xffffffff));
9323 	tg3_write_mem(tp,
9324 		      (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
9325 		       maxlen_flags);
9326 
9327 	if (!tg3_flag(tp, 5705_PLUS))
9328 		tg3_write_mem(tp,
9329 			      (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
9330 			      nic_addr);
9331 }
9332 
9333 
9334 static void tg3_coal_tx_init(struct tg3 *tp, struct ethtool_coalesce *ec)
9335 {
9336 	int i = 0;
9337 
9338 	if (!tg3_flag(tp, ENABLE_TSS)) {
9339 		tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
9340 		tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
9341 		tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
9342 	} else {
9343 		tw32(HOSTCC_TXCOL_TICKS, 0);
9344 		tw32(HOSTCC_TXMAX_FRAMES, 0);
9345 		tw32(HOSTCC_TXCOAL_MAXF_INT, 0);
9346 
9347 		for (; i < tp->txq_cnt; i++) {
9348 			u32 reg;
9349 
9350 			reg = HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18;
9351 			tw32(reg, ec->tx_coalesce_usecs);
9352 			reg = HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18;
9353 			tw32(reg, ec->tx_max_coalesced_frames);
9354 			reg = HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18;
9355 			tw32(reg, ec->tx_max_coalesced_frames_irq);
9356 		}
9357 	}
9358 
9359 	for (; i < tp->irq_max - 1; i++) {
9360 		tw32(HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18, 0);
9361 		tw32(HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18, 0);
9362 		tw32(HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
9363 	}
9364 }
9365 
9366 static void tg3_coal_rx_init(struct tg3 *tp, struct ethtool_coalesce *ec)
9367 {
9368 	int i = 0;
9369 	u32 limit = tp->rxq_cnt;
9370 
9371 	if (!tg3_flag(tp, ENABLE_RSS)) {
9372 		tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
9373 		tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
9374 		tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
9375 		limit--;
9376 	} else {
9377 		tw32(HOSTCC_RXCOL_TICKS, 0);
9378 		tw32(HOSTCC_RXMAX_FRAMES, 0);
9379 		tw32(HOSTCC_RXCOAL_MAXF_INT, 0);
9380 	}
9381 
9382 	for (; i < limit; i++) {
9383 		u32 reg;
9384 
9385 		reg = HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18;
9386 		tw32(reg, ec->rx_coalesce_usecs);
9387 		reg = HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18;
9388 		tw32(reg, ec->rx_max_coalesced_frames);
9389 		reg = HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18;
9390 		tw32(reg, ec->rx_max_coalesced_frames_irq);
9391 	}
9392 
9393 	for (; i < tp->irq_max - 1; i++) {
9394 		tw32(HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18, 0);
9395 		tw32(HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18, 0);
9396 		tw32(HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
9397 	}
9398 }
9399 
9400 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
9401 {
9402 	tg3_coal_tx_init(tp, ec);
9403 	tg3_coal_rx_init(tp, ec);
9404 
9405 	if (!tg3_flag(tp, 5705_PLUS)) {
9406 		u32 val = ec->stats_block_coalesce_usecs;
9407 
9408 		tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
9409 		tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
9410 
9411 		if (!tp->link_up)
9412 			val = 0;
9413 
9414 		tw32(HOSTCC_STAT_COAL_TICKS, val);
9415 	}
9416 }
9417 
9418 /* tp->lock is held. */
9419 static void tg3_tx_rcbs_disable(struct tg3 *tp)
9420 {
9421 	u32 txrcb, limit;
9422 
9423 	/* Disable all transmit rings but the first. */
9424 	if (!tg3_flag(tp, 5705_PLUS))
9425 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 16;
9426 	else if (tg3_flag(tp, 5717_PLUS))
9427 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 4;
9428 	else if (tg3_flag(tp, 57765_CLASS) ||
9429 		 tg3_asic_rev(tp) == ASIC_REV_5762)
9430 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 2;
9431 	else
9432 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
9433 
9434 	for (txrcb = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
9435 	     txrcb < limit; txrcb += TG3_BDINFO_SIZE)
9436 		tg3_write_mem(tp, txrcb + TG3_BDINFO_MAXLEN_FLAGS,
9437 			      BDINFO_FLAGS_DISABLED);
9438 }
9439 
9440 /* tp->lock is held. */
9441 static void tg3_tx_rcbs_init(struct tg3 *tp)
9442 {
9443 	int i = 0;
9444 	u32 txrcb = NIC_SRAM_SEND_RCB;
9445 
9446 	if (tg3_flag(tp, ENABLE_TSS))
9447 		i++;
9448 
9449 	for (; i < tp->irq_max; i++, txrcb += TG3_BDINFO_SIZE) {
9450 		struct tg3_napi *tnapi = &tp->napi[i];
9451 
9452 		if (!tnapi->tx_ring)
9453 			continue;
9454 
9455 		tg3_set_bdinfo(tp, txrcb, tnapi->tx_desc_mapping,
9456 			       (TG3_TX_RING_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT),
9457 			       NIC_SRAM_TX_BUFFER_DESC);
9458 	}
9459 }
9460 
9461 /* tp->lock is held. */
9462 static void tg3_rx_ret_rcbs_disable(struct tg3 *tp)
9463 {
9464 	u32 rxrcb, limit;
9465 
9466 	/* Disable all receive return rings but the first. */
9467 	if (tg3_flag(tp, 5717_PLUS))
9468 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 17;
9469 	else if (!tg3_flag(tp, 5705_PLUS))
9470 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 16;
9471 	else if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
9472 		 tg3_asic_rev(tp) == ASIC_REV_5762 ||
9473 		 tg3_flag(tp, 57765_CLASS))
9474 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 4;
9475 	else
9476 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
9477 
9478 	for (rxrcb = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
9479 	     rxrcb < limit; rxrcb += TG3_BDINFO_SIZE)
9480 		tg3_write_mem(tp, rxrcb + TG3_BDINFO_MAXLEN_FLAGS,
9481 			      BDINFO_FLAGS_DISABLED);
9482 }
9483 
9484 /* tp->lock is held. */
9485 static void tg3_rx_ret_rcbs_init(struct tg3 *tp)
9486 {
9487 	int i = 0;
9488 	u32 rxrcb = NIC_SRAM_RCV_RET_RCB;
9489 
9490 	if (tg3_flag(tp, ENABLE_RSS))
9491 		i++;
9492 
9493 	for (; i < tp->irq_max; i++, rxrcb += TG3_BDINFO_SIZE) {
9494 		struct tg3_napi *tnapi = &tp->napi[i];
9495 
9496 		if (!tnapi->rx_rcb)
9497 			continue;
9498 
9499 		tg3_set_bdinfo(tp, rxrcb, tnapi->rx_rcb_mapping,
9500 			       (tp->rx_ret_ring_mask + 1) <<
9501 				BDINFO_FLAGS_MAXLEN_SHIFT, 0);
9502 	}
9503 }
9504 
9505 /* tp->lock is held. */
9506 static void tg3_rings_reset(struct tg3 *tp)
9507 {
9508 	int i;
9509 	u32 stblk;
9510 	struct tg3_napi *tnapi = &tp->napi[0];
9511 
9512 	tg3_tx_rcbs_disable(tp);
9513 
9514 	tg3_rx_ret_rcbs_disable(tp);
9515 
9516 	/* Disable interrupts */
9517 	tw32_mailbox_f(tp->napi[0].int_mbox, 1);
9518 	tp->napi[0].chk_msi_cnt = 0;
9519 	tp->napi[0].last_rx_cons = 0;
9520 	tp->napi[0].last_tx_cons = 0;
9521 
9522 	/* Zero mailbox registers. */
9523 	if (tg3_flag(tp, SUPPORT_MSIX)) {
9524 		for (i = 1; i < tp->irq_max; i++) {
9525 			tp->napi[i].tx_prod = 0;
9526 			tp->napi[i].tx_cons = 0;
9527 			if (tg3_flag(tp, ENABLE_TSS))
9528 				tw32_mailbox(tp->napi[i].prodmbox, 0);
9529 			tw32_rx_mbox(tp->napi[i].consmbox, 0);
9530 			tw32_mailbox_f(tp->napi[i].int_mbox, 1);
9531 			tp->napi[i].chk_msi_cnt = 0;
9532 			tp->napi[i].last_rx_cons = 0;
9533 			tp->napi[i].last_tx_cons = 0;
9534 		}
9535 		if (!tg3_flag(tp, ENABLE_TSS))
9536 			tw32_mailbox(tp->napi[0].prodmbox, 0);
9537 	} else {
9538 		tp->napi[0].tx_prod = 0;
9539 		tp->napi[0].tx_cons = 0;
9540 		tw32_mailbox(tp->napi[0].prodmbox, 0);
9541 		tw32_rx_mbox(tp->napi[0].consmbox, 0);
9542 	}
9543 
9544 	/* Make sure the NIC-based send BD rings are disabled. */
9545 	if (!tg3_flag(tp, 5705_PLUS)) {
9546 		u32 mbox = MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW;
9547 		for (i = 0; i < 16; i++)
9548 			tw32_tx_mbox(mbox + i * 8, 0);
9549 	}
9550 
9551 	/* Clear status block in ram. */
9552 	memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
9553 
9554 	/* Set status block DMA address */
9555 	tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
9556 	     ((u64) tnapi->status_mapping >> 32));
9557 	tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
9558 	     ((u64) tnapi->status_mapping & 0xffffffff));
9559 
9560 	stblk = HOSTCC_STATBLCK_RING1;
9561 
9562 	for (i = 1, tnapi++; i < tp->irq_cnt; i++, tnapi++) {
9563 		u64 mapping = (u64)tnapi->status_mapping;
9564 		tw32(stblk + TG3_64BIT_REG_HIGH, mapping >> 32);
9565 		tw32(stblk + TG3_64BIT_REG_LOW, mapping & 0xffffffff);
9566 		stblk += 8;
9567 
9568 		/* Clear status block in ram. */
9569 		memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
9570 	}
9571 
9572 	tg3_tx_rcbs_init(tp);
9573 	tg3_rx_ret_rcbs_init(tp);
9574 }
9575 
9576 static void tg3_setup_rxbd_thresholds(struct tg3 *tp)
9577 {
9578 	u32 val, bdcache_maxcnt, host_rep_thresh, nic_rep_thresh;
9579 
9580 	if (!tg3_flag(tp, 5750_PLUS) ||
9581 	    tg3_flag(tp, 5780_CLASS) ||
9582 	    tg3_asic_rev(tp) == ASIC_REV_5750 ||
9583 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
9584 	    tg3_flag(tp, 57765_PLUS))
9585 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5700;
9586 	else if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
9587 		 tg3_asic_rev(tp) == ASIC_REV_5787)
9588 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5755;
9589 	else
9590 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5906;
9591 
9592 	nic_rep_thresh = min(bdcache_maxcnt / 2, tp->rx_std_max_post);
9593 	host_rep_thresh = max_t(u32, tp->rx_pending / 8, 1);
9594 
9595 	val = min(nic_rep_thresh, host_rep_thresh);
9596 	tw32(RCVBDI_STD_THRESH, val);
9597 
9598 	if (tg3_flag(tp, 57765_PLUS))
9599 		tw32(STD_REPLENISH_LWM, bdcache_maxcnt);
9600 
9601 	if (!tg3_flag(tp, JUMBO_CAPABLE) || tg3_flag(tp, 5780_CLASS))
9602 		return;
9603 
9604 	bdcache_maxcnt = TG3_SRAM_RX_JMB_BDCACHE_SIZE_5700;
9605 
9606 	host_rep_thresh = max_t(u32, tp->rx_jumbo_pending / 8, 1);
9607 
9608 	val = min(bdcache_maxcnt / 2, host_rep_thresh);
9609 	tw32(RCVBDI_JUMBO_THRESH, val);
9610 
9611 	if (tg3_flag(tp, 57765_PLUS))
9612 		tw32(JMB_REPLENISH_LWM, bdcache_maxcnt);
9613 }
9614 
9615 static inline u32 calc_crc(unsigned char *buf, int len)
9616 {
9617 	u32 reg;
9618 	u32 tmp;
9619 	int j, k;
9620 
9621 	reg = 0xffffffff;
9622 
9623 	for (j = 0; j < len; j++) {
9624 		reg ^= buf[j];
9625 
9626 		for (k = 0; k < 8; k++) {
9627 			tmp = reg & 0x01;
9628 
9629 			reg >>= 1;
9630 
9631 			if (tmp)
9632 				reg ^= 0xedb88320;
9633 		}
9634 	}
9635 
9636 	return ~reg;
9637 }
9638 
9639 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
9640 {
9641 	/* accept or reject all multicast frames */
9642 	tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
9643 	tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
9644 	tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
9645 	tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
9646 }
9647 
9648 static void __tg3_set_rx_mode(struct net_device *dev)
9649 {
9650 	struct tg3 *tp = netdev_priv(dev);
9651 	u32 rx_mode;
9652 
9653 	rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
9654 				  RX_MODE_KEEP_VLAN_TAG);
9655 
9656 #if !defined(CONFIG_VLAN_8021Q) && !defined(CONFIG_VLAN_8021Q_MODULE)
9657 	/* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
9658 	 * flag clear.
9659 	 */
9660 	if (!tg3_flag(tp, ENABLE_ASF))
9661 		rx_mode |= RX_MODE_KEEP_VLAN_TAG;
9662 #endif
9663 
9664 	if (dev->flags & IFF_PROMISC) {
9665 		/* Promiscuous mode. */
9666 		rx_mode |= RX_MODE_PROMISC;
9667 	} else if (dev->flags & IFF_ALLMULTI) {
9668 		/* Accept all multicast. */
9669 		tg3_set_multi(tp, 1);
9670 	} else if (netdev_mc_empty(dev)) {
9671 		/* Reject all multicast. */
9672 		tg3_set_multi(tp, 0);
9673 	} else {
9674 		/* Accept one or more multicast(s). */
9675 		struct netdev_hw_addr *ha;
9676 		u32 mc_filter[4] = { 0, };
9677 		u32 regidx;
9678 		u32 bit;
9679 		u32 crc;
9680 
9681 		netdev_for_each_mc_addr(ha, dev) {
9682 			crc = calc_crc(ha->addr, ETH_ALEN);
9683 			bit = ~crc & 0x7f;
9684 			regidx = (bit & 0x60) >> 5;
9685 			bit &= 0x1f;
9686 			mc_filter[regidx] |= (1 << bit);
9687 		}
9688 
9689 		tw32(MAC_HASH_REG_0, mc_filter[0]);
9690 		tw32(MAC_HASH_REG_1, mc_filter[1]);
9691 		tw32(MAC_HASH_REG_2, mc_filter[2]);
9692 		tw32(MAC_HASH_REG_3, mc_filter[3]);
9693 	}
9694 
9695 	if (netdev_uc_count(dev) > TG3_MAX_UCAST_ADDR(tp)) {
9696 		rx_mode |= RX_MODE_PROMISC;
9697 	} else if (!(dev->flags & IFF_PROMISC)) {
9698 		/* Add all entries into to the mac addr filter list */
9699 		int i = 0;
9700 		struct netdev_hw_addr *ha;
9701 
9702 		netdev_for_each_uc_addr(ha, dev) {
9703 			__tg3_set_one_mac_addr(tp, ha->addr,
9704 					       i + TG3_UCAST_ADDR_IDX(tp));
9705 			i++;
9706 		}
9707 	}
9708 
9709 	if (rx_mode != tp->rx_mode) {
9710 		tp->rx_mode = rx_mode;
9711 		tw32_f(MAC_RX_MODE, rx_mode);
9712 		udelay(10);
9713 	}
9714 }
9715 
9716 static void tg3_rss_init_dflt_indir_tbl(struct tg3 *tp, u32 qcnt)
9717 {
9718 	int i;
9719 
9720 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
9721 		tp->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, qcnt);
9722 }
9723 
9724 static void tg3_rss_check_indir_tbl(struct tg3 *tp)
9725 {
9726 	int i;
9727 
9728 	if (!tg3_flag(tp, SUPPORT_MSIX))
9729 		return;
9730 
9731 	if (tp->rxq_cnt == 1) {
9732 		memset(&tp->rss_ind_tbl[0], 0, sizeof(tp->rss_ind_tbl));
9733 		return;
9734 	}
9735 
9736 	/* Validate table against current IRQ count */
9737 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++) {
9738 		if (tp->rss_ind_tbl[i] >= tp->rxq_cnt)
9739 			break;
9740 	}
9741 
9742 	if (i != TG3_RSS_INDIR_TBL_SIZE)
9743 		tg3_rss_init_dflt_indir_tbl(tp, tp->rxq_cnt);
9744 }
9745 
9746 static void tg3_rss_write_indir_tbl(struct tg3 *tp)
9747 {
9748 	int i = 0;
9749 	u32 reg = MAC_RSS_INDIR_TBL_0;
9750 
9751 	while (i < TG3_RSS_INDIR_TBL_SIZE) {
9752 		u32 val = tp->rss_ind_tbl[i];
9753 		i++;
9754 		for (; i % 8; i++) {
9755 			val <<= 4;
9756 			val |= tp->rss_ind_tbl[i];
9757 		}
9758 		tw32(reg, val);
9759 		reg += 4;
9760 	}
9761 }
9762 
9763 static inline u32 tg3_lso_rd_dma_workaround_bit(struct tg3 *tp)
9764 {
9765 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
9766 		return TG3_LSO_RD_DMA_TX_LENGTH_WA_5719;
9767 	else
9768 		return TG3_LSO_RD_DMA_TX_LENGTH_WA_5720;
9769 }
9770 
9771 /* tp->lock is held. */
9772 static int tg3_reset_hw(struct tg3 *tp, bool reset_phy)
9773 {
9774 	u32 val, rdmac_mode;
9775 	int i, err, limit;
9776 	struct tg3_rx_prodring_set *tpr = &tp->napi[0].prodring;
9777 
9778 	tg3_disable_ints(tp);
9779 
9780 	tg3_stop_fw(tp);
9781 
9782 	tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
9783 
9784 	if (tg3_flag(tp, INIT_COMPLETE))
9785 		tg3_abort_hw(tp, 1);
9786 
9787 	if ((tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
9788 	    !(tp->phy_flags & TG3_PHYFLG_USER_CONFIGURED)) {
9789 		tg3_phy_pull_config(tp);
9790 		tg3_eee_pull_config(tp, NULL);
9791 		tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
9792 	}
9793 
9794 	/* Enable MAC control of LPI */
9795 	if (tp->phy_flags & TG3_PHYFLG_EEE_CAP)
9796 		tg3_setup_eee(tp);
9797 
9798 	if (reset_phy)
9799 		tg3_phy_reset(tp);
9800 
9801 	err = tg3_chip_reset(tp);
9802 	if (err)
9803 		return err;
9804 
9805 	tg3_write_sig_legacy(tp, RESET_KIND_INIT);
9806 
9807 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX) {
9808 		val = tr32(TG3_CPMU_CTRL);
9809 		val &= ~(CPMU_CTRL_LINK_AWARE_MODE | CPMU_CTRL_LINK_IDLE_MODE);
9810 		tw32(TG3_CPMU_CTRL, val);
9811 
9812 		val = tr32(TG3_CPMU_LSPD_10MB_CLK);
9813 		val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
9814 		val |= CPMU_LSPD_10MB_MACCLK_6_25;
9815 		tw32(TG3_CPMU_LSPD_10MB_CLK, val);
9816 
9817 		val = tr32(TG3_CPMU_LNK_AWARE_PWRMD);
9818 		val &= ~CPMU_LNK_AWARE_MACCLK_MASK;
9819 		val |= CPMU_LNK_AWARE_MACCLK_6_25;
9820 		tw32(TG3_CPMU_LNK_AWARE_PWRMD, val);
9821 
9822 		val = tr32(TG3_CPMU_HST_ACC);
9823 		val &= ~CPMU_HST_ACC_MACCLK_MASK;
9824 		val |= CPMU_HST_ACC_MACCLK_6_25;
9825 		tw32(TG3_CPMU_HST_ACC, val);
9826 	}
9827 
9828 	if (tg3_asic_rev(tp) == ASIC_REV_57780) {
9829 		val = tr32(PCIE_PWR_MGMT_THRESH) & ~PCIE_PWR_MGMT_L1_THRESH_MSK;
9830 		val |= PCIE_PWR_MGMT_EXT_ASPM_TMR_EN |
9831 		       PCIE_PWR_MGMT_L1_THRESH_4MS;
9832 		tw32(PCIE_PWR_MGMT_THRESH, val);
9833 
9834 		val = tr32(TG3_PCIE_EIDLE_DELAY) & ~TG3_PCIE_EIDLE_DELAY_MASK;
9835 		tw32(TG3_PCIE_EIDLE_DELAY, val | TG3_PCIE_EIDLE_DELAY_13_CLKS);
9836 
9837 		tw32(TG3_CORR_ERR_STAT, TG3_CORR_ERR_STAT_CLEAR);
9838 
9839 		val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
9840 		tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
9841 	}
9842 
9843 	if (tg3_flag(tp, L1PLLPD_EN)) {
9844 		u32 grc_mode = tr32(GRC_MODE);
9845 
9846 		/* Access the lower 1K of PL PCIE block registers. */
9847 		val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9848 		tw32(GRC_MODE, val | GRC_MODE_PCIE_PL_SEL);
9849 
9850 		val = tr32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL1);
9851 		tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL1,
9852 		     val | TG3_PCIE_PL_LO_PHYCTL1_L1PLLPD_EN);
9853 
9854 		tw32(GRC_MODE, grc_mode);
9855 	}
9856 
9857 	if (tg3_flag(tp, 57765_CLASS)) {
9858 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0) {
9859 			u32 grc_mode = tr32(GRC_MODE);
9860 
9861 			/* Access the lower 1K of PL PCIE block registers. */
9862 			val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9863 			tw32(GRC_MODE, val | GRC_MODE_PCIE_PL_SEL);
9864 
9865 			val = tr32(TG3_PCIE_TLDLPL_PORT +
9866 				   TG3_PCIE_PL_LO_PHYCTL5);
9867 			tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL5,
9868 			     val | TG3_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ);
9869 
9870 			tw32(GRC_MODE, grc_mode);
9871 		}
9872 
9873 		if (tg3_chip_rev(tp) != CHIPREV_57765_AX) {
9874 			u32 grc_mode;
9875 
9876 			/* Fix transmit hangs */
9877 			val = tr32(TG3_CPMU_PADRNG_CTL);
9878 			val |= TG3_CPMU_PADRNG_CTL_RDIV2;
9879 			tw32(TG3_CPMU_PADRNG_CTL, val);
9880 
9881 			grc_mode = tr32(GRC_MODE);
9882 
9883 			/* Access the lower 1K of DL PCIE block registers. */
9884 			val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9885 			tw32(GRC_MODE, val | GRC_MODE_PCIE_DL_SEL);
9886 
9887 			val = tr32(TG3_PCIE_TLDLPL_PORT +
9888 				   TG3_PCIE_DL_LO_FTSMAX);
9889 			val &= ~TG3_PCIE_DL_LO_FTSMAX_MSK;
9890 			tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_DL_LO_FTSMAX,
9891 			     val | TG3_PCIE_DL_LO_FTSMAX_VAL);
9892 
9893 			tw32(GRC_MODE, grc_mode);
9894 		}
9895 
9896 		val = tr32(TG3_CPMU_LSPD_10MB_CLK);
9897 		val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
9898 		val |= CPMU_LSPD_10MB_MACCLK_6_25;
9899 		tw32(TG3_CPMU_LSPD_10MB_CLK, val);
9900 	}
9901 
9902 	/* This works around an issue with Athlon chipsets on
9903 	 * B3 tigon3 silicon.  This bit has no effect on any
9904 	 * other revision.  But do not set this on PCI Express
9905 	 * chips and don't even touch the clocks if the CPMU is present.
9906 	 */
9907 	if (!tg3_flag(tp, CPMU_PRESENT)) {
9908 		if (!tg3_flag(tp, PCI_EXPRESS))
9909 			tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
9910 		tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
9911 	}
9912 
9913 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0 &&
9914 	    tg3_flag(tp, PCIX_MODE)) {
9915 		val = tr32(TG3PCI_PCISTATE);
9916 		val |= PCISTATE_RETRY_SAME_DMA;
9917 		tw32(TG3PCI_PCISTATE, val);
9918 	}
9919 
9920 	if (tg3_flag(tp, ENABLE_APE)) {
9921 		/* Allow reads and writes to the
9922 		 * APE register and memory space.
9923 		 */
9924 		val = tr32(TG3PCI_PCISTATE);
9925 		val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
9926 		       PCISTATE_ALLOW_APE_SHMEM_WR |
9927 		       PCISTATE_ALLOW_APE_PSPACE_WR;
9928 		tw32(TG3PCI_PCISTATE, val);
9929 	}
9930 
9931 	if (tg3_chip_rev(tp) == CHIPREV_5704_BX) {
9932 		/* Enable some hw fixes.  */
9933 		val = tr32(TG3PCI_MSI_DATA);
9934 		val |= (1 << 26) | (1 << 28) | (1 << 29);
9935 		tw32(TG3PCI_MSI_DATA, val);
9936 	}
9937 
9938 	/* Descriptor ring init may make accesses to the
9939 	 * NIC SRAM area to setup the TX descriptors, so we
9940 	 * can only do this after the hardware has been
9941 	 * successfully reset.
9942 	 */
9943 	err = tg3_init_rings(tp);
9944 	if (err)
9945 		return err;
9946 
9947 	if (tg3_flag(tp, 57765_PLUS)) {
9948 		val = tr32(TG3PCI_DMA_RW_CTRL) &
9949 		      ~DMA_RWCTRL_DIS_CACHE_ALIGNMENT;
9950 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0)
9951 			val &= ~DMA_RWCTRL_CRDRDR_RDMA_MRRS_MSK;
9952 		if (!tg3_flag(tp, 57765_CLASS) &&
9953 		    tg3_asic_rev(tp) != ASIC_REV_5717 &&
9954 		    tg3_asic_rev(tp) != ASIC_REV_5762)
9955 			val |= DMA_RWCTRL_TAGGED_STAT_WA;
9956 		tw32(TG3PCI_DMA_RW_CTRL, val | tp->dma_rwctrl);
9957 	} else if (tg3_asic_rev(tp) != ASIC_REV_5784 &&
9958 		   tg3_asic_rev(tp) != ASIC_REV_5761) {
9959 		/* This value is determined during the probe time DMA
9960 		 * engine test, tg3_test_dma.
9961 		 */
9962 		tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
9963 	}
9964 
9965 	tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
9966 			  GRC_MODE_4X_NIC_SEND_RINGS |
9967 			  GRC_MODE_NO_TX_PHDR_CSUM |
9968 			  GRC_MODE_NO_RX_PHDR_CSUM);
9969 	tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
9970 
9971 	/* Pseudo-header checksum is done by hardware logic and not
9972 	 * the offload processers, so make the chip do the pseudo-
9973 	 * header checksums on receive.  For transmit it is more
9974 	 * convenient to do the pseudo-header checksum in software
9975 	 * as Linux does that on transmit for us in all cases.
9976 	 */
9977 	tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
9978 
9979 	val = GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP;
9980 	if (tp->rxptpctl)
9981 		tw32(TG3_RX_PTP_CTL,
9982 		     tp->rxptpctl | TG3_RX_PTP_CTL_HWTS_INTERLOCK);
9983 
9984 	if (tg3_flag(tp, PTP_CAPABLE))
9985 		val |= GRC_MODE_TIME_SYNC_ENABLE;
9986 
9987 	tw32(GRC_MODE, tp->grc_mode | val);
9988 
9989 	/* Setup the timer prescalar register.  Clock is always 66Mhz. */
9990 	val = tr32(GRC_MISC_CFG);
9991 	val &= ~0xff;
9992 	val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
9993 	tw32(GRC_MISC_CFG, val);
9994 
9995 	/* Initialize MBUF/DESC pool. */
9996 	if (tg3_flag(tp, 5750_PLUS)) {
9997 		/* Do nothing.  */
9998 	} else if (tg3_asic_rev(tp) != ASIC_REV_5705) {
9999 		tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
10000 		if (tg3_asic_rev(tp) == ASIC_REV_5704)
10001 			tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
10002 		else
10003 			tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
10004 		tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
10005 		tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
10006 	} else if (tg3_flag(tp, TSO_CAPABLE)) {
10007 		int fw_len;
10008 
10009 		fw_len = tp->fw_len;
10010 		fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
10011 		tw32(BUFMGR_MB_POOL_ADDR,
10012 		     NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
10013 		tw32(BUFMGR_MB_POOL_SIZE,
10014 		     NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
10015 	}
10016 
10017 	if (tp->dev->mtu <= ETH_DATA_LEN) {
10018 		tw32(BUFMGR_MB_RDMA_LOW_WATER,
10019 		     tp->bufmgr_config.mbuf_read_dma_low_water);
10020 		tw32(BUFMGR_MB_MACRX_LOW_WATER,
10021 		     tp->bufmgr_config.mbuf_mac_rx_low_water);
10022 		tw32(BUFMGR_MB_HIGH_WATER,
10023 		     tp->bufmgr_config.mbuf_high_water);
10024 	} else {
10025 		tw32(BUFMGR_MB_RDMA_LOW_WATER,
10026 		     tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
10027 		tw32(BUFMGR_MB_MACRX_LOW_WATER,
10028 		     tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
10029 		tw32(BUFMGR_MB_HIGH_WATER,
10030 		     tp->bufmgr_config.mbuf_high_water_jumbo);
10031 	}
10032 	tw32(BUFMGR_DMA_LOW_WATER,
10033 	     tp->bufmgr_config.dma_low_water);
10034 	tw32(BUFMGR_DMA_HIGH_WATER,
10035 	     tp->bufmgr_config.dma_high_water);
10036 
10037 	val = BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE;
10038 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
10039 		val |= BUFMGR_MODE_NO_TX_UNDERRUN;
10040 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
10041 	    tg3_asic_rev(tp) == ASIC_REV_5762 ||
10042 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10043 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5720_A0)
10044 		val |= BUFMGR_MODE_MBLOW_ATTN_ENAB;
10045 	tw32(BUFMGR_MODE, val);
10046 	for (i = 0; i < 2000; i++) {
10047 		if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
10048 			break;
10049 		udelay(10);
10050 	}
10051 	if (i >= 2000) {
10052 		netdev_err(tp->dev, "%s cannot enable BUFMGR\n", __func__);
10053 		return -ENODEV;
10054 	}
10055 
10056 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5906_A1)
10057 		tw32(ISO_PKT_TX, (tr32(ISO_PKT_TX) & ~0x3) | 0x2);
10058 
10059 	tg3_setup_rxbd_thresholds(tp);
10060 
10061 	/* Initialize TG3_BDINFO's at:
10062 	 *  RCVDBDI_STD_BD:	standard eth size rx ring
10063 	 *  RCVDBDI_JUMBO_BD:	jumbo frame rx ring
10064 	 *  RCVDBDI_MINI_BD:	small frame rx ring (??? does not work)
10065 	 *
10066 	 * like so:
10067 	 *  TG3_BDINFO_HOST_ADDR:	high/low parts of DMA address of ring
10068 	 *  TG3_BDINFO_MAXLEN_FLAGS:	(rx max buffer size << 16) |
10069 	 *                              ring attribute flags
10070 	 *  TG3_BDINFO_NIC_ADDR:	location of descriptors in nic SRAM
10071 	 *
10072 	 * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
10073 	 * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
10074 	 *
10075 	 * The size of each ring is fixed in the firmware, but the location is
10076 	 * configurable.
10077 	 */
10078 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
10079 	     ((u64) tpr->rx_std_mapping >> 32));
10080 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
10081 	     ((u64) tpr->rx_std_mapping & 0xffffffff));
10082 	if (!tg3_flag(tp, 5717_PLUS))
10083 		tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
10084 		     NIC_SRAM_RX_BUFFER_DESC);
10085 
10086 	/* Disable the mini ring */
10087 	if (!tg3_flag(tp, 5705_PLUS))
10088 		tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
10089 		     BDINFO_FLAGS_DISABLED);
10090 
10091 	/* Program the jumbo buffer descriptor ring control
10092 	 * blocks on those devices that have them.
10093 	 */
10094 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10095 	    (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS))) {
10096 
10097 		if (tg3_flag(tp, JUMBO_RING_ENABLE)) {
10098 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
10099 			     ((u64) tpr->rx_jmb_mapping >> 32));
10100 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
10101 			     ((u64) tpr->rx_jmb_mapping & 0xffffffff));
10102 			val = TG3_RX_JMB_RING_SIZE(tp) <<
10103 			      BDINFO_FLAGS_MAXLEN_SHIFT;
10104 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
10105 			     val | BDINFO_FLAGS_USE_EXT_RECV);
10106 			if (!tg3_flag(tp, USE_JUMBO_BDFLAG) ||
10107 			    tg3_flag(tp, 57765_CLASS) ||
10108 			    tg3_asic_rev(tp) == ASIC_REV_5762)
10109 				tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
10110 				     NIC_SRAM_RX_JUMBO_BUFFER_DESC);
10111 		} else {
10112 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
10113 			     BDINFO_FLAGS_DISABLED);
10114 		}
10115 
10116 		if (tg3_flag(tp, 57765_PLUS)) {
10117 			val = TG3_RX_STD_RING_SIZE(tp);
10118 			val <<= BDINFO_FLAGS_MAXLEN_SHIFT;
10119 			val |= (TG3_RX_STD_DMA_SZ << 2);
10120 		} else
10121 			val = TG3_RX_STD_DMA_SZ << BDINFO_FLAGS_MAXLEN_SHIFT;
10122 	} else
10123 		val = TG3_RX_STD_MAX_SIZE_5700 << BDINFO_FLAGS_MAXLEN_SHIFT;
10124 
10125 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS, val);
10126 
10127 	tpr->rx_std_prod_idx = tp->rx_pending;
10128 	tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG, tpr->rx_std_prod_idx);
10129 
10130 	tpr->rx_jmb_prod_idx =
10131 		tg3_flag(tp, JUMBO_RING_ENABLE) ? tp->rx_jumbo_pending : 0;
10132 	tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG, tpr->rx_jmb_prod_idx);
10133 
10134 	tg3_rings_reset(tp);
10135 
10136 	/* Initialize MAC address and backoff seed. */
10137 	__tg3_set_mac_addr(tp, false);
10138 
10139 	/* MTU + ethernet header + FCS + optional VLAN tag */
10140 	tw32(MAC_RX_MTU_SIZE,
10141 	     tp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
10142 
10143 	/* The slot time is changed by tg3_setup_phy if we
10144 	 * run at gigabit with half duplex.
10145 	 */
10146 	val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
10147 	      (6 << TX_LENGTHS_IPG_SHIFT) |
10148 	      (32 << TX_LENGTHS_SLOT_TIME_SHIFT);
10149 
10150 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10151 	    tg3_asic_rev(tp) == ASIC_REV_5762)
10152 		val |= tr32(MAC_TX_LENGTHS) &
10153 		       (TX_LENGTHS_JMB_FRM_LEN_MSK |
10154 			TX_LENGTHS_CNT_DWN_VAL_MSK);
10155 
10156 	tw32(MAC_TX_LENGTHS, val);
10157 
10158 	/* Receive rules. */
10159 	tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
10160 	tw32(RCVLPC_CONFIG, 0x0181);
10161 
10162 	/* Calculate RDMAC_MODE setting early, we need it to determine
10163 	 * the RCVLPC_STATE_ENABLE mask.
10164 	 */
10165 	rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
10166 		      RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
10167 		      RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
10168 		      RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
10169 		      RDMAC_MODE_LNGREAD_ENAB);
10170 
10171 	if (tg3_asic_rev(tp) == ASIC_REV_5717)
10172 		rdmac_mode |= RDMAC_MODE_MULT_DMA_RD_DIS;
10173 
10174 	if (tg3_asic_rev(tp) == ASIC_REV_5784 ||
10175 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10176 	    tg3_asic_rev(tp) == ASIC_REV_57780)
10177 		rdmac_mode |= RDMAC_MODE_BD_SBD_CRPT_ENAB |
10178 			      RDMAC_MODE_MBUF_RBD_CRPT_ENAB |
10179 			      RDMAC_MODE_MBUF_SBD_CRPT_ENAB;
10180 
10181 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
10182 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
10183 		if (tg3_flag(tp, TSO_CAPABLE) &&
10184 		    tg3_asic_rev(tp) == ASIC_REV_5705) {
10185 			rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
10186 		} else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
10187 			   !tg3_flag(tp, IS_5788)) {
10188 			rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
10189 		}
10190 	}
10191 
10192 	if (tg3_flag(tp, PCI_EXPRESS))
10193 		rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
10194 
10195 	if (tg3_asic_rev(tp) == ASIC_REV_57766) {
10196 		tp->dma_limit = 0;
10197 		if (tp->dev->mtu <= ETH_DATA_LEN) {
10198 			rdmac_mode |= RDMAC_MODE_JMB_2K_MMRR;
10199 			tp->dma_limit = TG3_TX_BD_DMA_MAX_2K;
10200 		}
10201 	}
10202 
10203 	if (tg3_flag(tp, HW_TSO_1) ||
10204 	    tg3_flag(tp, HW_TSO_2) ||
10205 	    tg3_flag(tp, HW_TSO_3))
10206 		rdmac_mode |= RDMAC_MODE_IPV4_LSO_EN;
10207 
10208 	if (tg3_flag(tp, 57765_PLUS) ||
10209 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10210 	    tg3_asic_rev(tp) == ASIC_REV_57780)
10211 		rdmac_mode |= RDMAC_MODE_IPV6_LSO_EN;
10212 
10213 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10214 	    tg3_asic_rev(tp) == ASIC_REV_5762)
10215 		rdmac_mode |= tr32(RDMAC_MODE) & RDMAC_MODE_H2BNC_VLAN_DET;
10216 
10217 	if (tg3_asic_rev(tp) == ASIC_REV_5761 ||
10218 	    tg3_asic_rev(tp) == ASIC_REV_5784 ||
10219 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10220 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
10221 	    tg3_flag(tp, 57765_PLUS)) {
10222 		u32 tgtreg;
10223 
10224 		if (tg3_asic_rev(tp) == ASIC_REV_5762)
10225 			tgtreg = TG3_RDMA_RSRVCTRL_REG2;
10226 		else
10227 			tgtreg = TG3_RDMA_RSRVCTRL_REG;
10228 
10229 		val = tr32(tgtreg);
10230 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10231 		    tg3_asic_rev(tp) == ASIC_REV_5762) {
10232 			val &= ~(TG3_RDMA_RSRVCTRL_TXMRGN_MASK |
10233 				 TG3_RDMA_RSRVCTRL_FIFO_LWM_MASK |
10234 				 TG3_RDMA_RSRVCTRL_FIFO_HWM_MASK);
10235 			val |= TG3_RDMA_RSRVCTRL_TXMRGN_320B |
10236 			       TG3_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
10237 			       TG3_RDMA_RSRVCTRL_FIFO_HWM_1_5K;
10238 		}
10239 		tw32(tgtreg, val | TG3_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
10240 	}
10241 
10242 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
10243 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
10244 	    tg3_asic_rev(tp) == ASIC_REV_5762) {
10245 		u32 tgtreg;
10246 
10247 		if (tg3_asic_rev(tp) == ASIC_REV_5762)
10248 			tgtreg = TG3_LSO_RD_DMA_CRPTEN_CTRL2;
10249 		else
10250 			tgtreg = TG3_LSO_RD_DMA_CRPTEN_CTRL;
10251 
10252 		val = tr32(tgtreg);
10253 		tw32(tgtreg, val |
10254 		     TG3_LSO_RD_DMA_CRPTEN_CTRL_BLEN_BD_4K |
10255 		     TG3_LSO_RD_DMA_CRPTEN_CTRL_BLEN_LSO_4K);
10256 	}
10257 
10258 	/* Receive/send statistics. */
10259 	if (tg3_flag(tp, 5750_PLUS)) {
10260 		val = tr32(RCVLPC_STATS_ENABLE);
10261 		val &= ~RCVLPC_STATSENAB_DACK_FIX;
10262 		tw32(RCVLPC_STATS_ENABLE, val);
10263 	} else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
10264 		   tg3_flag(tp, TSO_CAPABLE)) {
10265 		val = tr32(RCVLPC_STATS_ENABLE);
10266 		val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
10267 		tw32(RCVLPC_STATS_ENABLE, val);
10268 	} else {
10269 		tw32(RCVLPC_STATS_ENABLE, 0xffffff);
10270 	}
10271 	tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
10272 	tw32(SNDDATAI_STATSENAB, 0xffffff);
10273 	tw32(SNDDATAI_STATSCTRL,
10274 	     (SNDDATAI_SCTRL_ENABLE |
10275 	      SNDDATAI_SCTRL_FASTUPD));
10276 
10277 	/* Setup host coalescing engine. */
10278 	tw32(HOSTCC_MODE, 0);
10279 	for (i = 0; i < 2000; i++) {
10280 		if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
10281 			break;
10282 		udelay(10);
10283 	}
10284 
10285 	__tg3_set_coalesce(tp, &tp->coal);
10286 
10287 	if (!tg3_flag(tp, 5705_PLUS)) {
10288 		/* Status/statistics block address.  See tg3_timer,
10289 		 * the tg3_periodic_fetch_stats call there, and
10290 		 * tg3_get_stats to see how this works for 5705/5750 chips.
10291 		 */
10292 		tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
10293 		     ((u64) tp->stats_mapping >> 32));
10294 		tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
10295 		     ((u64) tp->stats_mapping & 0xffffffff));
10296 		tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
10297 
10298 		tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
10299 
10300 		/* Clear statistics and status block memory areas */
10301 		for (i = NIC_SRAM_STATS_BLK;
10302 		     i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
10303 		     i += sizeof(u32)) {
10304 			tg3_write_mem(tp, i, 0);
10305 			udelay(40);
10306 		}
10307 	}
10308 
10309 	tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
10310 
10311 	tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
10312 	tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
10313 	if (!tg3_flag(tp, 5705_PLUS))
10314 		tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
10315 
10316 	if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
10317 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
10318 		/* reset to prevent losing 1st rx packet intermittently */
10319 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
10320 		udelay(10);
10321 	}
10322 
10323 	tp->mac_mode |= MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
10324 			MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE |
10325 			MAC_MODE_FHDE_ENABLE;
10326 	if (tg3_flag(tp, ENABLE_APE))
10327 		tp->mac_mode |= MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
10328 	if (!tg3_flag(tp, 5705_PLUS) &&
10329 	    !(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
10330 	    tg3_asic_rev(tp) != ASIC_REV_5700)
10331 		tp->mac_mode |= MAC_MODE_LINK_POLARITY;
10332 	tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
10333 	udelay(40);
10334 
10335 	/* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
10336 	 * If TG3_FLAG_IS_NIC is zero, we should read the
10337 	 * register to preserve the GPIO settings for LOMs. The GPIOs,
10338 	 * whether used as inputs or outputs, are set by boot code after
10339 	 * reset.
10340 	 */
10341 	if (!tg3_flag(tp, IS_NIC)) {
10342 		u32 gpio_mask;
10343 
10344 		gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE1 |
10345 			    GRC_LCLCTRL_GPIO_OE2 | GRC_LCLCTRL_GPIO_OUTPUT0 |
10346 			    GRC_LCLCTRL_GPIO_OUTPUT1 | GRC_LCLCTRL_GPIO_OUTPUT2;
10347 
10348 		if (tg3_asic_rev(tp) == ASIC_REV_5752)
10349 			gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
10350 				     GRC_LCLCTRL_GPIO_OUTPUT3;
10351 
10352 		if (tg3_asic_rev(tp) == ASIC_REV_5755)
10353 			gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
10354 
10355 		tp->grc_local_ctrl &= ~gpio_mask;
10356 		tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
10357 
10358 		/* GPIO1 must be driven high for eeprom write protect */
10359 		if (tg3_flag(tp, EEPROM_WRITE_PROT))
10360 			tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
10361 					       GRC_LCLCTRL_GPIO_OUTPUT1);
10362 	}
10363 	tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10364 	udelay(100);
10365 
10366 	if (tg3_flag(tp, USING_MSIX)) {
10367 		val = tr32(MSGINT_MODE);
10368 		val |= MSGINT_MODE_ENABLE;
10369 		if (tp->irq_cnt > 1)
10370 			val |= MSGINT_MODE_MULTIVEC_EN;
10371 		if (!tg3_flag(tp, 1SHOT_MSI))
10372 			val |= MSGINT_MODE_ONE_SHOT_DISABLE;
10373 		tw32(MSGINT_MODE, val);
10374 	}
10375 
10376 	if (!tg3_flag(tp, 5705_PLUS)) {
10377 		tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
10378 		udelay(40);
10379 	}
10380 
10381 	val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
10382 	       WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
10383 	       WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
10384 	       WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
10385 	       WDMAC_MODE_LNGREAD_ENAB);
10386 
10387 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
10388 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
10389 		if (tg3_flag(tp, TSO_CAPABLE) &&
10390 		    (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A1 ||
10391 		     tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A2)) {
10392 			/* nothing */
10393 		} else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
10394 			   !tg3_flag(tp, IS_5788)) {
10395 			val |= WDMAC_MODE_RX_ACCEL;
10396 		}
10397 	}
10398 
10399 	/* Enable host coalescing bug fix */
10400 	if (tg3_flag(tp, 5755_PLUS))
10401 		val |= WDMAC_MODE_STATUS_TAG_FIX;
10402 
10403 	if (tg3_asic_rev(tp) == ASIC_REV_5785)
10404 		val |= WDMAC_MODE_BURST_ALL_DATA;
10405 
10406 	tw32_f(WDMAC_MODE, val);
10407 	udelay(40);
10408 
10409 	if (tg3_flag(tp, PCIX_MODE)) {
10410 		u16 pcix_cmd;
10411 
10412 		pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
10413 				     &pcix_cmd);
10414 		if (tg3_asic_rev(tp) == ASIC_REV_5703) {
10415 			pcix_cmd &= ~PCI_X_CMD_MAX_READ;
10416 			pcix_cmd |= PCI_X_CMD_READ_2K;
10417 		} else if (tg3_asic_rev(tp) == ASIC_REV_5704) {
10418 			pcix_cmd &= ~(PCI_X_CMD_MAX_SPLIT | PCI_X_CMD_MAX_READ);
10419 			pcix_cmd |= PCI_X_CMD_READ_2K;
10420 		}
10421 		pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
10422 				      pcix_cmd);
10423 	}
10424 
10425 	tw32_f(RDMAC_MODE, rdmac_mode);
10426 	udelay(40);
10427 
10428 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
10429 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
10430 		for (i = 0; i < TG3_NUM_RDMA_CHANNELS; i++) {
10431 			if (tr32(TG3_RDMA_LENGTH + (i << 2)) > TG3_MAX_MTU(tp))
10432 				break;
10433 		}
10434 		if (i < TG3_NUM_RDMA_CHANNELS) {
10435 			val = tr32(TG3_LSO_RD_DMA_CRPTEN_CTRL);
10436 			val |= tg3_lso_rd_dma_workaround_bit(tp);
10437 			tw32(TG3_LSO_RD_DMA_CRPTEN_CTRL, val);
10438 			tg3_flag_set(tp, 5719_5720_RDMA_BUG);
10439 		}
10440 	}
10441 
10442 	tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
10443 	if (!tg3_flag(tp, 5705_PLUS))
10444 		tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
10445 
10446 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
10447 		tw32(SNDDATAC_MODE,
10448 		     SNDDATAC_MODE_ENABLE | SNDDATAC_MODE_CDELAY);
10449 	else
10450 		tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
10451 
10452 	tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
10453 	tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
10454 	val = RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ;
10455 	if (tg3_flag(tp, LRG_PROD_RING_CAP))
10456 		val |= RCVDBDI_MODE_LRG_RING_SZ;
10457 	tw32(RCVDBDI_MODE, val);
10458 	tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
10459 	if (tg3_flag(tp, HW_TSO_1) ||
10460 	    tg3_flag(tp, HW_TSO_2) ||
10461 	    tg3_flag(tp, HW_TSO_3))
10462 		tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
10463 	val = SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE;
10464 	if (tg3_flag(tp, ENABLE_TSS))
10465 		val |= SNDBDI_MODE_MULTI_TXQ_EN;
10466 	tw32(SNDBDI_MODE, val);
10467 	tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
10468 
10469 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0) {
10470 		err = tg3_load_5701_a0_firmware_fix(tp);
10471 		if (err)
10472 			return err;
10473 	}
10474 
10475 	if (tg3_asic_rev(tp) == ASIC_REV_57766) {
10476 		/* Ignore any errors for the firmware download. If download
10477 		 * fails, the device will operate with EEE disabled
10478 		 */
10479 		tg3_load_57766_firmware(tp);
10480 	}
10481 
10482 	if (tg3_flag(tp, TSO_CAPABLE)) {
10483 		err = tg3_load_tso_firmware(tp);
10484 		if (err)
10485 			return err;
10486 	}
10487 
10488 	tp->tx_mode = TX_MODE_ENABLE;
10489 
10490 	if (tg3_flag(tp, 5755_PLUS) ||
10491 	    tg3_asic_rev(tp) == ASIC_REV_5906)
10492 		tp->tx_mode |= TX_MODE_MBUF_LOCKUP_FIX;
10493 
10494 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10495 	    tg3_asic_rev(tp) == ASIC_REV_5762) {
10496 		val = TX_MODE_JMB_FRM_LEN | TX_MODE_CNT_DN_MODE;
10497 		tp->tx_mode &= ~val;
10498 		tp->tx_mode |= tr32(MAC_TX_MODE) & val;
10499 	}
10500 
10501 	tw32_f(MAC_TX_MODE, tp->tx_mode);
10502 	udelay(100);
10503 
10504 	if (tg3_flag(tp, ENABLE_RSS)) {
10505 		tg3_rss_write_indir_tbl(tp);
10506 
10507 		/* Setup the "secret" hash key. */
10508 		tw32(MAC_RSS_HASH_KEY_0, 0x5f865437);
10509 		tw32(MAC_RSS_HASH_KEY_1, 0xe4ac62cc);
10510 		tw32(MAC_RSS_HASH_KEY_2, 0x50103a45);
10511 		tw32(MAC_RSS_HASH_KEY_3, 0x36621985);
10512 		tw32(MAC_RSS_HASH_KEY_4, 0xbf14c0e8);
10513 		tw32(MAC_RSS_HASH_KEY_5, 0x1bc27a1e);
10514 		tw32(MAC_RSS_HASH_KEY_6, 0x84f4b556);
10515 		tw32(MAC_RSS_HASH_KEY_7, 0x094ea6fe);
10516 		tw32(MAC_RSS_HASH_KEY_8, 0x7dda01e7);
10517 		tw32(MAC_RSS_HASH_KEY_9, 0xc04d7481);
10518 	}
10519 
10520 	tp->rx_mode = RX_MODE_ENABLE;
10521 	if (tg3_flag(tp, 5755_PLUS))
10522 		tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
10523 
10524 	if (tg3_asic_rev(tp) == ASIC_REV_5762)
10525 		tp->rx_mode |= RX_MODE_IPV4_FRAG_FIX;
10526 
10527 	if (tg3_flag(tp, ENABLE_RSS))
10528 		tp->rx_mode |= RX_MODE_RSS_ENABLE |
10529 			       RX_MODE_RSS_ITBL_HASH_BITS_7 |
10530 			       RX_MODE_RSS_IPV6_HASH_EN |
10531 			       RX_MODE_RSS_TCP_IPV6_HASH_EN |
10532 			       RX_MODE_RSS_IPV4_HASH_EN |
10533 			       RX_MODE_RSS_TCP_IPV4_HASH_EN;
10534 
10535 	tw32_f(MAC_RX_MODE, tp->rx_mode);
10536 	udelay(10);
10537 
10538 	tw32(MAC_LED_CTRL, tp->led_ctrl);
10539 
10540 	tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
10541 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
10542 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
10543 		udelay(10);
10544 	}
10545 	tw32_f(MAC_RX_MODE, tp->rx_mode);
10546 	udelay(10);
10547 
10548 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
10549 		if ((tg3_asic_rev(tp) == ASIC_REV_5704) &&
10550 		    !(tp->phy_flags & TG3_PHYFLG_SERDES_PREEMPHASIS)) {
10551 			/* Set drive transmission level to 1.2V  */
10552 			/* only if the signal pre-emphasis bit is not set  */
10553 			val = tr32(MAC_SERDES_CFG);
10554 			val &= 0xfffff000;
10555 			val |= 0x880;
10556 			tw32(MAC_SERDES_CFG, val);
10557 		}
10558 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A1)
10559 			tw32(MAC_SERDES_CFG, 0x616000);
10560 	}
10561 
10562 	/* Prevent chip from dropping frames when flow control
10563 	 * is enabled.
10564 	 */
10565 	if (tg3_flag(tp, 57765_CLASS))
10566 		val = 1;
10567 	else
10568 		val = 2;
10569 	tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, val);
10570 
10571 	if (tg3_asic_rev(tp) == ASIC_REV_5704 &&
10572 	    (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
10573 		/* Use hardware link auto-negotiation */
10574 		tg3_flag_set(tp, HW_AUTONEG);
10575 	}
10576 
10577 	if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
10578 	    tg3_asic_rev(tp) == ASIC_REV_5714) {
10579 		u32 tmp;
10580 
10581 		tmp = tr32(SERDES_RX_CTRL);
10582 		tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
10583 		tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
10584 		tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
10585 		tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10586 	}
10587 
10588 	if (!tg3_flag(tp, USE_PHYLIB)) {
10589 		if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
10590 			tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
10591 
10592 		err = tg3_setup_phy(tp, false);
10593 		if (err)
10594 			return err;
10595 
10596 		if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
10597 		    !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
10598 			u32 tmp;
10599 
10600 			/* Clear CRC stats. */
10601 			if (!tg3_readphy(tp, MII_TG3_TEST1, &tmp)) {
10602 				tg3_writephy(tp, MII_TG3_TEST1,
10603 					     tmp | MII_TG3_TEST1_CRC_EN);
10604 				tg3_readphy(tp, MII_TG3_RXR_COUNTERS, &tmp);
10605 			}
10606 		}
10607 	}
10608 
10609 	__tg3_set_rx_mode(tp->dev);
10610 
10611 	/* Initialize receive rules. */
10612 	tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
10613 	tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
10614 	tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
10615 	tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
10616 
10617 	if (tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS))
10618 		limit = 8;
10619 	else
10620 		limit = 16;
10621 	if (tg3_flag(tp, ENABLE_ASF))
10622 		limit -= 4;
10623 	switch (limit) {
10624 	case 16:
10625 		tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
10626 	case 15:
10627 		tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
10628 	case 14:
10629 		tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
10630 	case 13:
10631 		tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
10632 	case 12:
10633 		tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
10634 	case 11:
10635 		tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
10636 	case 10:
10637 		tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
10638 	case 9:
10639 		tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
10640 	case 8:
10641 		tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
10642 	case 7:
10643 		tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
10644 	case 6:
10645 		tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
10646 	case 5:
10647 		tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
10648 	case 4:
10649 		/* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
10650 	case 3:
10651 		/* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
10652 	case 2:
10653 	case 1:
10654 
10655 	default:
10656 		break;
10657 	}
10658 
10659 	if (tg3_flag(tp, ENABLE_APE))
10660 		/* Write our heartbeat update interval to APE. */
10661 		tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
10662 				APE_HOST_HEARTBEAT_INT_DISABLE);
10663 
10664 	tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
10665 
10666 	return 0;
10667 }
10668 
10669 /* Called at device open time to get the chip ready for
10670  * packet processing.  Invoked with tp->lock held.
10671  */
10672 static int tg3_init_hw(struct tg3 *tp, bool reset_phy)
10673 {
10674 	/* Chip may have been just powered on. If so, the boot code may still
10675 	 * be running initialization. Wait for it to finish to avoid races in
10676 	 * accessing the hardware.
10677 	 */
10678 	tg3_enable_register_access(tp);
10679 	tg3_poll_fw(tp);
10680 
10681 	tg3_switch_clocks(tp);
10682 
10683 	tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
10684 
10685 	return tg3_reset_hw(tp, reset_phy);
10686 }
10687 
10688 static void tg3_sd_scan_scratchpad(struct tg3 *tp, struct tg3_ocir *ocir)
10689 {
10690 	int i;
10691 
10692 	for (i = 0; i < TG3_SD_NUM_RECS; i++, ocir++) {
10693 		u32 off = i * TG3_OCIR_LEN, len = TG3_OCIR_LEN;
10694 
10695 		tg3_ape_scratchpad_read(tp, (u32 *) ocir, off, len);
10696 		off += len;
10697 
10698 		if (ocir->signature != TG3_OCIR_SIG_MAGIC ||
10699 		    !(ocir->version_flags & TG3_OCIR_FLAG_ACTIVE))
10700 			memset(ocir, 0, TG3_OCIR_LEN);
10701 	}
10702 }
10703 
10704 /* sysfs attributes for hwmon */
10705 static ssize_t tg3_show_temp(struct device *dev,
10706 			     struct device_attribute *devattr, char *buf)
10707 {
10708 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
10709 	struct tg3 *tp = dev_get_drvdata(dev);
10710 	u32 temperature;
10711 
10712 	spin_lock_bh(&tp->lock);
10713 	tg3_ape_scratchpad_read(tp, &temperature, attr->index,
10714 				sizeof(temperature));
10715 	spin_unlock_bh(&tp->lock);
10716 	return sprintf(buf, "%u\n", temperature);
10717 }
10718 
10719 
10720 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, tg3_show_temp, NULL,
10721 			  TG3_TEMP_SENSOR_OFFSET);
10722 static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, tg3_show_temp, NULL,
10723 			  TG3_TEMP_CAUTION_OFFSET);
10724 static SENSOR_DEVICE_ATTR(temp1_max, S_IRUGO, tg3_show_temp, NULL,
10725 			  TG3_TEMP_MAX_OFFSET);
10726 
10727 static struct attribute *tg3_attrs[] = {
10728 	&sensor_dev_attr_temp1_input.dev_attr.attr,
10729 	&sensor_dev_attr_temp1_crit.dev_attr.attr,
10730 	&sensor_dev_attr_temp1_max.dev_attr.attr,
10731 	NULL
10732 };
10733 ATTRIBUTE_GROUPS(tg3);
10734 
10735 static void tg3_hwmon_close(struct tg3 *tp)
10736 {
10737 	if (tp->hwmon_dev) {
10738 		hwmon_device_unregister(tp->hwmon_dev);
10739 		tp->hwmon_dev = NULL;
10740 	}
10741 }
10742 
10743 static void tg3_hwmon_open(struct tg3 *tp)
10744 {
10745 	int i;
10746 	u32 size = 0;
10747 	struct pci_dev *pdev = tp->pdev;
10748 	struct tg3_ocir ocirs[TG3_SD_NUM_RECS];
10749 
10750 	tg3_sd_scan_scratchpad(tp, ocirs);
10751 
10752 	for (i = 0; i < TG3_SD_NUM_RECS; i++) {
10753 		if (!ocirs[i].src_data_length)
10754 			continue;
10755 
10756 		size += ocirs[i].src_hdr_length;
10757 		size += ocirs[i].src_data_length;
10758 	}
10759 
10760 	if (!size)
10761 		return;
10762 
10763 	tp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, "tg3",
10764 							  tp, tg3_groups);
10765 	if (IS_ERR(tp->hwmon_dev)) {
10766 		tp->hwmon_dev = NULL;
10767 		dev_err(&pdev->dev, "Cannot register hwmon device, aborting\n");
10768 	}
10769 }
10770 
10771 
10772 #define TG3_STAT_ADD32(PSTAT, REG) \
10773 do {	u32 __val = tr32(REG); \
10774 	(PSTAT)->low += __val; \
10775 	if ((PSTAT)->low < __val) \
10776 		(PSTAT)->high += 1; \
10777 } while (0)
10778 
10779 static void tg3_periodic_fetch_stats(struct tg3 *tp)
10780 {
10781 	struct tg3_hw_stats *sp = tp->hw_stats;
10782 
10783 	if (!tp->link_up)
10784 		return;
10785 
10786 	TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
10787 	TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
10788 	TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
10789 	TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
10790 	TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
10791 	TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
10792 	TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
10793 	TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
10794 	TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
10795 	TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
10796 	TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
10797 	TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
10798 	TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
10799 	if (unlikely(tg3_flag(tp, 5719_5720_RDMA_BUG) &&
10800 		     (sp->tx_ucast_packets.low + sp->tx_mcast_packets.low +
10801 		      sp->tx_bcast_packets.low) > TG3_NUM_RDMA_CHANNELS)) {
10802 		u32 val;
10803 
10804 		val = tr32(TG3_LSO_RD_DMA_CRPTEN_CTRL);
10805 		val &= ~tg3_lso_rd_dma_workaround_bit(tp);
10806 		tw32(TG3_LSO_RD_DMA_CRPTEN_CTRL, val);
10807 		tg3_flag_clear(tp, 5719_5720_RDMA_BUG);
10808 	}
10809 
10810 	TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
10811 	TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
10812 	TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
10813 	TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
10814 	TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
10815 	TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
10816 	TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
10817 	TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
10818 	TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
10819 	TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
10820 	TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
10821 	TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
10822 	TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
10823 	TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
10824 
10825 	TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
10826 	if (tg3_asic_rev(tp) != ASIC_REV_5717 &&
10827 	    tg3_asic_rev(tp) != ASIC_REV_5762 &&
10828 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5719_A0 &&
10829 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5720_A0) {
10830 		TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
10831 	} else {
10832 		u32 val = tr32(HOSTCC_FLOW_ATTN);
10833 		val = (val & HOSTCC_FLOW_ATTN_MBUF_LWM) ? 1 : 0;
10834 		if (val) {
10835 			tw32(HOSTCC_FLOW_ATTN, HOSTCC_FLOW_ATTN_MBUF_LWM);
10836 			sp->rx_discards.low += val;
10837 			if (sp->rx_discards.low < val)
10838 				sp->rx_discards.high += 1;
10839 		}
10840 		sp->mbuf_lwm_thresh_hit = sp->rx_discards;
10841 	}
10842 	TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
10843 }
10844 
10845 static void tg3_chk_missed_msi(struct tg3 *tp)
10846 {
10847 	u32 i;
10848 
10849 	for (i = 0; i < tp->irq_cnt; i++) {
10850 		struct tg3_napi *tnapi = &tp->napi[i];
10851 
10852 		if (tg3_has_work(tnapi)) {
10853 			if (tnapi->last_rx_cons == tnapi->rx_rcb_ptr &&
10854 			    tnapi->last_tx_cons == tnapi->tx_cons) {
10855 				if (tnapi->chk_msi_cnt < 1) {
10856 					tnapi->chk_msi_cnt++;
10857 					return;
10858 				}
10859 				tg3_msi(0, tnapi);
10860 			}
10861 		}
10862 		tnapi->chk_msi_cnt = 0;
10863 		tnapi->last_rx_cons = tnapi->rx_rcb_ptr;
10864 		tnapi->last_tx_cons = tnapi->tx_cons;
10865 	}
10866 }
10867 
10868 static void tg3_timer(unsigned long __opaque)
10869 {
10870 	struct tg3 *tp = (struct tg3 *) __opaque;
10871 
10872 	if (tp->irq_sync || tg3_flag(tp, RESET_TASK_PENDING))
10873 		goto restart_timer;
10874 
10875 	spin_lock(&tp->lock);
10876 
10877 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
10878 	    tg3_flag(tp, 57765_CLASS))
10879 		tg3_chk_missed_msi(tp);
10880 
10881 	if (tg3_flag(tp, FLUSH_POSTED_WRITES)) {
10882 		/* BCM4785: Flush posted writes from GbE to host memory. */
10883 		tr32(HOSTCC_MODE);
10884 	}
10885 
10886 	if (!tg3_flag(tp, TAGGED_STATUS)) {
10887 		/* All of this garbage is because when using non-tagged
10888 		 * IRQ status the mailbox/status_block protocol the chip
10889 		 * uses with the cpu is race prone.
10890 		 */
10891 		if (tp->napi[0].hw_status->status & SD_STATUS_UPDATED) {
10892 			tw32(GRC_LOCAL_CTRL,
10893 			     tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
10894 		} else {
10895 			tw32(HOSTCC_MODE, tp->coalesce_mode |
10896 			     HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW);
10897 		}
10898 
10899 		if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
10900 			spin_unlock(&tp->lock);
10901 			tg3_reset_task_schedule(tp);
10902 			goto restart_timer;
10903 		}
10904 	}
10905 
10906 	/* This part only runs once per second. */
10907 	if (!--tp->timer_counter) {
10908 		if (tg3_flag(tp, 5705_PLUS))
10909 			tg3_periodic_fetch_stats(tp);
10910 
10911 		if (tp->setlpicnt && !--tp->setlpicnt)
10912 			tg3_phy_eee_enable(tp);
10913 
10914 		if (tg3_flag(tp, USE_LINKCHG_REG)) {
10915 			u32 mac_stat;
10916 			int phy_event;
10917 
10918 			mac_stat = tr32(MAC_STATUS);
10919 
10920 			phy_event = 0;
10921 			if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT) {
10922 				if (mac_stat & MAC_STATUS_MI_INTERRUPT)
10923 					phy_event = 1;
10924 			} else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
10925 				phy_event = 1;
10926 
10927 			if (phy_event)
10928 				tg3_setup_phy(tp, false);
10929 		} else if (tg3_flag(tp, POLL_SERDES)) {
10930 			u32 mac_stat = tr32(MAC_STATUS);
10931 			int need_setup = 0;
10932 
10933 			if (tp->link_up &&
10934 			    (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
10935 				need_setup = 1;
10936 			}
10937 			if (!tp->link_up &&
10938 			    (mac_stat & (MAC_STATUS_PCS_SYNCED |
10939 					 MAC_STATUS_SIGNAL_DET))) {
10940 				need_setup = 1;
10941 			}
10942 			if (need_setup) {
10943 				if (!tp->serdes_counter) {
10944 					tw32_f(MAC_MODE,
10945 					     (tp->mac_mode &
10946 					      ~MAC_MODE_PORT_MODE_MASK));
10947 					udelay(40);
10948 					tw32_f(MAC_MODE, tp->mac_mode);
10949 					udelay(40);
10950 				}
10951 				tg3_setup_phy(tp, false);
10952 			}
10953 		} else if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
10954 			   tg3_flag(tp, 5780_CLASS)) {
10955 			tg3_serdes_parallel_detect(tp);
10956 		} else if (tg3_flag(tp, POLL_CPMU_LINK)) {
10957 			u32 cpmu = tr32(TG3_CPMU_STATUS);
10958 			bool link_up = !((cpmu & TG3_CPMU_STATUS_LINK_MASK) ==
10959 					 TG3_CPMU_STATUS_LINK_MASK);
10960 
10961 			if (link_up != tp->link_up)
10962 				tg3_setup_phy(tp, false);
10963 		}
10964 
10965 		tp->timer_counter = tp->timer_multiplier;
10966 	}
10967 
10968 	/* Heartbeat is only sent once every 2 seconds.
10969 	 *
10970 	 * The heartbeat is to tell the ASF firmware that the host
10971 	 * driver is still alive.  In the event that the OS crashes,
10972 	 * ASF needs to reset the hardware to free up the FIFO space
10973 	 * that may be filled with rx packets destined for the host.
10974 	 * If the FIFO is full, ASF will no longer function properly.
10975 	 *
10976 	 * Unintended resets have been reported on real time kernels
10977 	 * where the timer doesn't run on time.  Netpoll will also have
10978 	 * same problem.
10979 	 *
10980 	 * The new FWCMD_NICDRV_ALIVE3 command tells the ASF firmware
10981 	 * to check the ring condition when the heartbeat is expiring
10982 	 * before doing the reset.  This will prevent most unintended
10983 	 * resets.
10984 	 */
10985 	if (!--tp->asf_counter) {
10986 		if (tg3_flag(tp, ENABLE_ASF) && !tg3_flag(tp, ENABLE_APE)) {
10987 			tg3_wait_for_event_ack(tp);
10988 
10989 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
10990 				      FWCMD_NICDRV_ALIVE3);
10991 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
10992 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX,
10993 				      TG3_FW_UPDATE_TIMEOUT_SEC);
10994 
10995 			tg3_generate_fw_event(tp);
10996 		}
10997 		tp->asf_counter = tp->asf_multiplier;
10998 	}
10999 
11000 	spin_unlock(&tp->lock);
11001 
11002 restart_timer:
11003 	tp->timer.expires = jiffies + tp->timer_offset;
11004 	add_timer(&tp->timer);
11005 }
11006 
11007 static void tg3_timer_init(struct tg3 *tp)
11008 {
11009 	if (tg3_flag(tp, TAGGED_STATUS) &&
11010 	    tg3_asic_rev(tp) != ASIC_REV_5717 &&
11011 	    !tg3_flag(tp, 57765_CLASS))
11012 		tp->timer_offset = HZ;
11013 	else
11014 		tp->timer_offset = HZ / 10;
11015 
11016 	BUG_ON(tp->timer_offset > HZ);
11017 
11018 	tp->timer_multiplier = (HZ / tp->timer_offset);
11019 	tp->asf_multiplier = (HZ / tp->timer_offset) *
11020 			     TG3_FW_UPDATE_FREQ_SEC;
11021 
11022 	init_timer(&tp->timer);
11023 	tp->timer.data = (unsigned long) tp;
11024 	tp->timer.function = tg3_timer;
11025 }
11026 
11027 static void tg3_timer_start(struct tg3 *tp)
11028 {
11029 	tp->asf_counter   = tp->asf_multiplier;
11030 	tp->timer_counter = tp->timer_multiplier;
11031 
11032 	tp->timer.expires = jiffies + tp->timer_offset;
11033 	add_timer(&tp->timer);
11034 }
11035 
11036 static void tg3_timer_stop(struct tg3 *tp)
11037 {
11038 	del_timer_sync(&tp->timer);
11039 }
11040 
11041 /* Restart hardware after configuration changes, self-test, etc.
11042  * Invoked with tp->lock held.
11043  */
11044 static int tg3_restart_hw(struct tg3 *tp, bool reset_phy)
11045 	__releases(tp->lock)
11046 	__acquires(tp->lock)
11047 {
11048 	int err;
11049 
11050 	err = tg3_init_hw(tp, reset_phy);
11051 	if (err) {
11052 		netdev_err(tp->dev,
11053 			   "Failed to re-initialize device, aborting\n");
11054 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11055 		tg3_full_unlock(tp);
11056 		tg3_timer_stop(tp);
11057 		tp->irq_sync = 0;
11058 		tg3_napi_enable(tp);
11059 		dev_close(tp->dev);
11060 		tg3_full_lock(tp, 0);
11061 	}
11062 	return err;
11063 }
11064 
11065 static void tg3_reset_task(struct work_struct *work)
11066 {
11067 	struct tg3 *tp = container_of(work, struct tg3, reset_task);
11068 	int err;
11069 
11070 	tg3_full_lock(tp, 0);
11071 
11072 	if (!netif_running(tp->dev)) {
11073 		tg3_flag_clear(tp, RESET_TASK_PENDING);
11074 		tg3_full_unlock(tp);
11075 		return;
11076 	}
11077 
11078 	tg3_full_unlock(tp);
11079 
11080 	tg3_phy_stop(tp);
11081 
11082 	tg3_netif_stop(tp);
11083 
11084 	tg3_full_lock(tp, 1);
11085 
11086 	if (tg3_flag(tp, TX_RECOVERY_PENDING)) {
11087 		tp->write32_tx_mbox = tg3_write32_tx_mbox;
11088 		tp->write32_rx_mbox = tg3_write_flush_reg32;
11089 		tg3_flag_set(tp, MBOX_WRITE_REORDER);
11090 		tg3_flag_clear(tp, TX_RECOVERY_PENDING);
11091 	}
11092 
11093 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
11094 	err = tg3_init_hw(tp, true);
11095 	if (err)
11096 		goto out;
11097 
11098 	tg3_netif_start(tp);
11099 
11100 out:
11101 	tg3_full_unlock(tp);
11102 
11103 	if (!err)
11104 		tg3_phy_start(tp);
11105 
11106 	tg3_flag_clear(tp, RESET_TASK_PENDING);
11107 }
11108 
11109 static int tg3_request_irq(struct tg3 *tp, int irq_num)
11110 {
11111 	irq_handler_t fn;
11112 	unsigned long flags;
11113 	char *name;
11114 	struct tg3_napi *tnapi = &tp->napi[irq_num];
11115 
11116 	if (tp->irq_cnt == 1)
11117 		name = tp->dev->name;
11118 	else {
11119 		name = &tnapi->irq_lbl[0];
11120 		if (tnapi->tx_buffers && tnapi->rx_rcb)
11121 			snprintf(name, IFNAMSIZ,
11122 				 "%s-txrx-%d", tp->dev->name, irq_num);
11123 		else if (tnapi->tx_buffers)
11124 			snprintf(name, IFNAMSIZ,
11125 				 "%s-tx-%d", tp->dev->name, irq_num);
11126 		else if (tnapi->rx_rcb)
11127 			snprintf(name, IFNAMSIZ,
11128 				 "%s-rx-%d", tp->dev->name, irq_num);
11129 		else
11130 			snprintf(name, IFNAMSIZ,
11131 				 "%s-%d", tp->dev->name, irq_num);
11132 		name[IFNAMSIZ-1] = 0;
11133 	}
11134 
11135 	if (tg3_flag(tp, USING_MSI) || tg3_flag(tp, USING_MSIX)) {
11136 		fn = tg3_msi;
11137 		if (tg3_flag(tp, 1SHOT_MSI))
11138 			fn = tg3_msi_1shot;
11139 		flags = 0;
11140 	} else {
11141 		fn = tg3_interrupt;
11142 		if (tg3_flag(tp, TAGGED_STATUS))
11143 			fn = tg3_interrupt_tagged;
11144 		flags = IRQF_SHARED;
11145 	}
11146 
11147 	return request_irq(tnapi->irq_vec, fn, flags, name, tnapi);
11148 }
11149 
11150 static int tg3_test_interrupt(struct tg3 *tp)
11151 {
11152 	struct tg3_napi *tnapi = &tp->napi[0];
11153 	struct net_device *dev = tp->dev;
11154 	int err, i, intr_ok = 0;
11155 	u32 val;
11156 
11157 	if (!netif_running(dev))
11158 		return -ENODEV;
11159 
11160 	tg3_disable_ints(tp);
11161 
11162 	free_irq(tnapi->irq_vec, tnapi);
11163 
11164 	/*
11165 	 * Turn off MSI one shot mode.  Otherwise this test has no
11166 	 * observable way to know whether the interrupt was delivered.
11167 	 */
11168 	if (tg3_flag(tp, 57765_PLUS)) {
11169 		val = tr32(MSGINT_MODE) | MSGINT_MODE_ONE_SHOT_DISABLE;
11170 		tw32(MSGINT_MODE, val);
11171 	}
11172 
11173 	err = request_irq(tnapi->irq_vec, tg3_test_isr,
11174 			  IRQF_SHARED, dev->name, tnapi);
11175 	if (err)
11176 		return err;
11177 
11178 	tnapi->hw_status->status &= ~SD_STATUS_UPDATED;
11179 	tg3_enable_ints(tp);
11180 
11181 	tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
11182 	       tnapi->coal_now);
11183 
11184 	for (i = 0; i < 5; i++) {
11185 		u32 int_mbox, misc_host_ctrl;
11186 
11187 		int_mbox = tr32_mailbox(tnapi->int_mbox);
11188 		misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
11189 
11190 		if ((int_mbox != 0) ||
11191 		    (misc_host_ctrl & MISC_HOST_CTRL_MASK_PCI_INT)) {
11192 			intr_ok = 1;
11193 			break;
11194 		}
11195 
11196 		if (tg3_flag(tp, 57765_PLUS) &&
11197 		    tnapi->hw_status->status_tag != tnapi->last_tag)
11198 			tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
11199 
11200 		msleep(10);
11201 	}
11202 
11203 	tg3_disable_ints(tp);
11204 
11205 	free_irq(tnapi->irq_vec, tnapi);
11206 
11207 	err = tg3_request_irq(tp, 0);
11208 
11209 	if (err)
11210 		return err;
11211 
11212 	if (intr_ok) {
11213 		/* Reenable MSI one shot mode. */
11214 		if (tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, 1SHOT_MSI)) {
11215 			val = tr32(MSGINT_MODE) & ~MSGINT_MODE_ONE_SHOT_DISABLE;
11216 			tw32(MSGINT_MODE, val);
11217 		}
11218 		return 0;
11219 	}
11220 
11221 	return -EIO;
11222 }
11223 
11224 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
11225  * successfully restored
11226  */
11227 static int tg3_test_msi(struct tg3 *tp)
11228 {
11229 	int err;
11230 	u16 pci_cmd;
11231 
11232 	if (!tg3_flag(tp, USING_MSI))
11233 		return 0;
11234 
11235 	/* Turn off SERR reporting in case MSI terminates with Master
11236 	 * Abort.
11237 	 */
11238 	pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11239 	pci_write_config_word(tp->pdev, PCI_COMMAND,
11240 			      pci_cmd & ~PCI_COMMAND_SERR);
11241 
11242 	err = tg3_test_interrupt(tp);
11243 
11244 	pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11245 
11246 	if (!err)
11247 		return 0;
11248 
11249 	/* other failures */
11250 	if (err != -EIO)
11251 		return err;
11252 
11253 	/* MSI test failed, go back to INTx mode */
11254 	netdev_warn(tp->dev, "No interrupt was generated using MSI. Switching "
11255 		    "to INTx mode. Please report this failure to the PCI "
11256 		    "maintainer and include system chipset information\n");
11257 
11258 	free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
11259 
11260 	pci_disable_msi(tp->pdev);
11261 
11262 	tg3_flag_clear(tp, USING_MSI);
11263 	tp->napi[0].irq_vec = tp->pdev->irq;
11264 
11265 	err = tg3_request_irq(tp, 0);
11266 	if (err)
11267 		return err;
11268 
11269 	/* Need to reset the chip because the MSI cycle may have terminated
11270 	 * with Master Abort.
11271 	 */
11272 	tg3_full_lock(tp, 1);
11273 
11274 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11275 	err = tg3_init_hw(tp, true);
11276 
11277 	tg3_full_unlock(tp);
11278 
11279 	if (err)
11280 		free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
11281 
11282 	return err;
11283 }
11284 
11285 static int tg3_request_firmware(struct tg3 *tp)
11286 {
11287 	const struct tg3_firmware_hdr *fw_hdr;
11288 
11289 	if (request_firmware(&tp->fw, tp->fw_needed, &tp->pdev->dev)) {
11290 		netdev_err(tp->dev, "Failed to load firmware \"%s\"\n",
11291 			   tp->fw_needed);
11292 		return -ENOENT;
11293 	}
11294 
11295 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
11296 
11297 	/* Firmware blob starts with version numbers, followed by
11298 	 * start address and _full_ length including BSS sections
11299 	 * (which must be longer than the actual data, of course
11300 	 */
11301 
11302 	tp->fw_len = be32_to_cpu(fw_hdr->len);	/* includes bss */
11303 	if (tp->fw_len < (tp->fw->size - TG3_FW_HDR_LEN)) {
11304 		netdev_err(tp->dev, "bogus length %d in \"%s\"\n",
11305 			   tp->fw_len, tp->fw_needed);
11306 		release_firmware(tp->fw);
11307 		tp->fw = NULL;
11308 		return -EINVAL;
11309 	}
11310 
11311 	/* We no longer need firmware; we have it. */
11312 	tp->fw_needed = NULL;
11313 	return 0;
11314 }
11315 
11316 static u32 tg3_irq_count(struct tg3 *tp)
11317 {
11318 	u32 irq_cnt = max(tp->rxq_cnt, tp->txq_cnt);
11319 
11320 	if (irq_cnt > 1) {
11321 		/* We want as many rx rings enabled as there are cpus.
11322 		 * In multiqueue MSI-X mode, the first MSI-X vector
11323 		 * only deals with link interrupts, etc, so we add
11324 		 * one to the number of vectors we are requesting.
11325 		 */
11326 		irq_cnt = min_t(unsigned, irq_cnt + 1, tp->irq_max);
11327 	}
11328 
11329 	return irq_cnt;
11330 }
11331 
11332 static bool tg3_enable_msix(struct tg3 *tp)
11333 {
11334 	int i, rc;
11335 	struct msix_entry msix_ent[TG3_IRQ_MAX_VECS];
11336 
11337 	tp->txq_cnt = tp->txq_req;
11338 	tp->rxq_cnt = tp->rxq_req;
11339 	if (!tp->rxq_cnt)
11340 		tp->rxq_cnt = netif_get_num_default_rss_queues();
11341 	if (tp->rxq_cnt > tp->rxq_max)
11342 		tp->rxq_cnt = tp->rxq_max;
11343 
11344 	/* Disable multiple TX rings by default.  Simple round-robin hardware
11345 	 * scheduling of the TX rings can cause starvation of rings with
11346 	 * small packets when other rings have TSO or jumbo packets.
11347 	 */
11348 	if (!tp->txq_req)
11349 		tp->txq_cnt = 1;
11350 
11351 	tp->irq_cnt = tg3_irq_count(tp);
11352 
11353 	for (i = 0; i < tp->irq_max; i++) {
11354 		msix_ent[i].entry  = i;
11355 		msix_ent[i].vector = 0;
11356 	}
11357 
11358 	rc = pci_enable_msix_range(tp->pdev, msix_ent, 1, tp->irq_cnt);
11359 	if (rc < 0) {
11360 		return false;
11361 	} else if (rc < tp->irq_cnt) {
11362 		netdev_notice(tp->dev, "Requested %d MSI-X vectors, received %d\n",
11363 			      tp->irq_cnt, rc);
11364 		tp->irq_cnt = rc;
11365 		tp->rxq_cnt = max(rc - 1, 1);
11366 		if (tp->txq_cnt)
11367 			tp->txq_cnt = min(tp->rxq_cnt, tp->txq_max);
11368 	}
11369 
11370 	for (i = 0; i < tp->irq_max; i++)
11371 		tp->napi[i].irq_vec = msix_ent[i].vector;
11372 
11373 	if (netif_set_real_num_rx_queues(tp->dev, tp->rxq_cnt)) {
11374 		pci_disable_msix(tp->pdev);
11375 		return false;
11376 	}
11377 
11378 	if (tp->irq_cnt == 1)
11379 		return true;
11380 
11381 	tg3_flag_set(tp, ENABLE_RSS);
11382 
11383 	if (tp->txq_cnt > 1)
11384 		tg3_flag_set(tp, ENABLE_TSS);
11385 
11386 	netif_set_real_num_tx_queues(tp->dev, tp->txq_cnt);
11387 
11388 	return true;
11389 }
11390 
11391 static void tg3_ints_init(struct tg3 *tp)
11392 {
11393 	if ((tg3_flag(tp, SUPPORT_MSI) || tg3_flag(tp, SUPPORT_MSIX)) &&
11394 	    !tg3_flag(tp, TAGGED_STATUS)) {
11395 		/* All MSI supporting chips should support tagged
11396 		 * status.  Assert that this is the case.
11397 		 */
11398 		netdev_warn(tp->dev,
11399 			    "MSI without TAGGED_STATUS? Not using MSI\n");
11400 		goto defcfg;
11401 	}
11402 
11403 	if (tg3_flag(tp, SUPPORT_MSIX) && tg3_enable_msix(tp))
11404 		tg3_flag_set(tp, USING_MSIX);
11405 	else if (tg3_flag(tp, SUPPORT_MSI) && pci_enable_msi(tp->pdev) == 0)
11406 		tg3_flag_set(tp, USING_MSI);
11407 
11408 	if (tg3_flag(tp, USING_MSI) || tg3_flag(tp, USING_MSIX)) {
11409 		u32 msi_mode = tr32(MSGINT_MODE);
11410 		if (tg3_flag(tp, USING_MSIX) && tp->irq_cnt > 1)
11411 			msi_mode |= MSGINT_MODE_MULTIVEC_EN;
11412 		if (!tg3_flag(tp, 1SHOT_MSI))
11413 			msi_mode |= MSGINT_MODE_ONE_SHOT_DISABLE;
11414 		tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
11415 	}
11416 defcfg:
11417 	if (!tg3_flag(tp, USING_MSIX)) {
11418 		tp->irq_cnt = 1;
11419 		tp->napi[0].irq_vec = tp->pdev->irq;
11420 	}
11421 
11422 	if (tp->irq_cnt == 1) {
11423 		tp->txq_cnt = 1;
11424 		tp->rxq_cnt = 1;
11425 		netif_set_real_num_tx_queues(tp->dev, 1);
11426 		netif_set_real_num_rx_queues(tp->dev, 1);
11427 	}
11428 }
11429 
11430 static void tg3_ints_fini(struct tg3 *tp)
11431 {
11432 	if (tg3_flag(tp, USING_MSIX))
11433 		pci_disable_msix(tp->pdev);
11434 	else if (tg3_flag(tp, USING_MSI))
11435 		pci_disable_msi(tp->pdev);
11436 	tg3_flag_clear(tp, USING_MSI);
11437 	tg3_flag_clear(tp, USING_MSIX);
11438 	tg3_flag_clear(tp, ENABLE_RSS);
11439 	tg3_flag_clear(tp, ENABLE_TSS);
11440 }
11441 
11442 static int tg3_start(struct tg3 *tp, bool reset_phy, bool test_irq,
11443 		     bool init)
11444 {
11445 	struct net_device *dev = tp->dev;
11446 	int i, err;
11447 
11448 	/*
11449 	 * Setup interrupts first so we know how
11450 	 * many NAPI resources to allocate
11451 	 */
11452 	tg3_ints_init(tp);
11453 
11454 	tg3_rss_check_indir_tbl(tp);
11455 
11456 	/* The placement of this call is tied
11457 	 * to the setup and use of Host TX descriptors.
11458 	 */
11459 	err = tg3_alloc_consistent(tp);
11460 	if (err)
11461 		goto out_ints_fini;
11462 
11463 	tg3_napi_init(tp);
11464 
11465 	tg3_napi_enable(tp);
11466 
11467 	for (i = 0; i < tp->irq_cnt; i++) {
11468 		struct tg3_napi *tnapi = &tp->napi[i];
11469 		err = tg3_request_irq(tp, i);
11470 		if (err) {
11471 			for (i--; i >= 0; i--) {
11472 				tnapi = &tp->napi[i];
11473 				free_irq(tnapi->irq_vec, tnapi);
11474 			}
11475 			goto out_napi_fini;
11476 		}
11477 	}
11478 
11479 	tg3_full_lock(tp, 0);
11480 
11481 	if (init)
11482 		tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
11483 
11484 	err = tg3_init_hw(tp, reset_phy);
11485 	if (err) {
11486 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11487 		tg3_free_rings(tp);
11488 	}
11489 
11490 	tg3_full_unlock(tp);
11491 
11492 	if (err)
11493 		goto out_free_irq;
11494 
11495 	if (test_irq && tg3_flag(tp, USING_MSI)) {
11496 		err = tg3_test_msi(tp);
11497 
11498 		if (err) {
11499 			tg3_full_lock(tp, 0);
11500 			tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11501 			tg3_free_rings(tp);
11502 			tg3_full_unlock(tp);
11503 
11504 			goto out_napi_fini;
11505 		}
11506 
11507 		if (!tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, USING_MSI)) {
11508 			u32 val = tr32(PCIE_TRANSACTION_CFG);
11509 
11510 			tw32(PCIE_TRANSACTION_CFG,
11511 			     val | PCIE_TRANS_CFG_1SHOT_MSI);
11512 		}
11513 	}
11514 
11515 	tg3_phy_start(tp);
11516 
11517 	tg3_hwmon_open(tp);
11518 
11519 	tg3_full_lock(tp, 0);
11520 
11521 	tg3_timer_start(tp);
11522 	tg3_flag_set(tp, INIT_COMPLETE);
11523 	tg3_enable_ints(tp);
11524 
11525 	if (init)
11526 		tg3_ptp_init(tp);
11527 	else
11528 		tg3_ptp_resume(tp);
11529 
11530 
11531 	tg3_full_unlock(tp);
11532 
11533 	netif_tx_start_all_queues(dev);
11534 
11535 	/*
11536 	 * Reset loopback feature if it was turned on while the device was down
11537 	 * make sure that it's installed properly now.
11538 	 */
11539 	if (dev->features & NETIF_F_LOOPBACK)
11540 		tg3_set_loopback(dev, dev->features);
11541 
11542 	return 0;
11543 
11544 out_free_irq:
11545 	for (i = tp->irq_cnt - 1; i >= 0; i--) {
11546 		struct tg3_napi *tnapi = &tp->napi[i];
11547 		free_irq(tnapi->irq_vec, tnapi);
11548 	}
11549 
11550 out_napi_fini:
11551 	tg3_napi_disable(tp);
11552 	tg3_napi_fini(tp);
11553 	tg3_free_consistent(tp);
11554 
11555 out_ints_fini:
11556 	tg3_ints_fini(tp);
11557 
11558 	return err;
11559 }
11560 
11561 static void tg3_stop(struct tg3 *tp)
11562 {
11563 	int i;
11564 
11565 	tg3_reset_task_cancel(tp);
11566 	tg3_netif_stop(tp);
11567 
11568 	tg3_timer_stop(tp);
11569 
11570 	tg3_hwmon_close(tp);
11571 
11572 	tg3_phy_stop(tp);
11573 
11574 	tg3_full_lock(tp, 1);
11575 
11576 	tg3_disable_ints(tp);
11577 
11578 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11579 	tg3_free_rings(tp);
11580 	tg3_flag_clear(tp, INIT_COMPLETE);
11581 
11582 	tg3_full_unlock(tp);
11583 
11584 	for (i = tp->irq_cnt - 1; i >= 0; i--) {
11585 		struct tg3_napi *tnapi = &tp->napi[i];
11586 		free_irq(tnapi->irq_vec, tnapi);
11587 	}
11588 
11589 	tg3_ints_fini(tp);
11590 
11591 	tg3_napi_fini(tp);
11592 
11593 	tg3_free_consistent(tp);
11594 }
11595 
11596 static int tg3_open(struct net_device *dev)
11597 {
11598 	struct tg3 *tp = netdev_priv(dev);
11599 	int err;
11600 
11601 	if (tp->fw_needed) {
11602 		err = tg3_request_firmware(tp);
11603 		if (tg3_asic_rev(tp) == ASIC_REV_57766) {
11604 			if (err) {
11605 				netdev_warn(tp->dev, "EEE capability disabled\n");
11606 				tp->phy_flags &= ~TG3_PHYFLG_EEE_CAP;
11607 			} else if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
11608 				netdev_warn(tp->dev, "EEE capability restored\n");
11609 				tp->phy_flags |= TG3_PHYFLG_EEE_CAP;
11610 			}
11611 		} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0) {
11612 			if (err)
11613 				return err;
11614 		} else if (err) {
11615 			netdev_warn(tp->dev, "TSO capability disabled\n");
11616 			tg3_flag_clear(tp, TSO_CAPABLE);
11617 		} else if (!tg3_flag(tp, TSO_CAPABLE)) {
11618 			netdev_notice(tp->dev, "TSO capability restored\n");
11619 			tg3_flag_set(tp, TSO_CAPABLE);
11620 		}
11621 	}
11622 
11623 	tg3_carrier_off(tp);
11624 
11625 	err = tg3_power_up(tp);
11626 	if (err)
11627 		return err;
11628 
11629 	tg3_full_lock(tp, 0);
11630 
11631 	tg3_disable_ints(tp);
11632 	tg3_flag_clear(tp, INIT_COMPLETE);
11633 
11634 	tg3_full_unlock(tp);
11635 
11636 	err = tg3_start(tp,
11637 			!(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN),
11638 			true, true);
11639 	if (err) {
11640 		tg3_frob_aux_power(tp, false);
11641 		pci_set_power_state(tp->pdev, PCI_D3hot);
11642 	}
11643 
11644 	if (tg3_flag(tp, PTP_CAPABLE)) {
11645 		tp->ptp_clock = ptp_clock_register(&tp->ptp_info,
11646 						   &tp->pdev->dev);
11647 		if (IS_ERR(tp->ptp_clock))
11648 			tp->ptp_clock = NULL;
11649 	}
11650 
11651 	return err;
11652 }
11653 
11654 static int tg3_close(struct net_device *dev)
11655 {
11656 	struct tg3 *tp = netdev_priv(dev);
11657 
11658 	tg3_ptp_fini(tp);
11659 
11660 	tg3_stop(tp);
11661 
11662 	/* Clear stats across close / open calls */
11663 	memset(&tp->net_stats_prev, 0, sizeof(tp->net_stats_prev));
11664 	memset(&tp->estats_prev, 0, sizeof(tp->estats_prev));
11665 
11666 	if (pci_device_is_present(tp->pdev)) {
11667 		tg3_power_down_prepare(tp);
11668 
11669 		tg3_carrier_off(tp);
11670 	}
11671 	return 0;
11672 }
11673 
11674 static inline u64 get_stat64(tg3_stat64_t *val)
11675 {
11676        return ((u64)val->high << 32) | ((u64)val->low);
11677 }
11678 
11679 static u64 tg3_calc_crc_errors(struct tg3 *tp)
11680 {
11681 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11682 
11683 	if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
11684 	    (tg3_asic_rev(tp) == ASIC_REV_5700 ||
11685 	     tg3_asic_rev(tp) == ASIC_REV_5701)) {
11686 		u32 val;
11687 
11688 		if (!tg3_readphy(tp, MII_TG3_TEST1, &val)) {
11689 			tg3_writephy(tp, MII_TG3_TEST1,
11690 				     val | MII_TG3_TEST1_CRC_EN);
11691 			tg3_readphy(tp, MII_TG3_RXR_COUNTERS, &val);
11692 		} else
11693 			val = 0;
11694 
11695 		tp->phy_crc_errors += val;
11696 
11697 		return tp->phy_crc_errors;
11698 	}
11699 
11700 	return get_stat64(&hw_stats->rx_fcs_errors);
11701 }
11702 
11703 #define ESTAT_ADD(member) \
11704 	estats->member =	old_estats->member + \
11705 				get_stat64(&hw_stats->member)
11706 
11707 static void tg3_get_estats(struct tg3 *tp, struct tg3_ethtool_stats *estats)
11708 {
11709 	struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
11710 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11711 
11712 	ESTAT_ADD(rx_octets);
11713 	ESTAT_ADD(rx_fragments);
11714 	ESTAT_ADD(rx_ucast_packets);
11715 	ESTAT_ADD(rx_mcast_packets);
11716 	ESTAT_ADD(rx_bcast_packets);
11717 	ESTAT_ADD(rx_fcs_errors);
11718 	ESTAT_ADD(rx_align_errors);
11719 	ESTAT_ADD(rx_xon_pause_rcvd);
11720 	ESTAT_ADD(rx_xoff_pause_rcvd);
11721 	ESTAT_ADD(rx_mac_ctrl_rcvd);
11722 	ESTAT_ADD(rx_xoff_entered);
11723 	ESTAT_ADD(rx_frame_too_long_errors);
11724 	ESTAT_ADD(rx_jabbers);
11725 	ESTAT_ADD(rx_undersize_packets);
11726 	ESTAT_ADD(rx_in_length_errors);
11727 	ESTAT_ADD(rx_out_length_errors);
11728 	ESTAT_ADD(rx_64_or_less_octet_packets);
11729 	ESTAT_ADD(rx_65_to_127_octet_packets);
11730 	ESTAT_ADD(rx_128_to_255_octet_packets);
11731 	ESTAT_ADD(rx_256_to_511_octet_packets);
11732 	ESTAT_ADD(rx_512_to_1023_octet_packets);
11733 	ESTAT_ADD(rx_1024_to_1522_octet_packets);
11734 	ESTAT_ADD(rx_1523_to_2047_octet_packets);
11735 	ESTAT_ADD(rx_2048_to_4095_octet_packets);
11736 	ESTAT_ADD(rx_4096_to_8191_octet_packets);
11737 	ESTAT_ADD(rx_8192_to_9022_octet_packets);
11738 
11739 	ESTAT_ADD(tx_octets);
11740 	ESTAT_ADD(tx_collisions);
11741 	ESTAT_ADD(tx_xon_sent);
11742 	ESTAT_ADD(tx_xoff_sent);
11743 	ESTAT_ADD(tx_flow_control);
11744 	ESTAT_ADD(tx_mac_errors);
11745 	ESTAT_ADD(tx_single_collisions);
11746 	ESTAT_ADD(tx_mult_collisions);
11747 	ESTAT_ADD(tx_deferred);
11748 	ESTAT_ADD(tx_excessive_collisions);
11749 	ESTAT_ADD(tx_late_collisions);
11750 	ESTAT_ADD(tx_collide_2times);
11751 	ESTAT_ADD(tx_collide_3times);
11752 	ESTAT_ADD(tx_collide_4times);
11753 	ESTAT_ADD(tx_collide_5times);
11754 	ESTAT_ADD(tx_collide_6times);
11755 	ESTAT_ADD(tx_collide_7times);
11756 	ESTAT_ADD(tx_collide_8times);
11757 	ESTAT_ADD(tx_collide_9times);
11758 	ESTAT_ADD(tx_collide_10times);
11759 	ESTAT_ADD(tx_collide_11times);
11760 	ESTAT_ADD(tx_collide_12times);
11761 	ESTAT_ADD(tx_collide_13times);
11762 	ESTAT_ADD(tx_collide_14times);
11763 	ESTAT_ADD(tx_collide_15times);
11764 	ESTAT_ADD(tx_ucast_packets);
11765 	ESTAT_ADD(tx_mcast_packets);
11766 	ESTAT_ADD(tx_bcast_packets);
11767 	ESTAT_ADD(tx_carrier_sense_errors);
11768 	ESTAT_ADD(tx_discards);
11769 	ESTAT_ADD(tx_errors);
11770 
11771 	ESTAT_ADD(dma_writeq_full);
11772 	ESTAT_ADD(dma_write_prioq_full);
11773 	ESTAT_ADD(rxbds_empty);
11774 	ESTAT_ADD(rx_discards);
11775 	ESTAT_ADD(rx_errors);
11776 	ESTAT_ADD(rx_threshold_hit);
11777 
11778 	ESTAT_ADD(dma_readq_full);
11779 	ESTAT_ADD(dma_read_prioq_full);
11780 	ESTAT_ADD(tx_comp_queue_full);
11781 
11782 	ESTAT_ADD(ring_set_send_prod_index);
11783 	ESTAT_ADD(ring_status_update);
11784 	ESTAT_ADD(nic_irqs);
11785 	ESTAT_ADD(nic_avoided_irqs);
11786 	ESTAT_ADD(nic_tx_threshold_hit);
11787 
11788 	ESTAT_ADD(mbuf_lwm_thresh_hit);
11789 }
11790 
11791 static void tg3_get_nstats(struct tg3 *tp, struct rtnl_link_stats64 *stats)
11792 {
11793 	struct rtnl_link_stats64 *old_stats = &tp->net_stats_prev;
11794 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11795 
11796 	stats->rx_packets = old_stats->rx_packets +
11797 		get_stat64(&hw_stats->rx_ucast_packets) +
11798 		get_stat64(&hw_stats->rx_mcast_packets) +
11799 		get_stat64(&hw_stats->rx_bcast_packets);
11800 
11801 	stats->tx_packets = old_stats->tx_packets +
11802 		get_stat64(&hw_stats->tx_ucast_packets) +
11803 		get_stat64(&hw_stats->tx_mcast_packets) +
11804 		get_stat64(&hw_stats->tx_bcast_packets);
11805 
11806 	stats->rx_bytes = old_stats->rx_bytes +
11807 		get_stat64(&hw_stats->rx_octets);
11808 	stats->tx_bytes = old_stats->tx_bytes +
11809 		get_stat64(&hw_stats->tx_octets);
11810 
11811 	stats->rx_errors = old_stats->rx_errors +
11812 		get_stat64(&hw_stats->rx_errors);
11813 	stats->tx_errors = old_stats->tx_errors +
11814 		get_stat64(&hw_stats->tx_errors) +
11815 		get_stat64(&hw_stats->tx_mac_errors) +
11816 		get_stat64(&hw_stats->tx_carrier_sense_errors) +
11817 		get_stat64(&hw_stats->tx_discards);
11818 
11819 	stats->multicast = old_stats->multicast +
11820 		get_stat64(&hw_stats->rx_mcast_packets);
11821 	stats->collisions = old_stats->collisions +
11822 		get_stat64(&hw_stats->tx_collisions);
11823 
11824 	stats->rx_length_errors = old_stats->rx_length_errors +
11825 		get_stat64(&hw_stats->rx_frame_too_long_errors) +
11826 		get_stat64(&hw_stats->rx_undersize_packets);
11827 
11828 	stats->rx_frame_errors = old_stats->rx_frame_errors +
11829 		get_stat64(&hw_stats->rx_align_errors);
11830 	stats->tx_aborted_errors = old_stats->tx_aborted_errors +
11831 		get_stat64(&hw_stats->tx_discards);
11832 	stats->tx_carrier_errors = old_stats->tx_carrier_errors +
11833 		get_stat64(&hw_stats->tx_carrier_sense_errors);
11834 
11835 	stats->rx_crc_errors = old_stats->rx_crc_errors +
11836 		tg3_calc_crc_errors(tp);
11837 
11838 	stats->rx_missed_errors = old_stats->rx_missed_errors +
11839 		get_stat64(&hw_stats->rx_discards);
11840 
11841 	stats->rx_dropped = tp->rx_dropped;
11842 	stats->tx_dropped = tp->tx_dropped;
11843 }
11844 
11845 static int tg3_get_regs_len(struct net_device *dev)
11846 {
11847 	return TG3_REG_BLK_SIZE;
11848 }
11849 
11850 static void tg3_get_regs(struct net_device *dev,
11851 		struct ethtool_regs *regs, void *_p)
11852 {
11853 	struct tg3 *tp = netdev_priv(dev);
11854 
11855 	regs->version = 0;
11856 
11857 	memset(_p, 0, TG3_REG_BLK_SIZE);
11858 
11859 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
11860 		return;
11861 
11862 	tg3_full_lock(tp, 0);
11863 
11864 	tg3_dump_legacy_regs(tp, (u32 *)_p);
11865 
11866 	tg3_full_unlock(tp);
11867 }
11868 
11869 static int tg3_get_eeprom_len(struct net_device *dev)
11870 {
11871 	struct tg3 *tp = netdev_priv(dev);
11872 
11873 	return tp->nvram_size;
11874 }
11875 
11876 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
11877 {
11878 	struct tg3 *tp = netdev_priv(dev);
11879 	int ret;
11880 	u8  *pd;
11881 	u32 i, offset, len, b_offset, b_count;
11882 	__be32 val;
11883 
11884 	if (tg3_flag(tp, NO_NVRAM))
11885 		return -EINVAL;
11886 
11887 	offset = eeprom->offset;
11888 	len = eeprom->len;
11889 	eeprom->len = 0;
11890 
11891 	eeprom->magic = TG3_EEPROM_MAGIC;
11892 
11893 	if (offset & 3) {
11894 		/* adjustments to start on required 4 byte boundary */
11895 		b_offset = offset & 3;
11896 		b_count = 4 - b_offset;
11897 		if (b_count > len) {
11898 			/* i.e. offset=1 len=2 */
11899 			b_count = len;
11900 		}
11901 		ret = tg3_nvram_read_be32(tp, offset-b_offset, &val);
11902 		if (ret)
11903 			return ret;
11904 		memcpy(data, ((char *)&val) + b_offset, b_count);
11905 		len -= b_count;
11906 		offset += b_count;
11907 		eeprom->len += b_count;
11908 	}
11909 
11910 	/* read bytes up to the last 4 byte boundary */
11911 	pd = &data[eeprom->len];
11912 	for (i = 0; i < (len - (len & 3)); i += 4) {
11913 		ret = tg3_nvram_read_be32(tp, offset + i, &val);
11914 		if (ret) {
11915 			eeprom->len += i;
11916 			return ret;
11917 		}
11918 		memcpy(pd + i, &val, 4);
11919 	}
11920 	eeprom->len += i;
11921 
11922 	if (len & 3) {
11923 		/* read last bytes not ending on 4 byte boundary */
11924 		pd = &data[eeprom->len];
11925 		b_count = len & 3;
11926 		b_offset = offset + len - b_count;
11927 		ret = tg3_nvram_read_be32(tp, b_offset, &val);
11928 		if (ret)
11929 			return ret;
11930 		memcpy(pd, &val, b_count);
11931 		eeprom->len += b_count;
11932 	}
11933 	return 0;
11934 }
11935 
11936 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
11937 {
11938 	struct tg3 *tp = netdev_priv(dev);
11939 	int ret;
11940 	u32 offset, len, b_offset, odd_len;
11941 	u8 *buf;
11942 	__be32 start, end;
11943 
11944 	if (tg3_flag(tp, NO_NVRAM) ||
11945 	    eeprom->magic != TG3_EEPROM_MAGIC)
11946 		return -EINVAL;
11947 
11948 	offset = eeprom->offset;
11949 	len = eeprom->len;
11950 
11951 	if ((b_offset = (offset & 3))) {
11952 		/* adjustments to start on required 4 byte boundary */
11953 		ret = tg3_nvram_read_be32(tp, offset-b_offset, &start);
11954 		if (ret)
11955 			return ret;
11956 		len += b_offset;
11957 		offset &= ~3;
11958 		if (len < 4)
11959 			len = 4;
11960 	}
11961 
11962 	odd_len = 0;
11963 	if (len & 3) {
11964 		/* adjustments to end on required 4 byte boundary */
11965 		odd_len = 1;
11966 		len = (len + 3) & ~3;
11967 		ret = tg3_nvram_read_be32(tp, offset+len-4, &end);
11968 		if (ret)
11969 			return ret;
11970 	}
11971 
11972 	buf = data;
11973 	if (b_offset || odd_len) {
11974 		buf = kmalloc(len, GFP_KERNEL);
11975 		if (!buf)
11976 			return -ENOMEM;
11977 		if (b_offset)
11978 			memcpy(buf, &start, 4);
11979 		if (odd_len)
11980 			memcpy(buf+len-4, &end, 4);
11981 		memcpy(buf + b_offset, data, eeprom->len);
11982 	}
11983 
11984 	ret = tg3_nvram_write_block(tp, offset, len, buf);
11985 
11986 	if (buf != data)
11987 		kfree(buf);
11988 
11989 	return ret;
11990 }
11991 
11992 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
11993 {
11994 	struct tg3 *tp = netdev_priv(dev);
11995 
11996 	if (tg3_flag(tp, USE_PHYLIB)) {
11997 		struct phy_device *phydev;
11998 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
11999 			return -EAGAIN;
12000 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
12001 		return phy_ethtool_gset(phydev, cmd);
12002 	}
12003 
12004 	cmd->supported = (SUPPORTED_Autoneg);
12005 
12006 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12007 		cmd->supported |= (SUPPORTED_1000baseT_Half |
12008 				   SUPPORTED_1000baseT_Full);
12009 
12010 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
12011 		cmd->supported |= (SUPPORTED_100baseT_Half |
12012 				  SUPPORTED_100baseT_Full |
12013 				  SUPPORTED_10baseT_Half |
12014 				  SUPPORTED_10baseT_Full |
12015 				  SUPPORTED_TP);
12016 		cmd->port = PORT_TP;
12017 	} else {
12018 		cmd->supported |= SUPPORTED_FIBRE;
12019 		cmd->port = PORT_FIBRE;
12020 	}
12021 
12022 	cmd->advertising = tp->link_config.advertising;
12023 	if (tg3_flag(tp, PAUSE_AUTONEG)) {
12024 		if (tp->link_config.flowctrl & FLOW_CTRL_RX) {
12025 			if (tp->link_config.flowctrl & FLOW_CTRL_TX) {
12026 				cmd->advertising |= ADVERTISED_Pause;
12027 			} else {
12028 				cmd->advertising |= ADVERTISED_Pause |
12029 						    ADVERTISED_Asym_Pause;
12030 			}
12031 		} else if (tp->link_config.flowctrl & FLOW_CTRL_TX) {
12032 			cmd->advertising |= ADVERTISED_Asym_Pause;
12033 		}
12034 	}
12035 	if (netif_running(dev) && tp->link_up) {
12036 		ethtool_cmd_speed_set(cmd, tp->link_config.active_speed);
12037 		cmd->duplex = tp->link_config.active_duplex;
12038 		cmd->lp_advertising = tp->link_config.rmt_adv;
12039 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
12040 			if (tp->phy_flags & TG3_PHYFLG_MDIX_STATE)
12041 				cmd->eth_tp_mdix = ETH_TP_MDI_X;
12042 			else
12043 				cmd->eth_tp_mdix = ETH_TP_MDI;
12044 		}
12045 	} else {
12046 		ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
12047 		cmd->duplex = DUPLEX_UNKNOWN;
12048 		cmd->eth_tp_mdix = ETH_TP_MDI_INVALID;
12049 	}
12050 	cmd->phy_address = tp->phy_addr;
12051 	cmd->transceiver = XCVR_INTERNAL;
12052 	cmd->autoneg = tp->link_config.autoneg;
12053 	cmd->maxtxpkt = 0;
12054 	cmd->maxrxpkt = 0;
12055 	return 0;
12056 }
12057 
12058 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
12059 {
12060 	struct tg3 *tp = netdev_priv(dev);
12061 	u32 speed = ethtool_cmd_speed(cmd);
12062 
12063 	if (tg3_flag(tp, USE_PHYLIB)) {
12064 		struct phy_device *phydev;
12065 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
12066 			return -EAGAIN;
12067 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
12068 		return phy_ethtool_sset(phydev, cmd);
12069 	}
12070 
12071 	if (cmd->autoneg != AUTONEG_ENABLE &&
12072 	    cmd->autoneg != AUTONEG_DISABLE)
12073 		return -EINVAL;
12074 
12075 	if (cmd->autoneg == AUTONEG_DISABLE &&
12076 	    cmd->duplex != DUPLEX_FULL &&
12077 	    cmd->duplex != DUPLEX_HALF)
12078 		return -EINVAL;
12079 
12080 	if (cmd->autoneg == AUTONEG_ENABLE) {
12081 		u32 mask = ADVERTISED_Autoneg |
12082 			   ADVERTISED_Pause |
12083 			   ADVERTISED_Asym_Pause;
12084 
12085 		if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12086 			mask |= ADVERTISED_1000baseT_Half |
12087 				ADVERTISED_1000baseT_Full;
12088 
12089 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
12090 			mask |= ADVERTISED_100baseT_Half |
12091 				ADVERTISED_100baseT_Full |
12092 				ADVERTISED_10baseT_Half |
12093 				ADVERTISED_10baseT_Full |
12094 				ADVERTISED_TP;
12095 		else
12096 			mask |= ADVERTISED_FIBRE;
12097 
12098 		if (cmd->advertising & ~mask)
12099 			return -EINVAL;
12100 
12101 		mask &= (ADVERTISED_1000baseT_Half |
12102 			 ADVERTISED_1000baseT_Full |
12103 			 ADVERTISED_100baseT_Half |
12104 			 ADVERTISED_100baseT_Full |
12105 			 ADVERTISED_10baseT_Half |
12106 			 ADVERTISED_10baseT_Full);
12107 
12108 		cmd->advertising &= mask;
12109 	} else {
12110 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES) {
12111 			if (speed != SPEED_1000)
12112 				return -EINVAL;
12113 
12114 			if (cmd->duplex != DUPLEX_FULL)
12115 				return -EINVAL;
12116 		} else {
12117 			if (speed != SPEED_100 &&
12118 			    speed != SPEED_10)
12119 				return -EINVAL;
12120 		}
12121 	}
12122 
12123 	tg3_full_lock(tp, 0);
12124 
12125 	tp->link_config.autoneg = cmd->autoneg;
12126 	if (cmd->autoneg == AUTONEG_ENABLE) {
12127 		tp->link_config.advertising = (cmd->advertising |
12128 					      ADVERTISED_Autoneg);
12129 		tp->link_config.speed = SPEED_UNKNOWN;
12130 		tp->link_config.duplex = DUPLEX_UNKNOWN;
12131 	} else {
12132 		tp->link_config.advertising = 0;
12133 		tp->link_config.speed = speed;
12134 		tp->link_config.duplex = cmd->duplex;
12135 	}
12136 
12137 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
12138 
12139 	tg3_warn_mgmt_link_flap(tp);
12140 
12141 	if (netif_running(dev))
12142 		tg3_setup_phy(tp, true);
12143 
12144 	tg3_full_unlock(tp);
12145 
12146 	return 0;
12147 }
12148 
12149 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
12150 {
12151 	struct tg3 *tp = netdev_priv(dev);
12152 
12153 	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
12154 	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
12155 	strlcpy(info->fw_version, tp->fw_ver, sizeof(info->fw_version));
12156 	strlcpy(info->bus_info, pci_name(tp->pdev), sizeof(info->bus_info));
12157 }
12158 
12159 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
12160 {
12161 	struct tg3 *tp = netdev_priv(dev);
12162 
12163 	if (tg3_flag(tp, WOL_CAP) && device_can_wakeup(&tp->pdev->dev))
12164 		wol->supported = WAKE_MAGIC;
12165 	else
12166 		wol->supported = 0;
12167 	wol->wolopts = 0;
12168 	if (tg3_flag(tp, WOL_ENABLE) && device_can_wakeup(&tp->pdev->dev))
12169 		wol->wolopts = WAKE_MAGIC;
12170 	memset(&wol->sopass, 0, sizeof(wol->sopass));
12171 }
12172 
12173 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
12174 {
12175 	struct tg3 *tp = netdev_priv(dev);
12176 	struct device *dp = &tp->pdev->dev;
12177 
12178 	if (wol->wolopts & ~WAKE_MAGIC)
12179 		return -EINVAL;
12180 	if ((wol->wolopts & WAKE_MAGIC) &&
12181 	    !(tg3_flag(tp, WOL_CAP) && device_can_wakeup(dp)))
12182 		return -EINVAL;
12183 
12184 	device_set_wakeup_enable(dp, wol->wolopts & WAKE_MAGIC);
12185 
12186 	if (device_may_wakeup(dp))
12187 		tg3_flag_set(tp, WOL_ENABLE);
12188 	else
12189 		tg3_flag_clear(tp, WOL_ENABLE);
12190 
12191 	return 0;
12192 }
12193 
12194 static u32 tg3_get_msglevel(struct net_device *dev)
12195 {
12196 	struct tg3 *tp = netdev_priv(dev);
12197 	return tp->msg_enable;
12198 }
12199 
12200 static void tg3_set_msglevel(struct net_device *dev, u32 value)
12201 {
12202 	struct tg3 *tp = netdev_priv(dev);
12203 	tp->msg_enable = value;
12204 }
12205 
12206 static int tg3_nway_reset(struct net_device *dev)
12207 {
12208 	struct tg3 *tp = netdev_priv(dev);
12209 	int r;
12210 
12211 	if (!netif_running(dev))
12212 		return -EAGAIN;
12213 
12214 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
12215 		return -EINVAL;
12216 
12217 	tg3_warn_mgmt_link_flap(tp);
12218 
12219 	if (tg3_flag(tp, USE_PHYLIB)) {
12220 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
12221 			return -EAGAIN;
12222 		r = phy_start_aneg(tp->mdio_bus->phy_map[tp->phy_addr]);
12223 	} else {
12224 		u32 bmcr;
12225 
12226 		spin_lock_bh(&tp->lock);
12227 		r = -EINVAL;
12228 		tg3_readphy(tp, MII_BMCR, &bmcr);
12229 		if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
12230 		    ((bmcr & BMCR_ANENABLE) ||
12231 		     (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT))) {
12232 			tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
12233 						   BMCR_ANENABLE);
12234 			r = 0;
12235 		}
12236 		spin_unlock_bh(&tp->lock);
12237 	}
12238 
12239 	return r;
12240 }
12241 
12242 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
12243 {
12244 	struct tg3 *tp = netdev_priv(dev);
12245 
12246 	ering->rx_max_pending = tp->rx_std_ring_mask;
12247 	if (tg3_flag(tp, JUMBO_RING_ENABLE))
12248 		ering->rx_jumbo_max_pending = tp->rx_jmb_ring_mask;
12249 	else
12250 		ering->rx_jumbo_max_pending = 0;
12251 
12252 	ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
12253 
12254 	ering->rx_pending = tp->rx_pending;
12255 	if (tg3_flag(tp, JUMBO_RING_ENABLE))
12256 		ering->rx_jumbo_pending = tp->rx_jumbo_pending;
12257 	else
12258 		ering->rx_jumbo_pending = 0;
12259 
12260 	ering->tx_pending = tp->napi[0].tx_pending;
12261 }
12262 
12263 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
12264 {
12265 	struct tg3 *tp = netdev_priv(dev);
12266 	int i, irq_sync = 0, err = 0;
12267 
12268 	if ((ering->rx_pending > tp->rx_std_ring_mask) ||
12269 	    (ering->rx_jumbo_pending > tp->rx_jmb_ring_mask) ||
12270 	    (ering->tx_pending > TG3_TX_RING_SIZE - 1) ||
12271 	    (ering->tx_pending <= MAX_SKB_FRAGS) ||
12272 	    (tg3_flag(tp, TSO_BUG) &&
12273 	     (ering->tx_pending <= (MAX_SKB_FRAGS * 3))))
12274 		return -EINVAL;
12275 
12276 	if (netif_running(dev)) {
12277 		tg3_phy_stop(tp);
12278 		tg3_netif_stop(tp);
12279 		irq_sync = 1;
12280 	}
12281 
12282 	tg3_full_lock(tp, irq_sync);
12283 
12284 	tp->rx_pending = ering->rx_pending;
12285 
12286 	if (tg3_flag(tp, MAX_RXPEND_64) &&
12287 	    tp->rx_pending > 63)
12288 		tp->rx_pending = 63;
12289 	tp->rx_jumbo_pending = ering->rx_jumbo_pending;
12290 
12291 	for (i = 0; i < tp->irq_max; i++)
12292 		tp->napi[i].tx_pending = ering->tx_pending;
12293 
12294 	if (netif_running(dev)) {
12295 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12296 		err = tg3_restart_hw(tp, false);
12297 		if (!err)
12298 			tg3_netif_start(tp);
12299 	}
12300 
12301 	tg3_full_unlock(tp);
12302 
12303 	if (irq_sync && !err)
12304 		tg3_phy_start(tp);
12305 
12306 	return err;
12307 }
12308 
12309 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
12310 {
12311 	struct tg3 *tp = netdev_priv(dev);
12312 
12313 	epause->autoneg = !!tg3_flag(tp, PAUSE_AUTONEG);
12314 
12315 	if (tp->link_config.flowctrl & FLOW_CTRL_RX)
12316 		epause->rx_pause = 1;
12317 	else
12318 		epause->rx_pause = 0;
12319 
12320 	if (tp->link_config.flowctrl & FLOW_CTRL_TX)
12321 		epause->tx_pause = 1;
12322 	else
12323 		epause->tx_pause = 0;
12324 }
12325 
12326 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
12327 {
12328 	struct tg3 *tp = netdev_priv(dev);
12329 	int err = 0;
12330 
12331 	if (tp->link_config.autoneg == AUTONEG_ENABLE)
12332 		tg3_warn_mgmt_link_flap(tp);
12333 
12334 	if (tg3_flag(tp, USE_PHYLIB)) {
12335 		u32 newadv;
12336 		struct phy_device *phydev;
12337 
12338 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
12339 
12340 		if (!(phydev->supported & SUPPORTED_Pause) ||
12341 		    (!(phydev->supported & SUPPORTED_Asym_Pause) &&
12342 		     (epause->rx_pause != epause->tx_pause)))
12343 			return -EINVAL;
12344 
12345 		tp->link_config.flowctrl = 0;
12346 		if (epause->rx_pause) {
12347 			tp->link_config.flowctrl |= FLOW_CTRL_RX;
12348 
12349 			if (epause->tx_pause) {
12350 				tp->link_config.flowctrl |= FLOW_CTRL_TX;
12351 				newadv = ADVERTISED_Pause;
12352 			} else
12353 				newadv = ADVERTISED_Pause |
12354 					 ADVERTISED_Asym_Pause;
12355 		} else if (epause->tx_pause) {
12356 			tp->link_config.flowctrl |= FLOW_CTRL_TX;
12357 			newadv = ADVERTISED_Asym_Pause;
12358 		} else
12359 			newadv = 0;
12360 
12361 		if (epause->autoneg)
12362 			tg3_flag_set(tp, PAUSE_AUTONEG);
12363 		else
12364 			tg3_flag_clear(tp, PAUSE_AUTONEG);
12365 
12366 		if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
12367 			u32 oldadv = phydev->advertising &
12368 				     (ADVERTISED_Pause | ADVERTISED_Asym_Pause);
12369 			if (oldadv != newadv) {
12370 				phydev->advertising &=
12371 					~(ADVERTISED_Pause |
12372 					  ADVERTISED_Asym_Pause);
12373 				phydev->advertising |= newadv;
12374 				if (phydev->autoneg) {
12375 					/*
12376 					 * Always renegotiate the link to
12377 					 * inform our link partner of our
12378 					 * flow control settings, even if the
12379 					 * flow control is forced.  Let
12380 					 * tg3_adjust_link() do the final
12381 					 * flow control setup.
12382 					 */
12383 					return phy_start_aneg(phydev);
12384 				}
12385 			}
12386 
12387 			if (!epause->autoneg)
12388 				tg3_setup_flow_control(tp, 0, 0);
12389 		} else {
12390 			tp->link_config.advertising &=
12391 					~(ADVERTISED_Pause |
12392 					  ADVERTISED_Asym_Pause);
12393 			tp->link_config.advertising |= newadv;
12394 		}
12395 	} else {
12396 		int irq_sync = 0;
12397 
12398 		if (netif_running(dev)) {
12399 			tg3_netif_stop(tp);
12400 			irq_sync = 1;
12401 		}
12402 
12403 		tg3_full_lock(tp, irq_sync);
12404 
12405 		if (epause->autoneg)
12406 			tg3_flag_set(tp, PAUSE_AUTONEG);
12407 		else
12408 			tg3_flag_clear(tp, PAUSE_AUTONEG);
12409 		if (epause->rx_pause)
12410 			tp->link_config.flowctrl |= FLOW_CTRL_RX;
12411 		else
12412 			tp->link_config.flowctrl &= ~FLOW_CTRL_RX;
12413 		if (epause->tx_pause)
12414 			tp->link_config.flowctrl |= FLOW_CTRL_TX;
12415 		else
12416 			tp->link_config.flowctrl &= ~FLOW_CTRL_TX;
12417 
12418 		if (netif_running(dev)) {
12419 			tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12420 			err = tg3_restart_hw(tp, false);
12421 			if (!err)
12422 				tg3_netif_start(tp);
12423 		}
12424 
12425 		tg3_full_unlock(tp);
12426 	}
12427 
12428 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
12429 
12430 	return err;
12431 }
12432 
12433 static int tg3_get_sset_count(struct net_device *dev, int sset)
12434 {
12435 	switch (sset) {
12436 	case ETH_SS_TEST:
12437 		return TG3_NUM_TEST;
12438 	case ETH_SS_STATS:
12439 		return TG3_NUM_STATS;
12440 	default:
12441 		return -EOPNOTSUPP;
12442 	}
12443 }
12444 
12445 static int tg3_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
12446 			 u32 *rules __always_unused)
12447 {
12448 	struct tg3 *tp = netdev_priv(dev);
12449 
12450 	if (!tg3_flag(tp, SUPPORT_MSIX))
12451 		return -EOPNOTSUPP;
12452 
12453 	switch (info->cmd) {
12454 	case ETHTOOL_GRXRINGS:
12455 		if (netif_running(tp->dev))
12456 			info->data = tp->rxq_cnt;
12457 		else {
12458 			info->data = num_online_cpus();
12459 			if (info->data > TG3_RSS_MAX_NUM_QS)
12460 				info->data = TG3_RSS_MAX_NUM_QS;
12461 		}
12462 
12463 		/* The first interrupt vector only
12464 		 * handles link interrupts.
12465 		 */
12466 		info->data -= 1;
12467 		return 0;
12468 
12469 	default:
12470 		return -EOPNOTSUPP;
12471 	}
12472 }
12473 
12474 static u32 tg3_get_rxfh_indir_size(struct net_device *dev)
12475 {
12476 	u32 size = 0;
12477 	struct tg3 *tp = netdev_priv(dev);
12478 
12479 	if (tg3_flag(tp, SUPPORT_MSIX))
12480 		size = TG3_RSS_INDIR_TBL_SIZE;
12481 
12482 	return size;
12483 }
12484 
12485 static int tg3_get_rxfh_indir(struct net_device *dev, u32 *indir)
12486 {
12487 	struct tg3 *tp = netdev_priv(dev);
12488 	int i;
12489 
12490 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
12491 		indir[i] = tp->rss_ind_tbl[i];
12492 
12493 	return 0;
12494 }
12495 
12496 static int tg3_set_rxfh_indir(struct net_device *dev, const u32 *indir)
12497 {
12498 	struct tg3 *tp = netdev_priv(dev);
12499 	size_t i;
12500 
12501 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
12502 		tp->rss_ind_tbl[i] = indir[i];
12503 
12504 	if (!netif_running(dev) || !tg3_flag(tp, ENABLE_RSS))
12505 		return 0;
12506 
12507 	/* It is legal to write the indirection
12508 	 * table while the device is running.
12509 	 */
12510 	tg3_full_lock(tp, 0);
12511 	tg3_rss_write_indir_tbl(tp);
12512 	tg3_full_unlock(tp);
12513 
12514 	return 0;
12515 }
12516 
12517 static void tg3_get_channels(struct net_device *dev,
12518 			     struct ethtool_channels *channel)
12519 {
12520 	struct tg3 *tp = netdev_priv(dev);
12521 	u32 deflt_qs = netif_get_num_default_rss_queues();
12522 
12523 	channel->max_rx = tp->rxq_max;
12524 	channel->max_tx = tp->txq_max;
12525 
12526 	if (netif_running(dev)) {
12527 		channel->rx_count = tp->rxq_cnt;
12528 		channel->tx_count = tp->txq_cnt;
12529 	} else {
12530 		if (tp->rxq_req)
12531 			channel->rx_count = tp->rxq_req;
12532 		else
12533 			channel->rx_count = min(deflt_qs, tp->rxq_max);
12534 
12535 		if (tp->txq_req)
12536 			channel->tx_count = tp->txq_req;
12537 		else
12538 			channel->tx_count = min(deflt_qs, tp->txq_max);
12539 	}
12540 }
12541 
12542 static int tg3_set_channels(struct net_device *dev,
12543 			    struct ethtool_channels *channel)
12544 {
12545 	struct tg3 *tp = netdev_priv(dev);
12546 
12547 	if (!tg3_flag(tp, SUPPORT_MSIX))
12548 		return -EOPNOTSUPP;
12549 
12550 	if (channel->rx_count > tp->rxq_max ||
12551 	    channel->tx_count > tp->txq_max)
12552 		return -EINVAL;
12553 
12554 	tp->rxq_req = channel->rx_count;
12555 	tp->txq_req = channel->tx_count;
12556 
12557 	if (!netif_running(dev))
12558 		return 0;
12559 
12560 	tg3_stop(tp);
12561 
12562 	tg3_carrier_off(tp);
12563 
12564 	tg3_start(tp, true, false, false);
12565 
12566 	return 0;
12567 }
12568 
12569 static void tg3_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
12570 {
12571 	switch (stringset) {
12572 	case ETH_SS_STATS:
12573 		memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
12574 		break;
12575 	case ETH_SS_TEST:
12576 		memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
12577 		break;
12578 	default:
12579 		WARN_ON(1);	/* we need a WARN() */
12580 		break;
12581 	}
12582 }
12583 
12584 static int tg3_set_phys_id(struct net_device *dev,
12585 			    enum ethtool_phys_id_state state)
12586 {
12587 	struct tg3 *tp = netdev_priv(dev);
12588 
12589 	if (!netif_running(tp->dev))
12590 		return -EAGAIN;
12591 
12592 	switch (state) {
12593 	case ETHTOOL_ID_ACTIVE:
12594 		return 1;	/* cycle on/off once per second */
12595 
12596 	case ETHTOOL_ID_ON:
12597 		tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
12598 		     LED_CTRL_1000MBPS_ON |
12599 		     LED_CTRL_100MBPS_ON |
12600 		     LED_CTRL_10MBPS_ON |
12601 		     LED_CTRL_TRAFFIC_OVERRIDE |
12602 		     LED_CTRL_TRAFFIC_BLINK |
12603 		     LED_CTRL_TRAFFIC_LED);
12604 		break;
12605 
12606 	case ETHTOOL_ID_OFF:
12607 		tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
12608 		     LED_CTRL_TRAFFIC_OVERRIDE);
12609 		break;
12610 
12611 	case ETHTOOL_ID_INACTIVE:
12612 		tw32(MAC_LED_CTRL, tp->led_ctrl);
12613 		break;
12614 	}
12615 
12616 	return 0;
12617 }
12618 
12619 static void tg3_get_ethtool_stats(struct net_device *dev,
12620 				   struct ethtool_stats *estats, u64 *tmp_stats)
12621 {
12622 	struct tg3 *tp = netdev_priv(dev);
12623 
12624 	if (tp->hw_stats)
12625 		tg3_get_estats(tp, (struct tg3_ethtool_stats *)tmp_stats);
12626 	else
12627 		memset(tmp_stats, 0, sizeof(struct tg3_ethtool_stats));
12628 }
12629 
12630 static __be32 *tg3_vpd_readblock(struct tg3 *tp, u32 *vpdlen)
12631 {
12632 	int i;
12633 	__be32 *buf;
12634 	u32 offset = 0, len = 0;
12635 	u32 magic, val;
12636 
12637 	if (tg3_flag(tp, NO_NVRAM) || tg3_nvram_read(tp, 0, &magic))
12638 		return NULL;
12639 
12640 	if (magic == TG3_EEPROM_MAGIC) {
12641 		for (offset = TG3_NVM_DIR_START;
12642 		     offset < TG3_NVM_DIR_END;
12643 		     offset += TG3_NVM_DIRENT_SIZE) {
12644 			if (tg3_nvram_read(tp, offset, &val))
12645 				return NULL;
12646 
12647 			if ((val >> TG3_NVM_DIRTYPE_SHIFT) ==
12648 			    TG3_NVM_DIRTYPE_EXTVPD)
12649 				break;
12650 		}
12651 
12652 		if (offset != TG3_NVM_DIR_END) {
12653 			len = (val & TG3_NVM_DIRTYPE_LENMSK) * 4;
12654 			if (tg3_nvram_read(tp, offset + 4, &offset))
12655 				return NULL;
12656 
12657 			offset = tg3_nvram_logical_addr(tp, offset);
12658 		}
12659 	}
12660 
12661 	if (!offset || !len) {
12662 		offset = TG3_NVM_VPD_OFF;
12663 		len = TG3_NVM_VPD_LEN;
12664 	}
12665 
12666 	buf = kmalloc(len, GFP_KERNEL);
12667 	if (buf == NULL)
12668 		return NULL;
12669 
12670 	if (magic == TG3_EEPROM_MAGIC) {
12671 		for (i = 0; i < len; i += 4) {
12672 			/* The data is in little-endian format in NVRAM.
12673 			 * Use the big-endian read routines to preserve
12674 			 * the byte order as it exists in NVRAM.
12675 			 */
12676 			if (tg3_nvram_read_be32(tp, offset + i, &buf[i/4]))
12677 				goto error;
12678 		}
12679 	} else {
12680 		u8 *ptr;
12681 		ssize_t cnt;
12682 		unsigned int pos = 0;
12683 
12684 		ptr = (u8 *)&buf[0];
12685 		for (i = 0; pos < len && i < 3; i++, pos += cnt, ptr += cnt) {
12686 			cnt = pci_read_vpd(tp->pdev, pos,
12687 					   len - pos, ptr);
12688 			if (cnt == -ETIMEDOUT || cnt == -EINTR)
12689 				cnt = 0;
12690 			else if (cnt < 0)
12691 				goto error;
12692 		}
12693 		if (pos != len)
12694 			goto error;
12695 	}
12696 
12697 	*vpdlen = len;
12698 
12699 	return buf;
12700 
12701 error:
12702 	kfree(buf);
12703 	return NULL;
12704 }
12705 
12706 #define NVRAM_TEST_SIZE 0x100
12707 #define NVRAM_SELFBOOT_FORMAT1_0_SIZE	0x14
12708 #define NVRAM_SELFBOOT_FORMAT1_2_SIZE	0x18
12709 #define NVRAM_SELFBOOT_FORMAT1_3_SIZE	0x1c
12710 #define NVRAM_SELFBOOT_FORMAT1_4_SIZE	0x20
12711 #define NVRAM_SELFBOOT_FORMAT1_5_SIZE	0x24
12712 #define NVRAM_SELFBOOT_FORMAT1_6_SIZE	0x50
12713 #define NVRAM_SELFBOOT_HW_SIZE 0x20
12714 #define NVRAM_SELFBOOT_DATA_SIZE 0x1c
12715 
12716 static int tg3_test_nvram(struct tg3 *tp)
12717 {
12718 	u32 csum, magic, len;
12719 	__be32 *buf;
12720 	int i, j, k, err = 0, size;
12721 
12722 	if (tg3_flag(tp, NO_NVRAM))
12723 		return 0;
12724 
12725 	if (tg3_nvram_read(tp, 0, &magic) != 0)
12726 		return -EIO;
12727 
12728 	if (magic == TG3_EEPROM_MAGIC)
12729 		size = NVRAM_TEST_SIZE;
12730 	else if ((magic & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW) {
12731 		if ((magic & TG3_EEPROM_SB_FORMAT_MASK) ==
12732 		    TG3_EEPROM_SB_FORMAT_1) {
12733 			switch (magic & TG3_EEPROM_SB_REVISION_MASK) {
12734 			case TG3_EEPROM_SB_REVISION_0:
12735 				size = NVRAM_SELFBOOT_FORMAT1_0_SIZE;
12736 				break;
12737 			case TG3_EEPROM_SB_REVISION_2:
12738 				size = NVRAM_SELFBOOT_FORMAT1_2_SIZE;
12739 				break;
12740 			case TG3_EEPROM_SB_REVISION_3:
12741 				size = NVRAM_SELFBOOT_FORMAT1_3_SIZE;
12742 				break;
12743 			case TG3_EEPROM_SB_REVISION_4:
12744 				size = NVRAM_SELFBOOT_FORMAT1_4_SIZE;
12745 				break;
12746 			case TG3_EEPROM_SB_REVISION_5:
12747 				size = NVRAM_SELFBOOT_FORMAT1_5_SIZE;
12748 				break;
12749 			case TG3_EEPROM_SB_REVISION_6:
12750 				size = NVRAM_SELFBOOT_FORMAT1_6_SIZE;
12751 				break;
12752 			default:
12753 				return -EIO;
12754 			}
12755 		} else
12756 			return 0;
12757 	} else if ((magic & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
12758 		size = NVRAM_SELFBOOT_HW_SIZE;
12759 	else
12760 		return -EIO;
12761 
12762 	buf = kmalloc(size, GFP_KERNEL);
12763 	if (buf == NULL)
12764 		return -ENOMEM;
12765 
12766 	err = -EIO;
12767 	for (i = 0, j = 0; i < size; i += 4, j++) {
12768 		err = tg3_nvram_read_be32(tp, i, &buf[j]);
12769 		if (err)
12770 			break;
12771 	}
12772 	if (i < size)
12773 		goto out;
12774 
12775 	/* Selfboot format */
12776 	magic = be32_to_cpu(buf[0]);
12777 	if ((magic & TG3_EEPROM_MAGIC_FW_MSK) ==
12778 	    TG3_EEPROM_MAGIC_FW) {
12779 		u8 *buf8 = (u8 *) buf, csum8 = 0;
12780 
12781 		if ((magic & TG3_EEPROM_SB_REVISION_MASK) ==
12782 		    TG3_EEPROM_SB_REVISION_2) {
12783 			/* For rev 2, the csum doesn't include the MBA. */
12784 			for (i = 0; i < TG3_EEPROM_SB_F1R2_MBA_OFF; i++)
12785 				csum8 += buf8[i];
12786 			for (i = TG3_EEPROM_SB_F1R2_MBA_OFF + 4; i < size; i++)
12787 				csum8 += buf8[i];
12788 		} else {
12789 			for (i = 0; i < size; i++)
12790 				csum8 += buf8[i];
12791 		}
12792 
12793 		if (csum8 == 0) {
12794 			err = 0;
12795 			goto out;
12796 		}
12797 
12798 		err = -EIO;
12799 		goto out;
12800 	}
12801 
12802 	if ((magic & TG3_EEPROM_MAGIC_HW_MSK) ==
12803 	    TG3_EEPROM_MAGIC_HW) {
12804 		u8 data[NVRAM_SELFBOOT_DATA_SIZE];
12805 		u8 parity[NVRAM_SELFBOOT_DATA_SIZE];
12806 		u8 *buf8 = (u8 *) buf;
12807 
12808 		/* Separate the parity bits and the data bytes.  */
12809 		for (i = 0, j = 0, k = 0; i < NVRAM_SELFBOOT_HW_SIZE; i++) {
12810 			if ((i == 0) || (i == 8)) {
12811 				int l;
12812 				u8 msk;
12813 
12814 				for (l = 0, msk = 0x80; l < 7; l++, msk >>= 1)
12815 					parity[k++] = buf8[i] & msk;
12816 				i++;
12817 			} else if (i == 16) {
12818 				int l;
12819 				u8 msk;
12820 
12821 				for (l = 0, msk = 0x20; l < 6; l++, msk >>= 1)
12822 					parity[k++] = buf8[i] & msk;
12823 				i++;
12824 
12825 				for (l = 0, msk = 0x80; l < 8; l++, msk >>= 1)
12826 					parity[k++] = buf8[i] & msk;
12827 				i++;
12828 			}
12829 			data[j++] = buf8[i];
12830 		}
12831 
12832 		err = -EIO;
12833 		for (i = 0; i < NVRAM_SELFBOOT_DATA_SIZE; i++) {
12834 			u8 hw8 = hweight8(data[i]);
12835 
12836 			if ((hw8 & 0x1) && parity[i])
12837 				goto out;
12838 			else if (!(hw8 & 0x1) && !parity[i])
12839 				goto out;
12840 		}
12841 		err = 0;
12842 		goto out;
12843 	}
12844 
12845 	err = -EIO;
12846 
12847 	/* Bootstrap checksum at offset 0x10 */
12848 	csum = calc_crc((unsigned char *) buf, 0x10);
12849 	if (csum != le32_to_cpu(buf[0x10/4]))
12850 		goto out;
12851 
12852 	/* Manufacturing block starts at offset 0x74, checksum at 0xfc */
12853 	csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
12854 	if (csum != le32_to_cpu(buf[0xfc/4]))
12855 		goto out;
12856 
12857 	kfree(buf);
12858 
12859 	buf = tg3_vpd_readblock(tp, &len);
12860 	if (!buf)
12861 		return -ENOMEM;
12862 
12863 	i = pci_vpd_find_tag((u8 *)buf, 0, len, PCI_VPD_LRDT_RO_DATA);
12864 	if (i > 0) {
12865 		j = pci_vpd_lrdt_size(&((u8 *)buf)[i]);
12866 		if (j < 0)
12867 			goto out;
12868 
12869 		if (i + PCI_VPD_LRDT_TAG_SIZE + j > len)
12870 			goto out;
12871 
12872 		i += PCI_VPD_LRDT_TAG_SIZE;
12873 		j = pci_vpd_find_info_keyword((u8 *)buf, i, j,
12874 					      PCI_VPD_RO_KEYWORD_CHKSUM);
12875 		if (j > 0) {
12876 			u8 csum8 = 0;
12877 
12878 			j += PCI_VPD_INFO_FLD_HDR_SIZE;
12879 
12880 			for (i = 0; i <= j; i++)
12881 				csum8 += ((u8 *)buf)[i];
12882 
12883 			if (csum8)
12884 				goto out;
12885 		}
12886 	}
12887 
12888 	err = 0;
12889 
12890 out:
12891 	kfree(buf);
12892 	return err;
12893 }
12894 
12895 #define TG3_SERDES_TIMEOUT_SEC	2
12896 #define TG3_COPPER_TIMEOUT_SEC	6
12897 
12898 static int tg3_test_link(struct tg3 *tp)
12899 {
12900 	int i, max;
12901 
12902 	if (!netif_running(tp->dev))
12903 		return -ENODEV;
12904 
12905 	if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
12906 		max = TG3_SERDES_TIMEOUT_SEC;
12907 	else
12908 		max = TG3_COPPER_TIMEOUT_SEC;
12909 
12910 	for (i = 0; i < max; i++) {
12911 		if (tp->link_up)
12912 			return 0;
12913 
12914 		if (msleep_interruptible(1000))
12915 			break;
12916 	}
12917 
12918 	return -EIO;
12919 }
12920 
12921 /* Only test the commonly used registers */
12922 static int tg3_test_registers(struct tg3 *tp)
12923 {
12924 	int i, is_5705, is_5750;
12925 	u32 offset, read_mask, write_mask, val, save_val, read_val;
12926 	static struct {
12927 		u16 offset;
12928 		u16 flags;
12929 #define TG3_FL_5705	0x1
12930 #define TG3_FL_NOT_5705	0x2
12931 #define TG3_FL_NOT_5788	0x4
12932 #define TG3_FL_NOT_5750	0x8
12933 		u32 read_mask;
12934 		u32 write_mask;
12935 	} reg_tbl[] = {
12936 		/* MAC Control Registers */
12937 		{ MAC_MODE, TG3_FL_NOT_5705,
12938 			0x00000000, 0x00ef6f8c },
12939 		{ MAC_MODE, TG3_FL_5705,
12940 			0x00000000, 0x01ef6b8c },
12941 		{ MAC_STATUS, TG3_FL_NOT_5705,
12942 			0x03800107, 0x00000000 },
12943 		{ MAC_STATUS, TG3_FL_5705,
12944 			0x03800100, 0x00000000 },
12945 		{ MAC_ADDR_0_HIGH, 0x0000,
12946 			0x00000000, 0x0000ffff },
12947 		{ MAC_ADDR_0_LOW, 0x0000,
12948 			0x00000000, 0xffffffff },
12949 		{ MAC_RX_MTU_SIZE, 0x0000,
12950 			0x00000000, 0x0000ffff },
12951 		{ MAC_TX_MODE, 0x0000,
12952 			0x00000000, 0x00000070 },
12953 		{ MAC_TX_LENGTHS, 0x0000,
12954 			0x00000000, 0x00003fff },
12955 		{ MAC_RX_MODE, TG3_FL_NOT_5705,
12956 			0x00000000, 0x000007fc },
12957 		{ MAC_RX_MODE, TG3_FL_5705,
12958 			0x00000000, 0x000007dc },
12959 		{ MAC_HASH_REG_0, 0x0000,
12960 			0x00000000, 0xffffffff },
12961 		{ MAC_HASH_REG_1, 0x0000,
12962 			0x00000000, 0xffffffff },
12963 		{ MAC_HASH_REG_2, 0x0000,
12964 			0x00000000, 0xffffffff },
12965 		{ MAC_HASH_REG_3, 0x0000,
12966 			0x00000000, 0xffffffff },
12967 
12968 		/* Receive Data and Receive BD Initiator Control Registers. */
12969 		{ RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
12970 			0x00000000, 0xffffffff },
12971 		{ RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
12972 			0x00000000, 0xffffffff },
12973 		{ RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
12974 			0x00000000, 0x00000003 },
12975 		{ RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
12976 			0x00000000, 0xffffffff },
12977 		{ RCVDBDI_STD_BD+0, 0x0000,
12978 			0x00000000, 0xffffffff },
12979 		{ RCVDBDI_STD_BD+4, 0x0000,
12980 			0x00000000, 0xffffffff },
12981 		{ RCVDBDI_STD_BD+8, 0x0000,
12982 			0x00000000, 0xffff0002 },
12983 		{ RCVDBDI_STD_BD+0xc, 0x0000,
12984 			0x00000000, 0xffffffff },
12985 
12986 		/* Receive BD Initiator Control Registers. */
12987 		{ RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
12988 			0x00000000, 0xffffffff },
12989 		{ RCVBDI_STD_THRESH, TG3_FL_5705,
12990 			0x00000000, 0x000003ff },
12991 		{ RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
12992 			0x00000000, 0xffffffff },
12993 
12994 		/* Host Coalescing Control Registers. */
12995 		{ HOSTCC_MODE, TG3_FL_NOT_5705,
12996 			0x00000000, 0x00000004 },
12997 		{ HOSTCC_MODE, TG3_FL_5705,
12998 			0x00000000, 0x000000f6 },
12999 		{ HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
13000 			0x00000000, 0xffffffff },
13001 		{ HOSTCC_RXCOL_TICKS, TG3_FL_5705,
13002 			0x00000000, 0x000003ff },
13003 		{ HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
13004 			0x00000000, 0xffffffff },
13005 		{ HOSTCC_TXCOL_TICKS, TG3_FL_5705,
13006 			0x00000000, 0x000003ff },
13007 		{ HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
13008 			0x00000000, 0xffffffff },
13009 		{ HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
13010 			0x00000000, 0x000000ff },
13011 		{ HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
13012 			0x00000000, 0xffffffff },
13013 		{ HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
13014 			0x00000000, 0x000000ff },
13015 		{ HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
13016 			0x00000000, 0xffffffff },
13017 		{ HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
13018 			0x00000000, 0xffffffff },
13019 		{ HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
13020 			0x00000000, 0xffffffff },
13021 		{ HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
13022 			0x00000000, 0x000000ff },
13023 		{ HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
13024 			0x00000000, 0xffffffff },
13025 		{ HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
13026 			0x00000000, 0x000000ff },
13027 		{ HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
13028 			0x00000000, 0xffffffff },
13029 		{ HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
13030 			0x00000000, 0xffffffff },
13031 		{ HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
13032 			0x00000000, 0xffffffff },
13033 		{ HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
13034 			0x00000000, 0xffffffff },
13035 		{ HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
13036 			0x00000000, 0xffffffff },
13037 		{ HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
13038 			0xffffffff, 0x00000000 },
13039 		{ HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
13040 			0xffffffff, 0x00000000 },
13041 
13042 		/* Buffer Manager Control Registers. */
13043 		{ BUFMGR_MB_POOL_ADDR, TG3_FL_NOT_5750,
13044 			0x00000000, 0x007fff80 },
13045 		{ BUFMGR_MB_POOL_SIZE, TG3_FL_NOT_5750,
13046 			0x00000000, 0x007fffff },
13047 		{ BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
13048 			0x00000000, 0x0000003f },
13049 		{ BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
13050 			0x00000000, 0x000001ff },
13051 		{ BUFMGR_MB_HIGH_WATER, 0x0000,
13052 			0x00000000, 0x000001ff },
13053 		{ BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
13054 			0xffffffff, 0x00000000 },
13055 		{ BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
13056 			0xffffffff, 0x00000000 },
13057 
13058 		/* Mailbox Registers */
13059 		{ GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
13060 			0x00000000, 0x000001ff },
13061 		{ GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
13062 			0x00000000, 0x000001ff },
13063 		{ GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
13064 			0x00000000, 0x000007ff },
13065 		{ GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
13066 			0x00000000, 0x000001ff },
13067 
13068 		{ 0xffff, 0x0000, 0x00000000, 0x00000000 },
13069 	};
13070 
13071 	is_5705 = is_5750 = 0;
13072 	if (tg3_flag(tp, 5705_PLUS)) {
13073 		is_5705 = 1;
13074 		if (tg3_flag(tp, 5750_PLUS))
13075 			is_5750 = 1;
13076 	}
13077 
13078 	for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
13079 		if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
13080 			continue;
13081 
13082 		if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
13083 			continue;
13084 
13085 		if (tg3_flag(tp, IS_5788) &&
13086 		    (reg_tbl[i].flags & TG3_FL_NOT_5788))
13087 			continue;
13088 
13089 		if (is_5750 && (reg_tbl[i].flags & TG3_FL_NOT_5750))
13090 			continue;
13091 
13092 		offset = (u32) reg_tbl[i].offset;
13093 		read_mask = reg_tbl[i].read_mask;
13094 		write_mask = reg_tbl[i].write_mask;
13095 
13096 		/* Save the original register content */
13097 		save_val = tr32(offset);
13098 
13099 		/* Determine the read-only value. */
13100 		read_val = save_val & read_mask;
13101 
13102 		/* Write zero to the register, then make sure the read-only bits
13103 		 * are not changed and the read/write bits are all zeros.
13104 		 */
13105 		tw32(offset, 0);
13106 
13107 		val = tr32(offset);
13108 
13109 		/* Test the read-only and read/write bits. */
13110 		if (((val & read_mask) != read_val) || (val & write_mask))
13111 			goto out;
13112 
13113 		/* Write ones to all the bits defined by RdMask and WrMask, then
13114 		 * make sure the read-only bits are not changed and the
13115 		 * read/write bits are all ones.
13116 		 */
13117 		tw32(offset, read_mask | write_mask);
13118 
13119 		val = tr32(offset);
13120 
13121 		/* Test the read-only bits. */
13122 		if ((val & read_mask) != read_val)
13123 			goto out;
13124 
13125 		/* Test the read/write bits. */
13126 		if ((val & write_mask) != write_mask)
13127 			goto out;
13128 
13129 		tw32(offset, save_val);
13130 	}
13131 
13132 	return 0;
13133 
13134 out:
13135 	if (netif_msg_hw(tp))
13136 		netdev_err(tp->dev,
13137 			   "Register test failed at offset %x\n", offset);
13138 	tw32(offset, save_val);
13139 	return -EIO;
13140 }
13141 
13142 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
13143 {
13144 	static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
13145 	int i;
13146 	u32 j;
13147 
13148 	for (i = 0; i < ARRAY_SIZE(test_pattern); i++) {
13149 		for (j = 0; j < len; j += 4) {
13150 			u32 val;
13151 
13152 			tg3_write_mem(tp, offset + j, test_pattern[i]);
13153 			tg3_read_mem(tp, offset + j, &val);
13154 			if (val != test_pattern[i])
13155 				return -EIO;
13156 		}
13157 	}
13158 	return 0;
13159 }
13160 
13161 static int tg3_test_memory(struct tg3 *tp)
13162 {
13163 	static struct mem_entry {
13164 		u32 offset;
13165 		u32 len;
13166 	} mem_tbl_570x[] = {
13167 		{ 0x00000000, 0x00b50},
13168 		{ 0x00002000, 0x1c000},
13169 		{ 0xffffffff, 0x00000}
13170 	}, mem_tbl_5705[] = {
13171 		{ 0x00000100, 0x0000c},
13172 		{ 0x00000200, 0x00008},
13173 		{ 0x00004000, 0x00800},
13174 		{ 0x00006000, 0x01000},
13175 		{ 0x00008000, 0x02000},
13176 		{ 0x00010000, 0x0e000},
13177 		{ 0xffffffff, 0x00000}
13178 	}, mem_tbl_5755[] = {
13179 		{ 0x00000200, 0x00008},
13180 		{ 0x00004000, 0x00800},
13181 		{ 0x00006000, 0x00800},
13182 		{ 0x00008000, 0x02000},
13183 		{ 0x00010000, 0x0c000},
13184 		{ 0xffffffff, 0x00000}
13185 	}, mem_tbl_5906[] = {
13186 		{ 0x00000200, 0x00008},
13187 		{ 0x00004000, 0x00400},
13188 		{ 0x00006000, 0x00400},
13189 		{ 0x00008000, 0x01000},
13190 		{ 0x00010000, 0x01000},
13191 		{ 0xffffffff, 0x00000}
13192 	}, mem_tbl_5717[] = {
13193 		{ 0x00000200, 0x00008},
13194 		{ 0x00010000, 0x0a000},
13195 		{ 0x00020000, 0x13c00},
13196 		{ 0xffffffff, 0x00000}
13197 	}, mem_tbl_57765[] = {
13198 		{ 0x00000200, 0x00008},
13199 		{ 0x00004000, 0x00800},
13200 		{ 0x00006000, 0x09800},
13201 		{ 0x00010000, 0x0a000},
13202 		{ 0xffffffff, 0x00000}
13203 	};
13204 	struct mem_entry *mem_tbl;
13205 	int err = 0;
13206 	int i;
13207 
13208 	if (tg3_flag(tp, 5717_PLUS))
13209 		mem_tbl = mem_tbl_5717;
13210 	else if (tg3_flag(tp, 57765_CLASS) ||
13211 		 tg3_asic_rev(tp) == ASIC_REV_5762)
13212 		mem_tbl = mem_tbl_57765;
13213 	else if (tg3_flag(tp, 5755_PLUS))
13214 		mem_tbl = mem_tbl_5755;
13215 	else if (tg3_asic_rev(tp) == ASIC_REV_5906)
13216 		mem_tbl = mem_tbl_5906;
13217 	else if (tg3_flag(tp, 5705_PLUS))
13218 		mem_tbl = mem_tbl_5705;
13219 	else
13220 		mem_tbl = mem_tbl_570x;
13221 
13222 	for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
13223 		err = tg3_do_mem_test(tp, mem_tbl[i].offset, mem_tbl[i].len);
13224 		if (err)
13225 			break;
13226 	}
13227 
13228 	return err;
13229 }
13230 
13231 #define TG3_TSO_MSS		500
13232 
13233 #define TG3_TSO_IP_HDR_LEN	20
13234 #define TG3_TSO_TCP_HDR_LEN	20
13235 #define TG3_TSO_TCP_OPT_LEN	12
13236 
13237 static const u8 tg3_tso_header[] = {
13238 0x08, 0x00,
13239 0x45, 0x00, 0x00, 0x00,
13240 0x00, 0x00, 0x40, 0x00,
13241 0x40, 0x06, 0x00, 0x00,
13242 0x0a, 0x00, 0x00, 0x01,
13243 0x0a, 0x00, 0x00, 0x02,
13244 0x0d, 0x00, 0xe0, 0x00,
13245 0x00, 0x00, 0x01, 0x00,
13246 0x00, 0x00, 0x02, 0x00,
13247 0x80, 0x10, 0x10, 0x00,
13248 0x14, 0x09, 0x00, 0x00,
13249 0x01, 0x01, 0x08, 0x0a,
13250 0x11, 0x11, 0x11, 0x11,
13251 0x11, 0x11, 0x11, 0x11,
13252 };
13253 
13254 static int tg3_run_loopback(struct tg3 *tp, u32 pktsz, bool tso_loopback)
13255 {
13256 	u32 rx_start_idx, rx_idx, tx_idx, opaque_key;
13257 	u32 base_flags = 0, mss = 0, desc_idx, coal_now, data_off, val;
13258 	u32 budget;
13259 	struct sk_buff *skb;
13260 	u8 *tx_data, *rx_data;
13261 	dma_addr_t map;
13262 	int num_pkts, tx_len, rx_len, i, err;
13263 	struct tg3_rx_buffer_desc *desc;
13264 	struct tg3_napi *tnapi, *rnapi;
13265 	struct tg3_rx_prodring_set *tpr = &tp->napi[0].prodring;
13266 
13267 	tnapi = &tp->napi[0];
13268 	rnapi = &tp->napi[0];
13269 	if (tp->irq_cnt > 1) {
13270 		if (tg3_flag(tp, ENABLE_RSS))
13271 			rnapi = &tp->napi[1];
13272 		if (tg3_flag(tp, ENABLE_TSS))
13273 			tnapi = &tp->napi[1];
13274 	}
13275 	coal_now = tnapi->coal_now | rnapi->coal_now;
13276 
13277 	err = -EIO;
13278 
13279 	tx_len = pktsz;
13280 	skb = netdev_alloc_skb(tp->dev, tx_len);
13281 	if (!skb)
13282 		return -ENOMEM;
13283 
13284 	tx_data = skb_put(skb, tx_len);
13285 	memcpy(tx_data, tp->dev->dev_addr, ETH_ALEN);
13286 	memset(tx_data + ETH_ALEN, 0x0, 8);
13287 
13288 	tw32(MAC_RX_MTU_SIZE, tx_len + ETH_FCS_LEN);
13289 
13290 	if (tso_loopback) {
13291 		struct iphdr *iph = (struct iphdr *)&tx_data[ETH_HLEN];
13292 
13293 		u32 hdr_len = TG3_TSO_IP_HDR_LEN + TG3_TSO_TCP_HDR_LEN +
13294 			      TG3_TSO_TCP_OPT_LEN;
13295 
13296 		memcpy(tx_data + ETH_ALEN * 2, tg3_tso_header,
13297 		       sizeof(tg3_tso_header));
13298 		mss = TG3_TSO_MSS;
13299 
13300 		val = tx_len - ETH_ALEN * 2 - sizeof(tg3_tso_header);
13301 		num_pkts = DIV_ROUND_UP(val, TG3_TSO_MSS);
13302 
13303 		/* Set the total length field in the IP header */
13304 		iph->tot_len = htons((u16)(mss + hdr_len));
13305 
13306 		base_flags = (TXD_FLAG_CPU_PRE_DMA |
13307 			      TXD_FLAG_CPU_POST_DMA);
13308 
13309 		if (tg3_flag(tp, HW_TSO_1) ||
13310 		    tg3_flag(tp, HW_TSO_2) ||
13311 		    tg3_flag(tp, HW_TSO_3)) {
13312 			struct tcphdr *th;
13313 			val = ETH_HLEN + TG3_TSO_IP_HDR_LEN;
13314 			th = (struct tcphdr *)&tx_data[val];
13315 			th->check = 0;
13316 		} else
13317 			base_flags |= TXD_FLAG_TCPUDP_CSUM;
13318 
13319 		if (tg3_flag(tp, HW_TSO_3)) {
13320 			mss |= (hdr_len & 0xc) << 12;
13321 			if (hdr_len & 0x10)
13322 				base_flags |= 0x00000010;
13323 			base_flags |= (hdr_len & 0x3e0) << 5;
13324 		} else if (tg3_flag(tp, HW_TSO_2))
13325 			mss |= hdr_len << 9;
13326 		else if (tg3_flag(tp, HW_TSO_1) ||
13327 			 tg3_asic_rev(tp) == ASIC_REV_5705) {
13328 			mss |= (TG3_TSO_TCP_OPT_LEN << 9);
13329 		} else {
13330 			base_flags |= (TG3_TSO_TCP_OPT_LEN << 10);
13331 		}
13332 
13333 		data_off = ETH_ALEN * 2 + sizeof(tg3_tso_header);
13334 	} else {
13335 		num_pkts = 1;
13336 		data_off = ETH_HLEN;
13337 
13338 		if (tg3_flag(tp, USE_JUMBO_BDFLAG) &&
13339 		    tx_len > VLAN_ETH_FRAME_LEN)
13340 			base_flags |= TXD_FLAG_JMB_PKT;
13341 	}
13342 
13343 	for (i = data_off; i < tx_len; i++)
13344 		tx_data[i] = (u8) (i & 0xff);
13345 
13346 	map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
13347 	if (pci_dma_mapping_error(tp->pdev, map)) {
13348 		dev_kfree_skb(skb);
13349 		return -EIO;
13350 	}
13351 
13352 	val = tnapi->tx_prod;
13353 	tnapi->tx_buffers[val].skb = skb;
13354 	dma_unmap_addr_set(&tnapi->tx_buffers[val], mapping, map);
13355 
13356 	tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
13357 	       rnapi->coal_now);
13358 
13359 	udelay(10);
13360 
13361 	rx_start_idx = rnapi->hw_status->idx[0].rx_producer;
13362 
13363 	budget = tg3_tx_avail(tnapi);
13364 	if (tg3_tx_frag_set(tnapi, &val, &budget, map, tx_len,
13365 			    base_flags | TXD_FLAG_END, mss, 0)) {
13366 		tnapi->tx_buffers[val].skb = NULL;
13367 		dev_kfree_skb(skb);
13368 		return -EIO;
13369 	}
13370 
13371 	tnapi->tx_prod++;
13372 
13373 	/* Sync BD data before updating mailbox */
13374 	wmb();
13375 
13376 	tw32_tx_mbox(tnapi->prodmbox, tnapi->tx_prod);
13377 	tr32_mailbox(tnapi->prodmbox);
13378 
13379 	udelay(10);
13380 
13381 	/* 350 usec to allow enough time on some 10/100 Mbps devices.  */
13382 	for (i = 0; i < 35; i++) {
13383 		tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
13384 		       coal_now);
13385 
13386 		udelay(10);
13387 
13388 		tx_idx = tnapi->hw_status->idx[0].tx_consumer;
13389 		rx_idx = rnapi->hw_status->idx[0].rx_producer;
13390 		if ((tx_idx == tnapi->tx_prod) &&
13391 		    (rx_idx == (rx_start_idx + num_pkts)))
13392 			break;
13393 	}
13394 
13395 	tg3_tx_skb_unmap(tnapi, tnapi->tx_prod - 1, -1);
13396 	dev_kfree_skb(skb);
13397 
13398 	if (tx_idx != tnapi->tx_prod)
13399 		goto out;
13400 
13401 	if (rx_idx != rx_start_idx + num_pkts)
13402 		goto out;
13403 
13404 	val = data_off;
13405 	while (rx_idx != rx_start_idx) {
13406 		desc = &rnapi->rx_rcb[rx_start_idx++];
13407 		desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
13408 		opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
13409 
13410 		if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
13411 		    (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
13412 			goto out;
13413 
13414 		rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT)
13415 			 - ETH_FCS_LEN;
13416 
13417 		if (!tso_loopback) {
13418 			if (rx_len != tx_len)
13419 				goto out;
13420 
13421 			if (pktsz <= TG3_RX_STD_DMA_SZ - ETH_FCS_LEN) {
13422 				if (opaque_key != RXD_OPAQUE_RING_STD)
13423 					goto out;
13424 			} else {
13425 				if (opaque_key != RXD_OPAQUE_RING_JUMBO)
13426 					goto out;
13427 			}
13428 		} else if ((desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
13429 			   (desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
13430 			    >> RXD_TCPCSUM_SHIFT != 0xffff) {
13431 			goto out;
13432 		}
13433 
13434 		if (opaque_key == RXD_OPAQUE_RING_STD) {
13435 			rx_data = tpr->rx_std_buffers[desc_idx].data;
13436 			map = dma_unmap_addr(&tpr->rx_std_buffers[desc_idx],
13437 					     mapping);
13438 		} else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
13439 			rx_data = tpr->rx_jmb_buffers[desc_idx].data;
13440 			map = dma_unmap_addr(&tpr->rx_jmb_buffers[desc_idx],
13441 					     mapping);
13442 		} else
13443 			goto out;
13444 
13445 		pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len,
13446 					    PCI_DMA_FROMDEVICE);
13447 
13448 		rx_data += TG3_RX_OFFSET(tp);
13449 		for (i = data_off; i < rx_len; i++, val++) {
13450 			if (*(rx_data + i) != (u8) (val & 0xff))
13451 				goto out;
13452 		}
13453 	}
13454 
13455 	err = 0;
13456 
13457 	/* tg3_free_rings will unmap and free the rx_data */
13458 out:
13459 	return err;
13460 }
13461 
13462 #define TG3_STD_LOOPBACK_FAILED		1
13463 #define TG3_JMB_LOOPBACK_FAILED		2
13464 #define TG3_TSO_LOOPBACK_FAILED		4
13465 #define TG3_LOOPBACK_FAILED \
13466 	(TG3_STD_LOOPBACK_FAILED | \
13467 	 TG3_JMB_LOOPBACK_FAILED | \
13468 	 TG3_TSO_LOOPBACK_FAILED)
13469 
13470 static int tg3_test_loopback(struct tg3 *tp, u64 *data, bool do_extlpbk)
13471 {
13472 	int err = -EIO;
13473 	u32 eee_cap;
13474 	u32 jmb_pkt_sz = 9000;
13475 
13476 	if (tp->dma_limit)
13477 		jmb_pkt_sz = tp->dma_limit - ETH_HLEN;
13478 
13479 	eee_cap = tp->phy_flags & TG3_PHYFLG_EEE_CAP;
13480 	tp->phy_flags &= ~TG3_PHYFLG_EEE_CAP;
13481 
13482 	if (!netif_running(tp->dev)) {
13483 		data[TG3_MAC_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13484 		data[TG3_PHY_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13485 		if (do_extlpbk)
13486 			data[TG3_EXT_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13487 		goto done;
13488 	}
13489 
13490 	err = tg3_reset_hw(tp, true);
13491 	if (err) {
13492 		data[TG3_MAC_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13493 		data[TG3_PHY_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13494 		if (do_extlpbk)
13495 			data[TG3_EXT_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13496 		goto done;
13497 	}
13498 
13499 	if (tg3_flag(tp, ENABLE_RSS)) {
13500 		int i;
13501 
13502 		/* Reroute all rx packets to the 1st queue */
13503 		for (i = MAC_RSS_INDIR_TBL_0;
13504 		     i < MAC_RSS_INDIR_TBL_0 + TG3_RSS_INDIR_TBL_SIZE; i += 4)
13505 			tw32(i, 0x0);
13506 	}
13507 
13508 	/* HW errata - mac loopback fails in some cases on 5780.
13509 	 * Normal traffic and PHY loopback are not affected by
13510 	 * errata.  Also, the MAC loopback test is deprecated for
13511 	 * all newer ASIC revisions.
13512 	 */
13513 	if (tg3_asic_rev(tp) != ASIC_REV_5780 &&
13514 	    !tg3_flag(tp, CPMU_PRESENT)) {
13515 		tg3_mac_loopback(tp, true);
13516 
13517 		if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13518 			data[TG3_MAC_LOOPB_TEST] |= TG3_STD_LOOPBACK_FAILED;
13519 
13520 		if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13521 		    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13522 			data[TG3_MAC_LOOPB_TEST] |= TG3_JMB_LOOPBACK_FAILED;
13523 
13524 		tg3_mac_loopback(tp, false);
13525 	}
13526 
13527 	if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
13528 	    !tg3_flag(tp, USE_PHYLIB)) {
13529 		int i;
13530 
13531 		tg3_phy_lpbk_set(tp, 0, false);
13532 
13533 		/* Wait for link */
13534 		for (i = 0; i < 100; i++) {
13535 			if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
13536 				break;
13537 			mdelay(1);
13538 		}
13539 
13540 		if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13541 			data[TG3_PHY_LOOPB_TEST] |= TG3_STD_LOOPBACK_FAILED;
13542 		if (tg3_flag(tp, TSO_CAPABLE) &&
13543 		    tg3_run_loopback(tp, ETH_FRAME_LEN, true))
13544 			data[TG3_PHY_LOOPB_TEST] |= TG3_TSO_LOOPBACK_FAILED;
13545 		if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13546 		    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13547 			data[TG3_PHY_LOOPB_TEST] |= TG3_JMB_LOOPBACK_FAILED;
13548 
13549 		if (do_extlpbk) {
13550 			tg3_phy_lpbk_set(tp, 0, true);
13551 
13552 			/* All link indications report up, but the hardware
13553 			 * isn't really ready for about 20 msec.  Double it
13554 			 * to be sure.
13555 			 */
13556 			mdelay(40);
13557 
13558 			if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13559 				data[TG3_EXT_LOOPB_TEST] |=
13560 							TG3_STD_LOOPBACK_FAILED;
13561 			if (tg3_flag(tp, TSO_CAPABLE) &&
13562 			    tg3_run_loopback(tp, ETH_FRAME_LEN, true))
13563 				data[TG3_EXT_LOOPB_TEST] |=
13564 							TG3_TSO_LOOPBACK_FAILED;
13565 			if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13566 			    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13567 				data[TG3_EXT_LOOPB_TEST] |=
13568 							TG3_JMB_LOOPBACK_FAILED;
13569 		}
13570 
13571 		/* Re-enable gphy autopowerdown. */
13572 		if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
13573 			tg3_phy_toggle_apd(tp, true);
13574 	}
13575 
13576 	err = (data[TG3_MAC_LOOPB_TEST] | data[TG3_PHY_LOOPB_TEST] |
13577 	       data[TG3_EXT_LOOPB_TEST]) ? -EIO : 0;
13578 
13579 done:
13580 	tp->phy_flags |= eee_cap;
13581 
13582 	return err;
13583 }
13584 
13585 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
13586 			  u64 *data)
13587 {
13588 	struct tg3 *tp = netdev_priv(dev);
13589 	bool doextlpbk = etest->flags & ETH_TEST_FL_EXTERNAL_LB;
13590 
13591 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
13592 		if (tg3_power_up(tp)) {
13593 			etest->flags |= ETH_TEST_FL_FAILED;
13594 			memset(data, 1, sizeof(u64) * TG3_NUM_TEST);
13595 			return;
13596 		}
13597 		tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
13598 	}
13599 
13600 	memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
13601 
13602 	if (tg3_test_nvram(tp) != 0) {
13603 		etest->flags |= ETH_TEST_FL_FAILED;
13604 		data[TG3_NVRAM_TEST] = 1;
13605 	}
13606 	if (!doextlpbk && tg3_test_link(tp)) {
13607 		etest->flags |= ETH_TEST_FL_FAILED;
13608 		data[TG3_LINK_TEST] = 1;
13609 	}
13610 	if (etest->flags & ETH_TEST_FL_OFFLINE) {
13611 		int err, err2 = 0, irq_sync = 0;
13612 
13613 		if (netif_running(dev)) {
13614 			tg3_phy_stop(tp);
13615 			tg3_netif_stop(tp);
13616 			irq_sync = 1;
13617 		}
13618 
13619 		tg3_full_lock(tp, irq_sync);
13620 		tg3_halt(tp, RESET_KIND_SUSPEND, 1);
13621 		err = tg3_nvram_lock(tp);
13622 		tg3_halt_cpu(tp, RX_CPU_BASE);
13623 		if (!tg3_flag(tp, 5705_PLUS))
13624 			tg3_halt_cpu(tp, TX_CPU_BASE);
13625 		if (!err)
13626 			tg3_nvram_unlock(tp);
13627 
13628 		if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
13629 			tg3_phy_reset(tp);
13630 
13631 		if (tg3_test_registers(tp) != 0) {
13632 			etest->flags |= ETH_TEST_FL_FAILED;
13633 			data[TG3_REGISTER_TEST] = 1;
13634 		}
13635 
13636 		if (tg3_test_memory(tp) != 0) {
13637 			etest->flags |= ETH_TEST_FL_FAILED;
13638 			data[TG3_MEMORY_TEST] = 1;
13639 		}
13640 
13641 		if (doextlpbk)
13642 			etest->flags |= ETH_TEST_FL_EXTERNAL_LB_DONE;
13643 
13644 		if (tg3_test_loopback(tp, data, doextlpbk))
13645 			etest->flags |= ETH_TEST_FL_FAILED;
13646 
13647 		tg3_full_unlock(tp);
13648 
13649 		if (tg3_test_interrupt(tp) != 0) {
13650 			etest->flags |= ETH_TEST_FL_FAILED;
13651 			data[TG3_INTERRUPT_TEST] = 1;
13652 		}
13653 
13654 		tg3_full_lock(tp, 0);
13655 
13656 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
13657 		if (netif_running(dev)) {
13658 			tg3_flag_set(tp, INIT_COMPLETE);
13659 			err2 = tg3_restart_hw(tp, true);
13660 			if (!err2)
13661 				tg3_netif_start(tp);
13662 		}
13663 
13664 		tg3_full_unlock(tp);
13665 
13666 		if (irq_sync && !err2)
13667 			tg3_phy_start(tp);
13668 	}
13669 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
13670 		tg3_power_down_prepare(tp);
13671 
13672 }
13673 
13674 static int tg3_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
13675 {
13676 	struct tg3 *tp = netdev_priv(dev);
13677 	struct hwtstamp_config stmpconf;
13678 
13679 	if (!tg3_flag(tp, PTP_CAPABLE))
13680 		return -EOPNOTSUPP;
13681 
13682 	if (copy_from_user(&stmpconf, ifr->ifr_data, sizeof(stmpconf)))
13683 		return -EFAULT;
13684 
13685 	if (stmpconf.flags)
13686 		return -EINVAL;
13687 
13688 	if (stmpconf.tx_type != HWTSTAMP_TX_ON &&
13689 	    stmpconf.tx_type != HWTSTAMP_TX_OFF)
13690 		return -ERANGE;
13691 
13692 	switch (stmpconf.rx_filter) {
13693 	case HWTSTAMP_FILTER_NONE:
13694 		tp->rxptpctl = 0;
13695 		break;
13696 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
13697 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13698 			       TG3_RX_PTP_CTL_ALL_V1_EVENTS;
13699 		break;
13700 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
13701 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13702 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13703 		break;
13704 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
13705 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13706 			       TG3_RX_PTP_CTL_DELAY_REQ;
13707 		break;
13708 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
13709 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13710 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13711 		break;
13712 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
13713 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13714 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13715 		break;
13716 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
13717 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13718 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13719 		break;
13720 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
13721 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13722 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13723 		break;
13724 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
13725 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13726 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13727 		break;
13728 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
13729 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13730 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13731 		break;
13732 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
13733 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13734 			       TG3_RX_PTP_CTL_DELAY_REQ;
13735 		break;
13736 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
13737 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13738 			       TG3_RX_PTP_CTL_DELAY_REQ;
13739 		break;
13740 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
13741 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13742 			       TG3_RX_PTP_CTL_DELAY_REQ;
13743 		break;
13744 	default:
13745 		return -ERANGE;
13746 	}
13747 
13748 	if (netif_running(dev) && tp->rxptpctl)
13749 		tw32(TG3_RX_PTP_CTL,
13750 		     tp->rxptpctl | TG3_RX_PTP_CTL_HWTS_INTERLOCK);
13751 
13752 	if (stmpconf.tx_type == HWTSTAMP_TX_ON)
13753 		tg3_flag_set(tp, TX_TSTAMP_EN);
13754 	else
13755 		tg3_flag_clear(tp, TX_TSTAMP_EN);
13756 
13757 	return copy_to_user(ifr->ifr_data, &stmpconf, sizeof(stmpconf)) ?
13758 		-EFAULT : 0;
13759 }
13760 
13761 static int tg3_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
13762 {
13763 	struct tg3 *tp = netdev_priv(dev);
13764 	struct hwtstamp_config stmpconf;
13765 
13766 	if (!tg3_flag(tp, PTP_CAPABLE))
13767 		return -EOPNOTSUPP;
13768 
13769 	stmpconf.flags = 0;
13770 	stmpconf.tx_type = (tg3_flag(tp, TX_TSTAMP_EN) ?
13771 			    HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF);
13772 
13773 	switch (tp->rxptpctl) {
13774 	case 0:
13775 		stmpconf.rx_filter = HWTSTAMP_FILTER_NONE;
13776 		break;
13777 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_ALL_V1_EVENTS:
13778 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
13779 		break;
13780 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13781 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
13782 		break;
13783 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13784 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
13785 		break;
13786 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13787 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
13788 		break;
13789 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13790 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
13791 		break;
13792 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13793 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
13794 		break;
13795 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13796 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
13797 		break;
13798 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13799 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_SYNC;
13800 		break;
13801 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13802 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
13803 		break;
13804 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13805 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
13806 		break;
13807 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13808 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ;
13809 		break;
13810 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13811 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
13812 		break;
13813 	default:
13814 		WARN_ON_ONCE(1);
13815 		return -ERANGE;
13816 	}
13817 
13818 	return copy_to_user(ifr->ifr_data, &stmpconf, sizeof(stmpconf)) ?
13819 		-EFAULT : 0;
13820 }
13821 
13822 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
13823 {
13824 	struct mii_ioctl_data *data = if_mii(ifr);
13825 	struct tg3 *tp = netdev_priv(dev);
13826 	int err;
13827 
13828 	if (tg3_flag(tp, USE_PHYLIB)) {
13829 		struct phy_device *phydev;
13830 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
13831 			return -EAGAIN;
13832 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
13833 		return phy_mii_ioctl(phydev, ifr, cmd);
13834 	}
13835 
13836 	switch (cmd) {
13837 	case SIOCGMIIPHY:
13838 		data->phy_id = tp->phy_addr;
13839 
13840 		/* fallthru */
13841 	case SIOCGMIIREG: {
13842 		u32 mii_regval;
13843 
13844 		if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
13845 			break;			/* We have no PHY */
13846 
13847 		if (!netif_running(dev))
13848 			return -EAGAIN;
13849 
13850 		spin_lock_bh(&tp->lock);
13851 		err = __tg3_readphy(tp, data->phy_id & 0x1f,
13852 				    data->reg_num & 0x1f, &mii_regval);
13853 		spin_unlock_bh(&tp->lock);
13854 
13855 		data->val_out = mii_regval;
13856 
13857 		return err;
13858 	}
13859 
13860 	case SIOCSMIIREG:
13861 		if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
13862 			break;			/* We have no PHY */
13863 
13864 		if (!netif_running(dev))
13865 			return -EAGAIN;
13866 
13867 		spin_lock_bh(&tp->lock);
13868 		err = __tg3_writephy(tp, data->phy_id & 0x1f,
13869 				     data->reg_num & 0x1f, data->val_in);
13870 		spin_unlock_bh(&tp->lock);
13871 
13872 		return err;
13873 
13874 	case SIOCSHWTSTAMP:
13875 		return tg3_hwtstamp_set(dev, ifr);
13876 
13877 	case SIOCGHWTSTAMP:
13878 		return tg3_hwtstamp_get(dev, ifr);
13879 
13880 	default:
13881 		/* do nothing */
13882 		break;
13883 	}
13884 	return -EOPNOTSUPP;
13885 }
13886 
13887 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
13888 {
13889 	struct tg3 *tp = netdev_priv(dev);
13890 
13891 	memcpy(ec, &tp->coal, sizeof(*ec));
13892 	return 0;
13893 }
13894 
13895 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
13896 {
13897 	struct tg3 *tp = netdev_priv(dev);
13898 	u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
13899 	u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
13900 
13901 	if (!tg3_flag(tp, 5705_PLUS)) {
13902 		max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
13903 		max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
13904 		max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
13905 		min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
13906 	}
13907 
13908 	if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
13909 	    (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
13910 	    (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
13911 	    (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
13912 	    (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
13913 	    (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
13914 	    (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
13915 	    (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
13916 	    (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
13917 	    (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
13918 		return -EINVAL;
13919 
13920 	/* No rx interrupts will be generated if both are zero */
13921 	if ((ec->rx_coalesce_usecs == 0) &&
13922 	    (ec->rx_max_coalesced_frames == 0))
13923 		return -EINVAL;
13924 
13925 	/* No tx interrupts will be generated if both are zero */
13926 	if ((ec->tx_coalesce_usecs == 0) &&
13927 	    (ec->tx_max_coalesced_frames == 0))
13928 		return -EINVAL;
13929 
13930 	/* Only copy relevant parameters, ignore all others. */
13931 	tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
13932 	tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
13933 	tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
13934 	tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
13935 	tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
13936 	tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
13937 	tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
13938 	tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
13939 	tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
13940 
13941 	if (netif_running(dev)) {
13942 		tg3_full_lock(tp, 0);
13943 		__tg3_set_coalesce(tp, &tp->coal);
13944 		tg3_full_unlock(tp);
13945 	}
13946 	return 0;
13947 }
13948 
13949 static int tg3_set_eee(struct net_device *dev, struct ethtool_eee *edata)
13950 {
13951 	struct tg3 *tp = netdev_priv(dev);
13952 
13953 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
13954 		netdev_warn(tp->dev, "Board does not support EEE!\n");
13955 		return -EOPNOTSUPP;
13956 	}
13957 
13958 	if (edata->advertised != tp->eee.advertised) {
13959 		netdev_warn(tp->dev,
13960 			    "Direct manipulation of EEE advertisement is not supported\n");
13961 		return -EINVAL;
13962 	}
13963 
13964 	if (edata->tx_lpi_timer > TG3_CPMU_DBTMR1_LNKIDLE_MAX) {
13965 		netdev_warn(tp->dev,
13966 			    "Maximal Tx Lpi timer supported is %#x(u)\n",
13967 			    TG3_CPMU_DBTMR1_LNKIDLE_MAX);
13968 		return -EINVAL;
13969 	}
13970 
13971 	tp->eee = *edata;
13972 
13973 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
13974 	tg3_warn_mgmt_link_flap(tp);
13975 
13976 	if (netif_running(tp->dev)) {
13977 		tg3_full_lock(tp, 0);
13978 		tg3_setup_eee(tp);
13979 		tg3_phy_reset(tp);
13980 		tg3_full_unlock(tp);
13981 	}
13982 
13983 	return 0;
13984 }
13985 
13986 static int tg3_get_eee(struct net_device *dev, struct ethtool_eee *edata)
13987 {
13988 	struct tg3 *tp = netdev_priv(dev);
13989 
13990 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
13991 		netdev_warn(tp->dev,
13992 			    "Board does not support EEE!\n");
13993 		return -EOPNOTSUPP;
13994 	}
13995 
13996 	*edata = tp->eee;
13997 	return 0;
13998 }
13999 
14000 static const struct ethtool_ops tg3_ethtool_ops = {
14001 	.get_settings		= tg3_get_settings,
14002 	.set_settings		= tg3_set_settings,
14003 	.get_drvinfo		= tg3_get_drvinfo,
14004 	.get_regs_len		= tg3_get_regs_len,
14005 	.get_regs		= tg3_get_regs,
14006 	.get_wol		= tg3_get_wol,
14007 	.set_wol		= tg3_set_wol,
14008 	.get_msglevel		= tg3_get_msglevel,
14009 	.set_msglevel		= tg3_set_msglevel,
14010 	.nway_reset		= tg3_nway_reset,
14011 	.get_link		= ethtool_op_get_link,
14012 	.get_eeprom_len		= tg3_get_eeprom_len,
14013 	.get_eeprom		= tg3_get_eeprom,
14014 	.set_eeprom		= tg3_set_eeprom,
14015 	.get_ringparam		= tg3_get_ringparam,
14016 	.set_ringparam		= tg3_set_ringparam,
14017 	.get_pauseparam		= tg3_get_pauseparam,
14018 	.set_pauseparam		= tg3_set_pauseparam,
14019 	.self_test		= tg3_self_test,
14020 	.get_strings		= tg3_get_strings,
14021 	.set_phys_id		= tg3_set_phys_id,
14022 	.get_ethtool_stats	= tg3_get_ethtool_stats,
14023 	.get_coalesce		= tg3_get_coalesce,
14024 	.set_coalesce		= tg3_set_coalesce,
14025 	.get_sset_count		= tg3_get_sset_count,
14026 	.get_rxnfc		= tg3_get_rxnfc,
14027 	.get_rxfh_indir_size    = tg3_get_rxfh_indir_size,
14028 	.get_rxfh_indir		= tg3_get_rxfh_indir,
14029 	.set_rxfh_indir		= tg3_set_rxfh_indir,
14030 	.get_channels		= tg3_get_channels,
14031 	.set_channels		= tg3_set_channels,
14032 	.get_ts_info		= tg3_get_ts_info,
14033 	.get_eee		= tg3_get_eee,
14034 	.set_eee		= tg3_set_eee,
14035 };
14036 
14037 static struct rtnl_link_stats64 *tg3_get_stats64(struct net_device *dev,
14038 						struct rtnl_link_stats64 *stats)
14039 {
14040 	struct tg3 *tp = netdev_priv(dev);
14041 
14042 	spin_lock_bh(&tp->lock);
14043 	if (!tp->hw_stats) {
14044 		spin_unlock_bh(&tp->lock);
14045 		return &tp->net_stats_prev;
14046 	}
14047 
14048 	tg3_get_nstats(tp, stats);
14049 	spin_unlock_bh(&tp->lock);
14050 
14051 	return stats;
14052 }
14053 
14054 static void tg3_set_rx_mode(struct net_device *dev)
14055 {
14056 	struct tg3 *tp = netdev_priv(dev);
14057 
14058 	if (!netif_running(dev))
14059 		return;
14060 
14061 	tg3_full_lock(tp, 0);
14062 	__tg3_set_rx_mode(dev);
14063 	tg3_full_unlock(tp);
14064 }
14065 
14066 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
14067 			       int new_mtu)
14068 {
14069 	dev->mtu = new_mtu;
14070 
14071 	if (new_mtu > ETH_DATA_LEN) {
14072 		if (tg3_flag(tp, 5780_CLASS)) {
14073 			netdev_update_features(dev);
14074 			tg3_flag_clear(tp, TSO_CAPABLE);
14075 		} else {
14076 			tg3_flag_set(tp, JUMBO_RING_ENABLE);
14077 		}
14078 	} else {
14079 		if (tg3_flag(tp, 5780_CLASS)) {
14080 			tg3_flag_set(tp, TSO_CAPABLE);
14081 			netdev_update_features(dev);
14082 		}
14083 		tg3_flag_clear(tp, JUMBO_RING_ENABLE);
14084 	}
14085 }
14086 
14087 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
14088 {
14089 	struct tg3 *tp = netdev_priv(dev);
14090 	int err;
14091 	bool reset_phy = false;
14092 
14093 	if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
14094 		return -EINVAL;
14095 
14096 	if (!netif_running(dev)) {
14097 		/* We'll just catch it later when the
14098 		 * device is up'd.
14099 		 */
14100 		tg3_set_mtu(dev, tp, new_mtu);
14101 		return 0;
14102 	}
14103 
14104 	tg3_phy_stop(tp);
14105 
14106 	tg3_netif_stop(tp);
14107 
14108 	tg3_set_mtu(dev, tp, new_mtu);
14109 
14110 	tg3_full_lock(tp, 1);
14111 
14112 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
14113 
14114 	/* Reset PHY, otherwise the read DMA engine will be in a mode that
14115 	 * breaks all requests to 256 bytes.
14116 	 */
14117 	if (tg3_asic_rev(tp) == ASIC_REV_57766)
14118 		reset_phy = true;
14119 
14120 	err = tg3_restart_hw(tp, reset_phy);
14121 
14122 	if (!err)
14123 		tg3_netif_start(tp);
14124 
14125 	tg3_full_unlock(tp);
14126 
14127 	if (!err)
14128 		tg3_phy_start(tp);
14129 
14130 	return err;
14131 }
14132 
14133 static const struct net_device_ops tg3_netdev_ops = {
14134 	.ndo_open		= tg3_open,
14135 	.ndo_stop		= tg3_close,
14136 	.ndo_start_xmit		= tg3_start_xmit,
14137 	.ndo_get_stats64	= tg3_get_stats64,
14138 	.ndo_validate_addr	= eth_validate_addr,
14139 	.ndo_set_rx_mode	= tg3_set_rx_mode,
14140 	.ndo_set_mac_address	= tg3_set_mac_addr,
14141 	.ndo_do_ioctl		= tg3_ioctl,
14142 	.ndo_tx_timeout		= tg3_tx_timeout,
14143 	.ndo_change_mtu		= tg3_change_mtu,
14144 	.ndo_fix_features	= tg3_fix_features,
14145 	.ndo_set_features	= tg3_set_features,
14146 #ifdef CONFIG_NET_POLL_CONTROLLER
14147 	.ndo_poll_controller	= tg3_poll_controller,
14148 #endif
14149 };
14150 
14151 static void tg3_get_eeprom_size(struct tg3 *tp)
14152 {
14153 	u32 cursize, val, magic;
14154 
14155 	tp->nvram_size = EEPROM_CHIP_SIZE;
14156 
14157 	if (tg3_nvram_read(tp, 0, &magic) != 0)
14158 		return;
14159 
14160 	if ((magic != TG3_EEPROM_MAGIC) &&
14161 	    ((magic & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW) &&
14162 	    ((magic & TG3_EEPROM_MAGIC_HW_MSK) != TG3_EEPROM_MAGIC_HW))
14163 		return;
14164 
14165 	/*
14166 	 * Size the chip by reading offsets at increasing powers of two.
14167 	 * When we encounter our validation signature, we know the addressing
14168 	 * has wrapped around, and thus have our chip size.
14169 	 */
14170 	cursize = 0x10;
14171 
14172 	while (cursize < tp->nvram_size) {
14173 		if (tg3_nvram_read(tp, cursize, &val) != 0)
14174 			return;
14175 
14176 		if (val == magic)
14177 			break;
14178 
14179 		cursize <<= 1;
14180 	}
14181 
14182 	tp->nvram_size = cursize;
14183 }
14184 
14185 static void tg3_get_nvram_size(struct tg3 *tp)
14186 {
14187 	u32 val;
14188 
14189 	if (tg3_flag(tp, NO_NVRAM) || tg3_nvram_read(tp, 0, &val) != 0)
14190 		return;
14191 
14192 	/* Selfboot format */
14193 	if (val != TG3_EEPROM_MAGIC) {
14194 		tg3_get_eeprom_size(tp);
14195 		return;
14196 	}
14197 
14198 	if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
14199 		if (val != 0) {
14200 			/* This is confusing.  We want to operate on the
14201 			 * 16-bit value at offset 0xf2.  The tg3_nvram_read()
14202 			 * call will read from NVRAM and byteswap the data
14203 			 * according to the byteswapping settings for all
14204 			 * other register accesses.  This ensures the data we
14205 			 * want will always reside in the lower 16-bits.
14206 			 * However, the data in NVRAM is in LE format, which
14207 			 * means the data from the NVRAM read will always be
14208 			 * opposite the endianness of the CPU.  The 16-bit
14209 			 * byteswap then brings the data to CPU endianness.
14210 			 */
14211 			tp->nvram_size = swab16((u16)(val & 0x0000ffff)) * 1024;
14212 			return;
14213 		}
14214 	}
14215 	tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14216 }
14217 
14218 static void tg3_get_nvram_info(struct tg3 *tp)
14219 {
14220 	u32 nvcfg1;
14221 
14222 	nvcfg1 = tr32(NVRAM_CFG1);
14223 	if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
14224 		tg3_flag_set(tp, FLASH);
14225 	} else {
14226 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14227 		tw32(NVRAM_CFG1, nvcfg1);
14228 	}
14229 
14230 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
14231 	    tg3_flag(tp, 5780_CLASS)) {
14232 		switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
14233 		case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
14234 			tp->nvram_jedecnum = JEDEC_ATMEL;
14235 			tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
14236 			tg3_flag_set(tp, NVRAM_BUFFERED);
14237 			break;
14238 		case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
14239 			tp->nvram_jedecnum = JEDEC_ATMEL;
14240 			tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
14241 			break;
14242 		case FLASH_VENDOR_ATMEL_EEPROM:
14243 			tp->nvram_jedecnum = JEDEC_ATMEL;
14244 			tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14245 			tg3_flag_set(tp, NVRAM_BUFFERED);
14246 			break;
14247 		case FLASH_VENDOR_ST:
14248 			tp->nvram_jedecnum = JEDEC_ST;
14249 			tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
14250 			tg3_flag_set(tp, NVRAM_BUFFERED);
14251 			break;
14252 		case FLASH_VENDOR_SAIFUN:
14253 			tp->nvram_jedecnum = JEDEC_SAIFUN;
14254 			tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
14255 			break;
14256 		case FLASH_VENDOR_SST_SMALL:
14257 		case FLASH_VENDOR_SST_LARGE:
14258 			tp->nvram_jedecnum = JEDEC_SST;
14259 			tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
14260 			break;
14261 		}
14262 	} else {
14263 		tp->nvram_jedecnum = JEDEC_ATMEL;
14264 		tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
14265 		tg3_flag_set(tp, NVRAM_BUFFERED);
14266 	}
14267 }
14268 
14269 static void tg3_nvram_get_pagesize(struct tg3 *tp, u32 nvmcfg1)
14270 {
14271 	switch (nvmcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
14272 	case FLASH_5752PAGE_SIZE_256:
14273 		tp->nvram_pagesize = 256;
14274 		break;
14275 	case FLASH_5752PAGE_SIZE_512:
14276 		tp->nvram_pagesize = 512;
14277 		break;
14278 	case FLASH_5752PAGE_SIZE_1K:
14279 		tp->nvram_pagesize = 1024;
14280 		break;
14281 	case FLASH_5752PAGE_SIZE_2K:
14282 		tp->nvram_pagesize = 2048;
14283 		break;
14284 	case FLASH_5752PAGE_SIZE_4K:
14285 		tp->nvram_pagesize = 4096;
14286 		break;
14287 	case FLASH_5752PAGE_SIZE_264:
14288 		tp->nvram_pagesize = 264;
14289 		break;
14290 	case FLASH_5752PAGE_SIZE_528:
14291 		tp->nvram_pagesize = 528;
14292 		break;
14293 	}
14294 }
14295 
14296 static void tg3_get_5752_nvram_info(struct tg3 *tp)
14297 {
14298 	u32 nvcfg1;
14299 
14300 	nvcfg1 = tr32(NVRAM_CFG1);
14301 
14302 	/* NVRAM protection for TPM */
14303 	if (nvcfg1 & (1 << 27))
14304 		tg3_flag_set(tp, PROTECTED_NVRAM);
14305 
14306 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14307 	case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
14308 	case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
14309 		tp->nvram_jedecnum = JEDEC_ATMEL;
14310 		tg3_flag_set(tp, NVRAM_BUFFERED);
14311 		break;
14312 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14313 		tp->nvram_jedecnum = JEDEC_ATMEL;
14314 		tg3_flag_set(tp, NVRAM_BUFFERED);
14315 		tg3_flag_set(tp, FLASH);
14316 		break;
14317 	case FLASH_5752VENDOR_ST_M45PE10:
14318 	case FLASH_5752VENDOR_ST_M45PE20:
14319 	case FLASH_5752VENDOR_ST_M45PE40:
14320 		tp->nvram_jedecnum = JEDEC_ST;
14321 		tg3_flag_set(tp, NVRAM_BUFFERED);
14322 		tg3_flag_set(tp, FLASH);
14323 		break;
14324 	}
14325 
14326 	if (tg3_flag(tp, FLASH)) {
14327 		tg3_nvram_get_pagesize(tp, nvcfg1);
14328 	} else {
14329 		/* For eeprom, set pagesize to maximum eeprom size */
14330 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14331 
14332 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14333 		tw32(NVRAM_CFG1, nvcfg1);
14334 	}
14335 }
14336 
14337 static void tg3_get_5755_nvram_info(struct tg3 *tp)
14338 {
14339 	u32 nvcfg1, protect = 0;
14340 
14341 	nvcfg1 = tr32(NVRAM_CFG1);
14342 
14343 	/* NVRAM protection for TPM */
14344 	if (nvcfg1 & (1 << 27)) {
14345 		tg3_flag_set(tp, PROTECTED_NVRAM);
14346 		protect = 1;
14347 	}
14348 
14349 	nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
14350 	switch (nvcfg1) {
14351 	case FLASH_5755VENDOR_ATMEL_FLASH_1:
14352 	case FLASH_5755VENDOR_ATMEL_FLASH_2:
14353 	case FLASH_5755VENDOR_ATMEL_FLASH_3:
14354 	case FLASH_5755VENDOR_ATMEL_FLASH_5:
14355 		tp->nvram_jedecnum = JEDEC_ATMEL;
14356 		tg3_flag_set(tp, NVRAM_BUFFERED);
14357 		tg3_flag_set(tp, FLASH);
14358 		tp->nvram_pagesize = 264;
14359 		if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_1 ||
14360 		    nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_5)
14361 			tp->nvram_size = (protect ? 0x3e200 :
14362 					  TG3_NVRAM_SIZE_512KB);
14363 		else if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_2)
14364 			tp->nvram_size = (protect ? 0x1f200 :
14365 					  TG3_NVRAM_SIZE_256KB);
14366 		else
14367 			tp->nvram_size = (protect ? 0x1f200 :
14368 					  TG3_NVRAM_SIZE_128KB);
14369 		break;
14370 	case FLASH_5752VENDOR_ST_M45PE10:
14371 	case FLASH_5752VENDOR_ST_M45PE20:
14372 	case FLASH_5752VENDOR_ST_M45PE40:
14373 		tp->nvram_jedecnum = JEDEC_ST;
14374 		tg3_flag_set(tp, NVRAM_BUFFERED);
14375 		tg3_flag_set(tp, FLASH);
14376 		tp->nvram_pagesize = 256;
14377 		if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE10)
14378 			tp->nvram_size = (protect ?
14379 					  TG3_NVRAM_SIZE_64KB :
14380 					  TG3_NVRAM_SIZE_128KB);
14381 		else if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE20)
14382 			tp->nvram_size = (protect ?
14383 					  TG3_NVRAM_SIZE_64KB :
14384 					  TG3_NVRAM_SIZE_256KB);
14385 		else
14386 			tp->nvram_size = (protect ?
14387 					  TG3_NVRAM_SIZE_128KB :
14388 					  TG3_NVRAM_SIZE_512KB);
14389 		break;
14390 	}
14391 }
14392 
14393 static void tg3_get_5787_nvram_info(struct tg3 *tp)
14394 {
14395 	u32 nvcfg1;
14396 
14397 	nvcfg1 = tr32(NVRAM_CFG1);
14398 
14399 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14400 	case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
14401 	case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
14402 	case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
14403 	case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
14404 		tp->nvram_jedecnum = JEDEC_ATMEL;
14405 		tg3_flag_set(tp, NVRAM_BUFFERED);
14406 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14407 
14408 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14409 		tw32(NVRAM_CFG1, nvcfg1);
14410 		break;
14411 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14412 	case FLASH_5755VENDOR_ATMEL_FLASH_1:
14413 	case FLASH_5755VENDOR_ATMEL_FLASH_2:
14414 	case FLASH_5755VENDOR_ATMEL_FLASH_3:
14415 		tp->nvram_jedecnum = JEDEC_ATMEL;
14416 		tg3_flag_set(tp, NVRAM_BUFFERED);
14417 		tg3_flag_set(tp, FLASH);
14418 		tp->nvram_pagesize = 264;
14419 		break;
14420 	case FLASH_5752VENDOR_ST_M45PE10:
14421 	case FLASH_5752VENDOR_ST_M45PE20:
14422 	case FLASH_5752VENDOR_ST_M45PE40:
14423 		tp->nvram_jedecnum = JEDEC_ST;
14424 		tg3_flag_set(tp, NVRAM_BUFFERED);
14425 		tg3_flag_set(tp, FLASH);
14426 		tp->nvram_pagesize = 256;
14427 		break;
14428 	}
14429 }
14430 
14431 static void tg3_get_5761_nvram_info(struct tg3 *tp)
14432 {
14433 	u32 nvcfg1, protect = 0;
14434 
14435 	nvcfg1 = tr32(NVRAM_CFG1);
14436 
14437 	/* NVRAM protection for TPM */
14438 	if (nvcfg1 & (1 << 27)) {
14439 		tg3_flag_set(tp, PROTECTED_NVRAM);
14440 		protect = 1;
14441 	}
14442 
14443 	nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
14444 	switch (nvcfg1) {
14445 	case FLASH_5761VENDOR_ATMEL_ADB021D:
14446 	case FLASH_5761VENDOR_ATMEL_ADB041D:
14447 	case FLASH_5761VENDOR_ATMEL_ADB081D:
14448 	case FLASH_5761VENDOR_ATMEL_ADB161D:
14449 	case FLASH_5761VENDOR_ATMEL_MDB021D:
14450 	case FLASH_5761VENDOR_ATMEL_MDB041D:
14451 	case FLASH_5761VENDOR_ATMEL_MDB081D:
14452 	case FLASH_5761VENDOR_ATMEL_MDB161D:
14453 		tp->nvram_jedecnum = JEDEC_ATMEL;
14454 		tg3_flag_set(tp, NVRAM_BUFFERED);
14455 		tg3_flag_set(tp, FLASH);
14456 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14457 		tp->nvram_pagesize = 256;
14458 		break;
14459 	case FLASH_5761VENDOR_ST_A_M45PE20:
14460 	case FLASH_5761VENDOR_ST_A_M45PE40:
14461 	case FLASH_5761VENDOR_ST_A_M45PE80:
14462 	case FLASH_5761VENDOR_ST_A_M45PE16:
14463 	case FLASH_5761VENDOR_ST_M_M45PE20:
14464 	case FLASH_5761VENDOR_ST_M_M45PE40:
14465 	case FLASH_5761VENDOR_ST_M_M45PE80:
14466 	case FLASH_5761VENDOR_ST_M_M45PE16:
14467 		tp->nvram_jedecnum = JEDEC_ST;
14468 		tg3_flag_set(tp, NVRAM_BUFFERED);
14469 		tg3_flag_set(tp, FLASH);
14470 		tp->nvram_pagesize = 256;
14471 		break;
14472 	}
14473 
14474 	if (protect) {
14475 		tp->nvram_size = tr32(NVRAM_ADDR_LOCKOUT);
14476 	} else {
14477 		switch (nvcfg1) {
14478 		case FLASH_5761VENDOR_ATMEL_ADB161D:
14479 		case FLASH_5761VENDOR_ATMEL_MDB161D:
14480 		case FLASH_5761VENDOR_ST_A_M45PE16:
14481 		case FLASH_5761VENDOR_ST_M_M45PE16:
14482 			tp->nvram_size = TG3_NVRAM_SIZE_2MB;
14483 			break;
14484 		case FLASH_5761VENDOR_ATMEL_ADB081D:
14485 		case FLASH_5761VENDOR_ATMEL_MDB081D:
14486 		case FLASH_5761VENDOR_ST_A_M45PE80:
14487 		case FLASH_5761VENDOR_ST_M_M45PE80:
14488 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14489 			break;
14490 		case FLASH_5761VENDOR_ATMEL_ADB041D:
14491 		case FLASH_5761VENDOR_ATMEL_MDB041D:
14492 		case FLASH_5761VENDOR_ST_A_M45PE40:
14493 		case FLASH_5761VENDOR_ST_M_M45PE40:
14494 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14495 			break;
14496 		case FLASH_5761VENDOR_ATMEL_ADB021D:
14497 		case FLASH_5761VENDOR_ATMEL_MDB021D:
14498 		case FLASH_5761VENDOR_ST_A_M45PE20:
14499 		case FLASH_5761VENDOR_ST_M_M45PE20:
14500 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14501 			break;
14502 		}
14503 	}
14504 }
14505 
14506 static void tg3_get_5906_nvram_info(struct tg3 *tp)
14507 {
14508 	tp->nvram_jedecnum = JEDEC_ATMEL;
14509 	tg3_flag_set(tp, NVRAM_BUFFERED);
14510 	tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14511 }
14512 
14513 static void tg3_get_57780_nvram_info(struct tg3 *tp)
14514 {
14515 	u32 nvcfg1;
14516 
14517 	nvcfg1 = tr32(NVRAM_CFG1);
14518 
14519 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14520 	case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
14521 	case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
14522 		tp->nvram_jedecnum = JEDEC_ATMEL;
14523 		tg3_flag_set(tp, NVRAM_BUFFERED);
14524 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14525 
14526 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14527 		tw32(NVRAM_CFG1, nvcfg1);
14528 		return;
14529 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14530 	case FLASH_57780VENDOR_ATMEL_AT45DB011D:
14531 	case FLASH_57780VENDOR_ATMEL_AT45DB011B:
14532 	case FLASH_57780VENDOR_ATMEL_AT45DB021D:
14533 	case FLASH_57780VENDOR_ATMEL_AT45DB021B:
14534 	case FLASH_57780VENDOR_ATMEL_AT45DB041D:
14535 	case FLASH_57780VENDOR_ATMEL_AT45DB041B:
14536 		tp->nvram_jedecnum = JEDEC_ATMEL;
14537 		tg3_flag_set(tp, NVRAM_BUFFERED);
14538 		tg3_flag_set(tp, FLASH);
14539 
14540 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14541 		case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14542 		case FLASH_57780VENDOR_ATMEL_AT45DB011D:
14543 		case FLASH_57780VENDOR_ATMEL_AT45DB011B:
14544 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14545 			break;
14546 		case FLASH_57780VENDOR_ATMEL_AT45DB021D:
14547 		case FLASH_57780VENDOR_ATMEL_AT45DB021B:
14548 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14549 			break;
14550 		case FLASH_57780VENDOR_ATMEL_AT45DB041D:
14551 		case FLASH_57780VENDOR_ATMEL_AT45DB041B:
14552 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14553 			break;
14554 		}
14555 		break;
14556 	case FLASH_5752VENDOR_ST_M45PE10:
14557 	case FLASH_5752VENDOR_ST_M45PE20:
14558 	case FLASH_5752VENDOR_ST_M45PE40:
14559 		tp->nvram_jedecnum = JEDEC_ST;
14560 		tg3_flag_set(tp, NVRAM_BUFFERED);
14561 		tg3_flag_set(tp, FLASH);
14562 
14563 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14564 		case FLASH_5752VENDOR_ST_M45PE10:
14565 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14566 			break;
14567 		case FLASH_5752VENDOR_ST_M45PE20:
14568 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14569 			break;
14570 		case FLASH_5752VENDOR_ST_M45PE40:
14571 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14572 			break;
14573 		}
14574 		break;
14575 	default:
14576 		tg3_flag_set(tp, NO_NVRAM);
14577 		return;
14578 	}
14579 
14580 	tg3_nvram_get_pagesize(tp, nvcfg1);
14581 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14582 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14583 }
14584 
14585 
14586 static void tg3_get_5717_nvram_info(struct tg3 *tp)
14587 {
14588 	u32 nvcfg1;
14589 
14590 	nvcfg1 = tr32(NVRAM_CFG1);
14591 
14592 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14593 	case FLASH_5717VENDOR_ATMEL_EEPROM:
14594 	case FLASH_5717VENDOR_MICRO_EEPROM:
14595 		tp->nvram_jedecnum = JEDEC_ATMEL;
14596 		tg3_flag_set(tp, NVRAM_BUFFERED);
14597 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14598 
14599 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14600 		tw32(NVRAM_CFG1, nvcfg1);
14601 		return;
14602 	case FLASH_5717VENDOR_ATMEL_MDB011D:
14603 	case FLASH_5717VENDOR_ATMEL_ADB011B:
14604 	case FLASH_5717VENDOR_ATMEL_ADB011D:
14605 	case FLASH_5717VENDOR_ATMEL_MDB021D:
14606 	case FLASH_5717VENDOR_ATMEL_ADB021B:
14607 	case FLASH_5717VENDOR_ATMEL_ADB021D:
14608 	case FLASH_5717VENDOR_ATMEL_45USPT:
14609 		tp->nvram_jedecnum = JEDEC_ATMEL;
14610 		tg3_flag_set(tp, NVRAM_BUFFERED);
14611 		tg3_flag_set(tp, FLASH);
14612 
14613 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14614 		case FLASH_5717VENDOR_ATMEL_MDB021D:
14615 			/* Detect size with tg3_nvram_get_size() */
14616 			break;
14617 		case FLASH_5717VENDOR_ATMEL_ADB021B:
14618 		case FLASH_5717VENDOR_ATMEL_ADB021D:
14619 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14620 			break;
14621 		default:
14622 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14623 			break;
14624 		}
14625 		break;
14626 	case FLASH_5717VENDOR_ST_M_M25PE10:
14627 	case FLASH_5717VENDOR_ST_A_M25PE10:
14628 	case FLASH_5717VENDOR_ST_M_M45PE10:
14629 	case FLASH_5717VENDOR_ST_A_M45PE10:
14630 	case FLASH_5717VENDOR_ST_M_M25PE20:
14631 	case FLASH_5717VENDOR_ST_A_M25PE20:
14632 	case FLASH_5717VENDOR_ST_M_M45PE20:
14633 	case FLASH_5717VENDOR_ST_A_M45PE20:
14634 	case FLASH_5717VENDOR_ST_25USPT:
14635 	case FLASH_5717VENDOR_ST_45USPT:
14636 		tp->nvram_jedecnum = JEDEC_ST;
14637 		tg3_flag_set(tp, NVRAM_BUFFERED);
14638 		tg3_flag_set(tp, FLASH);
14639 
14640 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14641 		case FLASH_5717VENDOR_ST_M_M25PE20:
14642 		case FLASH_5717VENDOR_ST_M_M45PE20:
14643 			/* Detect size with tg3_nvram_get_size() */
14644 			break;
14645 		case FLASH_5717VENDOR_ST_A_M25PE20:
14646 		case FLASH_5717VENDOR_ST_A_M45PE20:
14647 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14648 			break;
14649 		default:
14650 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14651 			break;
14652 		}
14653 		break;
14654 	default:
14655 		tg3_flag_set(tp, NO_NVRAM);
14656 		return;
14657 	}
14658 
14659 	tg3_nvram_get_pagesize(tp, nvcfg1);
14660 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14661 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14662 }
14663 
14664 static void tg3_get_5720_nvram_info(struct tg3 *tp)
14665 {
14666 	u32 nvcfg1, nvmpinstrp;
14667 
14668 	nvcfg1 = tr32(NVRAM_CFG1);
14669 	nvmpinstrp = nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK;
14670 
14671 	if (tg3_asic_rev(tp) == ASIC_REV_5762) {
14672 		if (!(nvcfg1 & NVRAM_CFG1_5762VENDOR_MASK)) {
14673 			tg3_flag_set(tp, NO_NVRAM);
14674 			return;
14675 		}
14676 
14677 		switch (nvmpinstrp) {
14678 		case FLASH_5762_EEPROM_HD:
14679 			nvmpinstrp = FLASH_5720_EEPROM_HD;
14680 			break;
14681 		case FLASH_5762_EEPROM_LD:
14682 			nvmpinstrp = FLASH_5720_EEPROM_LD;
14683 			break;
14684 		case FLASH_5720VENDOR_M_ST_M45PE20:
14685 			/* This pinstrap supports multiple sizes, so force it
14686 			 * to read the actual size from location 0xf0.
14687 			 */
14688 			nvmpinstrp = FLASH_5720VENDOR_ST_45USPT;
14689 			break;
14690 		}
14691 	}
14692 
14693 	switch (nvmpinstrp) {
14694 	case FLASH_5720_EEPROM_HD:
14695 	case FLASH_5720_EEPROM_LD:
14696 		tp->nvram_jedecnum = JEDEC_ATMEL;
14697 		tg3_flag_set(tp, NVRAM_BUFFERED);
14698 
14699 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14700 		tw32(NVRAM_CFG1, nvcfg1);
14701 		if (nvmpinstrp == FLASH_5720_EEPROM_HD)
14702 			tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14703 		else
14704 			tp->nvram_pagesize = ATMEL_AT24C02_CHIP_SIZE;
14705 		return;
14706 	case FLASH_5720VENDOR_M_ATMEL_DB011D:
14707 	case FLASH_5720VENDOR_A_ATMEL_DB011B:
14708 	case FLASH_5720VENDOR_A_ATMEL_DB011D:
14709 	case FLASH_5720VENDOR_M_ATMEL_DB021D:
14710 	case FLASH_5720VENDOR_A_ATMEL_DB021B:
14711 	case FLASH_5720VENDOR_A_ATMEL_DB021D:
14712 	case FLASH_5720VENDOR_M_ATMEL_DB041D:
14713 	case FLASH_5720VENDOR_A_ATMEL_DB041B:
14714 	case FLASH_5720VENDOR_A_ATMEL_DB041D:
14715 	case FLASH_5720VENDOR_M_ATMEL_DB081D:
14716 	case FLASH_5720VENDOR_A_ATMEL_DB081D:
14717 	case FLASH_5720VENDOR_ATMEL_45USPT:
14718 		tp->nvram_jedecnum = JEDEC_ATMEL;
14719 		tg3_flag_set(tp, NVRAM_BUFFERED);
14720 		tg3_flag_set(tp, FLASH);
14721 
14722 		switch (nvmpinstrp) {
14723 		case FLASH_5720VENDOR_M_ATMEL_DB021D:
14724 		case FLASH_5720VENDOR_A_ATMEL_DB021B:
14725 		case FLASH_5720VENDOR_A_ATMEL_DB021D:
14726 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14727 			break;
14728 		case FLASH_5720VENDOR_M_ATMEL_DB041D:
14729 		case FLASH_5720VENDOR_A_ATMEL_DB041B:
14730 		case FLASH_5720VENDOR_A_ATMEL_DB041D:
14731 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14732 			break;
14733 		case FLASH_5720VENDOR_M_ATMEL_DB081D:
14734 		case FLASH_5720VENDOR_A_ATMEL_DB081D:
14735 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14736 			break;
14737 		default:
14738 			if (tg3_asic_rev(tp) != ASIC_REV_5762)
14739 				tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14740 			break;
14741 		}
14742 		break;
14743 	case FLASH_5720VENDOR_M_ST_M25PE10:
14744 	case FLASH_5720VENDOR_M_ST_M45PE10:
14745 	case FLASH_5720VENDOR_A_ST_M25PE10:
14746 	case FLASH_5720VENDOR_A_ST_M45PE10:
14747 	case FLASH_5720VENDOR_M_ST_M25PE20:
14748 	case FLASH_5720VENDOR_M_ST_M45PE20:
14749 	case FLASH_5720VENDOR_A_ST_M25PE20:
14750 	case FLASH_5720VENDOR_A_ST_M45PE20:
14751 	case FLASH_5720VENDOR_M_ST_M25PE40:
14752 	case FLASH_5720VENDOR_M_ST_M45PE40:
14753 	case FLASH_5720VENDOR_A_ST_M25PE40:
14754 	case FLASH_5720VENDOR_A_ST_M45PE40:
14755 	case FLASH_5720VENDOR_M_ST_M25PE80:
14756 	case FLASH_5720VENDOR_M_ST_M45PE80:
14757 	case FLASH_5720VENDOR_A_ST_M25PE80:
14758 	case FLASH_5720VENDOR_A_ST_M45PE80:
14759 	case FLASH_5720VENDOR_ST_25USPT:
14760 	case FLASH_5720VENDOR_ST_45USPT:
14761 		tp->nvram_jedecnum = JEDEC_ST;
14762 		tg3_flag_set(tp, NVRAM_BUFFERED);
14763 		tg3_flag_set(tp, FLASH);
14764 
14765 		switch (nvmpinstrp) {
14766 		case FLASH_5720VENDOR_M_ST_M25PE20:
14767 		case FLASH_5720VENDOR_M_ST_M45PE20:
14768 		case FLASH_5720VENDOR_A_ST_M25PE20:
14769 		case FLASH_5720VENDOR_A_ST_M45PE20:
14770 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14771 			break;
14772 		case FLASH_5720VENDOR_M_ST_M25PE40:
14773 		case FLASH_5720VENDOR_M_ST_M45PE40:
14774 		case FLASH_5720VENDOR_A_ST_M25PE40:
14775 		case FLASH_5720VENDOR_A_ST_M45PE40:
14776 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14777 			break;
14778 		case FLASH_5720VENDOR_M_ST_M25PE80:
14779 		case FLASH_5720VENDOR_M_ST_M45PE80:
14780 		case FLASH_5720VENDOR_A_ST_M25PE80:
14781 		case FLASH_5720VENDOR_A_ST_M45PE80:
14782 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14783 			break;
14784 		default:
14785 			if (tg3_asic_rev(tp) != ASIC_REV_5762)
14786 				tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14787 			break;
14788 		}
14789 		break;
14790 	default:
14791 		tg3_flag_set(tp, NO_NVRAM);
14792 		return;
14793 	}
14794 
14795 	tg3_nvram_get_pagesize(tp, nvcfg1);
14796 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14797 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14798 
14799 	if (tg3_asic_rev(tp) == ASIC_REV_5762) {
14800 		u32 val;
14801 
14802 		if (tg3_nvram_read(tp, 0, &val))
14803 			return;
14804 
14805 		if (val != TG3_EEPROM_MAGIC &&
14806 		    (val & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW)
14807 			tg3_flag_set(tp, NO_NVRAM);
14808 	}
14809 }
14810 
14811 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
14812 static void tg3_nvram_init(struct tg3 *tp)
14813 {
14814 	if (tg3_flag(tp, IS_SSB_CORE)) {
14815 		/* No NVRAM and EEPROM on the SSB Broadcom GigE core. */
14816 		tg3_flag_clear(tp, NVRAM);
14817 		tg3_flag_clear(tp, NVRAM_BUFFERED);
14818 		tg3_flag_set(tp, NO_NVRAM);
14819 		return;
14820 	}
14821 
14822 	tw32_f(GRC_EEPROM_ADDR,
14823 	     (EEPROM_ADDR_FSM_RESET |
14824 	      (EEPROM_DEFAULT_CLOCK_PERIOD <<
14825 	       EEPROM_ADDR_CLKPERD_SHIFT)));
14826 
14827 	msleep(1);
14828 
14829 	/* Enable seeprom accesses. */
14830 	tw32_f(GRC_LOCAL_CTRL,
14831 	     tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
14832 	udelay(100);
14833 
14834 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
14835 	    tg3_asic_rev(tp) != ASIC_REV_5701) {
14836 		tg3_flag_set(tp, NVRAM);
14837 
14838 		if (tg3_nvram_lock(tp)) {
14839 			netdev_warn(tp->dev,
14840 				    "Cannot get nvram lock, %s failed\n",
14841 				    __func__);
14842 			return;
14843 		}
14844 		tg3_enable_nvram_access(tp);
14845 
14846 		tp->nvram_size = 0;
14847 
14848 		if (tg3_asic_rev(tp) == ASIC_REV_5752)
14849 			tg3_get_5752_nvram_info(tp);
14850 		else if (tg3_asic_rev(tp) == ASIC_REV_5755)
14851 			tg3_get_5755_nvram_info(tp);
14852 		else if (tg3_asic_rev(tp) == ASIC_REV_5787 ||
14853 			 tg3_asic_rev(tp) == ASIC_REV_5784 ||
14854 			 tg3_asic_rev(tp) == ASIC_REV_5785)
14855 			tg3_get_5787_nvram_info(tp);
14856 		else if (tg3_asic_rev(tp) == ASIC_REV_5761)
14857 			tg3_get_5761_nvram_info(tp);
14858 		else if (tg3_asic_rev(tp) == ASIC_REV_5906)
14859 			tg3_get_5906_nvram_info(tp);
14860 		else if (tg3_asic_rev(tp) == ASIC_REV_57780 ||
14861 			 tg3_flag(tp, 57765_CLASS))
14862 			tg3_get_57780_nvram_info(tp);
14863 		else if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
14864 			 tg3_asic_rev(tp) == ASIC_REV_5719)
14865 			tg3_get_5717_nvram_info(tp);
14866 		else if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
14867 			 tg3_asic_rev(tp) == ASIC_REV_5762)
14868 			tg3_get_5720_nvram_info(tp);
14869 		else
14870 			tg3_get_nvram_info(tp);
14871 
14872 		if (tp->nvram_size == 0)
14873 			tg3_get_nvram_size(tp);
14874 
14875 		tg3_disable_nvram_access(tp);
14876 		tg3_nvram_unlock(tp);
14877 
14878 	} else {
14879 		tg3_flag_clear(tp, NVRAM);
14880 		tg3_flag_clear(tp, NVRAM_BUFFERED);
14881 
14882 		tg3_get_eeprom_size(tp);
14883 	}
14884 }
14885 
14886 struct subsys_tbl_ent {
14887 	u16 subsys_vendor, subsys_devid;
14888 	u32 phy_id;
14889 };
14890 
14891 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
14892 	/* Broadcom boards. */
14893 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14894 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700A6, TG3_PHY_ID_BCM5401 },
14895 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14896 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A5, TG3_PHY_ID_BCM5701 },
14897 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14898 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700T6, TG3_PHY_ID_BCM8002 },
14899 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14900 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700A9, 0 },
14901 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14902 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701T1, TG3_PHY_ID_BCM5701 },
14903 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14904 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701T8, TG3_PHY_ID_BCM5701 },
14905 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14906 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A7, 0 },
14907 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14908 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A10, TG3_PHY_ID_BCM5701 },
14909 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14910 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A12, TG3_PHY_ID_BCM5701 },
14911 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14912 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95703AX1, TG3_PHY_ID_BCM5703 },
14913 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
14914 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95703AX2, TG3_PHY_ID_BCM5703 },
14915 
14916 	/* 3com boards. */
14917 	{ TG3PCI_SUBVENDOR_ID_3COM,
14918 	  TG3PCI_SUBDEVICE_ID_3COM_3C996T, TG3_PHY_ID_BCM5401 },
14919 	{ TG3PCI_SUBVENDOR_ID_3COM,
14920 	  TG3PCI_SUBDEVICE_ID_3COM_3C996BT, TG3_PHY_ID_BCM5701 },
14921 	{ TG3PCI_SUBVENDOR_ID_3COM,
14922 	  TG3PCI_SUBDEVICE_ID_3COM_3C996SX, 0 },
14923 	{ TG3PCI_SUBVENDOR_ID_3COM,
14924 	  TG3PCI_SUBDEVICE_ID_3COM_3C1000T, TG3_PHY_ID_BCM5701 },
14925 	{ TG3PCI_SUBVENDOR_ID_3COM,
14926 	  TG3PCI_SUBDEVICE_ID_3COM_3C940BR01, TG3_PHY_ID_BCM5701 },
14927 
14928 	/* DELL boards. */
14929 	{ TG3PCI_SUBVENDOR_ID_DELL,
14930 	  TG3PCI_SUBDEVICE_ID_DELL_VIPER, TG3_PHY_ID_BCM5401 },
14931 	{ TG3PCI_SUBVENDOR_ID_DELL,
14932 	  TG3PCI_SUBDEVICE_ID_DELL_JAGUAR, TG3_PHY_ID_BCM5401 },
14933 	{ TG3PCI_SUBVENDOR_ID_DELL,
14934 	  TG3PCI_SUBDEVICE_ID_DELL_MERLOT, TG3_PHY_ID_BCM5411 },
14935 	{ TG3PCI_SUBVENDOR_ID_DELL,
14936 	  TG3PCI_SUBDEVICE_ID_DELL_SLIM_MERLOT, TG3_PHY_ID_BCM5411 },
14937 
14938 	/* Compaq boards. */
14939 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
14940 	  TG3PCI_SUBDEVICE_ID_COMPAQ_BANSHEE, TG3_PHY_ID_BCM5701 },
14941 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
14942 	  TG3PCI_SUBDEVICE_ID_COMPAQ_BANSHEE_2, TG3_PHY_ID_BCM5701 },
14943 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
14944 	  TG3PCI_SUBDEVICE_ID_COMPAQ_CHANGELING, 0 },
14945 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
14946 	  TG3PCI_SUBDEVICE_ID_COMPAQ_NC7780, TG3_PHY_ID_BCM5701 },
14947 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
14948 	  TG3PCI_SUBDEVICE_ID_COMPAQ_NC7780_2, TG3_PHY_ID_BCM5701 },
14949 
14950 	/* IBM boards. */
14951 	{ TG3PCI_SUBVENDOR_ID_IBM,
14952 	  TG3PCI_SUBDEVICE_ID_IBM_5703SAX2, 0 }
14953 };
14954 
14955 static struct subsys_tbl_ent *tg3_lookup_by_subsys(struct tg3 *tp)
14956 {
14957 	int i;
14958 
14959 	for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
14960 		if ((subsys_id_to_phy_id[i].subsys_vendor ==
14961 		     tp->pdev->subsystem_vendor) &&
14962 		    (subsys_id_to_phy_id[i].subsys_devid ==
14963 		     tp->pdev->subsystem_device))
14964 			return &subsys_id_to_phy_id[i];
14965 	}
14966 	return NULL;
14967 }
14968 
14969 static void tg3_get_eeprom_hw_cfg(struct tg3 *tp)
14970 {
14971 	u32 val;
14972 
14973 	tp->phy_id = TG3_PHY_ID_INVALID;
14974 	tp->led_ctrl = LED_CTRL_MODE_PHY_1;
14975 
14976 	/* Assume an onboard device and WOL capable by default.  */
14977 	tg3_flag_set(tp, EEPROM_WRITE_PROT);
14978 	tg3_flag_set(tp, WOL_CAP);
14979 
14980 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
14981 		if (!(tr32(PCIE_TRANSACTION_CFG) & PCIE_TRANS_CFG_LOM)) {
14982 			tg3_flag_clear(tp, EEPROM_WRITE_PROT);
14983 			tg3_flag_set(tp, IS_NIC);
14984 		}
14985 		val = tr32(VCPU_CFGSHDW);
14986 		if (val & VCPU_CFGSHDW_ASPM_DBNC)
14987 			tg3_flag_set(tp, ASPM_WORKAROUND);
14988 		if ((val & VCPU_CFGSHDW_WOL_ENABLE) &&
14989 		    (val & VCPU_CFGSHDW_WOL_MAGPKT)) {
14990 			tg3_flag_set(tp, WOL_ENABLE);
14991 			device_set_wakeup_enable(&tp->pdev->dev, true);
14992 		}
14993 		goto done;
14994 	}
14995 
14996 	tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
14997 	if (val == NIC_SRAM_DATA_SIG_MAGIC) {
14998 		u32 nic_cfg, led_cfg;
14999 		u32 cfg2 = 0, cfg4 = 0, cfg5 = 0;
15000 		u32 nic_phy_id, ver, eeprom_phy_id;
15001 		int eeprom_phy_serdes = 0;
15002 
15003 		tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
15004 		tp->nic_sram_data_cfg = nic_cfg;
15005 
15006 		tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
15007 		ver >>= NIC_SRAM_DATA_VER_SHIFT;
15008 		if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
15009 		    tg3_asic_rev(tp) != ASIC_REV_5701 &&
15010 		    tg3_asic_rev(tp) != ASIC_REV_5703 &&
15011 		    (ver > 0) && (ver < 0x100))
15012 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
15013 
15014 		if (tg3_asic_rev(tp) == ASIC_REV_5785)
15015 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_4, &cfg4);
15016 
15017 		if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
15018 		    tg3_asic_rev(tp) == ASIC_REV_5719 ||
15019 		    tg3_asic_rev(tp) == ASIC_REV_5720)
15020 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_5, &cfg5);
15021 
15022 		if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
15023 		    NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
15024 			eeprom_phy_serdes = 1;
15025 
15026 		tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
15027 		if (nic_phy_id != 0) {
15028 			u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
15029 			u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
15030 
15031 			eeprom_phy_id  = (id1 >> 16) << 10;
15032 			eeprom_phy_id |= (id2 & 0xfc00) << 16;
15033 			eeprom_phy_id |= (id2 & 0x03ff) <<  0;
15034 		} else
15035 			eeprom_phy_id = 0;
15036 
15037 		tp->phy_id = eeprom_phy_id;
15038 		if (eeprom_phy_serdes) {
15039 			if (!tg3_flag(tp, 5705_PLUS))
15040 				tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15041 			else
15042 				tp->phy_flags |= TG3_PHYFLG_MII_SERDES;
15043 		}
15044 
15045 		if (tg3_flag(tp, 5750_PLUS))
15046 			led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
15047 				    SHASTA_EXT_LED_MODE_MASK);
15048 		else
15049 			led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
15050 
15051 		switch (led_cfg) {
15052 		default:
15053 		case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
15054 			tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15055 			break;
15056 
15057 		case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
15058 			tp->led_ctrl = LED_CTRL_MODE_PHY_2;
15059 			break;
15060 
15061 		case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
15062 			tp->led_ctrl = LED_CTRL_MODE_MAC;
15063 
15064 			/* Default to PHY_1_MODE if 0 (MAC_MODE) is
15065 			 * read on some older 5700/5701 bootcode.
15066 			 */
15067 			if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
15068 			    tg3_asic_rev(tp) == ASIC_REV_5701)
15069 				tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15070 
15071 			break;
15072 
15073 		case SHASTA_EXT_LED_SHARED:
15074 			tp->led_ctrl = LED_CTRL_MODE_SHARED;
15075 			if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0 &&
15076 			    tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A1)
15077 				tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
15078 						 LED_CTRL_MODE_PHY_2);
15079 
15080 			if (tg3_flag(tp, 5717_PLUS) ||
15081 			    tg3_asic_rev(tp) == ASIC_REV_5762)
15082 				tp->led_ctrl |= LED_CTRL_BLINK_RATE_OVERRIDE |
15083 						LED_CTRL_BLINK_RATE_MASK;
15084 
15085 			break;
15086 
15087 		case SHASTA_EXT_LED_MAC:
15088 			tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
15089 			break;
15090 
15091 		case SHASTA_EXT_LED_COMBO:
15092 			tp->led_ctrl = LED_CTRL_MODE_COMBO;
15093 			if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0)
15094 				tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
15095 						 LED_CTRL_MODE_PHY_2);
15096 			break;
15097 
15098 		}
15099 
15100 		if ((tg3_asic_rev(tp) == ASIC_REV_5700 ||
15101 		     tg3_asic_rev(tp) == ASIC_REV_5701) &&
15102 		    tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
15103 			tp->led_ctrl = LED_CTRL_MODE_PHY_2;
15104 
15105 		if (tg3_chip_rev(tp) == CHIPREV_5784_AX)
15106 			tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15107 
15108 		if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP) {
15109 			tg3_flag_set(tp, EEPROM_WRITE_PROT);
15110 			if ((tp->pdev->subsystem_vendor ==
15111 			     PCI_VENDOR_ID_ARIMA) &&
15112 			    (tp->pdev->subsystem_device == 0x205a ||
15113 			     tp->pdev->subsystem_device == 0x2063))
15114 				tg3_flag_clear(tp, EEPROM_WRITE_PROT);
15115 		} else {
15116 			tg3_flag_clear(tp, EEPROM_WRITE_PROT);
15117 			tg3_flag_set(tp, IS_NIC);
15118 		}
15119 
15120 		if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
15121 			tg3_flag_set(tp, ENABLE_ASF);
15122 			if (tg3_flag(tp, 5750_PLUS))
15123 				tg3_flag_set(tp, ASF_NEW_HANDSHAKE);
15124 		}
15125 
15126 		if ((nic_cfg & NIC_SRAM_DATA_CFG_APE_ENABLE) &&
15127 		    tg3_flag(tp, 5750_PLUS))
15128 			tg3_flag_set(tp, ENABLE_APE);
15129 
15130 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES &&
15131 		    !(nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL))
15132 			tg3_flag_clear(tp, WOL_CAP);
15133 
15134 		if (tg3_flag(tp, WOL_CAP) &&
15135 		    (nic_cfg & NIC_SRAM_DATA_CFG_WOL_ENABLE)) {
15136 			tg3_flag_set(tp, WOL_ENABLE);
15137 			device_set_wakeup_enable(&tp->pdev->dev, true);
15138 		}
15139 
15140 		if (cfg2 & (1 << 17))
15141 			tp->phy_flags |= TG3_PHYFLG_CAPACITIVE_COUPLING;
15142 
15143 		/* serdes signal pre-emphasis in register 0x590 set by */
15144 		/* bootcode if bit 18 is set */
15145 		if (cfg2 & (1 << 18))
15146 			tp->phy_flags |= TG3_PHYFLG_SERDES_PREEMPHASIS;
15147 
15148 		if ((tg3_flag(tp, 57765_PLUS) ||
15149 		     (tg3_asic_rev(tp) == ASIC_REV_5784 &&
15150 		      tg3_chip_rev(tp) != CHIPREV_5784_AX)) &&
15151 		    (cfg2 & NIC_SRAM_DATA_CFG_2_APD_EN))
15152 			tp->phy_flags |= TG3_PHYFLG_ENABLE_APD;
15153 
15154 		if (tg3_flag(tp, PCI_EXPRESS)) {
15155 			u32 cfg3;
15156 
15157 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &cfg3);
15158 			if (tg3_asic_rev(tp) != ASIC_REV_5785 &&
15159 			    !tg3_flag(tp, 57765_PLUS) &&
15160 			    (cfg3 & NIC_SRAM_ASPM_DEBOUNCE))
15161 				tg3_flag_set(tp, ASPM_WORKAROUND);
15162 			if (cfg3 & NIC_SRAM_LNK_FLAP_AVOID)
15163 				tp->phy_flags |= TG3_PHYFLG_KEEP_LINK_ON_PWRDN;
15164 			if (cfg3 & NIC_SRAM_1G_ON_VAUX_OK)
15165 				tp->phy_flags |= TG3_PHYFLG_1G_ON_VAUX_OK;
15166 		}
15167 
15168 		if (cfg4 & NIC_SRAM_RGMII_INBAND_DISABLE)
15169 			tg3_flag_set(tp, RGMII_INBAND_DISABLE);
15170 		if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_RX_EN)
15171 			tg3_flag_set(tp, RGMII_EXT_IBND_RX_EN);
15172 		if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_TX_EN)
15173 			tg3_flag_set(tp, RGMII_EXT_IBND_TX_EN);
15174 
15175 		if (cfg5 & NIC_SRAM_DISABLE_1G_HALF_ADV)
15176 			tp->phy_flags |= TG3_PHYFLG_DISABLE_1G_HD_ADV;
15177 	}
15178 done:
15179 	if (tg3_flag(tp, WOL_CAP))
15180 		device_set_wakeup_enable(&tp->pdev->dev,
15181 					 tg3_flag(tp, WOL_ENABLE));
15182 	else
15183 		device_set_wakeup_capable(&tp->pdev->dev, false);
15184 }
15185 
15186 static int tg3_ape_otp_read(struct tg3 *tp, u32 offset, u32 *val)
15187 {
15188 	int i, err;
15189 	u32 val2, off = offset * 8;
15190 
15191 	err = tg3_nvram_lock(tp);
15192 	if (err)
15193 		return err;
15194 
15195 	tg3_ape_write32(tp, TG3_APE_OTP_ADDR, off | APE_OTP_ADDR_CPU_ENABLE);
15196 	tg3_ape_write32(tp, TG3_APE_OTP_CTRL, APE_OTP_CTRL_PROG_EN |
15197 			APE_OTP_CTRL_CMD_RD | APE_OTP_CTRL_START);
15198 	tg3_ape_read32(tp, TG3_APE_OTP_CTRL);
15199 	udelay(10);
15200 
15201 	for (i = 0; i < 100; i++) {
15202 		val2 = tg3_ape_read32(tp, TG3_APE_OTP_STATUS);
15203 		if (val2 & APE_OTP_STATUS_CMD_DONE) {
15204 			*val = tg3_ape_read32(tp, TG3_APE_OTP_RD_DATA);
15205 			break;
15206 		}
15207 		udelay(10);
15208 	}
15209 
15210 	tg3_ape_write32(tp, TG3_APE_OTP_CTRL, 0);
15211 
15212 	tg3_nvram_unlock(tp);
15213 	if (val2 & APE_OTP_STATUS_CMD_DONE)
15214 		return 0;
15215 
15216 	return -EBUSY;
15217 }
15218 
15219 static int tg3_issue_otp_command(struct tg3 *tp, u32 cmd)
15220 {
15221 	int i;
15222 	u32 val;
15223 
15224 	tw32(OTP_CTRL, cmd | OTP_CTRL_OTP_CMD_START);
15225 	tw32(OTP_CTRL, cmd);
15226 
15227 	/* Wait for up to 1 ms for command to execute. */
15228 	for (i = 0; i < 100; i++) {
15229 		val = tr32(OTP_STATUS);
15230 		if (val & OTP_STATUS_CMD_DONE)
15231 			break;
15232 		udelay(10);
15233 	}
15234 
15235 	return (val & OTP_STATUS_CMD_DONE) ? 0 : -EBUSY;
15236 }
15237 
15238 /* Read the gphy configuration from the OTP region of the chip.  The gphy
15239  * configuration is a 32-bit value that straddles the alignment boundary.
15240  * We do two 32-bit reads and then shift and merge the results.
15241  */
15242 static u32 tg3_read_otp_phycfg(struct tg3 *tp)
15243 {
15244 	u32 bhalf_otp, thalf_otp;
15245 
15246 	tw32(OTP_MODE, OTP_MODE_OTP_THRU_GRC);
15247 
15248 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_INIT))
15249 		return 0;
15250 
15251 	tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC1);
15252 
15253 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
15254 		return 0;
15255 
15256 	thalf_otp = tr32(OTP_READ_DATA);
15257 
15258 	tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC2);
15259 
15260 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
15261 		return 0;
15262 
15263 	bhalf_otp = tr32(OTP_READ_DATA);
15264 
15265 	return ((thalf_otp & 0x0000ffff) << 16) | (bhalf_otp >> 16);
15266 }
15267 
15268 static void tg3_phy_init_link_config(struct tg3 *tp)
15269 {
15270 	u32 adv = ADVERTISED_Autoneg;
15271 
15272 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
15273 		if (!(tp->phy_flags & TG3_PHYFLG_DISABLE_1G_HD_ADV))
15274 			adv |= ADVERTISED_1000baseT_Half;
15275 		adv |= ADVERTISED_1000baseT_Full;
15276 	}
15277 
15278 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
15279 		adv |= ADVERTISED_100baseT_Half |
15280 		       ADVERTISED_100baseT_Full |
15281 		       ADVERTISED_10baseT_Half |
15282 		       ADVERTISED_10baseT_Full |
15283 		       ADVERTISED_TP;
15284 	else
15285 		adv |= ADVERTISED_FIBRE;
15286 
15287 	tp->link_config.advertising = adv;
15288 	tp->link_config.speed = SPEED_UNKNOWN;
15289 	tp->link_config.duplex = DUPLEX_UNKNOWN;
15290 	tp->link_config.autoneg = AUTONEG_ENABLE;
15291 	tp->link_config.active_speed = SPEED_UNKNOWN;
15292 	tp->link_config.active_duplex = DUPLEX_UNKNOWN;
15293 
15294 	tp->old_link = -1;
15295 }
15296 
15297 static int tg3_phy_probe(struct tg3 *tp)
15298 {
15299 	u32 hw_phy_id_1, hw_phy_id_2;
15300 	u32 hw_phy_id, hw_phy_id_masked;
15301 	int err;
15302 
15303 	/* flow control autonegotiation is default behavior */
15304 	tg3_flag_set(tp, PAUSE_AUTONEG);
15305 	tp->link_config.flowctrl = FLOW_CTRL_TX | FLOW_CTRL_RX;
15306 
15307 	if (tg3_flag(tp, ENABLE_APE)) {
15308 		switch (tp->pci_fn) {
15309 		case 0:
15310 			tp->phy_ape_lock = TG3_APE_LOCK_PHY0;
15311 			break;
15312 		case 1:
15313 			tp->phy_ape_lock = TG3_APE_LOCK_PHY1;
15314 			break;
15315 		case 2:
15316 			tp->phy_ape_lock = TG3_APE_LOCK_PHY2;
15317 			break;
15318 		case 3:
15319 			tp->phy_ape_lock = TG3_APE_LOCK_PHY3;
15320 			break;
15321 		}
15322 	}
15323 
15324 	if (!tg3_flag(tp, ENABLE_ASF) &&
15325 	    !(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15326 	    !(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
15327 		tp->phy_flags &= ~(TG3_PHYFLG_1G_ON_VAUX_OK |
15328 				   TG3_PHYFLG_KEEP_LINK_ON_PWRDN);
15329 
15330 	if (tg3_flag(tp, USE_PHYLIB))
15331 		return tg3_phy_init(tp);
15332 
15333 	/* Reading the PHY ID register can conflict with ASF
15334 	 * firmware access to the PHY hardware.
15335 	 */
15336 	err = 0;
15337 	if (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)) {
15338 		hw_phy_id = hw_phy_id_masked = TG3_PHY_ID_INVALID;
15339 	} else {
15340 		/* Now read the physical PHY_ID from the chip and verify
15341 		 * that it is sane.  If it doesn't look good, we fall back
15342 		 * to either the hard-coded table based PHY_ID and failing
15343 		 * that the value found in the eeprom area.
15344 		 */
15345 		err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
15346 		err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
15347 
15348 		hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
15349 		hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
15350 		hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
15351 
15352 		hw_phy_id_masked = hw_phy_id & TG3_PHY_ID_MASK;
15353 	}
15354 
15355 	if (!err && TG3_KNOWN_PHY_ID(hw_phy_id_masked)) {
15356 		tp->phy_id = hw_phy_id;
15357 		if (hw_phy_id_masked == TG3_PHY_ID_BCM8002)
15358 			tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15359 		else
15360 			tp->phy_flags &= ~TG3_PHYFLG_PHY_SERDES;
15361 	} else {
15362 		if (tp->phy_id != TG3_PHY_ID_INVALID) {
15363 			/* Do nothing, phy ID already set up in
15364 			 * tg3_get_eeprom_hw_cfg().
15365 			 */
15366 		} else {
15367 			struct subsys_tbl_ent *p;
15368 
15369 			/* No eeprom signature?  Try the hardcoded
15370 			 * subsys device table.
15371 			 */
15372 			p = tg3_lookup_by_subsys(tp);
15373 			if (p) {
15374 				tp->phy_id = p->phy_id;
15375 			} else if (!tg3_flag(tp, IS_SSB_CORE)) {
15376 				/* For now we saw the IDs 0xbc050cd0,
15377 				 * 0xbc050f80 and 0xbc050c30 on devices
15378 				 * connected to an BCM4785 and there are
15379 				 * probably more. Just assume that the phy is
15380 				 * supported when it is connected to a SSB core
15381 				 * for now.
15382 				 */
15383 				return -ENODEV;
15384 			}
15385 
15386 			if (!tp->phy_id ||
15387 			    tp->phy_id == TG3_PHY_ID_BCM8002)
15388 				tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15389 		}
15390 	}
15391 
15392 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15393 	    (tg3_asic_rev(tp) == ASIC_REV_5719 ||
15394 	     tg3_asic_rev(tp) == ASIC_REV_5720 ||
15395 	     tg3_asic_rev(tp) == ASIC_REV_57766 ||
15396 	     tg3_asic_rev(tp) == ASIC_REV_5762 ||
15397 	     (tg3_asic_rev(tp) == ASIC_REV_5717 &&
15398 	      tg3_chip_rev_id(tp) != CHIPREV_ID_5717_A0) ||
15399 	     (tg3_asic_rev(tp) == ASIC_REV_57765 &&
15400 	      tg3_chip_rev_id(tp) != CHIPREV_ID_57765_A0))) {
15401 		tp->phy_flags |= TG3_PHYFLG_EEE_CAP;
15402 
15403 		tp->eee.supported = SUPPORTED_100baseT_Full |
15404 				    SUPPORTED_1000baseT_Full;
15405 		tp->eee.advertised = ADVERTISED_100baseT_Full |
15406 				     ADVERTISED_1000baseT_Full;
15407 		tp->eee.eee_enabled = 1;
15408 		tp->eee.tx_lpi_enabled = 1;
15409 		tp->eee.tx_lpi_timer = TG3_CPMU_DBTMR1_LNKIDLE_2047US;
15410 	}
15411 
15412 	tg3_phy_init_link_config(tp);
15413 
15414 	if (!(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
15415 	    !(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15416 	    !tg3_flag(tp, ENABLE_APE) &&
15417 	    !tg3_flag(tp, ENABLE_ASF)) {
15418 		u32 bmsr, dummy;
15419 
15420 		tg3_readphy(tp, MII_BMSR, &bmsr);
15421 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
15422 		    (bmsr & BMSR_LSTATUS))
15423 			goto skip_phy_reset;
15424 
15425 		err = tg3_phy_reset(tp);
15426 		if (err)
15427 			return err;
15428 
15429 		tg3_phy_set_wirespeed(tp);
15430 
15431 		if (!tg3_phy_copper_an_config_ok(tp, &dummy)) {
15432 			tg3_phy_autoneg_cfg(tp, tp->link_config.advertising,
15433 					    tp->link_config.flowctrl);
15434 
15435 			tg3_writephy(tp, MII_BMCR,
15436 				     BMCR_ANENABLE | BMCR_ANRESTART);
15437 		}
15438 	}
15439 
15440 skip_phy_reset:
15441 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
15442 		err = tg3_init_5401phy_dsp(tp);
15443 		if (err)
15444 			return err;
15445 
15446 		err = tg3_init_5401phy_dsp(tp);
15447 	}
15448 
15449 	return err;
15450 }
15451 
15452 static void tg3_read_vpd(struct tg3 *tp)
15453 {
15454 	u8 *vpd_data;
15455 	unsigned int block_end, rosize, len;
15456 	u32 vpdlen;
15457 	int j, i = 0;
15458 
15459 	vpd_data = (u8 *)tg3_vpd_readblock(tp, &vpdlen);
15460 	if (!vpd_data)
15461 		goto out_no_vpd;
15462 
15463 	i = pci_vpd_find_tag(vpd_data, 0, vpdlen, PCI_VPD_LRDT_RO_DATA);
15464 	if (i < 0)
15465 		goto out_not_found;
15466 
15467 	rosize = pci_vpd_lrdt_size(&vpd_data[i]);
15468 	block_end = i + PCI_VPD_LRDT_TAG_SIZE + rosize;
15469 	i += PCI_VPD_LRDT_TAG_SIZE;
15470 
15471 	if (block_end > vpdlen)
15472 		goto out_not_found;
15473 
15474 	j = pci_vpd_find_info_keyword(vpd_data, i, rosize,
15475 				      PCI_VPD_RO_KEYWORD_MFR_ID);
15476 	if (j > 0) {
15477 		len = pci_vpd_info_field_size(&vpd_data[j]);
15478 
15479 		j += PCI_VPD_INFO_FLD_HDR_SIZE;
15480 		if (j + len > block_end || len != 4 ||
15481 		    memcmp(&vpd_data[j], "1028", 4))
15482 			goto partno;
15483 
15484 		j = pci_vpd_find_info_keyword(vpd_data, i, rosize,
15485 					      PCI_VPD_RO_KEYWORD_VENDOR0);
15486 		if (j < 0)
15487 			goto partno;
15488 
15489 		len = pci_vpd_info_field_size(&vpd_data[j]);
15490 
15491 		j += PCI_VPD_INFO_FLD_HDR_SIZE;
15492 		if (j + len > block_end)
15493 			goto partno;
15494 
15495 		if (len >= sizeof(tp->fw_ver))
15496 			len = sizeof(tp->fw_ver) - 1;
15497 		memset(tp->fw_ver, 0, sizeof(tp->fw_ver));
15498 		snprintf(tp->fw_ver, sizeof(tp->fw_ver), "%.*s bc ", len,
15499 			 &vpd_data[j]);
15500 	}
15501 
15502 partno:
15503 	i = pci_vpd_find_info_keyword(vpd_data, i, rosize,
15504 				      PCI_VPD_RO_KEYWORD_PARTNO);
15505 	if (i < 0)
15506 		goto out_not_found;
15507 
15508 	len = pci_vpd_info_field_size(&vpd_data[i]);
15509 
15510 	i += PCI_VPD_INFO_FLD_HDR_SIZE;
15511 	if (len > TG3_BPN_SIZE ||
15512 	    (len + i) > vpdlen)
15513 		goto out_not_found;
15514 
15515 	memcpy(tp->board_part_number, &vpd_data[i], len);
15516 
15517 out_not_found:
15518 	kfree(vpd_data);
15519 	if (tp->board_part_number[0])
15520 		return;
15521 
15522 out_no_vpd:
15523 	if (tg3_asic_rev(tp) == ASIC_REV_5717) {
15524 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
15525 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C)
15526 			strcpy(tp->board_part_number, "BCM5717");
15527 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718)
15528 			strcpy(tp->board_part_number, "BCM5718");
15529 		else
15530 			goto nomatch;
15531 	} else if (tg3_asic_rev(tp) == ASIC_REV_57780) {
15532 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57780)
15533 			strcpy(tp->board_part_number, "BCM57780");
15534 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57760)
15535 			strcpy(tp->board_part_number, "BCM57760");
15536 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57790)
15537 			strcpy(tp->board_part_number, "BCM57790");
15538 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57788)
15539 			strcpy(tp->board_part_number, "BCM57788");
15540 		else
15541 			goto nomatch;
15542 	} else if (tg3_asic_rev(tp) == ASIC_REV_57765) {
15543 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761)
15544 			strcpy(tp->board_part_number, "BCM57761");
15545 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765)
15546 			strcpy(tp->board_part_number, "BCM57765");
15547 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57781)
15548 			strcpy(tp->board_part_number, "BCM57781");
15549 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57785)
15550 			strcpy(tp->board_part_number, "BCM57785");
15551 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791)
15552 			strcpy(tp->board_part_number, "BCM57791");
15553 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795)
15554 			strcpy(tp->board_part_number, "BCM57795");
15555 		else
15556 			goto nomatch;
15557 	} else if (tg3_asic_rev(tp) == ASIC_REV_57766) {
15558 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57762)
15559 			strcpy(tp->board_part_number, "BCM57762");
15560 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57766)
15561 			strcpy(tp->board_part_number, "BCM57766");
15562 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57782)
15563 			strcpy(tp->board_part_number, "BCM57782");
15564 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57786)
15565 			strcpy(tp->board_part_number, "BCM57786");
15566 		else
15567 			goto nomatch;
15568 	} else if (tg3_asic_rev(tp) == ASIC_REV_5906) {
15569 		strcpy(tp->board_part_number, "BCM95906");
15570 	} else {
15571 nomatch:
15572 		strcpy(tp->board_part_number, "none");
15573 	}
15574 }
15575 
15576 static int tg3_fw_img_is_valid(struct tg3 *tp, u32 offset)
15577 {
15578 	u32 val;
15579 
15580 	if (tg3_nvram_read(tp, offset, &val) ||
15581 	    (val & 0xfc000000) != 0x0c000000 ||
15582 	    tg3_nvram_read(tp, offset + 4, &val) ||
15583 	    val != 0)
15584 		return 0;
15585 
15586 	return 1;
15587 }
15588 
15589 static void tg3_read_bc_ver(struct tg3 *tp)
15590 {
15591 	u32 val, offset, start, ver_offset;
15592 	int i, dst_off;
15593 	bool newver = false;
15594 
15595 	if (tg3_nvram_read(tp, 0xc, &offset) ||
15596 	    tg3_nvram_read(tp, 0x4, &start))
15597 		return;
15598 
15599 	offset = tg3_nvram_logical_addr(tp, offset);
15600 
15601 	if (tg3_nvram_read(tp, offset, &val))
15602 		return;
15603 
15604 	if ((val & 0xfc000000) == 0x0c000000) {
15605 		if (tg3_nvram_read(tp, offset + 4, &val))
15606 			return;
15607 
15608 		if (val == 0)
15609 			newver = true;
15610 	}
15611 
15612 	dst_off = strlen(tp->fw_ver);
15613 
15614 	if (newver) {
15615 		if (TG3_VER_SIZE - dst_off < 16 ||
15616 		    tg3_nvram_read(tp, offset + 8, &ver_offset))
15617 			return;
15618 
15619 		offset = offset + ver_offset - start;
15620 		for (i = 0; i < 16; i += 4) {
15621 			__be32 v;
15622 			if (tg3_nvram_read_be32(tp, offset + i, &v))
15623 				return;
15624 
15625 			memcpy(tp->fw_ver + dst_off + i, &v, sizeof(v));
15626 		}
15627 	} else {
15628 		u32 major, minor;
15629 
15630 		if (tg3_nvram_read(tp, TG3_NVM_PTREV_BCVER, &ver_offset))
15631 			return;
15632 
15633 		major = (ver_offset & TG3_NVM_BCVER_MAJMSK) >>
15634 			TG3_NVM_BCVER_MAJSFT;
15635 		minor = ver_offset & TG3_NVM_BCVER_MINMSK;
15636 		snprintf(&tp->fw_ver[dst_off], TG3_VER_SIZE - dst_off,
15637 			 "v%d.%02d", major, minor);
15638 	}
15639 }
15640 
15641 static void tg3_read_hwsb_ver(struct tg3 *tp)
15642 {
15643 	u32 val, major, minor;
15644 
15645 	/* Use native endian representation */
15646 	if (tg3_nvram_read(tp, TG3_NVM_HWSB_CFG1, &val))
15647 		return;
15648 
15649 	major = (val & TG3_NVM_HWSB_CFG1_MAJMSK) >>
15650 		TG3_NVM_HWSB_CFG1_MAJSFT;
15651 	minor = (val & TG3_NVM_HWSB_CFG1_MINMSK) >>
15652 		TG3_NVM_HWSB_CFG1_MINSFT;
15653 
15654 	snprintf(&tp->fw_ver[0], 32, "sb v%d.%02d", major, minor);
15655 }
15656 
15657 static void tg3_read_sb_ver(struct tg3 *tp, u32 val)
15658 {
15659 	u32 offset, major, minor, build;
15660 
15661 	strncat(tp->fw_ver, "sb", TG3_VER_SIZE - strlen(tp->fw_ver) - 1);
15662 
15663 	if ((val & TG3_EEPROM_SB_FORMAT_MASK) != TG3_EEPROM_SB_FORMAT_1)
15664 		return;
15665 
15666 	switch (val & TG3_EEPROM_SB_REVISION_MASK) {
15667 	case TG3_EEPROM_SB_REVISION_0:
15668 		offset = TG3_EEPROM_SB_F1R0_EDH_OFF;
15669 		break;
15670 	case TG3_EEPROM_SB_REVISION_2:
15671 		offset = TG3_EEPROM_SB_F1R2_EDH_OFF;
15672 		break;
15673 	case TG3_EEPROM_SB_REVISION_3:
15674 		offset = TG3_EEPROM_SB_F1R3_EDH_OFF;
15675 		break;
15676 	case TG3_EEPROM_SB_REVISION_4:
15677 		offset = TG3_EEPROM_SB_F1R4_EDH_OFF;
15678 		break;
15679 	case TG3_EEPROM_SB_REVISION_5:
15680 		offset = TG3_EEPROM_SB_F1R5_EDH_OFF;
15681 		break;
15682 	case TG3_EEPROM_SB_REVISION_6:
15683 		offset = TG3_EEPROM_SB_F1R6_EDH_OFF;
15684 		break;
15685 	default:
15686 		return;
15687 	}
15688 
15689 	if (tg3_nvram_read(tp, offset, &val))
15690 		return;
15691 
15692 	build = (val & TG3_EEPROM_SB_EDH_BLD_MASK) >>
15693 		TG3_EEPROM_SB_EDH_BLD_SHFT;
15694 	major = (val & TG3_EEPROM_SB_EDH_MAJ_MASK) >>
15695 		TG3_EEPROM_SB_EDH_MAJ_SHFT;
15696 	minor =  val & TG3_EEPROM_SB_EDH_MIN_MASK;
15697 
15698 	if (minor > 99 || build > 26)
15699 		return;
15700 
15701 	offset = strlen(tp->fw_ver);
15702 	snprintf(&tp->fw_ver[offset], TG3_VER_SIZE - offset,
15703 		 " v%d.%02d", major, minor);
15704 
15705 	if (build > 0) {
15706 		offset = strlen(tp->fw_ver);
15707 		if (offset < TG3_VER_SIZE - 1)
15708 			tp->fw_ver[offset] = 'a' + build - 1;
15709 	}
15710 }
15711 
15712 static void tg3_read_mgmtfw_ver(struct tg3 *tp)
15713 {
15714 	u32 val, offset, start;
15715 	int i, vlen;
15716 
15717 	for (offset = TG3_NVM_DIR_START;
15718 	     offset < TG3_NVM_DIR_END;
15719 	     offset += TG3_NVM_DIRENT_SIZE) {
15720 		if (tg3_nvram_read(tp, offset, &val))
15721 			return;
15722 
15723 		if ((val >> TG3_NVM_DIRTYPE_SHIFT) == TG3_NVM_DIRTYPE_ASFINI)
15724 			break;
15725 	}
15726 
15727 	if (offset == TG3_NVM_DIR_END)
15728 		return;
15729 
15730 	if (!tg3_flag(tp, 5705_PLUS))
15731 		start = 0x08000000;
15732 	else if (tg3_nvram_read(tp, offset - 4, &start))
15733 		return;
15734 
15735 	if (tg3_nvram_read(tp, offset + 4, &offset) ||
15736 	    !tg3_fw_img_is_valid(tp, offset) ||
15737 	    tg3_nvram_read(tp, offset + 8, &val))
15738 		return;
15739 
15740 	offset += val - start;
15741 
15742 	vlen = strlen(tp->fw_ver);
15743 
15744 	tp->fw_ver[vlen++] = ',';
15745 	tp->fw_ver[vlen++] = ' ';
15746 
15747 	for (i = 0; i < 4; i++) {
15748 		__be32 v;
15749 		if (tg3_nvram_read_be32(tp, offset, &v))
15750 			return;
15751 
15752 		offset += sizeof(v);
15753 
15754 		if (vlen > TG3_VER_SIZE - sizeof(v)) {
15755 			memcpy(&tp->fw_ver[vlen], &v, TG3_VER_SIZE - vlen);
15756 			break;
15757 		}
15758 
15759 		memcpy(&tp->fw_ver[vlen], &v, sizeof(v));
15760 		vlen += sizeof(v);
15761 	}
15762 }
15763 
15764 static void tg3_probe_ncsi(struct tg3 *tp)
15765 {
15766 	u32 apedata;
15767 
15768 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
15769 	if (apedata != APE_SEG_SIG_MAGIC)
15770 		return;
15771 
15772 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
15773 	if (!(apedata & APE_FW_STATUS_READY))
15774 		return;
15775 
15776 	if (tg3_ape_read32(tp, TG3_APE_FW_FEATURES) & TG3_APE_FW_FEATURE_NCSI)
15777 		tg3_flag_set(tp, APE_HAS_NCSI);
15778 }
15779 
15780 static void tg3_read_dash_ver(struct tg3 *tp)
15781 {
15782 	int vlen;
15783 	u32 apedata;
15784 	char *fwtype;
15785 
15786 	apedata = tg3_ape_read32(tp, TG3_APE_FW_VERSION);
15787 
15788 	if (tg3_flag(tp, APE_HAS_NCSI))
15789 		fwtype = "NCSI";
15790 	else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725)
15791 		fwtype = "SMASH";
15792 	else
15793 		fwtype = "DASH";
15794 
15795 	vlen = strlen(tp->fw_ver);
15796 
15797 	snprintf(&tp->fw_ver[vlen], TG3_VER_SIZE - vlen, " %s v%d.%d.%d.%d",
15798 		 fwtype,
15799 		 (apedata & APE_FW_VERSION_MAJMSK) >> APE_FW_VERSION_MAJSFT,
15800 		 (apedata & APE_FW_VERSION_MINMSK) >> APE_FW_VERSION_MINSFT,
15801 		 (apedata & APE_FW_VERSION_REVMSK) >> APE_FW_VERSION_REVSFT,
15802 		 (apedata & APE_FW_VERSION_BLDMSK));
15803 }
15804 
15805 static void tg3_read_otp_ver(struct tg3 *tp)
15806 {
15807 	u32 val, val2;
15808 
15809 	if (tg3_asic_rev(tp) != ASIC_REV_5762)
15810 		return;
15811 
15812 	if (!tg3_ape_otp_read(tp, OTP_ADDRESS_MAGIC0, &val) &&
15813 	    !tg3_ape_otp_read(tp, OTP_ADDRESS_MAGIC0 + 4, &val2) &&
15814 	    TG3_OTP_MAGIC0_VALID(val)) {
15815 		u64 val64 = (u64) val << 32 | val2;
15816 		u32 ver = 0;
15817 		int i, vlen;
15818 
15819 		for (i = 0; i < 7; i++) {
15820 			if ((val64 & 0xff) == 0)
15821 				break;
15822 			ver = val64 & 0xff;
15823 			val64 >>= 8;
15824 		}
15825 		vlen = strlen(tp->fw_ver);
15826 		snprintf(&tp->fw_ver[vlen], TG3_VER_SIZE - vlen, " .%02d", ver);
15827 	}
15828 }
15829 
15830 static void tg3_read_fw_ver(struct tg3 *tp)
15831 {
15832 	u32 val;
15833 	bool vpd_vers = false;
15834 
15835 	if (tp->fw_ver[0] != 0)
15836 		vpd_vers = true;
15837 
15838 	if (tg3_flag(tp, NO_NVRAM)) {
15839 		strcat(tp->fw_ver, "sb");
15840 		tg3_read_otp_ver(tp);
15841 		return;
15842 	}
15843 
15844 	if (tg3_nvram_read(tp, 0, &val))
15845 		return;
15846 
15847 	if (val == TG3_EEPROM_MAGIC)
15848 		tg3_read_bc_ver(tp);
15849 	else if ((val & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW)
15850 		tg3_read_sb_ver(tp, val);
15851 	else if ((val & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
15852 		tg3_read_hwsb_ver(tp);
15853 
15854 	if (tg3_flag(tp, ENABLE_ASF)) {
15855 		if (tg3_flag(tp, ENABLE_APE)) {
15856 			tg3_probe_ncsi(tp);
15857 			if (!vpd_vers)
15858 				tg3_read_dash_ver(tp);
15859 		} else if (!vpd_vers) {
15860 			tg3_read_mgmtfw_ver(tp);
15861 		}
15862 	}
15863 
15864 	tp->fw_ver[TG3_VER_SIZE - 1] = 0;
15865 }
15866 
15867 static inline u32 tg3_rx_ret_ring_size(struct tg3 *tp)
15868 {
15869 	if (tg3_flag(tp, LRG_PROD_RING_CAP))
15870 		return TG3_RX_RET_MAX_SIZE_5717;
15871 	else if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS))
15872 		return TG3_RX_RET_MAX_SIZE_5700;
15873 	else
15874 		return TG3_RX_RET_MAX_SIZE_5705;
15875 }
15876 
15877 static DEFINE_PCI_DEVICE_TABLE(tg3_write_reorder_chipsets) = {
15878 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_FE_GATE_700C) },
15879 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8131_BRIDGE) },
15880 	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8385_0) },
15881 	{ },
15882 };
15883 
15884 static struct pci_dev *tg3_find_peer(struct tg3 *tp)
15885 {
15886 	struct pci_dev *peer;
15887 	unsigned int func, devnr = tp->pdev->devfn & ~7;
15888 
15889 	for (func = 0; func < 8; func++) {
15890 		peer = pci_get_slot(tp->pdev->bus, devnr | func);
15891 		if (peer && peer != tp->pdev)
15892 			break;
15893 		pci_dev_put(peer);
15894 	}
15895 	/* 5704 can be configured in single-port mode, set peer to
15896 	 * tp->pdev in that case.
15897 	 */
15898 	if (!peer) {
15899 		peer = tp->pdev;
15900 		return peer;
15901 	}
15902 
15903 	/*
15904 	 * We don't need to keep the refcount elevated; there's no way
15905 	 * to remove one half of this device without removing the other
15906 	 */
15907 	pci_dev_put(peer);
15908 
15909 	return peer;
15910 }
15911 
15912 static void tg3_detect_asic_rev(struct tg3 *tp, u32 misc_ctrl_reg)
15913 {
15914 	tp->pci_chip_rev_id = misc_ctrl_reg >> MISC_HOST_CTRL_CHIPREV_SHIFT;
15915 	if (tg3_asic_rev(tp) == ASIC_REV_USE_PROD_ID_REG) {
15916 		u32 reg;
15917 
15918 		/* All devices that use the alternate
15919 		 * ASIC REV location have a CPMU.
15920 		 */
15921 		tg3_flag_set(tp, CPMU_PRESENT);
15922 
15923 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
15924 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C ||
15925 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718 ||
15926 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5719 ||
15927 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5720 ||
15928 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57767 ||
15929 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57764 ||
15930 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5762 ||
15931 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725 ||
15932 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5727 ||
15933 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57787)
15934 			reg = TG3PCI_GEN2_PRODID_ASICREV;
15935 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57781 ||
15936 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57785 ||
15937 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761 ||
15938 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765 ||
15939 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791 ||
15940 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795 ||
15941 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57762 ||
15942 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57766 ||
15943 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57782 ||
15944 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57786)
15945 			reg = TG3PCI_GEN15_PRODID_ASICREV;
15946 		else
15947 			reg = TG3PCI_PRODID_ASICREV;
15948 
15949 		pci_read_config_dword(tp->pdev, reg, &tp->pci_chip_rev_id);
15950 	}
15951 
15952 	/* Wrong chip ID in 5752 A0. This code can be removed later
15953 	 * as A0 is not in production.
15954 	 */
15955 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5752_A0_HW)
15956 		tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
15957 
15958 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5717_C0)
15959 		tp->pci_chip_rev_id = CHIPREV_ID_5720_A0;
15960 
15961 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
15962 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
15963 	    tg3_asic_rev(tp) == ASIC_REV_5720)
15964 		tg3_flag_set(tp, 5717_PLUS);
15965 
15966 	if (tg3_asic_rev(tp) == ASIC_REV_57765 ||
15967 	    tg3_asic_rev(tp) == ASIC_REV_57766)
15968 		tg3_flag_set(tp, 57765_CLASS);
15969 
15970 	if (tg3_flag(tp, 57765_CLASS) || tg3_flag(tp, 5717_PLUS) ||
15971 	     tg3_asic_rev(tp) == ASIC_REV_5762)
15972 		tg3_flag_set(tp, 57765_PLUS);
15973 
15974 	/* Intentionally exclude ASIC_REV_5906 */
15975 	if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
15976 	    tg3_asic_rev(tp) == ASIC_REV_5787 ||
15977 	    tg3_asic_rev(tp) == ASIC_REV_5784 ||
15978 	    tg3_asic_rev(tp) == ASIC_REV_5761 ||
15979 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
15980 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
15981 	    tg3_flag(tp, 57765_PLUS))
15982 		tg3_flag_set(tp, 5755_PLUS);
15983 
15984 	if (tg3_asic_rev(tp) == ASIC_REV_5780 ||
15985 	    tg3_asic_rev(tp) == ASIC_REV_5714)
15986 		tg3_flag_set(tp, 5780_CLASS);
15987 
15988 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
15989 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
15990 	    tg3_asic_rev(tp) == ASIC_REV_5906 ||
15991 	    tg3_flag(tp, 5755_PLUS) ||
15992 	    tg3_flag(tp, 5780_CLASS))
15993 		tg3_flag_set(tp, 5750_PLUS);
15994 
15995 	if (tg3_asic_rev(tp) == ASIC_REV_5705 ||
15996 	    tg3_flag(tp, 5750_PLUS))
15997 		tg3_flag_set(tp, 5705_PLUS);
15998 }
15999 
16000 static bool tg3_10_100_only_device(struct tg3 *tp,
16001 				   const struct pci_device_id *ent)
16002 {
16003 	u32 grc_misc_cfg = tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK;
16004 
16005 	if ((tg3_asic_rev(tp) == ASIC_REV_5703 &&
16006 	     (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
16007 	    (tp->phy_flags & TG3_PHYFLG_IS_FET))
16008 		return true;
16009 
16010 	if (ent->driver_data & TG3_DRV_DATA_FLAG_10_100_ONLY) {
16011 		if (tg3_asic_rev(tp) == ASIC_REV_5705) {
16012 			if (ent->driver_data & TG3_DRV_DATA_FLAG_5705_10_100)
16013 				return true;
16014 		} else {
16015 			return true;
16016 		}
16017 	}
16018 
16019 	return false;
16020 }
16021 
16022 static int tg3_get_invariants(struct tg3 *tp, const struct pci_device_id *ent)
16023 {
16024 	u32 misc_ctrl_reg;
16025 	u32 pci_state_reg, grc_misc_cfg;
16026 	u32 val;
16027 	u16 pci_cmd;
16028 	int err;
16029 
16030 	/* Force memory write invalidate off.  If we leave it on,
16031 	 * then on 5700_BX chips we have to enable a workaround.
16032 	 * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
16033 	 * to match the cacheline size.  The Broadcom driver have this
16034 	 * workaround but turns MWI off all the times so never uses
16035 	 * it.  This seems to suggest that the workaround is insufficient.
16036 	 */
16037 	pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16038 	pci_cmd &= ~PCI_COMMAND_INVALIDATE;
16039 	pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16040 
16041 	/* Important! -- Make sure register accesses are byteswapped
16042 	 * correctly.  Also, for those chips that require it, make
16043 	 * sure that indirect register accesses are enabled before
16044 	 * the first operation.
16045 	 */
16046 	pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16047 			      &misc_ctrl_reg);
16048 	tp->misc_host_ctrl |= (misc_ctrl_reg &
16049 			       MISC_HOST_CTRL_CHIPREV);
16050 	pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16051 			       tp->misc_host_ctrl);
16052 
16053 	tg3_detect_asic_rev(tp, misc_ctrl_reg);
16054 
16055 	/* If we have 5702/03 A1 or A2 on certain ICH chipsets,
16056 	 * we need to disable memory and use config. cycles
16057 	 * only to access all registers. The 5702/03 chips
16058 	 * can mistakenly decode the special cycles from the
16059 	 * ICH chipsets as memory write cycles, causing corruption
16060 	 * of register and memory space. Only certain ICH bridges
16061 	 * will drive special cycles with non-zero data during the
16062 	 * address phase which can fall within the 5703's address
16063 	 * range. This is not an ICH bug as the PCI spec allows
16064 	 * non-zero address during special cycles. However, only
16065 	 * these ICH bridges are known to drive non-zero addresses
16066 	 * during special cycles.
16067 	 *
16068 	 * Since special cycles do not cross PCI bridges, we only
16069 	 * enable this workaround if the 5703 is on the secondary
16070 	 * bus of these ICH bridges.
16071 	 */
16072 	if ((tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A1) ||
16073 	    (tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A2)) {
16074 		static struct tg3_dev_id {
16075 			u32	vendor;
16076 			u32	device;
16077 			u32	rev;
16078 		} ich_chipsets[] = {
16079 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
16080 			  PCI_ANY_ID },
16081 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
16082 			  PCI_ANY_ID },
16083 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
16084 			  0xa },
16085 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
16086 			  PCI_ANY_ID },
16087 			{ },
16088 		};
16089 		struct tg3_dev_id *pci_id = &ich_chipsets[0];
16090 		struct pci_dev *bridge = NULL;
16091 
16092 		while (pci_id->vendor != 0) {
16093 			bridge = pci_get_device(pci_id->vendor, pci_id->device,
16094 						bridge);
16095 			if (!bridge) {
16096 				pci_id++;
16097 				continue;
16098 			}
16099 			if (pci_id->rev != PCI_ANY_ID) {
16100 				if (bridge->revision > pci_id->rev)
16101 					continue;
16102 			}
16103 			if (bridge->subordinate &&
16104 			    (bridge->subordinate->number ==
16105 			     tp->pdev->bus->number)) {
16106 				tg3_flag_set(tp, ICH_WORKAROUND);
16107 				pci_dev_put(bridge);
16108 				break;
16109 			}
16110 		}
16111 	}
16112 
16113 	if (tg3_asic_rev(tp) == ASIC_REV_5701) {
16114 		static struct tg3_dev_id {
16115 			u32	vendor;
16116 			u32	device;
16117 		} bridge_chipsets[] = {
16118 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_0 },
16119 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_1 },
16120 			{ },
16121 		};
16122 		struct tg3_dev_id *pci_id = &bridge_chipsets[0];
16123 		struct pci_dev *bridge = NULL;
16124 
16125 		while (pci_id->vendor != 0) {
16126 			bridge = pci_get_device(pci_id->vendor,
16127 						pci_id->device,
16128 						bridge);
16129 			if (!bridge) {
16130 				pci_id++;
16131 				continue;
16132 			}
16133 			if (bridge->subordinate &&
16134 			    (bridge->subordinate->number <=
16135 			     tp->pdev->bus->number) &&
16136 			    (bridge->subordinate->busn_res.end >=
16137 			     tp->pdev->bus->number)) {
16138 				tg3_flag_set(tp, 5701_DMA_BUG);
16139 				pci_dev_put(bridge);
16140 				break;
16141 			}
16142 		}
16143 	}
16144 
16145 	/* The EPB bridge inside 5714, 5715, and 5780 cannot support
16146 	 * DMA addresses > 40-bit. This bridge may have other additional
16147 	 * 57xx devices behind it in some 4-port NIC designs for example.
16148 	 * Any tg3 device found behind the bridge will also need the 40-bit
16149 	 * DMA workaround.
16150 	 */
16151 	if (tg3_flag(tp, 5780_CLASS)) {
16152 		tg3_flag_set(tp, 40BIT_DMA_BUG);
16153 		tp->msi_cap = tp->pdev->msi_cap;
16154 	} else {
16155 		struct pci_dev *bridge = NULL;
16156 
16157 		do {
16158 			bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
16159 						PCI_DEVICE_ID_SERVERWORKS_EPB,
16160 						bridge);
16161 			if (bridge && bridge->subordinate &&
16162 			    (bridge->subordinate->number <=
16163 			     tp->pdev->bus->number) &&
16164 			    (bridge->subordinate->busn_res.end >=
16165 			     tp->pdev->bus->number)) {
16166 				tg3_flag_set(tp, 40BIT_DMA_BUG);
16167 				pci_dev_put(bridge);
16168 				break;
16169 			}
16170 		} while (bridge);
16171 	}
16172 
16173 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16174 	    tg3_asic_rev(tp) == ASIC_REV_5714)
16175 		tp->pdev_peer = tg3_find_peer(tp);
16176 
16177 	/* Determine TSO capabilities */
16178 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0)
16179 		; /* Do nothing. HW bug. */
16180 	else if (tg3_flag(tp, 57765_PLUS))
16181 		tg3_flag_set(tp, HW_TSO_3);
16182 	else if (tg3_flag(tp, 5755_PLUS) ||
16183 		 tg3_asic_rev(tp) == ASIC_REV_5906)
16184 		tg3_flag_set(tp, HW_TSO_2);
16185 	else if (tg3_flag(tp, 5750_PLUS)) {
16186 		tg3_flag_set(tp, HW_TSO_1);
16187 		tg3_flag_set(tp, TSO_BUG);
16188 		if (tg3_asic_rev(tp) == ASIC_REV_5750 &&
16189 		    tg3_chip_rev_id(tp) >= CHIPREV_ID_5750_C2)
16190 			tg3_flag_clear(tp, TSO_BUG);
16191 	} else if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
16192 		   tg3_asic_rev(tp) != ASIC_REV_5701 &&
16193 		   tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
16194 		tg3_flag_set(tp, FW_TSO);
16195 		tg3_flag_set(tp, TSO_BUG);
16196 		if (tg3_asic_rev(tp) == ASIC_REV_5705)
16197 			tp->fw_needed = FIRMWARE_TG3TSO5;
16198 		else
16199 			tp->fw_needed = FIRMWARE_TG3TSO;
16200 	}
16201 
16202 	/* Selectively allow TSO based on operating conditions */
16203 	if (tg3_flag(tp, HW_TSO_1) ||
16204 	    tg3_flag(tp, HW_TSO_2) ||
16205 	    tg3_flag(tp, HW_TSO_3) ||
16206 	    tg3_flag(tp, FW_TSO)) {
16207 		/* For firmware TSO, assume ASF is disabled.
16208 		 * We'll disable TSO later if we discover ASF
16209 		 * is enabled in tg3_get_eeprom_hw_cfg().
16210 		 */
16211 		tg3_flag_set(tp, TSO_CAPABLE);
16212 	} else {
16213 		tg3_flag_clear(tp, TSO_CAPABLE);
16214 		tg3_flag_clear(tp, TSO_BUG);
16215 		tp->fw_needed = NULL;
16216 	}
16217 
16218 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0)
16219 		tp->fw_needed = FIRMWARE_TG3;
16220 
16221 	if (tg3_asic_rev(tp) == ASIC_REV_57766)
16222 		tp->fw_needed = FIRMWARE_TG357766;
16223 
16224 	tp->irq_max = 1;
16225 
16226 	if (tg3_flag(tp, 5750_PLUS)) {
16227 		tg3_flag_set(tp, SUPPORT_MSI);
16228 		if (tg3_chip_rev(tp) == CHIPREV_5750_AX ||
16229 		    tg3_chip_rev(tp) == CHIPREV_5750_BX ||
16230 		    (tg3_asic_rev(tp) == ASIC_REV_5714 &&
16231 		     tg3_chip_rev_id(tp) <= CHIPREV_ID_5714_A2 &&
16232 		     tp->pdev_peer == tp->pdev))
16233 			tg3_flag_clear(tp, SUPPORT_MSI);
16234 
16235 		if (tg3_flag(tp, 5755_PLUS) ||
16236 		    tg3_asic_rev(tp) == ASIC_REV_5906) {
16237 			tg3_flag_set(tp, 1SHOT_MSI);
16238 		}
16239 
16240 		if (tg3_flag(tp, 57765_PLUS)) {
16241 			tg3_flag_set(tp, SUPPORT_MSIX);
16242 			tp->irq_max = TG3_IRQ_MAX_VECS;
16243 		}
16244 	}
16245 
16246 	tp->txq_max = 1;
16247 	tp->rxq_max = 1;
16248 	if (tp->irq_max > 1) {
16249 		tp->rxq_max = TG3_RSS_MAX_NUM_QS;
16250 		tg3_rss_init_dflt_indir_tbl(tp, TG3_RSS_MAX_NUM_QS);
16251 
16252 		if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
16253 		    tg3_asic_rev(tp) == ASIC_REV_5720)
16254 			tp->txq_max = tp->irq_max - 1;
16255 	}
16256 
16257 	if (tg3_flag(tp, 5755_PLUS) ||
16258 	    tg3_asic_rev(tp) == ASIC_REV_5906)
16259 		tg3_flag_set(tp, SHORT_DMA_BUG);
16260 
16261 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
16262 		tp->dma_limit = TG3_TX_BD_DMA_MAX_4K;
16263 
16264 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16265 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
16266 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
16267 	    tg3_asic_rev(tp) == ASIC_REV_5762)
16268 		tg3_flag_set(tp, LRG_PROD_RING_CAP);
16269 
16270 	if (tg3_flag(tp, 57765_PLUS) &&
16271 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5719_A0)
16272 		tg3_flag_set(tp, USE_JUMBO_BDFLAG);
16273 
16274 	if (!tg3_flag(tp, 5705_PLUS) ||
16275 	    tg3_flag(tp, 5780_CLASS) ||
16276 	    tg3_flag(tp, USE_JUMBO_BDFLAG))
16277 		tg3_flag_set(tp, JUMBO_CAPABLE);
16278 
16279 	pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
16280 			      &pci_state_reg);
16281 
16282 	if (pci_is_pcie(tp->pdev)) {
16283 		u16 lnkctl;
16284 
16285 		tg3_flag_set(tp, PCI_EXPRESS);
16286 
16287 		pcie_capability_read_word(tp->pdev, PCI_EXP_LNKCTL, &lnkctl);
16288 		if (lnkctl & PCI_EXP_LNKCTL_CLKREQ_EN) {
16289 			if (tg3_asic_rev(tp) == ASIC_REV_5906) {
16290 				tg3_flag_clear(tp, HW_TSO_2);
16291 				tg3_flag_clear(tp, TSO_CAPABLE);
16292 			}
16293 			if (tg3_asic_rev(tp) == ASIC_REV_5784 ||
16294 			    tg3_asic_rev(tp) == ASIC_REV_5761 ||
16295 			    tg3_chip_rev_id(tp) == CHIPREV_ID_57780_A0 ||
16296 			    tg3_chip_rev_id(tp) == CHIPREV_ID_57780_A1)
16297 				tg3_flag_set(tp, CLKREQ_BUG);
16298 		} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5717_A0) {
16299 			tg3_flag_set(tp, L1PLLPD_EN);
16300 		}
16301 	} else if (tg3_asic_rev(tp) == ASIC_REV_5785) {
16302 		/* BCM5785 devices are effectively PCIe devices, and should
16303 		 * follow PCIe codepaths, but do not have a PCIe capabilities
16304 		 * section.
16305 		 */
16306 		tg3_flag_set(tp, PCI_EXPRESS);
16307 	} else if (!tg3_flag(tp, 5705_PLUS) ||
16308 		   tg3_flag(tp, 5780_CLASS)) {
16309 		tp->pcix_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_PCIX);
16310 		if (!tp->pcix_cap) {
16311 			dev_err(&tp->pdev->dev,
16312 				"Cannot find PCI-X capability, aborting\n");
16313 			return -EIO;
16314 		}
16315 
16316 		if (!(pci_state_reg & PCISTATE_CONV_PCI_MODE))
16317 			tg3_flag_set(tp, PCIX_MODE);
16318 	}
16319 
16320 	/* If we have an AMD 762 or VIA K8T800 chipset, write
16321 	 * reordering to the mailbox registers done by the host
16322 	 * controller can cause major troubles.  We read back from
16323 	 * every mailbox register write to force the writes to be
16324 	 * posted to the chip in order.
16325 	 */
16326 	if (pci_dev_present(tg3_write_reorder_chipsets) &&
16327 	    !tg3_flag(tp, PCI_EXPRESS))
16328 		tg3_flag_set(tp, MBOX_WRITE_REORDER);
16329 
16330 	pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
16331 			     &tp->pci_cacheline_sz);
16332 	pci_read_config_byte(tp->pdev, PCI_LATENCY_TIMER,
16333 			     &tp->pci_lat_timer);
16334 	if (tg3_asic_rev(tp) == ASIC_REV_5703 &&
16335 	    tp->pci_lat_timer < 64) {
16336 		tp->pci_lat_timer = 64;
16337 		pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
16338 				      tp->pci_lat_timer);
16339 	}
16340 
16341 	/* Important! -- It is critical that the PCI-X hw workaround
16342 	 * situation is decided before the first MMIO register access.
16343 	 */
16344 	if (tg3_chip_rev(tp) == CHIPREV_5700_BX) {
16345 		/* 5700 BX chips need to have their TX producer index
16346 		 * mailboxes written twice to workaround a bug.
16347 		 */
16348 		tg3_flag_set(tp, TXD_MBOX_HWBUG);
16349 
16350 		/* If we are in PCI-X mode, enable register write workaround.
16351 		 *
16352 		 * The workaround is to use indirect register accesses
16353 		 * for all chip writes not to mailbox registers.
16354 		 */
16355 		if (tg3_flag(tp, PCIX_MODE)) {
16356 			u32 pm_reg;
16357 
16358 			tg3_flag_set(tp, PCIX_TARGET_HWBUG);
16359 
16360 			/* The chip can have it's power management PCI config
16361 			 * space registers clobbered due to this bug.
16362 			 * So explicitly force the chip into D0 here.
16363 			 */
16364 			pci_read_config_dword(tp->pdev,
16365 					      tp->pdev->pm_cap + PCI_PM_CTRL,
16366 					      &pm_reg);
16367 			pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
16368 			pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
16369 			pci_write_config_dword(tp->pdev,
16370 					       tp->pdev->pm_cap + PCI_PM_CTRL,
16371 					       pm_reg);
16372 
16373 			/* Also, force SERR#/PERR# in PCI command. */
16374 			pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16375 			pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
16376 			pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16377 		}
16378 	}
16379 
16380 	if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
16381 		tg3_flag_set(tp, PCI_HIGH_SPEED);
16382 	if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
16383 		tg3_flag_set(tp, PCI_32BIT);
16384 
16385 	/* Chip-specific fixup from Broadcom driver */
16386 	if ((tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0) &&
16387 	    (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
16388 		pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
16389 		pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
16390 	}
16391 
16392 	/* Default fast path register access methods */
16393 	tp->read32 = tg3_read32;
16394 	tp->write32 = tg3_write32;
16395 	tp->read32_mbox = tg3_read32;
16396 	tp->write32_mbox = tg3_write32;
16397 	tp->write32_tx_mbox = tg3_write32;
16398 	tp->write32_rx_mbox = tg3_write32;
16399 
16400 	/* Various workaround register access methods */
16401 	if (tg3_flag(tp, PCIX_TARGET_HWBUG))
16402 		tp->write32 = tg3_write_indirect_reg32;
16403 	else if (tg3_asic_rev(tp) == ASIC_REV_5701 ||
16404 		 (tg3_flag(tp, PCI_EXPRESS) &&
16405 		  tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A0)) {
16406 		/*
16407 		 * Back to back register writes can cause problems on these
16408 		 * chips, the workaround is to read back all reg writes
16409 		 * except those to mailbox regs.
16410 		 *
16411 		 * See tg3_write_indirect_reg32().
16412 		 */
16413 		tp->write32 = tg3_write_flush_reg32;
16414 	}
16415 
16416 	if (tg3_flag(tp, TXD_MBOX_HWBUG) || tg3_flag(tp, MBOX_WRITE_REORDER)) {
16417 		tp->write32_tx_mbox = tg3_write32_tx_mbox;
16418 		if (tg3_flag(tp, MBOX_WRITE_REORDER))
16419 			tp->write32_rx_mbox = tg3_write_flush_reg32;
16420 	}
16421 
16422 	if (tg3_flag(tp, ICH_WORKAROUND)) {
16423 		tp->read32 = tg3_read_indirect_reg32;
16424 		tp->write32 = tg3_write_indirect_reg32;
16425 		tp->read32_mbox = tg3_read_indirect_mbox;
16426 		tp->write32_mbox = tg3_write_indirect_mbox;
16427 		tp->write32_tx_mbox = tg3_write_indirect_mbox;
16428 		tp->write32_rx_mbox = tg3_write_indirect_mbox;
16429 
16430 		iounmap(tp->regs);
16431 		tp->regs = NULL;
16432 
16433 		pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16434 		pci_cmd &= ~PCI_COMMAND_MEMORY;
16435 		pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16436 	}
16437 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
16438 		tp->read32_mbox = tg3_read32_mbox_5906;
16439 		tp->write32_mbox = tg3_write32_mbox_5906;
16440 		tp->write32_tx_mbox = tg3_write32_mbox_5906;
16441 		tp->write32_rx_mbox = tg3_write32_mbox_5906;
16442 	}
16443 
16444 	if (tp->write32 == tg3_write_indirect_reg32 ||
16445 	    (tg3_flag(tp, PCIX_MODE) &&
16446 	     (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16447 	      tg3_asic_rev(tp) == ASIC_REV_5701)))
16448 		tg3_flag_set(tp, SRAM_USE_CONFIG);
16449 
16450 	/* The memory arbiter has to be enabled in order for SRAM accesses
16451 	 * to succeed.  Normally on powerup the tg3 chip firmware will make
16452 	 * sure it is enabled, but other entities such as system netboot
16453 	 * code might disable it.
16454 	 */
16455 	val = tr32(MEMARB_MODE);
16456 	tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
16457 
16458 	tp->pci_fn = PCI_FUNC(tp->pdev->devfn) & 3;
16459 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16460 	    tg3_flag(tp, 5780_CLASS)) {
16461 		if (tg3_flag(tp, PCIX_MODE)) {
16462 			pci_read_config_dword(tp->pdev,
16463 					      tp->pcix_cap + PCI_X_STATUS,
16464 					      &val);
16465 			tp->pci_fn = val & 0x7;
16466 		}
16467 	} else if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16468 		   tg3_asic_rev(tp) == ASIC_REV_5719 ||
16469 		   tg3_asic_rev(tp) == ASIC_REV_5720) {
16470 		tg3_read_mem(tp, NIC_SRAM_CPMU_STATUS, &val);
16471 		if ((val & NIC_SRAM_CPMUSTAT_SIG_MSK) != NIC_SRAM_CPMUSTAT_SIG)
16472 			val = tr32(TG3_CPMU_STATUS);
16473 
16474 		if (tg3_asic_rev(tp) == ASIC_REV_5717)
16475 			tp->pci_fn = (val & TG3_CPMU_STATUS_FMSK_5717) ? 1 : 0;
16476 		else
16477 			tp->pci_fn = (val & TG3_CPMU_STATUS_FMSK_5719) >>
16478 				     TG3_CPMU_STATUS_FSHFT_5719;
16479 	}
16480 
16481 	if (tg3_flag(tp, FLUSH_POSTED_WRITES)) {
16482 		tp->write32_tx_mbox = tg3_write_flush_reg32;
16483 		tp->write32_rx_mbox = tg3_write_flush_reg32;
16484 	}
16485 
16486 	/* Get eeprom hw config before calling tg3_set_power_state().
16487 	 * In particular, the TG3_FLAG_IS_NIC flag must be
16488 	 * determined before calling tg3_set_power_state() so that
16489 	 * we know whether or not to switch out of Vaux power.
16490 	 * When the flag is set, it means that GPIO1 is used for eeprom
16491 	 * write protect and also implies that it is a LOM where GPIOs
16492 	 * are not used to switch power.
16493 	 */
16494 	tg3_get_eeprom_hw_cfg(tp);
16495 
16496 	if (tg3_flag(tp, FW_TSO) && tg3_flag(tp, ENABLE_ASF)) {
16497 		tg3_flag_clear(tp, TSO_CAPABLE);
16498 		tg3_flag_clear(tp, TSO_BUG);
16499 		tp->fw_needed = NULL;
16500 	}
16501 
16502 	if (tg3_flag(tp, ENABLE_APE)) {
16503 		/* Allow reads and writes to the
16504 		 * APE register and memory space.
16505 		 */
16506 		pci_state_reg |= PCISTATE_ALLOW_APE_CTLSPC_WR |
16507 				 PCISTATE_ALLOW_APE_SHMEM_WR |
16508 				 PCISTATE_ALLOW_APE_PSPACE_WR;
16509 		pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE,
16510 				       pci_state_reg);
16511 
16512 		tg3_ape_lock_init(tp);
16513 	}
16514 
16515 	/* Set up tp->grc_local_ctrl before calling
16516 	 * tg3_pwrsrc_switch_to_vmain().  GPIO1 driven high
16517 	 * will bring 5700's external PHY out of reset.
16518 	 * It is also used as eeprom write protect on LOMs.
16519 	 */
16520 	tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
16521 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16522 	    tg3_flag(tp, EEPROM_WRITE_PROT))
16523 		tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
16524 				       GRC_LCLCTRL_GPIO_OUTPUT1);
16525 	/* Unused GPIO3 must be driven as output on 5752 because there
16526 	 * are no pull-up resistors on unused GPIO pins.
16527 	 */
16528 	else if (tg3_asic_rev(tp) == ASIC_REV_5752)
16529 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
16530 
16531 	if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
16532 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
16533 	    tg3_flag(tp, 57765_CLASS))
16534 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
16535 
16536 	if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
16537 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
16538 		/* Turn off the debug UART. */
16539 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
16540 		if (tg3_flag(tp, IS_NIC))
16541 			/* Keep VMain power. */
16542 			tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
16543 					      GRC_LCLCTRL_GPIO_OUTPUT0;
16544 	}
16545 
16546 	if (tg3_asic_rev(tp) == ASIC_REV_5762)
16547 		tp->grc_local_ctrl |=
16548 			tr32(GRC_LOCAL_CTRL) & GRC_LCLCTRL_GPIO_UART_SEL;
16549 
16550 	/* Switch out of Vaux if it is a NIC */
16551 	tg3_pwrsrc_switch_to_vmain(tp);
16552 
16553 	/* Derive initial jumbo mode from MTU assigned in
16554 	 * ether_setup() via the alloc_etherdev() call
16555 	 */
16556 	if (tp->dev->mtu > ETH_DATA_LEN && !tg3_flag(tp, 5780_CLASS))
16557 		tg3_flag_set(tp, JUMBO_RING_ENABLE);
16558 
16559 	/* Determine WakeOnLan speed to use. */
16560 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16561 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
16562 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0 ||
16563 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B2) {
16564 		tg3_flag_clear(tp, WOL_SPEED_100MB);
16565 	} else {
16566 		tg3_flag_set(tp, WOL_SPEED_100MB);
16567 	}
16568 
16569 	if (tg3_asic_rev(tp) == ASIC_REV_5906)
16570 		tp->phy_flags |= TG3_PHYFLG_IS_FET;
16571 
16572 	/* A few boards don't want Ethernet@WireSpeed phy feature */
16573 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16574 	    (tg3_asic_rev(tp) == ASIC_REV_5705 &&
16575 	     (tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) &&
16576 	     (tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A1)) ||
16577 	    (tp->phy_flags & TG3_PHYFLG_IS_FET) ||
16578 	    (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
16579 		tp->phy_flags |= TG3_PHYFLG_NO_ETH_WIRE_SPEED;
16580 
16581 	if (tg3_chip_rev(tp) == CHIPREV_5703_AX ||
16582 	    tg3_chip_rev(tp) == CHIPREV_5704_AX)
16583 		tp->phy_flags |= TG3_PHYFLG_ADC_BUG;
16584 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0)
16585 		tp->phy_flags |= TG3_PHYFLG_5704_A0_BUG;
16586 
16587 	if (tg3_flag(tp, 5705_PLUS) &&
16588 	    !(tp->phy_flags & TG3_PHYFLG_IS_FET) &&
16589 	    tg3_asic_rev(tp) != ASIC_REV_5785 &&
16590 	    tg3_asic_rev(tp) != ASIC_REV_57780 &&
16591 	    !tg3_flag(tp, 57765_PLUS)) {
16592 		if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
16593 		    tg3_asic_rev(tp) == ASIC_REV_5787 ||
16594 		    tg3_asic_rev(tp) == ASIC_REV_5784 ||
16595 		    tg3_asic_rev(tp) == ASIC_REV_5761) {
16596 			if (tp->pdev->device != PCI_DEVICE_ID_TIGON3_5756 &&
16597 			    tp->pdev->device != PCI_DEVICE_ID_TIGON3_5722)
16598 				tp->phy_flags |= TG3_PHYFLG_JITTER_BUG;
16599 			if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5755M)
16600 				tp->phy_flags |= TG3_PHYFLG_ADJUST_TRIM;
16601 		} else
16602 			tp->phy_flags |= TG3_PHYFLG_BER_BUG;
16603 	}
16604 
16605 	if (tg3_asic_rev(tp) == ASIC_REV_5784 &&
16606 	    tg3_chip_rev(tp) != CHIPREV_5784_AX) {
16607 		tp->phy_otp = tg3_read_otp_phycfg(tp);
16608 		if (tp->phy_otp == 0)
16609 			tp->phy_otp = TG3_OTP_DEFAULT;
16610 	}
16611 
16612 	if (tg3_flag(tp, CPMU_PRESENT))
16613 		tp->mi_mode = MAC_MI_MODE_500KHZ_CONST;
16614 	else
16615 		tp->mi_mode = MAC_MI_MODE_BASE;
16616 
16617 	tp->coalesce_mode = 0;
16618 	if (tg3_chip_rev(tp) != CHIPREV_5700_AX &&
16619 	    tg3_chip_rev(tp) != CHIPREV_5700_BX)
16620 		tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
16621 
16622 	/* Set these bits to enable statistics workaround. */
16623 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16624 	    tg3_asic_rev(tp) == ASIC_REV_5762 ||
16625 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
16626 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5720_A0) {
16627 		tp->coalesce_mode |= HOSTCC_MODE_ATTN;
16628 		tp->grc_mode |= GRC_MODE_IRQ_ON_FLOW_ATTN;
16629 	}
16630 
16631 	if (tg3_asic_rev(tp) == ASIC_REV_5785 ||
16632 	    tg3_asic_rev(tp) == ASIC_REV_57780)
16633 		tg3_flag_set(tp, USE_PHYLIB);
16634 
16635 	err = tg3_mdio_init(tp);
16636 	if (err)
16637 		return err;
16638 
16639 	/* Initialize data/descriptor byte/word swapping. */
16640 	val = tr32(GRC_MODE);
16641 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
16642 	    tg3_asic_rev(tp) == ASIC_REV_5762)
16643 		val &= (GRC_MODE_BYTE_SWAP_B2HRX_DATA |
16644 			GRC_MODE_WORD_SWAP_B2HRX_DATA |
16645 			GRC_MODE_B2HRX_ENABLE |
16646 			GRC_MODE_HTX2B_ENABLE |
16647 			GRC_MODE_HOST_STACKUP);
16648 	else
16649 		val &= GRC_MODE_HOST_STACKUP;
16650 
16651 	tw32(GRC_MODE, val | tp->grc_mode);
16652 
16653 	tg3_switch_clocks(tp);
16654 
16655 	/* Clear this out for sanity. */
16656 	tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
16657 
16658 	/* Clear TG3PCI_REG_BASE_ADDR to prevent hangs. */
16659 	tw32(TG3PCI_REG_BASE_ADDR, 0);
16660 
16661 	pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
16662 			      &pci_state_reg);
16663 	if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
16664 	    !tg3_flag(tp, PCIX_TARGET_HWBUG)) {
16665 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
16666 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0 ||
16667 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B2 ||
16668 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B5) {
16669 			void __iomem *sram_base;
16670 
16671 			/* Write some dummy words into the SRAM status block
16672 			 * area, see if it reads back correctly.  If the return
16673 			 * value is bad, force enable the PCIX workaround.
16674 			 */
16675 			sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
16676 
16677 			writel(0x00000000, sram_base);
16678 			writel(0x00000000, sram_base + 4);
16679 			writel(0xffffffff, sram_base + 4);
16680 			if (readl(sram_base) != 0x00000000)
16681 				tg3_flag_set(tp, PCIX_TARGET_HWBUG);
16682 		}
16683 	}
16684 
16685 	udelay(50);
16686 	tg3_nvram_init(tp);
16687 
16688 	/* If the device has an NVRAM, no need to load patch firmware */
16689 	if (tg3_asic_rev(tp) == ASIC_REV_57766 &&
16690 	    !tg3_flag(tp, NO_NVRAM))
16691 		tp->fw_needed = NULL;
16692 
16693 	grc_misc_cfg = tr32(GRC_MISC_CFG);
16694 	grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
16695 
16696 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
16697 	    (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
16698 	     grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
16699 		tg3_flag_set(tp, IS_5788);
16700 
16701 	if (!tg3_flag(tp, IS_5788) &&
16702 	    tg3_asic_rev(tp) != ASIC_REV_5700)
16703 		tg3_flag_set(tp, TAGGED_STATUS);
16704 	if (tg3_flag(tp, TAGGED_STATUS)) {
16705 		tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
16706 				      HOSTCC_MODE_CLRTICK_TXBD);
16707 
16708 		tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
16709 		pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16710 				       tp->misc_host_ctrl);
16711 	}
16712 
16713 	/* Preserve the APE MAC_MODE bits */
16714 	if (tg3_flag(tp, ENABLE_APE))
16715 		tp->mac_mode = MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
16716 	else
16717 		tp->mac_mode = 0;
16718 
16719 	if (tg3_10_100_only_device(tp, ent))
16720 		tp->phy_flags |= TG3_PHYFLG_10_100_ONLY;
16721 
16722 	err = tg3_phy_probe(tp);
16723 	if (err) {
16724 		dev_err(&tp->pdev->dev, "phy probe failed, err %d\n", err);
16725 		/* ... but do not return immediately ... */
16726 		tg3_mdio_fini(tp);
16727 	}
16728 
16729 	tg3_read_vpd(tp);
16730 	tg3_read_fw_ver(tp);
16731 
16732 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
16733 		tp->phy_flags &= ~TG3_PHYFLG_USE_MI_INTERRUPT;
16734 	} else {
16735 		if (tg3_asic_rev(tp) == ASIC_REV_5700)
16736 			tp->phy_flags |= TG3_PHYFLG_USE_MI_INTERRUPT;
16737 		else
16738 			tp->phy_flags &= ~TG3_PHYFLG_USE_MI_INTERRUPT;
16739 	}
16740 
16741 	/* 5700 {AX,BX} chips have a broken status block link
16742 	 * change bit implementation, so we must use the
16743 	 * status register in those cases.
16744 	 */
16745 	if (tg3_asic_rev(tp) == ASIC_REV_5700)
16746 		tg3_flag_set(tp, USE_LINKCHG_REG);
16747 	else
16748 		tg3_flag_clear(tp, USE_LINKCHG_REG);
16749 
16750 	/* The led_ctrl is set during tg3_phy_probe, here we might
16751 	 * have to force the link status polling mechanism based
16752 	 * upon subsystem IDs.
16753 	 */
16754 	if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
16755 	    tg3_asic_rev(tp) == ASIC_REV_5701 &&
16756 	    !(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
16757 		tp->phy_flags |= TG3_PHYFLG_USE_MI_INTERRUPT;
16758 		tg3_flag_set(tp, USE_LINKCHG_REG);
16759 	}
16760 
16761 	/* For all SERDES we poll the MAC status register. */
16762 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
16763 		tg3_flag_set(tp, POLL_SERDES);
16764 	else
16765 		tg3_flag_clear(tp, POLL_SERDES);
16766 
16767 	if (tg3_flag(tp, ENABLE_APE) && tg3_flag(tp, ENABLE_ASF))
16768 		tg3_flag_set(tp, POLL_CPMU_LINK);
16769 
16770 	tp->rx_offset = NET_SKB_PAD + NET_IP_ALIGN;
16771 	tp->rx_copy_thresh = TG3_RX_COPY_THRESHOLD;
16772 	if (tg3_asic_rev(tp) == ASIC_REV_5701 &&
16773 	    tg3_flag(tp, PCIX_MODE)) {
16774 		tp->rx_offset = NET_SKB_PAD;
16775 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
16776 		tp->rx_copy_thresh = ~(u16)0;
16777 #endif
16778 	}
16779 
16780 	tp->rx_std_ring_mask = TG3_RX_STD_RING_SIZE(tp) - 1;
16781 	tp->rx_jmb_ring_mask = TG3_RX_JMB_RING_SIZE(tp) - 1;
16782 	tp->rx_ret_ring_mask = tg3_rx_ret_ring_size(tp) - 1;
16783 
16784 	tp->rx_std_max_post = tp->rx_std_ring_mask + 1;
16785 
16786 	/* Increment the rx prod index on the rx std ring by at most
16787 	 * 8 for these chips to workaround hw errata.
16788 	 */
16789 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
16790 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
16791 	    tg3_asic_rev(tp) == ASIC_REV_5755)
16792 		tp->rx_std_max_post = 8;
16793 
16794 	if (tg3_flag(tp, ASPM_WORKAROUND))
16795 		tp->pwrmgmt_thresh = tr32(PCIE_PWR_MGMT_THRESH) &
16796 				     PCIE_PWR_MGMT_L1_THRESH_MSK;
16797 
16798 	return err;
16799 }
16800 
16801 #ifdef CONFIG_SPARC
16802 static int tg3_get_macaddr_sparc(struct tg3 *tp)
16803 {
16804 	struct net_device *dev = tp->dev;
16805 	struct pci_dev *pdev = tp->pdev;
16806 	struct device_node *dp = pci_device_to_OF_node(pdev);
16807 	const unsigned char *addr;
16808 	int len;
16809 
16810 	addr = of_get_property(dp, "local-mac-address", &len);
16811 	if (addr && len == ETH_ALEN) {
16812 		memcpy(dev->dev_addr, addr, ETH_ALEN);
16813 		return 0;
16814 	}
16815 	return -ENODEV;
16816 }
16817 
16818 static int tg3_get_default_macaddr_sparc(struct tg3 *tp)
16819 {
16820 	struct net_device *dev = tp->dev;
16821 
16822 	memcpy(dev->dev_addr, idprom->id_ethaddr, ETH_ALEN);
16823 	return 0;
16824 }
16825 #endif
16826 
16827 static int tg3_get_device_address(struct tg3 *tp)
16828 {
16829 	struct net_device *dev = tp->dev;
16830 	u32 hi, lo, mac_offset;
16831 	int addr_ok = 0;
16832 	int err;
16833 
16834 #ifdef CONFIG_SPARC
16835 	if (!tg3_get_macaddr_sparc(tp))
16836 		return 0;
16837 #endif
16838 
16839 	if (tg3_flag(tp, IS_SSB_CORE)) {
16840 		err = ssb_gige_get_macaddr(tp->pdev, &dev->dev_addr[0]);
16841 		if (!err && is_valid_ether_addr(&dev->dev_addr[0]))
16842 			return 0;
16843 	}
16844 
16845 	mac_offset = 0x7c;
16846 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16847 	    tg3_flag(tp, 5780_CLASS)) {
16848 		if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
16849 			mac_offset = 0xcc;
16850 		if (tg3_nvram_lock(tp))
16851 			tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
16852 		else
16853 			tg3_nvram_unlock(tp);
16854 	} else if (tg3_flag(tp, 5717_PLUS)) {
16855 		if (tp->pci_fn & 1)
16856 			mac_offset = 0xcc;
16857 		if (tp->pci_fn > 1)
16858 			mac_offset += 0x18c;
16859 	} else if (tg3_asic_rev(tp) == ASIC_REV_5906)
16860 		mac_offset = 0x10;
16861 
16862 	/* First try to get it from MAC address mailbox. */
16863 	tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
16864 	if ((hi >> 16) == 0x484b) {
16865 		dev->dev_addr[0] = (hi >>  8) & 0xff;
16866 		dev->dev_addr[1] = (hi >>  0) & 0xff;
16867 
16868 		tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
16869 		dev->dev_addr[2] = (lo >> 24) & 0xff;
16870 		dev->dev_addr[3] = (lo >> 16) & 0xff;
16871 		dev->dev_addr[4] = (lo >>  8) & 0xff;
16872 		dev->dev_addr[5] = (lo >>  0) & 0xff;
16873 
16874 		/* Some old bootcode may report a 0 MAC address in SRAM */
16875 		addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
16876 	}
16877 	if (!addr_ok) {
16878 		/* Next, try NVRAM. */
16879 		if (!tg3_flag(tp, NO_NVRAM) &&
16880 		    !tg3_nvram_read_be32(tp, mac_offset + 0, &hi) &&
16881 		    !tg3_nvram_read_be32(tp, mac_offset + 4, &lo)) {
16882 			memcpy(&dev->dev_addr[0], ((char *)&hi) + 2, 2);
16883 			memcpy(&dev->dev_addr[2], (char *)&lo, sizeof(lo));
16884 		}
16885 		/* Finally just fetch it out of the MAC control regs. */
16886 		else {
16887 			hi = tr32(MAC_ADDR_0_HIGH);
16888 			lo = tr32(MAC_ADDR_0_LOW);
16889 
16890 			dev->dev_addr[5] = lo & 0xff;
16891 			dev->dev_addr[4] = (lo >> 8) & 0xff;
16892 			dev->dev_addr[3] = (lo >> 16) & 0xff;
16893 			dev->dev_addr[2] = (lo >> 24) & 0xff;
16894 			dev->dev_addr[1] = hi & 0xff;
16895 			dev->dev_addr[0] = (hi >> 8) & 0xff;
16896 		}
16897 	}
16898 
16899 	if (!is_valid_ether_addr(&dev->dev_addr[0])) {
16900 #ifdef CONFIG_SPARC
16901 		if (!tg3_get_default_macaddr_sparc(tp))
16902 			return 0;
16903 #endif
16904 		return -EINVAL;
16905 	}
16906 	return 0;
16907 }
16908 
16909 #define BOUNDARY_SINGLE_CACHELINE	1
16910 #define BOUNDARY_MULTI_CACHELINE	2
16911 
16912 static u32 tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
16913 {
16914 	int cacheline_size;
16915 	u8 byte;
16916 	int goal;
16917 
16918 	pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
16919 	if (byte == 0)
16920 		cacheline_size = 1024;
16921 	else
16922 		cacheline_size = (int) byte * 4;
16923 
16924 	/* On 5703 and later chips, the boundary bits have no
16925 	 * effect.
16926 	 */
16927 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
16928 	    tg3_asic_rev(tp) != ASIC_REV_5701 &&
16929 	    !tg3_flag(tp, PCI_EXPRESS))
16930 		goto out;
16931 
16932 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
16933 	goal = BOUNDARY_MULTI_CACHELINE;
16934 #else
16935 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
16936 	goal = BOUNDARY_SINGLE_CACHELINE;
16937 #else
16938 	goal = 0;
16939 #endif
16940 #endif
16941 
16942 	if (tg3_flag(tp, 57765_PLUS)) {
16943 		val = goal ? 0 : DMA_RWCTRL_DIS_CACHE_ALIGNMENT;
16944 		goto out;
16945 	}
16946 
16947 	if (!goal)
16948 		goto out;
16949 
16950 	/* PCI controllers on most RISC systems tend to disconnect
16951 	 * when a device tries to burst across a cache-line boundary.
16952 	 * Therefore, letting tg3 do so just wastes PCI bandwidth.
16953 	 *
16954 	 * Unfortunately, for PCI-E there are only limited
16955 	 * write-side controls for this, and thus for reads
16956 	 * we will still get the disconnects.  We'll also waste
16957 	 * these PCI cycles for both read and write for chips
16958 	 * other than 5700 and 5701 which do not implement the
16959 	 * boundary bits.
16960 	 */
16961 	if (tg3_flag(tp, PCIX_MODE) && !tg3_flag(tp, PCI_EXPRESS)) {
16962 		switch (cacheline_size) {
16963 		case 16:
16964 		case 32:
16965 		case 64:
16966 		case 128:
16967 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
16968 				val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
16969 					DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
16970 			} else {
16971 				val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
16972 					DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
16973 			}
16974 			break;
16975 
16976 		case 256:
16977 			val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
16978 				DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
16979 			break;
16980 
16981 		default:
16982 			val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
16983 				DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
16984 			break;
16985 		}
16986 	} else if (tg3_flag(tp, PCI_EXPRESS)) {
16987 		switch (cacheline_size) {
16988 		case 16:
16989 		case 32:
16990 		case 64:
16991 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
16992 				val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
16993 				val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
16994 				break;
16995 			}
16996 			/* fallthrough */
16997 		case 128:
16998 		default:
16999 			val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
17000 			val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
17001 			break;
17002 		}
17003 	} else {
17004 		switch (cacheline_size) {
17005 		case 16:
17006 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17007 				val |= (DMA_RWCTRL_READ_BNDRY_16 |
17008 					DMA_RWCTRL_WRITE_BNDRY_16);
17009 				break;
17010 			}
17011 			/* fallthrough */
17012 		case 32:
17013 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17014 				val |= (DMA_RWCTRL_READ_BNDRY_32 |
17015 					DMA_RWCTRL_WRITE_BNDRY_32);
17016 				break;
17017 			}
17018 			/* fallthrough */
17019 		case 64:
17020 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17021 				val |= (DMA_RWCTRL_READ_BNDRY_64 |
17022 					DMA_RWCTRL_WRITE_BNDRY_64);
17023 				break;
17024 			}
17025 			/* fallthrough */
17026 		case 128:
17027 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17028 				val |= (DMA_RWCTRL_READ_BNDRY_128 |
17029 					DMA_RWCTRL_WRITE_BNDRY_128);
17030 				break;
17031 			}
17032 			/* fallthrough */
17033 		case 256:
17034 			val |= (DMA_RWCTRL_READ_BNDRY_256 |
17035 				DMA_RWCTRL_WRITE_BNDRY_256);
17036 			break;
17037 		case 512:
17038 			val |= (DMA_RWCTRL_READ_BNDRY_512 |
17039 				DMA_RWCTRL_WRITE_BNDRY_512);
17040 			break;
17041 		case 1024:
17042 		default:
17043 			val |= (DMA_RWCTRL_READ_BNDRY_1024 |
17044 				DMA_RWCTRL_WRITE_BNDRY_1024);
17045 			break;
17046 		}
17047 	}
17048 
17049 out:
17050 	return val;
17051 }
17052 
17053 static int tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma,
17054 			   int size, bool to_device)
17055 {
17056 	struct tg3_internal_buffer_desc test_desc;
17057 	u32 sram_dma_descs;
17058 	int i, ret;
17059 
17060 	sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
17061 
17062 	tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
17063 	tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
17064 	tw32(RDMAC_STATUS, 0);
17065 	tw32(WDMAC_STATUS, 0);
17066 
17067 	tw32(BUFMGR_MODE, 0);
17068 	tw32(FTQ_RESET, 0);
17069 
17070 	test_desc.addr_hi = ((u64) buf_dma) >> 32;
17071 	test_desc.addr_lo = buf_dma & 0xffffffff;
17072 	test_desc.nic_mbuf = 0x00002100;
17073 	test_desc.len = size;
17074 
17075 	/*
17076 	 * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
17077 	 * the *second* time the tg3 driver was getting loaded after an
17078 	 * initial scan.
17079 	 *
17080 	 * Broadcom tells me:
17081 	 *   ...the DMA engine is connected to the GRC block and a DMA
17082 	 *   reset may affect the GRC block in some unpredictable way...
17083 	 *   The behavior of resets to individual blocks has not been tested.
17084 	 *
17085 	 * Broadcom noted the GRC reset will also reset all sub-components.
17086 	 */
17087 	if (to_device) {
17088 		test_desc.cqid_sqid = (13 << 8) | 2;
17089 
17090 		tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
17091 		udelay(40);
17092 	} else {
17093 		test_desc.cqid_sqid = (16 << 8) | 7;
17094 
17095 		tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
17096 		udelay(40);
17097 	}
17098 	test_desc.flags = 0x00000005;
17099 
17100 	for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
17101 		u32 val;
17102 
17103 		val = *(((u32 *)&test_desc) + i);
17104 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
17105 				       sram_dma_descs + (i * sizeof(u32)));
17106 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
17107 	}
17108 	pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
17109 
17110 	if (to_device)
17111 		tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
17112 	else
17113 		tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
17114 
17115 	ret = -ENODEV;
17116 	for (i = 0; i < 40; i++) {
17117 		u32 val;
17118 
17119 		if (to_device)
17120 			val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
17121 		else
17122 			val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
17123 		if ((val & 0xffff) == sram_dma_descs) {
17124 			ret = 0;
17125 			break;
17126 		}
17127 
17128 		udelay(100);
17129 	}
17130 
17131 	return ret;
17132 }
17133 
17134 #define TEST_BUFFER_SIZE	0x2000
17135 
17136 static DEFINE_PCI_DEVICE_TABLE(tg3_dma_wait_state_chipsets) = {
17137 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
17138 	{ },
17139 };
17140 
17141 static int tg3_test_dma(struct tg3 *tp)
17142 {
17143 	dma_addr_t buf_dma;
17144 	u32 *buf, saved_dma_rwctrl;
17145 	int ret = 0;
17146 
17147 	buf = dma_alloc_coherent(&tp->pdev->dev, TEST_BUFFER_SIZE,
17148 				 &buf_dma, GFP_KERNEL);
17149 	if (!buf) {
17150 		ret = -ENOMEM;
17151 		goto out_nofree;
17152 	}
17153 
17154 	tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
17155 			  (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
17156 
17157 	tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
17158 
17159 	if (tg3_flag(tp, 57765_PLUS))
17160 		goto out;
17161 
17162 	if (tg3_flag(tp, PCI_EXPRESS)) {
17163 		/* DMA read watermark not used on PCIE */
17164 		tp->dma_rwctrl |= 0x00180000;
17165 	} else if (!tg3_flag(tp, PCIX_MODE)) {
17166 		if (tg3_asic_rev(tp) == ASIC_REV_5705 ||
17167 		    tg3_asic_rev(tp) == ASIC_REV_5750)
17168 			tp->dma_rwctrl |= 0x003f0000;
17169 		else
17170 			tp->dma_rwctrl |= 0x003f000f;
17171 	} else {
17172 		if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
17173 		    tg3_asic_rev(tp) == ASIC_REV_5704) {
17174 			u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
17175 			u32 read_water = 0x7;
17176 
17177 			/* If the 5704 is behind the EPB bridge, we can
17178 			 * do the less restrictive ONE_DMA workaround for
17179 			 * better performance.
17180 			 */
17181 			if (tg3_flag(tp, 40BIT_DMA_BUG) &&
17182 			    tg3_asic_rev(tp) == ASIC_REV_5704)
17183 				tp->dma_rwctrl |= 0x8000;
17184 			else if (ccval == 0x6 || ccval == 0x7)
17185 				tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
17186 
17187 			if (tg3_asic_rev(tp) == ASIC_REV_5703)
17188 				read_water = 4;
17189 			/* Set bit 23 to enable PCIX hw bug fix */
17190 			tp->dma_rwctrl |=
17191 				(read_water << DMA_RWCTRL_READ_WATER_SHIFT) |
17192 				(0x3 << DMA_RWCTRL_WRITE_WATER_SHIFT) |
17193 				(1 << 23);
17194 		} else if (tg3_asic_rev(tp) == ASIC_REV_5780) {
17195 			/* 5780 always in PCIX mode */
17196 			tp->dma_rwctrl |= 0x00144000;
17197 		} else if (tg3_asic_rev(tp) == ASIC_REV_5714) {
17198 			/* 5714 always in PCIX mode */
17199 			tp->dma_rwctrl |= 0x00148000;
17200 		} else {
17201 			tp->dma_rwctrl |= 0x001b000f;
17202 		}
17203 	}
17204 	if (tg3_flag(tp, ONE_DMA_AT_ONCE))
17205 		tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
17206 
17207 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
17208 	    tg3_asic_rev(tp) == ASIC_REV_5704)
17209 		tp->dma_rwctrl &= 0xfffffff0;
17210 
17211 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
17212 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
17213 		/* Remove this if it causes problems for some boards. */
17214 		tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
17215 
17216 		/* On 5700/5701 chips, we need to set this bit.
17217 		 * Otherwise the chip will issue cacheline transactions
17218 		 * to streamable DMA memory with not all the byte
17219 		 * enables turned on.  This is an error on several
17220 		 * RISC PCI controllers, in particular sparc64.
17221 		 *
17222 		 * On 5703/5704 chips, this bit has been reassigned
17223 		 * a different meaning.  In particular, it is used
17224 		 * on those chips to enable a PCI-X workaround.
17225 		 */
17226 		tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
17227 	}
17228 
17229 	tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17230 
17231 
17232 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
17233 	    tg3_asic_rev(tp) != ASIC_REV_5701)
17234 		goto out;
17235 
17236 	/* It is best to perform DMA test with maximum write burst size
17237 	 * to expose the 5700/5701 write DMA bug.
17238 	 */
17239 	saved_dma_rwctrl = tp->dma_rwctrl;
17240 	tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17241 	tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17242 
17243 	while (1) {
17244 		u32 *p = buf, i;
17245 
17246 		for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
17247 			p[i] = i;
17248 
17249 		/* Send the buffer to the chip. */
17250 		ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, true);
17251 		if (ret) {
17252 			dev_err(&tp->pdev->dev,
17253 				"%s: Buffer write failed. err = %d\n",
17254 				__func__, ret);
17255 			break;
17256 		}
17257 
17258 		/* Now read it back. */
17259 		ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, false);
17260 		if (ret) {
17261 			dev_err(&tp->pdev->dev, "%s: Buffer read failed. "
17262 				"err = %d\n", __func__, ret);
17263 			break;
17264 		}
17265 
17266 		/* Verify it. */
17267 		for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
17268 			if (p[i] == i)
17269 				continue;
17270 
17271 			if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
17272 			    DMA_RWCTRL_WRITE_BNDRY_16) {
17273 				tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17274 				tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
17275 				tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17276 				break;
17277 			} else {
17278 				dev_err(&tp->pdev->dev,
17279 					"%s: Buffer corrupted on read back! "
17280 					"(%d != %d)\n", __func__, p[i], i);
17281 				ret = -ENODEV;
17282 				goto out;
17283 			}
17284 		}
17285 
17286 		if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
17287 			/* Success. */
17288 			ret = 0;
17289 			break;
17290 		}
17291 	}
17292 	if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
17293 	    DMA_RWCTRL_WRITE_BNDRY_16) {
17294 		/* DMA test passed without adjusting DMA boundary,
17295 		 * now look for chipsets that are known to expose the
17296 		 * DMA bug without failing the test.
17297 		 */
17298 		if (pci_dev_present(tg3_dma_wait_state_chipsets)) {
17299 			tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17300 			tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
17301 		} else {
17302 			/* Safe to use the calculated DMA boundary. */
17303 			tp->dma_rwctrl = saved_dma_rwctrl;
17304 		}
17305 
17306 		tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17307 	}
17308 
17309 out:
17310 	dma_free_coherent(&tp->pdev->dev, TEST_BUFFER_SIZE, buf, buf_dma);
17311 out_nofree:
17312 	return ret;
17313 }
17314 
17315 static void tg3_init_bufmgr_config(struct tg3 *tp)
17316 {
17317 	if (tg3_flag(tp, 57765_PLUS)) {
17318 		tp->bufmgr_config.mbuf_read_dma_low_water =
17319 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17320 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17321 			DEFAULT_MB_MACRX_LOW_WATER_57765;
17322 		tp->bufmgr_config.mbuf_high_water =
17323 			DEFAULT_MB_HIGH_WATER_57765;
17324 
17325 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17326 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17327 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17328 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO_57765;
17329 		tp->bufmgr_config.mbuf_high_water_jumbo =
17330 			DEFAULT_MB_HIGH_WATER_JUMBO_57765;
17331 	} else if (tg3_flag(tp, 5705_PLUS)) {
17332 		tp->bufmgr_config.mbuf_read_dma_low_water =
17333 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17334 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17335 			DEFAULT_MB_MACRX_LOW_WATER_5705;
17336 		tp->bufmgr_config.mbuf_high_water =
17337 			DEFAULT_MB_HIGH_WATER_5705;
17338 		if (tg3_asic_rev(tp) == ASIC_REV_5906) {
17339 			tp->bufmgr_config.mbuf_mac_rx_low_water =
17340 				DEFAULT_MB_MACRX_LOW_WATER_5906;
17341 			tp->bufmgr_config.mbuf_high_water =
17342 				DEFAULT_MB_HIGH_WATER_5906;
17343 		}
17344 
17345 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17346 			DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
17347 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17348 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
17349 		tp->bufmgr_config.mbuf_high_water_jumbo =
17350 			DEFAULT_MB_HIGH_WATER_JUMBO_5780;
17351 	} else {
17352 		tp->bufmgr_config.mbuf_read_dma_low_water =
17353 			DEFAULT_MB_RDMA_LOW_WATER;
17354 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17355 			DEFAULT_MB_MACRX_LOW_WATER;
17356 		tp->bufmgr_config.mbuf_high_water =
17357 			DEFAULT_MB_HIGH_WATER;
17358 
17359 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17360 			DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
17361 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17362 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
17363 		tp->bufmgr_config.mbuf_high_water_jumbo =
17364 			DEFAULT_MB_HIGH_WATER_JUMBO;
17365 	}
17366 
17367 	tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
17368 	tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
17369 }
17370 
17371 static char *tg3_phy_string(struct tg3 *tp)
17372 {
17373 	switch (tp->phy_id & TG3_PHY_ID_MASK) {
17374 	case TG3_PHY_ID_BCM5400:	return "5400";
17375 	case TG3_PHY_ID_BCM5401:	return "5401";
17376 	case TG3_PHY_ID_BCM5411:	return "5411";
17377 	case TG3_PHY_ID_BCM5701:	return "5701";
17378 	case TG3_PHY_ID_BCM5703:	return "5703";
17379 	case TG3_PHY_ID_BCM5704:	return "5704";
17380 	case TG3_PHY_ID_BCM5705:	return "5705";
17381 	case TG3_PHY_ID_BCM5750:	return "5750";
17382 	case TG3_PHY_ID_BCM5752:	return "5752";
17383 	case TG3_PHY_ID_BCM5714:	return "5714";
17384 	case TG3_PHY_ID_BCM5780:	return "5780";
17385 	case TG3_PHY_ID_BCM5755:	return "5755";
17386 	case TG3_PHY_ID_BCM5787:	return "5787";
17387 	case TG3_PHY_ID_BCM5784:	return "5784";
17388 	case TG3_PHY_ID_BCM5756:	return "5722/5756";
17389 	case TG3_PHY_ID_BCM5906:	return "5906";
17390 	case TG3_PHY_ID_BCM5761:	return "5761";
17391 	case TG3_PHY_ID_BCM5718C:	return "5718C";
17392 	case TG3_PHY_ID_BCM5718S:	return "5718S";
17393 	case TG3_PHY_ID_BCM57765:	return "57765";
17394 	case TG3_PHY_ID_BCM5719C:	return "5719C";
17395 	case TG3_PHY_ID_BCM5720C:	return "5720C";
17396 	case TG3_PHY_ID_BCM5762:	return "5762C";
17397 	case TG3_PHY_ID_BCM8002:	return "8002/serdes";
17398 	case 0:			return "serdes";
17399 	default:		return "unknown";
17400 	}
17401 }
17402 
17403 static char *tg3_bus_string(struct tg3 *tp, char *str)
17404 {
17405 	if (tg3_flag(tp, PCI_EXPRESS)) {
17406 		strcpy(str, "PCI Express");
17407 		return str;
17408 	} else if (tg3_flag(tp, PCIX_MODE)) {
17409 		u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
17410 
17411 		strcpy(str, "PCIX:");
17412 
17413 		if ((clock_ctrl == 7) ||
17414 		    ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
17415 		     GRC_MISC_CFG_BOARD_ID_5704CIOBE))
17416 			strcat(str, "133MHz");
17417 		else if (clock_ctrl == 0)
17418 			strcat(str, "33MHz");
17419 		else if (clock_ctrl == 2)
17420 			strcat(str, "50MHz");
17421 		else if (clock_ctrl == 4)
17422 			strcat(str, "66MHz");
17423 		else if (clock_ctrl == 6)
17424 			strcat(str, "100MHz");
17425 	} else {
17426 		strcpy(str, "PCI:");
17427 		if (tg3_flag(tp, PCI_HIGH_SPEED))
17428 			strcat(str, "66MHz");
17429 		else
17430 			strcat(str, "33MHz");
17431 	}
17432 	if (tg3_flag(tp, PCI_32BIT))
17433 		strcat(str, ":32-bit");
17434 	else
17435 		strcat(str, ":64-bit");
17436 	return str;
17437 }
17438 
17439 static void tg3_init_coal(struct tg3 *tp)
17440 {
17441 	struct ethtool_coalesce *ec = &tp->coal;
17442 
17443 	memset(ec, 0, sizeof(*ec));
17444 	ec->cmd = ETHTOOL_GCOALESCE;
17445 	ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
17446 	ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
17447 	ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
17448 	ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
17449 	ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
17450 	ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
17451 	ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
17452 	ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
17453 	ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
17454 
17455 	if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
17456 				 HOSTCC_MODE_CLRTICK_TXBD)) {
17457 		ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
17458 		ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
17459 		ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
17460 		ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
17461 	}
17462 
17463 	if (tg3_flag(tp, 5705_PLUS)) {
17464 		ec->rx_coalesce_usecs_irq = 0;
17465 		ec->tx_coalesce_usecs_irq = 0;
17466 		ec->stats_block_coalesce_usecs = 0;
17467 	}
17468 }
17469 
17470 static int tg3_init_one(struct pci_dev *pdev,
17471 				  const struct pci_device_id *ent)
17472 {
17473 	struct net_device *dev;
17474 	struct tg3 *tp;
17475 	int i, err;
17476 	u32 sndmbx, rcvmbx, intmbx;
17477 	char str[40];
17478 	u64 dma_mask, persist_dma_mask;
17479 	netdev_features_t features = 0;
17480 
17481 	printk_once(KERN_INFO "%s\n", version);
17482 
17483 	err = pci_enable_device(pdev);
17484 	if (err) {
17485 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
17486 		return err;
17487 	}
17488 
17489 	err = pci_request_regions(pdev, DRV_MODULE_NAME);
17490 	if (err) {
17491 		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
17492 		goto err_out_disable_pdev;
17493 	}
17494 
17495 	pci_set_master(pdev);
17496 
17497 	dev = alloc_etherdev_mq(sizeof(*tp), TG3_IRQ_MAX_VECS);
17498 	if (!dev) {
17499 		err = -ENOMEM;
17500 		goto err_out_free_res;
17501 	}
17502 
17503 	SET_NETDEV_DEV(dev, &pdev->dev);
17504 
17505 	tp = netdev_priv(dev);
17506 	tp->pdev = pdev;
17507 	tp->dev = dev;
17508 	tp->rx_mode = TG3_DEF_RX_MODE;
17509 	tp->tx_mode = TG3_DEF_TX_MODE;
17510 	tp->irq_sync = 1;
17511 
17512 	if (tg3_debug > 0)
17513 		tp->msg_enable = tg3_debug;
17514 	else
17515 		tp->msg_enable = TG3_DEF_MSG_ENABLE;
17516 
17517 	if (pdev_is_ssb_gige_core(pdev)) {
17518 		tg3_flag_set(tp, IS_SSB_CORE);
17519 		if (ssb_gige_must_flush_posted_writes(pdev))
17520 			tg3_flag_set(tp, FLUSH_POSTED_WRITES);
17521 		if (ssb_gige_one_dma_at_once(pdev))
17522 			tg3_flag_set(tp, ONE_DMA_AT_ONCE);
17523 		if (ssb_gige_have_roboswitch(pdev)) {
17524 			tg3_flag_set(tp, USE_PHYLIB);
17525 			tg3_flag_set(tp, ROBOSWITCH);
17526 		}
17527 		if (ssb_gige_is_rgmii(pdev))
17528 			tg3_flag_set(tp, RGMII_MODE);
17529 	}
17530 
17531 	/* The word/byte swap controls here control register access byte
17532 	 * swapping.  DMA data byte swapping is controlled in the GRC_MODE
17533 	 * setting below.
17534 	 */
17535 	tp->misc_host_ctrl =
17536 		MISC_HOST_CTRL_MASK_PCI_INT |
17537 		MISC_HOST_CTRL_WORD_SWAP |
17538 		MISC_HOST_CTRL_INDIR_ACCESS |
17539 		MISC_HOST_CTRL_PCISTATE_RW;
17540 
17541 	/* The NONFRM (non-frame) byte/word swap controls take effect
17542 	 * on descriptor entries, anything which isn't packet data.
17543 	 *
17544 	 * The StrongARM chips on the board (one for tx, one for rx)
17545 	 * are running in big-endian mode.
17546 	 */
17547 	tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
17548 			GRC_MODE_WSWAP_NONFRM_DATA);
17549 #ifdef __BIG_ENDIAN
17550 	tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
17551 #endif
17552 	spin_lock_init(&tp->lock);
17553 	spin_lock_init(&tp->indirect_lock);
17554 	INIT_WORK(&tp->reset_task, tg3_reset_task);
17555 
17556 	tp->regs = pci_ioremap_bar(pdev, BAR_0);
17557 	if (!tp->regs) {
17558 		dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
17559 		err = -ENOMEM;
17560 		goto err_out_free_dev;
17561 	}
17562 
17563 	if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
17564 	    tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761E ||
17565 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S ||
17566 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761SE ||
17567 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
17568 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C ||
17569 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718 ||
17570 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5719 ||
17571 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5720 ||
17572 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57767 ||
17573 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57764 ||
17574 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5762 ||
17575 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725 ||
17576 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5727 ||
17577 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57787) {
17578 		tg3_flag_set(tp, ENABLE_APE);
17579 		tp->aperegs = pci_ioremap_bar(pdev, BAR_2);
17580 		if (!tp->aperegs) {
17581 			dev_err(&pdev->dev,
17582 				"Cannot map APE registers, aborting\n");
17583 			err = -ENOMEM;
17584 			goto err_out_iounmap;
17585 		}
17586 	}
17587 
17588 	tp->rx_pending = TG3_DEF_RX_RING_PENDING;
17589 	tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
17590 
17591 	dev->ethtool_ops = &tg3_ethtool_ops;
17592 	dev->watchdog_timeo = TG3_TX_TIMEOUT;
17593 	dev->netdev_ops = &tg3_netdev_ops;
17594 	dev->irq = pdev->irq;
17595 
17596 	err = tg3_get_invariants(tp, ent);
17597 	if (err) {
17598 		dev_err(&pdev->dev,
17599 			"Problem fetching invariants of chip, aborting\n");
17600 		goto err_out_apeunmap;
17601 	}
17602 
17603 	/* The EPB bridge inside 5714, 5715, and 5780 and any
17604 	 * device behind the EPB cannot support DMA addresses > 40-bit.
17605 	 * On 64-bit systems with IOMMU, use 40-bit dma_mask.
17606 	 * On 64-bit systems without IOMMU, use 64-bit dma_mask and
17607 	 * do DMA address check in tg3_start_xmit().
17608 	 */
17609 	if (tg3_flag(tp, IS_5788))
17610 		persist_dma_mask = dma_mask = DMA_BIT_MASK(32);
17611 	else if (tg3_flag(tp, 40BIT_DMA_BUG)) {
17612 		persist_dma_mask = dma_mask = DMA_BIT_MASK(40);
17613 #ifdef CONFIG_HIGHMEM
17614 		dma_mask = DMA_BIT_MASK(64);
17615 #endif
17616 	} else
17617 		persist_dma_mask = dma_mask = DMA_BIT_MASK(64);
17618 
17619 	/* Configure DMA attributes. */
17620 	if (dma_mask > DMA_BIT_MASK(32)) {
17621 		err = pci_set_dma_mask(pdev, dma_mask);
17622 		if (!err) {
17623 			features |= NETIF_F_HIGHDMA;
17624 			err = pci_set_consistent_dma_mask(pdev,
17625 							  persist_dma_mask);
17626 			if (err < 0) {
17627 				dev_err(&pdev->dev, "Unable to obtain 64 bit "
17628 					"DMA for consistent allocations\n");
17629 				goto err_out_apeunmap;
17630 			}
17631 		}
17632 	}
17633 	if (err || dma_mask == DMA_BIT_MASK(32)) {
17634 		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
17635 		if (err) {
17636 			dev_err(&pdev->dev,
17637 				"No usable DMA configuration, aborting\n");
17638 			goto err_out_apeunmap;
17639 		}
17640 	}
17641 
17642 	tg3_init_bufmgr_config(tp);
17643 
17644 	/* 5700 B0 chips do not support checksumming correctly due
17645 	 * to hardware bugs.
17646 	 */
17647 	if (tg3_chip_rev_id(tp) != CHIPREV_ID_5700_B0) {
17648 		features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
17649 
17650 		if (tg3_flag(tp, 5755_PLUS))
17651 			features |= NETIF_F_IPV6_CSUM;
17652 	}
17653 
17654 	/* TSO is on by default on chips that support hardware TSO.
17655 	 * Firmware TSO on older chips gives lower performance, so it
17656 	 * is off by default, but can be enabled using ethtool.
17657 	 */
17658 	if ((tg3_flag(tp, HW_TSO_1) ||
17659 	     tg3_flag(tp, HW_TSO_2) ||
17660 	     tg3_flag(tp, HW_TSO_3)) &&
17661 	    (features & NETIF_F_IP_CSUM))
17662 		features |= NETIF_F_TSO;
17663 	if (tg3_flag(tp, HW_TSO_2) || tg3_flag(tp, HW_TSO_3)) {
17664 		if (features & NETIF_F_IPV6_CSUM)
17665 			features |= NETIF_F_TSO6;
17666 		if (tg3_flag(tp, HW_TSO_3) ||
17667 		    tg3_asic_rev(tp) == ASIC_REV_5761 ||
17668 		    (tg3_asic_rev(tp) == ASIC_REV_5784 &&
17669 		     tg3_chip_rev(tp) != CHIPREV_5784_AX) ||
17670 		    tg3_asic_rev(tp) == ASIC_REV_5785 ||
17671 		    tg3_asic_rev(tp) == ASIC_REV_57780)
17672 			features |= NETIF_F_TSO_ECN;
17673 	}
17674 
17675 	dev->features |= features | NETIF_F_HW_VLAN_CTAG_TX |
17676 			 NETIF_F_HW_VLAN_CTAG_RX;
17677 	dev->vlan_features |= features;
17678 
17679 	/*
17680 	 * Add loopback capability only for a subset of devices that support
17681 	 * MAC-LOOPBACK. Eventually this need to be enhanced to allow INT-PHY
17682 	 * loopback for the remaining devices.
17683 	 */
17684 	if (tg3_asic_rev(tp) != ASIC_REV_5780 &&
17685 	    !tg3_flag(tp, CPMU_PRESENT))
17686 		/* Add the loopback capability */
17687 		features |= NETIF_F_LOOPBACK;
17688 
17689 	dev->hw_features |= features;
17690 	dev->priv_flags |= IFF_UNICAST_FLT;
17691 
17692 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A1 &&
17693 	    !tg3_flag(tp, TSO_CAPABLE) &&
17694 	    !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
17695 		tg3_flag_set(tp, MAX_RXPEND_64);
17696 		tp->rx_pending = 63;
17697 	}
17698 
17699 	err = tg3_get_device_address(tp);
17700 	if (err) {
17701 		dev_err(&pdev->dev,
17702 			"Could not obtain valid ethernet address, aborting\n");
17703 		goto err_out_apeunmap;
17704 	}
17705 
17706 	/*
17707 	 * Reset chip in case UNDI or EFI driver did not shutdown
17708 	 * DMA self test will enable WDMAC and we'll see (spurious)
17709 	 * pending DMA on the PCI bus at that point.
17710 	 */
17711 	if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
17712 	    (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
17713 		tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
17714 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
17715 	}
17716 
17717 	err = tg3_test_dma(tp);
17718 	if (err) {
17719 		dev_err(&pdev->dev, "DMA engine test failed, aborting\n");
17720 		goto err_out_apeunmap;
17721 	}
17722 
17723 	intmbx = MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW;
17724 	rcvmbx = MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW;
17725 	sndmbx = MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW;
17726 	for (i = 0; i < tp->irq_max; i++) {
17727 		struct tg3_napi *tnapi = &tp->napi[i];
17728 
17729 		tnapi->tp = tp;
17730 		tnapi->tx_pending = TG3_DEF_TX_RING_PENDING;
17731 
17732 		tnapi->int_mbox = intmbx;
17733 		if (i <= 4)
17734 			intmbx += 0x8;
17735 		else
17736 			intmbx += 0x4;
17737 
17738 		tnapi->consmbox = rcvmbx;
17739 		tnapi->prodmbox = sndmbx;
17740 
17741 		if (i)
17742 			tnapi->coal_now = HOSTCC_MODE_COAL_VEC1_NOW << (i - 1);
17743 		else
17744 			tnapi->coal_now = HOSTCC_MODE_NOW;
17745 
17746 		if (!tg3_flag(tp, SUPPORT_MSIX))
17747 			break;
17748 
17749 		/*
17750 		 * If we support MSIX, we'll be using RSS.  If we're using
17751 		 * RSS, the first vector only handles link interrupts and the
17752 		 * remaining vectors handle rx and tx interrupts.  Reuse the
17753 		 * mailbox values for the next iteration.  The values we setup
17754 		 * above are still useful for the single vectored mode.
17755 		 */
17756 		if (!i)
17757 			continue;
17758 
17759 		rcvmbx += 0x8;
17760 
17761 		if (sndmbx & 0x4)
17762 			sndmbx -= 0x4;
17763 		else
17764 			sndmbx += 0xc;
17765 	}
17766 
17767 	tg3_init_coal(tp);
17768 
17769 	pci_set_drvdata(pdev, dev);
17770 
17771 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
17772 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
17773 	    tg3_asic_rev(tp) == ASIC_REV_5762)
17774 		tg3_flag_set(tp, PTP_CAPABLE);
17775 
17776 	tg3_timer_init(tp);
17777 
17778 	tg3_carrier_off(tp);
17779 
17780 	err = register_netdev(dev);
17781 	if (err) {
17782 		dev_err(&pdev->dev, "Cannot register net device, aborting\n");
17783 		goto err_out_apeunmap;
17784 	}
17785 
17786 	netdev_info(dev, "Tigon3 [partno(%s) rev %04x] (%s) MAC address %pM\n",
17787 		    tp->board_part_number,
17788 		    tg3_chip_rev_id(tp),
17789 		    tg3_bus_string(tp, str),
17790 		    dev->dev_addr);
17791 
17792 	if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
17793 		struct phy_device *phydev;
17794 		phydev = tp->mdio_bus->phy_map[tp->phy_addr];
17795 		netdev_info(dev,
17796 			    "attached PHY driver [%s] (mii_bus:phy_addr=%s)\n",
17797 			    phydev->drv->name, dev_name(&phydev->dev));
17798 	} else {
17799 		char *ethtype;
17800 
17801 		if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
17802 			ethtype = "10/100Base-TX";
17803 		else if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
17804 			ethtype = "1000Base-SX";
17805 		else
17806 			ethtype = "10/100/1000Base-T";
17807 
17808 		netdev_info(dev, "attached PHY is %s (%s Ethernet) "
17809 			    "(WireSpeed[%d], EEE[%d])\n",
17810 			    tg3_phy_string(tp), ethtype,
17811 			    (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED) == 0,
17812 			    (tp->phy_flags & TG3_PHYFLG_EEE_CAP) != 0);
17813 	}
17814 
17815 	netdev_info(dev, "RXcsums[%d] LinkChgREG[%d] MIirq[%d] ASF[%d] TSOcap[%d]\n",
17816 		    (dev->features & NETIF_F_RXCSUM) != 0,
17817 		    tg3_flag(tp, USE_LINKCHG_REG) != 0,
17818 		    (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT) != 0,
17819 		    tg3_flag(tp, ENABLE_ASF) != 0,
17820 		    tg3_flag(tp, TSO_CAPABLE) != 0);
17821 	netdev_info(dev, "dma_rwctrl[%08x] dma_mask[%d-bit]\n",
17822 		    tp->dma_rwctrl,
17823 		    pdev->dma_mask == DMA_BIT_MASK(32) ? 32 :
17824 		    ((u64)pdev->dma_mask) == DMA_BIT_MASK(40) ? 40 : 64);
17825 
17826 	pci_save_state(pdev);
17827 
17828 	return 0;
17829 
17830 err_out_apeunmap:
17831 	if (tp->aperegs) {
17832 		iounmap(tp->aperegs);
17833 		tp->aperegs = NULL;
17834 	}
17835 
17836 err_out_iounmap:
17837 	if (tp->regs) {
17838 		iounmap(tp->regs);
17839 		tp->regs = NULL;
17840 	}
17841 
17842 err_out_free_dev:
17843 	free_netdev(dev);
17844 
17845 err_out_free_res:
17846 	pci_release_regions(pdev);
17847 
17848 err_out_disable_pdev:
17849 	if (pci_is_enabled(pdev))
17850 		pci_disable_device(pdev);
17851 	return err;
17852 }
17853 
17854 static void tg3_remove_one(struct pci_dev *pdev)
17855 {
17856 	struct net_device *dev = pci_get_drvdata(pdev);
17857 
17858 	if (dev) {
17859 		struct tg3 *tp = netdev_priv(dev);
17860 
17861 		release_firmware(tp->fw);
17862 
17863 		tg3_reset_task_cancel(tp);
17864 
17865 		if (tg3_flag(tp, USE_PHYLIB)) {
17866 			tg3_phy_fini(tp);
17867 			tg3_mdio_fini(tp);
17868 		}
17869 
17870 		unregister_netdev(dev);
17871 		if (tp->aperegs) {
17872 			iounmap(tp->aperegs);
17873 			tp->aperegs = NULL;
17874 		}
17875 		if (tp->regs) {
17876 			iounmap(tp->regs);
17877 			tp->regs = NULL;
17878 		}
17879 		free_netdev(dev);
17880 		pci_release_regions(pdev);
17881 		pci_disable_device(pdev);
17882 	}
17883 }
17884 
17885 #ifdef CONFIG_PM_SLEEP
17886 static int tg3_suspend(struct device *device)
17887 {
17888 	struct pci_dev *pdev = to_pci_dev(device);
17889 	struct net_device *dev = pci_get_drvdata(pdev);
17890 	struct tg3 *tp = netdev_priv(dev);
17891 	int err = 0;
17892 
17893 	rtnl_lock();
17894 
17895 	if (!netif_running(dev))
17896 		goto unlock;
17897 
17898 	tg3_reset_task_cancel(tp);
17899 	tg3_phy_stop(tp);
17900 	tg3_netif_stop(tp);
17901 
17902 	tg3_timer_stop(tp);
17903 
17904 	tg3_full_lock(tp, 1);
17905 	tg3_disable_ints(tp);
17906 	tg3_full_unlock(tp);
17907 
17908 	netif_device_detach(dev);
17909 
17910 	tg3_full_lock(tp, 0);
17911 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
17912 	tg3_flag_clear(tp, INIT_COMPLETE);
17913 	tg3_full_unlock(tp);
17914 
17915 	err = tg3_power_down_prepare(tp);
17916 	if (err) {
17917 		int err2;
17918 
17919 		tg3_full_lock(tp, 0);
17920 
17921 		tg3_flag_set(tp, INIT_COMPLETE);
17922 		err2 = tg3_restart_hw(tp, true);
17923 		if (err2)
17924 			goto out;
17925 
17926 		tg3_timer_start(tp);
17927 
17928 		netif_device_attach(dev);
17929 		tg3_netif_start(tp);
17930 
17931 out:
17932 		tg3_full_unlock(tp);
17933 
17934 		if (!err2)
17935 			tg3_phy_start(tp);
17936 	}
17937 
17938 unlock:
17939 	rtnl_unlock();
17940 	return err;
17941 }
17942 
17943 static int tg3_resume(struct device *device)
17944 {
17945 	struct pci_dev *pdev = to_pci_dev(device);
17946 	struct net_device *dev = pci_get_drvdata(pdev);
17947 	struct tg3 *tp = netdev_priv(dev);
17948 	int err = 0;
17949 
17950 	rtnl_lock();
17951 
17952 	if (!netif_running(dev))
17953 		goto unlock;
17954 
17955 	netif_device_attach(dev);
17956 
17957 	tg3_full_lock(tp, 0);
17958 
17959 	tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
17960 
17961 	tg3_flag_set(tp, INIT_COMPLETE);
17962 	err = tg3_restart_hw(tp,
17963 			     !(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN));
17964 	if (err)
17965 		goto out;
17966 
17967 	tg3_timer_start(tp);
17968 
17969 	tg3_netif_start(tp);
17970 
17971 out:
17972 	tg3_full_unlock(tp);
17973 
17974 	if (!err)
17975 		tg3_phy_start(tp);
17976 
17977 unlock:
17978 	rtnl_unlock();
17979 	return err;
17980 }
17981 #endif /* CONFIG_PM_SLEEP */
17982 
17983 static SIMPLE_DEV_PM_OPS(tg3_pm_ops, tg3_suspend, tg3_resume);
17984 
17985 static void tg3_shutdown(struct pci_dev *pdev)
17986 {
17987 	struct net_device *dev = pci_get_drvdata(pdev);
17988 	struct tg3 *tp = netdev_priv(dev);
17989 
17990 	rtnl_lock();
17991 	netif_device_detach(dev);
17992 
17993 	if (netif_running(dev))
17994 		dev_close(dev);
17995 
17996 	if (system_state == SYSTEM_POWER_OFF)
17997 		tg3_power_down(tp);
17998 
17999 	rtnl_unlock();
18000 }
18001 
18002 /**
18003  * tg3_io_error_detected - called when PCI error is detected
18004  * @pdev: Pointer to PCI device
18005  * @state: The current pci connection state
18006  *
18007  * This function is called after a PCI bus error affecting
18008  * this device has been detected.
18009  */
18010 static pci_ers_result_t tg3_io_error_detected(struct pci_dev *pdev,
18011 					      pci_channel_state_t state)
18012 {
18013 	struct net_device *netdev = pci_get_drvdata(pdev);
18014 	struct tg3 *tp = netdev_priv(netdev);
18015 	pci_ers_result_t err = PCI_ERS_RESULT_NEED_RESET;
18016 
18017 	netdev_info(netdev, "PCI I/O error detected\n");
18018 
18019 	rtnl_lock();
18020 
18021 	/* We probably don't have netdev yet */
18022 	if (!netdev || !netif_running(netdev))
18023 		goto done;
18024 
18025 	tg3_phy_stop(tp);
18026 
18027 	tg3_netif_stop(tp);
18028 
18029 	tg3_timer_stop(tp);
18030 
18031 	/* Want to make sure that the reset task doesn't run */
18032 	tg3_reset_task_cancel(tp);
18033 
18034 	netif_device_detach(netdev);
18035 
18036 	/* Clean up software state, even if MMIO is blocked */
18037 	tg3_full_lock(tp, 0);
18038 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
18039 	tg3_full_unlock(tp);
18040 
18041 done:
18042 	if (state == pci_channel_io_perm_failure) {
18043 		if (netdev) {
18044 			tg3_napi_enable(tp);
18045 			dev_close(netdev);
18046 		}
18047 		err = PCI_ERS_RESULT_DISCONNECT;
18048 	} else {
18049 		pci_disable_device(pdev);
18050 	}
18051 
18052 	rtnl_unlock();
18053 
18054 	return err;
18055 }
18056 
18057 /**
18058  * tg3_io_slot_reset - called after the pci bus has been reset.
18059  * @pdev: Pointer to PCI device
18060  *
18061  * Restart the card from scratch, as if from a cold-boot.
18062  * At this point, the card has exprienced a hard reset,
18063  * followed by fixups by BIOS, and has its config space
18064  * set up identically to what it was at cold boot.
18065  */
18066 static pci_ers_result_t tg3_io_slot_reset(struct pci_dev *pdev)
18067 {
18068 	struct net_device *netdev = pci_get_drvdata(pdev);
18069 	struct tg3 *tp = netdev_priv(netdev);
18070 	pci_ers_result_t rc = PCI_ERS_RESULT_DISCONNECT;
18071 	int err;
18072 
18073 	rtnl_lock();
18074 
18075 	if (pci_enable_device(pdev)) {
18076 		dev_err(&pdev->dev,
18077 			"Cannot re-enable PCI device after reset.\n");
18078 		goto done;
18079 	}
18080 
18081 	pci_set_master(pdev);
18082 	pci_restore_state(pdev);
18083 	pci_save_state(pdev);
18084 
18085 	if (!netdev || !netif_running(netdev)) {
18086 		rc = PCI_ERS_RESULT_RECOVERED;
18087 		goto done;
18088 	}
18089 
18090 	err = tg3_power_up(tp);
18091 	if (err)
18092 		goto done;
18093 
18094 	rc = PCI_ERS_RESULT_RECOVERED;
18095 
18096 done:
18097 	if (rc != PCI_ERS_RESULT_RECOVERED && netdev && netif_running(netdev)) {
18098 		tg3_napi_enable(tp);
18099 		dev_close(netdev);
18100 	}
18101 	rtnl_unlock();
18102 
18103 	return rc;
18104 }
18105 
18106 /**
18107  * tg3_io_resume - called when traffic can start flowing again.
18108  * @pdev: Pointer to PCI device
18109  *
18110  * This callback is called when the error recovery driver tells
18111  * us that its OK to resume normal operation.
18112  */
18113 static void tg3_io_resume(struct pci_dev *pdev)
18114 {
18115 	struct net_device *netdev = pci_get_drvdata(pdev);
18116 	struct tg3 *tp = netdev_priv(netdev);
18117 	int err;
18118 
18119 	rtnl_lock();
18120 
18121 	if (!netif_running(netdev))
18122 		goto done;
18123 
18124 	tg3_full_lock(tp, 0);
18125 	tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
18126 	tg3_flag_set(tp, INIT_COMPLETE);
18127 	err = tg3_restart_hw(tp, true);
18128 	if (err) {
18129 		tg3_full_unlock(tp);
18130 		netdev_err(netdev, "Cannot restart hardware after reset.\n");
18131 		goto done;
18132 	}
18133 
18134 	netif_device_attach(netdev);
18135 
18136 	tg3_timer_start(tp);
18137 
18138 	tg3_netif_start(tp);
18139 
18140 	tg3_full_unlock(tp);
18141 
18142 	tg3_phy_start(tp);
18143 
18144 done:
18145 	rtnl_unlock();
18146 }
18147 
18148 static const struct pci_error_handlers tg3_err_handler = {
18149 	.error_detected	= tg3_io_error_detected,
18150 	.slot_reset	= tg3_io_slot_reset,
18151 	.resume		= tg3_io_resume
18152 };
18153 
18154 static struct pci_driver tg3_driver = {
18155 	.name		= DRV_MODULE_NAME,
18156 	.id_table	= tg3_pci_tbl,
18157 	.probe		= tg3_init_one,
18158 	.remove		= tg3_remove_one,
18159 	.err_handler	= &tg3_err_handler,
18160 	.driver.pm	= &tg3_pm_ops,
18161 	.shutdown	= tg3_shutdown,
18162 };
18163 
18164 module_pci_driver(tg3_driver);
18165