1 /*
2  * tg3.c: Broadcom Tigon3 ethernet driver.
3  *
4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
6  * Copyright (C) 2004 Sun Microsystems Inc.
7  * Copyright (C) 2005-2016 Broadcom Corporation.
8  * Copyright (C) 2016-2017 Broadcom Limited.
9  * Copyright (C) 2018 Broadcom. All Rights Reserved. The term "Broadcom"
10  * refers to Broadcom Inc. and/or its subsidiaries.
11  *
12  * Firmware is:
13  *	Derived from proprietary unpublished source code,
14  *	Copyright (C) 2000-2016 Broadcom Corporation.
15  *	Copyright (C) 2016-2017 Broadcom Ltd.
16  *	Copyright (C) 2018 Broadcom. All Rights Reserved. The term "Broadcom"
17  *	refers to Broadcom Inc. and/or its subsidiaries.
18  *
19  *	Permission is hereby granted for the distribution of this firmware
20  *	data in hexadecimal or equivalent format, provided this copyright
21  *	notice is accompanying it.
22  */
23 
24 
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/stringify.h>
28 #include <linux/kernel.h>
29 #include <linux/sched/signal.h>
30 #include <linux/types.h>
31 #include <linux/compiler.h>
32 #include <linux/slab.h>
33 #include <linux/delay.h>
34 #include <linux/in.h>
35 #include <linux/interrupt.h>
36 #include <linux/ioport.h>
37 #include <linux/pci.h>
38 #include <linux/netdevice.h>
39 #include <linux/etherdevice.h>
40 #include <linux/skbuff.h>
41 #include <linux/ethtool.h>
42 #include <linux/mdio.h>
43 #include <linux/mii.h>
44 #include <linux/phy.h>
45 #include <linux/brcmphy.h>
46 #include <linux/if.h>
47 #include <linux/if_vlan.h>
48 #include <linux/ip.h>
49 #include <linux/tcp.h>
50 #include <linux/workqueue.h>
51 #include <linux/prefetch.h>
52 #include <linux/dma-mapping.h>
53 #include <linux/firmware.h>
54 #include <linux/ssb/ssb_driver_gige.h>
55 #include <linux/hwmon.h>
56 #include <linux/hwmon-sysfs.h>
57 #include <linux/crc32poly.h>
58 
59 #include <net/checksum.h>
60 #include <net/ip.h>
61 
62 #include <linux/io.h>
63 #include <asm/byteorder.h>
64 #include <linux/uaccess.h>
65 
66 #include <uapi/linux/net_tstamp.h>
67 #include <linux/ptp_clock_kernel.h>
68 
69 #define BAR_0	0
70 #define BAR_2	2
71 
72 #include "tg3.h"
73 
74 /* Functions & macros to verify TG3_FLAGS types */
75 
76 static inline int _tg3_flag(enum TG3_FLAGS flag, unsigned long *bits)
77 {
78 	return test_bit(flag, bits);
79 }
80 
81 static inline void _tg3_flag_set(enum TG3_FLAGS flag, unsigned long *bits)
82 {
83 	set_bit(flag, bits);
84 }
85 
86 static inline void _tg3_flag_clear(enum TG3_FLAGS flag, unsigned long *bits)
87 {
88 	clear_bit(flag, bits);
89 }
90 
91 #define tg3_flag(tp, flag)				\
92 	_tg3_flag(TG3_FLAG_##flag, (tp)->tg3_flags)
93 #define tg3_flag_set(tp, flag)				\
94 	_tg3_flag_set(TG3_FLAG_##flag, (tp)->tg3_flags)
95 #define tg3_flag_clear(tp, flag)			\
96 	_tg3_flag_clear(TG3_FLAG_##flag, (tp)->tg3_flags)
97 
98 #define DRV_MODULE_NAME		"tg3"
99 /* DO NOT UPDATE TG3_*_NUM defines */
100 #define TG3_MAJ_NUM			3
101 #define TG3_MIN_NUM			137
102 
103 #define RESET_KIND_SHUTDOWN	0
104 #define RESET_KIND_INIT		1
105 #define RESET_KIND_SUSPEND	2
106 
107 #define TG3_DEF_RX_MODE		0
108 #define TG3_DEF_TX_MODE		0
109 #define TG3_DEF_MSG_ENABLE	  \
110 	(NETIF_MSG_DRV		| \
111 	 NETIF_MSG_PROBE	| \
112 	 NETIF_MSG_LINK		| \
113 	 NETIF_MSG_TIMER	| \
114 	 NETIF_MSG_IFDOWN	| \
115 	 NETIF_MSG_IFUP		| \
116 	 NETIF_MSG_RX_ERR	| \
117 	 NETIF_MSG_TX_ERR)
118 
119 #define TG3_GRC_LCLCTL_PWRSW_DELAY	100
120 
121 /* length of time before we decide the hardware is borked,
122  * and dev->tx_timeout() should be called to fix the problem
123  */
124 
125 #define TG3_TX_TIMEOUT			(5 * HZ)
126 
127 /* hardware minimum and maximum for a single frame's data payload */
128 #define TG3_MIN_MTU			ETH_ZLEN
129 #define TG3_MAX_MTU(tp)	\
130 	(tg3_flag(tp, JUMBO_CAPABLE) ? 9000 : 1500)
131 
132 /* These numbers seem to be hard coded in the NIC firmware somehow.
133  * You can't change the ring sizes, but you can change where you place
134  * them in the NIC onboard memory.
135  */
136 #define TG3_RX_STD_RING_SIZE(tp) \
137 	(tg3_flag(tp, LRG_PROD_RING_CAP) ? \
138 	 TG3_RX_STD_MAX_SIZE_5717 : TG3_RX_STD_MAX_SIZE_5700)
139 #define TG3_DEF_RX_RING_PENDING		200
140 #define TG3_RX_JMB_RING_SIZE(tp) \
141 	(tg3_flag(tp, LRG_PROD_RING_CAP) ? \
142 	 TG3_RX_JMB_MAX_SIZE_5717 : TG3_RX_JMB_MAX_SIZE_5700)
143 #define TG3_DEF_RX_JUMBO_RING_PENDING	100
144 
145 /* Do not place this n-ring entries value into the tp struct itself,
146  * we really want to expose these constants to GCC so that modulo et
147  * al.  operations are done with shifts and masks instead of with
148  * hw multiply/modulo instructions.  Another solution would be to
149  * replace things like '% foo' with '& (foo - 1)'.
150  */
151 
152 #define TG3_TX_RING_SIZE		512
153 #define TG3_DEF_TX_RING_PENDING		(TG3_TX_RING_SIZE - 1)
154 
155 #define TG3_RX_STD_RING_BYTES(tp) \
156 	(sizeof(struct tg3_rx_buffer_desc) * TG3_RX_STD_RING_SIZE(tp))
157 #define TG3_RX_JMB_RING_BYTES(tp) \
158 	(sizeof(struct tg3_ext_rx_buffer_desc) * TG3_RX_JMB_RING_SIZE(tp))
159 #define TG3_RX_RCB_RING_BYTES(tp) \
160 	(sizeof(struct tg3_rx_buffer_desc) * (tp->rx_ret_ring_mask + 1))
161 #define TG3_TX_RING_BYTES	(sizeof(struct tg3_tx_buffer_desc) * \
162 				 TG3_TX_RING_SIZE)
163 #define NEXT_TX(N)		(((N) + 1) & (TG3_TX_RING_SIZE - 1))
164 
165 #define TG3_DMA_BYTE_ENAB		64
166 
167 #define TG3_RX_STD_DMA_SZ		1536
168 #define TG3_RX_JMB_DMA_SZ		9046
169 
170 #define TG3_RX_DMA_TO_MAP_SZ(x)		((x) + TG3_DMA_BYTE_ENAB)
171 
172 #define TG3_RX_STD_MAP_SZ		TG3_RX_DMA_TO_MAP_SZ(TG3_RX_STD_DMA_SZ)
173 #define TG3_RX_JMB_MAP_SZ		TG3_RX_DMA_TO_MAP_SZ(TG3_RX_JMB_DMA_SZ)
174 
175 #define TG3_RX_STD_BUFF_RING_SIZE(tp) \
176 	(sizeof(struct ring_info) * TG3_RX_STD_RING_SIZE(tp))
177 
178 #define TG3_RX_JMB_BUFF_RING_SIZE(tp) \
179 	(sizeof(struct ring_info) * TG3_RX_JMB_RING_SIZE(tp))
180 
181 /* Due to a hardware bug, the 5701 can only DMA to memory addresses
182  * that are at least dword aligned when used in PCIX mode.  The driver
183  * works around this bug by double copying the packet.  This workaround
184  * is built into the normal double copy length check for efficiency.
185  *
186  * However, the double copy is only necessary on those architectures
187  * where unaligned memory accesses are inefficient.  For those architectures
188  * where unaligned memory accesses incur little penalty, we can reintegrate
189  * the 5701 in the normal rx path.  Doing so saves a device structure
190  * dereference by hardcoding the double copy threshold in place.
191  */
192 #define TG3_RX_COPY_THRESHOLD		256
193 #if NET_IP_ALIGN == 0 || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
194 	#define TG3_RX_COPY_THRESH(tp)	TG3_RX_COPY_THRESHOLD
195 #else
196 	#define TG3_RX_COPY_THRESH(tp)	((tp)->rx_copy_thresh)
197 #endif
198 
199 #if (NET_IP_ALIGN != 0)
200 #define TG3_RX_OFFSET(tp)	((tp)->rx_offset)
201 #else
202 #define TG3_RX_OFFSET(tp)	(NET_SKB_PAD)
203 #endif
204 
205 /* minimum number of free TX descriptors required to wake up TX process */
206 #define TG3_TX_WAKEUP_THRESH(tnapi)		((tnapi)->tx_pending / 4)
207 #define TG3_TX_BD_DMA_MAX_2K		2048
208 #define TG3_TX_BD_DMA_MAX_4K		4096
209 
210 #define TG3_RAW_IP_ALIGN 2
211 
212 #define TG3_MAX_UCAST_ADDR(tp) (tg3_flag((tp), ENABLE_ASF) ? 2 : 3)
213 #define TG3_UCAST_ADDR_IDX(tp) (tg3_flag((tp), ENABLE_ASF) ? 2 : 1)
214 
215 #define TG3_FW_UPDATE_TIMEOUT_SEC	5
216 #define TG3_FW_UPDATE_FREQ_SEC		(TG3_FW_UPDATE_TIMEOUT_SEC / 2)
217 
218 #define FIRMWARE_TG3		"tigon/tg3.bin"
219 #define FIRMWARE_TG357766	"tigon/tg357766.bin"
220 #define FIRMWARE_TG3TSO		"tigon/tg3_tso.bin"
221 #define FIRMWARE_TG3TSO5	"tigon/tg3_tso5.bin"
222 
223 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
224 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
225 MODULE_LICENSE("GPL");
226 MODULE_FIRMWARE(FIRMWARE_TG3);
227 MODULE_FIRMWARE(FIRMWARE_TG3TSO);
228 MODULE_FIRMWARE(FIRMWARE_TG3TSO5);
229 
230 static int tg3_debug = -1;	/* -1 == use TG3_DEF_MSG_ENABLE as value */
231 module_param(tg3_debug, int, 0);
232 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
233 
234 #define TG3_DRV_DATA_FLAG_10_100_ONLY	0x0001
235 #define TG3_DRV_DATA_FLAG_5705_10_100	0x0002
236 
237 static const struct pci_device_id tg3_pci_tbl[] = {
238 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
239 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
240 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
241 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
242 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
243 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
244 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
245 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
246 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
247 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
248 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
249 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
250 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
251 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
252 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
253 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
254 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
255 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
256 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901),
257 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
258 			TG3_DRV_DATA_FLAG_5705_10_100},
259 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2),
260 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
261 			TG3_DRV_DATA_FLAG_5705_10_100},
262 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
263 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F),
264 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY |
265 			TG3_DRV_DATA_FLAG_5705_10_100},
266 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
267 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
268 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
269 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
270 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
271 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F),
272 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
273 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
274 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
275 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
276 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
277 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F),
278 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
279 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
280 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
281 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
282 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
283 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
284 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
285 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
286 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5787M,
287 			PCI_VENDOR_ID_LENOVO,
288 			TG3PCI_SUBDEVICE_ID_LENOVO_5787M),
289 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
290 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
291 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F),
292 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
293 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
294 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
295 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
296 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
297 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
298 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
299 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
300 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
301 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
302 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
303 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
304 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
305 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
306 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
307 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761S)},
308 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761SE)},
309 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_G)},
310 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_F)},
311 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780,
312 			PCI_VENDOR_ID_AI, TG3PCI_SUBDEVICE_ID_ACER_57780_A),
313 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
314 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780,
315 			PCI_VENDOR_ID_AI, TG3PCI_SUBDEVICE_ID_ACER_57780_B),
316 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
317 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780)},
318 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)},
319 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790),
320 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
321 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57788)},
322 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717)},
323 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717_C)},
324 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5718)},
325 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57781)},
326 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57785)},
327 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57761)},
328 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57765)},
329 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57791),
330 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
331 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57795),
332 	 .driver_data = TG3_DRV_DATA_FLAG_10_100_ONLY},
333 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5719)},
334 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5720)},
335 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57762)},
336 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57766)},
337 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5762)},
338 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5725)},
339 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5727)},
340 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57764)},
341 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57767)},
342 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57787)},
343 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57782)},
344 	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57786)},
345 	{PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
346 	{PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
347 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
348 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
349 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
350 	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
351 	{PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
352 	{PCI_DEVICE(0x10cf, 0x11a2)}, /* Fujitsu 1000base-SX with BCM5703SKHB */
353 	{}
354 };
355 
356 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
357 
358 static const struct {
359 	const char string[ETH_GSTRING_LEN];
360 } ethtool_stats_keys[] = {
361 	{ "rx_octets" },
362 	{ "rx_fragments" },
363 	{ "rx_ucast_packets" },
364 	{ "rx_mcast_packets" },
365 	{ "rx_bcast_packets" },
366 	{ "rx_fcs_errors" },
367 	{ "rx_align_errors" },
368 	{ "rx_xon_pause_rcvd" },
369 	{ "rx_xoff_pause_rcvd" },
370 	{ "rx_mac_ctrl_rcvd" },
371 	{ "rx_xoff_entered" },
372 	{ "rx_frame_too_long_errors" },
373 	{ "rx_jabbers" },
374 	{ "rx_undersize_packets" },
375 	{ "rx_in_length_errors" },
376 	{ "rx_out_length_errors" },
377 	{ "rx_64_or_less_octet_packets" },
378 	{ "rx_65_to_127_octet_packets" },
379 	{ "rx_128_to_255_octet_packets" },
380 	{ "rx_256_to_511_octet_packets" },
381 	{ "rx_512_to_1023_octet_packets" },
382 	{ "rx_1024_to_1522_octet_packets" },
383 	{ "rx_1523_to_2047_octet_packets" },
384 	{ "rx_2048_to_4095_octet_packets" },
385 	{ "rx_4096_to_8191_octet_packets" },
386 	{ "rx_8192_to_9022_octet_packets" },
387 
388 	{ "tx_octets" },
389 	{ "tx_collisions" },
390 
391 	{ "tx_xon_sent" },
392 	{ "tx_xoff_sent" },
393 	{ "tx_flow_control" },
394 	{ "tx_mac_errors" },
395 	{ "tx_single_collisions" },
396 	{ "tx_mult_collisions" },
397 	{ "tx_deferred" },
398 	{ "tx_excessive_collisions" },
399 	{ "tx_late_collisions" },
400 	{ "tx_collide_2times" },
401 	{ "tx_collide_3times" },
402 	{ "tx_collide_4times" },
403 	{ "tx_collide_5times" },
404 	{ "tx_collide_6times" },
405 	{ "tx_collide_7times" },
406 	{ "tx_collide_8times" },
407 	{ "tx_collide_9times" },
408 	{ "tx_collide_10times" },
409 	{ "tx_collide_11times" },
410 	{ "tx_collide_12times" },
411 	{ "tx_collide_13times" },
412 	{ "tx_collide_14times" },
413 	{ "tx_collide_15times" },
414 	{ "tx_ucast_packets" },
415 	{ "tx_mcast_packets" },
416 	{ "tx_bcast_packets" },
417 	{ "tx_carrier_sense_errors" },
418 	{ "tx_discards" },
419 	{ "tx_errors" },
420 
421 	{ "dma_writeq_full" },
422 	{ "dma_write_prioq_full" },
423 	{ "rxbds_empty" },
424 	{ "rx_discards" },
425 	{ "rx_errors" },
426 	{ "rx_threshold_hit" },
427 
428 	{ "dma_readq_full" },
429 	{ "dma_read_prioq_full" },
430 	{ "tx_comp_queue_full" },
431 
432 	{ "ring_set_send_prod_index" },
433 	{ "ring_status_update" },
434 	{ "nic_irqs" },
435 	{ "nic_avoided_irqs" },
436 	{ "nic_tx_threshold_hit" },
437 
438 	{ "mbuf_lwm_thresh_hit" },
439 };
440 
441 #define TG3_NUM_STATS	ARRAY_SIZE(ethtool_stats_keys)
442 #define TG3_NVRAM_TEST		0
443 #define TG3_LINK_TEST		1
444 #define TG3_REGISTER_TEST	2
445 #define TG3_MEMORY_TEST		3
446 #define TG3_MAC_LOOPB_TEST	4
447 #define TG3_PHY_LOOPB_TEST	5
448 #define TG3_EXT_LOOPB_TEST	6
449 #define TG3_INTERRUPT_TEST	7
450 
451 
452 static const struct {
453 	const char string[ETH_GSTRING_LEN];
454 } ethtool_test_keys[] = {
455 	[TG3_NVRAM_TEST]	= { "nvram test        (online) " },
456 	[TG3_LINK_TEST]		= { "link test         (online) " },
457 	[TG3_REGISTER_TEST]	= { "register test     (offline)" },
458 	[TG3_MEMORY_TEST]	= { "memory test       (offline)" },
459 	[TG3_MAC_LOOPB_TEST]	= { "mac loopback test (offline)" },
460 	[TG3_PHY_LOOPB_TEST]	= { "phy loopback test (offline)" },
461 	[TG3_EXT_LOOPB_TEST]	= { "ext loopback test (offline)" },
462 	[TG3_INTERRUPT_TEST]	= { "interrupt test    (offline)" },
463 };
464 
465 #define TG3_NUM_TEST	ARRAY_SIZE(ethtool_test_keys)
466 
467 
468 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
469 {
470 	writel(val, tp->regs + off);
471 }
472 
473 static u32 tg3_read32(struct tg3 *tp, u32 off)
474 {
475 	return readl(tp->regs + off);
476 }
477 
478 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
479 {
480 	writel(val, tp->aperegs + off);
481 }
482 
483 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
484 {
485 	return readl(tp->aperegs + off);
486 }
487 
488 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
489 {
490 	unsigned long flags;
491 
492 	spin_lock_irqsave(&tp->indirect_lock, flags);
493 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
494 	pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
495 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
496 }
497 
498 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
499 {
500 	writel(val, tp->regs + off);
501 	readl(tp->regs + off);
502 }
503 
504 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
505 {
506 	unsigned long flags;
507 	u32 val;
508 
509 	spin_lock_irqsave(&tp->indirect_lock, flags);
510 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
511 	pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
512 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
513 	return val;
514 }
515 
516 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
517 {
518 	unsigned long flags;
519 
520 	if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
521 		pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
522 				       TG3_64BIT_REG_LOW, val);
523 		return;
524 	}
525 	if (off == TG3_RX_STD_PROD_IDX_REG) {
526 		pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
527 				       TG3_64BIT_REG_LOW, val);
528 		return;
529 	}
530 
531 	spin_lock_irqsave(&tp->indirect_lock, flags);
532 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
533 	pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
534 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
535 
536 	/* In indirect mode when disabling interrupts, we also need
537 	 * to clear the interrupt bit in the GRC local ctrl register.
538 	 */
539 	if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
540 	    (val == 0x1)) {
541 		pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
542 				       tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
543 	}
544 }
545 
546 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
547 {
548 	unsigned long flags;
549 	u32 val;
550 
551 	spin_lock_irqsave(&tp->indirect_lock, flags);
552 	pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
553 	pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
554 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
555 	return val;
556 }
557 
558 /* usec_wait specifies the wait time in usec when writing to certain registers
559  * where it is unsafe to read back the register without some delay.
560  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
561  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
562  */
563 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
564 {
565 	if (tg3_flag(tp, PCIX_TARGET_HWBUG) || tg3_flag(tp, ICH_WORKAROUND))
566 		/* Non-posted methods */
567 		tp->write32(tp, off, val);
568 	else {
569 		/* Posted method */
570 		tg3_write32(tp, off, val);
571 		if (usec_wait)
572 			udelay(usec_wait);
573 		tp->read32(tp, off);
574 	}
575 	/* Wait again after the read for the posted method to guarantee that
576 	 * the wait time is met.
577 	 */
578 	if (usec_wait)
579 		udelay(usec_wait);
580 }
581 
582 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
583 {
584 	tp->write32_mbox(tp, off, val);
585 	if (tg3_flag(tp, FLUSH_POSTED_WRITES) ||
586 	    (!tg3_flag(tp, MBOX_WRITE_REORDER) &&
587 	     !tg3_flag(tp, ICH_WORKAROUND)))
588 		tp->read32_mbox(tp, off);
589 }
590 
591 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
592 {
593 	void __iomem *mbox = tp->regs + off;
594 	writel(val, mbox);
595 	if (tg3_flag(tp, TXD_MBOX_HWBUG))
596 		writel(val, mbox);
597 	if (tg3_flag(tp, MBOX_WRITE_REORDER) ||
598 	    tg3_flag(tp, FLUSH_POSTED_WRITES))
599 		readl(mbox);
600 }
601 
602 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
603 {
604 	return readl(tp->regs + off + GRCMBOX_BASE);
605 }
606 
607 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
608 {
609 	writel(val, tp->regs + off + GRCMBOX_BASE);
610 }
611 
612 #define tw32_mailbox(reg, val)		tp->write32_mbox(tp, reg, val)
613 #define tw32_mailbox_f(reg, val)	tw32_mailbox_flush(tp, (reg), (val))
614 #define tw32_rx_mbox(reg, val)		tp->write32_rx_mbox(tp, reg, val)
615 #define tw32_tx_mbox(reg, val)		tp->write32_tx_mbox(tp, reg, val)
616 #define tr32_mailbox(reg)		tp->read32_mbox(tp, reg)
617 
618 #define tw32(reg, val)			tp->write32(tp, reg, val)
619 #define tw32_f(reg, val)		_tw32_flush(tp, (reg), (val), 0)
620 #define tw32_wait_f(reg, val, us)	_tw32_flush(tp, (reg), (val), (us))
621 #define tr32(reg)			tp->read32(tp, reg)
622 
623 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
624 {
625 	unsigned long flags;
626 
627 	if (tg3_asic_rev(tp) == ASIC_REV_5906 &&
628 	    (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
629 		return;
630 
631 	spin_lock_irqsave(&tp->indirect_lock, flags);
632 	if (tg3_flag(tp, SRAM_USE_CONFIG)) {
633 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
634 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
635 
636 		/* Always leave this as zero. */
637 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
638 	} else {
639 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
640 		tw32_f(TG3PCI_MEM_WIN_DATA, val);
641 
642 		/* Always leave this as zero. */
643 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
644 	}
645 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
646 }
647 
648 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
649 {
650 	unsigned long flags;
651 
652 	if (tg3_asic_rev(tp) == ASIC_REV_5906 &&
653 	    (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
654 		*val = 0;
655 		return;
656 	}
657 
658 	spin_lock_irqsave(&tp->indirect_lock, flags);
659 	if (tg3_flag(tp, SRAM_USE_CONFIG)) {
660 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
661 		pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
662 
663 		/* Always leave this as zero. */
664 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
665 	} else {
666 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
667 		*val = tr32(TG3PCI_MEM_WIN_DATA);
668 
669 		/* Always leave this as zero. */
670 		tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
671 	}
672 	spin_unlock_irqrestore(&tp->indirect_lock, flags);
673 }
674 
675 static void tg3_ape_lock_init(struct tg3 *tp)
676 {
677 	int i;
678 	u32 regbase, bit;
679 
680 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
681 		regbase = TG3_APE_LOCK_GRANT;
682 	else
683 		regbase = TG3_APE_PER_LOCK_GRANT;
684 
685 	/* Make sure the driver hasn't any stale locks. */
686 	for (i = TG3_APE_LOCK_PHY0; i <= TG3_APE_LOCK_GPIO; i++) {
687 		switch (i) {
688 		case TG3_APE_LOCK_PHY0:
689 		case TG3_APE_LOCK_PHY1:
690 		case TG3_APE_LOCK_PHY2:
691 		case TG3_APE_LOCK_PHY3:
692 			bit = APE_LOCK_GRANT_DRIVER;
693 			break;
694 		default:
695 			if (!tp->pci_fn)
696 				bit = APE_LOCK_GRANT_DRIVER;
697 			else
698 				bit = 1 << tp->pci_fn;
699 		}
700 		tg3_ape_write32(tp, regbase + 4 * i, bit);
701 	}
702 
703 }
704 
705 static int tg3_ape_lock(struct tg3 *tp, int locknum)
706 {
707 	int i, off;
708 	int ret = 0;
709 	u32 status, req, gnt, bit;
710 
711 	if (!tg3_flag(tp, ENABLE_APE))
712 		return 0;
713 
714 	switch (locknum) {
715 	case TG3_APE_LOCK_GPIO:
716 		if (tg3_asic_rev(tp) == ASIC_REV_5761)
717 			return 0;
718 		fallthrough;
719 	case TG3_APE_LOCK_GRC:
720 	case TG3_APE_LOCK_MEM:
721 		if (!tp->pci_fn)
722 			bit = APE_LOCK_REQ_DRIVER;
723 		else
724 			bit = 1 << tp->pci_fn;
725 		break;
726 	case TG3_APE_LOCK_PHY0:
727 	case TG3_APE_LOCK_PHY1:
728 	case TG3_APE_LOCK_PHY2:
729 	case TG3_APE_LOCK_PHY3:
730 		bit = APE_LOCK_REQ_DRIVER;
731 		break;
732 	default:
733 		return -EINVAL;
734 	}
735 
736 	if (tg3_asic_rev(tp) == ASIC_REV_5761) {
737 		req = TG3_APE_LOCK_REQ;
738 		gnt = TG3_APE_LOCK_GRANT;
739 	} else {
740 		req = TG3_APE_PER_LOCK_REQ;
741 		gnt = TG3_APE_PER_LOCK_GRANT;
742 	}
743 
744 	off = 4 * locknum;
745 
746 	tg3_ape_write32(tp, req + off, bit);
747 
748 	/* Wait for up to 1 millisecond to acquire lock. */
749 	for (i = 0; i < 100; i++) {
750 		status = tg3_ape_read32(tp, gnt + off);
751 		if (status == bit)
752 			break;
753 		if (pci_channel_offline(tp->pdev))
754 			break;
755 
756 		udelay(10);
757 	}
758 
759 	if (status != bit) {
760 		/* Revoke the lock request. */
761 		tg3_ape_write32(tp, gnt + off, bit);
762 		ret = -EBUSY;
763 	}
764 
765 	return ret;
766 }
767 
768 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
769 {
770 	u32 gnt, bit;
771 
772 	if (!tg3_flag(tp, ENABLE_APE))
773 		return;
774 
775 	switch (locknum) {
776 	case TG3_APE_LOCK_GPIO:
777 		if (tg3_asic_rev(tp) == ASIC_REV_5761)
778 			return;
779 		fallthrough;
780 	case TG3_APE_LOCK_GRC:
781 	case TG3_APE_LOCK_MEM:
782 		if (!tp->pci_fn)
783 			bit = APE_LOCK_GRANT_DRIVER;
784 		else
785 			bit = 1 << tp->pci_fn;
786 		break;
787 	case TG3_APE_LOCK_PHY0:
788 	case TG3_APE_LOCK_PHY1:
789 	case TG3_APE_LOCK_PHY2:
790 	case TG3_APE_LOCK_PHY3:
791 		bit = APE_LOCK_GRANT_DRIVER;
792 		break;
793 	default:
794 		return;
795 	}
796 
797 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
798 		gnt = TG3_APE_LOCK_GRANT;
799 	else
800 		gnt = TG3_APE_PER_LOCK_GRANT;
801 
802 	tg3_ape_write32(tp, gnt + 4 * locknum, bit);
803 }
804 
805 static int tg3_ape_event_lock(struct tg3 *tp, u32 timeout_us)
806 {
807 	u32 apedata;
808 
809 	while (timeout_us) {
810 		if (tg3_ape_lock(tp, TG3_APE_LOCK_MEM))
811 			return -EBUSY;
812 
813 		apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
814 		if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
815 			break;
816 
817 		tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
818 
819 		udelay(10);
820 		timeout_us -= (timeout_us > 10) ? 10 : timeout_us;
821 	}
822 
823 	return timeout_us ? 0 : -EBUSY;
824 }
825 
826 #ifdef CONFIG_TIGON3_HWMON
827 static int tg3_ape_wait_for_event(struct tg3 *tp, u32 timeout_us)
828 {
829 	u32 i, apedata;
830 
831 	for (i = 0; i < timeout_us / 10; i++) {
832 		apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
833 
834 		if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
835 			break;
836 
837 		udelay(10);
838 	}
839 
840 	return i == timeout_us / 10;
841 }
842 
843 static int tg3_ape_scratchpad_read(struct tg3 *tp, u32 *data, u32 base_off,
844 				   u32 len)
845 {
846 	int err;
847 	u32 i, bufoff, msgoff, maxlen, apedata;
848 
849 	if (!tg3_flag(tp, APE_HAS_NCSI))
850 		return 0;
851 
852 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
853 	if (apedata != APE_SEG_SIG_MAGIC)
854 		return -ENODEV;
855 
856 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
857 	if (!(apedata & APE_FW_STATUS_READY))
858 		return -EAGAIN;
859 
860 	bufoff = tg3_ape_read32(tp, TG3_APE_SEG_MSG_BUF_OFF) +
861 		 TG3_APE_SHMEM_BASE;
862 	msgoff = bufoff + 2 * sizeof(u32);
863 	maxlen = tg3_ape_read32(tp, TG3_APE_SEG_MSG_BUF_LEN);
864 
865 	while (len) {
866 		u32 length;
867 
868 		/* Cap xfer sizes to scratchpad limits. */
869 		length = (len > maxlen) ? maxlen : len;
870 		len -= length;
871 
872 		apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
873 		if (!(apedata & APE_FW_STATUS_READY))
874 			return -EAGAIN;
875 
876 		/* Wait for up to 1 msec for APE to service previous event. */
877 		err = tg3_ape_event_lock(tp, 1000);
878 		if (err)
879 			return err;
880 
881 		apedata = APE_EVENT_STATUS_DRIVER_EVNT |
882 			  APE_EVENT_STATUS_SCRTCHPD_READ |
883 			  APE_EVENT_STATUS_EVENT_PENDING;
884 		tg3_ape_write32(tp, TG3_APE_EVENT_STATUS, apedata);
885 
886 		tg3_ape_write32(tp, bufoff, base_off);
887 		tg3_ape_write32(tp, bufoff + sizeof(u32), length);
888 
889 		tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
890 		tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
891 
892 		base_off += length;
893 
894 		if (tg3_ape_wait_for_event(tp, 30000))
895 			return -EAGAIN;
896 
897 		for (i = 0; length; i += 4, length -= 4) {
898 			u32 val = tg3_ape_read32(tp, msgoff + i);
899 			memcpy(data, &val, sizeof(u32));
900 			data++;
901 		}
902 	}
903 
904 	return 0;
905 }
906 #endif
907 
908 static int tg3_ape_send_event(struct tg3 *tp, u32 event)
909 {
910 	int err;
911 	u32 apedata;
912 
913 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
914 	if (apedata != APE_SEG_SIG_MAGIC)
915 		return -EAGAIN;
916 
917 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
918 	if (!(apedata & APE_FW_STATUS_READY))
919 		return -EAGAIN;
920 
921 	/* Wait for up to 20 millisecond for APE to service previous event. */
922 	err = tg3_ape_event_lock(tp, 20000);
923 	if (err)
924 		return err;
925 
926 	tg3_ape_write32(tp, TG3_APE_EVENT_STATUS,
927 			event | APE_EVENT_STATUS_EVENT_PENDING);
928 
929 	tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
930 	tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
931 
932 	return 0;
933 }
934 
935 static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
936 {
937 	u32 event;
938 	u32 apedata;
939 
940 	if (!tg3_flag(tp, ENABLE_APE))
941 		return;
942 
943 	switch (kind) {
944 	case RESET_KIND_INIT:
945 		tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_COUNT, tp->ape_hb++);
946 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
947 				APE_HOST_SEG_SIG_MAGIC);
948 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
949 				APE_HOST_SEG_LEN_MAGIC);
950 		apedata = tg3_ape_read32(tp, TG3_APE_HOST_INIT_COUNT);
951 		tg3_ape_write32(tp, TG3_APE_HOST_INIT_COUNT, ++apedata);
952 		tg3_ape_write32(tp, TG3_APE_HOST_DRIVER_ID,
953 			APE_HOST_DRIVER_ID_MAGIC(TG3_MAJ_NUM, TG3_MIN_NUM));
954 		tg3_ape_write32(tp, TG3_APE_HOST_BEHAVIOR,
955 				APE_HOST_BEHAV_NO_PHYLOCK);
956 		tg3_ape_write32(tp, TG3_APE_HOST_DRVR_STATE,
957 				    TG3_APE_HOST_DRVR_STATE_START);
958 
959 		event = APE_EVENT_STATUS_STATE_START;
960 		break;
961 	case RESET_KIND_SHUTDOWN:
962 		if (device_may_wakeup(&tp->pdev->dev) &&
963 		    tg3_flag(tp, WOL_ENABLE)) {
964 			tg3_ape_write32(tp, TG3_APE_HOST_WOL_SPEED,
965 					    TG3_APE_HOST_WOL_SPEED_AUTO);
966 			apedata = TG3_APE_HOST_DRVR_STATE_WOL;
967 		} else
968 			apedata = TG3_APE_HOST_DRVR_STATE_UNLOAD;
969 
970 		tg3_ape_write32(tp, TG3_APE_HOST_DRVR_STATE, apedata);
971 
972 		event = APE_EVENT_STATUS_STATE_UNLOAD;
973 		break;
974 	default:
975 		return;
976 	}
977 
978 	event |= APE_EVENT_STATUS_DRIVER_EVNT | APE_EVENT_STATUS_STATE_CHNGE;
979 
980 	tg3_ape_send_event(tp, event);
981 }
982 
983 static void tg3_send_ape_heartbeat(struct tg3 *tp,
984 				   unsigned long interval)
985 {
986 	/* Check if hb interval has exceeded */
987 	if (!tg3_flag(tp, ENABLE_APE) ||
988 	    time_before(jiffies, tp->ape_hb_jiffies + interval))
989 		return;
990 
991 	tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_COUNT, tp->ape_hb++);
992 	tp->ape_hb_jiffies = jiffies;
993 }
994 
995 static void tg3_disable_ints(struct tg3 *tp)
996 {
997 	int i;
998 
999 	tw32(TG3PCI_MISC_HOST_CTRL,
1000 	     (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
1001 	for (i = 0; i < tp->irq_max; i++)
1002 		tw32_mailbox_f(tp->napi[i].int_mbox, 0x00000001);
1003 }
1004 
1005 static void tg3_enable_ints(struct tg3 *tp)
1006 {
1007 	int i;
1008 
1009 	tp->irq_sync = 0;
1010 	wmb();
1011 
1012 	tw32(TG3PCI_MISC_HOST_CTRL,
1013 	     (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
1014 
1015 	tp->coal_now = tp->coalesce_mode | HOSTCC_MODE_ENABLE;
1016 	for (i = 0; i < tp->irq_cnt; i++) {
1017 		struct tg3_napi *tnapi = &tp->napi[i];
1018 
1019 		tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
1020 		if (tg3_flag(tp, 1SHOT_MSI))
1021 			tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
1022 
1023 		tp->coal_now |= tnapi->coal_now;
1024 	}
1025 
1026 	/* Force an initial interrupt */
1027 	if (!tg3_flag(tp, TAGGED_STATUS) &&
1028 	    (tp->napi[0].hw_status->status & SD_STATUS_UPDATED))
1029 		tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
1030 	else
1031 		tw32(HOSTCC_MODE, tp->coal_now);
1032 
1033 	tp->coal_now &= ~(tp->napi[0].coal_now | tp->napi[1].coal_now);
1034 }
1035 
1036 static inline unsigned int tg3_has_work(struct tg3_napi *tnapi)
1037 {
1038 	struct tg3 *tp = tnapi->tp;
1039 	struct tg3_hw_status *sblk = tnapi->hw_status;
1040 	unsigned int work_exists = 0;
1041 
1042 	/* check for phy events */
1043 	if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
1044 		if (sblk->status & SD_STATUS_LINK_CHG)
1045 			work_exists = 1;
1046 	}
1047 
1048 	/* check for TX work to do */
1049 	if (sblk->idx[0].tx_consumer != tnapi->tx_cons)
1050 		work_exists = 1;
1051 
1052 	/* check for RX work to do */
1053 	if (tnapi->rx_rcb_prod_idx &&
1054 	    *(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
1055 		work_exists = 1;
1056 
1057 	return work_exists;
1058 }
1059 
1060 /* tg3_int_reenable
1061  *  similar to tg3_enable_ints, but it accurately determines whether there
1062  *  is new work pending and can return without flushing the PIO write
1063  *  which reenables interrupts
1064  */
1065 static void tg3_int_reenable(struct tg3_napi *tnapi)
1066 {
1067 	struct tg3 *tp = tnapi->tp;
1068 
1069 	tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
1070 
1071 	/* When doing tagged status, this work check is unnecessary.
1072 	 * The last_tag we write above tells the chip which piece of
1073 	 * work we've completed.
1074 	 */
1075 	if (!tg3_flag(tp, TAGGED_STATUS) && tg3_has_work(tnapi))
1076 		tw32(HOSTCC_MODE, tp->coalesce_mode |
1077 		     HOSTCC_MODE_ENABLE | tnapi->coal_now);
1078 }
1079 
1080 static void tg3_switch_clocks(struct tg3 *tp)
1081 {
1082 	u32 clock_ctrl;
1083 	u32 orig_clock_ctrl;
1084 
1085 	if (tg3_flag(tp, CPMU_PRESENT) || tg3_flag(tp, 5780_CLASS))
1086 		return;
1087 
1088 	clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
1089 
1090 	orig_clock_ctrl = clock_ctrl;
1091 	clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
1092 		       CLOCK_CTRL_CLKRUN_OENABLE |
1093 		       0x1f);
1094 	tp->pci_clock_ctrl = clock_ctrl;
1095 
1096 	if (tg3_flag(tp, 5705_PLUS)) {
1097 		if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
1098 			tw32_wait_f(TG3PCI_CLOCK_CTRL,
1099 				    clock_ctrl | CLOCK_CTRL_625_CORE, 40);
1100 		}
1101 	} else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
1102 		tw32_wait_f(TG3PCI_CLOCK_CTRL,
1103 			    clock_ctrl |
1104 			    (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
1105 			    40);
1106 		tw32_wait_f(TG3PCI_CLOCK_CTRL,
1107 			    clock_ctrl | (CLOCK_CTRL_ALTCLK),
1108 			    40);
1109 	}
1110 	tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
1111 }
1112 
1113 #define PHY_BUSY_LOOPS	5000
1114 
1115 static int __tg3_readphy(struct tg3 *tp, unsigned int phy_addr, int reg,
1116 			 u32 *val)
1117 {
1118 	u32 frame_val;
1119 	unsigned int loops;
1120 	int ret;
1121 
1122 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1123 		tw32_f(MAC_MI_MODE,
1124 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
1125 		udelay(80);
1126 	}
1127 
1128 	tg3_ape_lock(tp, tp->phy_ape_lock);
1129 
1130 	*val = 0x0;
1131 
1132 	frame_val  = ((phy_addr << MI_COM_PHY_ADDR_SHIFT) &
1133 		      MI_COM_PHY_ADDR_MASK);
1134 	frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
1135 		      MI_COM_REG_ADDR_MASK);
1136 	frame_val |= (MI_COM_CMD_READ | MI_COM_START);
1137 
1138 	tw32_f(MAC_MI_COM, frame_val);
1139 
1140 	loops = PHY_BUSY_LOOPS;
1141 	while (loops != 0) {
1142 		udelay(10);
1143 		frame_val = tr32(MAC_MI_COM);
1144 
1145 		if ((frame_val & MI_COM_BUSY) == 0) {
1146 			udelay(5);
1147 			frame_val = tr32(MAC_MI_COM);
1148 			break;
1149 		}
1150 		loops -= 1;
1151 	}
1152 
1153 	ret = -EBUSY;
1154 	if (loops != 0) {
1155 		*val = frame_val & MI_COM_DATA_MASK;
1156 		ret = 0;
1157 	}
1158 
1159 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1160 		tw32_f(MAC_MI_MODE, tp->mi_mode);
1161 		udelay(80);
1162 	}
1163 
1164 	tg3_ape_unlock(tp, tp->phy_ape_lock);
1165 
1166 	return ret;
1167 }
1168 
1169 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
1170 {
1171 	return __tg3_readphy(tp, tp->phy_addr, reg, val);
1172 }
1173 
1174 static int __tg3_writephy(struct tg3 *tp, unsigned int phy_addr, int reg,
1175 			  u32 val)
1176 {
1177 	u32 frame_val;
1178 	unsigned int loops;
1179 	int ret;
1180 
1181 	if ((tp->phy_flags & TG3_PHYFLG_IS_FET) &&
1182 	    (reg == MII_CTRL1000 || reg == MII_TG3_AUX_CTRL))
1183 		return 0;
1184 
1185 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1186 		tw32_f(MAC_MI_MODE,
1187 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
1188 		udelay(80);
1189 	}
1190 
1191 	tg3_ape_lock(tp, tp->phy_ape_lock);
1192 
1193 	frame_val  = ((phy_addr << MI_COM_PHY_ADDR_SHIFT) &
1194 		      MI_COM_PHY_ADDR_MASK);
1195 	frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
1196 		      MI_COM_REG_ADDR_MASK);
1197 	frame_val |= (val & MI_COM_DATA_MASK);
1198 	frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
1199 
1200 	tw32_f(MAC_MI_COM, frame_val);
1201 
1202 	loops = PHY_BUSY_LOOPS;
1203 	while (loops != 0) {
1204 		udelay(10);
1205 		frame_val = tr32(MAC_MI_COM);
1206 		if ((frame_val & MI_COM_BUSY) == 0) {
1207 			udelay(5);
1208 			frame_val = tr32(MAC_MI_COM);
1209 			break;
1210 		}
1211 		loops -= 1;
1212 	}
1213 
1214 	ret = -EBUSY;
1215 	if (loops != 0)
1216 		ret = 0;
1217 
1218 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
1219 		tw32_f(MAC_MI_MODE, tp->mi_mode);
1220 		udelay(80);
1221 	}
1222 
1223 	tg3_ape_unlock(tp, tp->phy_ape_lock);
1224 
1225 	return ret;
1226 }
1227 
1228 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
1229 {
1230 	return __tg3_writephy(tp, tp->phy_addr, reg, val);
1231 }
1232 
1233 static int tg3_phy_cl45_write(struct tg3 *tp, u32 devad, u32 addr, u32 val)
1234 {
1235 	int err;
1236 
1237 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
1238 	if (err)
1239 		goto done;
1240 
1241 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
1242 	if (err)
1243 		goto done;
1244 
1245 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
1246 			   MII_TG3_MMD_CTRL_DATA_NOINC | devad);
1247 	if (err)
1248 		goto done;
1249 
1250 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, val);
1251 
1252 done:
1253 	return err;
1254 }
1255 
1256 static int tg3_phy_cl45_read(struct tg3 *tp, u32 devad, u32 addr, u32 *val)
1257 {
1258 	int err;
1259 
1260 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
1261 	if (err)
1262 		goto done;
1263 
1264 	err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
1265 	if (err)
1266 		goto done;
1267 
1268 	err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
1269 			   MII_TG3_MMD_CTRL_DATA_NOINC | devad);
1270 	if (err)
1271 		goto done;
1272 
1273 	err = tg3_readphy(tp, MII_TG3_MMD_ADDRESS, val);
1274 
1275 done:
1276 	return err;
1277 }
1278 
1279 static int tg3_phydsp_read(struct tg3 *tp, u32 reg, u32 *val)
1280 {
1281 	int err;
1282 
1283 	err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1284 	if (!err)
1285 		err = tg3_readphy(tp, MII_TG3_DSP_RW_PORT, val);
1286 
1287 	return err;
1288 }
1289 
1290 static int tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
1291 {
1292 	int err;
1293 
1294 	err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1295 	if (!err)
1296 		err = tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
1297 
1298 	return err;
1299 }
1300 
1301 static int tg3_phy_auxctl_read(struct tg3 *tp, int reg, u32 *val)
1302 {
1303 	int err;
1304 
1305 	err = tg3_writephy(tp, MII_TG3_AUX_CTRL,
1306 			   (reg << MII_TG3_AUXCTL_MISC_RDSEL_SHIFT) |
1307 			   MII_TG3_AUXCTL_SHDWSEL_MISC);
1308 	if (!err)
1309 		err = tg3_readphy(tp, MII_TG3_AUX_CTRL, val);
1310 
1311 	return err;
1312 }
1313 
1314 static int tg3_phy_auxctl_write(struct tg3 *tp, int reg, u32 set)
1315 {
1316 	if (reg == MII_TG3_AUXCTL_SHDWSEL_MISC)
1317 		set |= MII_TG3_AUXCTL_MISC_WREN;
1318 
1319 	return tg3_writephy(tp, MII_TG3_AUX_CTRL, set | reg);
1320 }
1321 
1322 static int tg3_phy_toggle_auxctl_smdsp(struct tg3 *tp, bool enable)
1323 {
1324 	u32 val;
1325 	int err;
1326 
1327 	err = tg3_phy_auxctl_read(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
1328 
1329 	if (err)
1330 		return err;
1331 
1332 	if (enable)
1333 		val |= MII_TG3_AUXCTL_ACTL_SMDSP_ENA;
1334 	else
1335 		val &= ~MII_TG3_AUXCTL_ACTL_SMDSP_ENA;
1336 
1337 	err = tg3_phy_auxctl_write((tp), MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
1338 				   val | MII_TG3_AUXCTL_ACTL_TX_6DB);
1339 
1340 	return err;
1341 }
1342 
1343 static int tg3_phy_shdw_write(struct tg3 *tp, int reg, u32 val)
1344 {
1345 	return tg3_writephy(tp, MII_TG3_MISC_SHDW,
1346 			    reg | val | MII_TG3_MISC_SHDW_WREN);
1347 }
1348 
1349 static int tg3_bmcr_reset(struct tg3 *tp)
1350 {
1351 	u32 phy_control;
1352 	int limit, err;
1353 
1354 	/* OK, reset it, and poll the BMCR_RESET bit until it
1355 	 * clears or we time out.
1356 	 */
1357 	phy_control = BMCR_RESET;
1358 	err = tg3_writephy(tp, MII_BMCR, phy_control);
1359 	if (err != 0)
1360 		return -EBUSY;
1361 
1362 	limit = 5000;
1363 	while (limit--) {
1364 		err = tg3_readphy(tp, MII_BMCR, &phy_control);
1365 		if (err != 0)
1366 			return -EBUSY;
1367 
1368 		if ((phy_control & BMCR_RESET) == 0) {
1369 			udelay(40);
1370 			break;
1371 		}
1372 		udelay(10);
1373 	}
1374 	if (limit < 0)
1375 		return -EBUSY;
1376 
1377 	return 0;
1378 }
1379 
1380 static int tg3_mdio_read(struct mii_bus *bp, int mii_id, int reg)
1381 {
1382 	struct tg3 *tp = bp->priv;
1383 	u32 val;
1384 
1385 	spin_lock_bh(&tp->lock);
1386 
1387 	if (__tg3_readphy(tp, mii_id, reg, &val))
1388 		val = -EIO;
1389 
1390 	spin_unlock_bh(&tp->lock);
1391 
1392 	return val;
1393 }
1394 
1395 static int tg3_mdio_write(struct mii_bus *bp, int mii_id, int reg, u16 val)
1396 {
1397 	struct tg3 *tp = bp->priv;
1398 	u32 ret = 0;
1399 
1400 	spin_lock_bh(&tp->lock);
1401 
1402 	if (__tg3_writephy(tp, mii_id, reg, val))
1403 		ret = -EIO;
1404 
1405 	spin_unlock_bh(&tp->lock);
1406 
1407 	return ret;
1408 }
1409 
1410 static void tg3_mdio_config_5785(struct tg3 *tp)
1411 {
1412 	u32 val;
1413 	struct phy_device *phydev;
1414 
1415 	phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
1416 	switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1417 	case PHY_ID_BCM50610:
1418 	case PHY_ID_BCM50610M:
1419 		val = MAC_PHYCFG2_50610_LED_MODES;
1420 		break;
1421 	case PHY_ID_BCMAC131:
1422 		val = MAC_PHYCFG2_AC131_LED_MODES;
1423 		break;
1424 	case PHY_ID_RTL8211C:
1425 		val = MAC_PHYCFG2_RTL8211C_LED_MODES;
1426 		break;
1427 	case PHY_ID_RTL8201E:
1428 		val = MAC_PHYCFG2_RTL8201E_LED_MODES;
1429 		break;
1430 	default:
1431 		return;
1432 	}
1433 
1434 	if (phydev->interface != PHY_INTERFACE_MODE_RGMII) {
1435 		tw32(MAC_PHYCFG2, val);
1436 
1437 		val = tr32(MAC_PHYCFG1);
1438 		val &= ~(MAC_PHYCFG1_RGMII_INT |
1439 			 MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK);
1440 		val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT;
1441 		tw32(MAC_PHYCFG1, val);
1442 
1443 		return;
1444 	}
1445 
1446 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE))
1447 		val |= MAC_PHYCFG2_EMODE_MASK_MASK |
1448 		       MAC_PHYCFG2_FMODE_MASK_MASK |
1449 		       MAC_PHYCFG2_GMODE_MASK_MASK |
1450 		       MAC_PHYCFG2_ACT_MASK_MASK   |
1451 		       MAC_PHYCFG2_QUAL_MASK_MASK |
1452 		       MAC_PHYCFG2_INBAND_ENABLE;
1453 
1454 	tw32(MAC_PHYCFG2, val);
1455 
1456 	val = tr32(MAC_PHYCFG1);
1457 	val &= ~(MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK |
1458 		 MAC_PHYCFG1_RGMII_EXT_RX_DEC | MAC_PHYCFG1_RGMII_SND_STAT_EN);
1459 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1460 		if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1461 			val |= MAC_PHYCFG1_RGMII_EXT_RX_DEC;
1462 		if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1463 			val |= MAC_PHYCFG1_RGMII_SND_STAT_EN;
1464 	}
1465 	val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT |
1466 	       MAC_PHYCFG1_RGMII_INT | MAC_PHYCFG1_TXC_DRV;
1467 	tw32(MAC_PHYCFG1, val);
1468 
1469 	val = tr32(MAC_EXT_RGMII_MODE);
1470 	val &= ~(MAC_RGMII_MODE_RX_INT_B |
1471 		 MAC_RGMII_MODE_RX_QUALITY |
1472 		 MAC_RGMII_MODE_RX_ACTIVITY |
1473 		 MAC_RGMII_MODE_RX_ENG_DET |
1474 		 MAC_RGMII_MODE_TX_ENABLE |
1475 		 MAC_RGMII_MODE_TX_LOWPWR |
1476 		 MAC_RGMII_MODE_TX_RESET);
1477 	if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1478 		if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1479 			val |= MAC_RGMII_MODE_RX_INT_B |
1480 			       MAC_RGMII_MODE_RX_QUALITY |
1481 			       MAC_RGMII_MODE_RX_ACTIVITY |
1482 			       MAC_RGMII_MODE_RX_ENG_DET;
1483 		if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1484 			val |= MAC_RGMII_MODE_TX_ENABLE |
1485 			       MAC_RGMII_MODE_TX_LOWPWR |
1486 			       MAC_RGMII_MODE_TX_RESET;
1487 	}
1488 	tw32(MAC_EXT_RGMII_MODE, val);
1489 }
1490 
1491 static void tg3_mdio_start(struct tg3 *tp)
1492 {
1493 	tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
1494 	tw32_f(MAC_MI_MODE, tp->mi_mode);
1495 	udelay(80);
1496 
1497 	if (tg3_flag(tp, MDIOBUS_INITED) &&
1498 	    tg3_asic_rev(tp) == ASIC_REV_5785)
1499 		tg3_mdio_config_5785(tp);
1500 }
1501 
1502 static int tg3_mdio_init(struct tg3 *tp)
1503 {
1504 	int i;
1505 	u32 reg;
1506 	struct phy_device *phydev;
1507 
1508 	if (tg3_flag(tp, 5717_PLUS)) {
1509 		u32 is_serdes;
1510 
1511 		tp->phy_addr = tp->pci_fn + 1;
1512 
1513 		if (tg3_chip_rev_id(tp) != CHIPREV_ID_5717_A0)
1514 			is_serdes = tr32(SG_DIG_STATUS) & SG_DIG_IS_SERDES;
1515 		else
1516 			is_serdes = tr32(TG3_CPMU_PHY_STRAP) &
1517 				    TG3_CPMU_PHY_STRAP_IS_SERDES;
1518 		if (is_serdes)
1519 			tp->phy_addr += 7;
1520 	} else if (tg3_flag(tp, IS_SSB_CORE) && tg3_flag(tp, ROBOSWITCH)) {
1521 		int addr;
1522 
1523 		addr = ssb_gige_get_phyaddr(tp->pdev);
1524 		if (addr < 0)
1525 			return addr;
1526 		tp->phy_addr = addr;
1527 	} else
1528 		tp->phy_addr = TG3_PHY_MII_ADDR;
1529 
1530 	tg3_mdio_start(tp);
1531 
1532 	if (!tg3_flag(tp, USE_PHYLIB) || tg3_flag(tp, MDIOBUS_INITED))
1533 		return 0;
1534 
1535 	tp->mdio_bus = mdiobus_alloc();
1536 	if (tp->mdio_bus == NULL)
1537 		return -ENOMEM;
1538 
1539 	tp->mdio_bus->name     = "tg3 mdio bus";
1540 	snprintf(tp->mdio_bus->id, MII_BUS_ID_SIZE, "%x",
1541 		 (tp->pdev->bus->number << 8) | tp->pdev->devfn);
1542 	tp->mdio_bus->priv     = tp;
1543 	tp->mdio_bus->parent   = &tp->pdev->dev;
1544 	tp->mdio_bus->read     = &tg3_mdio_read;
1545 	tp->mdio_bus->write    = &tg3_mdio_write;
1546 	tp->mdio_bus->phy_mask = ~(1 << tp->phy_addr);
1547 
1548 	/* The bus registration will look for all the PHYs on the mdio bus.
1549 	 * Unfortunately, it does not ensure the PHY is powered up before
1550 	 * accessing the PHY ID registers.  A chip reset is the
1551 	 * quickest way to bring the device back to an operational state..
1552 	 */
1553 	if (tg3_readphy(tp, MII_BMCR, &reg) || (reg & BMCR_PDOWN))
1554 		tg3_bmcr_reset(tp);
1555 
1556 	i = mdiobus_register(tp->mdio_bus);
1557 	if (i) {
1558 		dev_warn(&tp->pdev->dev, "mdiobus_reg failed (0x%x)\n", i);
1559 		mdiobus_free(tp->mdio_bus);
1560 		return i;
1561 	}
1562 
1563 	phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
1564 
1565 	if (!phydev || !phydev->drv) {
1566 		dev_warn(&tp->pdev->dev, "No PHY devices\n");
1567 		mdiobus_unregister(tp->mdio_bus);
1568 		mdiobus_free(tp->mdio_bus);
1569 		return -ENODEV;
1570 	}
1571 
1572 	switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1573 	case PHY_ID_BCM57780:
1574 		phydev->interface = PHY_INTERFACE_MODE_GMII;
1575 		phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1576 		break;
1577 	case PHY_ID_BCM50610:
1578 	case PHY_ID_BCM50610M:
1579 		phydev->dev_flags |= PHY_BRCM_CLEAR_RGMII_MODE |
1580 				     PHY_BRCM_RX_REFCLK_UNUSED |
1581 				     PHY_BRCM_DIS_TXCRXC_NOENRGY |
1582 				     PHY_BRCM_AUTO_PWRDWN_ENABLE;
1583 		fallthrough;
1584 	case PHY_ID_RTL8211C:
1585 		phydev->interface = PHY_INTERFACE_MODE_RGMII;
1586 		break;
1587 	case PHY_ID_RTL8201E:
1588 	case PHY_ID_BCMAC131:
1589 		phydev->interface = PHY_INTERFACE_MODE_MII;
1590 		phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1591 		tp->phy_flags |= TG3_PHYFLG_IS_FET;
1592 		break;
1593 	}
1594 
1595 	tg3_flag_set(tp, MDIOBUS_INITED);
1596 
1597 	if (tg3_asic_rev(tp) == ASIC_REV_5785)
1598 		tg3_mdio_config_5785(tp);
1599 
1600 	return 0;
1601 }
1602 
1603 static void tg3_mdio_fini(struct tg3 *tp)
1604 {
1605 	if (tg3_flag(tp, MDIOBUS_INITED)) {
1606 		tg3_flag_clear(tp, MDIOBUS_INITED);
1607 		mdiobus_unregister(tp->mdio_bus);
1608 		mdiobus_free(tp->mdio_bus);
1609 	}
1610 }
1611 
1612 /* tp->lock is held. */
1613 static inline void tg3_generate_fw_event(struct tg3 *tp)
1614 {
1615 	u32 val;
1616 
1617 	val = tr32(GRC_RX_CPU_EVENT);
1618 	val |= GRC_RX_CPU_DRIVER_EVENT;
1619 	tw32_f(GRC_RX_CPU_EVENT, val);
1620 
1621 	tp->last_event_jiffies = jiffies;
1622 }
1623 
1624 #define TG3_FW_EVENT_TIMEOUT_USEC 2500
1625 
1626 /* tp->lock is held. */
1627 static void tg3_wait_for_event_ack(struct tg3 *tp)
1628 {
1629 	int i;
1630 	unsigned int delay_cnt;
1631 	long time_remain;
1632 
1633 	/* If enough time has passed, no wait is necessary. */
1634 	time_remain = (long)(tp->last_event_jiffies + 1 +
1635 		      usecs_to_jiffies(TG3_FW_EVENT_TIMEOUT_USEC)) -
1636 		      (long)jiffies;
1637 	if (time_remain < 0)
1638 		return;
1639 
1640 	/* Check if we can shorten the wait time. */
1641 	delay_cnt = jiffies_to_usecs(time_remain);
1642 	if (delay_cnt > TG3_FW_EVENT_TIMEOUT_USEC)
1643 		delay_cnt = TG3_FW_EVENT_TIMEOUT_USEC;
1644 	delay_cnt = (delay_cnt >> 3) + 1;
1645 
1646 	for (i = 0; i < delay_cnt; i++) {
1647 		if (!(tr32(GRC_RX_CPU_EVENT) & GRC_RX_CPU_DRIVER_EVENT))
1648 			break;
1649 		if (pci_channel_offline(tp->pdev))
1650 			break;
1651 
1652 		udelay(8);
1653 	}
1654 }
1655 
1656 /* tp->lock is held. */
1657 static void tg3_phy_gather_ump_data(struct tg3 *tp, u32 *data)
1658 {
1659 	u32 reg, val;
1660 
1661 	val = 0;
1662 	if (!tg3_readphy(tp, MII_BMCR, &reg))
1663 		val = reg << 16;
1664 	if (!tg3_readphy(tp, MII_BMSR, &reg))
1665 		val |= (reg & 0xffff);
1666 	*data++ = val;
1667 
1668 	val = 0;
1669 	if (!tg3_readphy(tp, MII_ADVERTISE, &reg))
1670 		val = reg << 16;
1671 	if (!tg3_readphy(tp, MII_LPA, &reg))
1672 		val |= (reg & 0xffff);
1673 	*data++ = val;
1674 
1675 	val = 0;
1676 	if (!(tp->phy_flags & TG3_PHYFLG_MII_SERDES)) {
1677 		if (!tg3_readphy(tp, MII_CTRL1000, &reg))
1678 			val = reg << 16;
1679 		if (!tg3_readphy(tp, MII_STAT1000, &reg))
1680 			val |= (reg & 0xffff);
1681 	}
1682 	*data++ = val;
1683 
1684 	if (!tg3_readphy(tp, MII_PHYADDR, &reg))
1685 		val = reg << 16;
1686 	else
1687 		val = 0;
1688 	*data++ = val;
1689 }
1690 
1691 /* tp->lock is held. */
1692 static void tg3_ump_link_report(struct tg3 *tp)
1693 {
1694 	u32 data[4];
1695 
1696 	if (!tg3_flag(tp, 5780_CLASS) || !tg3_flag(tp, ENABLE_ASF))
1697 		return;
1698 
1699 	tg3_phy_gather_ump_data(tp, data);
1700 
1701 	tg3_wait_for_event_ack(tp);
1702 
1703 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_LINK_UPDATE);
1704 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 14);
1705 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x0, data[0]);
1706 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x4, data[1]);
1707 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0x8, data[2]);
1708 	tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 0xc, data[3]);
1709 
1710 	tg3_generate_fw_event(tp);
1711 }
1712 
1713 /* tp->lock is held. */
1714 static void tg3_stop_fw(struct tg3 *tp)
1715 {
1716 	if (tg3_flag(tp, ENABLE_ASF) && !tg3_flag(tp, ENABLE_APE)) {
1717 		/* Wait for RX cpu to ACK the previous event. */
1718 		tg3_wait_for_event_ack(tp);
1719 
1720 		tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
1721 
1722 		tg3_generate_fw_event(tp);
1723 
1724 		/* Wait for RX cpu to ACK this event. */
1725 		tg3_wait_for_event_ack(tp);
1726 	}
1727 }
1728 
1729 /* tp->lock is held. */
1730 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
1731 {
1732 	tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
1733 		      NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
1734 
1735 	if (tg3_flag(tp, ASF_NEW_HANDSHAKE)) {
1736 		switch (kind) {
1737 		case RESET_KIND_INIT:
1738 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1739 				      DRV_STATE_START);
1740 			break;
1741 
1742 		case RESET_KIND_SHUTDOWN:
1743 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1744 				      DRV_STATE_UNLOAD);
1745 			break;
1746 
1747 		case RESET_KIND_SUSPEND:
1748 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1749 				      DRV_STATE_SUSPEND);
1750 			break;
1751 
1752 		default:
1753 			break;
1754 		}
1755 	}
1756 }
1757 
1758 /* tp->lock is held. */
1759 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
1760 {
1761 	if (tg3_flag(tp, ASF_NEW_HANDSHAKE)) {
1762 		switch (kind) {
1763 		case RESET_KIND_INIT:
1764 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1765 				      DRV_STATE_START_DONE);
1766 			break;
1767 
1768 		case RESET_KIND_SHUTDOWN:
1769 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1770 				      DRV_STATE_UNLOAD_DONE);
1771 			break;
1772 
1773 		default:
1774 			break;
1775 		}
1776 	}
1777 }
1778 
1779 /* tp->lock is held. */
1780 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
1781 {
1782 	if (tg3_flag(tp, ENABLE_ASF)) {
1783 		switch (kind) {
1784 		case RESET_KIND_INIT:
1785 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1786 				      DRV_STATE_START);
1787 			break;
1788 
1789 		case RESET_KIND_SHUTDOWN:
1790 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1791 				      DRV_STATE_UNLOAD);
1792 			break;
1793 
1794 		case RESET_KIND_SUSPEND:
1795 			tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
1796 				      DRV_STATE_SUSPEND);
1797 			break;
1798 
1799 		default:
1800 			break;
1801 		}
1802 	}
1803 }
1804 
1805 static int tg3_poll_fw(struct tg3 *tp)
1806 {
1807 	int i;
1808 	u32 val;
1809 
1810 	if (tg3_flag(tp, NO_FWARE_REPORTED))
1811 		return 0;
1812 
1813 	if (tg3_flag(tp, IS_SSB_CORE)) {
1814 		/* We don't use firmware. */
1815 		return 0;
1816 	}
1817 
1818 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
1819 		/* Wait up to 20ms for init done. */
1820 		for (i = 0; i < 200; i++) {
1821 			if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
1822 				return 0;
1823 			if (pci_channel_offline(tp->pdev))
1824 				return -ENODEV;
1825 
1826 			udelay(100);
1827 		}
1828 		return -ENODEV;
1829 	}
1830 
1831 	/* Wait for firmware initialization to complete. */
1832 	for (i = 0; i < 100000; i++) {
1833 		tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
1834 		if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1835 			break;
1836 		if (pci_channel_offline(tp->pdev)) {
1837 			if (!tg3_flag(tp, NO_FWARE_REPORTED)) {
1838 				tg3_flag_set(tp, NO_FWARE_REPORTED);
1839 				netdev_info(tp->dev, "No firmware running\n");
1840 			}
1841 
1842 			break;
1843 		}
1844 
1845 		udelay(10);
1846 	}
1847 
1848 	/* Chip might not be fitted with firmware.  Some Sun onboard
1849 	 * parts are configured like that.  So don't signal the timeout
1850 	 * of the above loop as an error, but do report the lack of
1851 	 * running firmware once.
1852 	 */
1853 	if (i >= 100000 && !tg3_flag(tp, NO_FWARE_REPORTED)) {
1854 		tg3_flag_set(tp, NO_FWARE_REPORTED);
1855 
1856 		netdev_info(tp->dev, "No firmware running\n");
1857 	}
1858 
1859 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0) {
1860 		/* The 57765 A0 needs a little more
1861 		 * time to do some important work.
1862 		 */
1863 		mdelay(10);
1864 	}
1865 
1866 	return 0;
1867 }
1868 
1869 static void tg3_link_report(struct tg3 *tp)
1870 {
1871 	if (!netif_carrier_ok(tp->dev)) {
1872 		netif_info(tp, link, tp->dev, "Link is down\n");
1873 		tg3_ump_link_report(tp);
1874 	} else if (netif_msg_link(tp)) {
1875 		netdev_info(tp->dev, "Link is up at %d Mbps, %s duplex\n",
1876 			    (tp->link_config.active_speed == SPEED_1000 ?
1877 			     1000 :
1878 			     (tp->link_config.active_speed == SPEED_100 ?
1879 			      100 : 10)),
1880 			    (tp->link_config.active_duplex == DUPLEX_FULL ?
1881 			     "full" : "half"));
1882 
1883 		netdev_info(tp->dev, "Flow control is %s for TX and %s for RX\n",
1884 			    (tp->link_config.active_flowctrl & FLOW_CTRL_TX) ?
1885 			    "on" : "off",
1886 			    (tp->link_config.active_flowctrl & FLOW_CTRL_RX) ?
1887 			    "on" : "off");
1888 
1889 		if (tp->phy_flags & TG3_PHYFLG_EEE_CAP)
1890 			netdev_info(tp->dev, "EEE is %s\n",
1891 				    tp->setlpicnt ? "enabled" : "disabled");
1892 
1893 		tg3_ump_link_report(tp);
1894 	}
1895 
1896 	tp->link_up = netif_carrier_ok(tp->dev);
1897 }
1898 
1899 static u32 tg3_decode_flowctrl_1000T(u32 adv)
1900 {
1901 	u32 flowctrl = 0;
1902 
1903 	if (adv & ADVERTISE_PAUSE_CAP) {
1904 		flowctrl |= FLOW_CTRL_RX;
1905 		if (!(adv & ADVERTISE_PAUSE_ASYM))
1906 			flowctrl |= FLOW_CTRL_TX;
1907 	} else if (adv & ADVERTISE_PAUSE_ASYM)
1908 		flowctrl |= FLOW_CTRL_TX;
1909 
1910 	return flowctrl;
1911 }
1912 
1913 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1914 {
1915 	u16 miireg;
1916 
1917 	if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1918 		miireg = ADVERTISE_1000XPAUSE;
1919 	else if (flow_ctrl & FLOW_CTRL_TX)
1920 		miireg = ADVERTISE_1000XPSE_ASYM;
1921 	else if (flow_ctrl & FLOW_CTRL_RX)
1922 		miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1923 	else
1924 		miireg = 0;
1925 
1926 	return miireg;
1927 }
1928 
1929 static u32 tg3_decode_flowctrl_1000X(u32 adv)
1930 {
1931 	u32 flowctrl = 0;
1932 
1933 	if (adv & ADVERTISE_1000XPAUSE) {
1934 		flowctrl |= FLOW_CTRL_RX;
1935 		if (!(adv & ADVERTISE_1000XPSE_ASYM))
1936 			flowctrl |= FLOW_CTRL_TX;
1937 	} else if (adv & ADVERTISE_1000XPSE_ASYM)
1938 		flowctrl |= FLOW_CTRL_TX;
1939 
1940 	return flowctrl;
1941 }
1942 
1943 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1944 {
1945 	u8 cap = 0;
1946 
1947 	if (lcladv & rmtadv & ADVERTISE_1000XPAUSE) {
1948 		cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1949 	} else if (lcladv & rmtadv & ADVERTISE_1000XPSE_ASYM) {
1950 		if (lcladv & ADVERTISE_1000XPAUSE)
1951 			cap = FLOW_CTRL_RX;
1952 		if (rmtadv & ADVERTISE_1000XPAUSE)
1953 			cap = FLOW_CTRL_TX;
1954 	}
1955 
1956 	return cap;
1957 }
1958 
1959 static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv)
1960 {
1961 	u8 autoneg;
1962 	u8 flowctrl = 0;
1963 	u32 old_rx_mode = tp->rx_mode;
1964 	u32 old_tx_mode = tp->tx_mode;
1965 
1966 	if (tg3_flag(tp, USE_PHYLIB))
1967 		autoneg = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr)->autoneg;
1968 	else
1969 		autoneg = tp->link_config.autoneg;
1970 
1971 	if (autoneg == AUTONEG_ENABLE && tg3_flag(tp, PAUSE_AUTONEG)) {
1972 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
1973 			flowctrl = tg3_resolve_flowctrl_1000X(lcladv, rmtadv);
1974 		else
1975 			flowctrl = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
1976 	} else
1977 		flowctrl = tp->link_config.flowctrl;
1978 
1979 	tp->link_config.active_flowctrl = flowctrl;
1980 
1981 	if (flowctrl & FLOW_CTRL_RX)
1982 		tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1983 	else
1984 		tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1985 
1986 	if (old_rx_mode != tp->rx_mode)
1987 		tw32_f(MAC_RX_MODE, tp->rx_mode);
1988 
1989 	if (flowctrl & FLOW_CTRL_TX)
1990 		tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1991 	else
1992 		tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1993 
1994 	if (old_tx_mode != tp->tx_mode)
1995 		tw32_f(MAC_TX_MODE, tp->tx_mode);
1996 }
1997 
1998 static void tg3_adjust_link(struct net_device *dev)
1999 {
2000 	u8 oldflowctrl, linkmesg = 0;
2001 	u32 mac_mode, lcl_adv, rmt_adv;
2002 	struct tg3 *tp = netdev_priv(dev);
2003 	struct phy_device *phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
2004 
2005 	spin_lock_bh(&tp->lock);
2006 
2007 	mac_mode = tp->mac_mode & ~(MAC_MODE_PORT_MODE_MASK |
2008 				    MAC_MODE_HALF_DUPLEX);
2009 
2010 	oldflowctrl = tp->link_config.active_flowctrl;
2011 
2012 	if (phydev->link) {
2013 		lcl_adv = 0;
2014 		rmt_adv = 0;
2015 
2016 		if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10)
2017 			mac_mode |= MAC_MODE_PORT_MODE_MII;
2018 		else if (phydev->speed == SPEED_1000 ||
2019 			 tg3_asic_rev(tp) != ASIC_REV_5785)
2020 			mac_mode |= MAC_MODE_PORT_MODE_GMII;
2021 		else
2022 			mac_mode |= MAC_MODE_PORT_MODE_MII;
2023 
2024 		if (phydev->duplex == DUPLEX_HALF)
2025 			mac_mode |= MAC_MODE_HALF_DUPLEX;
2026 		else {
2027 			lcl_adv = mii_advertise_flowctrl(
2028 				  tp->link_config.flowctrl);
2029 
2030 			if (phydev->pause)
2031 				rmt_adv = LPA_PAUSE_CAP;
2032 			if (phydev->asym_pause)
2033 				rmt_adv |= LPA_PAUSE_ASYM;
2034 		}
2035 
2036 		tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
2037 	} else
2038 		mac_mode |= MAC_MODE_PORT_MODE_GMII;
2039 
2040 	if (mac_mode != tp->mac_mode) {
2041 		tp->mac_mode = mac_mode;
2042 		tw32_f(MAC_MODE, tp->mac_mode);
2043 		udelay(40);
2044 	}
2045 
2046 	if (tg3_asic_rev(tp) == ASIC_REV_5785) {
2047 		if (phydev->speed == SPEED_10)
2048 			tw32(MAC_MI_STAT,
2049 			     MAC_MI_STAT_10MBPS_MODE |
2050 			     MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
2051 		else
2052 			tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
2053 	}
2054 
2055 	if (phydev->speed == SPEED_1000 && phydev->duplex == DUPLEX_HALF)
2056 		tw32(MAC_TX_LENGTHS,
2057 		     ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2058 		      (6 << TX_LENGTHS_IPG_SHIFT) |
2059 		      (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
2060 	else
2061 		tw32(MAC_TX_LENGTHS,
2062 		     ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2063 		      (6 << TX_LENGTHS_IPG_SHIFT) |
2064 		      (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
2065 
2066 	if (phydev->link != tp->old_link ||
2067 	    phydev->speed != tp->link_config.active_speed ||
2068 	    phydev->duplex != tp->link_config.active_duplex ||
2069 	    oldflowctrl != tp->link_config.active_flowctrl)
2070 		linkmesg = 1;
2071 
2072 	tp->old_link = phydev->link;
2073 	tp->link_config.active_speed = phydev->speed;
2074 	tp->link_config.active_duplex = phydev->duplex;
2075 
2076 	spin_unlock_bh(&tp->lock);
2077 
2078 	if (linkmesg)
2079 		tg3_link_report(tp);
2080 }
2081 
2082 static int tg3_phy_init(struct tg3 *tp)
2083 {
2084 	struct phy_device *phydev;
2085 
2086 	if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED)
2087 		return 0;
2088 
2089 	/* Bring the PHY back to a known state. */
2090 	tg3_bmcr_reset(tp);
2091 
2092 	phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
2093 
2094 	/* Attach the MAC to the PHY. */
2095 	phydev = phy_connect(tp->dev, phydev_name(phydev),
2096 			     tg3_adjust_link, phydev->interface);
2097 	if (IS_ERR(phydev)) {
2098 		dev_err(&tp->pdev->dev, "Could not attach to PHY\n");
2099 		return PTR_ERR(phydev);
2100 	}
2101 
2102 	/* Mask with MAC supported features. */
2103 	switch (phydev->interface) {
2104 	case PHY_INTERFACE_MODE_GMII:
2105 	case PHY_INTERFACE_MODE_RGMII:
2106 		if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
2107 			phy_set_max_speed(phydev, SPEED_1000);
2108 			phy_support_asym_pause(phydev);
2109 			break;
2110 		}
2111 		fallthrough;
2112 	case PHY_INTERFACE_MODE_MII:
2113 		phy_set_max_speed(phydev, SPEED_100);
2114 		phy_support_asym_pause(phydev);
2115 		break;
2116 	default:
2117 		phy_disconnect(mdiobus_get_phy(tp->mdio_bus, tp->phy_addr));
2118 		return -EINVAL;
2119 	}
2120 
2121 	tp->phy_flags |= TG3_PHYFLG_IS_CONNECTED;
2122 
2123 	phy_attached_info(phydev);
2124 
2125 	return 0;
2126 }
2127 
2128 static void tg3_phy_start(struct tg3 *tp)
2129 {
2130 	struct phy_device *phydev;
2131 
2132 	if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
2133 		return;
2134 
2135 	phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
2136 
2137 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
2138 		tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
2139 		phydev->speed = tp->link_config.speed;
2140 		phydev->duplex = tp->link_config.duplex;
2141 		phydev->autoneg = tp->link_config.autoneg;
2142 		ethtool_convert_legacy_u32_to_link_mode(
2143 			phydev->advertising, tp->link_config.advertising);
2144 	}
2145 
2146 	phy_start(phydev);
2147 
2148 	phy_start_aneg(phydev);
2149 }
2150 
2151 static void tg3_phy_stop(struct tg3 *tp)
2152 {
2153 	if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
2154 		return;
2155 
2156 	phy_stop(mdiobus_get_phy(tp->mdio_bus, tp->phy_addr));
2157 }
2158 
2159 static void tg3_phy_fini(struct tg3 *tp)
2160 {
2161 	if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
2162 		phy_disconnect(mdiobus_get_phy(tp->mdio_bus, tp->phy_addr));
2163 		tp->phy_flags &= ~TG3_PHYFLG_IS_CONNECTED;
2164 	}
2165 }
2166 
2167 static int tg3_phy_set_extloopbk(struct tg3 *tp)
2168 {
2169 	int err;
2170 	u32 val;
2171 
2172 	if (tp->phy_flags & TG3_PHYFLG_IS_FET)
2173 		return 0;
2174 
2175 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
2176 		/* Cannot do read-modify-write on 5401 */
2177 		err = tg3_phy_auxctl_write(tp,
2178 					   MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
2179 					   MII_TG3_AUXCTL_ACTL_EXTLOOPBK |
2180 					   0x4c20);
2181 		goto done;
2182 	}
2183 
2184 	err = tg3_phy_auxctl_read(tp,
2185 				  MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
2186 	if (err)
2187 		return err;
2188 
2189 	val |= MII_TG3_AUXCTL_ACTL_EXTLOOPBK;
2190 	err = tg3_phy_auxctl_write(tp,
2191 				   MII_TG3_AUXCTL_SHDWSEL_AUXCTL, val);
2192 
2193 done:
2194 	return err;
2195 }
2196 
2197 static void tg3_phy_fet_toggle_apd(struct tg3 *tp, bool enable)
2198 {
2199 	u32 phytest;
2200 
2201 	if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
2202 		u32 phy;
2203 
2204 		tg3_writephy(tp, MII_TG3_FET_TEST,
2205 			     phytest | MII_TG3_FET_SHADOW_EN);
2206 		if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXSTAT2, &phy)) {
2207 			if (enable)
2208 				phy |= MII_TG3_FET_SHDW_AUXSTAT2_APD;
2209 			else
2210 				phy &= ~MII_TG3_FET_SHDW_AUXSTAT2_APD;
2211 			tg3_writephy(tp, MII_TG3_FET_SHDW_AUXSTAT2, phy);
2212 		}
2213 		tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
2214 	}
2215 }
2216 
2217 static void tg3_phy_toggle_apd(struct tg3 *tp, bool enable)
2218 {
2219 	u32 reg;
2220 
2221 	if (!tg3_flag(tp, 5705_PLUS) ||
2222 	    (tg3_flag(tp, 5717_PLUS) &&
2223 	     (tp->phy_flags & TG3_PHYFLG_MII_SERDES)))
2224 		return;
2225 
2226 	if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2227 		tg3_phy_fet_toggle_apd(tp, enable);
2228 		return;
2229 	}
2230 
2231 	reg = MII_TG3_MISC_SHDW_SCR5_LPED |
2232 	      MII_TG3_MISC_SHDW_SCR5_DLPTLM |
2233 	      MII_TG3_MISC_SHDW_SCR5_SDTL |
2234 	      MII_TG3_MISC_SHDW_SCR5_C125OE;
2235 	if (tg3_asic_rev(tp) != ASIC_REV_5784 || !enable)
2236 		reg |= MII_TG3_MISC_SHDW_SCR5_DLLAPD;
2237 
2238 	tg3_phy_shdw_write(tp, MII_TG3_MISC_SHDW_SCR5_SEL, reg);
2239 
2240 
2241 	reg = MII_TG3_MISC_SHDW_APD_WKTM_84MS;
2242 	if (enable)
2243 		reg |= MII_TG3_MISC_SHDW_APD_ENABLE;
2244 
2245 	tg3_phy_shdw_write(tp, MII_TG3_MISC_SHDW_APD_SEL, reg);
2246 }
2247 
2248 static void tg3_phy_toggle_automdix(struct tg3 *tp, bool enable)
2249 {
2250 	u32 phy;
2251 
2252 	if (!tg3_flag(tp, 5705_PLUS) ||
2253 	    (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
2254 		return;
2255 
2256 	if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2257 		u32 ephy;
2258 
2259 		if (!tg3_readphy(tp, MII_TG3_FET_TEST, &ephy)) {
2260 			u32 reg = MII_TG3_FET_SHDW_MISCCTRL;
2261 
2262 			tg3_writephy(tp, MII_TG3_FET_TEST,
2263 				     ephy | MII_TG3_FET_SHADOW_EN);
2264 			if (!tg3_readphy(tp, reg, &phy)) {
2265 				if (enable)
2266 					phy |= MII_TG3_FET_SHDW_MISCCTRL_MDIX;
2267 				else
2268 					phy &= ~MII_TG3_FET_SHDW_MISCCTRL_MDIX;
2269 				tg3_writephy(tp, reg, phy);
2270 			}
2271 			tg3_writephy(tp, MII_TG3_FET_TEST, ephy);
2272 		}
2273 	} else {
2274 		int ret;
2275 
2276 		ret = tg3_phy_auxctl_read(tp,
2277 					  MII_TG3_AUXCTL_SHDWSEL_MISC, &phy);
2278 		if (!ret) {
2279 			if (enable)
2280 				phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
2281 			else
2282 				phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
2283 			tg3_phy_auxctl_write(tp,
2284 					     MII_TG3_AUXCTL_SHDWSEL_MISC, phy);
2285 		}
2286 	}
2287 }
2288 
2289 static void tg3_phy_set_wirespeed(struct tg3 *tp)
2290 {
2291 	int ret;
2292 	u32 val;
2293 
2294 	if (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED)
2295 		return;
2296 
2297 	ret = tg3_phy_auxctl_read(tp, MII_TG3_AUXCTL_SHDWSEL_MISC, &val);
2298 	if (!ret)
2299 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_MISC,
2300 				     val | MII_TG3_AUXCTL_MISC_WIRESPD_EN);
2301 }
2302 
2303 static void tg3_phy_apply_otp(struct tg3 *tp)
2304 {
2305 	u32 otp, phy;
2306 
2307 	if (!tp->phy_otp)
2308 		return;
2309 
2310 	otp = tp->phy_otp;
2311 
2312 	if (tg3_phy_toggle_auxctl_smdsp(tp, true))
2313 		return;
2314 
2315 	phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
2316 	phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
2317 	tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
2318 
2319 	phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
2320 	      ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
2321 	tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
2322 
2323 	phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
2324 	phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
2325 	tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
2326 
2327 	phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
2328 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
2329 
2330 	phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
2331 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
2332 
2333 	phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
2334 	      ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
2335 	tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
2336 
2337 	tg3_phy_toggle_auxctl_smdsp(tp, false);
2338 }
2339 
2340 static void tg3_eee_pull_config(struct tg3 *tp, struct ethtool_eee *eee)
2341 {
2342 	u32 val;
2343 	struct ethtool_eee *dest = &tp->eee;
2344 
2345 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
2346 		return;
2347 
2348 	if (eee)
2349 		dest = eee;
2350 
2351 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, TG3_CL45_D7_EEERES_STAT, &val))
2352 		return;
2353 
2354 	/* Pull eee_active */
2355 	if (val == TG3_CL45_D7_EEERES_STAT_LP_1000T ||
2356 	    val == TG3_CL45_D7_EEERES_STAT_LP_100TX) {
2357 		dest->eee_active = 1;
2358 	} else
2359 		dest->eee_active = 0;
2360 
2361 	/* Pull lp advertised settings */
2362 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, MDIO_AN_EEE_LPABLE, &val))
2363 		return;
2364 	dest->lp_advertised = mmd_eee_adv_to_ethtool_adv_t(val);
2365 
2366 	/* Pull advertised and eee_enabled settings */
2367 	if (tg3_phy_cl45_read(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, &val))
2368 		return;
2369 	dest->eee_enabled = !!val;
2370 	dest->advertised = mmd_eee_adv_to_ethtool_adv_t(val);
2371 
2372 	/* Pull tx_lpi_enabled */
2373 	val = tr32(TG3_CPMU_EEE_MODE);
2374 	dest->tx_lpi_enabled = !!(val & TG3_CPMU_EEEMD_LPI_IN_TX);
2375 
2376 	/* Pull lpi timer value */
2377 	dest->tx_lpi_timer = tr32(TG3_CPMU_EEE_DBTMR1) & 0xffff;
2378 }
2379 
2380 static void tg3_phy_eee_adjust(struct tg3 *tp, bool current_link_up)
2381 {
2382 	u32 val;
2383 
2384 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
2385 		return;
2386 
2387 	tp->setlpicnt = 0;
2388 
2389 	if (tp->link_config.autoneg == AUTONEG_ENABLE &&
2390 	    current_link_up &&
2391 	    tp->link_config.active_duplex == DUPLEX_FULL &&
2392 	    (tp->link_config.active_speed == SPEED_100 ||
2393 	     tp->link_config.active_speed == SPEED_1000)) {
2394 		u32 eeectl;
2395 
2396 		if (tp->link_config.active_speed == SPEED_1000)
2397 			eeectl = TG3_CPMU_EEE_CTRL_EXIT_16_5_US;
2398 		else
2399 			eeectl = TG3_CPMU_EEE_CTRL_EXIT_36_US;
2400 
2401 		tw32(TG3_CPMU_EEE_CTRL, eeectl);
2402 
2403 		tg3_eee_pull_config(tp, NULL);
2404 		if (tp->eee.eee_active)
2405 			tp->setlpicnt = 2;
2406 	}
2407 
2408 	if (!tp->setlpicnt) {
2409 		if (current_link_up &&
2410 		   !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2411 			tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, 0x0000);
2412 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2413 		}
2414 
2415 		val = tr32(TG3_CPMU_EEE_MODE);
2416 		tw32(TG3_CPMU_EEE_MODE, val & ~TG3_CPMU_EEEMD_LPI_ENABLE);
2417 	}
2418 }
2419 
2420 static void tg3_phy_eee_enable(struct tg3 *tp)
2421 {
2422 	u32 val;
2423 
2424 	if (tp->link_config.active_speed == SPEED_1000 &&
2425 	    (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2426 	     tg3_asic_rev(tp) == ASIC_REV_5719 ||
2427 	     tg3_flag(tp, 57765_CLASS)) &&
2428 	    !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2429 		val = MII_TG3_DSP_TAP26_ALNOKO |
2430 		      MII_TG3_DSP_TAP26_RMRXSTO;
2431 		tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
2432 		tg3_phy_toggle_auxctl_smdsp(tp, false);
2433 	}
2434 
2435 	val = tr32(TG3_CPMU_EEE_MODE);
2436 	tw32(TG3_CPMU_EEE_MODE, val | TG3_CPMU_EEEMD_LPI_ENABLE);
2437 }
2438 
2439 static int tg3_wait_macro_done(struct tg3 *tp)
2440 {
2441 	int limit = 100;
2442 
2443 	while (limit--) {
2444 		u32 tmp32;
2445 
2446 		if (!tg3_readphy(tp, MII_TG3_DSP_CONTROL, &tmp32)) {
2447 			if ((tmp32 & 0x1000) == 0)
2448 				break;
2449 		}
2450 	}
2451 	if (limit < 0)
2452 		return -EBUSY;
2453 
2454 	return 0;
2455 }
2456 
2457 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
2458 {
2459 	static const u32 test_pat[4][6] = {
2460 	{ 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
2461 	{ 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
2462 	{ 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
2463 	{ 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
2464 	};
2465 	int chan;
2466 
2467 	for (chan = 0; chan < 4; chan++) {
2468 		int i;
2469 
2470 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2471 			     (chan * 0x2000) | 0x0200);
2472 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
2473 
2474 		for (i = 0; i < 6; i++)
2475 			tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
2476 				     test_pat[chan][i]);
2477 
2478 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
2479 		if (tg3_wait_macro_done(tp)) {
2480 			*resetp = 1;
2481 			return -EBUSY;
2482 		}
2483 
2484 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2485 			     (chan * 0x2000) | 0x0200);
2486 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0082);
2487 		if (tg3_wait_macro_done(tp)) {
2488 			*resetp = 1;
2489 			return -EBUSY;
2490 		}
2491 
2492 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0802);
2493 		if (tg3_wait_macro_done(tp)) {
2494 			*resetp = 1;
2495 			return -EBUSY;
2496 		}
2497 
2498 		for (i = 0; i < 6; i += 2) {
2499 			u32 low, high;
2500 
2501 			if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
2502 			    tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
2503 			    tg3_wait_macro_done(tp)) {
2504 				*resetp = 1;
2505 				return -EBUSY;
2506 			}
2507 			low &= 0x7fff;
2508 			high &= 0x000f;
2509 			if (low != test_pat[chan][i] ||
2510 			    high != test_pat[chan][i+1]) {
2511 				tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
2512 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
2513 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
2514 
2515 				return -EBUSY;
2516 			}
2517 		}
2518 	}
2519 
2520 	return 0;
2521 }
2522 
2523 static int tg3_phy_reset_chanpat(struct tg3 *tp)
2524 {
2525 	int chan;
2526 
2527 	for (chan = 0; chan < 4; chan++) {
2528 		int i;
2529 
2530 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
2531 			     (chan * 0x2000) | 0x0200);
2532 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
2533 		for (i = 0; i < 6; i++)
2534 			tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
2535 		tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
2536 		if (tg3_wait_macro_done(tp))
2537 			return -EBUSY;
2538 	}
2539 
2540 	return 0;
2541 }
2542 
2543 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
2544 {
2545 	u32 reg32, phy9_orig;
2546 	int retries, do_phy_reset, err;
2547 
2548 	retries = 10;
2549 	do_phy_reset = 1;
2550 	do {
2551 		if (do_phy_reset) {
2552 			err = tg3_bmcr_reset(tp);
2553 			if (err)
2554 				return err;
2555 			do_phy_reset = 0;
2556 		}
2557 
2558 		/* Disable transmitter and interrupt.  */
2559 		if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
2560 			continue;
2561 
2562 		reg32 |= 0x3000;
2563 		tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2564 
2565 		/* Set full-duplex, 1000 mbps.  */
2566 		tg3_writephy(tp, MII_BMCR,
2567 			     BMCR_FULLDPLX | BMCR_SPEED1000);
2568 
2569 		/* Set to master mode.  */
2570 		if (tg3_readphy(tp, MII_CTRL1000, &phy9_orig))
2571 			continue;
2572 
2573 		tg3_writephy(tp, MII_CTRL1000,
2574 			     CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER);
2575 
2576 		err = tg3_phy_toggle_auxctl_smdsp(tp, true);
2577 		if (err)
2578 			return err;
2579 
2580 		/* Block the PHY control access.  */
2581 		tg3_phydsp_write(tp, 0x8005, 0x0800);
2582 
2583 		err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
2584 		if (!err)
2585 			break;
2586 	} while (--retries);
2587 
2588 	err = tg3_phy_reset_chanpat(tp);
2589 	if (err)
2590 		return err;
2591 
2592 	tg3_phydsp_write(tp, 0x8005, 0x0000);
2593 
2594 	tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
2595 	tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0000);
2596 
2597 	tg3_phy_toggle_auxctl_smdsp(tp, false);
2598 
2599 	tg3_writephy(tp, MII_CTRL1000, phy9_orig);
2600 
2601 	err = tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32);
2602 	if (err)
2603 		return err;
2604 
2605 	reg32 &= ~0x3000;
2606 	tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2607 
2608 	return 0;
2609 }
2610 
2611 static void tg3_carrier_off(struct tg3 *tp)
2612 {
2613 	netif_carrier_off(tp->dev);
2614 	tp->link_up = false;
2615 }
2616 
2617 static void tg3_warn_mgmt_link_flap(struct tg3 *tp)
2618 {
2619 	if (tg3_flag(tp, ENABLE_ASF))
2620 		netdev_warn(tp->dev,
2621 			    "Management side-band traffic will be interrupted during phy settings change\n");
2622 }
2623 
2624 /* This will reset the tigon3 PHY if there is no valid
2625  * link unless the FORCE argument is non-zero.
2626  */
2627 static int tg3_phy_reset(struct tg3 *tp)
2628 {
2629 	u32 val, cpmuctrl;
2630 	int err;
2631 
2632 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
2633 		val = tr32(GRC_MISC_CFG);
2634 		tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
2635 		udelay(40);
2636 	}
2637 	err  = tg3_readphy(tp, MII_BMSR, &val);
2638 	err |= tg3_readphy(tp, MII_BMSR, &val);
2639 	if (err != 0)
2640 		return -EBUSY;
2641 
2642 	if (netif_running(tp->dev) && tp->link_up) {
2643 		netif_carrier_off(tp->dev);
2644 		tg3_link_report(tp);
2645 	}
2646 
2647 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
2648 	    tg3_asic_rev(tp) == ASIC_REV_5704 ||
2649 	    tg3_asic_rev(tp) == ASIC_REV_5705) {
2650 		err = tg3_phy_reset_5703_4_5(tp);
2651 		if (err)
2652 			return err;
2653 		goto out;
2654 	}
2655 
2656 	cpmuctrl = 0;
2657 	if (tg3_asic_rev(tp) == ASIC_REV_5784 &&
2658 	    tg3_chip_rev(tp) != CHIPREV_5784_AX) {
2659 		cpmuctrl = tr32(TG3_CPMU_CTRL);
2660 		if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
2661 			tw32(TG3_CPMU_CTRL,
2662 			     cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
2663 	}
2664 
2665 	err = tg3_bmcr_reset(tp);
2666 	if (err)
2667 		return err;
2668 
2669 	if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
2670 		val = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
2671 		tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, val);
2672 
2673 		tw32(TG3_CPMU_CTRL, cpmuctrl);
2674 	}
2675 
2676 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX ||
2677 	    tg3_chip_rev(tp) == CHIPREV_5761_AX) {
2678 		val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2679 		if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
2680 		    CPMU_LSPD_1000MB_MACCLK_12_5) {
2681 			val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2682 			udelay(40);
2683 			tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2684 		}
2685 	}
2686 
2687 	if (tg3_flag(tp, 5717_PLUS) &&
2688 	    (tp->phy_flags & TG3_PHYFLG_MII_SERDES))
2689 		return 0;
2690 
2691 	tg3_phy_apply_otp(tp);
2692 
2693 	if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
2694 		tg3_phy_toggle_apd(tp, true);
2695 	else
2696 		tg3_phy_toggle_apd(tp, false);
2697 
2698 out:
2699 	if ((tp->phy_flags & TG3_PHYFLG_ADC_BUG) &&
2700 	    !tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2701 		tg3_phydsp_write(tp, 0x201f, 0x2aaa);
2702 		tg3_phydsp_write(tp, 0x000a, 0x0323);
2703 		tg3_phy_toggle_auxctl_smdsp(tp, false);
2704 	}
2705 
2706 	if (tp->phy_flags & TG3_PHYFLG_5704_A0_BUG) {
2707 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2708 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2709 	}
2710 
2711 	if (tp->phy_flags & TG3_PHYFLG_BER_BUG) {
2712 		if (!tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2713 			tg3_phydsp_write(tp, 0x000a, 0x310b);
2714 			tg3_phydsp_write(tp, 0x201f, 0x9506);
2715 			tg3_phydsp_write(tp, 0x401f, 0x14e2);
2716 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2717 		}
2718 	} else if (tp->phy_flags & TG3_PHYFLG_JITTER_BUG) {
2719 		if (!tg3_phy_toggle_auxctl_smdsp(tp, true)) {
2720 			tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
2721 			if (tp->phy_flags & TG3_PHYFLG_ADJUST_TRIM) {
2722 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
2723 				tg3_writephy(tp, MII_TG3_TEST1,
2724 					     MII_TG3_TEST1_TRIM_EN | 0x4);
2725 			} else
2726 				tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
2727 
2728 			tg3_phy_toggle_auxctl_smdsp(tp, false);
2729 		}
2730 	}
2731 
2732 	/* Set Extended packet length bit (bit 14) on all chips that */
2733 	/* support jumbo frames */
2734 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
2735 		/* Cannot do read-modify-write on 5401 */
2736 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
2737 	} else if (tg3_flag(tp, JUMBO_CAPABLE)) {
2738 		/* Set bit 14 with read-modify-write to preserve other bits */
2739 		err = tg3_phy_auxctl_read(tp,
2740 					  MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
2741 		if (!err)
2742 			tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
2743 					   val | MII_TG3_AUXCTL_ACTL_EXTPKTLEN);
2744 	}
2745 
2746 	/* Set phy register 0x10 bit 0 to high fifo elasticity to support
2747 	 * jumbo frames transmission.
2748 	 */
2749 	if (tg3_flag(tp, JUMBO_CAPABLE)) {
2750 		if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &val))
2751 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
2752 				     val | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
2753 	}
2754 
2755 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
2756 		/* adjust output voltage */
2757 		tg3_writephy(tp, MII_TG3_FET_PTEST, 0x12);
2758 	}
2759 
2760 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5762_A0)
2761 		tg3_phydsp_write(tp, 0xffb, 0x4000);
2762 
2763 	tg3_phy_toggle_automdix(tp, true);
2764 	tg3_phy_set_wirespeed(tp);
2765 	return 0;
2766 }
2767 
2768 #define TG3_GPIO_MSG_DRVR_PRES		 0x00000001
2769 #define TG3_GPIO_MSG_NEED_VAUX		 0x00000002
2770 #define TG3_GPIO_MSG_MASK		 (TG3_GPIO_MSG_DRVR_PRES | \
2771 					  TG3_GPIO_MSG_NEED_VAUX)
2772 #define TG3_GPIO_MSG_ALL_DRVR_PRES_MASK \
2773 	((TG3_GPIO_MSG_DRVR_PRES << 0) | \
2774 	 (TG3_GPIO_MSG_DRVR_PRES << 4) | \
2775 	 (TG3_GPIO_MSG_DRVR_PRES << 8) | \
2776 	 (TG3_GPIO_MSG_DRVR_PRES << 12))
2777 
2778 #define TG3_GPIO_MSG_ALL_NEED_VAUX_MASK \
2779 	((TG3_GPIO_MSG_NEED_VAUX << 0) | \
2780 	 (TG3_GPIO_MSG_NEED_VAUX << 4) | \
2781 	 (TG3_GPIO_MSG_NEED_VAUX << 8) | \
2782 	 (TG3_GPIO_MSG_NEED_VAUX << 12))
2783 
2784 static inline u32 tg3_set_function_status(struct tg3 *tp, u32 newstat)
2785 {
2786 	u32 status, shift;
2787 
2788 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2789 	    tg3_asic_rev(tp) == ASIC_REV_5719)
2790 		status = tg3_ape_read32(tp, TG3_APE_GPIO_MSG);
2791 	else
2792 		status = tr32(TG3_CPMU_DRV_STATUS);
2793 
2794 	shift = TG3_APE_GPIO_MSG_SHIFT + 4 * tp->pci_fn;
2795 	status &= ~(TG3_GPIO_MSG_MASK << shift);
2796 	status |= (newstat << shift);
2797 
2798 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2799 	    tg3_asic_rev(tp) == ASIC_REV_5719)
2800 		tg3_ape_write32(tp, TG3_APE_GPIO_MSG, status);
2801 	else
2802 		tw32(TG3_CPMU_DRV_STATUS, status);
2803 
2804 	return status >> TG3_APE_GPIO_MSG_SHIFT;
2805 }
2806 
2807 static inline int tg3_pwrsrc_switch_to_vmain(struct tg3 *tp)
2808 {
2809 	if (!tg3_flag(tp, IS_NIC))
2810 		return 0;
2811 
2812 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2813 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
2814 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
2815 		if (tg3_ape_lock(tp, TG3_APE_LOCK_GPIO))
2816 			return -EIO;
2817 
2818 		tg3_set_function_status(tp, TG3_GPIO_MSG_DRVR_PRES);
2819 
2820 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl,
2821 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2822 
2823 		tg3_ape_unlock(tp, TG3_APE_LOCK_GPIO);
2824 	} else {
2825 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl,
2826 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2827 	}
2828 
2829 	return 0;
2830 }
2831 
2832 static void tg3_pwrsrc_die_with_vmain(struct tg3 *tp)
2833 {
2834 	u32 grc_local_ctrl;
2835 
2836 	if (!tg3_flag(tp, IS_NIC) ||
2837 	    tg3_asic_rev(tp) == ASIC_REV_5700 ||
2838 	    tg3_asic_rev(tp) == ASIC_REV_5701)
2839 		return;
2840 
2841 	grc_local_ctrl = tp->grc_local_ctrl | GRC_LCLCTRL_GPIO_OE1;
2842 
2843 	tw32_wait_f(GRC_LOCAL_CTRL,
2844 		    grc_local_ctrl | GRC_LCLCTRL_GPIO_OUTPUT1,
2845 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2846 
2847 	tw32_wait_f(GRC_LOCAL_CTRL,
2848 		    grc_local_ctrl,
2849 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2850 
2851 	tw32_wait_f(GRC_LOCAL_CTRL,
2852 		    grc_local_ctrl | GRC_LCLCTRL_GPIO_OUTPUT1,
2853 		    TG3_GRC_LCLCTL_PWRSW_DELAY);
2854 }
2855 
2856 static void tg3_pwrsrc_switch_to_vaux(struct tg3 *tp)
2857 {
2858 	if (!tg3_flag(tp, IS_NIC))
2859 		return;
2860 
2861 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
2862 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
2863 		tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2864 			    (GRC_LCLCTRL_GPIO_OE0 |
2865 			     GRC_LCLCTRL_GPIO_OE1 |
2866 			     GRC_LCLCTRL_GPIO_OE2 |
2867 			     GRC_LCLCTRL_GPIO_OUTPUT0 |
2868 			     GRC_LCLCTRL_GPIO_OUTPUT1),
2869 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2870 	} else if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
2871 		   tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
2872 		/* The 5761 non-e device swaps GPIO 0 and GPIO 2. */
2873 		u32 grc_local_ctrl = GRC_LCLCTRL_GPIO_OE0 |
2874 				     GRC_LCLCTRL_GPIO_OE1 |
2875 				     GRC_LCLCTRL_GPIO_OE2 |
2876 				     GRC_LCLCTRL_GPIO_OUTPUT0 |
2877 				     GRC_LCLCTRL_GPIO_OUTPUT1 |
2878 				     tp->grc_local_ctrl;
2879 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2880 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2881 
2882 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT2;
2883 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2884 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2885 
2886 		grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT0;
2887 		tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl,
2888 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2889 	} else {
2890 		u32 no_gpio2;
2891 		u32 grc_local_ctrl = 0;
2892 
2893 		/* Workaround to prevent overdrawing Amps. */
2894 		if (tg3_asic_rev(tp) == ASIC_REV_5714) {
2895 			grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
2896 			tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2897 				    grc_local_ctrl,
2898 				    TG3_GRC_LCLCTL_PWRSW_DELAY);
2899 		}
2900 
2901 		/* On 5753 and variants, GPIO2 cannot be used. */
2902 		no_gpio2 = tp->nic_sram_data_cfg &
2903 			   NIC_SRAM_DATA_CFG_NO_GPIO2;
2904 
2905 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
2906 				  GRC_LCLCTRL_GPIO_OE1 |
2907 				  GRC_LCLCTRL_GPIO_OE2 |
2908 				  GRC_LCLCTRL_GPIO_OUTPUT1 |
2909 				  GRC_LCLCTRL_GPIO_OUTPUT2;
2910 		if (no_gpio2) {
2911 			grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
2912 					    GRC_LCLCTRL_GPIO_OUTPUT2);
2913 		}
2914 		tw32_wait_f(GRC_LOCAL_CTRL,
2915 			    tp->grc_local_ctrl | grc_local_ctrl,
2916 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2917 
2918 		grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
2919 
2920 		tw32_wait_f(GRC_LOCAL_CTRL,
2921 			    tp->grc_local_ctrl | grc_local_ctrl,
2922 			    TG3_GRC_LCLCTL_PWRSW_DELAY);
2923 
2924 		if (!no_gpio2) {
2925 			grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
2926 			tw32_wait_f(GRC_LOCAL_CTRL,
2927 				    tp->grc_local_ctrl | grc_local_ctrl,
2928 				    TG3_GRC_LCLCTL_PWRSW_DELAY);
2929 		}
2930 	}
2931 }
2932 
2933 static void tg3_frob_aux_power_5717(struct tg3 *tp, bool wol_enable)
2934 {
2935 	u32 msg = 0;
2936 
2937 	/* Serialize power state transitions */
2938 	if (tg3_ape_lock(tp, TG3_APE_LOCK_GPIO))
2939 		return;
2940 
2941 	if (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE) || wol_enable)
2942 		msg = TG3_GPIO_MSG_NEED_VAUX;
2943 
2944 	msg = tg3_set_function_status(tp, msg);
2945 
2946 	if (msg & TG3_GPIO_MSG_ALL_DRVR_PRES_MASK)
2947 		goto done;
2948 
2949 	if (msg & TG3_GPIO_MSG_ALL_NEED_VAUX_MASK)
2950 		tg3_pwrsrc_switch_to_vaux(tp);
2951 	else
2952 		tg3_pwrsrc_die_with_vmain(tp);
2953 
2954 done:
2955 	tg3_ape_unlock(tp, TG3_APE_LOCK_GPIO);
2956 }
2957 
2958 static void tg3_frob_aux_power(struct tg3 *tp, bool include_wol)
2959 {
2960 	bool need_vaux = false;
2961 
2962 	/* The GPIOs do something completely different on 57765. */
2963 	if (!tg3_flag(tp, IS_NIC) || tg3_flag(tp, 57765_CLASS))
2964 		return;
2965 
2966 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
2967 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
2968 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
2969 		tg3_frob_aux_power_5717(tp, include_wol ?
2970 					tg3_flag(tp, WOL_ENABLE) != 0 : 0);
2971 		return;
2972 	}
2973 
2974 	if (tp->pdev_peer && tp->pdev_peer != tp->pdev) {
2975 		struct net_device *dev_peer;
2976 
2977 		dev_peer = pci_get_drvdata(tp->pdev_peer);
2978 
2979 		/* remove_one() may have been run on the peer. */
2980 		if (dev_peer) {
2981 			struct tg3 *tp_peer = netdev_priv(dev_peer);
2982 
2983 			if (tg3_flag(tp_peer, INIT_COMPLETE))
2984 				return;
2985 
2986 			if ((include_wol && tg3_flag(tp_peer, WOL_ENABLE)) ||
2987 			    tg3_flag(tp_peer, ENABLE_ASF))
2988 				need_vaux = true;
2989 		}
2990 	}
2991 
2992 	if ((include_wol && tg3_flag(tp, WOL_ENABLE)) ||
2993 	    tg3_flag(tp, ENABLE_ASF))
2994 		need_vaux = true;
2995 
2996 	if (need_vaux)
2997 		tg3_pwrsrc_switch_to_vaux(tp);
2998 	else
2999 		tg3_pwrsrc_die_with_vmain(tp);
3000 }
3001 
3002 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
3003 {
3004 	if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
3005 		return 1;
3006 	else if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411) {
3007 		if (speed != SPEED_10)
3008 			return 1;
3009 	} else if (speed == SPEED_10)
3010 		return 1;
3011 
3012 	return 0;
3013 }
3014 
3015 static bool tg3_phy_power_bug(struct tg3 *tp)
3016 {
3017 	switch (tg3_asic_rev(tp)) {
3018 	case ASIC_REV_5700:
3019 	case ASIC_REV_5704:
3020 		return true;
3021 	case ASIC_REV_5780:
3022 		if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
3023 			return true;
3024 		return false;
3025 	case ASIC_REV_5717:
3026 		if (!tp->pci_fn)
3027 			return true;
3028 		return false;
3029 	case ASIC_REV_5719:
3030 	case ASIC_REV_5720:
3031 		if ((tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
3032 		    !tp->pci_fn)
3033 			return true;
3034 		return false;
3035 	}
3036 
3037 	return false;
3038 }
3039 
3040 static bool tg3_phy_led_bug(struct tg3 *tp)
3041 {
3042 	switch (tg3_asic_rev(tp)) {
3043 	case ASIC_REV_5719:
3044 	case ASIC_REV_5720:
3045 		if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
3046 		    !tp->pci_fn)
3047 			return true;
3048 		return false;
3049 	}
3050 
3051 	return false;
3052 }
3053 
3054 static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power)
3055 {
3056 	u32 val;
3057 
3058 	if (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)
3059 		return;
3060 
3061 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
3062 		if (tg3_asic_rev(tp) == ASIC_REV_5704) {
3063 			u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
3064 			u32 serdes_cfg = tr32(MAC_SERDES_CFG);
3065 
3066 			sg_dig_ctrl |=
3067 				SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
3068 			tw32(SG_DIG_CTRL, sg_dig_ctrl);
3069 			tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
3070 		}
3071 		return;
3072 	}
3073 
3074 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
3075 		tg3_bmcr_reset(tp);
3076 		val = tr32(GRC_MISC_CFG);
3077 		tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
3078 		udelay(40);
3079 		return;
3080 	} else if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
3081 		u32 phytest;
3082 		if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
3083 			u32 phy;
3084 
3085 			tg3_writephy(tp, MII_ADVERTISE, 0);
3086 			tg3_writephy(tp, MII_BMCR,
3087 				     BMCR_ANENABLE | BMCR_ANRESTART);
3088 
3089 			tg3_writephy(tp, MII_TG3_FET_TEST,
3090 				     phytest | MII_TG3_FET_SHADOW_EN);
3091 			if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXMODE4, &phy)) {
3092 				phy |= MII_TG3_FET_SHDW_AUXMODE4_SBPD;
3093 				tg3_writephy(tp,
3094 					     MII_TG3_FET_SHDW_AUXMODE4,
3095 					     phy);
3096 			}
3097 			tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
3098 		}
3099 		return;
3100 	} else if (do_low_power) {
3101 		if (!tg3_phy_led_bug(tp))
3102 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
3103 				     MII_TG3_EXT_CTRL_FORCE_LED_OFF);
3104 
3105 		val = MII_TG3_AUXCTL_PCTL_100TX_LPWR |
3106 		      MII_TG3_AUXCTL_PCTL_SPR_ISOLATE |
3107 		      MII_TG3_AUXCTL_PCTL_VREG_11V;
3108 		tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, val);
3109 	}
3110 
3111 	/* The PHY should not be powered down on some chips because
3112 	 * of bugs.
3113 	 */
3114 	if (tg3_phy_power_bug(tp))
3115 		return;
3116 
3117 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX ||
3118 	    tg3_chip_rev(tp) == CHIPREV_5761_AX) {
3119 		val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
3120 		val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
3121 		val |= CPMU_LSPD_1000MB_MACCLK_12_5;
3122 		tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
3123 	}
3124 
3125 	tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
3126 }
3127 
3128 /* tp->lock is held. */
3129 static int tg3_nvram_lock(struct tg3 *tp)
3130 {
3131 	if (tg3_flag(tp, NVRAM)) {
3132 		int i;
3133 
3134 		if (tp->nvram_lock_cnt == 0) {
3135 			tw32(NVRAM_SWARB, SWARB_REQ_SET1);
3136 			for (i = 0; i < 8000; i++) {
3137 				if (tr32(NVRAM_SWARB) & SWARB_GNT1)
3138 					break;
3139 				udelay(20);
3140 			}
3141 			if (i == 8000) {
3142 				tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
3143 				return -ENODEV;
3144 			}
3145 		}
3146 		tp->nvram_lock_cnt++;
3147 	}
3148 	return 0;
3149 }
3150 
3151 /* tp->lock is held. */
3152 static void tg3_nvram_unlock(struct tg3 *tp)
3153 {
3154 	if (tg3_flag(tp, NVRAM)) {
3155 		if (tp->nvram_lock_cnt > 0)
3156 			tp->nvram_lock_cnt--;
3157 		if (tp->nvram_lock_cnt == 0)
3158 			tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
3159 	}
3160 }
3161 
3162 /* tp->lock is held. */
3163 static void tg3_enable_nvram_access(struct tg3 *tp)
3164 {
3165 	if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
3166 		u32 nvaccess = tr32(NVRAM_ACCESS);
3167 
3168 		tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
3169 	}
3170 }
3171 
3172 /* tp->lock is held. */
3173 static void tg3_disable_nvram_access(struct tg3 *tp)
3174 {
3175 	if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
3176 		u32 nvaccess = tr32(NVRAM_ACCESS);
3177 
3178 		tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
3179 	}
3180 }
3181 
3182 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
3183 					u32 offset, u32 *val)
3184 {
3185 	u32 tmp;
3186 	int i;
3187 
3188 	if (offset > EEPROM_ADDR_ADDR_MASK || (offset % 4) != 0)
3189 		return -EINVAL;
3190 
3191 	tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
3192 					EEPROM_ADDR_DEVID_MASK |
3193 					EEPROM_ADDR_READ);
3194 	tw32(GRC_EEPROM_ADDR,
3195 	     tmp |
3196 	     (0 << EEPROM_ADDR_DEVID_SHIFT) |
3197 	     ((offset << EEPROM_ADDR_ADDR_SHIFT) &
3198 	      EEPROM_ADDR_ADDR_MASK) |
3199 	     EEPROM_ADDR_READ | EEPROM_ADDR_START);
3200 
3201 	for (i = 0; i < 1000; i++) {
3202 		tmp = tr32(GRC_EEPROM_ADDR);
3203 
3204 		if (tmp & EEPROM_ADDR_COMPLETE)
3205 			break;
3206 		msleep(1);
3207 	}
3208 	if (!(tmp & EEPROM_ADDR_COMPLETE))
3209 		return -EBUSY;
3210 
3211 	tmp = tr32(GRC_EEPROM_DATA);
3212 
3213 	/*
3214 	 * The data will always be opposite the native endian
3215 	 * format.  Perform a blind byteswap to compensate.
3216 	 */
3217 	*val = swab32(tmp);
3218 
3219 	return 0;
3220 }
3221 
3222 #define NVRAM_CMD_TIMEOUT 10000
3223 
3224 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
3225 {
3226 	int i;
3227 
3228 	tw32(NVRAM_CMD, nvram_cmd);
3229 	for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
3230 		usleep_range(10, 40);
3231 		if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
3232 			udelay(10);
3233 			break;
3234 		}
3235 	}
3236 
3237 	if (i == NVRAM_CMD_TIMEOUT)
3238 		return -EBUSY;
3239 
3240 	return 0;
3241 }
3242 
3243 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
3244 {
3245 	if (tg3_flag(tp, NVRAM) &&
3246 	    tg3_flag(tp, NVRAM_BUFFERED) &&
3247 	    tg3_flag(tp, FLASH) &&
3248 	    !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
3249 	    (tp->nvram_jedecnum == JEDEC_ATMEL))
3250 
3251 		addr = ((addr / tp->nvram_pagesize) <<
3252 			ATMEL_AT45DB0X1B_PAGE_POS) +
3253 		       (addr % tp->nvram_pagesize);
3254 
3255 	return addr;
3256 }
3257 
3258 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
3259 {
3260 	if (tg3_flag(tp, NVRAM) &&
3261 	    tg3_flag(tp, NVRAM_BUFFERED) &&
3262 	    tg3_flag(tp, FLASH) &&
3263 	    !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
3264 	    (tp->nvram_jedecnum == JEDEC_ATMEL))
3265 
3266 		addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
3267 			tp->nvram_pagesize) +
3268 		       (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
3269 
3270 	return addr;
3271 }
3272 
3273 /* NOTE: Data read in from NVRAM is byteswapped according to
3274  * the byteswapping settings for all other register accesses.
3275  * tg3 devices are BE devices, so on a BE machine, the data
3276  * returned will be exactly as it is seen in NVRAM.  On a LE
3277  * machine, the 32-bit value will be byteswapped.
3278  */
3279 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
3280 {
3281 	int ret;
3282 
3283 	if (!tg3_flag(tp, NVRAM))
3284 		return tg3_nvram_read_using_eeprom(tp, offset, val);
3285 
3286 	offset = tg3_nvram_phys_addr(tp, offset);
3287 
3288 	if (offset > NVRAM_ADDR_MSK)
3289 		return -EINVAL;
3290 
3291 	ret = tg3_nvram_lock(tp);
3292 	if (ret)
3293 		return ret;
3294 
3295 	tg3_enable_nvram_access(tp);
3296 
3297 	tw32(NVRAM_ADDR, offset);
3298 	ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
3299 		NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
3300 
3301 	if (ret == 0)
3302 		*val = tr32(NVRAM_RDDATA);
3303 
3304 	tg3_disable_nvram_access(tp);
3305 
3306 	tg3_nvram_unlock(tp);
3307 
3308 	return ret;
3309 }
3310 
3311 /* Ensures NVRAM data is in bytestream format. */
3312 static int tg3_nvram_read_be32(struct tg3 *tp, u32 offset, __be32 *val)
3313 {
3314 	u32 v;
3315 	int res = tg3_nvram_read(tp, offset, &v);
3316 	if (!res)
3317 		*val = cpu_to_be32(v);
3318 	return res;
3319 }
3320 
3321 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
3322 				    u32 offset, u32 len, u8 *buf)
3323 {
3324 	int i, j, rc = 0;
3325 	u32 val;
3326 
3327 	for (i = 0; i < len; i += 4) {
3328 		u32 addr;
3329 		__be32 data;
3330 
3331 		addr = offset + i;
3332 
3333 		memcpy(&data, buf + i, 4);
3334 
3335 		/*
3336 		 * The SEEPROM interface expects the data to always be opposite
3337 		 * the native endian format.  We accomplish this by reversing
3338 		 * all the operations that would have been performed on the
3339 		 * data from a call to tg3_nvram_read_be32().
3340 		 */
3341 		tw32(GRC_EEPROM_DATA, swab32(be32_to_cpu(data)));
3342 
3343 		val = tr32(GRC_EEPROM_ADDR);
3344 		tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
3345 
3346 		val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
3347 			EEPROM_ADDR_READ);
3348 		tw32(GRC_EEPROM_ADDR, val |
3349 			(0 << EEPROM_ADDR_DEVID_SHIFT) |
3350 			(addr & EEPROM_ADDR_ADDR_MASK) |
3351 			EEPROM_ADDR_START |
3352 			EEPROM_ADDR_WRITE);
3353 
3354 		for (j = 0; j < 1000; j++) {
3355 			val = tr32(GRC_EEPROM_ADDR);
3356 
3357 			if (val & EEPROM_ADDR_COMPLETE)
3358 				break;
3359 			msleep(1);
3360 		}
3361 		if (!(val & EEPROM_ADDR_COMPLETE)) {
3362 			rc = -EBUSY;
3363 			break;
3364 		}
3365 	}
3366 
3367 	return rc;
3368 }
3369 
3370 /* offset and length are dword aligned */
3371 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
3372 		u8 *buf)
3373 {
3374 	int ret = 0;
3375 	u32 pagesize = tp->nvram_pagesize;
3376 	u32 pagemask = pagesize - 1;
3377 	u32 nvram_cmd;
3378 	u8 *tmp;
3379 
3380 	tmp = kmalloc(pagesize, GFP_KERNEL);
3381 	if (tmp == NULL)
3382 		return -ENOMEM;
3383 
3384 	while (len) {
3385 		int j;
3386 		u32 phy_addr, page_off, size;
3387 
3388 		phy_addr = offset & ~pagemask;
3389 
3390 		for (j = 0; j < pagesize; j += 4) {
3391 			ret = tg3_nvram_read_be32(tp, phy_addr + j,
3392 						  (__be32 *) (tmp + j));
3393 			if (ret)
3394 				break;
3395 		}
3396 		if (ret)
3397 			break;
3398 
3399 		page_off = offset & pagemask;
3400 		size = pagesize;
3401 		if (len < size)
3402 			size = len;
3403 
3404 		len -= size;
3405 
3406 		memcpy(tmp + page_off, buf, size);
3407 
3408 		offset = offset + (pagesize - page_off);
3409 
3410 		tg3_enable_nvram_access(tp);
3411 
3412 		/*
3413 		 * Before we can erase the flash page, we need
3414 		 * to issue a special "write enable" command.
3415 		 */
3416 		nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3417 
3418 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3419 			break;
3420 
3421 		/* Erase the target page */
3422 		tw32(NVRAM_ADDR, phy_addr);
3423 
3424 		nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
3425 			NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
3426 
3427 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3428 			break;
3429 
3430 		/* Issue another write enable to start the write. */
3431 		nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3432 
3433 		if (tg3_nvram_exec_cmd(tp, nvram_cmd))
3434 			break;
3435 
3436 		for (j = 0; j < pagesize; j += 4) {
3437 			__be32 data;
3438 
3439 			data = *((__be32 *) (tmp + j));
3440 
3441 			tw32(NVRAM_WRDATA, be32_to_cpu(data));
3442 
3443 			tw32(NVRAM_ADDR, phy_addr + j);
3444 
3445 			nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
3446 				NVRAM_CMD_WR;
3447 
3448 			if (j == 0)
3449 				nvram_cmd |= NVRAM_CMD_FIRST;
3450 			else if (j == (pagesize - 4))
3451 				nvram_cmd |= NVRAM_CMD_LAST;
3452 
3453 			ret = tg3_nvram_exec_cmd(tp, nvram_cmd);
3454 			if (ret)
3455 				break;
3456 		}
3457 		if (ret)
3458 			break;
3459 	}
3460 
3461 	nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3462 	tg3_nvram_exec_cmd(tp, nvram_cmd);
3463 
3464 	kfree(tmp);
3465 
3466 	return ret;
3467 }
3468 
3469 /* offset and length are dword aligned */
3470 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
3471 		u8 *buf)
3472 {
3473 	int i, ret = 0;
3474 
3475 	for (i = 0; i < len; i += 4, offset += 4) {
3476 		u32 page_off, phy_addr, nvram_cmd;
3477 		__be32 data;
3478 
3479 		memcpy(&data, buf + i, 4);
3480 		tw32(NVRAM_WRDATA, be32_to_cpu(data));
3481 
3482 		page_off = offset % tp->nvram_pagesize;
3483 
3484 		phy_addr = tg3_nvram_phys_addr(tp, offset);
3485 
3486 		nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
3487 
3488 		if (page_off == 0 || i == 0)
3489 			nvram_cmd |= NVRAM_CMD_FIRST;
3490 		if (page_off == (tp->nvram_pagesize - 4))
3491 			nvram_cmd |= NVRAM_CMD_LAST;
3492 
3493 		if (i == (len - 4))
3494 			nvram_cmd |= NVRAM_CMD_LAST;
3495 
3496 		if ((nvram_cmd & NVRAM_CMD_FIRST) ||
3497 		    !tg3_flag(tp, FLASH) ||
3498 		    !tg3_flag(tp, 57765_PLUS))
3499 			tw32(NVRAM_ADDR, phy_addr);
3500 
3501 		if (tg3_asic_rev(tp) != ASIC_REV_5752 &&
3502 		    !tg3_flag(tp, 5755_PLUS) &&
3503 		    (tp->nvram_jedecnum == JEDEC_ST) &&
3504 		    (nvram_cmd & NVRAM_CMD_FIRST)) {
3505 			u32 cmd;
3506 
3507 			cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
3508 			ret = tg3_nvram_exec_cmd(tp, cmd);
3509 			if (ret)
3510 				break;
3511 		}
3512 		if (!tg3_flag(tp, FLASH)) {
3513 			/* We always do complete word writes to eeprom. */
3514 			nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
3515 		}
3516 
3517 		ret = tg3_nvram_exec_cmd(tp, nvram_cmd);
3518 		if (ret)
3519 			break;
3520 	}
3521 	return ret;
3522 }
3523 
3524 /* offset and length are dword aligned */
3525 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
3526 {
3527 	int ret;
3528 
3529 	if (tg3_flag(tp, EEPROM_WRITE_PROT)) {
3530 		tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
3531 		       ~GRC_LCLCTRL_GPIO_OUTPUT1);
3532 		udelay(40);
3533 	}
3534 
3535 	if (!tg3_flag(tp, NVRAM)) {
3536 		ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
3537 	} else {
3538 		u32 grc_mode;
3539 
3540 		ret = tg3_nvram_lock(tp);
3541 		if (ret)
3542 			return ret;
3543 
3544 		tg3_enable_nvram_access(tp);
3545 		if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM))
3546 			tw32(NVRAM_WRITE1, 0x406);
3547 
3548 		grc_mode = tr32(GRC_MODE);
3549 		tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
3550 
3551 		if (tg3_flag(tp, NVRAM_BUFFERED) || !tg3_flag(tp, FLASH)) {
3552 			ret = tg3_nvram_write_block_buffered(tp, offset, len,
3553 				buf);
3554 		} else {
3555 			ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
3556 				buf);
3557 		}
3558 
3559 		grc_mode = tr32(GRC_MODE);
3560 		tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
3561 
3562 		tg3_disable_nvram_access(tp);
3563 		tg3_nvram_unlock(tp);
3564 	}
3565 
3566 	if (tg3_flag(tp, EEPROM_WRITE_PROT)) {
3567 		tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
3568 		udelay(40);
3569 	}
3570 
3571 	return ret;
3572 }
3573 
3574 #define RX_CPU_SCRATCH_BASE	0x30000
3575 #define RX_CPU_SCRATCH_SIZE	0x04000
3576 #define TX_CPU_SCRATCH_BASE	0x34000
3577 #define TX_CPU_SCRATCH_SIZE	0x04000
3578 
3579 /* tp->lock is held. */
3580 static int tg3_pause_cpu(struct tg3 *tp, u32 cpu_base)
3581 {
3582 	int i;
3583 	const int iters = 10000;
3584 
3585 	for (i = 0; i < iters; i++) {
3586 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3587 		tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
3588 		if (tr32(cpu_base + CPU_MODE) & CPU_MODE_HALT)
3589 			break;
3590 		if (pci_channel_offline(tp->pdev))
3591 			return -EBUSY;
3592 	}
3593 
3594 	return (i == iters) ? -EBUSY : 0;
3595 }
3596 
3597 /* tp->lock is held. */
3598 static int tg3_rxcpu_pause(struct tg3 *tp)
3599 {
3600 	int rc = tg3_pause_cpu(tp, RX_CPU_BASE);
3601 
3602 	tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
3603 	tw32_f(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
3604 	udelay(10);
3605 
3606 	return rc;
3607 }
3608 
3609 /* tp->lock is held. */
3610 static int tg3_txcpu_pause(struct tg3 *tp)
3611 {
3612 	return tg3_pause_cpu(tp, TX_CPU_BASE);
3613 }
3614 
3615 /* tp->lock is held. */
3616 static void tg3_resume_cpu(struct tg3 *tp, u32 cpu_base)
3617 {
3618 	tw32(cpu_base + CPU_STATE, 0xffffffff);
3619 	tw32_f(cpu_base + CPU_MODE,  0x00000000);
3620 }
3621 
3622 /* tp->lock is held. */
3623 static void tg3_rxcpu_resume(struct tg3 *tp)
3624 {
3625 	tg3_resume_cpu(tp, RX_CPU_BASE);
3626 }
3627 
3628 /* tp->lock is held. */
3629 static int tg3_halt_cpu(struct tg3 *tp, u32 cpu_base)
3630 {
3631 	int rc;
3632 
3633 	BUG_ON(cpu_base == TX_CPU_BASE && tg3_flag(tp, 5705_PLUS));
3634 
3635 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
3636 		u32 val = tr32(GRC_VCPU_EXT_CTRL);
3637 
3638 		tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
3639 		return 0;
3640 	}
3641 	if (cpu_base == RX_CPU_BASE) {
3642 		rc = tg3_rxcpu_pause(tp);
3643 	} else {
3644 		/*
3645 		 * There is only an Rx CPU for the 5750 derivative in the
3646 		 * BCM4785.
3647 		 */
3648 		if (tg3_flag(tp, IS_SSB_CORE))
3649 			return 0;
3650 
3651 		rc = tg3_txcpu_pause(tp);
3652 	}
3653 
3654 	if (rc) {
3655 		netdev_err(tp->dev, "%s timed out, %s CPU\n",
3656 			   __func__, cpu_base == RX_CPU_BASE ? "RX" : "TX");
3657 		return -ENODEV;
3658 	}
3659 
3660 	/* Clear firmware's nvram arbitration. */
3661 	if (tg3_flag(tp, NVRAM))
3662 		tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
3663 	return 0;
3664 }
3665 
3666 static int tg3_fw_data_len(struct tg3 *tp,
3667 			   const struct tg3_firmware_hdr *fw_hdr)
3668 {
3669 	int fw_len;
3670 
3671 	/* Non fragmented firmware have one firmware header followed by a
3672 	 * contiguous chunk of data to be written. The length field in that
3673 	 * header is not the length of data to be written but the complete
3674 	 * length of the bss. The data length is determined based on
3675 	 * tp->fw->size minus headers.
3676 	 *
3677 	 * Fragmented firmware have a main header followed by multiple
3678 	 * fragments. Each fragment is identical to non fragmented firmware
3679 	 * with a firmware header followed by a contiguous chunk of data. In
3680 	 * the main header, the length field is unused and set to 0xffffffff.
3681 	 * In each fragment header the length is the entire size of that
3682 	 * fragment i.e. fragment data + header length. Data length is
3683 	 * therefore length field in the header minus TG3_FW_HDR_LEN.
3684 	 */
3685 	if (tp->fw_len == 0xffffffff)
3686 		fw_len = be32_to_cpu(fw_hdr->len);
3687 	else
3688 		fw_len = tp->fw->size;
3689 
3690 	return (fw_len - TG3_FW_HDR_LEN) / sizeof(u32);
3691 }
3692 
3693 /* tp->lock is held. */
3694 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base,
3695 				 u32 cpu_scratch_base, int cpu_scratch_size,
3696 				 const struct tg3_firmware_hdr *fw_hdr)
3697 {
3698 	int err, i;
3699 	void (*write_op)(struct tg3 *, u32, u32);
3700 	int total_len = tp->fw->size;
3701 
3702 	if (cpu_base == TX_CPU_BASE && tg3_flag(tp, 5705_PLUS)) {
3703 		netdev_err(tp->dev,
3704 			   "%s: Trying to load TX cpu firmware which is 5705\n",
3705 			   __func__);
3706 		return -EINVAL;
3707 	}
3708 
3709 	if (tg3_flag(tp, 5705_PLUS) && tg3_asic_rev(tp) != ASIC_REV_57766)
3710 		write_op = tg3_write_mem;
3711 	else
3712 		write_op = tg3_write_indirect_reg32;
3713 
3714 	if (tg3_asic_rev(tp) != ASIC_REV_57766) {
3715 		/* It is possible that bootcode is still loading at this point.
3716 		 * Get the nvram lock first before halting the cpu.
3717 		 */
3718 		int lock_err = tg3_nvram_lock(tp);
3719 		err = tg3_halt_cpu(tp, cpu_base);
3720 		if (!lock_err)
3721 			tg3_nvram_unlock(tp);
3722 		if (err)
3723 			goto out;
3724 
3725 		for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
3726 			write_op(tp, cpu_scratch_base + i, 0);
3727 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3728 		tw32(cpu_base + CPU_MODE,
3729 		     tr32(cpu_base + CPU_MODE) | CPU_MODE_HALT);
3730 	} else {
3731 		/* Subtract additional main header for fragmented firmware and
3732 		 * advance to the first fragment
3733 		 */
3734 		total_len -= TG3_FW_HDR_LEN;
3735 		fw_hdr++;
3736 	}
3737 
3738 	do {
3739 		u32 *fw_data = (u32 *)(fw_hdr + 1);
3740 		for (i = 0; i < tg3_fw_data_len(tp, fw_hdr); i++)
3741 			write_op(tp, cpu_scratch_base +
3742 				     (be32_to_cpu(fw_hdr->base_addr) & 0xffff) +
3743 				     (i * sizeof(u32)),
3744 				 be32_to_cpu(fw_data[i]));
3745 
3746 		total_len -= be32_to_cpu(fw_hdr->len);
3747 
3748 		/* Advance to next fragment */
3749 		fw_hdr = (struct tg3_firmware_hdr *)
3750 			 ((void *)fw_hdr + be32_to_cpu(fw_hdr->len));
3751 	} while (total_len > 0);
3752 
3753 	err = 0;
3754 
3755 out:
3756 	return err;
3757 }
3758 
3759 /* tp->lock is held. */
3760 static int tg3_pause_cpu_and_set_pc(struct tg3 *tp, u32 cpu_base, u32 pc)
3761 {
3762 	int i;
3763 	const int iters = 5;
3764 
3765 	tw32(cpu_base + CPU_STATE, 0xffffffff);
3766 	tw32_f(cpu_base + CPU_PC, pc);
3767 
3768 	for (i = 0; i < iters; i++) {
3769 		if (tr32(cpu_base + CPU_PC) == pc)
3770 			break;
3771 		tw32(cpu_base + CPU_STATE, 0xffffffff);
3772 		tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
3773 		tw32_f(cpu_base + CPU_PC, pc);
3774 		udelay(1000);
3775 	}
3776 
3777 	return (i == iters) ? -EBUSY : 0;
3778 }
3779 
3780 /* tp->lock is held. */
3781 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
3782 {
3783 	const struct tg3_firmware_hdr *fw_hdr;
3784 	int err;
3785 
3786 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3787 
3788 	/* Firmware blob starts with version numbers, followed by
3789 	   start address and length. We are setting complete length.
3790 	   length = end_address_of_bss - start_address_of_text.
3791 	   Remainder is the blob to be loaded contiguously
3792 	   from start address. */
3793 
3794 	err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
3795 				    RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
3796 				    fw_hdr);
3797 	if (err)
3798 		return err;
3799 
3800 	err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
3801 				    TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
3802 				    fw_hdr);
3803 	if (err)
3804 		return err;
3805 
3806 	/* Now startup only the RX cpu. */
3807 	err = tg3_pause_cpu_and_set_pc(tp, RX_CPU_BASE,
3808 				       be32_to_cpu(fw_hdr->base_addr));
3809 	if (err) {
3810 		netdev_err(tp->dev, "%s fails to set RX CPU PC, is %08x "
3811 			   "should be %08x\n", __func__,
3812 			   tr32(RX_CPU_BASE + CPU_PC),
3813 				be32_to_cpu(fw_hdr->base_addr));
3814 		return -ENODEV;
3815 	}
3816 
3817 	tg3_rxcpu_resume(tp);
3818 
3819 	return 0;
3820 }
3821 
3822 static int tg3_validate_rxcpu_state(struct tg3 *tp)
3823 {
3824 	const int iters = 1000;
3825 	int i;
3826 	u32 val;
3827 
3828 	/* Wait for boot code to complete initialization and enter service
3829 	 * loop. It is then safe to download service patches
3830 	 */
3831 	for (i = 0; i < iters; i++) {
3832 		if (tr32(RX_CPU_HWBKPT) == TG3_SBROM_IN_SERVICE_LOOP)
3833 			break;
3834 
3835 		udelay(10);
3836 	}
3837 
3838 	if (i == iters) {
3839 		netdev_err(tp->dev, "Boot code not ready for service patches\n");
3840 		return -EBUSY;
3841 	}
3842 
3843 	val = tg3_read_indirect_reg32(tp, TG3_57766_FW_HANDSHAKE);
3844 	if (val & 0xff) {
3845 		netdev_warn(tp->dev,
3846 			    "Other patches exist. Not downloading EEE patch\n");
3847 		return -EEXIST;
3848 	}
3849 
3850 	return 0;
3851 }
3852 
3853 /* tp->lock is held. */
3854 static void tg3_load_57766_firmware(struct tg3 *tp)
3855 {
3856 	struct tg3_firmware_hdr *fw_hdr;
3857 
3858 	if (!tg3_flag(tp, NO_NVRAM))
3859 		return;
3860 
3861 	if (tg3_validate_rxcpu_state(tp))
3862 		return;
3863 
3864 	if (!tp->fw)
3865 		return;
3866 
3867 	/* This firmware blob has a different format than older firmware
3868 	 * releases as given below. The main difference is we have fragmented
3869 	 * data to be written to non-contiguous locations.
3870 	 *
3871 	 * In the beginning we have a firmware header identical to other
3872 	 * firmware which consists of version, base addr and length. The length
3873 	 * here is unused and set to 0xffffffff.
3874 	 *
3875 	 * This is followed by a series of firmware fragments which are
3876 	 * individually identical to previous firmware. i.e. they have the
3877 	 * firmware header and followed by data for that fragment. The version
3878 	 * field of the individual fragment header is unused.
3879 	 */
3880 
3881 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3882 	if (be32_to_cpu(fw_hdr->base_addr) != TG3_57766_FW_BASE_ADDR)
3883 		return;
3884 
3885 	if (tg3_rxcpu_pause(tp))
3886 		return;
3887 
3888 	/* tg3_load_firmware_cpu() will always succeed for the 57766 */
3889 	tg3_load_firmware_cpu(tp, 0, TG3_57766_FW_BASE_ADDR, 0, fw_hdr);
3890 
3891 	tg3_rxcpu_resume(tp);
3892 }
3893 
3894 /* tp->lock is held. */
3895 static int tg3_load_tso_firmware(struct tg3 *tp)
3896 {
3897 	const struct tg3_firmware_hdr *fw_hdr;
3898 	unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
3899 	int err;
3900 
3901 	if (!tg3_flag(tp, FW_TSO))
3902 		return 0;
3903 
3904 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
3905 
3906 	/* Firmware blob starts with version numbers, followed by
3907 	   start address and length. We are setting complete length.
3908 	   length = end_address_of_bss - start_address_of_text.
3909 	   Remainder is the blob to be loaded contiguously
3910 	   from start address. */
3911 
3912 	cpu_scratch_size = tp->fw_len;
3913 
3914 	if (tg3_asic_rev(tp) == ASIC_REV_5705) {
3915 		cpu_base = RX_CPU_BASE;
3916 		cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
3917 	} else {
3918 		cpu_base = TX_CPU_BASE;
3919 		cpu_scratch_base = TX_CPU_SCRATCH_BASE;
3920 		cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
3921 	}
3922 
3923 	err = tg3_load_firmware_cpu(tp, cpu_base,
3924 				    cpu_scratch_base, cpu_scratch_size,
3925 				    fw_hdr);
3926 	if (err)
3927 		return err;
3928 
3929 	/* Now startup the cpu. */
3930 	err = tg3_pause_cpu_and_set_pc(tp, cpu_base,
3931 				       be32_to_cpu(fw_hdr->base_addr));
3932 	if (err) {
3933 		netdev_err(tp->dev,
3934 			   "%s fails to set CPU PC, is %08x should be %08x\n",
3935 			   __func__, tr32(cpu_base + CPU_PC),
3936 			   be32_to_cpu(fw_hdr->base_addr));
3937 		return -ENODEV;
3938 	}
3939 
3940 	tg3_resume_cpu(tp, cpu_base);
3941 	return 0;
3942 }
3943 
3944 /* tp->lock is held. */
3945 static void __tg3_set_one_mac_addr(struct tg3 *tp, const u8 *mac_addr,
3946 				   int index)
3947 {
3948 	u32 addr_high, addr_low;
3949 
3950 	addr_high = ((mac_addr[0] << 8) | mac_addr[1]);
3951 	addr_low = ((mac_addr[2] << 24) | (mac_addr[3] << 16) |
3952 		    (mac_addr[4] <<  8) | mac_addr[5]);
3953 
3954 	if (index < 4) {
3955 		tw32(MAC_ADDR_0_HIGH + (index * 8), addr_high);
3956 		tw32(MAC_ADDR_0_LOW + (index * 8), addr_low);
3957 	} else {
3958 		index -= 4;
3959 		tw32(MAC_EXTADDR_0_HIGH + (index * 8), addr_high);
3960 		tw32(MAC_EXTADDR_0_LOW + (index * 8), addr_low);
3961 	}
3962 }
3963 
3964 /* tp->lock is held. */
3965 static void __tg3_set_mac_addr(struct tg3 *tp, bool skip_mac_1)
3966 {
3967 	u32 addr_high;
3968 	int i;
3969 
3970 	for (i = 0; i < 4; i++) {
3971 		if (i == 1 && skip_mac_1)
3972 			continue;
3973 		__tg3_set_one_mac_addr(tp, tp->dev->dev_addr, i);
3974 	}
3975 
3976 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
3977 	    tg3_asic_rev(tp) == ASIC_REV_5704) {
3978 		for (i = 4; i < 16; i++)
3979 			__tg3_set_one_mac_addr(tp, tp->dev->dev_addr, i);
3980 	}
3981 
3982 	addr_high = (tp->dev->dev_addr[0] +
3983 		     tp->dev->dev_addr[1] +
3984 		     tp->dev->dev_addr[2] +
3985 		     tp->dev->dev_addr[3] +
3986 		     tp->dev->dev_addr[4] +
3987 		     tp->dev->dev_addr[5]) &
3988 		TX_BACKOFF_SEED_MASK;
3989 	tw32(MAC_TX_BACKOFF_SEED, addr_high);
3990 }
3991 
3992 static void tg3_enable_register_access(struct tg3 *tp)
3993 {
3994 	/*
3995 	 * Make sure register accesses (indirect or otherwise) will function
3996 	 * correctly.
3997 	 */
3998 	pci_write_config_dword(tp->pdev,
3999 			       TG3PCI_MISC_HOST_CTRL, tp->misc_host_ctrl);
4000 }
4001 
4002 static int tg3_power_up(struct tg3 *tp)
4003 {
4004 	int err;
4005 
4006 	tg3_enable_register_access(tp);
4007 
4008 	err = pci_set_power_state(tp->pdev, PCI_D0);
4009 	if (!err) {
4010 		/* Switch out of Vaux if it is a NIC */
4011 		tg3_pwrsrc_switch_to_vmain(tp);
4012 	} else {
4013 		netdev_err(tp->dev, "Transition to D0 failed\n");
4014 	}
4015 
4016 	return err;
4017 }
4018 
4019 static int tg3_setup_phy(struct tg3 *, bool);
4020 
4021 static int tg3_power_down_prepare(struct tg3 *tp)
4022 {
4023 	u32 misc_host_ctrl;
4024 	bool device_should_wake, do_low_power;
4025 
4026 	tg3_enable_register_access(tp);
4027 
4028 	/* Restore the CLKREQ setting. */
4029 	if (tg3_flag(tp, CLKREQ_BUG))
4030 		pcie_capability_set_word(tp->pdev, PCI_EXP_LNKCTL,
4031 					 PCI_EXP_LNKCTL_CLKREQ_EN);
4032 
4033 	misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
4034 	tw32(TG3PCI_MISC_HOST_CTRL,
4035 	     misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
4036 
4037 	device_should_wake = device_may_wakeup(&tp->pdev->dev) &&
4038 			     tg3_flag(tp, WOL_ENABLE);
4039 
4040 	if (tg3_flag(tp, USE_PHYLIB)) {
4041 		do_low_power = false;
4042 		if ((tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) &&
4043 		    !(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4044 			__ETHTOOL_DECLARE_LINK_MODE_MASK(advertising) = { 0, };
4045 			struct phy_device *phydev;
4046 			u32 phyid;
4047 
4048 			phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
4049 
4050 			tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
4051 
4052 			tp->link_config.speed = phydev->speed;
4053 			tp->link_config.duplex = phydev->duplex;
4054 			tp->link_config.autoneg = phydev->autoneg;
4055 			ethtool_convert_link_mode_to_legacy_u32(
4056 				&tp->link_config.advertising,
4057 				phydev->advertising);
4058 
4059 			linkmode_set_bit(ETHTOOL_LINK_MODE_TP_BIT, advertising);
4060 			linkmode_set_bit(ETHTOOL_LINK_MODE_Pause_BIT,
4061 					 advertising);
4062 			linkmode_set_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
4063 					 advertising);
4064 			linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT,
4065 					 advertising);
4066 
4067 			if (tg3_flag(tp, ENABLE_ASF) || device_should_wake) {
4068 				if (tg3_flag(tp, WOL_SPEED_100MB)) {
4069 					linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT,
4070 							 advertising);
4071 					linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT,
4072 							 advertising);
4073 					linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT,
4074 							 advertising);
4075 				} else {
4076 					linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT,
4077 							 advertising);
4078 				}
4079 			}
4080 
4081 			linkmode_copy(phydev->advertising, advertising);
4082 			phy_start_aneg(phydev);
4083 
4084 			phyid = phydev->drv->phy_id & phydev->drv->phy_id_mask;
4085 			if (phyid != PHY_ID_BCMAC131) {
4086 				phyid &= PHY_BCM_OUI_MASK;
4087 				if (phyid == PHY_BCM_OUI_1 ||
4088 				    phyid == PHY_BCM_OUI_2 ||
4089 				    phyid == PHY_BCM_OUI_3)
4090 					do_low_power = true;
4091 			}
4092 		}
4093 	} else {
4094 		do_low_power = true;
4095 
4096 		if (!(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER))
4097 			tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
4098 
4099 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
4100 			tg3_setup_phy(tp, false);
4101 	}
4102 
4103 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
4104 		u32 val;
4105 
4106 		val = tr32(GRC_VCPU_EXT_CTRL);
4107 		tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
4108 	} else if (!tg3_flag(tp, ENABLE_ASF)) {
4109 		int i;
4110 		u32 val;
4111 
4112 		for (i = 0; i < 200; i++) {
4113 			tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
4114 			if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
4115 				break;
4116 			msleep(1);
4117 		}
4118 	}
4119 	if (tg3_flag(tp, WOL_CAP))
4120 		tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
4121 						     WOL_DRV_STATE_SHUTDOWN |
4122 						     WOL_DRV_WOL |
4123 						     WOL_SET_MAGIC_PKT);
4124 
4125 	if (device_should_wake) {
4126 		u32 mac_mode;
4127 
4128 		if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
4129 			if (do_low_power &&
4130 			    !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
4131 				tg3_phy_auxctl_write(tp,
4132 					       MII_TG3_AUXCTL_SHDWSEL_PWRCTL,
4133 					       MII_TG3_AUXCTL_PCTL_WOL_EN |
4134 					       MII_TG3_AUXCTL_PCTL_100TX_LPWR |
4135 					       MII_TG3_AUXCTL_PCTL_CL_AB_TXDAC);
4136 				udelay(40);
4137 			}
4138 
4139 			if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
4140 				mac_mode = MAC_MODE_PORT_MODE_GMII;
4141 			else if (tp->phy_flags &
4142 				 TG3_PHYFLG_KEEP_LINK_ON_PWRDN) {
4143 				if (tp->link_config.active_speed == SPEED_1000)
4144 					mac_mode = MAC_MODE_PORT_MODE_GMII;
4145 				else
4146 					mac_mode = MAC_MODE_PORT_MODE_MII;
4147 			} else
4148 				mac_mode = MAC_MODE_PORT_MODE_MII;
4149 
4150 			mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
4151 			if (tg3_asic_rev(tp) == ASIC_REV_5700) {
4152 				u32 speed = tg3_flag(tp, WOL_SPEED_100MB) ?
4153 					     SPEED_100 : SPEED_10;
4154 				if (tg3_5700_link_polarity(tp, speed))
4155 					mac_mode |= MAC_MODE_LINK_POLARITY;
4156 				else
4157 					mac_mode &= ~MAC_MODE_LINK_POLARITY;
4158 			}
4159 		} else {
4160 			mac_mode = MAC_MODE_PORT_MODE_TBI;
4161 		}
4162 
4163 		if (!tg3_flag(tp, 5750_PLUS))
4164 			tw32(MAC_LED_CTRL, tp->led_ctrl);
4165 
4166 		mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
4167 		if ((tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS)) &&
4168 		    (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)))
4169 			mac_mode |= MAC_MODE_KEEP_FRAME_IN_WOL;
4170 
4171 		if (tg3_flag(tp, ENABLE_APE))
4172 			mac_mode |= MAC_MODE_APE_TX_EN |
4173 				    MAC_MODE_APE_RX_EN |
4174 				    MAC_MODE_TDE_ENABLE;
4175 
4176 		tw32_f(MAC_MODE, mac_mode);
4177 		udelay(100);
4178 
4179 		tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
4180 		udelay(10);
4181 	}
4182 
4183 	if (!tg3_flag(tp, WOL_SPEED_100MB) &&
4184 	    (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4185 	     tg3_asic_rev(tp) == ASIC_REV_5701)) {
4186 		u32 base_val;
4187 
4188 		base_val = tp->pci_clock_ctrl;
4189 		base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
4190 			     CLOCK_CTRL_TXCLK_DISABLE);
4191 
4192 		tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
4193 			    CLOCK_CTRL_PWRDOWN_PLL133, 40);
4194 	} else if (tg3_flag(tp, 5780_CLASS) ||
4195 		   tg3_flag(tp, CPMU_PRESENT) ||
4196 		   tg3_asic_rev(tp) == ASIC_REV_5906) {
4197 		/* do nothing */
4198 	} else if (!(tg3_flag(tp, 5750_PLUS) && tg3_flag(tp, ENABLE_ASF))) {
4199 		u32 newbits1, newbits2;
4200 
4201 		if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4202 		    tg3_asic_rev(tp) == ASIC_REV_5701) {
4203 			newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
4204 				    CLOCK_CTRL_TXCLK_DISABLE |
4205 				    CLOCK_CTRL_ALTCLK);
4206 			newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
4207 		} else if (tg3_flag(tp, 5705_PLUS)) {
4208 			newbits1 = CLOCK_CTRL_625_CORE;
4209 			newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
4210 		} else {
4211 			newbits1 = CLOCK_CTRL_ALTCLK;
4212 			newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
4213 		}
4214 
4215 		tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
4216 			    40);
4217 
4218 		tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
4219 			    40);
4220 
4221 		if (!tg3_flag(tp, 5705_PLUS)) {
4222 			u32 newbits3;
4223 
4224 			if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4225 			    tg3_asic_rev(tp) == ASIC_REV_5701) {
4226 				newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
4227 					    CLOCK_CTRL_TXCLK_DISABLE |
4228 					    CLOCK_CTRL_44MHZ_CORE);
4229 			} else {
4230 				newbits3 = CLOCK_CTRL_44MHZ_CORE;
4231 			}
4232 
4233 			tw32_wait_f(TG3PCI_CLOCK_CTRL,
4234 				    tp->pci_clock_ctrl | newbits3, 40);
4235 		}
4236 	}
4237 
4238 	if (!(device_should_wake) && !tg3_flag(tp, ENABLE_ASF))
4239 		tg3_power_down_phy(tp, do_low_power);
4240 
4241 	tg3_frob_aux_power(tp, true);
4242 
4243 	/* Workaround for unstable PLL clock */
4244 	if ((!tg3_flag(tp, IS_SSB_CORE)) &&
4245 	    ((tg3_chip_rev(tp) == CHIPREV_5750_AX) ||
4246 	     (tg3_chip_rev(tp) == CHIPREV_5750_BX))) {
4247 		u32 val = tr32(0x7d00);
4248 
4249 		val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
4250 		tw32(0x7d00, val);
4251 		if (!tg3_flag(tp, ENABLE_ASF)) {
4252 			int err;
4253 
4254 			err = tg3_nvram_lock(tp);
4255 			tg3_halt_cpu(tp, RX_CPU_BASE);
4256 			if (!err)
4257 				tg3_nvram_unlock(tp);
4258 		}
4259 	}
4260 
4261 	tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
4262 
4263 	tg3_ape_driver_state_change(tp, RESET_KIND_SHUTDOWN);
4264 
4265 	return 0;
4266 }
4267 
4268 static void tg3_power_down(struct tg3 *tp)
4269 {
4270 	pci_wake_from_d3(tp->pdev, tg3_flag(tp, WOL_ENABLE));
4271 	pci_set_power_state(tp->pdev, PCI_D3hot);
4272 }
4273 
4274 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u32 *speed, u8 *duplex)
4275 {
4276 	switch (val & MII_TG3_AUX_STAT_SPDMASK) {
4277 	case MII_TG3_AUX_STAT_10HALF:
4278 		*speed = SPEED_10;
4279 		*duplex = DUPLEX_HALF;
4280 		break;
4281 
4282 	case MII_TG3_AUX_STAT_10FULL:
4283 		*speed = SPEED_10;
4284 		*duplex = DUPLEX_FULL;
4285 		break;
4286 
4287 	case MII_TG3_AUX_STAT_100HALF:
4288 		*speed = SPEED_100;
4289 		*duplex = DUPLEX_HALF;
4290 		break;
4291 
4292 	case MII_TG3_AUX_STAT_100FULL:
4293 		*speed = SPEED_100;
4294 		*duplex = DUPLEX_FULL;
4295 		break;
4296 
4297 	case MII_TG3_AUX_STAT_1000HALF:
4298 		*speed = SPEED_1000;
4299 		*duplex = DUPLEX_HALF;
4300 		break;
4301 
4302 	case MII_TG3_AUX_STAT_1000FULL:
4303 		*speed = SPEED_1000;
4304 		*duplex = DUPLEX_FULL;
4305 		break;
4306 
4307 	default:
4308 		if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
4309 			*speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
4310 				 SPEED_10;
4311 			*duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
4312 				  DUPLEX_HALF;
4313 			break;
4314 		}
4315 		*speed = SPEED_UNKNOWN;
4316 		*duplex = DUPLEX_UNKNOWN;
4317 		break;
4318 	}
4319 }
4320 
4321 static int tg3_phy_autoneg_cfg(struct tg3 *tp, u32 advertise, u32 flowctrl)
4322 {
4323 	int err = 0;
4324 	u32 val, new_adv;
4325 
4326 	new_adv = ADVERTISE_CSMA;
4327 	new_adv |= ethtool_adv_to_mii_adv_t(advertise) & ADVERTISE_ALL;
4328 	new_adv |= mii_advertise_flowctrl(flowctrl);
4329 
4330 	err = tg3_writephy(tp, MII_ADVERTISE, new_adv);
4331 	if (err)
4332 		goto done;
4333 
4334 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4335 		new_adv = ethtool_adv_to_mii_ctrl1000_t(advertise);
4336 
4337 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4338 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0)
4339 			new_adv |= CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER;
4340 
4341 		err = tg3_writephy(tp, MII_CTRL1000, new_adv);
4342 		if (err)
4343 			goto done;
4344 	}
4345 
4346 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
4347 		goto done;
4348 
4349 	tw32(TG3_CPMU_EEE_MODE,
4350 	     tr32(TG3_CPMU_EEE_MODE) & ~TG3_CPMU_EEEMD_LPI_ENABLE);
4351 
4352 	err = tg3_phy_toggle_auxctl_smdsp(tp, true);
4353 	if (!err) {
4354 		u32 err2;
4355 
4356 		val = 0;
4357 		/* Advertise 100-BaseTX EEE ability */
4358 		if (advertise & ADVERTISED_100baseT_Full)
4359 			val |= MDIO_AN_EEE_ADV_100TX;
4360 		/* Advertise 1000-BaseT EEE ability */
4361 		if (advertise & ADVERTISED_1000baseT_Full)
4362 			val |= MDIO_AN_EEE_ADV_1000T;
4363 
4364 		if (!tp->eee.eee_enabled) {
4365 			val = 0;
4366 			tp->eee.advertised = 0;
4367 		} else {
4368 			tp->eee.advertised = advertise &
4369 					     (ADVERTISED_100baseT_Full |
4370 					      ADVERTISED_1000baseT_Full);
4371 		}
4372 
4373 		err = tg3_phy_cl45_write(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, val);
4374 		if (err)
4375 			val = 0;
4376 
4377 		switch (tg3_asic_rev(tp)) {
4378 		case ASIC_REV_5717:
4379 		case ASIC_REV_57765:
4380 		case ASIC_REV_57766:
4381 		case ASIC_REV_5719:
4382 			/* If we advertised any eee advertisements above... */
4383 			if (val)
4384 				val = MII_TG3_DSP_TAP26_ALNOKO |
4385 				      MII_TG3_DSP_TAP26_RMRXSTO |
4386 				      MII_TG3_DSP_TAP26_OPCSINPT;
4387 			tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
4388 			fallthrough;
4389 		case ASIC_REV_5720:
4390 		case ASIC_REV_5762:
4391 			if (!tg3_phydsp_read(tp, MII_TG3_DSP_CH34TP2, &val))
4392 				tg3_phydsp_write(tp, MII_TG3_DSP_CH34TP2, val |
4393 						 MII_TG3_DSP_CH34TP2_HIBW01);
4394 		}
4395 
4396 		err2 = tg3_phy_toggle_auxctl_smdsp(tp, false);
4397 		if (!err)
4398 			err = err2;
4399 	}
4400 
4401 done:
4402 	return err;
4403 }
4404 
4405 static void tg3_phy_copper_begin(struct tg3 *tp)
4406 {
4407 	if (tp->link_config.autoneg == AUTONEG_ENABLE ||
4408 	    (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4409 		u32 adv, fc;
4410 
4411 		if ((tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) &&
4412 		    !(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)) {
4413 			adv = ADVERTISED_10baseT_Half |
4414 			      ADVERTISED_10baseT_Full;
4415 			if (tg3_flag(tp, WOL_SPEED_100MB))
4416 				adv |= ADVERTISED_100baseT_Half |
4417 				       ADVERTISED_100baseT_Full;
4418 			if (tp->phy_flags & TG3_PHYFLG_1G_ON_VAUX_OK) {
4419 				if (!(tp->phy_flags &
4420 				      TG3_PHYFLG_DISABLE_1G_HD_ADV))
4421 					adv |= ADVERTISED_1000baseT_Half;
4422 				adv |= ADVERTISED_1000baseT_Full;
4423 			}
4424 
4425 			fc = FLOW_CTRL_TX | FLOW_CTRL_RX;
4426 		} else {
4427 			adv = tp->link_config.advertising;
4428 			if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
4429 				adv &= ~(ADVERTISED_1000baseT_Half |
4430 					 ADVERTISED_1000baseT_Full);
4431 
4432 			fc = tp->link_config.flowctrl;
4433 		}
4434 
4435 		tg3_phy_autoneg_cfg(tp, adv, fc);
4436 
4437 		if ((tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) &&
4438 		    (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN)) {
4439 			/* Normally during power down we want to autonegotiate
4440 			 * the lowest possible speed for WOL. However, to avoid
4441 			 * link flap, we leave it untouched.
4442 			 */
4443 			return;
4444 		}
4445 
4446 		tg3_writephy(tp, MII_BMCR,
4447 			     BMCR_ANENABLE | BMCR_ANRESTART);
4448 	} else {
4449 		int i;
4450 		u32 bmcr, orig_bmcr;
4451 
4452 		tp->link_config.active_speed = tp->link_config.speed;
4453 		tp->link_config.active_duplex = tp->link_config.duplex;
4454 
4455 		if (tg3_asic_rev(tp) == ASIC_REV_5714) {
4456 			/* With autoneg disabled, 5715 only links up when the
4457 			 * advertisement register has the configured speed
4458 			 * enabled.
4459 			 */
4460 			tg3_writephy(tp, MII_ADVERTISE, ADVERTISE_ALL);
4461 		}
4462 
4463 		bmcr = 0;
4464 		switch (tp->link_config.speed) {
4465 		default:
4466 		case SPEED_10:
4467 			break;
4468 
4469 		case SPEED_100:
4470 			bmcr |= BMCR_SPEED100;
4471 			break;
4472 
4473 		case SPEED_1000:
4474 			bmcr |= BMCR_SPEED1000;
4475 			break;
4476 		}
4477 
4478 		if (tp->link_config.duplex == DUPLEX_FULL)
4479 			bmcr |= BMCR_FULLDPLX;
4480 
4481 		if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
4482 		    (bmcr != orig_bmcr)) {
4483 			tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
4484 			for (i = 0; i < 1500; i++) {
4485 				u32 tmp;
4486 
4487 				udelay(10);
4488 				if (tg3_readphy(tp, MII_BMSR, &tmp) ||
4489 				    tg3_readphy(tp, MII_BMSR, &tmp))
4490 					continue;
4491 				if (!(tmp & BMSR_LSTATUS)) {
4492 					udelay(40);
4493 					break;
4494 				}
4495 			}
4496 			tg3_writephy(tp, MII_BMCR, bmcr);
4497 			udelay(40);
4498 		}
4499 	}
4500 }
4501 
4502 static int tg3_phy_pull_config(struct tg3 *tp)
4503 {
4504 	int err;
4505 	u32 val;
4506 
4507 	err = tg3_readphy(tp, MII_BMCR, &val);
4508 	if (err)
4509 		goto done;
4510 
4511 	if (!(val & BMCR_ANENABLE)) {
4512 		tp->link_config.autoneg = AUTONEG_DISABLE;
4513 		tp->link_config.advertising = 0;
4514 		tg3_flag_clear(tp, PAUSE_AUTONEG);
4515 
4516 		err = -EIO;
4517 
4518 		switch (val & (BMCR_SPEED1000 | BMCR_SPEED100)) {
4519 		case 0:
4520 			if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
4521 				goto done;
4522 
4523 			tp->link_config.speed = SPEED_10;
4524 			break;
4525 		case BMCR_SPEED100:
4526 			if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
4527 				goto done;
4528 
4529 			tp->link_config.speed = SPEED_100;
4530 			break;
4531 		case BMCR_SPEED1000:
4532 			if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4533 				tp->link_config.speed = SPEED_1000;
4534 				break;
4535 			}
4536 			fallthrough;
4537 		default:
4538 			goto done;
4539 		}
4540 
4541 		if (val & BMCR_FULLDPLX)
4542 			tp->link_config.duplex = DUPLEX_FULL;
4543 		else
4544 			tp->link_config.duplex = DUPLEX_HALF;
4545 
4546 		tp->link_config.flowctrl = FLOW_CTRL_RX | FLOW_CTRL_TX;
4547 
4548 		err = 0;
4549 		goto done;
4550 	}
4551 
4552 	tp->link_config.autoneg = AUTONEG_ENABLE;
4553 	tp->link_config.advertising = ADVERTISED_Autoneg;
4554 	tg3_flag_set(tp, PAUSE_AUTONEG);
4555 
4556 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
4557 		u32 adv;
4558 
4559 		err = tg3_readphy(tp, MII_ADVERTISE, &val);
4560 		if (err)
4561 			goto done;
4562 
4563 		adv = mii_adv_to_ethtool_adv_t(val & ADVERTISE_ALL);
4564 		tp->link_config.advertising |= adv | ADVERTISED_TP;
4565 
4566 		tp->link_config.flowctrl = tg3_decode_flowctrl_1000T(val);
4567 	} else {
4568 		tp->link_config.advertising |= ADVERTISED_FIBRE;
4569 	}
4570 
4571 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4572 		u32 adv;
4573 
4574 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
4575 			err = tg3_readphy(tp, MII_CTRL1000, &val);
4576 			if (err)
4577 				goto done;
4578 
4579 			adv = mii_ctrl1000_to_ethtool_adv_t(val);
4580 		} else {
4581 			err = tg3_readphy(tp, MII_ADVERTISE, &val);
4582 			if (err)
4583 				goto done;
4584 
4585 			adv = tg3_decode_flowctrl_1000X(val);
4586 			tp->link_config.flowctrl = adv;
4587 
4588 			val &= (ADVERTISE_1000XHALF | ADVERTISE_1000XFULL);
4589 			adv = mii_adv_to_ethtool_adv_x(val);
4590 		}
4591 
4592 		tp->link_config.advertising |= adv;
4593 	}
4594 
4595 done:
4596 	return err;
4597 }
4598 
4599 static int tg3_init_5401phy_dsp(struct tg3 *tp)
4600 {
4601 	int err;
4602 
4603 	/* Turn off tap power management. */
4604 	/* Set Extended packet length bit */
4605 	err = tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
4606 
4607 	err |= tg3_phydsp_write(tp, 0x0012, 0x1804);
4608 	err |= tg3_phydsp_write(tp, 0x0013, 0x1204);
4609 	err |= tg3_phydsp_write(tp, 0x8006, 0x0132);
4610 	err |= tg3_phydsp_write(tp, 0x8006, 0x0232);
4611 	err |= tg3_phydsp_write(tp, 0x201f, 0x0a20);
4612 
4613 	udelay(40);
4614 
4615 	return err;
4616 }
4617 
4618 static bool tg3_phy_eee_config_ok(struct tg3 *tp)
4619 {
4620 	struct ethtool_eee eee;
4621 
4622 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
4623 		return true;
4624 
4625 	tg3_eee_pull_config(tp, &eee);
4626 
4627 	if (tp->eee.eee_enabled) {
4628 		if (tp->eee.advertised != eee.advertised ||
4629 		    tp->eee.tx_lpi_timer != eee.tx_lpi_timer ||
4630 		    tp->eee.tx_lpi_enabled != eee.tx_lpi_enabled)
4631 			return false;
4632 	} else {
4633 		/* EEE is disabled but we're advertising */
4634 		if (eee.advertised)
4635 			return false;
4636 	}
4637 
4638 	return true;
4639 }
4640 
4641 static bool tg3_phy_copper_an_config_ok(struct tg3 *tp, u32 *lcladv)
4642 {
4643 	u32 advmsk, tgtadv, advertising;
4644 
4645 	advertising = tp->link_config.advertising;
4646 	tgtadv = ethtool_adv_to_mii_adv_t(advertising) & ADVERTISE_ALL;
4647 
4648 	advmsk = ADVERTISE_ALL;
4649 	if (tp->link_config.active_duplex == DUPLEX_FULL) {
4650 		tgtadv |= mii_advertise_flowctrl(tp->link_config.flowctrl);
4651 		advmsk |= ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
4652 	}
4653 
4654 	if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
4655 		return false;
4656 
4657 	if ((*lcladv & advmsk) != tgtadv)
4658 		return false;
4659 
4660 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4661 		u32 tg3_ctrl;
4662 
4663 		tgtadv = ethtool_adv_to_mii_ctrl1000_t(advertising);
4664 
4665 		if (tg3_readphy(tp, MII_CTRL1000, &tg3_ctrl))
4666 			return false;
4667 
4668 		if (tgtadv &&
4669 		    (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4670 		     tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0)) {
4671 			tgtadv |= CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER;
4672 			tg3_ctrl &= (ADVERTISE_1000HALF | ADVERTISE_1000FULL |
4673 				     CTL1000_AS_MASTER | CTL1000_ENABLE_MASTER);
4674 		} else {
4675 			tg3_ctrl &= (ADVERTISE_1000HALF | ADVERTISE_1000FULL);
4676 		}
4677 
4678 		if (tg3_ctrl != tgtadv)
4679 			return false;
4680 	}
4681 
4682 	return true;
4683 }
4684 
4685 static bool tg3_phy_copper_fetch_rmtadv(struct tg3 *tp, u32 *rmtadv)
4686 {
4687 	u32 lpeth = 0;
4688 
4689 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
4690 		u32 val;
4691 
4692 		if (tg3_readphy(tp, MII_STAT1000, &val))
4693 			return false;
4694 
4695 		lpeth = mii_stat1000_to_ethtool_lpa_t(val);
4696 	}
4697 
4698 	if (tg3_readphy(tp, MII_LPA, rmtadv))
4699 		return false;
4700 
4701 	lpeth |= mii_lpa_to_ethtool_lpa_t(*rmtadv);
4702 	tp->link_config.rmt_adv = lpeth;
4703 
4704 	return true;
4705 }
4706 
4707 static bool tg3_test_and_report_link_chg(struct tg3 *tp, bool curr_link_up)
4708 {
4709 	if (curr_link_up != tp->link_up) {
4710 		if (curr_link_up) {
4711 			netif_carrier_on(tp->dev);
4712 		} else {
4713 			netif_carrier_off(tp->dev);
4714 			if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
4715 				tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4716 		}
4717 
4718 		tg3_link_report(tp);
4719 		return true;
4720 	}
4721 
4722 	return false;
4723 }
4724 
4725 static void tg3_clear_mac_status(struct tg3 *tp)
4726 {
4727 	tw32(MAC_EVENT, 0);
4728 
4729 	tw32_f(MAC_STATUS,
4730 	       MAC_STATUS_SYNC_CHANGED |
4731 	       MAC_STATUS_CFG_CHANGED |
4732 	       MAC_STATUS_MI_COMPLETION |
4733 	       MAC_STATUS_LNKSTATE_CHANGED);
4734 	udelay(40);
4735 }
4736 
4737 static void tg3_setup_eee(struct tg3 *tp)
4738 {
4739 	u32 val;
4740 
4741 	val = TG3_CPMU_EEE_LNKIDL_PCIE_NL0 |
4742 	      TG3_CPMU_EEE_LNKIDL_UART_IDL;
4743 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0)
4744 		val |= TG3_CPMU_EEE_LNKIDL_APE_TX_MT;
4745 
4746 	tw32_f(TG3_CPMU_EEE_LNKIDL_CTRL, val);
4747 
4748 	tw32_f(TG3_CPMU_EEE_CTRL,
4749 	       TG3_CPMU_EEE_CTRL_EXIT_20_1_US);
4750 
4751 	val = TG3_CPMU_EEEMD_ERLY_L1_XIT_DET |
4752 	      (tp->eee.tx_lpi_enabled ? TG3_CPMU_EEEMD_LPI_IN_TX : 0) |
4753 	      TG3_CPMU_EEEMD_LPI_IN_RX |
4754 	      TG3_CPMU_EEEMD_EEE_ENABLE;
4755 
4756 	if (tg3_asic_rev(tp) != ASIC_REV_5717)
4757 		val |= TG3_CPMU_EEEMD_SND_IDX_DET_EN;
4758 
4759 	if (tg3_flag(tp, ENABLE_APE))
4760 		val |= TG3_CPMU_EEEMD_APE_TX_DET_EN;
4761 
4762 	tw32_f(TG3_CPMU_EEE_MODE, tp->eee.eee_enabled ? val : 0);
4763 
4764 	tw32_f(TG3_CPMU_EEE_DBTMR1,
4765 	       TG3_CPMU_DBTMR1_PCIEXIT_2047US |
4766 	       (tp->eee.tx_lpi_timer & 0xffff));
4767 
4768 	tw32_f(TG3_CPMU_EEE_DBTMR2,
4769 	       TG3_CPMU_DBTMR2_APE_TX_2047US |
4770 	       TG3_CPMU_DBTMR2_TXIDXEQ_2047US);
4771 }
4772 
4773 static int tg3_setup_copper_phy(struct tg3 *tp, bool force_reset)
4774 {
4775 	bool current_link_up;
4776 	u32 bmsr, val;
4777 	u32 lcl_adv, rmt_adv;
4778 	u32 current_speed;
4779 	u8 current_duplex;
4780 	int i, err;
4781 
4782 	tg3_clear_mac_status(tp);
4783 
4784 	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
4785 		tw32_f(MAC_MI_MODE,
4786 		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
4787 		udelay(80);
4788 	}
4789 
4790 	tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, 0);
4791 
4792 	/* Some third-party PHYs need to be reset on link going
4793 	 * down.
4794 	 */
4795 	if ((tg3_asic_rev(tp) == ASIC_REV_5703 ||
4796 	     tg3_asic_rev(tp) == ASIC_REV_5704 ||
4797 	     tg3_asic_rev(tp) == ASIC_REV_5705) &&
4798 	    tp->link_up) {
4799 		tg3_readphy(tp, MII_BMSR, &bmsr);
4800 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4801 		    !(bmsr & BMSR_LSTATUS))
4802 			force_reset = true;
4803 	}
4804 	if (force_reset)
4805 		tg3_phy_reset(tp);
4806 
4807 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
4808 		tg3_readphy(tp, MII_BMSR, &bmsr);
4809 		if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
4810 		    !tg3_flag(tp, INIT_COMPLETE))
4811 			bmsr = 0;
4812 
4813 		if (!(bmsr & BMSR_LSTATUS)) {
4814 			err = tg3_init_5401phy_dsp(tp);
4815 			if (err)
4816 				return err;
4817 
4818 			tg3_readphy(tp, MII_BMSR, &bmsr);
4819 			for (i = 0; i < 1000; i++) {
4820 				udelay(10);
4821 				if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4822 				    (bmsr & BMSR_LSTATUS)) {
4823 					udelay(40);
4824 					break;
4825 				}
4826 			}
4827 
4828 			if ((tp->phy_id & TG3_PHY_ID_REV_MASK) ==
4829 			    TG3_PHY_REV_BCM5401_B0 &&
4830 			    !(bmsr & BMSR_LSTATUS) &&
4831 			    tp->link_config.active_speed == SPEED_1000) {
4832 				err = tg3_phy_reset(tp);
4833 				if (!err)
4834 					err = tg3_init_5401phy_dsp(tp);
4835 				if (err)
4836 					return err;
4837 			}
4838 		}
4839 	} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
4840 		   tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0) {
4841 		/* 5701 {A0,B0} CRC bug workaround */
4842 		tg3_writephy(tp, 0x15, 0x0a75);
4843 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
4844 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
4845 		tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
4846 	}
4847 
4848 	/* Clear pending interrupts... */
4849 	tg3_readphy(tp, MII_TG3_ISTAT, &val);
4850 	tg3_readphy(tp, MII_TG3_ISTAT, &val);
4851 
4852 	if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT)
4853 		tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
4854 	else if (!(tp->phy_flags & TG3_PHYFLG_IS_FET))
4855 		tg3_writephy(tp, MII_TG3_IMASK, ~0);
4856 
4857 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
4858 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
4859 		if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
4860 			tg3_writephy(tp, MII_TG3_EXT_CTRL,
4861 				     MII_TG3_EXT_CTRL_LNK3_LED_MODE);
4862 		else
4863 			tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
4864 	}
4865 
4866 	current_link_up = false;
4867 	current_speed = SPEED_UNKNOWN;
4868 	current_duplex = DUPLEX_UNKNOWN;
4869 	tp->phy_flags &= ~TG3_PHYFLG_MDIX_STATE;
4870 	tp->link_config.rmt_adv = 0;
4871 
4872 	if (tp->phy_flags & TG3_PHYFLG_CAPACITIVE_COUPLING) {
4873 		err = tg3_phy_auxctl_read(tp,
4874 					  MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
4875 					  &val);
4876 		if (!err && !(val & (1 << 10))) {
4877 			tg3_phy_auxctl_write(tp,
4878 					     MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
4879 					     val | (1 << 10));
4880 			goto relink;
4881 		}
4882 	}
4883 
4884 	bmsr = 0;
4885 	for (i = 0; i < 100; i++) {
4886 		tg3_readphy(tp, MII_BMSR, &bmsr);
4887 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
4888 		    (bmsr & BMSR_LSTATUS))
4889 			break;
4890 		udelay(40);
4891 	}
4892 
4893 	if (bmsr & BMSR_LSTATUS) {
4894 		u32 aux_stat, bmcr;
4895 
4896 		tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
4897 		for (i = 0; i < 2000; i++) {
4898 			udelay(10);
4899 			if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
4900 			    aux_stat)
4901 				break;
4902 		}
4903 
4904 		tg3_aux_stat_to_speed_duplex(tp, aux_stat,
4905 					     &current_speed,
4906 					     &current_duplex);
4907 
4908 		bmcr = 0;
4909 		for (i = 0; i < 200; i++) {
4910 			tg3_readphy(tp, MII_BMCR, &bmcr);
4911 			if (tg3_readphy(tp, MII_BMCR, &bmcr))
4912 				continue;
4913 			if (bmcr && bmcr != 0x7fff)
4914 				break;
4915 			udelay(10);
4916 		}
4917 
4918 		lcl_adv = 0;
4919 		rmt_adv = 0;
4920 
4921 		tp->link_config.active_speed = current_speed;
4922 		tp->link_config.active_duplex = current_duplex;
4923 
4924 		if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4925 			bool eee_config_ok = tg3_phy_eee_config_ok(tp);
4926 
4927 			if ((bmcr & BMCR_ANENABLE) &&
4928 			    eee_config_ok &&
4929 			    tg3_phy_copper_an_config_ok(tp, &lcl_adv) &&
4930 			    tg3_phy_copper_fetch_rmtadv(tp, &rmt_adv))
4931 				current_link_up = true;
4932 
4933 			/* EEE settings changes take effect only after a phy
4934 			 * reset.  If we have skipped a reset due to Link Flap
4935 			 * Avoidance being enabled, do it now.
4936 			 */
4937 			if (!eee_config_ok &&
4938 			    (tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
4939 			    !force_reset) {
4940 				tg3_setup_eee(tp);
4941 				tg3_phy_reset(tp);
4942 			}
4943 		} else {
4944 			if (!(bmcr & BMCR_ANENABLE) &&
4945 			    tp->link_config.speed == current_speed &&
4946 			    tp->link_config.duplex == current_duplex) {
4947 				current_link_up = true;
4948 			}
4949 		}
4950 
4951 		if (current_link_up &&
4952 		    tp->link_config.active_duplex == DUPLEX_FULL) {
4953 			u32 reg, bit;
4954 
4955 			if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
4956 				reg = MII_TG3_FET_GEN_STAT;
4957 				bit = MII_TG3_FET_GEN_STAT_MDIXSTAT;
4958 			} else {
4959 				reg = MII_TG3_EXT_STAT;
4960 				bit = MII_TG3_EXT_STAT_MDIX;
4961 			}
4962 
4963 			if (!tg3_readphy(tp, reg, &val) && (val & bit))
4964 				tp->phy_flags |= TG3_PHYFLG_MDIX_STATE;
4965 
4966 			tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
4967 		}
4968 	}
4969 
4970 relink:
4971 	if (!current_link_up || (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
4972 		tg3_phy_copper_begin(tp);
4973 
4974 		if (tg3_flag(tp, ROBOSWITCH)) {
4975 			current_link_up = true;
4976 			/* FIXME: when BCM5325 switch is used use 100 MBit/s */
4977 			current_speed = SPEED_1000;
4978 			current_duplex = DUPLEX_FULL;
4979 			tp->link_config.active_speed = current_speed;
4980 			tp->link_config.active_duplex = current_duplex;
4981 		}
4982 
4983 		tg3_readphy(tp, MII_BMSR, &bmsr);
4984 		if ((!tg3_readphy(tp, MII_BMSR, &bmsr) && (bmsr & BMSR_LSTATUS)) ||
4985 		    (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
4986 			current_link_up = true;
4987 	}
4988 
4989 	tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
4990 	if (current_link_up) {
4991 		if (tp->link_config.active_speed == SPEED_100 ||
4992 		    tp->link_config.active_speed == SPEED_10)
4993 			tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
4994 		else
4995 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4996 	} else if (tp->phy_flags & TG3_PHYFLG_IS_FET)
4997 		tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
4998 	else
4999 		tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5000 
5001 	/* In order for the 5750 core in BCM4785 chip to work properly
5002 	 * in RGMII mode, the Led Control Register must be set up.
5003 	 */
5004 	if (tg3_flag(tp, RGMII_MODE)) {
5005 		u32 led_ctrl = tr32(MAC_LED_CTRL);
5006 		led_ctrl &= ~(LED_CTRL_1000MBPS_ON | LED_CTRL_100MBPS_ON);
5007 
5008 		if (tp->link_config.active_speed == SPEED_10)
5009 			led_ctrl |= LED_CTRL_LNKLED_OVERRIDE;
5010 		else if (tp->link_config.active_speed == SPEED_100)
5011 			led_ctrl |= (LED_CTRL_LNKLED_OVERRIDE |
5012 				     LED_CTRL_100MBPS_ON);
5013 		else if (tp->link_config.active_speed == SPEED_1000)
5014 			led_ctrl |= (LED_CTRL_LNKLED_OVERRIDE |
5015 				     LED_CTRL_1000MBPS_ON);
5016 
5017 		tw32(MAC_LED_CTRL, led_ctrl);
5018 		udelay(40);
5019 	}
5020 
5021 	tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
5022 	if (tp->link_config.active_duplex == DUPLEX_HALF)
5023 		tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
5024 
5025 	if (tg3_asic_rev(tp) == ASIC_REV_5700) {
5026 		if (current_link_up &&
5027 		    tg3_5700_link_polarity(tp, tp->link_config.active_speed))
5028 			tp->mac_mode |= MAC_MODE_LINK_POLARITY;
5029 		else
5030 			tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
5031 	}
5032 
5033 	/* ??? Without this setting Netgear GA302T PHY does not
5034 	 * ??? send/receive packets...
5035 	 */
5036 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411 &&
5037 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5700_ALTIMA) {
5038 		tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
5039 		tw32_f(MAC_MI_MODE, tp->mi_mode);
5040 		udelay(80);
5041 	}
5042 
5043 	tw32_f(MAC_MODE, tp->mac_mode);
5044 	udelay(40);
5045 
5046 	tg3_phy_eee_adjust(tp, current_link_up);
5047 
5048 	if (tg3_flag(tp, USE_LINKCHG_REG)) {
5049 		/* Polled via timer. */
5050 		tw32_f(MAC_EVENT, 0);
5051 	} else {
5052 		tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5053 	}
5054 	udelay(40);
5055 
5056 	if (tg3_asic_rev(tp) == ASIC_REV_5700 &&
5057 	    current_link_up &&
5058 	    tp->link_config.active_speed == SPEED_1000 &&
5059 	    (tg3_flag(tp, PCIX_MODE) || tg3_flag(tp, PCI_HIGH_SPEED))) {
5060 		udelay(120);
5061 		tw32_f(MAC_STATUS,
5062 		     (MAC_STATUS_SYNC_CHANGED |
5063 		      MAC_STATUS_CFG_CHANGED));
5064 		udelay(40);
5065 		tg3_write_mem(tp,
5066 			      NIC_SRAM_FIRMWARE_MBOX,
5067 			      NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
5068 	}
5069 
5070 	/* Prevent send BD corruption. */
5071 	if (tg3_flag(tp, CLKREQ_BUG)) {
5072 		if (tp->link_config.active_speed == SPEED_100 ||
5073 		    tp->link_config.active_speed == SPEED_10)
5074 			pcie_capability_clear_word(tp->pdev, PCI_EXP_LNKCTL,
5075 						   PCI_EXP_LNKCTL_CLKREQ_EN);
5076 		else
5077 			pcie_capability_set_word(tp->pdev, PCI_EXP_LNKCTL,
5078 						 PCI_EXP_LNKCTL_CLKREQ_EN);
5079 	}
5080 
5081 	tg3_test_and_report_link_chg(tp, current_link_up);
5082 
5083 	return 0;
5084 }
5085 
5086 struct tg3_fiber_aneginfo {
5087 	int state;
5088 #define ANEG_STATE_UNKNOWN		0
5089 #define ANEG_STATE_AN_ENABLE		1
5090 #define ANEG_STATE_RESTART_INIT		2
5091 #define ANEG_STATE_RESTART		3
5092 #define ANEG_STATE_DISABLE_LINK_OK	4
5093 #define ANEG_STATE_ABILITY_DETECT_INIT	5
5094 #define ANEG_STATE_ABILITY_DETECT	6
5095 #define ANEG_STATE_ACK_DETECT_INIT	7
5096 #define ANEG_STATE_ACK_DETECT		8
5097 #define ANEG_STATE_COMPLETE_ACK_INIT	9
5098 #define ANEG_STATE_COMPLETE_ACK		10
5099 #define ANEG_STATE_IDLE_DETECT_INIT	11
5100 #define ANEG_STATE_IDLE_DETECT		12
5101 #define ANEG_STATE_LINK_OK		13
5102 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT	14
5103 #define ANEG_STATE_NEXT_PAGE_WAIT	15
5104 
5105 	u32 flags;
5106 #define MR_AN_ENABLE		0x00000001
5107 #define MR_RESTART_AN		0x00000002
5108 #define MR_AN_COMPLETE		0x00000004
5109 #define MR_PAGE_RX		0x00000008
5110 #define MR_NP_LOADED		0x00000010
5111 #define MR_TOGGLE_TX		0x00000020
5112 #define MR_LP_ADV_FULL_DUPLEX	0x00000040
5113 #define MR_LP_ADV_HALF_DUPLEX	0x00000080
5114 #define MR_LP_ADV_SYM_PAUSE	0x00000100
5115 #define MR_LP_ADV_ASYM_PAUSE	0x00000200
5116 #define MR_LP_ADV_REMOTE_FAULT1	0x00000400
5117 #define MR_LP_ADV_REMOTE_FAULT2	0x00000800
5118 #define MR_LP_ADV_NEXT_PAGE	0x00001000
5119 #define MR_TOGGLE_RX		0x00002000
5120 #define MR_NP_RX		0x00004000
5121 
5122 #define MR_LINK_OK		0x80000000
5123 
5124 	unsigned long link_time, cur_time;
5125 
5126 	u32 ability_match_cfg;
5127 	int ability_match_count;
5128 
5129 	char ability_match, idle_match, ack_match;
5130 
5131 	u32 txconfig, rxconfig;
5132 #define ANEG_CFG_NP		0x00000080
5133 #define ANEG_CFG_ACK		0x00000040
5134 #define ANEG_CFG_RF2		0x00000020
5135 #define ANEG_CFG_RF1		0x00000010
5136 #define ANEG_CFG_PS2		0x00000001
5137 #define ANEG_CFG_PS1		0x00008000
5138 #define ANEG_CFG_HD		0x00004000
5139 #define ANEG_CFG_FD		0x00002000
5140 #define ANEG_CFG_INVAL		0x00001f06
5141 
5142 };
5143 #define ANEG_OK		0
5144 #define ANEG_DONE	1
5145 #define ANEG_TIMER_ENAB	2
5146 #define ANEG_FAILED	-1
5147 
5148 #define ANEG_STATE_SETTLE_TIME	10000
5149 
5150 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
5151 				   struct tg3_fiber_aneginfo *ap)
5152 {
5153 	u16 flowctrl;
5154 	unsigned long delta;
5155 	u32 rx_cfg_reg;
5156 	int ret;
5157 
5158 	if (ap->state == ANEG_STATE_UNKNOWN) {
5159 		ap->rxconfig = 0;
5160 		ap->link_time = 0;
5161 		ap->cur_time = 0;
5162 		ap->ability_match_cfg = 0;
5163 		ap->ability_match_count = 0;
5164 		ap->ability_match = 0;
5165 		ap->idle_match = 0;
5166 		ap->ack_match = 0;
5167 	}
5168 	ap->cur_time++;
5169 
5170 	if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
5171 		rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
5172 
5173 		if (rx_cfg_reg != ap->ability_match_cfg) {
5174 			ap->ability_match_cfg = rx_cfg_reg;
5175 			ap->ability_match = 0;
5176 			ap->ability_match_count = 0;
5177 		} else {
5178 			if (++ap->ability_match_count > 1) {
5179 				ap->ability_match = 1;
5180 				ap->ability_match_cfg = rx_cfg_reg;
5181 			}
5182 		}
5183 		if (rx_cfg_reg & ANEG_CFG_ACK)
5184 			ap->ack_match = 1;
5185 		else
5186 			ap->ack_match = 0;
5187 
5188 		ap->idle_match = 0;
5189 	} else {
5190 		ap->idle_match = 1;
5191 		ap->ability_match_cfg = 0;
5192 		ap->ability_match_count = 0;
5193 		ap->ability_match = 0;
5194 		ap->ack_match = 0;
5195 
5196 		rx_cfg_reg = 0;
5197 	}
5198 
5199 	ap->rxconfig = rx_cfg_reg;
5200 	ret = ANEG_OK;
5201 
5202 	switch (ap->state) {
5203 	case ANEG_STATE_UNKNOWN:
5204 		if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
5205 			ap->state = ANEG_STATE_AN_ENABLE;
5206 
5207 		fallthrough;
5208 	case ANEG_STATE_AN_ENABLE:
5209 		ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
5210 		if (ap->flags & MR_AN_ENABLE) {
5211 			ap->link_time = 0;
5212 			ap->cur_time = 0;
5213 			ap->ability_match_cfg = 0;
5214 			ap->ability_match_count = 0;
5215 			ap->ability_match = 0;
5216 			ap->idle_match = 0;
5217 			ap->ack_match = 0;
5218 
5219 			ap->state = ANEG_STATE_RESTART_INIT;
5220 		} else {
5221 			ap->state = ANEG_STATE_DISABLE_LINK_OK;
5222 		}
5223 		break;
5224 
5225 	case ANEG_STATE_RESTART_INIT:
5226 		ap->link_time = ap->cur_time;
5227 		ap->flags &= ~(MR_NP_LOADED);
5228 		ap->txconfig = 0;
5229 		tw32(MAC_TX_AUTO_NEG, 0);
5230 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5231 		tw32_f(MAC_MODE, tp->mac_mode);
5232 		udelay(40);
5233 
5234 		ret = ANEG_TIMER_ENAB;
5235 		ap->state = ANEG_STATE_RESTART;
5236 
5237 		fallthrough;
5238 	case ANEG_STATE_RESTART:
5239 		delta = ap->cur_time - ap->link_time;
5240 		if (delta > ANEG_STATE_SETTLE_TIME)
5241 			ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
5242 		else
5243 			ret = ANEG_TIMER_ENAB;
5244 		break;
5245 
5246 	case ANEG_STATE_DISABLE_LINK_OK:
5247 		ret = ANEG_DONE;
5248 		break;
5249 
5250 	case ANEG_STATE_ABILITY_DETECT_INIT:
5251 		ap->flags &= ~(MR_TOGGLE_TX);
5252 		ap->txconfig = ANEG_CFG_FD;
5253 		flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5254 		if (flowctrl & ADVERTISE_1000XPAUSE)
5255 			ap->txconfig |= ANEG_CFG_PS1;
5256 		if (flowctrl & ADVERTISE_1000XPSE_ASYM)
5257 			ap->txconfig |= ANEG_CFG_PS2;
5258 		tw32(MAC_TX_AUTO_NEG, ap->txconfig);
5259 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5260 		tw32_f(MAC_MODE, tp->mac_mode);
5261 		udelay(40);
5262 
5263 		ap->state = ANEG_STATE_ABILITY_DETECT;
5264 		break;
5265 
5266 	case ANEG_STATE_ABILITY_DETECT:
5267 		if (ap->ability_match != 0 && ap->rxconfig != 0)
5268 			ap->state = ANEG_STATE_ACK_DETECT_INIT;
5269 		break;
5270 
5271 	case ANEG_STATE_ACK_DETECT_INIT:
5272 		ap->txconfig |= ANEG_CFG_ACK;
5273 		tw32(MAC_TX_AUTO_NEG, ap->txconfig);
5274 		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
5275 		tw32_f(MAC_MODE, tp->mac_mode);
5276 		udelay(40);
5277 
5278 		ap->state = ANEG_STATE_ACK_DETECT;
5279 
5280 		fallthrough;
5281 	case ANEG_STATE_ACK_DETECT:
5282 		if (ap->ack_match != 0) {
5283 			if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
5284 			    (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
5285 				ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
5286 			} else {
5287 				ap->state = ANEG_STATE_AN_ENABLE;
5288 			}
5289 		} else if (ap->ability_match != 0 &&
5290 			   ap->rxconfig == 0) {
5291 			ap->state = ANEG_STATE_AN_ENABLE;
5292 		}
5293 		break;
5294 
5295 	case ANEG_STATE_COMPLETE_ACK_INIT:
5296 		if (ap->rxconfig & ANEG_CFG_INVAL) {
5297 			ret = ANEG_FAILED;
5298 			break;
5299 		}
5300 		ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
5301 			       MR_LP_ADV_HALF_DUPLEX |
5302 			       MR_LP_ADV_SYM_PAUSE |
5303 			       MR_LP_ADV_ASYM_PAUSE |
5304 			       MR_LP_ADV_REMOTE_FAULT1 |
5305 			       MR_LP_ADV_REMOTE_FAULT2 |
5306 			       MR_LP_ADV_NEXT_PAGE |
5307 			       MR_TOGGLE_RX |
5308 			       MR_NP_RX);
5309 		if (ap->rxconfig & ANEG_CFG_FD)
5310 			ap->flags |= MR_LP_ADV_FULL_DUPLEX;
5311 		if (ap->rxconfig & ANEG_CFG_HD)
5312 			ap->flags |= MR_LP_ADV_HALF_DUPLEX;
5313 		if (ap->rxconfig & ANEG_CFG_PS1)
5314 			ap->flags |= MR_LP_ADV_SYM_PAUSE;
5315 		if (ap->rxconfig & ANEG_CFG_PS2)
5316 			ap->flags |= MR_LP_ADV_ASYM_PAUSE;
5317 		if (ap->rxconfig & ANEG_CFG_RF1)
5318 			ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
5319 		if (ap->rxconfig & ANEG_CFG_RF2)
5320 			ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
5321 		if (ap->rxconfig & ANEG_CFG_NP)
5322 			ap->flags |= MR_LP_ADV_NEXT_PAGE;
5323 
5324 		ap->link_time = ap->cur_time;
5325 
5326 		ap->flags ^= (MR_TOGGLE_TX);
5327 		if (ap->rxconfig & 0x0008)
5328 			ap->flags |= MR_TOGGLE_RX;
5329 		if (ap->rxconfig & ANEG_CFG_NP)
5330 			ap->flags |= MR_NP_RX;
5331 		ap->flags |= MR_PAGE_RX;
5332 
5333 		ap->state = ANEG_STATE_COMPLETE_ACK;
5334 		ret = ANEG_TIMER_ENAB;
5335 		break;
5336 
5337 	case ANEG_STATE_COMPLETE_ACK:
5338 		if (ap->ability_match != 0 &&
5339 		    ap->rxconfig == 0) {
5340 			ap->state = ANEG_STATE_AN_ENABLE;
5341 			break;
5342 		}
5343 		delta = ap->cur_time - ap->link_time;
5344 		if (delta > ANEG_STATE_SETTLE_TIME) {
5345 			if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
5346 				ap->state = ANEG_STATE_IDLE_DETECT_INIT;
5347 			} else {
5348 				if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
5349 				    !(ap->flags & MR_NP_RX)) {
5350 					ap->state = ANEG_STATE_IDLE_DETECT_INIT;
5351 				} else {
5352 					ret = ANEG_FAILED;
5353 				}
5354 			}
5355 		}
5356 		break;
5357 
5358 	case ANEG_STATE_IDLE_DETECT_INIT:
5359 		ap->link_time = ap->cur_time;
5360 		tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
5361 		tw32_f(MAC_MODE, tp->mac_mode);
5362 		udelay(40);
5363 
5364 		ap->state = ANEG_STATE_IDLE_DETECT;
5365 		ret = ANEG_TIMER_ENAB;
5366 		break;
5367 
5368 	case ANEG_STATE_IDLE_DETECT:
5369 		if (ap->ability_match != 0 &&
5370 		    ap->rxconfig == 0) {
5371 			ap->state = ANEG_STATE_AN_ENABLE;
5372 			break;
5373 		}
5374 		delta = ap->cur_time - ap->link_time;
5375 		if (delta > ANEG_STATE_SETTLE_TIME) {
5376 			/* XXX another gem from the Broadcom driver :( */
5377 			ap->state = ANEG_STATE_LINK_OK;
5378 		}
5379 		break;
5380 
5381 	case ANEG_STATE_LINK_OK:
5382 		ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
5383 		ret = ANEG_DONE;
5384 		break;
5385 
5386 	case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
5387 		/* ??? unimplemented */
5388 		break;
5389 
5390 	case ANEG_STATE_NEXT_PAGE_WAIT:
5391 		/* ??? unimplemented */
5392 		break;
5393 
5394 	default:
5395 		ret = ANEG_FAILED;
5396 		break;
5397 	}
5398 
5399 	return ret;
5400 }
5401 
5402 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
5403 {
5404 	int res = 0;
5405 	struct tg3_fiber_aneginfo aninfo;
5406 	int status = ANEG_FAILED;
5407 	unsigned int tick;
5408 	u32 tmp;
5409 
5410 	tw32_f(MAC_TX_AUTO_NEG, 0);
5411 
5412 	tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
5413 	tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
5414 	udelay(40);
5415 
5416 	tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
5417 	udelay(40);
5418 
5419 	memset(&aninfo, 0, sizeof(aninfo));
5420 	aninfo.flags |= MR_AN_ENABLE;
5421 	aninfo.state = ANEG_STATE_UNKNOWN;
5422 	aninfo.cur_time = 0;
5423 	tick = 0;
5424 	while (++tick < 195000) {
5425 		status = tg3_fiber_aneg_smachine(tp, &aninfo);
5426 		if (status == ANEG_DONE || status == ANEG_FAILED)
5427 			break;
5428 
5429 		udelay(1);
5430 	}
5431 
5432 	tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
5433 	tw32_f(MAC_MODE, tp->mac_mode);
5434 	udelay(40);
5435 
5436 	*txflags = aninfo.txconfig;
5437 	*rxflags = aninfo.flags;
5438 
5439 	if (status == ANEG_DONE &&
5440 	    (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
5441 			     MR_LP_ADV_FULL_DUPLEX)))
5442 		res = 1;
5443 
5444 	return res;
5445 }
5446 
5447 static void tg3_init_bcm8002(struct tg3 *tp)
5448 {
5449 	u32 mac_status = tr32(MAC_STATUS);
5450 	int i;
5451 
5452 	/* Reset when initting first time or we have a link. */
5453 	if (tg3_flag(tp, INIT_COMPLETE) &&
5454 	    !(mac_status & MAC_STATUS_PCS_SYNCED))
5455 		return;
5456 
5457 	/* Set PLL lock range. */
5458 	tg3_writephy(tp, 0x16, 0x8007);
5459 
5460 	/* SW reset */
5461 	tg3_writephy(tp, MII_BMCR, BMCR_RESET);
5462 
5463 	/* Wait for reset to complete. */
5464 	/* XXX schedule_timeout() ... */
5465 	for (i = 0; i < 500; i++)
5466 		udelay(10);
5467 
5468 	/* Config mode; select PMA/Ch 1 regs. */
5469 	tg3_writephy(tp, 0x10, 0x8411);
5470 
5471 	/* Enable auto-lock and comdet, select txclk for tx. */
5472 	tg3_writephy(tp, 0x11, 0x0a10);
5473 
5474 	tg3_writephy(tp, 0x18, 0x00a0);
5475 	tg3_writephy(tp, 0x16, 0x41ff);
5476 
5477 	/* Assert and deassert POR. */
5478 	tg3_writephy(tp, 0x13, 0x0400);
5479 	udelay(40);
5480 	tg3_writephy(tp, 0x13, 0x0000);
5481 
5482 	tg3_writephy(tp, 0x11, 0x0a50);
5483 	udelay(40);
5484 	tg3_writephy(tp, 0x11, 0x0a10);
5485 
5486 	/* Wait for signal to stabilize */
5487 	/* XXX schedule_timeout() ... */
5488 	for (i = 0; i < 15000; i++)
5489 		udelay(10);
5490 
5491 	/* Deselect the channel register so we can read the PHYID
5492 	 * later.
5493 	 */
5494 	tg3_writephy(tp, 0x10, 0x8011);
5495 }
5496 
5497 static bool tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
5498 {
5499 	u16 flowctrl;
5500 	bool current_link_up;
5501 	u32 sg_dig_ctrl, sg_dig_status;
5502 	u32 serdes_cfg, expected_sg_dig_ctrl;
5503 	int workaround, port_a;
5504 
5505 	serdes_cfg = 0;
5506 	workaround = 0;
5507 	port_a = 1;
5508 	current_link_up = false;
5509 
5510 	if (tg3_chip_rev_id(tp) != CHIPREV_ID_5704_A0 &&
5511 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5704_A1) {
5512 		workaround = 1;
5513 		if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
5514 			port_a = 0;
5515 
5516 		/* preserve bits 0-11,13,14 for signal pre-emphasis */
5517 		/* preserve bits 20-23 for voltage regulator */
5518 		serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
5519 	}
5520 
5521 	sg_dig_ctrl = tr32(SG_DIG_CTRL);
5522 
5523 	if (tp->link_config.autoneg != AUTONEG_ENABLE) {
5524 		if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
5525 			if (workaround) {
5526 				u32 val = serdes_cfg;
5527 
5528 				if (port_a)
5529 					val |= 0xc010000;
5530 				else
5531 					val |= 0x4010000;
5532 				tw32_f(MAC_SERDES_CFG, val);
5533 			}
5534 
5535 			tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
5536 		}
5537 		if (mac_status & MAC_STATUS_PCS_SYNCED) {
5538 			tg3_setup_flow_control(tp, 0, 0);
5539 			current_link_up = true;
5540 		}
5541 		goto out;
5542 	}
5543 
5544 	/* Want auto-negotiation.  */
5545 	expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
5546 
5547 	flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5548 	if (flowctrl & ADVERTISE_1000XPAUSE)
5549 		expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
5550 	if (flowctrl & ADVERTISE_1000XPSE_ASYM)
5551 		expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
5552 
5553 	if (sg_dig_ctrl != expected_sg_dig_ctrl) {
5554 		if ((tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT) &&
5555 		    tp->serdes_counter &&
5556 		    ((mac_status & (MAC_STATUS_PCS_SYNCED |
5557 				    MAC_STATUS_RCVD_CFG)) ==
5558 		     MAC_STATUS_PCS_SYNCED)) {
5559 			tp->serdes_counter--;
5560 			current_link_up = true;
5561 			goto out;
5562 		}
5563 restart_autoneg:
5564 		if (workaround)
5565 			tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
5566 		tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
5567 		udelay(5);
5568 		tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
5569 
5570 		tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
5571 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5572 	} else if (mac_status & (MAC_STATUS_PCS_SYNCED |
5573 				 MAC_STATUS_SIGNAL_DET)) {
5574 		sg_dig_status = tr32(SG_DIG_STATUS);
5575 		mac_status = tr32(MAC_STATUS);
5576 
5577 		if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
5578 		    (mac_status & MAC_STATUS_PCS_SYNCED)) {
5579 			u32 local_adv = 0, remote_adv = 0;
5580 
5581 			if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
5582 				local_adv |= ADVERTISE_1000XPAUSE;
5583 			if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
5584 				local_adv |= ADVERTISE_1000XPSE_ASYM;
5585 
5586 			if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
5587 				remote_adv |= LPA_1000XPAUSE;
5588 			if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
5589 				remote_adv |= LPA_1000XPAUSE_ASYM;
5590 
5591 			tp->link_config.rmt_adv =
5592 					   mii_adv_to_ethtool_adv_x(remote_adv);
5593 
5594 			tg3_setup_flow_control(tp, local_adv, remote_adv);
5595 			current_link_up = true;
5596 			tp->serdes_counter = 0;
5597 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5598 		} else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
5599 			if (tp->serdes_counter)
5600 				tp->serdes_counter--;
5601 			else {
5602 				if (workaround) {
5603 					u32 val = serdes_cfg;
5604 
5605 					if (port_a)
5606 						val |= 0xc010000;
5607 					else
5608 						val |= 0x4010000;
5609 
5610 					tw32_f(MAC_SERDES_CFG, val);
5611 				}
5612 
5613 				tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
5614 				udelay(40);
5615 
5616 				/* Link parallel detection - link is up */
5617 				/* only if we have PCS_SYNC and not */
5618 				/* receiving config code words */
5619 				mac_status = tr32(MAC_STATUS);
5620 				if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
5621 				    !(mac_status & MAC_STATUS_RCVD_CFG)) {
5622 					tg3_setup_flow_control(tp, 0, 0);
5623 					current_link_up = true;
5624 					tp->phy_flags |=
5625 						TG3_PHYFLG_PARALLEL_DETECT;
5626 					tp->serdes_counter =
5627 						SERDES_PARALLEL_DET_TIMEOUT;
5628 				} else
5629 					goto restart_autoneg;
5630 			}
5631 		}
5632 	} else {
5633 		tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
5634 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5635 	}
5636 
5637 out:
5638 	return current_link_up;
5639 }
5640 
5641 static bool tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
5642 {
5643 	bool current_link_up = false;
5644 
5645 	if (!(mac_status & MAC_STATUS_PCS_SYNCED))
5646 		goto out;
5647 
5648 	if (tp->link_config.autoneg == AUTONEG_ENABLE) {
5649 		u32 txflags, rxflags;
5650 		int i;
5651 
5652 		if (fiber_autoneg(tp, &txflags, &rxflags)) {
5653 			u32 local_adv = 0, remote_adv = 0;
5654 
5655 			if (txflags & ANEG_CFG_PS1)
5656 				local_adv |= ADVERTISE_1000XPAUSE;
5657 			if (txflags & ANEG_CFG_PS2)
5658 				local_adv |= ADVERTISE_1000XPSE_ASYM;
5659 
5660 			if (rxflags & MR_LP_ADV_SYM_PAUSE)
5661 				remote_adv |= LPA_1000XPAUSE;
5662 			if (rxflags & MR_LP_ADV_ASYM_PAUSE)
5663 				remote_adv |= LPA_1000XPAUSE_ASYM;
5664 
5665 			tp->link_config.rmt_adv =
5666 					   mii_adv_to_ethtool_adv_x(remote_adv);
5667 
5668 			tg3_setup_flow_control(tp, local_adv, remote_adv);
5669 
5670 			current_link_up = true;
5671 		}
5672 		for (i = 0; i < 30; i++) {
5673 			udelay(20);
5674 			tw32_f(MAC_STATUS,
5675 			       (MAC_STATUS_SYNC_CHANGED |
5676 				MAC_STATUS_CFG_CHANGED));
5677 			udelay(40);
5678 			if ((tr32(MAC_STATUS) &
5679 			     (MAC_STATUS_SYNC_CHANGED |
5680 			      MAC_STATUS_CFG_CHANGED)) == 0)
5681 				break;
5682 		}
5683 
5684 		mac_status = tr32(MAC_STATUS);
5685 		if (!current_link_up &&
5686 		    (mac_status & MAC_STATUS_PCS_SYNCED) &&
5687 		    !(mac_status & MAC_STATUS_RCVD_CFG))
5688 			current_link_up = true;
5689 	} else {
5690 		tg3_setup_flow_control(tp, 0, 0);
5691 
5692 		/* Forcing 1000FD link up. */
5693 		current_link_up = true;
5694 
5695 		tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
5696 		udelay(40);
5697 
5698 		tw32_f(MAC_MODE, tp->mac_mode);
5699 		udelay(40);
5700 	}
5701 
5702 out:
5703 	return current_link_up;
5704 }
5705 
5706 static int tg3_setup_fiber_phy(struct tg3 *tp, bool force_reset)
5707 {
5708 	u32 orig_pause_cfg;
5709 	u32 orig_active_speed;
5710 	u8 orig_active_duplex;
5711 	u32 mac_status;
5712 	bool current_link_up;
5713 	int i;
5714 
5715 	orig_pause_cfg = tp->link_config.active_flowctrl;
5716 	orig_active_speed = tp->link_config.active_speed;
5717 	orig_active_duplex = tp->link_config.active_duplex;
5718 
5719 	if (!tg3_flag(tp, HW_AUTONEG) &&
5720 	    tp->link_up &&
5721 	    tg3_flag(tp, INIT_COMPLETE)) {
5722 		mac_status = tr32(MAC_STATUS);
5723 		mac_status &= (MAC_STATUS_PCS_SYNCED |
5724 			       MAC_STATUS_SIGNAL_DET |
5725 			       MAC_STATUS_CFG_CHANGED |
5726 			       MAC_STATUS_RCVD_CFG);
5727 		if (mac_status == (MAC_STATUS_PCS_SYNCED |
5728 				   MAC_STATUS_SIGNAL_DET)) {
5729 			tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
5730 					    MAC_STATUS_CFG_CHANGED));
5731 			return 0;
5732 		}
5733 	}
5734 
5735 	tw32_f(MAC_TX_AUTO_NEG, 0);
5736 
5737 	tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
5738 	tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
5739 	tw32_f(MAC_MODE, tp->mac_mode);
5740 	udelay(40);
5741 
5742 	if (tp->phy_id == TG3_PHY_ID_BCM8002)
5743 		tg3_init_bcm8002(tp);
5744 
5745 	/* Enable link change event even when serdes polling.  */
5746 	tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5747 	udelay(40);
5748 
5749 	tp->link_config.rmt_adv = 0;
5750 	mac_status = tr32(MAC_STATUS);
5751 
5752 	if (tg3_flag(tp, HW_AUTONEG))
5753 		current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
5754 	else
5755 		current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
5756 
5757 	tp->napi[0].hw_status->status =
5758 		(SD_STATUS_UPDATED |
5759 		 (tp->napi[0].hw_status->status & ~SD_STATUS_LINK_CHG));
5760 
5761 	for (i = 0; i < 100; i++) {
5762 		tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
5763 				    MAC_STATUS_CFG_CHANGED));
5764 		udelay(5);
5765 		if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
5766 					 MAC_STATUS_CFG_CHANGED |
5767 					 MAC_STATUS_LNKSTATE_CHANGED)) == 0)
5768 			break;
5769 	}
5770 
5771 	mac_status = tr32(MAC_STATUS);
5772 	if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
5773 		current_link_up = false;
5774 		if (tp->link_config.autoneg == AUTONEG_ENABLE &&
5775 		    tp->serdes_counter == 0) {
5776 			tw32_f(MAC_MODE, (tp->mac_mode |
5777 					  MAC_MODE_SEND_CONFIGS));
5778 			udelay(1);
5779 			tw32_f(MAC_MODE, tp->mac_mode);
5780 		}
5781 	}
5782 
5783 	if (current_link_up) {
5784 		tp->link_config.active_speed = SPEED_1000;
5785 		tp->link_config.active_duplex = DUPLEX_FULL;
5786 		tw32(MAC_LED_CTRL, (tp->led_ctrl |
5787 				    LED_CTRL_LNKLED_OVERRIDE |
5788 				    LED_CTRL_1000MBPS_ON));
5789 	} else {
5790 		tp->link_config.active_speed = SPEED_UNKNOWN;
5791 		tp->link_config.active_duplex = DUPLEX_UNKNOWN;
5792 		tw32(MAC_LED_CTRL, (tp->led_ctrl |
5793 				    LED_CTRL_LNKLED_OVERRIDE |
5794 				    LED_CTRL_TRAFFIC_OVERRIDE));
5795 	}
5796 
5797 	if (!tg3_test_and_report_link_chg(tp, current_link_up)) {
5798 		u32 now_pause_cfg = tp->link_config.active_flowctrl;
5799 		if (orig_pause_cfg != now_pause_cfg ||
5800 		    orig_active_speed != tp->link_config.active_speed ||
5801 		    orig_active_duplex != tp->link_config.active_duplex)
5802 			tg3_link_report(tp);
5803 	}
5804 
5805 	return 0;
5806 }
5807 
5808 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, bool force_reset)
5809 {
5810 	int err = 0;
5811 	u32 bmsr, bmcr;
5812 	u32 current_speed = SPEED_UNKNOWN;
5813 	u8 current_duplex = DUPLEX_UNKNOWN;
5814 	bool current_link_up = false;
5815 	u32 local_adv, remote_adv, sgsr;
5816 
5817 	if ((tg3_asic_rev(tp) == ASIC_REV_5719 ||
5818 	     tg3_asic_rev(tp) == ASIC_REV_5720) &&
5819 	     !tg3_readphy(tp, SERDES_TG3_1000X_STATUS, &sgsr) &&
5820 	     (sgsr & SERDES_TG3_SGMII_MODE)) {
5821 
5822 		if (force_reset)
5823 			tg3_phy_reset(tp);
5824 
5825 		tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
5826 
5827 		if (!(sgsr & SERDES_TG3_LINK_UP)) {
5828 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5829 		} else {
5830 			current_link_up = true;
5831 			if (sgsr & SERDES_TG3_SPEED_1000) {
5832 				current_speed = SPEED_1000;
5833 				tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5834 			} else if (sgsr & SERDES_TG3_SPEED_100) {
5835 				current_speed = SPEED_100;
5836 				tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
5837 			} else {
5838 				current_speed = SPEED_10;
5839 				tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
5840 			}
5841 
5842 			if (sgsr & SERDES_TG3_FULL_DUPLEX)
5843 				current_duplex = DUPLEX_FULL;
5844 			else
5845 				current_duplex = DUPLEX_HALF;
5846 		}
5847 
5848 		tw32_f(MAC_MODE, tp->mac_mode);
5849 		udelay(40);
5850 
5851 		tg3_clear_mac_status(tp);
5852 
5853 		goto fiber_setup_done;
5854 	}
5855 
5856 	tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
5857 	tw32_f(MAC_MODE, tp->mac_mode);
5858 	udelay(40);
5859 
5860 	tg3_clear_mac_status(tp);
5861 
5862 	if (force_reset)
5863 		tg3_phy_reset(tp);
5864 
5865 	tp->link_config.rmt_adv = 0;
5866 
5867 	err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5868 	err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5869 	if (tg3_asic_rev(tp) == ASIC_REV_5714) {
5870 		if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
5871 			bmsr |= BMSR_LSTATUS;
5872 		else
5873 			bmsr &= ~BMSR_LSTATUS;
5874 	}
5875 
5876 	err |= tg3_readphy(tp, MII_BMCR, &bmcr);
5877 
5878 	if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
5879 	    (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
5880 		/* do nothing, just check for link up at the end */
5881 	} else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
5882 		u32 adv, newadv;
5883 
5884 		err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
5885 		newadv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
5886 				 ADVERTISE_1000XPAUSE |
5887 				 ADVERTISE_1000XPSE_ASYM |
5888 				 ADVERTISE_SLCT);
5889 
5890 		newadv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
5891 		newadv |= ethtool_adv_to_mii_adv_x(tp->link_config.advertising);
5892 
5893 		if ((newadv != adv) || !(bmcr & BMCR_ANENABLE)) {
5894 			tg3_writephy(tp, MII_ADVERTISE, newadv);
5895 			bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
5896 			tg3_writephy(tp, MII_BMCR, bmcr);
5897 
5898 			tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5899 			tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
5900 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5901 
5902 			return err;
5903 		}
5904 	} else {
5905 		u32 new_bmcr;
5906 
5907 		bmcr &= ~BMCR_SPEED1000;
5908 		new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
5909 
5910 		if (tp->link_config.duplex == DUPLEX_FULL)
5911 			new_bmcr |= BMCR_FULLDPLX;
5912 
5913 		if (new_bmcr != bmcr) {
5914 			/* BMCR_SPEED1000 is a reserved bit that needs
5915 			 * to be set on write.
5916 			 */
5917 			new_bmcr |= BMCR_SPEED1000;
5918 
5919 			/* Force a linkdown */
5920 			if (tp->link_up) {
5921 				u32 adv;
5922 
5923 				err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
5924 				adv &= ~(ADVERTISE_1000XFULL |
5925 					 ADVERTISE_1000XHALF |
5926 					 ADVERTISE_SLCT);
5927 				tg3_writephy(tp, MII_ADVERTISE, adv);
5928 				tg3_writephy(tp, MII_BMCR, bmcr |
5929 							   BMCR_ANRESTART |
5930 							   BMCR_ANENABLE);
5931 				udelay(10);
5932 				tg3_carrier_off(tp);
5933 			}
5934 			tg3_writephy(tp, MII_BMCR, new_bmcr);
5935 			bmcr = new_bmcr;
5936 			err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5937 			err |= tg3_readphy(tp, MII_BMSR, &bmsr);
5938 			if (tg3_asic_rev(tp) == ASIC_REV_5714) {
5939 				if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
5940 					bmsr |= BMSR_LSTATUS;
5941 				else
5942 					bmsr &= ~BMSR_LSTATUS;
5943 			}
5944 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
5945 		}
5946 	}
5947 
5948 	if (bmsr & BMSR_LSTATUS) {
5949 		current_speed = SPEED_1000;
5950 		current_link_up = true;
5951 		if (bmcr & BMCR_FULLDPLX)
5952 			current_duplex = DUPLEX_FULL;
5953 		else
5954 			current_duplex = DUPLEX_HALF;
5955 
5956 		local_adv = 0;
5957 		remote_adv = 0;
5958 
5959 		if (bmcr & BMCR_ANENABLE) {
5960 			u32 common;
5961 
5962 			err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
5963 			err |= tg3_readphy(tp, MII_LPA, &remote_adv);
5964 			common = local_adv & remote_adv;
5965 			if (common & (ADVERTISE_1000XHALF |
5966 				      ADVERTISE_1000XFULL)) {
5967 				if (common & ADVERTISE_1000XFULL)
5968 					current_duplex = DUPLEX_FULL;
5969 				else
5970 					current_duplex = DUPLEX_HALF;
5971 
5972 				tp->link_config.rmt_adv =
5973 					   mii_adv_to_ethtool_adv_x(remote_adv);
5974 			} else if (!tg3_flag(tp, 5780_CLASS)) {
5975 				/* Link is up via parallel detect */
5976 			} else {
5977 				current_link_up = false;
5978 			}
5979 		}
5980 	}
5981 
5982 fiber_setup_done:
5983 	if (current_link_up && current_duplex == DUPLEX_FULL)
5984 		tg3_setup_flow_control(tp, local_adv, remote_adv);
5985 
5986 	tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
5987 	if (tp->link_config.active_duplex == DUPLEX_HALF)
5988 		tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
5989 
5990 	tw32_f(MAC_MODE, tp->mac_mode);
5991 	udelay(40);
5992 
5993 	tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
5994 
5995 	tp->link_config.active_speed = current_speed;
5996 	tp->link_config.active_duplex = current_duplex;
5997 
5998 	tg3_test_and_report_link_chg(tp, current_link_up);
5999 	return err;
6000 }
6001 
6002 static void tg3_serdes_parallel_detect(struct tg3 *tp)
6003 {
6004 	if (tp->serdes_counter) {
6005 		/* Give autoneg time to complete. */
6006 		tp->serdes_counter--;
6007 		return;
6008 	}
6009 
6010 	if (!tp->link_up &&
6011 	    (tp->link_config.autoneg == AUTONEG_ENABLE)) {
6012 		u32 bmcr;
6013 
6014 		tg3_readphy(tp, MII_BMCR, &bmcr);
6015 		if (bmcr & BMCR_ANENABLE) {
6016 			u32 phy1, phy2;
6017 
6018 			/* Select shadow register 0x1f */
6019 			tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x7c00);
6020 			tg3_readphy(tp, MII_TG3_MISC_SHDW, &phy1);
6021 
6022 			/* Select expansion interrupt status register */
6023 			tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
6024 					 MII_TG3_DSP_EXP1_INT_STAT);
6025 			tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6026 			tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6027 
6028 			if ((phy1 & 0x10) && !(phy2 & 0x20)) {
6029 				/* We have signal detect and not receiving
6030 				 * config code words, link is up by parallel
6031 				 * detection.
6032 				 */
6033 
6034 				bmcr &= ~BMCR_ANENABLE;
6035 				bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
6036 				tg3_writephy(tp, MII_BMCR, bmcr);
6037 				tp->phy_flags |= TG3_PHYFLG_PARALLEL_DETECT;
6038 			}
6039 		}
6040 	} else if (tp->link_up &&
6041 		   (tp->link_config.autoneg == AUTONEG_ENABLE) &&
6042 		   (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
6043 		u32 phy2;
6044 
6045 		/* Select expansion interrupt status register */
6046 		tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
6047 				 MII_TG3_DSP_EXP1_INT_STAT);
6048 		tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
6049 		if (phy2 & 0x20) {
6050 			u32 bmcr;
6051 
6052 			/* Config code words received, turn on autoneg. */
6053 			tg3_readphy(tp, MII_BMCR, &bmcr);
6054 			tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
6055 
6056 			tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
6057 
6058 		}
6059 	}
6060 }
6061 
6062 static int tg3_setup_phy(struct tg3 *tp, bool force_reset)
6063 {
6064 	u32 val;
6065 	int err;
6066 
6067 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
6068 		err = tg3_setup_fiber_phy(tp, force_reset);
6069 	else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
6070 		err = tg3_setup_fiber_mii_phy(tp, force_reset);
6071 	else
6072 		err = tg3_setup_copper_phy(tp, force_reset);
6073 
6074 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX) {
6075 		u32 scale;
6076 
6077 		val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
6078 		if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
6079 			scale = 65;
6080 		else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
6081 			scale = 6;
6082 		else
6083 			scale = 12;
6084 
6085 		val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
6086 		val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
6087 		tw32(GRC_MISC_CFG, val);
6088 	}
6089 
6090 	val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
6091 	      (6 << TX_LENGTHS_IPG_SHIFT);
6092 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
6093 	    tg3_asic_rev(tp) == ASIC_REV_5762)
6094 		val |= tr32(MAC_TX_LENGTHS) &
6095 		       (TX_LENGTHS_JMB_FRM_LEN_MSK |
6096 			TX_LENGTHS_CNT_DWN_VAL_MSK);
6097 
6098 	if (tp->link_config.active_speed == SPEED_1000 &&
6099 	    tp->link_config.active_duplex == DUPLEX_HALF)
6100 		tw32(MAC_TX_LENGTHS, val |
6101 		     (0xff << TX_LENGTHS_SLOT_TIME_SHIFT));
6102 	else
6103 		tw32(MAC_TX_LENGTHS, val |
6104 		     (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
6105 
6106 	if (!tg3_flag(tp, 5705_PLUS)) {
6107 		if (tp->link_up) {
6108 			tw32(HOSTCC_STAT_COAL_TICKS,
6109 			     tp->coal.stats_block_coalesce_usecs);
6110 		} else {
6111 			tw32(HOSTCC_STAT_COAL_TICKS, 0);
6112 		}
6113 	}
6114 
6115 	if (tg3_flag(tp, ASPM_WORKAROUND)) {
6116 		val = tr32(PCIE_PWR_MGMT_THRESH);
6117 		if (!tp->link_up)
6118 			val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
6119 			      tp->pwrmgmt_thresh;
6120 		else
6121 			val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
6122 		tw32(PCIE_PWR_MGMT_THRESH, val);
6123 	}
6124 
6125 	return err;
6126 }
6127 
6128 /* tp->lock must be held */
6129 static u64 tg3_refclk_read(struct tg3 *tp, struct ptp_system_timestamp *sts)
6130 {
6131 	u64 stamp;
6132 
6133 	ptp_read_system_prets(sts);
6134 	stamp = tr32(TG3_EAV_REF_CLCK_LSB);
6135 	ptp_read_system_postts(sts);
6136 	stamp |= (u64)tr32(TG3_EAV_REF_CLCK_MSB) << 32;
6137 
6138 	return stamp;
6139 }
6140 
6141 /* tp->lock must be held */
6142 static void tg3_refclk_write(struct tg3 *tp, u64 newval)
6143 {
6144 	u32 clock_ctl = tr32(TG3_EAV_REF_CLCK_CTL);
6145 
6146 	tw32(TG3_EAV_REF_CLCK_CTL, clock_ctl | TG3_EAV_REF_CLCK_CTL_STOP);
6147 	tw32(TG3_EAV_REF_CLCK_LSB, newval & 0xffffffff);
6148 	tw32(TG3_EAV_REF_CLCK_MSB, newval >> 32);
6149 	tw32_f(TG3_EAV_REF_CLCK_CTL, clock_ctl | TG3_EAV_REF_CLCK_CTL_RESUME);
6150 }
6151 
6152 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync);
6153 static inline void tg3_full_unlock(struct tg3 *tp);
6154 static int tg3_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info)
6155 {
6156 	struct tg3 *tp = netdev_priv(dev);
6157 
6158 	info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
6159 				SOF_TIMESTAMPING_RX_SOFTWARE |
6160 				SOF_TIMESTAMPING_SOFTWARE;
6161 
6162 	if (tg3_flag(tp, PTP_CAPABLE)) {
6163 		info->so_timestamping |= SOF_TIMESTAMPING_TX_HARDWARE |
6164 					SOF_TIMESTAMPING_RX_HARDWARE |
6165 					SOF_TIMESTAMPING_RAW_HARDWARE;
6166 	}
6167 
6168 	if (tp->ptp_clock)
6169 		info->phc_index = ptp_clock_index(tp->ptp_clock);
6170 	else
6171 		info->phc_index = -1;
6172 
6173 	info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
6174 
6175 	info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
6176 			   (1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
6177 			   (1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
6178 			   (1 << HWTSTAMP_FILTER_PTP_V2_L4_EVENT);
6179 	return 0;
6180 }
6181 
6182 static int tg3_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
6183 {
6184 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6185 	bool neg_adj = false;
6186 	u32 correction = 0;
6187 
6188 	if (ppb < 0) {
6189 		neg_adj = true;
6190 		ppb = -ppb;
6191 	}
6192 
6193 	/* Frequency adjustment is performed using hardware with a 24 bit
6194 	 * accumulator and a programmable correction value. On each clk, the
6195 	 * correction value gets added to the accumulator and when it
6196 	 * overflows, the time counter is incremented/decremented.
6197 	 *
6198 	 * So conversion from ppb to correction value is
6199 	 *		ppb * (1 << 24) / 1000000000
6200 	 */
6201 	correction = div_u64((u64)ppb * (1 << 24), 1000000000ULL) &
6202 		     TG3_EAV_REF_CLK_CORRECT_MASK;
6203 
6204 	tg3_full_lock(tp, 0);
6205 
6206 	if (correction)
6207 		tw32(TG3_EAV_REF_CLK_CORRECT_CTL,
6208 		     TG3_EAV_REF_CLK_CORRECT_EN |
6209 		     (neg_adj ? TG3_EAV_REF_CLK_CORRECT_NEG : 0) | correction);
6210 	else
6211 		tw32(TG3_EAV_REF_CLK_CORRECT_CTL, 0);
6212 
6213 	tg3_full_unlock(tp);
6214 
6215 	return 0;
6216 }
6217 
6218 static int tg3_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
6219 {
6220 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6221 
6222 	tg3_full_lock(tp, 0);
6223 	tp->ptp_adjust += delta;
6224 	tg3_full_unlock(tp);
6225 
6226 	return 0;
6227 }
6228 
6229 static int tg3_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts,
6230 			    struct ptp_system_timestamp *sts)
6231 {
6232 	u64 ns;
6233 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6234 
6235 	tg3_full_lock(tp, 0);
6236 	ns = tg3_refclk_read(tp, sts);
6237 	ns += tp->ptp_adjust;
6238 	tg3_full_unlock(tp);
6239 
6240 	*ts = ns_to_timespec64(ns);
6241 
6242 	return 0;
6243 }
6244 
6245 static int tg3_ptp_settime(struct ptp_clock_info *ptp,
6246 			   const struct timespec64 *ts)
6247 {
6248 	u64 ns;
6249 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6250 
6251 	ns = timespec64_to_ns(ts);
6252 
6253 	tg3_full_lock(tp, 0);
6254 	tg3_refclk_write(tp, ns);
6255 	tp->ptp_adjust = 0;
6256 	tg3_full_unlock(tp);
6257 
6258 	return 0;
6259 }
6260 
6261 static int tg3_ptp_enable(struct ptp_clock_info *ptp,
6262 			  struct ptp_clock_request *rq, int on)
6263 {
6264 	struct tg3 *tp = container_of(ptp, struct tg3, ptp_info);
6265 	u32 clock_ctl;
6266 	int rval = 0;
6267 
6268 	switch (rq->type) {
6269 	case PTP_CLK_REQ_PEROUT:
6270 		/* Reject requests with unsupported flags */
6271 		if (rq->perout.flags)
6272 			return -EOPNOTSUPP;
6273 
6274 		if (rq->perout.index != 0)
6275 			return -EINVAL;
6276 
6277 		tg3_full_lock(tp, 0);
6278 		clock_ctl = tr32(TG3_EAV_REF_CLCK_CTL);
6279 		clock_ctl &= ~TG3_EAV_CTL_TSYNC_GPIO_MASK;
6280 
6281 		if (on) {
6282 			u64 nsec;
6283 
6284 			nsec = rq->perout.start.sec * 1000000000ULL +
6285 			       rq->perout.start.nsec;
6286 
6287 			if (rq->perout.period.sec || rq->perout.period.nsec) {
6288 				netdev_warn(tp->dev,
6289 					    "Device supports only a one-shot timesync output, period must be 0\n");
6290 				rval = -EINVAL;
6291 				goto err_out;
6292 			}
6293 
6294 			if (nsec & (1ULL << 63)) {
6295 				netdev_warn(tp->dev,
6296 					    "Start value (nsec) is over limit. Maximum size of start is only 63 bits\n");
6297 				rval = -EINVAL;
6298 				goto err_out;
6299 			}
6300 
6301 			tw32(TG3_EAV_WATCHDOG0_LSB, (nsec & 0xffffffff));
6302 			tw32(TG3_EAV_WATCHDOG0_MSB,
6303 			     TG3_EAV_WATCHDOG0_EN |
6304 			     ((nsec >> 32) & TG3_EAV_WATCHDOG_MSB_MASK));
6305 
6306 			tw32(TG3_EAV_REF_CLCK_CTL,
6307 			     clock_ctl | TG3_EAV_CTL_TSYNC_WDOG0);
6308 		} else {
6309 			tw32(TG3_EAV_WATCHDOG0_MSB, 0);
6310 			tw32(TG3_EAV_REF_CLCK_CTL, clock_ctl);
6311 		}
6312 
6313 err_out:
6314 		tg3_full_unlock(tp);
6315 		return rval;
6316 
6317 	default:
6318 		break;
6319 	}
6320 
6321 	return -EOPNOTSUPP;
6322 }
6323 
6324 static const struct ptp_clock_info tg3_ptp_caps = {
6325 	.owner		= THIS_MODULE,
6326 	.name		= "tg3 clock",
6327 	.max_adj	= 250000000,
6328 	.n_alarm	= 0,
6329 	.n_ext_ts	= 0,
6330 	.n_per_out	= 1,
6331 	.n_pins		= 0,
6332 	.pps		= 0,
6333 	.adjfreq	= tg3_ptp_adjfreq,
6334 	.adjtime	= tg3_ptp_adjtime,
6335 	.gettimex64	= tg3_ptp_gettimex,
6336 	.settime64	= tg3_ptp_settime,
6337 	.enable		= tg3_ptp_enable,
6338 };
6339 
6340 static void tg3_hwclock_to_timestamp(struct tg3 *tp, u64 hwclock,
6341 				     struct skb_shared_hwtstamps *timestamp)
6342 {
6343 	memset(timestamp, 0, sizeof(struct skb_shared_hwtstamps));
6344 	timestamp->hwtstamp  = ns_to_ktime((hwclock & TG3_TSTAMP_MASK) +
6345 					   tp->ptp_adjust);
6346 }
6347 
6348 /* tp->lock must be held */
6349 static void tg3_ptp_init(struct tg3 *tp)
6350 {
6351 	if (!tg3_flag(tp, PTP_CAPABLE))
6352 		return;
6353 
6354 	/* Initialize the hardware clock to the system time. */
6355 	tg3_refclk_write(tp, ktime_to_ns(ktime_get_real()));
6356 	tp->ptp_adjust = 0;
6357 	tp->ptp_info = tg3_ptp_caps;
6358 }
6359 
6360 /* tp->lock must be held */
6361 static void tg3_ptp_resume(struct tg3 *tp)
6362 {
6363 	if (!tg3_flag(tp, PTP_CAPABLE))
6364 		return;
6365 
6366 	tg3_refclk_write(tp, ktime_to_ns(ktime_get_real()) + tp->ptp_adjust);
6367 	tp->ptp_adjust = 0;
6368 }
6369 
6370 static void tg3_ptp_fini(struct tg3 *tp)
6371 {
6372 	if (!tg3_flag(tp, PTP_CAPABLE) || !tp->ptp_clock)
6373 		return;
6374 
6375 	ptp_clock_unregister(tp->ptp_clock);
6376 	tp->ptp_clock = NULL;
6377 	tp->ptp_adjust = 0;
6378 }
6379 
6380 static inline int tg3_irq_sync(struct tg3 *tp)
6381 {
6382 	return tp->irq_sync;
6383 }
6384 
6385 static inline void tg3_rd32_loop(struct tg3 *tp, u32 *dst, u32 off, u32 len)
6386 {
6387 	int i;
6388 
6389 	dst = (u32 *)((u8 *)dst + off);
6390 	for (i = 0; i < len; i += sizeof(u32))
6391 		*dst++ = tr32(off + i);
6392 }
6393 
6394 static void tg3_dump_legacy_regs(struct tg3 *tp, u32 *regs)
6395 {
6396 	tg3_rd32_loop(tp, regs, TG3PCI_VENDOR, 0xb0);
6397 	tg3_rd32_loop(tp, regs, MAILBOX_INTERRUPT_0, 0x200);
6398 	tg3_rd32_loop(tp, regs, MAC_MODE, 0x4f0);
6399 	tg3_rd32_loop(tp, regs, SNDDATAI_MODE, 0xe0);
6400 	tg3_rd32_loop(tp, regs, SNDDATAC_MODE, 0x04);
6401 	tg3_rd32_loop(tp, regs, SNDBDS_MODE, 0x80);
6402 	tg3_rd32_loop(tp, regs, SNDBDI_MODE, 0x48);
6403 	tg3_rd32_loop(tp, regs, SNDBDC_MODE, 0x04);
6404 	tg3_rd32_loop(tp, regs, RCVLPC_MODE, 0x20);
6405 	tg3_rd32_loop(tp, regs, RCVLPC_SELLST_BASE, 0x15c);
6406 	tg3_rd32_loop(tp, regs, RCVDBDI_MODE, 0x0c);
6407 	tg3_rd32_loop(tp, regs, RCVDBDI_JUMBO_BD, 0x3c);
6408 	tg3_rd32_loop(tp, regs, RCVDBDI_BD_PROD_IDX_0, 0x44);
6409 	tg3_rd32_loop(tp, regs, RCVDCC_MODE, 0x04);
6410 	tg3_rd32_loop(tp, regs, RCVBDI_MODE, 0x20);
6411 	tg3_rd32_loop(tp, regs, RCVCC_MODE, 0x14);
6412 	tg3_rd32_loop(tp, regs, RCVLSC_MODE, 0x08);
6413 	tg3_rd32_loop(tp, regs, MBFREE_MODE, 0x08);
6414 	tg3_rd32_loop(tp, regs, HOSTCC_MODE, 0x100);
6415 
6416 	if (tg3_flag(tp, SUPPORT_MSIX))
6417 		tg3_rd32_loop(tp, regs, HOSTCC_RXCOL_TICKS_VEC1, 0x180);
6418 
6419 	tg3_rd32_loop(tp, regs, MEMARB_MODE, 0x10);
6420 	tg3_rd32_loop(tp, regs, BUFMGR_MODE, 0x58);
6421 	tg3_rd32_loop(tp, regs, RDMAC_MODE, 0x08);
6422 	tg3_rd32_loop(tp, regs, WDMAC_MODE, 0x08);
6423 	tg3_rd32_loop(tp, regs, RX_CPU_MODE, 0x04);
6424 	tg3_rd32_loop(tp, regs, RX_CPU_STATE, 0x04);
6425 	tg3_rd32_loop(tp, regs, RX_CPU_PGMCTR, 0x04);
6426 	tg3_rd32_loop(tp, regs, RX_CPU_HWBKPT, 0x04);
6427 
6428 	if (!tg3_flag(tp, 5705_PLUS)) {
6429 		tg3_rd32_loop(tp, regs, TX_CPU_MODE, 0x04);
6430 		tg3_rd32_loop(tp, regs, TX_CPU_STATE, 0x04);
6431 		tg3_rd32_loop(tp, regs, TX_CPU_PGMCTR, 0x04);
6432 	}
6433 
6434 	tg3_rd32_loop(tp, regs, GRCMBOX_INTERRUPT_0, 0x110);
6435 	tg3_rd32_loop(tp, regs, FTQ_RESET, 0x120);
6436 	tg3_rd32_loop(tp, regs, MSGINT_MODE, 0x0c);
6437 	tg3_rd32_loop(tp, regs, DMAC_MODE, 0x04);
6438 	tg3_rd32_loop(tp, regs, GRC_MODE, 0x4c);
6439 
6440 	if (tg3_flag(tp, NVRAM))
6441 		tg3_rd32_loop(tp, regs, NVRAM_CMD, 0x24);
6442 }
6443 
6444 static void tg3_dump_state(struct tg3 *tp)
6445 {
6446 	int i;
6447 	u32 *regs;
6448 
6449 	regs = kzalloc(TG3_REG_BLK_SIZE, GFP_ATOMIC);
6450 	if (!regs)
6451 		return;
6452 
6453 	if (tg3_flag(tp, PCI_EXPRESS)) {
6454 		/* Read up to but not including private PCI registers */
6455 		for (i = 0; i < TG3_PCIE_TLDLPL_PORT; i += sizeof(u32))
6456 			regs[i / sizeof(u32)] = tr32(i);
6457 	} else
6458 		tg3_dump_legacy_regs(tp, regs);
6459 
6460 	for (i = 0; i < TG3_REG_BLK_SIZE / sizeof(u32); i += 4) {
6461 		if (!regs[i + 0] && !regs[i + 1] &&
6462 		    !regs[i + 2] && !regs[i + 3])
6463 			continue;
6464 
6465 		netdev_err(tp->dev, "0x%08x: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
6466 			   i * 4,
6467 			   regs[i + 0], regs[i + 1], regs[i + 2], regs[i + 3]);
6468 	}
6469 
6470 	kfree(regs);
6471 
6472 	for (i = 0; i < tp->irq_cnt; i++) {
6473 		struct tg3_napi *tnapi = &tp->napi[i];
6474 
6475 		/* SW status block */
6476 		netdev_err(tp->dev,
6477 			 "%d: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
6478 			   i,
6479 			   tnapi->hw_status->status,
6480 			   tnapi->hw_status->status_tag,
6481 			   tnapi->hw_status->rx_jumbo_consumer,
6482 			   tnapi->hw_status->rx_consumer,
6483 			   tnapi->hw_status->rx_mini_consumer,
6484 			   tnapi->hw_status->idx[0].rx_producer,
6485 			   tnapi->hw_status->idx[0].tx_consumer);
6486 
6487 		netdev_err(tp->dev,
6488 		"%d: NAPI info [%08x:%08x:(%04x:%04x:%04x):%04x:(%04x:%04x:%04x:%04x)]\n",
6489 			   i,
6490 			   tnapi->last_tag, tnapi->last_irq_tag,
6491 			   tnapi->tx_prod, tnapi->tx_cons, tnapi->tx_pending,
6492 			   tnapi->rx_rcb_ptr,
6493 			   tnapi->prodring.rx_std_prod_idx,
6494 			   tnapi->prodring.rx_std_cons_idx,
6495 			   tnapi->prodring.rx_jmb_prod_idx,
6496 			   tnapi->prodring.rx_jmb_cons_idx);
6497 	}
6498 }
6499 
6500 /* This is called whenever we suspect that the system chipset is re-
6501  * ordering the sequence of MMIO to the tx send mailbox. The symptom
6502  * is bogus tx completions. We try to recover by setting the
6503  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
6504  * in the workqueue.
6505  */
6506 static void tg3_tx_recover(struct tg3 *tp)
6507 {
6508 	BUG_ON(tg3_flag(tp, MBOX_WRITE_REORDER) ||
6509 	       tp->write32_tx_mbox == tg3_write_indirect_mbox);
6510 
6511 	netdev_warn(tp->dev,
6512 		    "The system may be re-ordering memory-mapped I/O "
6513 		    "cycles to the network device, attempting to recover. "
6514 		    "Please report the problem to the driver maintainer "
6515 		    "and include system chipset information.\n");
6516 
6517 	tg3_flag_set(tp, TX_RECOVERY_PENDING);
6518 }
6519 
6520 static inline u32 tg3_tx_avail(struct tg3_napi *tnapi)
6521 {
6522 	/* Tell compiler to fetch tx indices from memory. */
6523 	barrier();
6524 	return tnapi->tx_pending -
6525 	       ((tnapi->tx_prod - tnapi->tx_cons) & (TG3_TX_RING_SIZE - 1));
6526 }
6527 
6528 /* Tigon3 never reports partial packet sends.  So we do not
6529  * need special logic to handle SKBs that have not had all
6530  * of their frags sent yet, like SunGEM does.
6531  */
6532 static void tg3_tx(struct tg3_napi *tnapi)
6533 {
6534 	struct tg3 *tp = tnapi->tp;
6535 	u32 hw_idx = tnapi->hw_status->idx[0].tx_consumer;
6536 	u32 sw_idx = tnapi->tx_cons;
6537 	struct netdev_queue *txq;
6538 	int index = tnapi - tp->napi;
6539 	unsigned int pkts_compl = 0, bytes_compl = 0;
6540 
6541 	if (tg3_flag(tp, ENABLE_TSS))
6542 		index--;
6543 
6544 	txq = netdev_get_tx_queue(tp->dev, index);
6545 
6546 	while (sw_idx != hw_idx) {
6547 		struct tg3_tx_ring_info *ri = &tnapi->tx_buffers[sw_idx];
6548 		struct sk_buff *skb = ri->skb;
6549 		int i, tx_bug = 0;
6550 
6551 		if (unlikely(skb == NULL)) {
6552 			tg3_tx_recover(tp);
6553 			return;
6554 		}
6555 
6556 		if (tnapi->tx_ring[sw_idx].len_flags & TXD_FLAG_HWTSTAMP) {
6557 			struct skb_shared_hwtstamps timestamp;
6558 			u64 hwclock = tr32(TG3_TX_TSTAMP_LSB);
6559 			hwclock |= (u64)tr32(TG3_TX_TSTAMP_MSB) << 32;
6560 
6561 			tg3_hwclock_to_timestamp(tp, hwclock, &timestamp);
6562 
6563 			skb_tstamp_tx(skb, &timestamp);
6564 		}
6565 
6566 		dma_unmap_single(&tp->pdev->dev, dma_unmap_addr(ri, mapping),
6567 				 skb_headlen(skb), DMA_TO_DEVICE);
6568 
6569 		ri->skb = NULL;
6570 
6571 		while (ri->fragmented) {
6572 			ri->fragmented = false;
6573 			sw_idx = NEXT_TX(sw_idx);
6574 			ri = &tnapi->tx_buffers[sw_idx];
6575 		}
6576 
6577 		sw_idx = NEXT_TX(sw_idx);
6578 
6579 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
6580 			ri = &tnapi->tx_buffers[sw_idx];
6581 			if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
6582 				tx_bug = 1;
6583 
6584 			dma_unmap_page(&tp->pdev->dev,
6585 				       dma_unmap_addr(ri, mapping),
6586 				       skb_frag_size(&skb_shinfo(skb)->frags[i]),
6587 				       DMA_TO_DEVICE);
6588 
6589 			while (ri->fragmented) {
6590 				ri->fragmented = false;
6591 				sw_idx = NEXT_TX(sw_idx);
6592 				ri = &tnapi->tx_buffers[sw_idx];
6593 			}
6594 
6595 			sw_idx = NEXT_TX(sw_idx);
6596 		}
6597 
6598 		pkts_compl++;
6599 		bytes_compl += skb->len;
6600 
6601 		dev_consume_skb_any(skb);
6602 
6603 		if (unlikely(tx_bug)) {
6604 			tg3_tx_recover(tp);
6605 			return;
6606 		}
6607 	}
6608 
6609 	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
6610 
6611 	tnapi->tx_cons = sw_idx;
6612 
6613 	/* Need to make the tx_cons update visible to tg3_start_xmit()
6614 	 * before checking for netif_queue_stopped().  Without the
6615 	 * memory barrier, there is a small possibility that tg3_start_xmit()
6616 	 * will miss it and cause the queue to be stopped forever.
6617 	 */
6618 	smp_mb();
6619 
6620 	if (unlikely(netif_tx_queue_stopped(txq) &&
6621 		     (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))) {
6622 		__netif_tx_lock(txq, smp_processor_id());
6623 		if (netif_tx_queue_stopped(txq) &&
6624 		    (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))
6625 			netif_tx_wake_queue(txq);
6626 		__netif_tx_unlock(txq);
6627 	}
6628 }
6629 
6630 static void tg3_frag_free(bool is_frag, void *data)
6631 {
6632 	if (is_frag)
6633 		skb_free_frag(data);
6634 	else
6635 		kfree(data);
6636 }
6637 
6638 static void tg3_rx_data_free(struct tg3 *tp, struct ring_info *ri, u32 map_sz)
6639 {
6640 	unsigned int skb_size = SKB_DATA_ALIGN(map_sz + TG3_RX_OFFSET(tp)) +
6641 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
6642 
6643 	if (!ri->data)
6644 		return;
6645 
6646 	dma_unmap_single(&tp->pdev->dev, dma_unmap_addr(ri, mapping), map_sz,
6647 			 DMA_FROM_DEVICE);
6648 	tg3_frag_free(skb_size <= PAGE_SIZE, ri->data);
6649 	ri->data = NULL;
6650 }
6651 
6652 
6653 /* Returns size of skb allocated or < 0 on error.
6654  *
6655  * We only need to fill in the address because the other members
6656  * of the RX descriptor are invariant, see tg3_init_rings.
6657  *
6658  * Note the purposeful assymetry of cpu vs. chip accesses.  For
6659  * posting buffers we only dirty the first cache line of the RX
6660  * descriptor (containing the address).  Whereas for the RX status
6661  * buffers the cpu only reads the last cacheline of the RX descriptor
6662  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
6663  */
6664 static int tg3_alloc_rx_data(struct tg3 *tp, struct tg3_rx_prodring_set *tpr,
6665 			     u32 opaque_key, u32 dest_idx_unmasked,
6666 			     unsigned int *frag_size)
6667 {
6668 	struct tg3_rx_buffer_desc *desc;
6669 	struct ring_info *map;
6670 	u8 *data;
6671 	dma_addr_t mapping;
6672 	int skb_size, data_size, dest_idx;
6673 
6674 	switch (opaque_key) {
6675 	case RXD_OPAQUE_RING_STD:
6676 		dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
6677 		desc = &tpr->rx_std[dest_idx];
6678 		map = &tpr->rx_std_buffers[dest_idx];
6679 		data_size = tp->rx_pkt_map_sz;
6680 		break;
6681 
6682 	case RXD_OPAQUE_RING_JUMBO:
6683 		dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
6684 		desc = &tpr->rx_jmb[dest_idx].std;
6685 		map = &tpr->rx_jmb_buffers[dest_idx];
6686 		data_size = TG3_RX_JMB_MAP_SZ;
6687 		break;
6688 
6689 	default:
6690 		return -EINVAL;
6691 	}
6692 
6693 	/* Do not overwrite any of the map or rp information
6694 	 * until we are sure we can commit to a new buffer.
6695 	 *
6696 	 * Callers depend upon this behavior and assume that
6697 	 * we leave everything unchanged if we fail.
6698 	 */
6699 	skb_size = SKB_DATA_ALIGN(data_size + TG3_RX_OFFSET(tp)) +
6700 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
6701 	if (skb_size <= PAGE_SIZE) {
6702 		data = napi_alloc_frag(skb_size);
6703 		*frag_size = skb_size;
6704 	} else {
6705 		data = kmalloc(skb_size, GFP_ATOMIC);
6706 		*frag_size = 0;
6707 	}
6708 	if (!data)
6709 		return -ENOMEM;
6710 
6711 	mapping = dma_map_single(&tp->pdev->dev, data + TG3_RX_OFFSET(tp),
6712 				 data_size, DMA_FROM_DEVICE);
6713 	if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) {
6714 		tg3_frag_free(skb_size <= PAGE_SIZE, data);
6715 		return -EIO;
6716 	}
6717 
6718 	map->data = data;
6719 	dma_unmap_addr_set(map, mapping, mapping);
6720 
6721 	desc->addr_hi = ((u64)mapping >> 32);
6722 	desc->addr_lo = ((u64)mapping & 0xffffffff);
6723 
6724 	return data_size;
6725 }
6726 
6727 /* We only need to move over in the address because the other
6728  * members of the RX descriptor are invariant.  See notes above
6729  * tg3_alloc_rx_data for full details.
6730  */
6731 static void tg3_recycle_rx(struct tg3_napi *tnapi,
6732 			   struct tg3_rx_prodring_set *dpr,
6733 			   u32 opaque_key, int src_idx,
6734 			   u32 dest_idx_unmasked)
6735 {
6736 	struct tg3 *tp = tnapi->tp;
6737 	struct tg3_rx_buffer_desc *src_desc, *dest_desc;
6738 	struct ring_info *src_map, *dest_map;
6739 	struct tg3_rx_prodring_set *spr = &tp->napi[0].prodring;
6740 	int dest_idx;
6741 
6742 	switch (opaque_key) {
6743 	case RXD_OPAQUE_RING_STD:
6744 		dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
6745 		dest_desc = &dpr->rx_std[dest_idx];
6746 		dest_map = &dpr->rx_std_buffers[dest_idx];
6747 		src_desc = &spr->rx_std[src_idx];
6748 		src_map = &spr->rx_std_buffers[src_idx];
6749 		break;
6750 
6751 	case RXD_OPAQUE_RING_JUMBO:
6752 		dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
6753 		dest_desc = &dpr->rx_jmb[dest_idx].std;
6754 		dest_map = &dpr->rx_jmb_buffers[dest_idx];
6755 		src_desc = &spr->rx_jmb[src_idx].std;
6756 		src_map = &spr->rx_jmb_buffers[src_idx];
6757 		break;
6758 
6759 	default:
6760 		return;
6761 	}
6762 
6763 	dest_map->data = src_map->data;
6764 	dma_unmap_addr_set(dest_map, mapping,
6765 			   dma_unmap_addr(src_map, mapping));
6766 	dest_desc->addr_hi = src_desc->addr_hi;
6767 	dest_desc->addr_lo = src_desc->addr_lo;
6768 
6769 	/* Ensure that the update to the skb happens after the physical
6770 	 * addresses have been transferred to the new BD location.
6771 	 */
6772 	smp_wmb();
6773 
6774 	src_map->data = NULL;
6775 }
6776 
6777 /* The RX ring scheme is composed of multiple rings which post fresh
6778  * buffers to the chip, and one special ring the chip uses to report
6779  * status back to the host.
6780  *
6781  * The special ring reports the status of received packets to the
6782  * host.  The chip does not write into the original descriptor the
6783  * RX buffer was obtained from.  The chip simply takes the original
6784  * descriptor as provided by the host, updates the status and length
6785  * field, then writes this into the next status ring entry.
6786  *
6787  * Each ring the host uses to post buffers to the chip is described
6788  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
6789  * it is first placed into the on-chip ram.  When the packet's length
6790  * is known, it walks down the TG3_BDINFO entries to select the ring.
6791  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
6792  * which is within the range of the new packet's length is chosen.
6793  *
6794  * The "separate ring for rx status" scheme may sound queer, but it makes
6795  * sense from a cache coherency perspective.  If only the host writes
6796  * to the buffer post rings, and only the chip writes to the rx status
6797  * rings, then cache lines never move beyond shared-modified state.
6798  * If both the host and chip were to write into the same ring, cache line
6799  * eviction could occur since both entities want it in an exclusive state.
6800  */
6801 static int tg3_rx(struct tg3_napi *tnapi, int budget)
6802 {
6803 	struct tg3 *tp = tnapi->tp;
6804 	u32 work_mask, rx_std_posted = 0;
6805 	u32 std_prod_idx, jmb_prod_idx;
6806 	u32 sw_idx = tnapi->rx_rcb_ptr;
6807 	u16 hw_idx;
6808 	int received;
6809 	struct tg3_rx_prodring_set *tpr = &tnapi->prodring;
6810 
6811 	hw_idx = *(tnapi->rx_rcb_prod_idx);
6812 	/*
6813 	 * We need to order the read of hw_idx and the read of
6814 	 * the opaque cookie.
6815 	 */
6816 	rmb();
6817 	work_mask = 0;
6818 	received = 0;
6819 	std_prod_idx = tpr->rx_std_prod_idx;
6820 	jmb_prod_idx = tpr->rx_jmb_prod_idx;
6821 	while (sw_idx != hw_idx && budget > 0) {
6822 		struct ring_info *ri;
6823 		struct tg3_rx_buffer_desc *desc = &tnapi->rx_rcb[sw_idx];
6824 		unsigned int len;
6825 		struct sk_buff *skb;
6826 		dma_addr_t dma_addr;
6827 		u32 opaque_key, desc_idx, *post_ptr;
6828 		u8 *data;
6829 		u64 tstamp = 0;
6830 
6831 		desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
6832 		opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
6833 		if (opaque_key == RXD_OPAQUE_RING_STD) {
6834 			ri = &tp->napi[0].prodring.rx_std_buffers[desc_idx];
6835 			dma_addr = dma_unmap_addr(ri, mapping);
6836 			data = ri->data;
6837 			post_ptr = &std_prod_idx;
6838 			rx_std_posted++;
6839 		} else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
6840 			ri = &tp->napi[0].prodring.rx_jmb_buffers[desc_idx];
6841 			dma_addr = dma_unmap_addr(ri, mapping);
6842 			data = ri->data;
6843 			post_ptr = &jmb_prod_idx;
6844 		} else
6845 			goto next_pkt_nopost;
6846 
6847 		work_mask |= opaque_key;
6848 
6849 		if (desc->err_vlan & RXD_ERR_MASK) {
6850 		drop_it:
6851 			tg3_recycle_rx(tnapi, tpr, opaque_key,
6852 				       desc_idx, *post_ptr);
6853 		drop_it_no_recycle:
6854 			/* Other statistics kept track of by card. */
6855 			tp->rx_dropped++;
6856 			goto next_pkt;
6857 		}
6858 
6859 		prefetch(data + TG3_RX_OFFSET(tp));
6860 		len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) -
6861 		      ETH_FCS_LEN;
6862 
6863 		if ((desc->type_flags & RXD_FLAG_PTPSTAT_MASK) ==
6864 		     RXD_FLAG_PTPSTAT_PTPV1 ||
6865 		    (desc->type_flags & RXD_FLAG_PTPSTAT_MASK) ==
6866 		     RXD_FLAG_PTPSTAT_PTPV2) {
6867 			tstamp = tr32(TG3_RX_TSTAMP_LSB);
6868 			tstamp |= (u64)tr32(TG3_RX_TSTAMP_MSB) << 32;
6869 		}
6870 
6871 		if (len > TG3_RX_COPY_THRESH(tp)) {
6872 			int skb_size;
6873 			unsigned int frag_size;
6874 
6875 			skb_size = tg3_alloc_rx_data(tp, tpr, opaque_key,
6876 						    *post_ptr, &frag_size);
6877 			if (skb_size < 0)
6878 				goto drop_it;
6879 
6880 			dma_unmap_single(&tp->pdev->dev, dma_addr, skb_size,
6881 					 DMA_FROM_DEVICE);
6882 
6883 			/* Ensure that the update to the data happens
6884 			 * after the usage of the old DMA mapping.
6885 			 */
6886 			smp_wmb();
6887 
6888 			ri->data = NULL;
6889 
6890 			skb = build_skb(data, frag_size);
6891 			if (!skb) {
6892 				tg3_frag_free(frag_size != 0, data);
6893 				goto drop_it_no_recycle;
6894 			}
6895 			skb_reserve(skb, TG3_RX_OFFSET(tp));
6896 		} else {
6897 			tg3_recycle_rx(tnapi, tpr, opaque_key,
6898 				       desc_idx, *post_ptr);
6899 
6900 			skb = netdev_alloc_skb(tp->dev,
6901 					       len + TG3_RAW_IP_ALIGN);
6902 			if (skb == NULL)
6903 				goto drop_it_no_recycle;
6904 
6905 			skb_reserve(skb, TG3_RAW_IP_ALIGN);
6906 			dma_sync_single_for_cpu(&tp->pdev->dev, dma_addr, len,
6907 						DMA_FROM_DEVICE);
6908 			memcpy(skb->data,
6909 			       data + TG3_RX_OFFSET(tp),
6910 			       len);
6911 			dma_sync_single_for_device(&tp->pdev->dev, dma_addr,
6912 						   len, DMA_FROM_DEVICE);
6913 		}
6914 
6915 		skb_put(skb, len);
6916 		if (tstamp)
6917 			tg3_hwclock_to_timestamp(tp, tstamp,
6918 						 skb_hwtstamps(skb));
6919 
6920 		if ((tp->dev->features & NETIF_F_RXCSUM) &&
6921 		    (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
6922 		    (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
6923 		      >> RXD_TCPCSUM_SHIFT) == 0xffff))
6924 			skb->ip_summed = CHECKSUM_UNNECESSARY;
6925 		else
6926 			skb_checksum_none_assert(skb);
6927 
6928 		skb->protocol = eth_type_trans(skb, tp->dev);
6929 
6930 		if (len > (tp->dev->mtu + ETH_HLEN) &&
6931 		    skb->protocol != htons(ETH_P_8021Q) &&
6932 		    skb->protocol != htons(ETH_P_8021AD)) {
6933 			dev_kfree_skb_any(skb);
6934 			goto drop_it_no_recycle;
6935 		}
6936 
6937 		if (desc->type_flags & RXD_FLAG_VLAN &&
6938 		    !(tp->rx_mode & RX_MODE_KEEP_VLAN_TAG))
6939 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
6940 					       desc->err_vlan & RXD_VLAN_MASK);
6941 
6942 		napi_gro_receive(&tnapi->napi, skb);
6943 
6944 		received++;
6945 		budget--;
6946 
6947 next_pkt:
6948 		(*post_ptr)++;
6949 
6950 		if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
6951 			tpr->rx_std_prod_idx = std_prod_idx &
6952 					       tp->rx_std_ring_mask;
6953 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
6954 				     tpr->rx_std_prod_idx);
6955 			work_mask &= ~RXD_OPAQUE_RING_STD;
6956 			rx_std_posted = 0;
6957 		}
6958 next_pkt_nopost:
6959 		sw_idx++;
6960 		sw_idx &= tp->rx_ret_ring_mask;
6961 
6962 		/* Refresh hw_idx to see if there is new work */
6963 		if (sw_idx == hw_idx) {
6964 			hw_idx = *(tnapi->rx_rcb_prod_idx);
6965 			rmb();
6966 		}
6967 	}
6968 
6969 	/* ACK the status ring. */
6970 	tnapi->rx_rcb_ptr = sw_idx;
6971 	tw32_rx_mbox(tnapi->consmbox, sw_idx);
6972 
6973 	/* Refill RX ring(s). */
6974 	if (!tg3_flag(tp, ENABLE_RSS)) {
6975 		/* Sync BD data before updating mailbox */
6976 		wmb();
6977 
6978 		if (work_mask & RXD_OPAQUE_RING_STD) {
6979 			tpr->rx_std_prod_idx = std_prod_idx &
6980 					       tp->rx_std_ring_mask;
6981 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
6982 				     tpr->rx_std_prod_idx);
6983 		}
6984 		if (work_mask & RXD_OPAQUE_RING_JUMBO) {
6985 			tpr->rx_jmb_prod_idx = jmb_prod_idx &
6986 					       tp->rx_jmb_ring_mask;
6987 			tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
6988 				     tpr->rx_jmb_prod_idx);
6989 		}
6990 	} else if (work_mask) {
6991 		/* rx_std_buffers[] and rx_jmb_buffers[] entries must be
6992 		 * updated before the producer indices can be updated.
6993 		 */
6994 		smp_wmb();
6995 
6996 		tpr->rx_std_prod_idx = std_prod_idx & tp->rx_std_ring_mask;
6997 		tpr->rx_jmb_prod_idx = jmb_prod_idx & tp->rx_jmb_ring_mask;
6998 
6999 		if (tnapi != &tp->napi[1]) {
7000 			tp->rx_refill = true;
7001 			napi_schedule(&tp->napi[1].napi);
7002 		}
7003 	}
7004 
7005 	return received;
7006 }
7007 
7008 static void tg3_poll_link(struct tg3 *tp)
7009 {
7010 	/* handle link change and other phy events */
7011 	if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
7012 		struct tg3_hw_status *sblk = tp->napi[0].hw_status;
7013 
7014 		if (sblk->status & SD_STATUS_LINK_CHG) {
7015 			sblk->status = SD_STATUS_UPDATED |
7016 				       (sblk->status & ~SD_STATUS_LINK_CHG);
7017 			spin_lock(&tp->lock);
7018 			if (tg3_flag(tp, USE_PHYLIB)) {
7019 				tw32_f(MAC_STATUS,
7020 				     (MAC_STATUS_SYNC_CHANGED |
7021 				      MAC_STATUS_CFG_CHANGED |
7022 				      MAC_STATUS_MI_COMPLETION |
7023 				      MAC_STATUS_LNKSTATE_CHANGED));
7024 				udelay(40);
7025 			} else
7026 				tg3_setup_phy(tp, false);
7027 			spin_unlock(&tp->lock);
7028 		}
7029 	}
7030 }
7031 
7032 static int tg3_rx_prodring_xfer(struct tg3 *tp,
7033 				struct tg3_rx_prodring_set *dpr,
7034 				struct tg3_rx_prodring_set *spr)
7035 {
7036 	u32 si, di, cpycnt, src_prod_idx;
7037 	int i, err = 0;
7038 
7039 	while (1) {
7040 		src_prod_idx = spr->rx_std_prod_idx;
7041 
7042 		/* Make sure updates to the rx_std_buffers[] entries and the
7043 		 * standard producer index are seen in the correct order.
7044 		 */
7045 		smp_rmb();
7046 
7047 		if (spr->rx_std_cons_idx == src_prod_idx)
7048 			break;
7049 
7050 		if (spr->rx_std_cons_idx < src_prod_idx)
7051 			cpycnt = src_prod_idx - spr->rx_std_cons_idx;
7052 		else
7053 			cpycnt = tp->rx_std_ring_mask + 1 -
7054 				 spr->rx_std_cons_idx;
7055 
7056 		cpycnt = min(cpycnt,
7057 			     tp->rx_std_ring_mask + 1 - dpr->rx_std_prod_idx);
7058 
7059 		si = spr->rx_std_cons_idx;
7060 		di = dpr->rx_std_prod_idx;
7061 
7062 		for (i = di; i < di + cpycnt; i++) {
7063 			if (dpr->rx_std_buffers[i].data) {
7064 				cpycnt = i - di;
7065 				err = -ENOSPC;
7066 				break;
7067 			}
7068 		}
7069 
7070 		if (!cpycnt)
7071 			break;
7072 
7073 		/* Ensure that updates to the rx_std_buffers ring and the
7074 		 * shadowed hardware producer ring from tg3_recycle_skb() are
7075 		 * ordered correctly WRT the skb check above.
7076 		 */
7077 		smp_rmb();
7078 
7079 		memcpy(&dpr->rx_std_buffers[di],
7080 		       &spr->rx_std_buffers[si],
7081 		       cpycnt * sizeof(struct ring_info));
7082 
7083 		for (i = 0; i < cpycnt; i++, di++, si++) {
7084 			struct tg3_rx_buffer_desc *sbd, *dbd;
7085 			sbd = &spr->rx_std[si];
7086 			dbd = &dpr->rx_std[di];
7087 			dbd->addr_hi = sbd->addr_hi;
7088 			dbd->addr_lo = sbd->addr_lo;
7089 		}
7090 
7091 		spr->rx_std_cons_idx = (spr->rx_std_cons_idx + cpycnt) &
7092 				       tp->rx_std_ring_mask;
7093 		dpr->rx_std_prod_idx = (dpr->rx_std_prod_idx + cpycnt) &
7094 				       tp->rx_std_ring_mask;
7095 	}
7096 
7097 	while (1) {
7098 		src_prod_idx = spr->rx_jmb_prod_idx;
7099 
7100 		/* Make sure updates to the rx_jmb_buffers[] entries and
7101 		 * the jumbo producer index are seen in the correct order.
7102 		 */
7103 		smp_rmb();
7104 
7105 		if (spr->rx_jmb_cons_idx == src_prod_idx)
7106 			break;
7107 
7108 		if (spr->rx_jmb_cons_idx < src_prod_idx)
7109 			cpycnt = src_prod_idx - spr->rx_jmb_cons_idx;
7110 		else
7111 			cpycnt = tp->rx_jmb_ring_mask + 1 -
7112 				 spr->rx_jmb_cons_idx;
7113 
7114 		cpycnt = min(cpycnt,
7115 			     tp->rx_jmb_ring_mask + 1 - dpr->rx_jmb_prod_idx);
7116 
7117 		si = spr->rx_jmb_cons_idx;
7118 		di = dpr->rx_jmb_prod_idx;
7119 
7120 		for (i = di; i < di + cpycnt; i++) {
7121 			if (dpr->rx_jmb_buffers[i].data) {
7122 				cpycnt = i - di;
7123 				err = -ENOSPC;
7124 				break;
7125 			}
7126 		}
7127 
7128 		if (!cpycnt)
7129 			break;
7130 
7131 		/* Ensure that updates to the rx_jmb_buffers ring and the
7132 		 * shadowed hardware producer ring from tg3_recycle_skb() are
7133 		 * ordered correctly WRT the skb check above.
7134 		 */
7135 		smp_rmb();
7136 
7137 		memcpy(&dpr->rx_jmb_buffers[di],
7138 		       &spr->rx_jmb_buffers[si],
7139 		       cpycnt * sizeof(struct ring_info));
7140 
7141 		for (i = 0; i < cpycnt; i++, di++, si++) {
7142 			struct tg3_rx_buffer_desc *sbd, *dbd;
7143 			sbd = &spr->rx_jmb[si].std;
7144 			dbd = &dpr->rx_jmb[di].std;
7145 			dbd->addr_hi = sbd->addr_hi;
7146 			dbd->addr_lo = sbd->addr_lo;
7147 		}
7148 
7149 		spr->rx_jmb_cons_idx = (spr->rx_jmb_cons_idx + cpycnt) &
7150 				       tp->rx_jmb_ring_mask;
7151 		dpr->rx_jmb_prod_idx = (dpr->rx_jmb_prod_idx + cpycnt) &
7152 				       tp->rx_jmb_ring_mask;
7153 	}
7154 
7155 	return err;
7156 }
7157 
7158 static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget)
7159 {
7160 	struct tg3 *tp = tnapi->tp;
7161 
7162 	/* run TX completion thread */
7163 	if (tnapi->hw_status->idx[0].tx_consumer != tnapi->tx_cons) {
7164 		tg3_tx(tnapi);
7165 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7166 			return work_done;
7167 	}
7168 
7169 	if (!tnapi->rx_rcb_prod_idx)
7170 		return work_done;
7171 
7172 	/* run RX thread, within the bounds set by NAPI.
7173 	 * All RX "locking" is done by ensuring outside
7174 	 * code synchronizes with tg3->napi.poll()
7175 	 */
7176 	if (*(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
7177 		work_done += tg3_rx(tnapi, budget - work_done);
7178 
7179 	if (tg3_flag(tp, ENABLE_RSS) && tnapi == &tp->napi[1]) {
7180 		struct tg3_rx_prodring_set *dpr = &tp->napi[0].prodring;
7181 		int i, err = 0;
7182 		u32 std_prod_idx = dpr->rx_std_prod_idx;
7183 		u32 jmb_prod_idx = dpr->rx_jmb_prod_idx;
7184 
7185 		tp->rx_refill = false;
7186 		for (i = 1; i <= tp->rxq_cnt; i++)
7187 			err |= tg3_rx_prodring_xfer(tp, dpr,
7188 						    &tp->napi[i].prodring);
7189 
7190 		wmb();
7191 
7192 		if (std_prod_idx != dpr->rx_std_prod_idx)
7193 			tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
7194 				     dpr->rx_std_prod_idx);
7195 
7196 		if (jmb_prod_idx != dpr->rx_jmb_prod_idx)
7197 			tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
7198 				     dpr->rx_jmb_prod_idx);
7199 
7200 		if (err)
7201 			tw32_f(HOSTCC_MODE, tp->coal_now);
7202 	}
7203 
7204 	return work_done;
7205 }
7206 
7207 static inline void tg3_reset_task_schedule(struct tg3 *tp)
7208 {
7209 	if (!test_and_set_bit(TG3_FLAG_RESET_TASK_PENDING, tp->tg3_flags))
7210 		schedule_work(&tp->reset_task);
7211 }
7212 
7213 static inline void tg3_reset_task_cancel(struct tg3 *tp)
7214 {
7215 	if (test_and_clear_bit(TG3_FLAG_RESET_TASK_PENDING, tp->tg3_flags))
7216 		cancel_work_sync(&tp->reset_task);
7217 	tg3_flag_clear(tp, TX_RECOVERY_PENDING);
7218 }
7219 
7220 static int tg3_poll_msix(struct napi_struct *napi, int budget)
7221 {
7222 	struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
7223 	struct tg3 *tp = tnapi->tp;
7224 	int work_done = 0;
7225 	struct tg3_hw_status *sblk = tnapi->hw_status;
7226 
7227 	while (1) {
7228 		work_done = tg3_poll_work(tnapi, work_done, budget);
7229 
7230 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7231 			goto tx_recovery;
7232 
7233 		if (unlikely(work_done >= budget))
7234 			break;
7235 
7236 		/* tp->last_tag is used in tg3_int_reenable() below
7237 		 * to tell the hw how much work has been processed,
7238 		 * so we must read it before checking for more work.
7239 		 */
7240 		tnapi->last_tag = sblk->status_tag;
7241 		tnapi->last_irq_tag = tnapi->last_tag;
7242 		rmb();
7243 
7244 		/* check for RX/TX work to do */
7245 		if (likely(sblk->idx[0].tx_consumer == tnapi->tx_cons &&
7246 			   *(tnapi->rx_rcb_prod_idx) == tnapi->rx_rcb_ptr)) {
7247 
7248 			/* This test here is not race free, but will reduce
7249 			 * the number of interrupts by looping again.
7250 			 */
7251 			if (tnapi == &tp->napi[1] && tp->rx_refill)
7252 				continue;
7253 
7254 			napi_complete_done(napi, work_done);
7255 			/* Reenable interrupts. */
7256 			tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
7257 
7258 			/* This test here is synchronized by napi_schedule()
7259 			 * and napi_complete() to close the race condition.
7260 			 */
7261 			if (unlikely(tnapi == &tp->napi[1] && tp->rx_refill)) {
7262 				tw32(HOSTCC_MODE, tp->coalesce_mode |
7263 						  HOSTCC_MODE_ENABLE |
7264 						  tnapi->coal_now);
7265 			}
7266 			break;
7267 		}
7268 	}
7269 
7270 	tg3_send_ape_heartbeat(tp, TG3_APE_HB_INTERVAL << 1);
7271 	return work_done;
7272 
7273 tx_recovery:
7274 	/* work_done is guaranteed to be less than budget. */
7275 	napi_complete(napi);
7276 	tg3_reset_task_schedule(tp);
7277 	return work_done;
7278 }
7279 
7280 static void tg3_process_error(struct tg3 *tp)
7281 {
7282 	u32 val;
7283 	bool real_error = false;
7284 
7285 	if (tg3_flag(tp, ERROR_PROCESSED))
7286 		return;
7287 
7288 	/* Check Flow Attention register */
7289 	val = tr32(HOSTCC_FLOW_ATTN);
7290 	if (val & ~HOSTCC_FLOW_ATTN_MBUF_LWM) {
7291 		netdev_err(tp->dev, "FLOW Attention error.  Resetting chip.\n");
7292 		real_error = true;
7293 	}
7294 
7295 	if (tr32(MSGINT_STATUS) & ~MSGINT_STATUS_MSI_REQ) {
7296 		netdev_err(tp->dev, "MSI Status error.  Resetting chip.\n");
7297 		real_error = true;
7298 	}
7299 
7300 	if (tr32(RDMAC_STATUS) || tr32(WDMAC_STATUS)) {
7301 		netdev_err(tp->dev, "DMA Status error.  Resetting chip.\n");
7302 		real_error = true;
7303 	}
7304 
7305 	if (!real_error)
7306 		return;
7307 
7308 	tg3_dump_state(tp);
7309 
7310 	tg3_flag_set(tp, ERROR_PROCESSED);
7311 	tg3_reset_task_schedule(tp);
7312 }
7313 
7314 static int tg3_poll(struct napi_struct *napi, int budget)
7315 {
7316 	struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
7317 	struct tg3 *tp = tnapi->tp;
7318 	int work_done = 0;
7319 	struct tg3_hw_status *sblk = tnapi->hw_status;
7320 
7321 	while (1) {
7322 		if (sblk->status & SD_STATUS_ERROR)
7323 			tg3_process_error(tp);
7324 
7325 		tg3_poll_link(tp);
7326 
7327 		work_done = tg3_poll_work(tnapi, work_done, budget);
7328 
7329 		if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
7330 			goto tx_recovery;
7331 
7332 		if (unlikely(work_done >= budget))
7333 			break;
7334 
7335 		if (tg3_flag(tp, TAGGED_STATUS)) {
7336 			/* tp->last_tag is used in tg3_int_reenable() below
7337 			 * to tell the hw how much work has been processed,
7338 			 * so we must read it before checking for more work.
7339 			 */
7340 			tnapi->last_tag = sblk->status_tag;
7341 			tnapi->last_irq_tag = tnapi->last_tag;
7342 			rmb();
7343 		} else
7344 			sblk->status &= ~SD_STATUS_UPDATED;
7345 
7346 		if (likely(!tg3_has_work(tnapi))) {
7347 			napi_complete_done(napi, work_done);
7348 			tg3_int_reenable(tnapi);
7349 			break;
7350 		}
7351 	}
7352 
7353 	tg3_send_ape_heartbeat(tp, TG3_APE_HB_INTERVAL << 1);
7354 	return work_done;
7355 
7356 tx_recovery:
7357 	/* work_done is guaranteed to be less than budget. */
7358 	napi_complete(napi);
7359 	tg3_reset_task_schedule(tp);
7360 	return work_done;
7361 }
7362 
7363 static void tg3_napi_disable(struct tg3 *tp)
7364 {
7365 	int i;
7366 
7367 	for (i = tp->irq_cnt - 1; i >= 0; i--)
7368 		napi_disable(&tp->napi[i].napi);
7369 }
7370 
7371 static void tg3_napi_enable(struct tg3 *tp)
7372 {
7373 	int i;
7374 
7375 	for (i = 0; i < tp->irq_cnt; i++)
7376 		napi_enable(&tp->napi[i].napi);
7377 }
7378 
7379 static void tg3_napi_init(struct tg3 *tp)
7380 {
7381 	int i;
7382 
7383 	netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll, 64);
7384 	for (i = 1; i < tp->irq_cnt; i++)
7385 		netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix, 64);
7386 }
7387 
7388 static void tg3_napi_fini(struct tg3 *tp)
7389 {
7390 	int i;
7391 
7392 	for (i = 0; i < tp->irq_cnt; i++)
7393 		netif_napi_del(&tp->napi[i].napi);
7394 }
7395 
7396 static inline void tg3_netif_stop(struct tg3 *tp)
7397 {
7398 	netif_trans_update(tp->dev);	/* prevent tx timeout */
7399 	tg3_napi_disable(tp);
7400 	netif_carrier_off(tp->dev);
7401 	netif_tx_disable(tp->dev);
7402 }
7403 
7404 /* tp->lock must be held */
7405 static inline void tg3_netif_start(struct tg3 *tp)
7406 {
7407 	tg3_ptp_resume(tp);
7408 
7409 	/* NOTE: unconditional netif_tx_wake_all_queues is only
7410 	 * appropriate so long as all callers are assured to
7411 	 * have free tx slots (such as after tg3_init_hw)
7412 	 */
7413 	netif_tx_wake_all_queues(tp->dev);
7414 
7415 	if (tp->link_up)
7416 		netif_carrier_on(tp->dev);
7417 
7418 	tg3_napi_enable(tp);
7419 	tp->napi[0].hw_status->status |= SD_STATUS_UPDATED;
7420 	tg3_enable_ints(tp);
7421 }
7422 
7423 static void tg3_irq_quiesce(struct tg3 *tp)
7424 	__releases(tp->lock)
7425 	__acquires(tp->lock)
7426 {
7427 	int i;
7428 
7429 	BUG_ON(tp->irq_sync);
7430 
7431 	tp->irq_sync = 1;
7432 	smp_mb();
7433 
7434 	spin_unlock_bh(&tp->lock);
7435 
7436 	for (i = 0; i < tp->irq_cnt; i++)
7437 		synchronize_irq(tp->napi[i].irq_vec);
7438 
7439 	spin_lock_bh(&tp->lock);
7440 }
7441 
7442 /* Fully shutdown all tg3 driver activity elsewhere in the system.
7443  * If irq_sync is non-zero, then the IRQ handler must be synchronized
7444  * with as well.  Most of the time, this is not necessary except when
7445  * shutting down the device.
7446  */
7447 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
7448 {
7449 	spin_lock_bh(&tp->lock);
7450 	if (irq_sync)
7451 		tg3_irq_quiesce(tp);
7452 }
7453 
7454 static inline void tg3_full_unlock(struct tg3 *tp)
7455 {
7456 	spin_unlock_bh(&tp->lock);
7457 }
7458 
7459 /* One-shot MSI handler - Chip automatically disables interrupt
7460  * after sending MSI so driver doesn't have to do it.
7461  */
7462 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
7463 {
7464 	struct tg3_napi *tnapi = dev_id;
7465 	struct tg3 *tp = tnapi->tp;
7466 
7467 	prefetch(tnapi->hw_status);
7468 	if (tnapi->rx_rcb)
7469 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7470 
7471 	if (likely(!tg3_irq_sync(tp)))
7472 		napi_schedule(&tnapi->napi);
7473 
7474 	return IRQ_HANDLED;
7475 }
7476 
7477 /* MSI ISR - No need to check for interrupt sharing and no need to
7478  * flush status block and interrupt mailbox. PCI ordering rules
7479  * guarantee that MSI will arrive after the status block.
7480  */
7481 static irqreturn_t tg3_msi(int irq, void *dev_id)
7482 {
7483 	struct tg3_napi *tnapi = dev_id;
7484 	struct tg3 *tp = tnapi->tp;
7485 
7486 	prefetch(tnapi->hw_status);
7487 	if (tnapi->rx_rcb)
7488 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7489 	/*
7490 	 * Writing any value to intr-mbox-0 clears PCI INTA# and
7491 	 * chip-internal interrupt pending events.
7492 	 * Writing non-zero to intr-mbox-0 additional tells the
7493 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7494 	 * event coalescing.
7495 	 */
7496 	tw32_mailbox(tnapi->int_mbox, 0x00000001);
7497 	if (likely(!tg3_irq_sync(tp)))
7498 		napi_schedule(&tnapi->napi);
7499 
7500 	return IRQ_RETVAL(1);
7501 }
7502 
7503 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
7504 {
7505 	struct tg3_napi *tnapi = dev_id;
7506 	struct tg3 *tp = tnapi->tp;
7507 	struct tg3_hw_status *sblk = tnapi->hw_status;
7508 	unsigned int handled = 1;
7509 
7510 	/* In INTx mode, it is possible for the interrupt to arrive at
7511 	 * the CPU before the status block posted prior to the interrupt.
7512 	 * Reading the PCI State register will confirm whether the
7513 	 * interrupt is ours and will flush the status block.
7514 	 */
7515 	if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
7516 		if (tg3_flag(tp, CHIP_RESETTING) ||
7517 		    (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7518 			handled = 0;
7519 			goto out;
7520 		}
7521 	}
7522 
7523 	/*
7524 	 * Writing any value to intr-mbox-0 clears PCI INTA# and
7525 	 * chip-internal interrupt pending events.
7526 	 * Writing non-zero to intr-mbox-0 additional tells the
7527 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7528 	 * event coalescing.
7529 	 *
7530 	 * Flush the mailbox to de-assert the IRQ immediately to prevent
7531 	 * spurious interrupts.  The flush impacts performance but
7532 	 * excessive spurious interrupts can be worse in some cases.
7533 	 */
7534 	tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
7535 	if (tg3_irq_sync(tp))
7536 		goto out;
7537 	sblk->status &= ~SD_STATUS_UPDATED;
7538 	if (likely(tg3_has_work(tnapi))) {
7539 		prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7540 		napi_schedule(&tnapi->napi);
7541 	} else {
7542 		/* No work, shared interrupt perhaps?  re-enable
7543 		 * interrupts, and flush that PCI write
7544 		 */
7545 		tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
7546 			       0x00000000);
7547 	}
7548 out:
7549 	return IRQ_RETVAL(handled);
7550 }
7551 
7552 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
7553 {
7554 	struct tg3_napi *tnapi = dev_id;
7555 	struct tg3 *tp = tnapi->tp;
7556 	struct tg3_hw_status *sblk = tnapi->hw_status;
7557 	unsigned int handled = 1;
7558 
7559 	/* In INTx mode, it is possible for the interrupt to arrive at
7560 	 * the CPU before the status block posted prior to the interrupt.
7561 	 * Reading the PCI State register will confirm whether the
7562 	 * interrupt is ours and will flush the status block.
7563 	 */
7564 	if (unlikely(sblk->status_tag == tnapi->last_irq_tag)) {
7565 		if (tg3_flag(tp, CHIP_RESETTING) ||
7566 		    (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7567 			handled = 0;
7568 			goto out;
7569 		}
7570 	}
7571 
7572 	/*
7573 	 * writing any value to intr-mbox-0 clears PCI INTA# and
7574 	 * chip-internal interrupt pending events.
7575 	 * writing non-zero to intr-mbox-0 additional tells the
7576 	 * NIC to stop sending us irqs, engaging "in-intr-handler"
7577 	 * event coalescing.
7578 	 *
7579 	 * Flush the mailbox to de-assert the IRQ immediately to prevent
7580 	 * spurious interrupts.  The flush impacts performance but
7581 	 * excessive spurious interrupts can be worse in some cases.
7582 	 */
7583 	tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
7584 
7585 	/*
7586 	 * In a shared interrupt configuration, sometimes other devices'
7587 	 * interrupts will scream.  We record the current status tag here
7588 	 * so that the above check can report that the screaming interrupts
7589 	 * are unhandled.  Eventually they will be silenced.
7590 	 */
7591 	tnapi->last_irq_tag = sblk->status_tag;
7592 
7593 	if (tg3_irq_sync(tp))
7594 		goto out;
7595 
7596 	prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
7597 
7598 	napi_schedule(&tnapi->napi);
7599 
7600 out:
7601 	return IRQ_RETVAL(handled);
7602 }
7603 
7604 /* ISR for interrupt test */
7605 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
7606 {
7607 	struct tg3_napi *tnapi = dev_id;
7608 	struct tg3 *tp = tnapi->tp;
7609 	struct tg3_hw_status *sblk = tnapi->hw_status;
7610 
7611 	if ((sblk->status & SD_STATUS_UPDATED) ||
7612 	    !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
7613 		tg3_disable_ints(tp);
7614 		return IRQ_RETVAL(1);
7615 	}
7616 	return IRQ_RETVAL(0);
7617 }
7618 
7619 #ifdef CONFIG_NET_POLL_CONTROLLER
7620 static void tg3_poll_controller(struct net_device *dev)
7621 {
7622 	int i;
7623 	struct tg3 *tp = netdev_priv(dev);
7624 
7625 	if (tg3_irq_sync(tp))
7626 		return;
7627 
7628 	for (i = 0; i < tp->irq_cnt; i++)
7629 		tg3_interrupt(tp->napi[i].irq_vec, &tp->napi[i]);
7630 }
7631 #endif
7632 
7633 static void tg3_tx_timeout(struct net_device *dev, unsigned int txqueue)
7634 {
7635 	struct tg3 *tp = netdev_priv(dev);
7636 
7637 	if (netif_msg_tx_err(tp)) {
7638 		netdev_err(dev, "transmit timed out, resetting\n");
7639 		tg3_dump_state(tp);
7640 	}
7641 
7642 	tg3_reset_task_schedule(tp);
7643 }
7644 
7645 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
7646 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
7647 {
7648 	u32 base = (u32) mapping & 0xffffffff;
7649 
7650 	return base + len + 8 < base;
7651 }
7652 
7653 /* Test for TSO DMA buffers that cross into regions which are within MSS bytes
7654  * of any 4GB boundaries: 4G, 8G, etc
7655  */
7656 static inline int tg3_4g_tso_overflow_test(struct tg3 *tp, dma_addr_t mapping,
7657 					   u32 len, u32 mss)
7658 {
7659 	if (tg3_asic_rev(tp) == ASIC_REV_5762 && mss) {
7660 		u32 base = (u32) mapping & 0xffffffff;
7661 
7662 		return ((base + len + (mss & 0x3fff)) < base);
7663 	}
7664 	return 0;
7665 }
7666 
7667 /* Test for DMA addresses > 40-bit */
7668 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
7669 					  int len)
7670 {
7671 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
7672 	if (tg3_flag(tp, 40BIT_DMA_BUG))
7673 		return ((u64) mapping + len) > DMA_BIT_MASK(40);
7674 	return 0;
7675 #else
7676 	return 0;
7677 #endif
7678 }
7679 
7680 static inline void tg3_tx_set_bd(struct tg3_tx_buffer_desc *txbd,
7681 				 dma_addr_t mapping, u32 len, u32 flags,
7682 				 u32 mss, u32 vlan)
7683 {
7684 	txbd->addr_hi = ((u64) mapping >> 32);
7685 	txbd->addr_lo = ((u64) mapping & 0xffffffff);
7686 	txbd->len_flags = (len << TXD_LEN_SHIFT) | (flags & 0x0000ffff);
7687 	txbd->vlan_tag = (mss << TXD_MSS_SHIFT) | (vlan << TXD_VLAN_TAG_SHIFT);
7688 }
7689 
7690 static bool tg3_tx_frag_set(struct tg3_napi *tnapi, u32 *entry, u32 *budget,
7691 			    dma_addr_t map, u32 len, u32 flags,
7692 			    u32 mss, u32 vlan)
7693 {
7694 	struct tg3 *tp = tnapi->tp;
7695 	bool hwbug = false;
7696 
7697 	if (tg3_flag(tp, SHORT_DMA_BUG) && len <= 8)
7698 		hwbug = true;
7699 
7700 	if (tg3_4g_overflow_test(map, len))
7701 		hwbug = true;
7702 
7703 	if (tg3_4g_tso_overflow_test(tp, map, len, mss))
7704 		hwbug = true;
7705 
7706 	if (tg3_40bit_overflow_test(tp, map, len))
7707 		hwbug = true;
7708 
7709 	if (tp->dma_limit) {
7710 		u32 prvidx = *entry;
7711 		u32 tmp_flag = flags & ~TXD_FLAG_END;
7712 		while (len > tp->dma_limit && *budget) {
7713 			u32 frag_len = tp->dma_limit;
7714 			len -= tp->dma_limit;
7715 
7716 			/* Avoid the 8byte DMA problem */
7717 			if (len <= 8) {
7718 				len += tp->dma_limit / 2;
7719 				frag_len = tp->dma_limit / 2;
7720 			}
7721 
7722 			tnapi->tx_buffers[*entry].fragmented = true;
7723 
7724 			tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7725 				      frag_len, tmp_flag, mss, vlan);
7726 			*budget -= 1;
7727 			prvidx = *entry;
7728 			*entry = NEXT_TX(*entry);
7729 
7730 			map += frag_len;
7731 		}
7732 
7733 		if (len) {
7734 			if (*budget) {
7735 				tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7736 					      len, flags, mss, vlan);
7737 				*budget -= 1;
7738 				*entry = NEXT_TX(*entry);
7739 			} else {
7740 				hwbug = true;
7741 				tnapi->tx_buffers[prvidx].fragmented = false;
7742 			}
7743 		}
7744 	} else {
7745 		tg3_tx_set_bd(&tnapi->tx_ring[*entry], map,
7746 			      len, flags, mss, vlan);
7747 		*entry = NEXT_TX(*entry);
7748 	}
7749 
7750 	return hwbug;
7751 }
7752 
7753 static void tg3_tx_skb_unmap(struct tg3_napi *tnapi, u32 entry, int last)
7754 {
7755 	int i;
7756 	struct sk_buff *skb;
7757 	struct tg3_tx_ring_info *txb = &tnapi->tx_buffers[entry];
7758 
7759 	skb = txb->skb;
7760 	txb->skb = NULL;
7761 
7762 	dma_unmap_single(&tnapi->tp->pdev->dev, dma_unmap_addr(txb, mapping),
7763 			 skb_headlen(skb), DMA_TO_DEVICE);
7764 
7765 	while (txb->fragmented) {
7766 		txb->fragmented = false;
7767 		entry = NEXT_TX(entry);
7768 		txb = &tnapi->tx_buffers[entry];
7769 	}
7770 
7771 	for (i = 0; i <= last; i++) {
7772 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
7773 
7774 		entry = NEXT_TX(entry);
7775 		txb = &tnapi->tx_buffers[entry];
7776 
7777 		dma_unmap_page(&tnapi->tp->pdev->dev,
7778 			       dma_unmap_addr(txb, mapping),
7779 			       skb_frag_size(frag), DMA_TO_DEVICE);
7780 
7781 		while (txb->fragmented) {
7782 			txb->fragmented = false;
7783 			entry = NEXT_TX(entry);
7784 			txb = &tnapi->tx_buffers[entry];
7785 		}
7786 	}
7787 }
7788 
7789 /* Workaround 4GB and 40-bit hardware DMA bugs. */
7790 static int tigon3_dma_hwbug_workaround(struct tg3_napi *tnapi,
7791 				       struct sk_buff **pskb,
7792 				       u32 *entry, u32 *budget,
7793 				       u32 base_flags, u32 mss, u32 vlan)
7794 {
7795 	struct tg3 *tp = tnapi->tp;
7796 	struct sk_buff *new_skb, *skb = *pskb;
7797 	dma_addr_t new_addr = 0;
7798 	int ret = 0;
7799 
7800 	if (tg3_asic_rev(tp) != ASIC_REV_5701)
7801 		new_skb = skb_copy(skb, GFP_ATOMIC);
7802 	else {
7803 		int more_headroom = 4 - ((unsigned long)skb->data & 3);
7804 
7805 		new_skb = skb_copy_expand(skb,
7806 					  skb_headroom(skb) + more_headroom,
7807 					  skb_tailroom(skb), GFP_ATOMIC);
7808 	}
7809 
7810 	if (!new_skb) {
7811 		ret = -1;
7812 	} else {
7813 		/* New SKB is guaranteed to be linear. */
7814 		new_addr = dma_map_single(&tp->pdev->dev, new_skb->data,
7815 					  new_skb->len, DMA_TO_DEVICE);
7816 		/* Make sure the mapping succeeded */
7817 		if (dma_mapping_error(&tp->pdev->dev, new_addr)) {
7818 			dev_kfree_skb_any(new_skb);
7819 			ret = -1;
7820 		} else {
7821 			u32 save_entry = *entry;
7822 
7823 			base_flags |= TXD_FLAG_END;
7824 
7825 			tnapi->tx_buffers[*entry].skb = new_skb;
7826 			dma_unmap_addr_set(&tnapi->tx_buffers[*entry],
7827 					   mapping, new_addr);
7828 
7829 			if (tg3_tx_frag_set(tnapi, entry, budget, new_addr,
7830 					    new_skb->len, base_flags,
7831 					    mss, vlan)) {
7832 				tg3_tx_skb_unmap(tnapi, save_entry, -1);
7833 				dev_kfree_skb_any(new_skb);
7834 				ret = -1;
7835 			}
7836 		}
7837 	}
7838 
7839 	dev_consume_skb_any(skb);
7840 	*pskb = new_skb;
7841 	return ret;
7842 }
7843 
7844 static bool tg3_tso_bug_gso_check(struct tg3_napi *tnapi, struct sk_buff *skb)
7845 {
7846 	/* Check if we will never have enough descriptors,
7847 	 * as gso_segs can be more than current ring size
7848 	 */
7849 	return skb_shinfo(skb)->gso_segs < tnapi->tx_pending / 3;
7850 }
7851 
7852 static netdev_tx_t tg3_start_xmit(struct sk_buff *, struct net_device *);
7853 
7854 /* Use GSO to workaround all TSO packets that meet HW bug conditions
7855  * indicated in tg3_tx_frag_set()
7856  */
7857 static int tg3_tso_bug(struct tg3 *tp, struct tg3_napi *tnapi,
7858 		       struct netdev_queue *txq, struct sk_buff *skb)
7859 {
7860 	u32 frag_cnt_est = skb_shinfo(skb)->gso_segs * 3;
7861 	struct sk_buff *segs, *seg, *next;
7862 
7863 	/* Estimate the number of fragments in the worst case */
7864 	if (unlikely(tg3_tx_avail(tnapi) <= frag_cnt_est)) {
7865 		netif_tx_stop_queue(txq);
7866 
7867 		/* netif_tx_stop_queue() must be done before checking
7868 		 * checking tx index in tg3_tx_avail() below, because in
7869 		 * tg3_tx(), we update tx index before checking for
7870 		 * netif_tx_queue_stopped().
7871 		 */
7872 		smp_mb();
7873 		if (tg3_tx_avail(tnapi) <= frag_cnt_est)
7874 			return NETDEV_TX_BUSY;
7875 
7876 		netif_tx_wake_queue(txq);
7877 	}
7878 
7879 	segs = skb_gso_segment(skb, tp->dev->features &
7880 				    ~(NETIF_F_TSO | NETIF_F_TSO6));
7881 	if (IS_ERR(segs) || !segs)
7882 		goto tg3_tso_bug_end;
7883 
7884 	skb_list_walk_safe(segs, seg, next) {
7885 		skb_mark_not_on_list(seg);
7886 		tg3_start_xmit(seg, tp->dev);
7887 	}
7888 
7889 tg3_tso_bug_end:
7890 	dev_consume_skb_any(skb);
7891 
7892 	return NETDEV_TX_OK;
7893 }
7894 
7895 /* hard_start_xmit for all devices */
7896 static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
7897 {
7898 	struct tg3 *tp = netdev_priv(dev);
7899 	u32 len, entry, base_flags, mss, vlan = 0;
7900 	u32 budget;
7901 	int i = -1, would_hit_hwbug;
7902 	dma_addr_t mapping;
7903 	struct tg3_napi *tnapi;
7904 	struct netdev_queue *txq;
7905 	unsigned int last;
7906 	struct iphdr *iph = NULL;
7907 	struct tcphdr *tcph = NULL;
7908 	__sum16 tcp_csum = 0, ip_csum = 0;
7909 	__be16 ip_tot_len = 0;
7910 
7911 	txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
7912 	tnapi = &tp->napi[skb_get_queue_mapping(skb)];
7913 	if (tg3_flag(tp, ENABLE_TSS))
7914 		tnapi++;
7915 
7916 	budget = tg3_tx_avail(tnapi);
7917 
7918 	/* We are running in BH disabled context with netif_tx_lock
7919 	 * and TX reclaim runs via tp->napi.poll inside of a software
7920 	 * interrupt.  Furthermore, IRQ processing runs lockless so we have
7921 	 * no IRQ context deadlocks to worry about either.  Rejoice!
7922 	 */
7923 	if (unlikely(budget <= (skb_shinfo(skb)->nr_frags + 1))) {
7924 		if (!netif_tx_queue_stopped(txq)) {
7925 			netif_tx_stop_queue(txq);
7926 
7927 			/* This is a hard error, log it. */
7928 			netdev_err(dev,
7929 				   "BUG! Tx Ring full when queue awake!\n");
7930 		}
7931 		return NETDEV_TX_BUSY;
7932 	}
7933 
7934 	entry = tnapi->tx_prod;
7935 	base_flags = 0;
7936 
7937 	mss = skb_shinfo(skb)->gso_size;
7938 	if (mss) {
7939 		u32 tcp_opt_len, hdr_len;
7940 
7941 		if (skb_cow_head(skb, 0))
7942 			goto drop;
7943 
7944 		iph = ip_hdr(skb);
7945 		tcp_opt_len = tcp_optlen(skb);
7946 
7947 		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb) - ETH_HLEN;
7948 
7949 		/* HW/FW can not correctly segment packets that have been
7950 		 * vlan encapsulated.
7951 		 */
7952 		if (skb->protocol == htons(ETH_P_8021Q) ||
7953 		    skb->protocol == htons(ETH_P_8021AD)) {
7954 			if (tg3_tso_bug_gso_check(tnapi, skb))
7955 				return tg3_tso_bug(tp, tnapi, txq, skb);
7956 			goto drop;
7957 		}
7958 
7959 		if (!skb_is_gso_v6(skb)) {
7960 			if (unlikely((ETH_HLEN + hdr_len) > 80) &&
7961 			    tg3_flag(tp, TSO_BUG)) {
7962 				if (tg3_tso_bug_gso_check(tnapi, skb))
7963 					return tg3_tso_bug(tp, tnapi, txq, skb);
7964 				goto drop;
7965 			}
7966 			ip_csum = iph->check;
7967 			ip_tot_len = iph->tot_len;
7968 			iph->check = 0;
7969 			iph->tot_len = htons(mss + hdr_len);
7970 		}
7971 
7972 		base_flags |= (TXD_FLAG_CPU_PRE_DMA |
7973 			       TXD_FLAG_CPU_POST_DMA);
7974 
7975 		tcph = tcp_hdr(skb);
7976 		tcp_csum = tcph->check;
7977 
7978 		if (tg3_flag(tp, HW_TSO_1) ||
7979 		    tg3_flag(tp, HW_TSO_2) ||
7980 		    tg3_flag(tp, HW_TSO_3)) {
7981 			tcph->check = 0;
7982 			base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
7983 		} else {
7984 			tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
7985 							 0, IPPROTO_TCP, 0);
7986 		}
7987 
7988 		if (tg3_flag(tp, HW_TSO_3)) {
7989 			mss |= (hdr_len & 0xc) << 12;
7990 			if (hdr_len & 0x10)
7991 				base_flags |= 0x00000010;
7992 			base_flags |= (hdr_len & 0x3e0) << 5;
7993 		} else if (tg3_flag(tp, HW_TSO_2))
7994 			mss |= hdr_len << 9;
7995 		else if (tg3_flag(tp, HW_TSO_1) ||
7996 			 tg3_asic_rev(tp) == ASIC_REV_5705) {
7997 			if (tcp_opt_len || iph->ihl > 5) {
7998 				int tsflags;
7999 
8000 				tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
8001 				mss |= (tsflags << 11);
8002 			}
8003 		} else {
8004 			if (tcp_opt_len || iph->ihl > 5) {
8005 				int tsflags;
8006 
8007 				tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
8008 				base_flags |= tsflags << 12;
8009 			}
8010 		}
8011 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
8012 		/* HW/FW can not correctly checksum packets that have been
8013 		 * vlan encapsulated.
8014 		 */
8015 		if (skb->protocol == htons(ETH_P_8021Q) ||
8016 		    skb->protocol == htons(ETH_P_8021AD)) {
8017 			if (skb_checksum_help(skb))
8018 				goto drop;
8019 		} else  {
8020 			base_flags |= TXD_FLAG_TCPUDP_CSUM;
8021 		}
8022 	}
8023 
8024 	if (tg3_flag(tp, USE_JUMBO_BDFLAG) &&
8025 	    !mss && skb->len > VLAN_ETH_FRAME_LEN)
8026 		base_flags |= TXD_FLAG_JMB_PKT;
8027 
8028 	if (skb_vlan_tag_present(skb)) {
8029 		base_flags |= TXD_FLAG_VLAN;
8030 		vlan = skb_vlan_tag_get(skb);
8031 	}
8032 
8033 	if ((unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) &&
8034 	    tg3_flag(tp, TX_TSTAMP_EN)) {
8035 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
8036 		base_flags |= TXD_FLAG_HWTSTAMP;
8037 	}
8038 
8039 	len = skb_headlen(skb);
8040 
8041 	mapping = dma_map_single(&tp->pdev->dev, skb->data, len,
8042 				 DMA_TO_DEVICE);
8043 	if (dma_mapping_error(&tp->pdev->dev, mapping))
8044 		goto drop;
8045 
8046 
8047 	tnapi->tx_buffers[entry].skb = skb;
8048 	dma_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, mapping);
8049 
8050 	would_hit_hwbug = 0;
8051 
8052 	if (tg3_flag(tp, 5701_DMA_BUG))
8053 		would_hit_hwbug = 1;
8054 
8055 	if (tg3_tx_frag_set(tnapi, &entry, &budget, mapping, len, base_flags |
8056 			  ((skb_shinfo(skb)->nr_frags == 0) ? TXD_FLAG_END : 0),
8057 			    mss, vlan)) {
8058 		would_hit_hwbug = 1;
8059 	} else if (skb_shinfo(skb)->nr_frags > 0) {
8060 		u32 tmp_mss = mss;
8061 
8062 		if (!tg3_flag(tp, HW_TSO_1) &&
8063 		    !tg3_flag(tp, HW_TSO_2) &&
8064 		    !tg3_flag(tp, HW_TSO_3))
8065 			tmp_mss = 0;
8066 
8067 		/* Now loop through additional data
8068 		 * fragments, and queue them.
8069 		 */
8070 		last = skb_shinfo(skb)->nr_frags - 1;
8071 		for (i = 0; i <= last; i++) {
8072 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
8073 
8074 			len = skb_frag_size(frag);
8075 			mapping = skb_frag_dma_map(&tp->pdev->dev, frag, 0,
8076 						   len, DMA_TO_DEVICE);
8077 
8078 			tnapi->tx_buffers[entry].skb = NULL;
8079 			dma_unmap_addr_set(&tnapi->tx_buffers[entry], mapping,
8080 					   mapping);
8081 			if (dma_mapping_error(&tp->pdev->dev, mapping))
8082 				goto dma_error;
8083 
8084 			if (!budget ||
8085 			    tg3_tx_frag_set(tnapi, &entry, &budget, mapping,
8086 					    len, base_flags |
8087 					    ((i == last) ? TXD_FLAG_END : 0),
8088 					    tmp_mss, vlan)) {
8089 				would_hit_hwbug = 1;
8090 				break;
8091 			}
8092 		}
8093 	}
8094 
8095 	if (would_hit_hwbug) {
8096 		tg3_tx_skb_unmap(tnapi, tnapi->tx_prod, i);
8097 
8098 		if (mss && tg3_tso_bug_gso_check(tnapi, skb)) {
8099 			/* If it's a TSO packet, do GSO instead of
8100 			 * allocating and copying to a large linear SKB
8101 			 */
8102 			if (ip_tot_len) {
8103 				iph->check = ip_csum;
8104 				iph->tot_len = ip_tot_len;
8105 			}
8106 			tcph->check = tcp_csum;
8107 			return tg3_tso_bug(tp, tnapi, txq, skb);
8108 		}
8109 
8110 		/* If the workaround fails due to memory/mapping
8111 		 * failure, silently drop this packet.
8112 		 */
8113 		entry = tnapi->tx_prod;
8114 		budget = tg3_tx_avail(tnapi);
8115 		if (tigon3_dma_hwbug_workaround(tnapi, &skb, &entry, &budget,
8116 						base_flags, mss, vlan))
8117 			goto drop_nofree;
8118 	}
8119 
8120 	skb_tx_timestamp(skb);
8121 	netdev_tx_sent_queue(txq, skb->len);
8122 
8123 	/* Sync BD data before updating mailbox */
8124 	wmb();
8125 
8126 	tnapi->tx_prod = entry;
8127 	if (unlikely(tg3_tx_avail(tnapi) <= (MAX_SKB_FRAGS + 1))) {
8128 		netif_tx_stop_queue(txq);
8129 
8130 		/* netif_tx_stop_queue() must be done before checking
8131 		 * checking tx index in tg3_tx_avail() below, because in
8132 		 * tg3_tx(), we update tx index before checking for
8133 		 * netif_tx_queue_stopped().
8134 		 */
8135 		smp_mb();
8136 		if (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi))
8137 			netif_tx_wake_queue(txq);
8138 	}
8139 
8140 	if (!netdev_xmit_more() || netif_xmit_stopped(txq)) {
8141 		/* Packets are ready, update Tx producer idx on card. */
8142 		tw32_tx_mbox(tnapi->prodmbox, entry);
8143 	}
8144 
8145 	return NETDEV_TX_OK;
8146 
8147 dma_error:
8148 	tg3_tx_skb_unmap(tnapi, tnapi->tx_prod, --i);
8149 	tnapi->tx_buffers[tnapi->tx_prod].skb = NULL;
8150 drop:
8151 	dev_kfree_skb_any(skb);
8152 drop_nofree:
8153 	tp->tx_dropped++;
8154 	return NETDEV_TX_OK;
8155 }
8156 
8157 static void tg3_mac_loopback(struct tg3 *tp, bool enable)
8158 {
8159 	if (enable) {
8160 		tp->mac_mode &= ~(MAC_MODE_HALF_DUPLEX |
8161 				  MAC_MODE_PORT_MODE_MASK);
8162 
8163 		tp->mac_mode |= MAC_MODE_PORT_INT_LPBACK;
8164 
8165 		if (!tg3_flag(tp, 5705_PLUS))
8166 			tp->mac_mode |= MAC_MODE_LINK_POLARITY;
8167 
8168 		if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
8169 			tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
8170 		else
8171 			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
8172 	} else {
8173 		tp->mac_mode &= ~MAC_MODE_PORT_INT_LPBACK;
8174 
8175 		if (tg3_flag(tp, 5705_PLUS) ||
8176 		    (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) ||
8177 		    tg3_asic_rev(tp) == ASIC_REV_5700)
8178 			tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
8179 	}
8180 
8181 	tw32(MAC_MODE, tp->mac_mode);
8182 	udelay(40);
8183 }
8184 
8185 static int tg3_phy_lpbk_set(struct tg3 *tp, u32 speed, bool extlpbk)
8186 {
8187 	u32 val, bmcr, mac_mode, ptest = 0;
8188 
8189 	tg3_phy_toggle_apd(tp, false);
8190 	tg3_phy_toggle_automdix(tp, false);
8191 
8192 	if (extlpbk && tg3_phy_set_extloopbk(tp))
8193 		return -EIO;
8194 
8195 	bmcr = BMCR_FULLDPLX;
8196 	switch (speed) {
8197 	case SPEED_10:
8198 		break;
8199 	case SPEED_100:
8200 		bmcr |= BMCR_SPEED100;
8201 		break;
8202 	case SPEED_1000:
8203 	default:
8204 		if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
8205 			speed = SPEED_100;
8206 			bmcr |= BMCR_SPEED100;
8207 		} else {
8208 			speed = SPEED_1000;
8209 			bmcr |= BMCR_SPEED1000;
8210 		}
8211 	}
8212 
8213 	if (extlpbk) {
8214 		if (!(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
8215 			tg3_readphy(tp, MII_CTRL1000, &val);
8216 			val |= CTL1000_AS_MASTER |
8217 			       CTL1000_ENABLE_MASTER;
8218 			tg3_writephy(tp, MII_CTRL1000, val);
8219 		} else {
8220 			ptest = MII_TG3_FET_PTEST_TRIM_SEL |
8221 				MII_TG3_FET_PTEST_TRIM_2;
8222 			tg3_writephy(tp, MII_TG3_FET_PTEST, ptest);
8223 		}
8224 	} else
8225 		bmcr |= BMCR_LOOPBACK;
8226 
8227 	tg3_writephy(tp, MII_BMCR, bmcr);
8228 
8229 	/* The write needs to be flushed for the FETs */
8230 	if (tp->phy_flags & TG3_PHYFLG_IS_FET)
8231 		tg3_readphy(tp, MII_BMCR, &bmcr);
8232 
8233 	udelay(40);
8234 
8235 	if ((tp->phy_flags & TG3_PHYFLG_IS_FET) &&
8236 	    tg3_asic_rev(tp) == ASIC_REV_5785) {
8237 		tg3_writephy(tp, MII_TG3_FET_PTEST, ptest |
8238 			     MII_TG3_FET_PTEST_FRC_TX_LINK |
8239 			     MII_TG3_FET_PTEST_FRC_TX_LOCK);
8240 
8241 		/* The write needs to be flushed for the AC131 */
8242 		tg3_readphy(tp, MII_TG3_FET_PTEST, &val);
8243 	}
8244 
8245 	/* Reset to prevent losing 1st rx packet intermittently */
8246 	if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
8247 	    tg3_flag(tp, 5780_CLASS)) {
8248 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
8249 		udelay(10);
8250 		tw32_f(MAC_RX_MODE, tp->rx_mode);
8251 	}
8252 
8253 	mac_mode = tp->mac_mode &
8254 		   ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
8255 	if (speed == SPEED_1000)
8256 		mac_mode |= MAC_MODE_PORT_MODE_GMII;
8257 	else
8258 		mac_mode |= MAC_MODE_PORT_MODE_MII;
8259 
8260 	if (tg3_asic_rev(tp) == ASIC_REV_5700) {
8261 		u32 masked_phy_id = tp->phy_id & TG3_PHY_ID_MASK;
8262 
8263 		if (masked_phy_id == TG3_PHY_ID_BCM5401)
8264 			mac_mode &= ~MAC_MODE_LINK_POLARITY;
8265 		else if (masked_phy_id == TG3_PHY_ID_BCM5411)
8266 			mac_mode |= MAC_MODE_LINK_POLARITY;
8267 
8268 		tg3_writephy(tp, MII_TG3_EXT_CTRL,
8269 			     MII_TG3_EXT_CTRL_LNK3_LED_MODE);
8270 	}
8271 
8272 	tw32(MAC_MODE, mac_mode);
8273 	udelay(40);
8274 
8275 	return 0;
8276 }
8277 
8278 static void tg3_set_loopback(struct net_device *dev, netdev_features_t features)
8279 {
8280 	struct tg3 *tp = netdev_priv(dev);
8281 
8282 	if (features & NETIF_F_LOOPBACK) {
8283 		if (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK)
8284 			return;
8285 
8286 		spin_lock_bh(&tp->lock);
8287 		tg3_mac_loopback(tp, true);
8288 		netif_carrier_on(tp->dev);
8289 		spin_unlock_bh(&tp->lock);
8290 		netdev_info(dev, "Internal MAC loopback mode enabled.\n");
8291 	} else {
8292 		if (!(tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
8293 			return;
8294 
8295 		spin_lock_bh(&tp->lock);
8296 		tg3_mac_loopback(tp, false);
8297 		/* Force link status check */
8298 		tg3_setup_phy(tp, true);
8299 		spin_unlock_bh(&tp->lock);
8300 		netdev_info(dev, "Internal MAC loopback mode disabled.\n");
8301 	}
8302 }
8303 
8304 static netdev_features_t tg3_fix_features(struct net_device *dev,
8305 	netdev_features_t features)
8306 {
8307 	struct tg3 *tp = netdev_priv(dev);
8308 
8309 	if (dev->mtu > ETH_DATA_LEN && tg3_flag(tp, 5780_CLASS))
8310 		features &= ~NETIF_F_ALL_TSO;
8311 
8312 	return features;
8313 }
8314 
8315 static int tg3_set_features(struct net_device *dev, netdev_features_t features)
8316 {
8317 	netdev_features_t changed = dev->features ^ features;
8318 
8319 	if ((changed & NETIF_F_LOOPBACK) && netif_running(dev))
8320 		tg3_set_loopback(dev, features);
8321 
8322 	return 0;
8323 }
8324 
8325 static void tg3_rx_prodring_free(struct tg3 *tp,
8326 				 struct tg3_rx_prodring_set *tpr)
8327 {
8328 	int i;
8329 
8330 	if (tpr != &tp->napi[0].prodring) {
8331 		for (i = tpr->rx_std_cons_idx; i != tpr->rx_std_prod_idx;
8332 		     i = (i + 1) & tp->rx_std_ring_mask)
8333 			tg3_rx_data_free(tp, &tpr->rx_std_buffers[i],
8334 					tp->rx_pkt_map_sz);
8335 
8336 		if (tg3_flag(tp, JUMBO_CAPABLE)) {
8337 			for (i = tpr->rx_jmb_cons_idx;
8338 			     i != tpr->rx_jmb_prod_idx;
8339 			     i = (i + 1) & tp->rx_jmb_ring_mask) {
8340 				tg3_rx_data_free(tp, &tpr->rx_jmb_buffers[i],
8341 						TG3_RX_JMB_MAP_SZ);
8342 			}
8343 		}
8344 
8345 		return;
8346 	}
8347 
8348 	for (i = 0; i <= tp->rx_std_ring_mask; i++)
8349 		tg3_rx_data_free(tp, &tpr->rx_std_buffers[i],
8350 				tp->rx_pkt_map_sz);
8351 
8352 	if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS)) {
8353 		for (i = 0; i <= tp->rx_jmb_ring_mask; i++)
8354 			tg3_rx_data_free(tp, &tpr->rx_jmb_buffers[i],
8355 					TG3_RX_JMB_MAP_SZ);
8356 	}
8357 }
8358 
8359 /* Initialize rx rings for packet processing.
8360  *
8361  * The chip has been shut down and the driver detached from
8362  * the networking, so no interrupts or new tx packets will
8363  * end up in the driver.  tp->{tx,}lock are held and thus
8364  * we may not sleep.
8365  */
8366 static int tg3_rx_prodring_alloc(struct tg3 *tp,
8367 				 struct tg3_rx_prodring_set *tpr)
8368 {
8369 	u32 i, rx_pkt_dma_sz;
8370 
8371 	tpr->rx_std_cons_idx = 0;
8372 	tpr->rx_std_prod_idx = 0;
8373 	tpr->rx_jmb_cons_idx = 0;
8374 	tpr->rx_jmb_prod_idx = 0;
8375 
8376 	if (tpr != &tp->napi[0].prodring) {
8377 		memset(&tpr->rx_std_buffers[0], 0,
8378 		       TG3_RX_STD_BUFF_RING_SIZE(tp));
8379 		if (tpr->rx_jmb_buffers)
8380 			memset(&tpr->rx_jmb_buffers[0], 0,
8381 			       TG3_RX_JMB_BUFF_RING_SIZE(tp));
8382 		goto done;
8383 	}
8384 
8385 	/* Zero out all descriptors. */
8386 	memset(tpr->rx_std, 0, TG3_RX_STD_RING_BYTES(tp));
8387 
8388 	rx_pkt_dma_sz = TG3_RX_STD_DMA_SZ;
8389 	if (tg3_flag(tp, 5780_CLASS) &&
8390 	    tp->dev->mtu > ETH_DATA_LEN)
8391 		rx_pkt_dma_sz = TG3_RX_JMB_DMA_SZ;
8392 	tp->rx_pkt_map_sz = TG3_RX_DMA_TO_MAP_SZ(rx_pkt_dma_sz);
8393 
8394 	/* Initialize invariants of the rings, we only set this
8395 	 * stuff once.  This works because the card does not
8396 	 * write into the rx buffer posting rings.
8397 	 */
8398 	for (i = 0; i <= tp->rx_std_ring_mask; i++) {
8399 		struct tg3_rx_buffer_desc *rxd;
8400 
8401 		rxd = &tpr->rx_std[i];
8402 		rxd->idx_len = rx_pkt_dma_sz << RXD_LEN_SHIFT;
8403 		rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
8404 		rxd->opaque = (RXD_OPAQUE_RING_STD |
8405 			       (i << RXD_OPAQUE_INDEX_SHIFT));
8406 	}
8407 
8408 	/* Now allocate fresh SKBs for each rx ring. */
8409 	for (i = 0; i < tp->rx_pending; i++) {
8410 		unsigned int frag_size;
8411 
8412 		if (tg3_alloc_rx_data(tp, tpr, RXD_OPAQUE_RING_STD, i,
8413 				      &frag_size) < 0) {
8414 			netdev_warn(tp->dev,
8415 				    "Using a smaller RX standard ring. Only "
8416 				    "%d out of %d buffers were allocated "
8417 				    "successfully\n", i, tp->rx_pending);
8418 			if (i == 0)
8419 				goto initfail;
8420 			tp->rx_pending = i;
8421 			break;
8422 		}
8423 	}
8424 
8425 	if (!tg3_flag(tp, JUMBO_CAPABLE) || tg3_flag(tp, 5780_CLASS))
8426 		goto done;
8427 
8428 	memset(tpr->rx_jmb, 0, TG3_RX_JMB_RING_BYTES(tp));
8429 
8430 	if (!tg3_flag(tp, JUMBO_RING_ENABLE))
8431 		goto done;
8432 
8433 	for (i = 0; i <= tp->rx_jmb_ring_mask; i++) {
8434 		struct tg3_rx_buffer_desc *rxd;
8435 
8436 		rxd = &tpr->rx_jmb[i].std;
8437 		rxd->idx_len = TG3_RX_JMB_DMA_SZ << RXD_LEN_SHIFT;
8438 		rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
8439 				  RXD_FLAG_JUMBO;
8440 		rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
8441 		       (i << RXD_OPAQUE_INDEX_SHIFT));
8442 	}
8443 
8444 	for (i = 0; i < tp->rx_jumbo_pending; i++) {
8445 		unsigned int frag_size;
8446 
8447 		if (tg3_alloc_rx_data(tp, tpr, RXD_OPAQUE_RING_JUMBO, i,
8448 				      &frag_size) < 0) {
8449 			netdev_warn(tp->dev,
8450 				    "Using a smaller RX jumbo ring. Only %d "
8451 				    "out of %d buffers were allocated "
8452 				    "successfully\n", i, tp->rx_jumbo_pending);
8453 			if (i == 0)
8454 				goto initfail;
8455 			tp->rx_jumbo_pending = i;
8456 			break;
8457 		}
8458 	}
8459 
8460 done:
8461 	return 0;
8462 
8463 initfail:
8464 	tg3_rx_prodring_free(tp, tpr);
8465 	return -ENOMEM;
8466 }
8467 
8468 static void tg3_rx_prodring_fini(struct tg3 *tp,
8469 				 struct tg3_rx_prodring_set *tpr)
8470 {
8471 	kfree(tpr->rx_std_buffers);
8472 	tpr->rx_std_buffers = NULL;
8473 	kfree(tpr->rx_jmb_buffers);
8474 	tpr->rx_jmb_buffers = NULL;
8475 	if (tpr->rx_std) {
8476 		dma_free_coherent(&tp->pdev->dev, TG3_RX_STD_RING_BYTES(tp),
8477 				  tpr->rx_std, tpr->rx_std_mapping);
8478 		tpr->rx_std = NULL;
8479 	}
8480 	if (tpr->rx_jmb) {
8481 		dma_free_coherent(&tp->pdev->dev, TG3_RX_JMB_RING_BYTES(tp),
8482 				  tpr->rx_jmb, tpr->rx_jmb_mapping);
8483 		tpr->rx_jmb = NULL;
8484 	}
8485 }
8486 
8487 static int tg3_rx_prodring_init(struct tg3 *tp,
8488 				struct tg3_rx_prodring_set *tpr)
8489 {
8490 	tpr->rx_std_buffers = kzalloc(TG3_RX_STD_BUFF_RING_SIZE(tp),
8491 				      GFP_KERNEL);
8492 	if (!tpr->rx_std_buffers)
8493 		return -ENOMEM;
8494 
8495 	tpr->rx_std = dma_alloc_coherent(&tp->pdev->dev,
8496 					 TG3_RX_STD_RING_BYTES(tp),
8497 					 &tpr->rx_std_mapping,
8498 					 GFP_KERNEL);
8499 	if (!tpr->rx_std)
8500 		goto err_out;
8501 
8502 	if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS)) {
8503 		tpr->rx_jmb_buffers = kzalloc(TG3_RX_JMB_BUFF_RING_SIZE(tp),
8504 					      GFP_KERNEL);
8505 		if (!tpr->rx_jmb_buffers)
8506 			goto err_out;
8507 
8508 		tpr->rx_jmb = dma_alloc_coherent(&tp->pdev->dev,
8509 						 TG3_RX_JMB_RING_BYTES(tp),
8510 						 &tpr->rx_jmb_mapping,
8511 						 GFP_KERNEL);
8512 		if (!tpr->rx_jmb)
8513 			goto err_out;
8514 	}
8515 
8516 	return 0;
8517 
8518 err_out:
8519 	tg3_rx_prodring_fini(tp, tpr);
8520 	return -ENOMEM;
8521 }
8522 
8523 /* Free up pending packets in all rx/tx rings.
8524  *
8525  * The chip has been shut down and the driver detached from
8526  * the networking, so no interrupts or new tx packets will
8527  * end up in the driver.  tp->{tx,}lock is not held and we are not
8528  * in an interrupt context and thus may sleep.
8529  */
8530 static void tg3_free_rings(struct tg3 *tp)
8531 {
8532 	int i, j;
8533 
8534 	for (j = 0; j < tp->irq_cnt; j++) {
8535 		struct tg3_napi *tnapi = &tp->napi[j];
8536 
8537 		tg3_rx_prodring_free(tp, &tnapi->prodring);
8538 
8539 		if (!tnapi->tx_buffers)
8540 			continue;
8541 
8542 		for (i = 0; i < TG3_TX_RING_SIZE; i++) {
8543 			struct sk_buff *skb = tnapi->tx_buffers[i].skb;
8544 
8545 			if (!skb)
8546 				continue;
8547 
8548 			tg3_tx_skb_unmap(tnapi, i,
8549 					 skb_shinfo(skb)->nr_frags - 1);
8550 
8551 			dev_consume_skb_any(skb);
8552 		}
8553 		netdev_tx_reset_queue(netdev_get_tx_queue(tp->dev, j));
8554 	}
8555 }
8556 
8557 /* Initialize tx/rx rings for packet processing.
8558  *
8559  * The chip has been shut down and the driver detached from
8560  * the networking, so no interrupts or new tx packets will
8561  * end up in the driver.  tp->{tx,}lock are held and thus
8562  * we may not sleep.
8563  */
8564 static int tg3_init_rings(struct tg3 *tp)
8565 {
8566 	int i;
8567 
8568 	/* Free up all the SKBs. */
8569 	tg3_free_rings(tp);
8570 
8571 	for (i = 0; i < tp->irq_cnt; i++) {
8572 		struct tg3_napi *tnapi = &tp->napi[i];
8573 
8574 		tnapi->last_tag = 0;
8575 		tnapi->last_irq_tag = 0;
8576 		tnapi->hw_status->status = 0;
8577 		tnapi->hw_status->status_tag = 0;
8578 		memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
8579 
8580 		tnapi->tx_prod = 0;
8581 		tnapi->tx_cons = 0;
8582 		if (tnapi->tx_ring)
8583 			memset(tnapi->tx_ring, 0, TG3_TX_RING_BYTES);
8584 
8585 		tnapi->rx_rcb_ptr = 0;
8586 		if (tnapi->rx_rcb)
8587 			memset(tnapi->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
8588 
8589 		if (tnapi->prodring.rx_std &&
8590 		    tg3_rx_prodring_alloc(tp, &tnapi->prodring)) {
8591 			tg3_free_rings(tp);
8592 			return -ENOMEM;
8593 		}
8594 	}
8595 
8596 	return 0;
8597 }
8598 
8599 static void tg3_mem_tx_release(struct tg3 *tp)
8600 {
8601 	int i;
8602 
8603 	for (i = 0; i < tp->irq_max; i++) {
8604 		struct tg3_napi *tnapi = &tp->napi[i];
8605 
8606 		if (tnapi->tx_ring) {
8607 			dma_free_coherent(&tp->pdev->dev, TG3_TX_RING_BYTES,
8608 				tnapi->tx_ring, tnapi->tx_desc_mapping);
8609 			tnapi->tx_ring = NULL;
8610 		}
8611 
8612 		kfree(tnapi->tx_buffers);
8613 		tnapi->tx_buffers = NULL;
8614 	}
8615 }
8616 
8617 static int tg3_mem_tx_acquire(struct tg3 *tp)
8618 {
8619 	int i;
8620 	struct tg3_napi *tnapi = &tp->napi[0];
8621 
8622 	/* If multivector TSS is enabled, vector 0 does not handle
8623 	 * tx interrupts.  Don't allocate any resources for it.
8624 	 */
8625 	if (tg3_flag(tp, ENABLE_TSS))
8626 		tnapi++;
8627 
8628 	for (i = 0; i < tp->txq_cnt; i++, tnapi++) {
8629 		tnapi->tx_buffers = kcalloc(TG3_TX_RING_SIZE,
8630 					    sizeof(struct tg3_tx_ring_info),
8631 					    GFP_KERNEL);
8632 		if (!tnapi->tx_buffers)
8633 			goto err_out;
8634 
8635 		tnapi->tx_ring = dma_alloc_coherent(&tp->pdev->dev,
8636 						    TG3_TX_RING_BYTES,
8637 						    &tnapi->tx_desc_mapping,
8638 						    GFP_KERNEL);
8639 		if (!tnapi->tx_ring)
8640 			goto err_out;
8641 	}
8642 
8643 	return 0;
8644 
8645 err_out:
8646 	tg3_mem_tx_release(tp);
8647 	return -ENOMEM;
8648 }
8649 
8650 static void tg3_mem_rx_release(struct tg3 *tp)
8651 {
8652 	int i;
8653 
8654 	for (i = 0; i < tp->irq_max; i++) {
8655 		struct tg3_napi *tnapi = &tp->napi[i];
8656 
8657 		tg3_rx_prodring_fini(tp, &tnapi->prodring);
8658 
8659 		if (!tnapi->rx_rcb)
8660 			continue;
8661 
8662 		dma_free_coherent(&tp->pdev->dev,
8663 				  TG3_RX_RCB_RING_BYTES(tp),
8664 				  tnapi->rx_rcb,
8665 				  tnapi->rx_rcb_mapping);
8666 		tnapi->rx_rcb = NULL;
8667 	}
8668 }
8669 
8670 static int tg3_mem_rx_acquire(struct tg3 *tp)
8671 {
8672 	unsigned int i, limit;
8673 
8674 	limit = tp->rxq_cnt;
8675 
8676 	/* If RSS is enabled, we need a (dummy) producer ring
8677 	 * set on vector zero.  This is the true hw prodring.
8678 	 */
8679 	if (tg3_flag(tp, ENABLE_RSS))
8680 		limit++;
8681 
8682 	for (i = 0; i < limit; i++) {
8683 		struct tg3_napi *tnapi = &tp->napi[i];
8684 
8685 		if (tg3_rx_prodring_init(tp, &tnapi->prodring))
8686 			goto err_out;
8687 
8688 		/* If multivector RSS is enabled, vector 0
8689 		 * does not handle rx or tx interrupts.
8690 		 * Don't allocate any resources for it.
8691 		 */
8692 		if (!i && tg3_flag(tp, ENABLE_RSS))
8693 			continue;
8694 
8695 		tnapi->rx_rcb = dma_alloc_coherent(&tp->pdev->dev,
8696 						   TG3_RX_RCB_RING_BYTES(tp),
8697 						   &tnapi->rx_rcb_mapping,
8698 						   GFP_KERNEL);
8699 		if (!tnapi->rx_rcb)
8700 			goto err_out;
8701 	}
8702 
8703 	return 0;
8704 
8705 err_out:
8706 	tg3_mem_rx_release(tp);
8707 	return -ENOMEM;
8708 }
8709 
8710 /*
8711  * Must not be invoked with interrupt sources disabled and
8712  * the hardware shutdown down.
8713  */
8714 static void tg3_free_consistent(struct tg3 *tp)
8715 {
8716 	int i;
8717 
8718 	for (i = 0; i < tp->irq_cnt; i++) {
8719 		struct tg3_napi *tnapi = &tp->napi[i];
8720 
8721 		if (tnapi->hw_status) {
8722 			dma_free_coherent(&tp->pdev->dev, TG3_HW_STATUS_SIZE,
8723 					  tnapi->hw_status,
8724 					  tnapi->status_mapping);
8725 			tnapi->hw_status = NULL;
8726 		}
8727 	}
8728 
8729 	tg3_mem_rx_release(tp);
8730 	tg3_mem_tx_release(tp);
8731 
8732 	/* tp->hw_stats can be referenced safely:
8733 	 *     1. under rtnl_lock
8734 	 *     2. or under tp->lock if TG3_FLAG_INIT_COMPLETE is set.
8735 	 */
8736 	if (tp->hw_stats) {
8737 		dma_free_coherent(&tp->pdev->dev, sizeof(struct tg3_hw_stats),
8738 				  tp->hw_stats, tp->stats_mapping);
8739 		tp->hw_stats = NULL;
8740 	}
8741 }
8742 
8743 /*
8744  * Must not be invoked with interrupt sources disabled and
8745  * the hardware shutdown down.  Can sleep.
8746  */
8747 static int tg3_alloc_consistent(struct tg3 *tp)
8748 {
8749 	int i;
8750 
8751 	tp->hw_stats = dma_alloc_coherent(&tp->pdev->dev,
8752 					  sizeof(struct tg3_hw_stats),
8753 					  &tp->stats_mapping, GFP_KERNEL);
8754 	if (!tp->hw_stats)
8755 		goto err_out;
8756 
8757 	for (i = 0; i < tp->irq_cnt; i++) {
8758 		struct tg3_napi *tnapi = &tp->napi[i];
8759 		struct tg3_hw_status *sblk;
8760 
8761 		tnapi->hw_status = dma_alloc_coherent(&tp->pdev->dev,
8762 						      TG3_HW_STATUS_SIZE,
8763 						      &tnapi->status_mapping,
8764 						      GFP_KERNEL);
8765 		if (!tnapi->hw_status)
8766 			goto err_out;
8767 
8768 		sblk = tnapi->hw_status;
8769 
8770 		if (tg3_flag(tp, ENABLE_RSS)) {
8771 			u16 *prodptr = NULL;
8772 
8773 			/*
8774 			 * When RSS is enabled, the status block format changes
8775 			 * slightly.  The "rx_jumbo_consumer", "reserved",
8776 			 * and "rx_mini_consumer" members get mapped to the
8777 			 * other three rx return ring producer indexes.
8778 			 */
8779 			switch (i) {
8780 			case 1:
8781 				prodptr = &sblk->idx[0].rx_producer;
8782 				break;
8783 			case 2:
8784 				prodptr = &sblk->rx_jumbo_consumer;
8785 				break;
8786 			case 3:
8787 				prodptr = &sblk->reserved;
8788 				break;
8789 			case 4:
8790 				prodptr = &sblk->rx_mini_consumer;
8791 				break;
8792 			}
8793 			tnapi->rx_rcb_prod_idx = prodptr;
8794 		} else {
8795 			tnapi->rx_rcb_prod_idx = &sblk->idx[0].rx_producer;
8796 		}
8797 	}
8798 
8799 	if (tg3_mem_tx_acquire(tp) || tg3_mem_rx_acquire(tp))
8800 		goto err_out;
8801 
8802 	return 0;
8803 
8804 err_out:
8805 	tg3_free_consistent(tp);
8806 	return -ENOMEM;
8807 }
8808 
8809 #define MAX_WAIT_CNT 1000
8810 
8811 /* To stop a block, clear the enable bit and poll till it
8812  * clears.  tp->lock is held.
8813  */
8814 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, bool silent)
8815 {
8816 	unsigned int i;
8817 	u32 val;
8818 
8819 	if (tg3_flag(tp, 5705_PLUS)) {
8820 		switch (ofs) {
8821 		case RCVLSC_MODE:
8822 		case DMAC_MODE:
8823 		case MBFREE_MODE:
8824 		case BUFMGR_MODE:
8825 		case MEMARB_MODE:
8826 			/* We can't enable/disable these bits of the
8827 			 * 5705/5750, just say success.
8828 			 */
8829 			return 0;
8830 
8831 		default:
8832 			break;
8833 		}
8834 	}
8835 
8836 	val = tr32(ofs);
8837 	val &= ~enable_bit;
8838 	tw32_f(ofs, val);
8839 
8840 	for (i = 0; i < MAX_WAIT_CNT; i++) {
8841 		if (pci_channel_offline(tp->pdev)) {
8842 			dev_err(&tp->pdev->dev,
8843 				"tg3_stop_block device offline, "
8844 				"ofs=%lx enable_bit=%x\n",
8845 				ofs, enable_bit);
8846 			return -ENODEV;
8847 		}
8848 
8849 		udelay(100);
8850 		val = tr32(ofs);
8851 		if ((val & enable_bit) == 0)
8852 			break;
8853 	}
8854 
8855 	if (i == MAX_WAIT_CNT && !silent) {
8856 		dev_err(&tp->pdev->dev,
8857 			"tg3_stop_block timed out, ofs=%lx enable_bit=%x\n",
8858 			ofs, enable_bit);
8859 		return -ENODEV;
8860 	}
8861 
8862 	return 0;
8863 }
8864 
8865 /* tp->lock is held. */
8866 static int tg3_abort_hw(struct tg3 *tp, bool silent)
8867 {
8868 	int i, err;
8869 
8870 	tg3_disable_ints(tp);
8871 
8872 	if (pci_channel_offline(tp->pdev)) {
8873 		tp->rx_mode &= ~(RX_MODE_ENABLE | TX_MODE_ENABLE);
8874 		tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
8875 		err = -ENODEV;
8876 		goto err_no_dev;
8877 	}
8878 
8879 	tp->rx_mode &= ~RX_MODE_ENABLE;
8880 	tw32_f(MAC_RX_MODE, tp->rx_mode);
8881 	udelay(10);
8882 
8883 	err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
8884 	err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
8885 	err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
8886 	err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
8887 	err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
8888 	err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
8889 
8890 	err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
8891 	err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
8892 	err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
8893 	err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
8894 	err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
8895 	err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
8896 	err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
8897 
8898 	tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
8899 	tw32_f(MAC_MODE, tp->mac_mode);
8900 	udelay(40);
8901 
8902 	tp->tx_mode &= ~TX_MODE_ENABLE;
8903 	tw32_f(MAC_TX_MODE, tp->tx_mode);
8904 
8905 	for (i = 0; i < MAX_WAIT_CNT; i++) {
8906 		udelay(100);
8907 		if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
8908 			break;
8909 	}
8910 	if (i >= MAX_WAIT_CNT) {
8911 		dev_err(&tp->pdev->dev,
8912 			"%s timed out, TX_MODE_ENABLE will not clear "
8913 			"MAC_TX_MODE=%08x\n", __func__, tr32(MAC_TX_MODE));
8914 		err |= -ENODEV;
8915 	}
8916 
8917 	err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
8918 	err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
8919 	err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
8920 
8921 	tw32(FTQ_RESET, 0xffffffff);
8922 	tw32(FTQ_RESET, 0x00000000);
8923 
8924 	err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
8925 	err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
8926 
8927 err_no_dev:
8928 	for (i = 0; i < tp->irq_cnt; i++) {
8929 		struct tg3_napi *tnapi = &tp->napi[i];
8930 		if (tnapi->hw_status)
8931 			memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
8932 	}
8933 
8934 	return err;
8935 }
8936 
8937 /* Save PCI command register before chip reset */
8938 static void tg3_save_pci_state(struct tg3 *tp)
8939 {
8940 	pci_read_config_word(tp->pdev, PCI_COMMAND, &tp->pci_cmd);
8941 }
8942 
8943 /* Restore PCI state after chip reset */
8944 static void tg3_restore_pci_state(struct tg3 *tp)
8945 {
8946 	u32 val;
8947 
8948 	/* Re-enable indirect register accesses. */
8949 	pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
8950 			       tp->misc_host_ctrl);
8951 
8952 	/* Set MAX PCI retry to zero. */
8953 	val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
8954 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0 &&
8955 	    tg3_flag(tp, PCIX_MODE))
8956 		val |= PCISTATE_RETRY_SAME_DMA;
8957 	/* Allow reads and writes to the APE register and memory space. */
8958 	if (tg3_flag(tp, ENABLE_APE))
8959 		val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
8960 		       PCISTATE_ALLOW_APE_SHMEM_WR |
8961 		       PCISTATE_ALLOW_APE_PSPACE_WR;
8962 	pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
8963 
8964 	pci_write_config_word(tp->pdev, PCI_COMMAND, tp->pci_cmd);
8965 
8966 	if (!tg3_flag(tp, PCI_EXPRESS)) {
8967 		pci_write_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
8968 				      tp->pci_cacheline_sz);
8969 		pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
8970 				      tp->pci_lat_timer);
8971 	}
8972 
8973 	/* Make sure PCI-X relaxed ordering bit is clear. */
8974 	if (tg3_flag(tp, PCIX_MODE)) {
8975 		u16 pcix_cmd;
8976 
8977 		pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
8978 				     &pcix_cmd);
8979 		pcix_cmd &= ~PCI_X_CMD_ERO;
8980 		pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
8981 				      pcix_cmd);
8982 	}
8983 
8984 	if (tg3_flag(tp, 5780_CLASS)) {
8985 
8986 		/* Chip reset on 5780 will reset MSI enable bit,
8987 		 * so need to restore it.
8988 		 */
8989 		if (tg3_flag(tp, USING_MSI)) {
8990 			u16 ctrl;
8991 
8992 			pci_read_config_word(tp->pdev,
8993 					     tp->msi_cap + PCI_MSI_FLAGS,
8994 					     &ctrl);
8995 			pci_write_config_word(tp->pdev,
8996 					      tp->msi_cap + PCI_MSI_FLAGS,
8997 					      ctrl | PCI_MSI_FLAGS_ENABLE);
8998 			val = tr32(MSGINT_MODE);
8999 			tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
9000 		}
9001 	}
9002 }
9003 
9004 static void tg3_override_clk(struct tg3 *tp)
9005 {
9006 	u32 val;
9007 
9008 	switch (tg3_asic_rev(tp)) {
9009 	case ASIC_REV_5717:
9010 		val = tr32(TG3_CPMU_CLCK_ORIDE_ENABLE);
9011 		tw32(TG3_CPMU_CLCK_ORIDE_ENABLE, val |
9012 		     TG3_CPMU_MAC_ORIDE_ENABLE);
9013 		break;
9014 
9015 	case ASIC_REV_5719:
9016 	case ASIC_REV_5720:
9017 		tw32(TG3_CPMU_CLCK_ORIDE, CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
9018 		break;
9019 
9020 	default:
9021 		return;
9022 	}
9023 }
9024 
9025 static void tg3_restore_clk(struct tg3 *tp)
9026 {
9027 	u32 val;
9028 
9029 	switch (tg3_asic_rev(tp)) {
9030 	case ASIC_REV_5717:
9031 		val = tr32(TG3_CPMU_CLCK_ORIDE_ENABLE);
9032 		tw32(TG3_CPMU_CLCK_ORIDE_ENABLE,
9033 		     val & ~TG3_CPMU_MAC_ORIDE_ENABLE);
9034 		break;
9035 
9036 	case ASIC_REV_5719:
9037 	case ASIC_REV_5720:
9038 		val = tr32(TG3_CPMU_CLCK_ORIDE);
9039 		tw32(TG3_CPMU_CLCK_ORIDE, val & ~CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
9040 		break;
9041 
9042 	default:
9043 		return;
9044 	}
9045 }
9046 
9047 /* tp->lock is held. */
9048 static int tg3_chip_reset(struct tg3 *tp)
9049 	__releases(tp->lock)
9050 	__acquires(tp->lock)
9051 {
9052 	u32 val;
9053 	void (*write_op)(struct tg3 *, u32, u32);
9054 	int i, err;
9055 
9056 	if (!pci_device_is_present(tp->pdev))
9057 		return -ENODEV;
9058 
9059 	tg3_nvram_lock(tp);
9060 
9061 	tg3_ape_lock(tp, TG3_APE_LOCK_GRC);
9062 
9063 	/* No matching tg3_nvram_unlock() after this because
9064 	 * chip reset below will undo the nvram lock.
9065 	 */
9066 	tp->nvram_lock_cnt = 0;
9067 
9068 	/* GRC_MISC_CFG core clock reset will clear the memory
9069 	 * enable bit in PCI register 4 and the MSI enable bit
9070 	 * on some chips, so we save relevant registers here.
9071 	 */
9072 	tg3_save_pci_state(tp);
9073 
9074 	if (tg3_asic_rev(tp) == ASIC_REV_5752 ||
9075 	    tg3_flag(tp, 5755_PLUS))
9076 		tw32(GRC_FASTBOOT_PC, 0);
9077 
9078 	/*
9079 	 * We must avoid the readl() that normally takes place.
9080 	 * It locks machines, causes machine checks, and other
9081 	 * fun things.  So, temporarily disable the 5701
9082 	 * hardware workaround, while we do the reset.
9083 	 */
9084 	write_op = tp->write32;
9085 	if (write_op == tg3_write_flush_reg32)
9086 		tp->write32 = tg3_write32;
9087 
9088 	/* Prevent the irq handler from reading or writing PCI registers
9089 	 * during chip reset when the memory enable bit in the PCI command
9090 	 * register may be cleared.  The chip does not generate interrupt
9091 	 * at this time, but the irq handler may still be called due to irq
9092 	 * sharing or irqpoll.
9093 	 */
9094 	tg3_flag_set(tp, CHIP_RESETTING);
9095 	for (i = 0; i < tp->irq_cnt; i++) {
9096 		struct tg3_napi *tnapi = &tp->napi[i];
9097 		if (tnapi->hw_status) {
9098 			tnapi->hw_status->status = 0;
9099 			tnapi->hw_status->status_tag = 0;
9100 		}
9101 		tnapi->last_tag = 0;
9102 		tnapi->last_irq_tag = 0;
9103 	}
9104 	smp_mb();
9105 
9106 	tg3_full_unlock(tp);
9107 
9108 	for (i = 0; i < tp->irq_cnt; i++)
9109 		synchronize_irq(tp->napi[i].irq_vec);
9110 
9111 	tg3_full_lock(tp, 0);
9112 
9113 	if (tg3_asic_rev(tp) == ASIC_REV_57780) {
9114 		val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
9115 		tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
9116 	}
9117 
9118 	/* do the reset */
9119 	val = GRC_MISC_CFG_CORECLK_RESET;
9120 
9121 	if (tg3_flag(tp, PCI_EXPRESS)) {
9122 		/* Force PCIe 1.0a mode */
9123 		if (tg3_asic_rev(tp) != ASIC_REV_5785 &&
9124 		    !tg3_flag(tp, 57765_PLUS) &&
9125 		    tr32(TG3_PCIE_PHY_TSTCTL) ==
9126 		    (TG3_PCIE_PHY_TSTCTL_PCIE10 | TG3_PCIE_PHY_TSTCTL_PSCRAM))
9127 			tw32(TG3_PCIE_PHY_TSTCTL, TG3_PCIE_PHY_TSTCTL_PSCRAM);
9128 
9129 		if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0) {
9130 			tw32(GRC_MISC_CFG, (1 << 29));
9131 			val |= (1 << 29);
9132 		}
9133 	}
9134 
9135 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
9136 		tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
9137 		tw32(GRC_VCPU_EXT_CTRL,
9138 		     tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
9139 	}
9140 
9141 	/* Set the clock to the highest frequency to avoid timeouts. With link
9142 	 * aware mode, the clock speed could be slow and bootcode does not
9143 	 * complete within the expected time. Override the clock to allow the
9144 	 * bootcode to finish sooner and then restore it.
9145 	 */
9146 	tg3_override_clk(tp);
9147 
9148 	/* Manage gphy power for all CPMU absent PCIe devices. */
9149 	if (tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, CPMU_PRESENT))
9150 		val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
9151 
9152 	tw32(GRC_MISC_CFG, val);
9153 
9154 	/* restore 5701 hardware bug workaround write method */
9155 	tp->write32 = write_op;
9156 
9157 	/* Unfortunately, we have to delay before the PCI read back.
9158 	 * Some 575X chips even will not respond to a PCI cfg access
9159 	 * when the reset command is given to the chip.
9160 	 *
9161 	 * How do these hardware designers expect things to work
9162 	 * properly if the PCI write is posted for a long period
9163 	 * of time?  It is always necessary to have some method by
9164 	 * which a register read back can occur to push the write
9165 	 * out which does the reset.
9166 	 *
9167 	 * For most tg3 variants the trick below was working.
9168 	 * Ho hum...
9169 	 */
9170 	udelay(120);
9171 
9172 	/* Flush PCI posted writes.  The normal MMIO registers
9173 	 * are inaccessible at this time so this is the only
9174 	 * way to make this reliably (actually, this is no longer
9175 	 * the case, see above).  I tried to use indirect
9176 	 * register read/write but this upset some 5701 variants.
9177 	 */
9178 	pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
9179 
9180 	udelay(120);
9181 
9182 	if (tg3_flag(tp, PCI_EXPRESS) && pci_is_pcie(tp->pdev)) {
9183 		u16 val16;
9184 
9185 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A0) {
9186 			int j;
9187 			u32 cfg_val;
9188 
9189 			/* Wait for link training to complete.  */
9190 			for (j = 0; j < 5000; j++)
9191 				udelay(100);
9192 
9193 			pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
9194 			pci_write_config_dword(tp->pdev, 0xc4,
9195 					       cfg_val | (1 << 15));
9196 		}
9197 
9198 		/* Clear the "no snoop" and "relaxed ordering" bits. */
9199 		val16 = PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN;
9200 		/*
9201 		 * Older PCIe devices only support the 128 byte
9202 		 * MPS setting.  Enforce the restriction.
9203 		 */
9204 		if (!tg3_flag(tp, CPMU_PRESENT))
9205 			val16 |= PCI_EXP_DEVCTL_PAYLOAD;
9206 		pcie_capability_clear_word(tp->pdev, PCI_EXP_DEVCTL, val16);
9207 
9208 		/* Clear error status */
9209 		pcie_capability_write_word(tp->pdev, PCI_EXP_DEVSTA,
9210 				      PCI_EXP_DEVSTA_CED |
9211 				      PCI_EXP_DEVSTA_NFED |
9212 				      PCI_EXP_DEVSTA_FED |
9213 				      PCI_EXP_DEVSTA_URD);
9214 	}
9215 
9216 	tg3_restore_pci_state(tp);
9217 
9218 	tg3_flag_clear(tp, CHIP_RESETTING);
9219 	tg3_flag_clear(tp, ERROR_PROCESSED);
9220 
9221 	val = 0;
9222 	if (tg3_flag(tp, 5780_CLASS))
9223 		val = tr32(MEMARB_MODE);
9224 	tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
9225 
9226 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A3) {
9227 		tg3_stop_fw(tp);
9228 		tw32(0x5000, 0x400);
9229 	}
9230 
9231 	if (tg3_flag(tp, IS_SSB_CORE)) {
9232 		/*
9233 		 * BCM4785: In order to avoid repercussions from using
9234 		 * potentially defective internal ROM, stop the Rx RISC CPU,
9235 		 * which is not required.
9236 		 */
9237 		tg3_stop_fw(tp);
9238 		tg3_halt_cpu(tp, RX_CPU_BASE);
9239 	}
9240 
9241 	err = tg3_poll_fw(tp);
9242 	if (err)
9243 		return err;
9244 
9245 	tw32(GRC_MODE, tp->grc_mode);
9246 
9247 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A0) {
9248 		val = tr32(0xc4);
9249 
9250 		tw32(0xc4, val | (1 << 15));
9251 	}
9252 
9253 	if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
9254 	    tg3_asic_rev(tp) == ASIC_REV_5705) {
9255 		tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
9256 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A0)
9257 			tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
9258 		tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
9259 	}
9260 
9261 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
9262 		tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
9263 		val = tp->mac_mode;
9264 	} else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
9265 		tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
9266 		val = tp->mac_mode;
9267 	} else
9268 		val = 0;
9269 
9270 	tw32_f(MAC_MODE, val);
9271 	udelay(40);
9272 
9273 	tg3_ape_unlock(tp, TG3_APE_LOCK_GRC);
9274 
9275 	tg3_mdio_start(tp);
9276 
9277 	if (tg3_flag(tp, PCI_EXPRESS) &&
9278 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0 &&
9279 	    tg3_asic_rev(tp) != ASIC_REV_5785 &&
9280 	    !tg3_flag(tp, 57765_PLUS)) {
9281 		val = tr32(0x7c00);
9282 
9283 		tw32(0x7c00, val | (1 << 25));
9284 	}
9285 
9286 	tg3_restore_clk(tp);
9287 
9288 	/* Increase the core clock speed to fix tx timeout issue for 5762
9289 	 * with 100Mbps link speed.
9290 	 */
9291 	if (tg3_asic_rev(tp) == ASIC_REV_5762) {
9292 		val = tr32(TG3_CPMU_CLCK_ORIDE_ENABLE);
9293 		tw32(TG3_CPMU_CLCK_ORIDE_ENABLE, val |
9294 		     TG3_CPMU_MAC_ORIDE_ENABLE);
9295 	}
9296 
9297 	/* Reprobe ASF enable state.  */
9298 	tg3_flag_clear(tp, ENABLE_ASF);
9299 	tp->phy_flags &= ~(TG3_PHYFLG_1G_ON_VAUX_OK |
9300 			   TG3_PHYFLG_KEEP_LINK_ON_PWRDN);
9301 
9302 	tg3_flag_clear(tp, ASF_NEW_HANDSHAKE);
9303 	tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
9304 	if (val == NIC_SRAM_DATA_SIG_MAGIC) {
9305 		u32 nic_cfg;
9306 
9307 		tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
9308 		if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
9309 			tg3_flag_set(tp, ENABLE_ASF);
9310 			tp->last_event_jiffies = jiffies;
9311 			if (tg3_flag(tp, 5750_PLUS))
9312 				tg3_flag_set(tp, ASF_NEW_HANDSHAKE);
9313 
9314 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &nic_cfg);
9315 			if (nic_cfg & NIC_SRAM_1G_ON_VAUX_OK)
9316 				tp->phy_flags |= TG3_PHYFLG_1G_ON_VAUX_OK;
9317 			if (nic_cfg & NIC_SRAM_LNK_FLAP_AVOID)
9318 				tp->phy_flags |= TG3_PHYFLG_KEEP_LINK_ON_PWRDN;
9319 		}
9320 	}
9321 
9322 	return 0;
9323 }
9324 
9325 static void tg3_get_nstats(struct tg3 *, struct rtnl_link_stats64 *);
9326 static void tg3_get_estats(struct tg3 *, struct tg3_ethtool_stats *);
9327 static void __tg3_set_rx_mode(struct net_device *);
9328 
9329 /* tp->lock is held. */
9330 static int tg3_halt(struct tg3 *tp, int kind, bool silent)
9331 {
9332 	int err;
9333 
9334 	tg3_stop_fw(tp);
9335 
9336 	tg3_write_sig_pre_reset(tp, kind);
9337 
9338 	tg3_abort_hw(tp, silent);
9339 	err = tg3_chip_reset(tp);
9340 
9341 	__tg3_set_mac_addr(tp, false);
9342 
9343 	tg3_write_sig_legacy(tp, kind);
9344 	tg3_write_sig_post_reset(tp, kind);
9345 
9346 	if (tp->hw_stats) {
9347 		/* Save the stats across chip resets... */
9348 		tg3_get_nstats(tp, &tp->net_stats_prev);
9349 		tg3_get_estats(tp, &tp->estats_prev);
9350 
9351 		/* And make sure the next sample is new data */
9352 		memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
9353 	}
9354 
9355 	return err;
9356 }
9357 
9358 static int tg3_set_mac_addr(struct net_device *dev, void *p)
9359 {
9360 	struct tg3 *tp = netdev_priv(dev);
9361 	struct sockaddr *addr = p;
9362 	int err = 0;
9363 	bool skip_mac_1 = false;
9364 
9365 	if (!is_valid_ether_addr(addr->sa_data))
9366 		return -EADDRNOTAVAIL;
9367 
9368 	eth_hw_addr_set(dev, addr->sa_data);
9369 
9370 	if (!netif_running(dev))
9371 		return 0;
9372 
9373 	if (tg3_flag(tp, ENABLE_ASF)) {
9374 		u32 addr0_high, addr0_low, addr1_high, addr1_low;
9375 
9376 		addr0_high = tr32(MAC_ADDR_0_HIGH);
9377 		addr0_low = tr32(MAC_ADDR_0_LOW);
9378 		addr1_high = tr32(MAC_ADDR_1_HIGH);
9379 		addr1_low = tr32(MAC_ADDR_1_LOW);
9380 
9381 		/* Skip MAC addr 1 if ASF is using it. */
9382 		if ((addr0_high != addr1_high || addr0_low != addr1_low) &&
9383 		    !(addr1_high == 0 && addr1_low == 0))
9384 			skip_mac_1 = true;
9385 	}
9386 	spin_lock_bh(&tp->lock);
9387 	__tg3_set_mac_addr(tp, skip_mac_1);
9388 	__tg3_set_rx_mode(dev);
9389 	spin_unlock_bh(&tp->lock);
9390 
9391 	return err;
9392 }
9393 
9394 /* tp->lock is held. */
9395 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
9396 			   dma_addr_t mapping, u32 maxlen_flags,
9397 			   u32 nic_addr)
9398 {
9399 	tg3_write_mem(tp,
9400 		      (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
9401 		      ((u64) mapping >> 32));
9402 	tg3_write_mem(tp,
9403 		      (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
9404 		      ((u64) mapping & 0xffffffff));
9405 	tg3_write_mem(tp,
9406 		      (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
9407 		       maxlen_flags);
9408 
9409 	if (!tg3_flag(tp, 5705_PLUS))
9410 		tg3_write_mem(tp,
9411 			      (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
9412 			      nic_addr);
9413 }
9414 
9415 
9416 static void tg3_coal_tx_init(struct tg3 *tp, struct ethtool_coalesce *ec)
9417 {
9418 	int i = 0;
9419 
9420 	if (!tg3_flag(tp, ENABLE_TSS)) {
9421 		tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
9422 		tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
9423 		tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
9424 	} else {
9425 		tw32(HOSTCC_TXCOL_TICKS, 0);
9426 		tw32(HOSTCC_TXMAX_FRAMES, 0);
9427 		tw32(HOSTCC_TXCOAL_MAXF_INT, 0);
9428 
9429 		for (; i < tp->txq_cnt; i++) {
9430 			u32 reg;
9431 
9432 			reg = HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18;
9433 			tw32(reg, ec->tx_coalesce_usecs);
9434 			reg = HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18;
9435 			tw32(reg, ec->tx_max_coalesced_frames);
9436 			reg = HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18;
9437 			tw32(reg, ec->tx_max_coalesced_frames_irq);
9438 		}
9439 	}
9440 
9441 	for (; i < tp->irq_max - 1; i++) {
9442 		tw32(HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18, 0);
9443 		tw32(HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18, 0);
9444 		tw32(HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
9445 	}
9446 }
9447 
9448 static void tg3_coal_rx_init(struct tg3 *tp, struct ethtool_coalesce *ec)
9449 {
9450 	int i = 0;
9451 	u32 limit = tp->rxq_cnt;
9452 
9453 	if (!tg3_flag(tp, ENABLE_RSS)) {
9454 		tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
9455 		tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
9456 		tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
9457 		limit--;
9458 	} else {
9459 		tw32(HOSTCC_RXCOL_TICKS, 0);
9460 		tw32(HOSTCC_RXMAX_FRAMES, 0);
9461 		tw32(HOSTCC_RXCOAL_MAXF_INT, 0);
9462 	}
9463 
9464 	for (; i < limit; i++) {
9465 		u32 reg;
9466 
9467 		reg = HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18;
9468 		tw32(reg, ec->rx_coalesce_usecs);
9469 		reg = HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18;
9470 		tw32(reg, ec->rx_max_coalesced_frames);
9471 		reg = HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18;
9472 		tw32(reg, ec->rx_max_coalesced_frames_irq);
9473 	}
9474 
9475 	for (; i < tp->irq_max - 1; i++) {
9476 		tw32(HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18, 0);
9477 		tw32(HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18, 0);
9478 		tw32(HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
9479 	}
9480 }
9481 
9482 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
9483 {
9484 	tg3_coal_tx_init(tp, ec);
9485 	tg3_coal_rx_init(tp, ec);
9486 
9487 	if (!tg3_flag(tp, 5705_PLUS)) {
9488 		u32 val = ec->stats_block_coalesce_usecs;
9489 
9490 		tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
9491 		tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
9492 
9493 		if (!tp->link_up)
9494 			val = 0;
9495 
9496 		tw32(HOSTCC_STAT_COAL_TICKS, val);
9497 	}
9498 }
9499 
9500 /* tp->lock is held. */
9501 static void tg3_tx_rcbs_disable(struct tg3 *tp)
9502 {
9503 	u32 txrcb, limit;
9504 
9505 	/* Disable all transmit rings but the first. */
9506 	if (!tg3_flag(tp, 5705_PLUS))
9507 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 16;
9508 	else if (tg3_flag(tp, 5717_PLUS))
9509 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 4;
9510 	else if (tg3_flag(tp, 57765_CLASS) ||
9511 		 tg3_asic_rev(tp) == ASIC_REV_5762)
9512 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 2;
9513 	else
9514 		limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
9515 
9516 	for (txrcb = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
9517 	     txrcb < limit; txrcb += TG3_BDINFO_SIZE)
9518 		tg3_write_mem(tp, txrcb + TG3_BDINFO_MAXLEN_FLAGS,
9519 			      BDINFO_FLAGS_DISABLED);
9520 }
9521 
9522 /* tp->lock is held. */
9523 static void tg3_tx_rcbs_init(struct tg3 *tp)
9524 {
9525 	int i = 0;
9526 	u32 txrcb = NIC_SRAM_SEND_RCB;
9527 
9528 	if (tg3_flag(tp, ENABLE_TSS))
9529 		i++;
9530 
9531 	for (; i < tp->irq_max; i++, txrcb += TG3_BDINFO_SIZE) {
9532 		struct tg3_napi *tnapi = &tp->napi[i];
9533 
9534 		if (!tnapi->tx_ring)
9535 			continue;
9536 
9537 		tg3_set_bdinfo(tp, txrcb, tnapi->tx_desc_mapping,
9538 			       (TG3_TX_RING_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT),
9539 			       NIC_SRAM_TX_BUFFER_DESC);
9540 	}
9541 }
9542 
9543 /* tp->lock is held. */
9544 static void tg3_rx_ret_rcbs_disable(struct tg3 *tp)
9545 {
9546 	u32 rxrcb, limit;
9547 
9548 	/* Disable all receive return rings but the first. */
9549 	if (tg3_flag(tp, 5717_PLUS))
9550 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 17;
9551 	else if (!tg3_flag(tp, 5705_PLUS))
9552 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 16;
9553 	else if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
9554 		 tg3_asic_rev(tp) == ASIC_REV_5762 ||
9555 		 tg3_flag(tp, 57765_CLASS))
9556 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 4;
9557 	else
9558 		limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
9559 
9560 	for (rxrcb = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
9561 	     rxrcb < limit; rxrcb += TG3_BDINFO_SIZE)
9562 		tg3_write_mem(tp, rxrcb + TG3_BDINFO_MAXLEN_FLAGS,
9563 			      BDINFO_FLAGS_DISABLED);
9564 }
9565 
9566 /* tp->lock is held. */
9567 static void tg3_rx_ret_rcbs_init(struct tg3 *tp)
9568 {
9569 	int i = 0;
9570 	u32 rxrcb = NIC_SRAM_RCV_RET_RCB;
9571 
9572 	if (tg3_flag(tp, ENABLE_RSS))
9573 		i++;
9574 
9575 	for (; i < tp->irq_max; i++, rxrcb += TG3_BDINFO_SIZE) {
9576 		struct tg3_napi *tnapi = &tp->napi[i];
9577 
9578 		if (!tnapi->rx_rcb)
9579 			continue;
9580 
9581 		tg3_set_bdinfo(tp, rxrcb, tnapi->rx_rcb_mapping,
9582 			       (tp->rx_ret_ring_mask + 1) <<
9583 				BDINFO_FLAGS_MAXLEN_SHIFT, 0);
9584 	}
9585 }
9586 
9587 /* tp->lock is held. */
9588 static void tg3_rings_reset(struct tg3 *tp)
9589 {
9590 	int i;
9591 	u32 stblk;
9592 	struct tg3_napi *tnapi = &tp->napi[0];
9593 
9594 	tg3_tx_rcbs_disable(tp);
9595 
9596 	tg3_rx_ret_rcbs_disable(tp);
9597 
9598 	/* Disable interrupts */
9599 	tw32_mailbox_f(tp->napi[0].int_mbox, 1);
9600 	tp->napi[0].chk_msi_cnt = 0;
9601 	tp->napi[0].last_rx_cons = 0;
9602 	tp->napi[0].last_tx_cons = 0;
9603 
9604 	/* Zero mailbox registers. */
9605 	if (tg3_flag(tp, SUPPORT_MSIX)) {
9606 		for (i = 1; i < tp->irq_max; i++) {
9607 			tp->napi[i].tx_prod = 0;
9608 			tp->napi[i].tx_cons = 0;
9609 			if (tg3_flag(tp, ENABLE_TSS))
9610 				tw32_mailbox(tp->napi[i].prodmbox, 0);
9611 			tw32_rx_mbox(tp->napi[i].consmbox, 0);
9612 			tw32_mailbox_f(tp->napi[i].int_mbox, 1);
9613 			tp->napi[i].chk_msi_cnt = 0;
9614 			tp->napi[i].last_rx_cons = 0;
9615 			tp->napi[i].last_tx_cons = 0;
9616 		}
9617 		if (!tg3_flag(tp, ENABLE_TSS))
9618 			tw32_mailbox(tp->napi[0].prodmbox, 0);
9619 	} else {
9620 		tp->napi[0].tx_prod = 0;
9621 		tp->napi[0].tx_cons = 0;
9622 		tw32_mailbox(tp->napi[0].prodmbox, 0);
9623 		tw32_rx_mbox(tp->napi[0].consmbox, 0);
9624 	}
9625 
9626 	/* Make sure the NIC-based send BD rings are disabled. */
9627 	if (!tg3_flag(tp, 5705_PLUS)) {
9628 		u32 mbox = MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW;
9629 		for (i = 0; i < 16; i++)
9630 			tw32_tx_mbox(mbox + i * 8, 0);
9631 	}
9632 
9633 	/* Clear status block in ram. */
9634 	memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
9635 
9636 	/* Set status block DMA address */
9637 	tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
9638 	     ((u64) tnapi->status_mapping >> 32));
9639 	tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
9640 	     ((u64) tnapi->status_mapping & 0xffffffff));
9641 
9642 	stblk = HOSTCC_STATBLCK_RING1;
9643 
9644 	for (i = 1, tnapi++; i < tp->irq_cnt; i++, tnapi++) {
9645 		u64 mapping = (u64)tnapi->status_mapping;
9646 		tw32(stblk + TG3_64BIT_REG_HIGH, mapping >> 32);
9647 		tw32(stblk + TG3_64BIT_REG_LOW, mapping & 0xffffffff);
9648 		stblk += 8;
9649 
9650 		/* Clear status block in ram. */
9651 		memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
9652 	}
9653 
9654 	tg3_tx_rcbs_init(tp);
9655 	tg3_rx_ret_rcbs_init(tp);
9656 }
9657 
9658 static void tg3_setup_rxbd_thresholds(struct tg3 *tp)
9659 {
9660 	u32 val, bdcache_maxcnt, host_rep_thresh, nic_rep_thresh;
9661 
9662 	if (!tg3_flag(tp, 5750_PLUS) ||
9663 	    tg3_flag(tp, 5780_CLASS) ||
9664 	    tg3_asic_rev(tp) == ASIC_REV_5750 ||
9665 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
9666 	    tg3_flag(tp, 57765_PLUS))
9667 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5700;
9668 	else if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
9669 		 tg3_asic_rev(tp) == ASIC_REV_5787)
9670 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5755;
9671 	else
9672 		bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5906;
9673 
9674 	nic_rep_thresh = min(bdcache_maxcnt / 2, tp->rx_std_max_post);
9675 	host_rep_thresh = max_t(u32, tp->rx_pending / 8, 1);
9676 
9677 	val = min(nic_rep_thresh, host_rep_thresh);
9678 	tw32(RCVBDI_STD_THRESH, val);
9679 
9680 	if (tg3_flag(tp, 57765_PLUS))
9681 		tw32(STD_REPLENISH_LWM, bdcache_maxcnt);
9682 
9683 	if (!tg3_flag(tp, JUMBO_CAPABLE) || tg3_flag(tp, 5780_CLASS))
9684 		return;
9685 
9686 	bdcache_maxcnt = TG3_SRAM_RX_JMB_BDCACHE_SIZE_5700;
9687 
9688 	host_rep_thresh = max_t(u32, tp->rx_jumbo_pending / 8, 1);
9689 
9690 	val = min(bdcache_maxcnt / 2, host_rep_thresh);
9691 	tw32(RCVBDI_JUMBO_THRESH, val);
9692 
9693 	if (tg3_flag(tp, 57765_PLUS))
9694 		tw32(JMB_REPLENISH_LWM, bdcache_maxcnt);
9695 }
9696 
9697 static inline u32 calc_crc(unsigned char *buf, int len)
9698 {
9699 	u32 reg;
9700 	u32 tmp;
9701 	int j, k;
9702 
9703 	reg = 0xffffffff;
9704 
9705 	for (j = 0; j < len; j++) {
9706 		reg ^= buf[j];
9707 
9708 		for (k = 0; k < 8; k++) {
9709 			tmp = reg & 0x01;
9710 
9711 			reg >>= 1;
9712 
9713 			if (tmp)
9714 				reg ^= CRC32_POLY_LE;
9715 		}
9716 	}
9717 
9718 	return ~reg;
9719 }
9720 
9721 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
9722 {
9723 	/* accept or reject all multicast frames */
9724 	tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
9725 	tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
9726 	tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
9727 	tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
9728 }
9729 
9730 static void __tg3_set_rx_mode(struct net_device *dev)
9731 {
9732 	struct tg3 *tp = netdev_priv(dev);
9733 	u32 rx_mode;
9734 
9735 	rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
9736 				  RX_MODE_KEEP_VLAN_TAG);
9737 
9738 #if !defined(CONFIG_VLAN_8021Q) && !defined(CONFIG_VLAN_8021Q_MODULE)
9739 	/* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
9740 	 * flag clear.
9741 	 */
9742 	if (!tg3_flag(tp, ENABLE_ASF))
9743 		rx_mode |= RX_MODE_KEEP_VLAN_TAG;
9744 #endif
9745 
9746 	if (dev->flags & IFF_PROMISC) {
9747 		/* Promiscuous mode. */
9748 		rx_mode |= RX_MODE_PROMISC;
9749 	} else if (dev->flags & IFF_ALLMULTI) {
9750 		/* Accept all multicast. */
9751 		tg3_set_multi(tp, 1);
9752 	} else if (netdev_mc_empty(dev)) {
9753 		/* Reject all multicast. */
9754 		tg3_set_multi(tp, 0);
9755 	} else {
9756 		/* Accept one or more multicast(s). */
9757 		struct netdev_hw_addr *ha;
9758 		u32 mc_filter[4] = { 0, };
9759 		u32 regidx;
9760 		u32 bit;
9761 		u32 crc;
9762 
9763 		netdev_for_each_mc_addr(ha, dev) {
9764 			crc = calc_crc(ha->addr, ETH_ALEN);
9765 			bit = ~crc & 0x7f;
9766 			regidx = (bit & 0x60) >> 5;
9767 			bit &= 0x1f;
9768 			mc_filter[regidx] |= (1 << bit);
9769 		}
9770 
9771 		tw32(MAC_HASH_REG_0, mc_filter[0]);
9772 		tw32(MAC_HASH_REG_1, mc_filter[1]);
9773 		tw32(MAC_HASH_REG_2, mc_filter[2]);
9774 		tw32(MAC_HASH_REG_3, mc_filter[3]);
9775 	}
9776 
9777 	if (netdev_uc_count(dev) > TG3_MAX_UCAST_ADDR(tp)) {
9778 		rx_mode |= RX_MODE_PROMISC;
9779 	} else if (!(dev->flags & IFF_PROMISC)) {
9780 		/* Add all entries into to the mac addr filter list */
9781 		int i = 0;
9782 		struct netdev_hw_addr *ha;
9783 
9784 		netdev_for_each_uc_addr(ha, dev) {
9785 			__tg3_set_one_mac_addr(tp, ha->addr,
9786 					       i + TG3_UCAST_ADDR_IDX(tp));
9787 			i++;
9788 		}
9789 	}
9790 
9791 	if (rx_mode != tp->rx_mode) {
9792 		tp->rx_mode = rx_mode;
9793 		tw32_f(MAC_RX_MODE, rx_mode);
9794 		udelay(10);
9795 	}
9796 }
9797 
9798 static void tg3_rss_init_dflt_indir_tbl(struct tg3 *tp, u32 qcnt)
9799 {
9800 	int i;
9801 
9802 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
9803 		tp->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, qcnt);
9804 }
9805 
9806 static void tg3_rss_check_indir_tbl(struct tg3 *tp)
9807 {
9808 	int i;
9809 
9810 	if (!tg3_flag(tp, SUPPORT_MSIX))
9811 		return;
9812 
9813 	if (tp->rxq_cnt == 1) {
9814 		memset(&tp->rss_ind_tbl[0], 0, sizeof(tp->rss_ind_tbl));
9815 		return;
9816 	}
9817 
9818 	/* Validate table against current IRQ count */
9819 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++) {
9820 		if (tp->rss_ind_tbl[i] >= tp->rxq_cnt)
9821 			break;
9822 	}
9823 
9824 	if (i != TG3_RSS_INDIR_TBL_SIZE)
9825 		tg3_rss_init_dflt_indir_tbl(tp, tp->rxq_cnt);
9826 }
9827 
9828 static void tg3_rss_write_indir_tbl(struct tg3 *tp)
9829 {
9830 	int i = 0;
9831 	u32 reg = MAC_RSS_INDIR_TBL_0;
9832 
9833 	while (i < TG3_RSS_INDIR_TBL_SIZE) {
9834 		u32 val = tp->rss_ind_tbl[i];
9835 		i++;
9836 		for (; i % 8; i++) {
9837 			val <<= 4;
9838 			val |= tp->rss_ind_tbl[i];
9839 		}
9840 		tw32(reg, val);
9841 		reg += 4;
9842 	}
9843 }
9844 
9845 static inline u32 tg3_lso_rd_dma_workaround_bit(struct tg3 *tp)
9846 {
9847 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
9848 		return TG3_LSO_RD_DMA_TX_LENGTH_WA_5719;
9849 	else
9850 		return TG3_LSO_RD_DMA_TX_LENGTH_WA_5720;
9851 }
9852 
9853 /* tp->lock is held. */
9854 static int tg3_reset_hw(struct tg3 *tp, bool reset_phy)
9855 {
9856 	u32 val, rdmac_mode;
9857 	int i, err, limit;
9858 	struct tg3_rx_prodring_set *tpr = &tp->napi[0].prodring;
9859 
9860 	tg3_disable_ints(tp);
9861 
9862 	tg3_stop_fw(tp);
9863 
9864 	tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
9865 
9866 	if (tg3_flag(tp, INIT_COMPLETE))
9867 		tg3_abort_hw(tp, 1);
9868 
9869 	if ((tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
9870 	    !(tp->phy_flags & TG3_PHYFLG_USER_CONFIGURED)) {
9871 		tg3_phy_pull_config(tp);
9872 		tg3_eee_pull_config(tp, NULL);
9873 		tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
9874 	}
9875 
9876 	/* Enable MAC control of LPI */
9877 	if (tp->phy_flags & TG3_PHYFLG_EEE_CAP)
9878 		tg3_setup_eee(tp);
9879 
9880 	if (reset_phy)
9881 		tg3_phy_reset(tp);
9882 
9883 	err = tg3_chip_reset(tp);
9884 	if (err)
9885 		return err;
9886 
9887 	tg3_write_sig_legacy(tp, RESET_KIND_INIT);
9888 
9889 	if (tg3_chip_rev(tp) == CHIPREV_5784_AX) {
9890 		val = tr32(TG3_CPMU_CTRL);
9891 		val &= ~(CPMU_CTRL_LINK_AWARE_MODE | CPMU_CTRL_LINK_IDLE_MODE);
9892 		tw32(TG3_CPMU_CTRL, val);
9893 
9894 		val = tr32(TG3_CPMU_LSPD_10MB_CLK);
9895 		val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
9896 		val |= CPMU_LSPD_10MB_MACCLK_6_25;
9897 		tw32(TG3_CPMU_LSPD_10MB_CLK, val);
9898 
9899 		val = tr32(TG3_CPMU_LNK_AWARE_PWRMD);
9900 		val &= ~CPMU_LNK_AWARE_MACCLK_MASK;
9901 		val |= CPMU_LNK_AWARE_MACCLK_6_25;
9902 		tw32(TG3_CPMU_LNK_AWARE_PWRMD, val);
9903 
9904 		val = tr32(TG3_CPMU_HST_ACC);
9905 		val &= ~CPMU_HST_ACC_MACCLK_MASK;
9906 		val |= CPMU_HST_ACC_MACCLK_6_25;
9907 		tw32(TG3_CPMU_HST_ACC, val);
9908 	}
9909 
9910 	if (tg3_asic_rev(tp) == ASIC_REV_57780) {
9911 		val = tr32(PCIE_PWR_MGMT_THRESH) & ~PCIE_PWR_MGMT_L1_THRESH_MSK;
9912 		val |= PCIE_PWR_MGMT_EXT_ASPM_TMR_EN |
9913 		       PCIE_PWR_MGMT_L1_THRESH_4MS;
9914 		tw32(PCIE_PWR_MGMT_THRESH, val);
9915 
9916 		val = tr32(TG3_PCIE_EIDLE_DELAY) & ~TG3_PCIE_EIDLE_DELAY_MASK;
9917 		tw32(TG3_PCIE_EIDLE_DELAY, val | TG3_PCIE_EIDLE_DELAY_13_CLKS);
9918 
9919 		tw32(TG3_CORR_ERR_STAT, TG3_CORR_ERR_STAT_CLEAR);
9920 
9921 		val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
9922 		tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
9923 	}
9924 
9925 	if (tg3_flag(tp, L1PLLPD_EN)) {
9926 		u32 grc_mode = tr32(GRC_MODE);
9927 
9928 		/* Access the lower 1K of PL PCIE block registers. */
9929 		val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9930 		tw32(GRC_MODE, val | GRC_MODE_PCIE_PL_SEL);
9931 
9932 		val = tr32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL1);
9933 		tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL1,
9934 		     val | TG3_PCIE_PL_LO_PHYCTL1_L1PLLPD_EN);
9935 
9936 		tw32(GRC_MODE, grc_mode);
9937 	}
9938 
9939 	if (tg3_flag(tp, 57765_CLASS)) {
9940 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0) {
9941 			u32 grc_mode = tr32(GRC_MODE);
9942 
9943 			/* Access the lower 1K of PL PCIE block registers. */
9944 			val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9945 			tw32(GRC_MODE, val | GRC_MODE_PCIE_PL_SEL);
9946 
9947 			val = tr32(TG3_PCIE_TLDLPL_PORT +
9948 				   TG3_PCIE_PL_LO_PHYCTL5);
9949 			tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL5,
9950 			     val | TG3_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ);
9951 
9952 			tw32(GRC_MODE, grc_mode);
9953 		}
9954 
9955 		if (tg3_chip_rev(tp) != CHIPREV_57765_AX) {
9956 			u32 grc_mode;
9957 
9958 			/* Fix transmit hangs */
9959 			val = tr32(TG3_CPMU_PADRNG_CTL);
9960 			val |= TG3_CPMU_PADRNG_CTL_RDIV2;
9961 			tw32(TG3_CPMU_PADRNG_CTL, val);
9962 
9963 			grc_mode = tr32(GRC_MODE);
9964 
9965 			/* Access the lower 1K of DL PCIE block registers. */
9966 			val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
9967 			tw32(GRC_MODE, val | GRC_MODE_PCIE_DL_SEL);
9968 
9969 			val = tr32(TG3_PCIE_TLDLPL_PORT +
9970 				   TG3_PCIE_DL_LO_FTSMAX);
9971 			val &= ~TG3_PCIE_DL_LO_FTSMAX_MSK;
9972 			tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_DL_LO_FTSMAX,
9973 			     val | TG3_PCIE_DL_LO_FTSMAX_VAL);
9974 
9975 			tw32(GRC_MODE, grc_mode);
9976 		}
9977 
9978 		val = tr32(TG3_CPMU_LSPD_10MB_CLK);
9979 		val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
9980 		val |= CPMU_LSPD_10MB_MACCLK_6_25;
9981 		tw32(TG3_CPMU_LSPD_10MB_CLK, val);
9982 	}
9983 
9984 	/* This works around an issue with Athlon chipsets on
9985 	 * B3 tigon3 silicon.  This bit has no effect on any
9986 	 * other revision.  But do not set this on PCI Express
9987 	 * chips and don't even touch the clocks if the CPMU is present.
9988 	 */
9989 	if (!tg3_flag(tp, CPMU_PRESENT)) {
9990 		if (!tg3_flag(tp, PCI_EXPRESS))
9991 			tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
9992 		tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
9993 	}
9994 
9995 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0 &&
9996 	    tg3_flag(tp, PCIX_MODE)) {
9997 		val = tr32(TG3PCI_PCISTATE);
9998 		val |= PCISTATE_RETRY_SAME_DMA;
9999 		tw32(TG3PCI_PCISTATE, val);
10000 	}
10001 
10002 	if (tg3_flag(tp, ENABLE_APE)) {
10003 		/* Allow reads and writes to the
10004 		 * APE register and memory space.
10005 		 */
10006 		val = tr32(TG3PCI_PCISTATE);
10007 		val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
10008 		       PCISTATE_ALLOW_APE_SHMEM_WR |
10009 		       PCISTATE_ALLOW_APE_PSPACE_WR;
10010 		tw32(TG3PCI_PCISTATE, val);
10011 	}
10012 
10013 	if (tg3_chip_rev(tp) == CHIPREV_5704_BX) {
10014 		/* Enable some hw fixes.  */
10015 		val = tr32(TG3PCI_MSI_DATA);
10016 		val |= (1 << 26) | (1 << 28) | (1 << 29);
10017 		tw32(TG3PCI_MSI_DATA, val);
10018 	}
10019 
10020 	/* Descriptor ring init may make accesses to the
10021 	 * NIC SRAM area to setup the TX descriptors, so we
10022 	 * can only do this after the hardware has been
10023 	 * successfully reset.
10024 	 */
10025 	err = tg3_init_rings(tp);
10026 	if (err)
10027 		return err;
10028 
10029 	if (tg3_flag(tp, 57765_PLUS)) {
10030 		val = tr32(TG3PCI_DMA_RW_CTRL) &
10031 		      ~DMA_RWCTRL_DIS_CACHE_ALIGNMENT;
10032 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_57765_A0)
10033 			val &= ~DMA_RWCTRL_CRDRDR_RDMA_MRRS_MSK;
10034 		if (!tg3_flag(tp, 57765_CLASS) &&
10035 		    tg3_asic_rev(tp) != ASIC_REV_5717 &&
10036 		    tg3_asic_rev(tp) != ASIC_REV_5762)
10037 			val |= DMA_RWCTRL_TAGGED_STAT_WA;
10038 		tw32(TG3PCI_DMA_RW_CTRL, val | tp->dma_rwctrl);
10039 	} else if (tg3_asic_rev(tp) != ASIC_REV_5784 &&
10040 		   tg3_asic_rev(tp) != ASIC_REV_5761) {
10041 		/* This value is determined during the probe time DMA
10042 		 * engine test, tg3_test_dma.
10043 		 */
10044 		tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
10045 	}
10046 
10047 	tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
10048 			  GRC_MODE_4X_NIC_SEND_RINGS |
10049 			  GRC_MODE_NO_TX_PHDR_CSUM |
10050 			  GRC_MODE_NO_RX_PHDR_CSUM);
10051 	tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
10052 
10053 	/* Pseudo-header checksum is done by hardware logic and not
10054 	 * the offload processers, so make the chip do the pseudo-
10055 	 * header checksums on receive.  For transmit it is more
10056 	 * convenient to do the pseudo-header checksum in software
10057 	 * as Linux does that on transmit for us in all cases.
10058 	 */
10059 	tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
10060 
10061 	val = GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP;
10062 	if (tp->rxptpctl)
10063 		tw32(TG3_RX_PTP_CTL,
10064 		     tp->rxptpctl | TG3_RX_PTP_CTL_HWTS_INTERLOCK);
10065 
10066 	if (tg3_flag(tp, PTP_CAPABLE))
10067 		val |= GRC_MODE_TIME_SYNC_ENABLE;
10068 
10069 	tw32(GRC_MODE, tp->grc_mode | val);
10070 
10071 	/* On one of the AMD platform, MRRS is restricted to 4000 because of
10072 	 * south bridge limitation. As a workaround, Driver is setting MRRS
10073 	 * to 2048 instead of default 4096.
10074 	 */
10075 	if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
10076 	    tp->pdev->subsystem_device == TG3PCI_SUBDEVICE_ID_DELL_5762) {
10077 		val = tr32(TG3PCI_DEV_STATUS_CTRL) & ~MAX_READ_REQ_MASK;
10078 		tw32(TG3PCI_DEV_STATUS_CTRL, val | MAX_READ_REQ_SIZE_2048);
10079 	}
10080 
10081 	/* Setup the timer prescalar register.  Clock is always 66Mhz. */
10082 	val = tr32(GRC_MISC_CFG);
10083 	val &= ~0xff;
10084 	val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
10085 	tw32(GRC_MISC_CFG, val);
10086 
10087 	/* Initialize MBUF/DESC pool. */
10088 	if (tg3_flag(tp, 5750_PLUS)) {
10089 		/* Do nothing.  */
10090 	} else if (tg3_asic_rev(tp) != ASIC_REV_5705) {
10091 		tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
10092 		if (tg3_asic_rev(tp) == ASIC_REV_5704)
10093 			tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
10094 		else
10095 			tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
10096 		tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
10097 		tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
10098 	} else if (tg3_flag(tp, TSO_CAPABLE)) {
10099 		int fw_len;
10100 
10101 		fw_len = tp->fw_len;
10102 		fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
10103 		tw32(BUFMGR_MB_POOL_ADDR,
10104 		     NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
10105 		tw32(BUFMGR_MB_POOL_SIZE,
10106 		     NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
10107 	}
10108 
10109 	if (tp->dev->mtu <= ETH_DATA_LEN) {
10110 		tw32(BUFMGR_MB_RDMA_LOW_WATER,
10111 		     tp->bufmgr_config.mbuf_read_dma_low_water);
10112 		tw32(BUFMGR_MB_MACRX_LOW_WATER,
10113 		     tp->bufmgr_config.mbuf_mac_rx_low_water);
10114 		tw32(BUFMGR_MB_HIGH_WATER,
10115 		     tp->bufmgr_config.mbuf_high_water);
10116 	} else {
10117 		tw32(BUFMGR_MB_RDMA_LOW_WATER,
10118 		     tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
10119 		tw32(BUFMGR_MB_MACRX_LOW_WATER,
10120 		     tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
10121 		tw32(BUFMGR_MB_HIGH_WATER,
10122 		     tp->bufmgr_config.mbuf_high_water_jumbo);
10123 	}
10124 	tw32(BUFMGR_DMA_LOW_WATER,
10125 	     tp->bufmgr_config.dma_low_water);
10126 	tw32(BUFMGR_DMA_HIGH_WATER,
10127 	     tp->bufmgr_config.dma_high_water);
10128 
10129 	val = BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE;
10130 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
10131 		val |= BUFMGR_MODE_NO_TX_UNDERRUN;
10132 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
10133 	    tg3_asic_rev(tp) == ASIC_REV_5762 ||
10134 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10135 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5720_A0)
10136 		val |= BUFMGR_MODE_MBLOW_ATTN_ENAB;
10137 	tw32(BUFMGR_MODE, val);
10138 	for (i = 0; i < 2000; i++) {
10139 		if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
10140 			break;
10141 		udelay(10);
10142 	}
10143 	if (i >= 2000) {
10144 		netdev_err(tp->dev, "%s cannot enable BUFMGR\n", __func__);
10145 		return -ENODEV;
10146 	}
10147 
10148 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5906_A1)
10149 		tw32(ISO_PKT_TX, (tr32(ISO_PKT_TX) & ~0x3) | 0x2);
10150 
10151 	tg3_setup_rxbd_thresholds(tp);
10152 
10153 	/* Initialize TG3_BDINFO's at:
10154 	 *  RCVDBDI_STD_BD:	standard eth size rx ring
10155 	 *  RCVDBDI_JUMBO_BD:	jumbo frame rx ring
10156 	 *  RCVDBDI_MINI_BD:	small frame rx ring (??? does not work)
10157 	 *
10158 	 * like so:
10159 	 *  TG3_BDINFO_HOST_ADDR:	high/low parts of DMA address of ring
10160 	 *  TG3_BDINFO_MAXLEN_FLAGS:	(rx max buffer size << 16) |
10161 	 *                              ring attribute flags
10162 	 *  TG3_BDINFO_NIC_ADDR:	location of descriptors in nic SRAM
10163 	 *
10164 	 * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
10165 	 * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
10166 	 *
10167 	 * The size of each ring is fixed in the firmware, but the location is
10168 	 * configurable.
10169 	 */
10170 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
10171 	     ((u64) tpr->rx_std_mapping >> 32));
10172 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
10173 	     ((u64) tpr->rx_std_mapping & 0xffffffff));
10174 	if (!tg3_flag(tp, 5717_PLUS))
10175 		tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
10176 		     NIC_SRAM_RX_BUFFER_DESC);
10177 
10178 	/* Disable the mini ring */
10179 	if (!tg3_flag(tp, 5705_PLUS))
10180 		tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
10181 		     BDINFO_FLAGS_DISABLED);
10182 
10183 	/* Program the jumbo buffer descriptor ring control
10184 	 * blocks on those devices that have them.
10185 	 */
10186 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10187 	    (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS))) {
10188 
10189 		if (tg3_flag(tp, JUMBO_RING_ENABLE)) {
10190 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
10191 			     ((u64) tpr->rx_jmb_mapping >> 32));
10192 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
10193 			     ((u64) tpr->rx_jmb_mapping & 0xffffffff));
10194 			val = TG3_RX_JMB_RING_SIZE(tp) <<
10195 			      BDINFO_FLAGS_MAXLEN_SHIFT;
10196 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
10197 			     val | BDINFO_FLAGS_USE_EXT_RECV);
10198 			if (!tg3_flag(tp, USE_JUMBO_BDFLAG) ||
10199 			    tg3_flag(tp, 57765_CLASS) ||
10200 			    tg3_asic_rev(tp) == ASIC_REV_5762)
10201 				tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
10202 				     NIC_SRAM_RX_JUMBO_BUFFER_DESC);
10203 		} else {
10204 			tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
10205 			     BDINFO_FLAGS_DISABLED);
10206 		}
10207 
10208 		if (tg3_flag(tp, 57765_PLUS)) {
10209 			val = TG3_RX_STD_RING_SIZE(tp);
10210 			val <<= BDINFO_FLAGS_MAXLEN_SHIFT;
10211 			val |= (TG3_RX_STD_DMA_SZ << 2);
10212 		} else
10213 			val = TG3_RX_STD_DMA_SZ << BDINFO_FLAGS_MAXLEN_SHIFT;
10214 	} else
10215 		val = TG3_RX_STD_MAX_SIZE_5700 << BDINFO_FLAGS_MAXLEN_SHIFT;
10216 
10217 	tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS, val);
10218 
10219 	tpr->rx_std_prod_idx = tp->rx_pending;
10220 	tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG, tpr->rx_std_prod_idx);
10221 
10222 	tpr->rx_jmb_prod_idx =
10223 		tg3_flag(tp, JUMBO_RING_ENABLE) ? tp->rx_jumbo_pending : 0;
10224 	tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG, tpr->rx_jmb_prod_idx);
10225 
10226 	tg3_rings_reset(tp);
10227 
10228 	/* Initialize MAC address and backoff seed. */
10229 	__tg3_set_mac_addr(tp, false);
10230 
10231 	/* MTU + ethernet header + FCS + optional VLAN tag */
10232 	tw32(MAC_RX_MTU_SIZE,
10233 	     tp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
10234 
10235 	/* The slot time is changed by tg3_setup_phy if we
10236 	 * run at gigabit with half duplex.
10237 	 */
10238 	val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
10239 	      (6 << TX_LENGTHS_IPG_SHIFT) |
10240 	      (32 << TX_LENGTHS_SLOT_TIME_SHIFT);
10241 
10242 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10243 	    tg3_asic_rev(tp) == ASIC_REV_5762)
10244 		val |= tr32(MAC_TX_LENGTHS) &
10245 		       (TX_LENGTHS_JMB_FRM_LEN_MSK |
10246 			TX_LENGTHS_CNT_DWN_VAL_MSK);
10247 
10248 	tw32(MAC_TX_LENGTHS, val);
10249 
10250 	/* Receive rules. */
10251 	tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
10252 	tw32(RCVLPC_CONFIG, 0x0181);
10253 
10254 	/* Calculate RDMAC_MODE setting early, we need it to determine
10255 	 * the RCVLPC_STATE_ENABLE mask.
10256 	 */
10257 	rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
10258 		      RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
10259 		      RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
10260 		      RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
10261 		      RDMAC_MODE_LNGREAD_ENAB);
10262 
10263 	if (tg3_asic_rev(tp) == ASIC_REV_5717)
10264 		rdmac_mode |= RDMAC_MODE_MULT_DMA_RD_DIS;
10265 
10266 	if (tg3_asic_rev(tp) == ASIC_REV_5784 ||
10267 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10268 	    tg3_asic_rev(tp) == ASIC_REV_57780)
10269 		rdmac_mode |= RDMAC_MODE_BD_SBD_CRPT_ENAB |
10270 			      RDMAC_MODE_MBUF_RBD_CRPT_ENAB |
10271 			      RDMAC_MODE_MBUF_SBD_CRPT_ENAB;
10272 
10273 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
10274 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
10275 		if (tg3_flag(tp, TSO_CAPABLE)) {
10276 			rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
10277 		} else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
10278 			   !tg3_flag(tp, IS_5788)) {
10279 			rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
10280 		}
10281 	}
10282 
10283 	if (tg3_flag(tp, PCI_EXPRESS))
10284 		rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
10285 
10286 	if (tg3_asic_rev(tp) == ASIC_REV_57766) {
10287 		tp->dma_limit = 0;
10288 		if (tp->dev->mtu <= ETH_DATA_LEN) {
10289 			rdmac_mode |= RDMAC_MODE_JMB_2K_MMRR;
10290 			tp->dma_limit = TG3_TX_BD_DMA_MAX_2K;
10291 		}
10292 	}
10293 
10294 	if (tg3_flag(tp, HW_TSO_1) ||
10295 	    tg3_flag(tp, HW_TSO_2) ||
10296 	    tg3_flag(tp, HW_TSO_3))
10297 		rdmac_mode |= RDMAC_MODE_IPV4_LSO_EN;
10298 
10299 	if (tg3_flag(tp, 57765_PLUS) ||
10300 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10301 	    tg3_asic_rev(tp) == ASIC_REV_57780)
10302 		rdmac_mode |= RDMAC_MODE_IPV6_LSO_EN;
10303 
10304 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10305 	    tg3_asic_rev(tp) == ASIC_REV_5762)
10306 		rdmac_mode |= tr32(RDMAC_MODE) & RDMAC_MODE_H2BNC_VLAN_DET;
10307 
10308 	if (tg3_asic_rev(tp) == ASIC_REV_5761 ||
10309 	    tg3_asic_rev(tp) == ASIC_REV_5784 ||
10310 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
10311 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
10312 	    tg3_flag(tp, 57765_PLUS)) {
10313 		u32 tgtreg;
10314 
10315 		if (tg3_asic_rev(tp) == ASIC_REV_5762)
10316 			tgtreg = TG3_RDMA_RSRVCTRL_REG2;
10317 		else
10318 			tgtreg = TG3_RDMA_RSRVCTRL_REG;
10319 
10320 		val = tr32(tgtreg);
10321 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
10322 		    tg3_asic_rev(tp) == ASIC_REV_5762) {
10323 			val &= ~(TG3_RDMA_RSRVCTRL_TXMRGN_MASK |
10324 				 TG3_RDMA_RSRVCTRL_FIFO_LWM_MASK |
10325 				 TG3_RDMA_RSRVCTRL_FIFO_HWM_MASK);
10326 			val |= TG3_RDMA_RSRVCTRL_TXMRGN_320B |
10327 			       TG3_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
10328 			       TG3_RDMA_RSRVCTRL_FIFO_HWM_1_5K;
10329 		}
10330 		tw32(tgtreg, val | TG3_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
10331 	}
10332 
10333 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
10334 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
10335 	    tg3_asic_rev(tp) == ASIC_REV_5762) {
10336 		u32 tgtreg;
10337 
10338 		if (tg3_asic_rev(tp) == ASIC_REV_5762)
10339 			tgtreg = TG3_LSO_RD_DMA_CRPTEN_CTRL2;
10340 		else
10341 			tgtreg = TG3_LSO_RD_DMA_CRPTEN_CTRL;
10342 
10343 		val = tr32(tgtreg);
10344 		tw32(tgtreg, val |
10345 		     TG3_LSO_RD_DMA_CRPTEN_CTRL_BLEN_BD_4K |
10346 		     TG3_LSO_RD_DMA_CRPTEN_CTRL_BLEN_LSO_4K);
10347 	}
10348 
10349 	/* Receive/send statistics. */
10350 	if (tg3_flag(tp, 5750_PLUS)) {
10351 		val = tr32(RCVLPC_STATS_ENABLE);
10352 		val &= ~RCVLPC_STATSENAB_DACK_FIX;
10353 		tw32(RCVLPC_STATS_ENABLE, val);
10354 	} else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
10355 		   tg3_flag(tp, TSO_CAPABLE)) {
10356 		val = tr32(RCVLPC_STATS_ENABLE);
10357 		val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
10358 		tw32(RCVLPC_STATS_ENABLE, val);
10359 	} else {
10360 		tw32(RCVLPC_STATS_ENABLE, 0xffffff);
10361 	}
10362 	tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
10363 	tw32(SNDDATAI_STATSENAB, 0xffffff);
10364 	tw32(SNDDATAI_STATSCTRL,
10365 	     (SNDDATAI_SCTRL_ENABLE |
10366 	      SNDDATAI_SCTRL_FASTUPD));
10367 
10368 	/* Setup host coalescing engine. */
10369 	tw32(HOSTCC_MODE, 0);
10370 	for (i = 0; i < 2000; i++) {
10371 		if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
10372 			break;
10373 		udelay(10);
10374 	}
10375 
10376 	__tg3_set_coalesce(tp, &tp->coal);
10377 
10378 	if (!tg3_flag(tp, 5705_PLUS)) {
10379 		/* Status/statistics block address.  See tg3_timer,
10380 		 * the tg3_periodic_fetch_stats call there, and
10381 		 * tg3_get_stats to see how this works for 5705/5750 chips.
10382 		 */
10383 		tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
10384 		     ((u64) tp->stats_mapping >> 32));
10385 		tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
10386 		     ((u64) tp->stats_mapping & 0xffffffff));
10387 		tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
10388 
10389 		tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
10390 
10391 		/* Clear statistics and status block memory areas */
10392 		for (i = NIC_SRAM_STATS_BLK;
10393 		     i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
10394 		     i += sizeof(u32)) {
10395 			tg3_write_mem(tp, i, 0);
10396 			udelay(40);
10397 		}
10398 	}
10399 
10400 	tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
10401 
10402 	tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
10403 	tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
10404 	if (!tg3_flag(tp, 5705_PLUS))
10405 		tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
10406 
10407 	if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
10408 		tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
10409 		/* reset to prevent losing 1st rx packet intermittently */
10410 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
10411 		udelay(10);
10412 	}
10413 
10414 	tp->mac_mode |= MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
10415 			MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE |
10416 			MAC_MODE_FHDE_ENABLE;
10417 	if (tg3_flag(tp, ENABLE_APE))
10418 		tp->mac_mode |= MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
10419 	if (!tg3_flag(tp, 5705_PLUS) &&
10420 	    !(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
10421 	    tg3_asic_rev(tp) != ASIC_REV_5700)
10422 		tp->mac_mode |= MAC_MODE_LINK_POLARITY;
10423 	tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
10424 	udelay(40);
10425 
10426 	/* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
10427 	 * If TG3_FLAG_IS_NIC is zero, we should read the
10428 	 * register to preserve the GPIO settings for LOMs. The GPIOs,
10429 	 * whether used as inputs or outputs, are set by boot code after
10430 	 * reset.
10431 	 */
10432 	if (!tg3_flag(tp, IS_NIC)) {
10433 		u32 gpio_mask;
10434 
10435 		gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE1 |
10436 			    GRC_LCLCTRL_GPIO_OE2 | GRC_LCLCTRL_GPIO_OUTPUT0 |
10437 			    GRC_LCLCTRL_GPIO_OUTPUT1 | GRC_LCLCTRL_GPIO_OUTPUT2;
10438 
10439 		if (tg3_asic_rev(tp) == ASIC_REV_5752)
10440 			gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
10441 				     GRC_LCLCTRL_GPIO_OUTPUT3;
10442 
10443 		if (tg3_asic_rev(tp) == ASIC_REV_5755)
10444 			gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
10445 
10446 		tp->grc_local_ctrl &= ~gpio_mask;
10447 		tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
10448 
10449 		/* GPIO1 must be driven high for eeprom write protect */
10450 		if (tg3_flag(tp, EEPROM_WRITE_PROT))
10451 			tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
10452 					       GRC_LCLCTRL_GPIO_OUTPUT1);
10453 	}
10454 	tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10455 	udelay(100);
10456 
10457 	if (tg3_flag(tp, USING_MSIX)) {
10458 		val = tr32(MSGINT_MODE);
10459 		val |= MSGINT_MODE_ENABLE;
10460 		if (tp->irq_cnt > 1)
10461 			val |= MSGINT_MODE_MULTIVEC_EN;
10462 		if (!tg3_flag(tp, 1SHOT_MSI))
10463 			val |= MSGINT_MODE_ONE_SHOT_DISABLE;
10464 		tw32(MSGINT_MODE, val);
10465 	}
10466 
10467 	if (!tg3_flag(tp, 5705_PLUS)) {
10468 		tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
10469 		udelay(40);
10470 	}
10471 
10472 	val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
10473 	       WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
10474 	       WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
10475 	       WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
10476 	       WDMAC_MODE_LNGREAD_ENAB);
10477 
10478 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
10479 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
10480 		if (tg3_flag(tp, TSO_CAPABLE) &&
10481 		    (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A1 ||
10482 		     tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A2)) {
10483 			/* nothing */
10484 		} else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
10485 			   !tg3_flag(tp, IS_5788)) {
10486 			val |= WDMAC_MODE_RX_ACCEL;
10487 		}
10488 	}
10489 
10490 	/* Enable host coalescing bug fix */
10491 	if (tg3_flag(tp, 5755_PLUS))
10492 		val |= WDMAC_MODE_STATUS_TAG_FIX;
10493 
10494 	if (tg3_asic_rev(tp) == ASIC_REV_5785)
10495 		val |= WDMAC_MODE_BURST_ALL_DATA;
10496 
10497 	tw32_f(WDMAC_MODE, val);
10498 	udelay(40);
10499 
10500 	if (tg3_flag(tp, PCIX_MODE)) {
10501 		u16 pcix_cmd;
10502 
10503 		pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
10504 				     &pcix_cmd);
10505 		if (tg3_asic_rev(tp) == ASIC_REV_5703) {
10506 			pcix_cmd &= ~PCI_X_CMD_MAX_READ;
10507 			pcix_cmd |= PCI_X_CMD_READ_2K;
10508 		} else if (tg3_asic_rev(tp) == ASIC_REV_5704) {
10509 			pcix_cmd &= ~(PCI_X_CMD_MAX_SPLIT | PCI_X_CMD_MAX_READ);
10510 			pcix_cmd |= PCI_X_CMD_READ_2K;
10511 		}
10512 		pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
10513 				      pcix_cmd);
10514 	}
10515 
10516 	tw32_f(RDMAC_MODE, rdmac_mode);
10517 	udelay(40);
10518 
10519 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
10520 	    tg3_asic_rev(tp) == ASIC_REV_5720) {
10521 		for (i = 0; i < TG3_NUM_RDMA_CHANNELS; i++) {
10522 			if (tr32(TG3_RDMA_LENGTH + (i << 2)) > TG3_MAX_MTU(tp))
10523 				break;
10524 		}
10525 		if (i < TG3_NUM_RDMA_CHANNELS) {
10526 			val = tr32(TG3_LSO_RD_DMA_CRPTEN_CTRL);
10527 			val |= tg3_lso_rd_dma_workaround_bit(tp);
10528 			tw32(TG3_LSO_RD_DMA_CRPTEN_CTRL, val);
10529 			tg3_flag_set(tp, 5719_5720_RDMA_BUG);
10530 		}
10531 	}
10532 
10533 	tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
10534 	if (!tg3_flag(tp, 5705_PLUS))
10535 		tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
10536 
10537 	if (tg3_asic_rev(tp) == ASIC_REV_5761)
10538 		tw32(SNDDATAC_MODE,
10539 		     SNDDATAC_MODE_ENABLE | SNDDATAC_MODE_CDELAY);
10540 	else
10541 		tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
10542 
10543 	tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
10544 	tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
10545 	val = RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ;
10546 	if (tg3_flag(tp, LRG_PROD_RING_CAP))
10547 		val |= RCVDBDI_MODE_LRG_RING_SZ;
10548 	tw32(RCVDBDI_MODE, val);
10549 	tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
10550 	if (tg3_flag(tp, HW_TSO_1) ||
10551 	    tg3_flag(tp, HW_TSO_2) ||
10552 	    tg3_flag(tp, HW_TSO_3))
10553 		tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
10554 	val = SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE;
10555 	if (tg3_flag(tp, ENABLE_TSS))
10556 		val |= SNDBDI_MODE_MULTI_TXQ_EN;
10557 	tw32(SNDBDI_MODE, val);
10558 	tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
10559 
10560 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0) {
10561 		err = tg3_load_5701_a0_firmware_fix(tp);
10562 		if (err)
10563 			return err;
10564 	}
10565 
10566 	if (tg3_asic_rev(tp) == ASIC_REV_57766) {
10567 		/* Ignore any errors for the firmware download. If download
10568 		 * fails, the device will operate with EEE disabled
10569 		 */
10570 		tg3_load_57766_firmware(tp);
10571 	}
10572 
10573 	if (tg3_flag(tp, TSO_CAPABLE)) {
10574 		err = tg3_load_tso_firmware(tp);
10575 		if (err)
10576 			return err;
10577 	}
10578 
10579 	tp->tx_mode = TX_MODE_ENABLE;
10580 
10581 	if (tg3_flag(tp, 5755_PLUS) ||
10582 	    tg3_asic_rev(tp) == ASIC_REV_5906)
10583 		tp->tx_mode |= TX_MODE_MBUF_LOCKUP_FIX;
10584 
10585 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
10586 	    tg3_asic_rev(tp) == ASIC_REV_5762) {
10587 		val = TX_MODE_JMB_FRM_LEN | TX_MODE_CNT_DN_MODE;
10588 		tp->tx_mode &= ~val;
10589 		tp->tx_mode |= tr32(MAC_TX_MODE) & val;
10590 	}
10591 
10592 	tw32_f(MAC_TX_MODE, tp->tx_mode);
10593 	udelay(100);
10594 
10595 	if (tg3_flag(tp, ENABLE_RSS)) {
10596 		u32 rss_key[10];
10597 
10598 		tg3_rss_write_indir_tbl(tp);
10599 
10600 		netdev_rss_key_fill(rss_key, 10 * sizeof(u32));
10601 
10602 		for (i = 0; i < 10 ; i++)
10603 			tw32(MAC_RSS_HASH_KEY_0 + i*4, rss_key[i]);
10604 	}
10605 
10606 	tp->rx_mode = RX_MODE_ENABLE;
10607 	if (tg3_flag(tp, 5755_PLUS))
10608 		tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
10609 
10610 	if (tg3_asic_rev(tp) == ASIC_REV_5762)
10611 		tp->rx_mode |= RX_MODE_IPV4_FRAG_FIX;
10612 
10613 	if (tg3_flag(tp, ENABLE_RSS))
10614 		tp->rx_mode |= RX_MODE_RSS_ENABLE |
10615 			       RX_MODE_RSS_ITBL_HASH_BITS_7 |
10616 			       RX_MODE_RSS_IPV6_HASH_EN |
10617 			       RX_MODE_RSS_TCP_IPV6_HASH_EN |
10618 			       RX_MODE_RSS_IPV4_HASH_EN |
10619 			       RX_MODE_RSS_TCP_IPV4_HASH_EN;
10620 
10621 	tw32_f(MAC_RX_MODE, tp->rx_mode);
10622 	udelay(10);
10623 
10624 	tw32(MAC_LED_CTRL, tp->led_ctrl);
10625 
10626 	tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
10627 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
10628 		tw32_f(MAC_RX_MODE, RX_MODE_RESET);
10629 		udelay(10);
10630 	}
10631 	tw32_f(MAC_RX_MODE, tp->rx_mode);
10632 	udelay(10);
10633 
10634 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
10635 		if ((tg3_asic_rev(tp) == ASIC_REV_5704) &&
10636 		    !(tp->phy_flags & TG3_PHYFLG_SERDES_PREEMPHASIS)) {
10637 			/* Set drive transmission level to 1.2V  */
10638 			/* only if the signal pre-emphasis bit is not set  */
10639 			val = tr32(MAC_SERDES_CFG);
10640 			val &= 0xfffff000;
10641 			val |= 0x880;
10642 			tw32(MAC_SERDES_CFG, val);
10643 		}
10644 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A1)
10645 			tw32(MAC_SERDES_CFG, 0x616000);
10646 	}
10647 
10648 	/* Prevent chip from dropping frames when flow control
10649 	 * is enabled.
10650 	 */
10651 	if (tg3_flag(tp, 57765_CLASS))
10652 		val = 1;
10653 	else
10654 		val = 2;
10655 	tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, val);
10656 
10657 	if (tg3_asic_rev(tp) == ASIC_REV_5704 &&
10658 	    (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
10659 		/* Use hardware link auto-negotiation */
10660 		tg3_flag_set(tp, HW_AUTONEG);
10661 	}
10662 
10663 	if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
10664 	    tg3_asic_rev(tp) == ASIC_REV_5714) {
10665 		u32 tmp;
10666 
10667 		tmp = tr32(SERDES_RX_CTRL);
10668 		tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
10669 		tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
10670 		tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
10671 		tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
10672 	}
10673 
10674 	if (!tg3_flag(tp, USE_PHYLIB)) {
10675 		if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
10676 			tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
10677 
10678 		err = tg3_setup_phy(tp, false);
10679 		if (err)
10680 			return err;
10681 
10682 		if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
10683 		    !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
10684 			u32 tmp;
10685 
10686 			/* Clear CRC stats. */
10687 			if (!tg3_readphy(tp, MII_TG3_TEST1, &tmp)) {
10688 				tg3_writephy(tp, MII_TG3_TEST1,
10689 					     tmp | MII_TG3_TEST1_CRC_EN);
10690 				tg3_readphy(tp, MII_TG3_RXR_COUNTERS, &tmp);
10691 			}
10692 		}
10693 	}
10694 
10695 	__tg3_set_rx_mode(tp->dev);
10696 
10697 	/* Initialize receive rules. */
10698 	tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
10699 	tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
10700 	tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
10701 	tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
10702 
10703 	if (tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS))
10704 		limit = 8;
10705 	else
10706 		limit = 16;
10707 	if (tg3_flag(tp, ENABLE_ASF))
10708 		limit -= 4;
10709 	switch (limit) {
10710 	case 16:
10711 		tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
10712 		fallthrough;
10713 	case 15:
10714 		tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
10715 		fallthrough;
10716 	case 14:
10717 		tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
10718 		fallthrough;
10719 	case 13:
10720 		tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
10721 		fallthrough;
10722 	case 12:
10723 		tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
10724 		fallthrough;
10725 	case 11:
10726 		tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
10727 		fallthrough;
10728 	case 10:
10729 		tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
10730 		fallthrough;
10731 	case 9:
10732 		tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
10733 		fallthrough;
10734 	case 8:
10735 		tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
10736 		fallthrough;
10737 	case 7:
10738 		tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
10739 		fallthrough;
10740 	case 6:
10741 		tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
10742 		fallthrough;
10743 	case 5:
10744 		tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
10745 		fallthrough;
10746 	case 4:
10747 		/* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
10748 	case 3:
10749 		/* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
10750 	case 2:
10751 	case 1:
10752 
10753 	default:
10754 		break;
10755 	}
10756 
10757 	if (tg3_flag(tp, ENABLE_APE))
10758 		/* Write our heartbeat update interval to APE. */
10759 		tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
10760 				APE_HOST_HEARTBEAT_INT_5SEC);
10761 
10762 	tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
10763 
10764 	return 0;
10765 }
10766 
10767 /* Called at device open time to get the chip ready for
10768  * packet processing.  Invoked with tp->lock held.
10769  */
10770 static int tg3_init_hw(struct tg3 *tp, bool reset_phy)
10771 {
10772 	/* Chip may have been just powered on. If so, the boot code may still
10773 	 * be running initialization. Wait for it to finish to avoid races in
10774 	 * accessing the hardware.
10775 	 */
10776 	tg3_enable_register_access(tp);
10777 	tg3_poll_fw(tp);
10778 
10779 	tg3_switch_clocks(tp);
10780 
10781 	tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
10782 
10783 	return tg3_reset_hw(tp, reset_phy);
10784 }
10785 
10786 #ifdef CONFIG_TIGON3_HWMON
10787 static void tg3_sd_scan_scratchpad(struct tg3 *tp, struct tg3_ocir *ocir)
10788 {
10789 	u32 off, len = TG3_OCIR_LEN;
10790 	int i;
10791 
10792 	for (i = 0, off = 0; i < TG3_SD_NUM_RECS; i++, ocir++, off += len) {
10793 		tg3_ape_scratchpad_read(tp, (u32 *) ocir, off, len);
10794 
10795 		if (ocir->signature != TG3_OCIR_SIG_MAGIC ||
10796 		    !(ocir->version_flags & TG3_OCIR_FLAG_ACTIVE))
10797 			memset(ocir, 0, len);
10798 	}
10799 }
10800 
10801 /* sysfs attributes for hwmon */
10802 static ssize_t tg3_show_temp(struct device *dev,
10803 			     struct device_attribute *devattr, char *buf)
10804 {
10805 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
10806 	struct tg3 *tp = dev_get_drvdata(dev);
10807 	u32 temperature;
10808 
10809 	spin_lock_bh(&tp->lock);
10810 	tg3_ape_scratchpad_read(tp, &temperature, attr->index,
10811 				sizeof(temperature));
10812 	spin_unlock_bh(&tp->lock);
10813 	return sprintf(buf, "%u\n", temperature * 1000);
10814 }
10815 
10816 
10817 static SENSOR_DEVICE_ATTR(temp1_input, 0444, tg3_show_temp, NULL,
10818 			  TG3_TEMP_SENSOR_OFFSET);
10819 static SENSOR_DEVICE_ATTR(temp1_crit, 0444, tg3_show_temp, NULL,
10820 			  TG3_TEMP_CAUTION_OFFSET);
10821 static SENSOR_DEVICE_ATTR(temp1_max, 0444, tg3_show_temp, NULL,
10822 			  TG3_TEMP_MAX_OFFSET);
10823 
10824 static struct attribute *tg3_attrs[] = {
10825 	&sensor_dev_attr_temp1_input.dev_attr.attr,
10826 	&sensor_dev_attr_temp1_crit.dev_attr.attr,
10827 	&sensor_dev_attr_temp1_max.dev_attr.attr,
10828 	NULL
10829 };
10830 ATTRIBUTE_GROUPS(tg3);
10831 
10832 static void tg3_hwmon_close(struct tg3 *tp)
10833 {
10834 	if (tp->hwmon_dev) {
10835 		hwmon_device_unregister(tp->hwmon_dev);
10836 		tp->hwmon_dev = NULL;
10837 	}
10838 }
10839 
10840 static void tg3_hwmon_open(struct tg3 *tp)
10841 {
10842 	int i;
10843 	u32 size = 0;
10844 	struct pci_dev *pdev = tp->pdev;
10845 	struct tg3_ocir ocirs[TG3_SD_NUM_RECS];
10846 
10847 	tg3_sd_scan_scratchpad(tp, ocirs);
10848 
10849 	for (i = 0; i < TG3_SD_NUM_RECS; i++) {
10850 		if (!ocirs[i].src_data_length)
10851 			continue;
10852 
10853 		size += ocirs[i].src_hdr_length;
10854 		size += ocirs[i].src_data_length;
10855 	}
10856 
10857 	if (!size)
10858 		return;
10859 
10860 	tp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev, "tg3",
10861 							  tp, tg3_groups);
10862 	if (IS_ERR(tp->hwmon_dev)) {
10863 		tp->hwmon_dev = NULL;
10864 		dev_err(&pdev->dev, "Cannot register hwmon device, aborting\n");
10865 	}
10866 }
10867 #else
10868 static inline void tg3_hwmon_close(struct tg3 *tp) { }
10869 static inline void tg3_hwmon_open(struct tg3 *tp) { }
10870 #endif /* CONFIG_TIGON3_HWMON */
10871 
10872 
10873 #define TG3_STAT_ADD32(PSTAT, REG) \
10874 do {	u32 __val = tr32(REG); \
10875 	(PSTAT)->low += __val; \
10876 	if ((PSTAT)->low < __val) \
10877 		(PSTAT)->high += 1; \
10878 } while (0)
10879 
10880 static void tg3_periodic_fetch_stats(struct tg3 *tp)
10881 {
10882 	struct tg3_hw_stats *sp = tp->hw_stats;
10883 
10884 	if (!tp->link_up)
10885 		return;
10886 
10887 	TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
10888 	TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
10889 	TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
10890 	TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
10891 	TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
10892 	TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
10893 	TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
10894 	TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
10895 	TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
10896 	TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
10897 	TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
10898 	TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
10899 	TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
10900 	if (unlikely(tg3_flag(tp, 5719_5720_RDMA_BUG) &&
10901 		     (sp->tx_ucast_packets.low + sp->tx_mcast_packets.low +
10902 		      sp->tx_bcast_packets.low) > TG3_NUM_RDMA_CHANNELS)) {
10903 		u32 val;
10904 
10905 		val = tr32(TG3_LSO_RD_DMA_CRPTEN_CTRL);
10906 		val &= ~tg3_lso_rd_dma_workaround_bit(tp);
10907 		tw32(TG3_LSO_RD_DMA_CRPTEN_CTRL, val);
10908 		tg3_flag_clear(tp, 5719_5720_RDMA_BUG);
10909 	}
10910 
10911 	TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
10912 	TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
10913 	TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
10914 	TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
10915 	TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
10916 	TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
10917 	TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
10918 	TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
10919 	TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
10920 	TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
10921 	TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
10922 	TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
10923 	TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
10924 	TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
10925 
10926 	TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
10927 	if (tg3_asic_rev(tp) != ASIC_REV_5717 &&
10928 	    tg3_asic_rev(tp) != ASIC_REV_5762 &&
10929 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5719_A0 &&
10930 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5720_A0) {
10931 		TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
10932 	} else {
10933 		u32 val = tr32(HOSTCC_FLOW_ATTN);
10934 		val = (val & HOSTCC_FLOW_ATTN_MBUF_LWM) ? 1 : 0;
10935 		if (val) {
10936 			tw32(HOSTCC_FLOW_ATTN, HOSTCC_FLOW_ATTN_MBUF_LWM);
10937 			sp->rx_discards.low += val;
10938 			if (sp->rx_discards.low < val)
10939 				sp->rx_discards.high += 1;
10940 		}
10941 		sp->mbuf_lwm_thresh_hit = sp->rx_discards;
10942 	}
10943 	TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
10944 }
10945 
10946 static void tg3_chk_missed_msi(struct tg3 *tp)
10947 {
10948 	u32 i;
10949 
10950 	for (i = 0; i < tp->irq_cnt; i++) {
10951 		struct tg3_napi *tnapi = &tp->napi[i];
10952 
10953 		if (tg3_has_work(tnapi)) {
10954 			if (tnapi->last_rx_cons == tnapi->rx_rcb_ptr &&
10955 			    tnapi->last_tx_cons == tnapi->tx_cons) {
10956 				if (tnapi->chk_msi_cnt < 1) {
10957 					tnapi->chk_msi_cnt++;
10958 					return;
10959 				}
10960 				tg3_msi(0, tnapi);
10961 			}
10962 		}
10963 		tnapi->chk_msi_cnt = 0;
10964 		tnapi->last_rx_cons = tnapi->rx_rcb_ptr;
10965 		tnapi->last_tx_cons = tnapi->tx_cons;
10966 	}
10967 }
10968 
10969 static void tg3_timer(struct timer_list *t)
10970 {
10971 	struct tg3 *tp = from_timer(tp, t, timer);
10972 
10973 	spin_lock(&tp->lock);
10974 
10975 	if (tp->irq_sync || tg3_flag(tp, RESET_TASK_PENDING)) {
10976 		spin_unlock(&tp->lock);
10977 		goto restart_timer;
10978 	}
10979 
10980 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
10981 	    tg3_flag(tp, 57765_CLASS))
10982 		tg3_chk_missed_msi(tp);
10983 
10984 	if (tg3_flag(tp, FLUSH_POSTED_WRITES)) {
10985 		/* BCM4785: Flush posted writes from GbE to host memory. */
10986 		tr32(HOSTCC_MODE);
10987 	}
10988 
10989 	if (!tg3_flag(tp, TAGGED_STATUS)) {
10990 		/* All of this garbage is because when using non-tagged
10991 		 * IRQ status the mailbox/status_block protocol the chip
10992 		 * uses with the cpu is race prone.
10993 		 */
10994 		if (tp->napi[0].hw_status->status & SD_STATUS_UPDATED) {
10995 			tw32(GRC_LOCAL_CTRL,
10996 			     tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
10997 		} else {
10998 			tw32(HOSTCC_MODE, tp->coalesce_mode |
10999 			     HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW);
11000 		}
11001 
11002 		if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
11003 			spin_unlock(&tp->lock);
11004 			tg3_reset_task_schedule(tp);
11005 			goto restart_timer;
11006 		}
11007 	}
11008 
11009 	/* This part only runs once per second. */
11010 	if (!--tp->timer_counter) {
11011 		if (tg3_flag(tp, 5705_PLUS))
11012 			tg3_periodic_fetch_stats(tp);
11013 
11014 		if (tp->setlpicnt && !--tp->setlpicnt)
11015 			tg3_phy_eee_enable(tp);
11016 
11017 		if (tg3_flag(tp, USE_LINKCHG_REG)) {
11018 			u32 mac_stat;
11019 			int phy_event;
11020 
11021 			mac_stat = tr32(MAC_STATUS);
11022 
11023 			phy_event = 0;
11024 			if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT) {
11025 				if (mac_stat & MAC_STATUS_MI_INTERRUPT)
11026 					phy_event = 1;
11027 			} else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
11028 				phy_event = 1;
11029 
11030 			if (phy_event)
11031 				tg3_setup_phy(tp, false);
11032 		} else if (tg3_flag(tp, POLL_SERDES)) {
11033 			u32 mac_stat = tr32(MAC_STATUS);
11034 			int need_setup = 0;
11035 
11036 			if (tp->link_up &&
11037 			    (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
11038 				need_setup = 1;
11039 			}
11040 			if (!tp->link_up &&
11041 			    (mac_stat & (MAC_STATUS_PCS_SYNCED |
11042 					 MAC_STATUS_SIGNAL_DET))) {
11043 				need_setup = 1;
11044 			}
11045 			if (need_setup) {
11046 				if (!tp->serdes_counter) {
11047 					tw32_f(MAC_MODE,
11048 					     (tp->mac_mode &
11049 					      ~MAC_MODE_PORT_MODE_MASK));
11050 					udelay(40);
11051 					tw32_f(MAC_MODE, tp->mac_mode);
11052 					udelay(40);
11053 				}
11054 				tg3_setup_phy(tp, false);
11055 			}
11056 		} else if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
11057 			   tg3_flag(tp, 5780_CLASS)) {
11058 			tg3_serdes_parallel_detect(tp);
11059 		} else if (tg3_flag(tp, POLL_CPMU_LINK)) {
11060 			u32 cpmu = tr32(TG3_CPMU_STATUS);
11061 			bool link_up = !((cpmu & TG3_CPMU_STATUS_LINK_MASK) ==
11062 					 TG3_CPMU_STATUS_LINK_MASK);
11063 
11064 			if (link_up != tp->link_up)
11065 				tg3_setup_phy(tp, false);
11066 		}
11067 
11068 		tp->timer_counter = tp->timer_multiplier;
11069 	}
11070 
11071 	/* Heartbeat is only sent once every 2 seconds.
11072 	 *
11073 	 * The heartbeat is to tell the ASF firmware that the host
11074 	 * driver is still alive.  In the event that the OS crashes,
11075 	 * ASF needs to reset the hardware to free up the FIFO space
11076 	 * that may be filled with rx packets destined for the host.
11077 	 * If the FIFO is full, ASF will no longer function properly.
11078 	 *
11079 	 * Unintended resets have been reported on real time kernels
11080 	 * where the timer doesn't run on time.  Netpoll will also have
11081 	 * same problem.
11082 	 *
11083 	 * The new FWCMD_NICDRV_ALIVE3 command tells the ASF firmware
11084 	 * to check the ring condition when the heartbeat is expiring
11085 	 * before doing the reset.  This will prevent most unintended
11086 	 * resets.
11087 	 */
11088 	if (!--tp->asf_counter) {
11089 		if (tg3_flag(tp, ENABLE_ASF) && !tg3_flag(tp, ENABLE_APE)) {
11090 			tg3_wait_for_event_ack(tp);
11091 
11092 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
11093 				      FWCMD_NICDRV_ALIVE3);
11094 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
11095 			tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX,
11096 				      TG3_FW_UPDATE_TIMEOUT_SEC);
11097 
11098 			tg3_generate_fw_event(tp);
11099 		}
11100 		tp->asf_counter = tp->asf_multiplier;
11101 	}
11102 
11103 	/* Update the APE heartbeat every 5 seconds.*/
11104 	tg3_send_ape_heartbeat(tp, TG3_APE_HB_INTERVAL);
11105 
11106 	spin_unlock(&tp->lock);
11107 
11108 restart_timer:
11109 	tp->timer.expires = jiffies + tp->timer_offset;
11110 	add_timer(&tp->timer);
11111 }
11112 
11113 static void tg3_timer_init(struct tg3 *tp)
11114 {
11115 	if (tg3_flag(tp, TAGGED_STATUS) &&
11116 	    tg3_asic_rev(tp) != ASIC_REV_5717 &&
11117 	    !tg3_flag(tp, 57765_CLASS))
11118 		tp->timer_offset = HZ;
11119 	else
11120 		tp->timer_offset = HZ / 10;
11121 
11122 	BUG_ON(tp->timer_offset > HZ);
11123 
11124 	tp->timer_multiplier = (HZ / tp->timer_offset);
11125 	tp->asf_multiplier = (HZ / tp->timer_offset) *
11126 			     TG3_FW_UPDATE_FREQ_SEC;
11127 
11128 	timer_setup(&tp->timer, tg3_timer, 0);
11129 }
11130 
11131 static void tg3_timer_start(struct tg3 *tp)
11132 {
11133 	tp->asf_counter   = tp->asf_multiplier;
11134 	tp->timer_counter = tp->timer_multiplier;
11135 
11136 	tp->timer.expires = jiffies + tp->timer_offset;
11137 	add_timer(&tp->timer);
11138 }
11139 
11140 static void tg3_timer_stop(struct tg3 *tp)
11141 {
11142 	del_timer_sync(&tp->timer);
11143 }
11144 
11145 /* Restart hardware after configuration changes, self-test, etc.
11146  * Invoked with tp->lock held.
11147  */
11148 static int tg3_restart_hw(struct tg3 *tp, bool reset_phy)
11149 	__releases(tp->lock)
11150 	__acquires(tp->lock)
11151 {
11152 	int err;
11153 
11154 	err = tg3_init_hw(tp, reset_phy);
11155 	if (err) {
11156 		netdev_err(tp->dev,
11157 			   "Failed to re-initialize device, aborting\n");
11158 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11159 		tg3_full_unlock(tp);
11160 		tg3_timer_stop(tp);
11161 		tp->irq_sync = 0;
11162 		tg3_napi_enable(tp);
11163 		dev_close(tp->dev);
11164 		tg3_full_lock(tp, 0);
11165 	}
11166 	return err;
11167 }
11168 
11169 static void tg3_reset_task(struct work_struct *work)
11170 {
11171 	struct tg3 *tp = container_of(work, struct tg3, reset_task);
11172 	int err;
11173 
11174 	rtnl_lock();
11175 	tg3_full_lock(tp, 0);
11176 
11177 	if (!netif_running(tp->dev)) {
11178 		tg3_flag_clear(tp, RESET_TASK_PENDING);
11179 		tg3_full_unlock(tp);
11180 		rtnl_unlock();
11181 		return;
11182 	}
11183 
11184 	tg3_full_unlock(tp);
11185 
11186 	tg3_phy_stop(tp);
11187 
11188 	tg3_netif_stop(tp);
11189 
11190 	tg3_full_lock(tp, 1);
11191 
11192 	if (tg3_flag(tp, TX_RECOVERY_PENDING)) {
11193 		tp->write32_tx_mbox = tg3_write32_tx_mbox;
11194 		tp->write32_rx_mbox = tg3_write_flush_reg32;
11195 		tg3_flag_set(tp, MBOX_WRITE_REORDER);
11196 		tg3_flag_clear(tp, TX_RECOVERY_PENDING);
11197 	}
11198 
11199 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
11200 	err = tg3_init_hw(tp, true);
11201 	if (err) {
11202 		tg3_full_unlock(tp);
11203 		tp->irq_sync = 0;
11204 		tg3_napi_enable(tp);
11205 		/* Clear this flag so that tg3_reset_task_cancel() will not
11206 		 * call cancel_work_sync() and wait forever.
11207 		 */
11208 		tg3_flag_clear(tp, RESET_TASK_PENDING);
11209 		dev_close(tp->dev);
11210 		goto out;
11211 	}
11212 
11213 	tg3_netif_start(tp);
11214 	tg3_full_unlock(tp);
11215 	tg3_phy_start(tp);
11216 	tg3_flag_clear(tp, RESET_TASK_PENDING);
11217 out:
11218 	rtnl_unlock();
11219 }
11220 
11221 static int tg3_request_irq(struct tg3 *tp, int irq_num)
11222 {
11223 	irq_handler_t fn;
11224 	unsigned long flags;
11225 	char *name;
11226 	struct tg3_napi *tnapi = &tp->napi[irq_num];
11227 
11228 	if (tp->irq_cnt == 1)
11229 		name = tp->dev->name;
11230 	else {
11231 		name = &tnapi->irq_lbl[0];
11232 		if (tnapi->tx_buffers && tnapi->rx_rcb)
11233 			snprintf(name, IFNAMSIZ,
11234 				 "%s-txrx-%d", tp->dev->name, irq_num);
11235 		else if (tnapi->tx_buffers)
11236 			snprintf(name, IFNAMSIZ,
11237 				 "%s-tx-%d", tp->dev->name, irq_num);
11238 		else if (tnapi->rx_rcb)
11239 			snprintf(name, IFNAMSIZ,
11240 				 "%s-rx-%d", tp->dev->name, irq_num);
11241 		else
11242 			snprintf(name, IFNAMSIZ,
11243 				 "%s-%d", tp->dev->name, irq_num);
11244 		name[IFNAMSIZ-1] = 0;
11245 	}
11246 
11247 	if (tg3_flag(tp, USING_MSI) || tg3_flag(tp, USING_MSIX)) {
11248 		fn = tg3_msi;
11249 		if (tg3_flag(tp, 1SHOT_MSI))
11250 			fn = tg3_msi_1shot;
11251 		flags = 0;
11252 	} else {
11253 		fn = tg3_interrupt;
11254 		if (tg3_flag(tp, TAGGED_STATUS))
11255 			fn = tg3_interrupt_tagged;
11256 		flags = IRQF_SHARED;
11257 	}
11258 
11259 	return request_irq(tnapi->irq_vec, fn, flags, name, tnapi);
11260 }
11261 
11262 static int tg3_test_interrupt(struct tg3 *tp)
11263 {
11264 	struct tg3_napi *tnapi = &tp->napi[0];
11265 	struct net_device *dev = tp->dev;
11266 	int err, i, intr_ok = 0;
11267 	u32 val;
11268 
11269 	if (!netif_running(dev))
11270 		return -ENODEV;
11271 
11272 	tg3_disable_ints(tp);
11273 
11274 	free_irq(tnapi->irq_vec, tnapi);
11275 
11276 	/*
11277 	 * Turn off MSI one shot mode.  Otherwise this test has no
11278 	 * observable way to know whether the interrupt was delivered.
11279 	 */
11280 	if (tg3_flag(tp, 57765_PLUS)) {
11281 		val = tr32(MSGINT_MODE) | MSGINT_MODE_ONE_SHOT_DISABLE;
11282 		tw32(MSGINT_MODE, val);
11283 	}
11284 
11285 	err = request_irq(tnapi->irq_vec, tg3_test_isr,
11286 			  IRQF_SHARED, dev->name, tnapi);
11287 	if (err)
11288 		return err;
11289 
11290 	tnapi->hw_status->status &= ~SD_STATUS_UPDATED;
11291 	tg3_enable_ints(tp);
11292 
11293 	tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
11294 	       tnapi->coal_now);
11295 
11296 	for (i = 0; i < 5; i++) {
11297 		u32 int_mbox, misc_host_ctrl;
11298 
11299 		int_mbox = tr32_mailbox(tnapi->int_mbox);
11300 		misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
11301 
11302 		if ((int_mbox != 0) ||
11303 		    (misc_host_ctrl & MISC_HOST_CTRL_MASK_PCI_INT)) {
11304 			intr_ok = 1;
11305 			break;
11306 		}
11307 
11308 		if (tg3_flag(tp, 57765_PLUS) &&
11309 		    tnapi->hw_status->status_tag != tnapi->last_tag)
11310 			tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
11311 
11312 		msleep(10);
11313 	}
11314 
11315 	tg3_disable_ints(tp);
11316 
11317 	free_irq(tnapi->irq_vec, tnapi);
11318 
11319 	err = tg3_request_irq(tp, 0);
11320 
11321 	if (err)
11322 		return err;
11323 
11324 	if (intr_ok) {
11325 		/* Reenable MSI one shot mode. */
11326 		if (tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, 1SHOT_MSI)) {
11327 			val = tr32(MSGINT_MODE) & ~MSGINT_MODE_ONE_SHOT_DISABLE;
11328 			tw32(MSGINT_MODE, val);
11329 		}
11330 		return 0;
11331 	}
11332 
11333 	return -EIO;
11334 }
11335 
11336 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
11337  * successfully restored
11338  */
11339 static int tg3_test_msi(struct tg3 *tp)
11340 {
11341 	int err;
11342 	u16 pci_cmd;
11343 
11344 	if (!tg3_flag(tp, USING_MSI))
11345 		return 0;
11346 
11347 	/* Turn off SERR reporting in case MSI terminates with Master
11348 	 * Abort.
11349 	 */
11350 	pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
11351 	pci_write_config_word(tp->pdev, PCI_COMMAND,
11352 			      pci_cmd & ~PCI_COMMAND_SERR);
11353 
11354 	err = tg3_test_interrupt(tp);
11355 
11356 	pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
11357 
11358 	if (!err)
11359 		return 0;
11360 
11361 	/* other failures */
11362 	if (err != -EIO)
11363 		return err;
11364 
11365 	/* MSI test failed, go back to INTx mode */
11366 	netdev_warn(tp->dev, "No interrupt was generated using MSI. Switching "
11367 		    "to INTx mode. Please report this failure to the PCI "
11368 		    "maintainer and include system chipset information\n");
11369 
11370 	free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
11371 
11372 	pci_disable_msi(tp->pdev);
11373 
11374 	tg3_flag_clear(tp, USING_MSI);
11375 	tp->napi[0].irq_vec = tp->pdev->irq;
11376 
11377 	err = tg3_request_irq(tp, 0);
11378 	if (err)
11379 		return err;
11380 
11381 	/* Need to reset the chip because the MSI cycle may have terminated
11382 	 * with Master Abort.
11383 	 */
11384 	tg3_full_lock(tp, 1);
11385 
11386 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11387 	err = tg3_init_hw(tp, true);
11388 
11389 	tg3_full_unlock(tp);
11390 
11391 	if (err)
11392 		free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
11393 
11394 	return err;
11395 }
11396 
11397 static int tg3_request_firmware(struct tg3 *tp)
11398 {
11399 	const struct tg3_firmware_hdr *fw_hdr;
11400 
11401 	if (request_firmware(&tp->fw, tp->fw_needed, &tp->pdev->dev)) {
11402 		netdev_err(tp->dev, "Failed to load firmware \"%s\"\n",
11403 			   tp->fw_needed);
11404 		return -ENOENT;
11405 	}
11406 
11407 	fw_hdr = (struct tg3_firmware_hdr *)tp->fw->data;
11408 
11409 	/* Firmware blob starts with version numbers, followed by
11410 	 * start address and _full_ length including BSS sections
11411 	 * (which must be longer than the actual data, of course
11412 	 */
11413 
11414 	tp->fw_len = be32_to_cpu(fw_hdr->len);	/* includes bss */
11415 	if (tp->fw_len < (tp->fw->size - TG3_FW_HDR_LEN)) {
11416 		netdev_err(tp->dev, "bogus length %d in \"%s\"\n",
11417 			   tp->fw_len, tp->fw_needed);
11418 		release_firmware(tp->fw);
11419 		tp->fw = NULL;
11420 		return -EINVAL;
11421 	}
11422 
11423 	/* We no longer need firmware; we have it. */
11424 	tp->fw_needed = NULL;
11425 	return 0;
11426 }
11427 
11428 static u32 tg3_irq_count(struct tg3 *tp)
11429 {
11430 	u32 irq_cnt = max(tp->rxq_cnt, tp->txq_cnt);
11431 
11432 	if (irq_cnt > 1) {
11433 		/* We want as many rx rings enabled as there are cpus.
11434 		 * In multiqueue MSI-X mode, the first MSI-X vector
11435 		 * only deals with link interrupts, etc, so we add
11436 		 * one to the number of vectors we are requesting.
11437 		 */
11438 		irq_cnt = min_t(unsigned, irq_cnt + 1, tp->irq_max);
11439 	}
11440 
11441 	return irq_cnt;
11442 }
11443 
11444 static bool tg3_enable_msix(struct tg3 *tp)
11445 {
11446 	int i, rc;
11447 	struct msix_entry msix_ent[TG3_IRQ_MAX_VECS];
11448 
11449 	tp->txq_cnt = tp->txq_req;
11450 	tp->rxq_cnt = tp->rxq_req;
11451 	if (!tp->rxq_cnt)
11452 		tp->rxq_cnt = netif_get_num_default_rss_queues();
11453 	if (tp->rxq_cnt > tp->rxq_max)
11454 		tp->rxq_cnt = tp->rxq_max;
11455 
11456 	/* Disable multiple TX rings by default.  Simple round-robin hardware
11457 	 * scheduling of the TX rings can cause starvation of rings with
11458 	 * small packets when other rings have TSO or jumbo packets.
11459 	 */
11460 	if (!tp->txq_req)
11461 		tp->txq_cnt = 1;
11462 
11463 	tp->irq_cnt = tg3_irq_count(tp);
11464 
11465 	for (i = 0; i < tp->irq_max; i++) {
11466 		msix_ent[i].entry  = i;
11467 		msix_ent[i].vector = 0;
11468 	}
11469 
11470 	rc = pci_enable_msix_range(tp->pdev, msix_ent, 1, tp->irq_cnt);
11471 	if (rc < 0) {
11472 		return false;
11473 	} else if (rc < tp->irq_cnt) {
11474 		netdev_notice(tp->dev, "Requested %d MSI-X vectors, received %d\n",
11475 			      tp->irq_cnt, rc);
11476 		tp->irq_cnt = rc;
11477 		tp->rxq_cnt = max(rc - 1, 1);
11478 		if (tp->txq_cnt)
11479 			tp->txq_cnt = min(tp->rxq_cnt, tp->txq_max);
11480 	}
11481 
11482 	for (i = 0; i < tp->irq_max; i++)
11483 		tp->napi[i].irq_vec = msix_ent[i].vector;
11484 
11485 	if (netif_set_real_num_rx_queues(tp->dev, tp->rxq_cnt)) {
11486 		pci_disable_msix(tp->pdev);
11487 		return false;
11488 	}
11489 
11490 	if (tp->irq_cnt == 1)
11491 		return true;
11492 
11493 	tg3_flag_set(tp, ENABLE_RSS);
11494 
11495 	if (tp->txq_cnt > 1)
11496 		tg3_flag_set(tp, ENABLE_TSS);
11497 
11498 	netif_set_real_num_tx_queues(tp->dev, tp->txq_cnt);
11499 
11500 	return true;
11501 }
11502 
11503 static void tg3_ints_init(struct tg3 *tp)
11504 {
11505 	if ((tg3_flag(tp, SUPPORT_MSI) || tg3_flag(tp, SUPPORT_MSIX)) &&
11506 	    !tg3_flag(tp, TAGGED_STATUS)) {
11507 		/* All MSI supporting chips should support tagged
11508 		 * status.  Assert that this is the case.
11509 		 */
11510 		netdev_warn(tp->dev,
11511 			    "MSI without TAGGED_STATUS? Not using MSI\n");
11512 		goto defcfg;
11513 	}
11514 
11515 	if (tg3_flag(tp, SUPPORT_MSIX) && tg3_enable_msix(tp))
11516 		tg3_flag_set(tp, USING_MSIX);
11517 	else if (tg3_flag(tp, SUPPORT_MSI) && pci_enable_msi(tp->pdev) == 0)
11518 		tg3_flag_set(tp, USING_MSI);
11519 
11520 	if (tg3_flag(tp, USING_MSI) || tg3_flag(tp, USING_MSIX)) {
11521 		u32 msi_mode = tr32(MSGINT_MODE);
11522 		if (tg3_flag(tp, USING_MSIX) && tp->irq_cnt > 1)
11523 			msi_mode |= MSGINT_MODE_MULTIVEC_EN;
11524 		if (!tg3_flag(tp, 1SHOT_MSI))
11525 			msi_mode |= MSGINT_MODE_ONE_SHOT_DISABLE;
11526 		tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
11527 	}
11528 defcfg:
11529 	if (!tg3_flag(tp, USING_MSIX)) {
11530 		tp->irq_cnt = 1;
11531 		tp->napi[0].irq_vec = tp->pdev->irq;
11532 	}
11533 
11534 	if (tp->irq_cnt == 1) {
11535 		tp->txq_cnt = 1;
11536 		tp->rxq_cnt = 1;
11537 		netif_set_real_num_tx_queues(tp->dev, 1);
11538 		netif_set_real_num_rx_queues(tp->dev, 1);
11539 	}
11540 }
11541 
11542 static void tg3_ints_fini(struct tg3 *tp)
11543 {
11544 	if (tg3_flag(tp, USING_MSIX))
11545 		pci_disable_msix(tp->pdev);
11546 	else if (tg3_flag(tp, USING_MSI))
11547 		pci_disable_msi(tp->pdev);
11548 	tg3_flag_clear(tp, USING_MSI);
11549 	tg3_flag_clear(tp, USING_MSIX);
11550 	tg3_flag_clear(tp, ENABLE_RSS);
11551 	tg3_flag_clear(tp, ENABLE_TSS);
11552 }
11553 
11554 static int tg3_start(struct tg3 *tp, bool reset_phy, bool test_irq,
11555 		     bool init)
11556 {
11557 	struct net_device *dev = tp->dev;
11558 	int i, err;
11559 
11560 	/*
11561 	 * Setup interrupts first so we know how
11562 	 * many NAPI resources to allocate
11563 	 */
11564 	tg3_ints_init(tp);
11565 
11566 	tg3_rss_check_indir_tbl(tp);
11567 
11568 	/* The placement of this call is tied
11569 	 * to the setup and use of Host TX descriptors.
11570 	 */
11571 	err = tg3_alloc_consistent(tp);
11572 	if (err)
11573 		goto out_ints_fini;
11574 
11575 	tg3_napi_init(tp);
11576 
11577 	tg3_napi_enable(tp);
11578 
11579 	for (i = 0; i < tp->irq_cnt; i++) {
11580 		err = tg3_request_irq(tp, i);
11581 		if (err) {
11582 			for (i--; i >= 0; i--) {
11583 				struct tg3_napi *tnapi = &tp->napi[i];
11584 
11585 				free_irq(tnapi->irq_vec, tnapi);
11586 			}
11587 			goto out_napi_fini;
11588 		}
11589 	}
11590 
11591 	tg3_full_lock(tp, 0);
11592 
11593 	if (init)
11594 		tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
11595 
11596 	err = tg3_init_hw(tp, reset_phy);
11597 	if (err) {
11598 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11599 		tg3_free_rings(tp);
11600 	}
11601 
11602 	tg3_full_unlock(tp);
11603 
11604 	if (err)
11605 		goto out_free_irq;
11606 
11607 	if (test_irq && tg3_flag(tp, USING_MSI)) {
11608 		err = tg3_test_msi(tp);
11609 
11610 		if (err) {
11611 			tg3_full_lock(tp, 0);
11612 			tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11613 			tg3_free_rings(tp);
11614 			tg3_full_unlock(tp);
11615 
11616 			goto out_napi_fini;
11617 		}
11618 
11619 		if (!tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, USING_MSI)) {
11620 			u32 val = tr32(PCIE_TRANSACTION_CFG);
11621 
11622 			tw32(PCIE_TRANSACTION_CFG,
11623 			     val | PCIE_TRANS_CFG_1SHOT_MSI);
11624 		}
11625 	}
11626 
11627 	tg3_phy_start(tp);
11628 
11629 	tg3_hwmon_open(tp);
11630 
11631 	tg3_full_lock(tp, 0);
11632 
11633 	tg3_timer_start(tp);
11634 	tg3_flag_set(tp, INIT_COMPLETE);
11635 	tg3_enable_ints(tp);
11636 
11637 	tg3_ptp_resume(tp);
11638 
11639 	tg3_full_unlock(tp);
11640 
11641 	netif_tx_start_all_queues(dev);
11642 
11643 	/*
11644 	 * Reset loopback feature if it was turned on while the device was down
11645 	 * make sure that it's installed properly now.
11646 	 */
11647 	if (dev->features & NETIF_F_LOOPBACK)
11648 		tg3_set_loopback(dev, dev->features);
11649 
11650 	return 0;
11651 
11652 out_free_irq:
11653 	for (i = tp->irq_cnt - 1; i >= 0; i--) {
11654 		struct tg3_napi *tnapi = &tp->napi[i];
11655 		free_irq(tnapi->irq_vec, tnapi);
11656 	}
11657 
11658 out_napi_fini:
11659 	tg3_napi_disable(tp);
11660 	tg3_napi_fini(tp);
11661 	tg3_free_consistent(tp);
11662 
11663 out_ints_fini:
11664 	tg3_ints_fini(tp);
11665 
11666 	return err;
11667 }
11668 
11669 static void tg3_stop(struct tg3 *tp)
11670 {
11671 	int i;
11672 
11673 	tg3_reset_task_cancel(tp);
11674 	tg3_netif_stop(tp);
11675 
11676 	tg3_timer_stop(tp);
11677 
11678 	tg3_hwmon_close(tp);
11679 
11680 	tg3_phy_stop(tp);
11681 
11682 	tg3_full_lock(tp, 1);
11683 
11684 	tg3_disable_ints(tp);
11685 
11686 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11687 	tg3_free_rings(tp);
11688 	tg3_flag_clear(tp, INIT_COMPLETE);
11689 
11690 	tg3_full_unlock(tp);
11691 
11692 	for (i = tp->irq_cnt - 1; i >= 0; i--) {
11693 		struct tg3_napi *tnapi = &tp->napi[i];
11694 		free_irq(tnapi->irq_vec, tnapi);
11695 	}
11696 
11697 	tg3_ints_fini(tp);
11698 
11699 	tg3_napi_fini(tp);
11700 
11701 	tg3_free_consistent(tp);
11702 }
11703 
11704 static int tg3_open(struct net_device *dev)
11705 {
11706 	struct tg3 *tp = netdev_priv(dev);
11707 	int err;
11708 
11709 	if (tp->pcierr_recovery) {
11710 		netdev_err(dev, "Failed to open device. PCI error recovery "
11711 			   "in progress\n");
11712 		return -EAGAIN;
11713 	}
11714 
11715 	if (tp->fw_needed) {
11716 		err = tg3_request_firmware(tp);
11717 		if (tg3_asic_rev(tp) == ASIC_REV_57766) {
11718 			if (err) {
11719 				netdev_warn(tp->dev, "EEE capability disabled\n");
11720 				tp->phy_flags &= ~TG3_PHYFLG_EEE_CAP;
11721 			} else if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
11722 				netdev_warn(tp->dev, "EEE capability restored\n");
11723 				tp->phy_flags |= TG3_PHYFLG_EEE_CAP;
11724 			}
11725 		} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0) {
11726 			if (err)
11727 				return err;
11728 		} else if (err) {
11729 			netdev_warn(tp->dev, "TSO capability disabled\n");
11730 			tg3_flag_clear(tp, TSO_CAPABLE);
11731 		} else if (!tg3_flag(tp, TSO_CAPABLE)) {
11732 			netdev_notice(tp->dev, "TSO capability restored\n");
11733 			tg3_flag_set(tp, TSO_CAPABLE);
11734 		}
11735 	}
11736 
11737 	tg3_carrier_off(tp);
11738 
11739 	err = tg3_power_up(tp);
11740 	if (err)
11741 		return err;
11742 
11743 	tg3_full_lock(tp, 0);
11744 
11745 	tg3_disable_ints(tp);
11746 	tg3_flag_clear(tp, INIT_COMPLETE);
11747 
11748 	tg3_full_unlock(tp);
11749 
11750 	err = tg3_start(tp,
11751 			!(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN),
11752 			true, true);
11753 	if (err) {
11754 		tg3_frob_aux_power(tp, false);
11755 		pci_set_power_state(tp->pdev, PCI_D3hot);
11756 	}
11757 
11758 	return err;
11759 }
11760 
11761 static int tg3_close(struct net_device *dev)
11762 {
11763 	struct tg3 *tp = netdev_priv(dev);
11764 
11765 	if (tp->pcierr_recovery) {
11766 		netdev_err(dev, "Failed to close device. PCI error recovery "
11767 			   "in progress\n");
11768 		return -EAGAIN;
11769 	}
11770 
11771 	tg3_stop(tp);
11772 
11773 	if (pci_device_is_present(tp->pdev)) {
11774 		tg3_power_down_prepare(tp);
11775 
11776 		tg3_carrier_off(tp);
11777 	}
11778 	return 0;
11779 }
11780 
11781 static inline u64 get_stat64(tg3_stat64_t *val)
11782 {
11783        return ((u64)val->high << 32) | ((u64)val->low);
11784 }
11785 
11786 static u64 tg3_calc_crc_errors(struct tg3 *tp)
11787 {
11788 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11789 
11790 	if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
11791 	    (tg3_asic_rev(tp) == ASIC_REV_5700 ||
11792 	     tg3_asic_rev(tp) == ASIC_REV_5701)) {
11793 		u32 val;
11794 
11795 		if (!tg3_readphy(tp, MII_TG3_TEST1, &val)) {
11796 			tg3_writephy(tp, MII_TG3_TEST1,
11797 				     val | MII_TG3_TEST1_CRC_EN);
11798 			tg3_readphy(tp, MII_TG3_RXR_COUNTERS, &val);
11799 		} else
11800 			val = 0;
11801 
11802 		tp->phy_crc_errors += val;
11803 
11804 		return tp->phy_crc_errors;
11805 	}
11806 
11807 	return get_stat64(&hw_stats->rx_fcs_errors);
11808 }
11809 
11810 #define ESTAT_ADD(member) \
11811 	estats->member =	old_estats->member + \
11812 				get_stat64(&hw_stats->member)
11813 
11814 static void tg3_get_estats(struct tg3 *tp, struct tg3_ethtool_stats *estats)
11815 {
11816 	struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
11817 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11818 
11819 	ESTAT_ADD(rx_octets);
11820 	ESTAT_ADD(rx_fragments);
11821 	ESTAT_ADD(rx_ucast_packets);
11822 	ESTAT_ADD(rx_mcast_packets);
11823 	ESTAT_ADD(rx_bcast_packets);
11824 	ESTAT_ADD(rx_fcs_errors);
11825 	ESTAT_ADD(rx_align_errors);
11826 	ESTAT_ADD(rx_xon_pause_rcvd);
11827 	ESTAT_ADD(rx_xoff_pause_rcvd);
11828 	ESTAT_ADD(rx_mac_ctrl_rcvd);
11829 	ESTAT_ADD(rx_xoff_entered);
11830 	ESTAT_ADD(rx_frame_too_long_errors);
11831 	ESTAT_ADD(rx_jabbers);
11832 	ESTAT_ADD(rx_undersize_packets);
11833 	ESTAT_ADD(rx_in_length_errors);
11834 	ESTAT_ADD(rx_out_length_errors);
11835 	ESTAT_ADD(rx_64_or_less_octet_packets);
11836 	ESTAT_ADD(rx_65_to_127_octet_packets);
11837 	ESTAT_ADD(rx_128_to_255_octet_packets);
11838 	ESTAT_ADD(rx_256_to_511_octet_packets);
11839 	ESTAT_ADD(rx_512_to_1023_octet_packets);
11840 	ESTAT_ADD(rx_1024_to_1522_octet_packets);
11841 	ESTAT_ADD(rx_1523_to_2047_octet_packets);
11842 	ESTAT_ADD(rx_2048_to_4095_octet_packets);
11843 	ESTAT_ADD(rx_4096_to_8191_octet_packets);
11844 	ESTAT_ADD(rx_8192_to_9022_octet_packets);
11845 
11846 	ESTAT_ADD(tx_octets);
11847 	ESTAT_ADD(tx_collisions);
11848 	ESTAT_ADD(tx_xon_sent);
11849 	ESTAT_ADD(tx_xoff_sent);
11850 	ESTAT_ADD(tx_flow_control);
11851 	ESTAT_ADD(tx_mac_errors);
11852 	ESTAT_ADD(tx_single_collisions);
11853 	ESTAT_ADD(tx_mult_collisions);
11854 	ESTAT_ADD(tx_deferred);
11855 	ESTAT_ADD(tx_excessive_collisions);
11856 	ESTAT_ADD(tx_late_collisions);
11857 	ESTAT_ADD(tx_collide_2times);
11858 	ESTAT_ADD(tx_collide_3times);
11859 	ESTAT_ADD(tx_collide_4times);
11860 	ESTAT_ADD(tx_collide_5times);
11861 	ESTAT_ADD(tx_collide_6times);
11862 	ESTAT_ADD(tx_collide_7times);
11863 	ESTAT_ADD(tx_collide_8times);
11864 	ESTAT_ADD(tx_collide_9times);
11865 	ESTAT_ADD(tx_collide_10times);
11866 	ESTAT_ADD(tx_collide_11times);
11867 	ESTAT_ADD(tx_collide_12times);
11868 	ESTAT_ADD(tx_collide_13times);
11869 	ESTAT_ADD(tx_collide_14times);
11870 	ESTAT_ADD(tx_collide_15times);
11871 	ESTAT_ADD(tx_ucast_packets);
11872 	ESTAT_ADD(tx_mcast_packets);
11873 	ESTAT_ADD(tx_bcast_packets);
11874 	ESTAT_ADD(tx_carrier_sense_errors);
11875 	ESTAT_ADD(tx_discards);
11876 	ESTAT_ADD(tx_errors);
11877 
11878 	ESTAT_ADD(dma_writeq_full);
11879 	ESTAT_ADD(dma_write_prioq_full);
11880 	ESTAT_ADD(rxbds_empty);
11881 	ESTAT_ADD(rx_discards);
11882 	ESTAT_ADD(rx_errors);
11883 	ESTAT_ADD(rx_threshold_hit);
11884 
11885 	ESTAT_ADD(dma_readq_full);
11886 	ESTAT_ADD(dma_read_prioq_full);
11887 	ESTAT_ADD(tx_comp_queue_full);
11888 
11889 	ESTAT_ADD(ring_set_send_prod_index);
11890 	ESTAT_ADD(ring_status_update);
11891 	ESTAT_ADD(nic_irqs);
11892 	ESTAT_ADD(nic_avoided_irqs);
11893 	ESTAT_ADD(nic_tx_threshold_hit);
11894 
11895 	ESTAT_ADD(mbuf_lwm_thresh_hit);
11896 }
11897 
11898 static void tg3_get_nstats(struct tg3 *tp, struct rtnl_link_stats64 *stats)
11899 {
11900 	struct rtnl_link_stats64 *old_stats = &tp->net_stats_prev;
11901 	struct tg3_hw_stats *hw_stats = tp->hw_stats;
11902 
11903 	stats->rx_packets = old_stats->rx_packets +
11904 		get_stat64(&hw_stats->rx_ucast_packets) +
11905 		get_stat64(&hw_stats->rx_mcast_packets) +
11906 		get_stat64(&hw_stats->rx_bcast_packets);
11907 
11908 	stats->tx_packets = old_stats->tx_packets +
11909 		get_stat64(&hw_stats->tx_ucast_packets) +
11910 		get_stat64(&hw_stats->tx_mcast_packets) +
11911 		get_stat64(&hw_stats->tx_bcast_packets);
11912 
11913 	stats->rx_bytes = old_stats->rx_bytes +
11914 		get_stat64(&hw_stats->rx_octets);
11915 	stats->tx_bytes = old_stats->tx_bytes +
11916 		get_stat64(&hw_stats->tx_octets);
11917 
11918 	stats->rx_errors = old_stats->rx_errors +
11919 		get_stat64(&hw_stats->rx_errors);
11920 	stats->tx_errors = old_stats->tx_errors +
11921 		get_stat64(&hw_stats->tx_errors) +
11922 		get_stat64(&hw_stats->tx_mac_errors) +
11923 		get_stat64(&hw_stats->tx_carrier_sense_errors) +
11924 		get_stat64(&hw_stats->tx_discards);
11925 
11926 	stats->multicast = old_stats->multicast +
11927 		get_stat64(&hw_stats->rx_mcast_packets);
11928 	stats->collisions = old_stats->collisions +
11929 		get_stat64(&hw_stats->tx_collisions);
11930 
11931 	stats->rx_length_errors = old_stats->rx_length_errors +
11932 		get_stat64(&hw_stats->rx_frame_too_long_errors) +
11933 		get_stat64(&hw_stats->rx_undersize_packets);
11934 
11935 	stats->rx_frame_errors = old_stats->rx_frame_errors +
11936 		get_stat64(&hw_stats->rx_align_errors);
11937 	stats->tx_aborted_errors = old_stats->tx_aborted_errors +
11938 		get_stat64(&hw_stats->tx_discards);
11939 	stats->tx_carrier_errors = old_stats->tx_carrier_errors +
11940 		get_stat64(&hw_stats->tx_carrier_sense_errors);
11941 
11942 	stats->rx_crc_errors = old_stats->rx_crc_errors +
11943 		tg3_calc_crc_errors(tp);
11944 
11945 	stats->rx_missed_errors = old_stats->rx_missed_errors +
11946 		get_stat64(&hw_stats->rx_discards);
11947 
11948 	stats->rx_dropped = tp->rx_dropped;
11949 	stats->tx_dropped = tp->tx_dropped;
11950 }
11951 
11952 static int tg3_get_regs_len(struct net_device *dev)
11953 {
11954 	return TG3_REG_BLK_SIZE;
11955 }
11956 
11957 static void tg3_get_regs(struct net_device *dev,
11958 		struct ethtool_regs *regs, void *_p)
11959 {
11960 	struct tg3 *tp = netdev_priv(dev);
11961 
11962 	regs->version = 0;
11963 
11964 	memset(_p, 0, TG3_REG_BLK_SIZE);
11965 
11966 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
11967 		return;
11968 
11969 	tg3_full_lock(tp, 0);
11970 
11971 	tg3_dump_legacy_regs(tp, (u32 *)_p);
11972 
11973 	tg3_full_unlock(tp);
11974 }
11975 
11976 static int tg3_get_eeprom_len(struct net_device *dev)
11977 {
11978 	struct tg3 *tp = netdev_priv(dev);
11979 
11980 	return tp->nvram_size;
11981 }
11982 
11983 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
11984 {
11985 	struct tg3 *tp = netdev_priv(dev);
11986 	int ret, cpmu_restore = 0;
11987 	u8  *pd;
11988 	u32 i, offset, len, b_offset, b_count, cpmu_val = 0;
11989 	__be32 val;
11990 
11991 	if (tg3_flag(tp, NO_NVRAM))
11992 		return -EINVAL;
11993 
11994 	offset = eeprom->offset;
11995 	len = eeprom->len;
11996 	eeprom->len = 0;
11997 
11998 	eeprom->magic = TG3_EEPROM_MAGIC;
11999 
12000 	/* Override clock, link aware and link idle modes */
12001 	if (tg3_flag(tp, CPMU_PRESENT)) {
12002 		cpmu_val = tr32(TG3_CPMU_CTRL);
12003 		if (cpmu_val & (CPMU_CTRL_LINK_AWARE_MODE |
12004 				CPMU_CTRL_LINK_IDLE_MODE)) {
12005 			tw32(TG3_CPMU_CTRL, cpmu_val &
12006 					    ~(CPMU_CTRL_LINK_AWARE_MODE |
12007 					     CPMU_CTRL_LINK_IDLE_MODE));
12008 			cpmu_restore = 1;
12009 		}
12010 	}
12011 	tg3_override_clk(tp);
12012 
12013 	if (offset & 3) {
12014 		/* adjustments to start on required 4 byte boundary */
12015 		b_offset = offset & 3;
12016 		b_count = 4 - b_offset;
12017 		if (b_count > len) {
12018 			/* i.e. offset=1 len=2 */
12019 			b_count = len;
12020 		}
12021 		ret = tg3_nvram_read_be32(tp, offset-b_offset, &val);
12022 		if (ret)
12023 			goto eeprom_done;
12024 		memcpy(data, ((char *)&val) + b_offset, b_count);
12025 		len -= b_count;
12026 		offset += b_count;
12027 		eeprom->len += b_count;
12028 	}
12029 
12030 	/* read bytes up to the last 4 byte boundary */
12031 	pd = &data[eeprom->len];
12032 	for (i = 0; i < (len - (len & 3)); i += 4) {
12033 		ret = tg3_nvram_read_be32(tp, offset + i, &val);
12034 		if (ret) {
12035 			if (i)
12036 				i -= 4;
12037 			eeprom->len += i;
12038 			goto eeprom_done;
12039 		}
12040 		memcpy(pd + i, &val, 4);
12041 		if (need_resched()) {
12042 			if (signal_pending(current)) {
12043 				eeprom->len += i;
12044 				ret = -EINTR;
12045 				goto eeprom_done;
12046 			}
12047 			cond_resched();
12048 		}
12049 	}
12050 	eeprom->len += i;
12051 
12052 	if (len & 3) {
12053 		/* read last bytes not ending on 4 byte boundary */
12054 		pd = &data[eeprom->len];
12055 		b_count = len & 3;
12056 		b_offset = offset + len - b_count;
12057 		ret = tg3_nvram_read_be32(tp, b_offset, &val);
12058 		if (ret)
12059 			goto eeprom_done;
12060 		memcpy(pd, &val, b_count);
12061 		eeprom->len += b_count;
12062 	}
12063 	ret = 0;
12064 
12065 eeprom_done:
12066 	/* Restore clock, link aware and link idle modes */
12067 	tg3_restore_clk(tp);
12068 	if (cpmu_restore)
12069 		tw32(TG3_CPMU_CTRL, cpmu_val);
12070 
12071 	return ret;
12072 }
12073 
12074 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
12075 {
12076 	struct tg3 *tp = netdev_priv(dev);
12077 	int ret;
12078 	u32 offset, len, b_offset, odd_len;
12079 	u8 *buf;
12080 	__be32 start = 0, end;
12081 
12082 	if (tg3_flag(tp, NO_NVRAM) ||
12083 	    eeprom->magic != TG3_EEPROM_MAGIC)
12084 		return -EINVAL;
12085 
12086 	offset = eeprom->offset;
12087 	len = eeprom->len;
12088 
12089 	if ((b_offset = (offset & 3))) {
12090 		/* adjustments to start on required 4 byte boundary */
12091 		ret = tg3_nvram_read_be32(tp, offset-b_offset, &start);
12092 		if (ret)
12093 			return ret;
12094 		len += b_offset;
12095 		offset &= ~3;
12096 		if (len < 4)
12097 			len = 4;
12098 	}
12099 
12100 	odd_len = 0;
12101 	if (len & 3) {
12102 		/* adjustments to end on required 4 byte boundary */
12103 		odd_len = 1;
12104 		len = (len + 3) & ~3;
12105 		ret = tg3_nvram_read_be32(tp, offset+len-4, &end);
12106 		if (ret)
12107 			return ret;
12108 	}
12109 
12110 	buf = data;
12111 	if (b_offset || odd_len) {
12112 		buf = kmalloc(len, GFP_KERNEL);
12113 		if (!buf)
12114 			return -ENOMEM;
12115 		if (b_offset)
12116 			memcpy(buf, &start, 4);
12117 		if (odd_len)
12118 			memcpy(buf+len-4, &end, 4);
12119 		memcpy(buf + b_offset, data, eeprom->len);
12120 	}
12121 
12122 	ret = tg3_nvram_write_block(tp, offset, len, buf);
12123 
12124 	if (buf != data)
12125 		kfree(buf);
12126 
12127 	return ret;
12128 }
12129 
12130 static int tg3_get_link_ksettings(struct net_device *dev,
12131 				  struct ethtool_link_ksettings *cmd)
12132 {
12133 	struct tg3 *tp = netdev_priv(dev);
12134 	u32 supported, advertising;
12135 
12136 	if (tg3_flag(tp, USE_PHYLIB)) {
12137 		struct phy_device *phydev;
12138 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
12139 			return -EAGAIN;
12140 		phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
12141 		phy_ethtool_ksettings_get(phydev, cmd);
12142 
12143 		return 0;
12144 	}
12145 
12146 	supported = (SUPPORTED_Autoneg);
12147 
12148 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12149 		supported |= (SUPPORTED_1000baseT_Half |
12150 			      SUPPORTED_1000baseT_Full);
12151 
12152 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
12153 		supported |= (SUPPORTED_100baseT_Half |
12154 			      SUPPORTED_100baseT_Full |
12155 			      SUPPORTED_10baseT_Half |
12156 			      SUPPORTED_10baseT_Full |
12157 			      SUPPORTED_TP);
12158 		cmd->base.port = PORT_TP;
12159 	} else {
12160 		supported |= SUPPORTED_FIBRE;
12161 		cmd->base.port = PORT_FIBRE;
12162 	}
12163 	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
12164 						supported);
12165 
12166 	advertising = tp->link_config.advertising;
12167 	if (tg3_flag(tp, PAUSE_AUTONEG)) {
12168 		if (tp->link_config.flowctrl & FLOW_CTRL_RX) {
12169 			if (tp->link_config.flowctrl & FLOW_CTRL_TX) {
12170 				advertising |= ADVERTISED_Pause;
12171 			} else {
12172 				advertising |= ADVERTISED_Pause |
12173 					ADVERTISED_Asym_Pause;
12174 			}
12175 		} else if (tp->link_config.flowctrl & FLOW_CTRL_TX) {
12176 			advertising |= ADVERTISED_Asym_Pause;
12177 		}
12178 	}
12179 	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
12180 						advertising);
12181 
12182 	if (netif_running(dev) && tp->link_up) {
12183 		cmd->base.speed = tp->link_config.active_speed;
12184 		cmd->base.duplex = tp->link_config.active_duplex;
12185 		ethtool_convert_legacy_u32_to_link_mode(
12186 			cmd->link_modes.lp_advertising,
12187 			tp->link_config.rmt_adv);
12188 
12189 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
12190 			if (tp->phy_flags & TG3_PHYFLG_MDIX_STATE)
12191 				cmd->base.eth_tp_mdix = ETH_TP_MDI_X;
12192 			else
12193 				cmd->base.eth_tp_mdix = ETH_TP_MDI;
12194 		}
12195 	} else {
12196 		cmd->base.speed = SPEED_UNKNOWN;
12197 		cmd->base.duplex = DUPLEX_UNKNOWN;
12198 		cmd->base.eth_tp_mdix = ETH_TP_MDI_INVALID;
12199 	}
12200 	cmd->base.phy_address = tp->phy_addr;
12201 	cmd->base.autoneg = tp->link_config.autoneg;
12202 	return 0;
12203 }
12204 
12205 static int tg3_set_link_ksettings(struct net_device *dev,
12206 				  const struct ethtool_link_ksettings *cmd)
12207 {
12208 	struct tg3 *tp = netdev_priv(dev);
12209 	u32 speed = cmd->base.speed;
12210 	u32 advertising;
12211 
12212 	if (tg3_flag(tp, USE_PHYLIB)) {
12213 		struct phy_device *phydev;
12214 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
12215 			return -EAGAIN;
12216 		phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
12217 		return phy_ethtool_ksettings_set(phydev, cmd);
12218 	}
12219 
12220 	if (cmd->base.autoneg != AUTONEG_ENABLE &&
12221 	    cmd->base.autoneg != AUTONEG_DISABLE)
12222 		return -EINVAL;
12223 
12224 	if (cmd->base.autoneg == AUTONEG_DISABLE &&
12225 	    cmd->base.duplex != DUPLEX_FULL &&
12226 	    cmd->base.duplex != DUPLEX_HALF)
12227 		return -EINVAL;
12228 
12229 	ethtool_convert_link_mode_to_legacy_u32(&advertising,
12230 						cmd->link_modes.advertising);
12231 
12232 	if (cmd->base.autoneg == AUTONEG_ENABLE) {
12233 		u32 mask = ADVERTISED_Autoneg |
12234 			   ADVERTISED_Pause |
12235 			   ADVERTISED_Asym_Pause;
12236 
12237 		if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12238 			mask |= ADVERTISED_1000baseT_Half |
12239 				ADVERTISED_1000baseT_Full;
12240 
12241 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
12242 			mask |= ADVERTISED_100baseT_Half |
12243 				ADVERTISED_100baseT_Full |
12244 				ADVERTISED_10baseT_Half |
12245 				ADVERTISED_10baseT_Full |
12246 				ADVERTISED_TP;
12247 		else
12248 			mask |= ADVERTISED_FIBRE;
12249 
12250 		if (advertising & ~mask)
12251 			return -EINVAL;
12252 
12253 		mask &= (ADVERTISED_1000baseT_Half |
12254 			 ADVERTISED_1000baseT_Full |
12255 			 ADVERTISED_100baseT_Half |
12256 			 ADVERTISED_100baseT_Full |
12257 			 ADVERTISED_10baseT_Half |
12258 			 ADVERTISED_10baseT_Full);
12259 
12260 		advertising &= mask;
12261 	} else {
12262 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES) {
12263 			if (speed != SPEED_1000)
12264 				return -EINVAL;
12265 
12266 			if (cmd->base.duplex != DUPLEX_FULL)
12267 				return -EINVAL;
12268 		} else {
12269 			if (speed != SPEED_100 &&
12270 			    speed != SPEED_10)
12271 				return -EINVAL;
12272 		}
12273 	}
12274 
12275 	tg3_full_lock(tp, 0);
12276 
12277 	tp->link_config.autoneg = cmd->base.autoneg;
12278 	if (cmd->base.autoneg == AUTONEG_ENABLE) {
12279 		tp->link_config.advertising = (advertising |
12280 					      ADVERTISED_Autoneg);
12281 		tp->link_config.speed = SPEED_UNKNOWN;
12282 		tp->link_config.duplex = DUPLEX_UNKNOWN;
12283 	} else {
12284 		tp->link_config.advertising = 0;
12285 		tp->link_config.speed = speed;
12286 		tp->link_config.duplex = cmd->base.duplex;
12287 	}
12288 
12289 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
12290 
12291 	tg3_warn_mgmt_link_flap(tp);
12292 
12293 	if (netif_running(dev))
12294 		tg3_setup_phy(tp, true);
12295 
12296 	tg3_full_unlock(tp);
12297 
12298 	return 0;
12299 }
12300 
12301 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
12302 {
12303 	struct tg3 *tp = netdev_priv(dev);
12304 
12305 	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
12306 	strlcpy(info->fw_version, tp->fw_ver, sizeof(info->fw_version));
12307 	strlcpy(info->bus_info, pci_name(tp->pdev), sizeof(info->bus_info));
12308 }
12309 
12310 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
12311 {
12312 	struct tg3 *tp = netdev_priv(dev);
12313 
12314 	if (tg3_flag(tp, WOL_CAP) && device_can_wakeup(&tp->pdev->dev))
12315 		wol->supported = WAKE_MAGIC;
12316 	else
12317 		wol->supported = 0;
12318 	wol->wolopts = 0;
12319 	if (tg3_flag(tp, WOL_ENABLE) && device_can_wakeup(&tp->pdev->dev))
12320 		wol->wolopts = WAKE_MAGIC;
12321 	memset(&wol->sopass, 0, sizeof(wol->sopass));
12322 }
12323 
12324 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
12325 {
12326 	struct tg3 *tp = netdev_priv(dev);
12327 	struct device *dp = &tp->pdev->dev;
12328 
12329 	if (wol->wolopts & ~WAKE_MAGIC)
12330 		return -EINVAL;
12331 	if ((wol->wolopts & WAKE_MAGIC) &&
12332 	    !(tg3_flag(tp, WOL_CAP) && device_can_wakeup(dp)))
12333 		return -EINVAL;
12334 
12335 	device_set_wakeup_enable(dp, wol->wolopts & WAKE_MAGIC);
12336 
12337 	if (device_may_wakeup(dp))
12338 		tg3_flag_set(tp, WOL_ENABLE);
12339 	else
12340 		tg3_flag_clear(tp, WOL_ENABLE);
12341 
12342 	return 0;
12343 }
12344 
12345 static u32 tg3_get_msglevel(struct net_device *dev)
12346 {
12347 	struct tg3 *tp = netdev_priv(dev);
12348 	return tp->msg_enable;
12349 }
12350 
12351 static void tg3_set_msglevel(struct net_device *dev, u32 value)
12352 {
12353 	struct tg3 *tp = netdev_priv(dev);
12354 	tp->msg_enable = value;
12355 }
12356 
12357 static int tg3_nway_reset(struct net_device *dev)
12358 {
12359 	struct tg3 *tp = netdev_priv(dev);
12360 	int r;
12361 
12362 	if (!netif_running(dev))
12363 		return -EAGAIN;
12364 
12365 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
12366 		return -EINVAL;
12367 
12368 	tg3_warn_mgmt_link_flap(tp);
12369 
12370 	if (tg3_flag(tp, USE_PHYLIB)) {
12371 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
12372 			return -EAGAIN;
12373 		r = phy_start_aneg(mdiobus_get_phy(tp->mdio_bus, tp->phy_addr));
12374 	} else {
12375 		u32 bmcr;
12376 
12377 		spin_lock_bh(&tp->lock);
12378 		r = -EINVAL;
12379 		tg3_readphy(tp, MII_BMCR, &bmcr);
12380 		if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
12381 		    ((bmcr & BMCR_ANENABLE) ||
12382 		     (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT))) {
12383 			tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
12384 						   BMCR_ANENABLE);
12385 			r = 0;
12386 		}
12387 		spin_unlock_bh(&tp->lock);
12388 	}
12389 
12390 	return r;
12391 }
12392 
12393 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
12394 {
12395 	struct tg3 *tp = netdev_priv(dev);
12396 
12397 	ering->rx_max_pending = tp->rx_std_ring_mask;
12398 	if (tg3_flag(tp, JUMBO_RING_ENABLE))
12399 		ering->rx_jumbo_max_pending = tp->rx_jmb_ring_mask;
12400 	else
12401 		ering->rx_jumbo_max_pending = 0;
12402 
12403 	ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
12404 
12405 	ering->rx_pending = tp->rx_pending;
12406 	if (tg3_flag(tp, JUMBO_RING_ENABLE))
12407 		ering->rx_jumbo_pending = tp->rx_jumbo_pending;
12408 	else
12409 		ering->rx_jumbo_pending = 0;
12410 
12411 	ering->tx_pending = tp->napi[0].tx_pending;
12412 }
12413 
12414 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
12415 {
12416 	struct tg3 *tp = netdev_priv(dev);
12417 	int i, irq_sync = 0, err = 0;
12418 	bool reset_phy = false;
12419 
12420 	if ((ering->rx_pending > tp->rx_std_ring_mask) ||
12421 	    (ering->rx_jumbo_pending > tp->rx_jmb_ring_mask) ||
12422 	    (ering->tx_pending > TG3_TX_RING_SIZE - 1) ||
12423 	    (ering->tx_pending <= MAX_SKB_FRAGS) ||
12424 	    (tg3_flag(tp, TSO_BUG) &&
12425 	     (ering->tx_pending <= (MAX_SKB_FRAGS * 3))))
12426 		return -EINVAL;
12427 
12428 	if (netif_running(dev)) {
12429 		tg3_phy_stop(tp);
12430 		tg3_netif_stop(tp);
12431 		irq_sync = 1;
12432 	}
12433 
12434 	tg3_full_lock(tp, irq_sync);
12435 
12436 	tp->rx_pending = ering->rx_pending;
12437 
12438 	if (tg3_flag(tp, MAX_RXPEND_64) &&
12439 	    tp->rx_pending > 63)
12440 		tp->rx_pending = 63;
12441 
12442 	if (tg3_flag(tp, JUMBO_RING_ENABLE))
12443 		tp->rx_jumbo_pending = ering->rx_jumbo_pending;
12444 
12445 	for (i = 0; i < tp->irq_max; i++)
12446 		tp->napi[i].tx_pending = ering->tx_pending;
12447 
12448 	if (netif_running(dev)) {
12449 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12450 		/* Reset PHY to avoid PHY lock up */
12451 		if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
12452 		    tg3_asic_rev(tp) == ASIC_REV_5719 ||
12453 		    tg3_asic_rev(tp) == ASIC_REV_5720)
12454 			reset_phy = true;
12455 
12456 		err = tg3_restart_hw(tp, reset_phy);
12457 		if (!err)
12458 			tg3_netif_start(tp);
12459 	}
12460 
12461 	tg3_full_unlock(tp);
12462 
12463 	if (irq_sync && !err)
12464 		tg3_phy_start(tp);
12465 
12466 	return err;
12467 }
12468 
12469 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
12470 {
12471 	struct tg3 *tp = netdev_priv(dev);
12472 
12473 	epause->autoneg = !!tg3_flag(tp, PAUSE_AUTONEG);
12474 
12475 	if (tp->link_config.flowctrl & FLOW_CTRL_RX)
12476 		epause->rx_pause = 1;
12477 	else
12478 		epause->rx_pause = 0;
12479 
12480 	if (tp->link_config.flowctrl & FLOW_CTRL_TX)
12481 		epause->tx_pause = 1;
12482 	else
12483 		epause->tx_pause = 0;
12484 }
12485 
12486 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
12487 {
12488 	struct tg3 *tp = netdev_priv(dev);
12489 	int err = 0;
12490 	bool reset_phy = false;
12491 
12492 	if (tp->link_config.autoneg == AUTONEG_ENABLE)
12493 		tg3_warn_mgmt_link_flap(tp);
12494 
12495 	if (tg3_flag(tp, USE_PHYLIB)) {
12496 		struct phy_device *phydev;
12497 
12498 		phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
12499 
12500 		if (!phy_validate_pause(phydev, epause))
12501 			return -EINVAL;
12502 
12503 		tp->link_config.flowctrl = 0;
12504 		phy_set_asym_pause(phydev, epause->rx_pause, epause->tx_pause);
12505 		if (epause->rx_pause) {
12506 			tp->link_config.flowctrl |= FLOW_CTRL_RX;
12507 
12508 			if (epause->tx_pause) {
12509 				tp->link_config.flowctrl |= FLOW_CTRL_TX;
12510 			}
12511 		} else if (epause->tx_pause) {
12512 			tp->link_config.flowctrl |= FLOW_CTRL_TX;
12513 		}
12514 
12515 		if (epause->autoneg)
12516 			tg3_flag_set(tp, PAUSE_AUTONEG);
12517 		else
12518 			tg3_flag_clear(tp, PAUSE_AUTONEG);
12519 
12520 		if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
12521 			if (phydev->autoneg) {
12522 				/* phy_set_asym_pause() will
12523 				 * renegotiate the link to inform our
12524 				 * link partner of our flow control
12525 				 * settings, even if the flow control
12526 				 * is forced.  Let tg3_adjust_link()
12527 				 * do the final flow control setup.
12528 				 */
12529 				return 0;
12530 			}
12531 
12532 			if (!epause->autoneg)
12533 				tg3_setup_flow_control(tp, 0, 0);
12534 		}
12535 	} else {
12536 		int irq_sync = 0;
12537 
12538 		if (netif_running(dev)) {
12539 			tg3_netif_stop(tp);
12540 			irq_sync = 1;
12541 		}
12542 
12543 		tg3_full_lock(tp, irq_sync);
12544 
12545 		if (epause->autoneg)
12546 			tg3_flag_set(tp, PAUSE_AUTONEG);
12547 		else
12548 			tg3_flag_clear(tp, PAUSE_AUTONEG);
12549 		if (epause->rx_pause)
12550 			tp->link_config.flowctrl |= FLOW_CTRL_RX;
12551 		else
12552 			tp->link_config.flowctrl &= ~FLOW_CTRL_RX;
12553 		if (epause->tx_pause)
12554 			tp->link_config.flowctrl |= FLOW_CTRL_TX;
12555 		else
12556 			tp->link_config.flowctrl &= ~FLOW_CTRL_TX;
12557 
12558 		if (netif_running(dev)) {
12559 			tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
12560 			/* Reset PHY to avoid PHY lock up */
12561 			if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
12562 			    tg3_asic_rev(tp) == ASIC_REV_5719 ||
12563 			    tg3_asic_rev(tp) == ASIC_REV_5720)
12564 				reset_phy = true;
12565 
12566 			err = tg3_restart_hw(tp, reset_phy);
12567 			if (!err)
12568 				tg3_netif_start(tp);
12569 		}
12570 
12571 		tg3_full_unlock(tp);
12572 	}
12573 
12574 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
12575 
12576 	return err;
12577 }
12578 
12579 static int tg3_get_sset_count(struct net_device *dev, int sset)
12580 {
12581 	switch (sset) {
12582 	case ETH_SS_TEST:
12583 		return TG3_NUM_TEST;
12584 	case ETH_SS_STATS:
12585 		return TG3_NUM_STATS;
12586 	default:
12587 		return -EOPNOTSUPP;
12588 	}
12589 }
12590 
12591 static int tg3_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
12592 			 u32 *rules __always_unused)
12593 {
12594 	struct tg3 *tp = netdev_priv(dev);
12595 
12596 	if (!tg3_flag(tp, SUPPORT_MSIX))
12597 		return -EOPNOTSUPP;
12598 
12599 	switch (info->cmd) {
12600 	case ETHTOOL_GRXRINGS:
12601 		if (netif_running(tp->dev))
12602 			info->data = tp->rxq_cnt;
12603 		else {
12604 			info->data = num_online_cpus();
12605 			if (info->data > TG3_RSS_MAX_NUM_QS)
12606 				info->data = TG3_RSS_MAX_NUM_QS;
12607 		}
12608 
12609 		return 0;
12610 
12611 	default:
12612 		return -EOPNOTSUPP;
12613 	}
12614 }
12615 
12616 static u32 tg3_get_rxfh_indir_size(struct net_device *dev)
12617 {
12618 	u32 size = 0;
12619 	struct tg3 *tp = netdev_priv(dev);
12620 
12621 	if (tg3_flag(tp, SUPPORT_MSIX))
12622 		size = TG3_RSS_INDIR_TBL_SIZE;
12623 
12624 	return size;
12625 }
12626 
12627 static int tg3_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc)
12628 {
12629 	struct tg3 *tp = netdev_priv(dev);
12630 	int i;
12631 
12632 	if (hfunc)
12633 		*hfunc = ETH_RSS_HASH_TOP;
12634 	if (!indir)
12635 		return 0;
12636 
12637 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
12638 		indir[i] = tp->rss_ind_tbl[i];
12639 
12640 	return 0;
12641 }
12642 
12643 static int tg3_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key,
12644 			const u8 hfunc)
12645 {
12646 	struct tg3 *tp = netdev_priv(dev);
12647 	size_t i;
12648 
12649 	/* We require at least one supported parameter to be changed and no
12650 	 * change in any of the unsupported parameters
12651 	 */
12652 	if (key ||
12653 	    (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
12654 		return -EOPNOTSUPP;
12655 
12656 	if (!indir)
12657 		return 0;
12658 
12659 	for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++)
12660 		tp->rss_ind_tbl[i] = indir[i];
12661 
12662 	if (!netif_running(dev) || !tg3_flag(tp, ENABLE_RSS))
12663 		return 0;
12664 
12665 	/* It is legal to write the indirection
12666 	 * table while the device is running.
12667 	 */
12668 	tg3_full_lock(tp, 0);
12669 	tg3_rss_write_indir_tbl(tp);
12670 	tg3_full_unlock(tp);
12671 
12672 	return 0;
12673 }
12674 
12675 static void tg3_get_channels(struct net_device *dev,
12676 			     struct ethtool_channels *channel)
12677 {
12678 	struct tg3 *tp = netdev_priv(dev);
12679 	u32 deflt_qs = netif_get_num_default_rss_queues();
12680 
12681 	channel->max_rx = tp->rxq_max;
12682 	channel->max_tx = tp->txq_max;
12683 
12684 	if (netif_running(dev)) {
12685 		channel->rx_count = tp->rxq_cnt;
12686 		channel->tx_count = tp->txq_cnt;
12687 	} else {
12688 		if (tp->rxq_req)
12689 			channel->rx_count = tp->rxq_req;
12690 		else
12691 			channel->rx_count = min(deflt_qs, tp->rxq_max);
12692 
12693 		if (tp->txq_req)
12694 			channel->tx_count = tp->txq_req;
12695 		else
12696 			channel->tx_count = min(deflt_qs, tp->txq_max);
12697 	}
12698 }
12699 
12700 static int tg3_set_channels(struct net_device *dev,
12701 			    struct ethtool_channels *channel)
12702 {
12703 	struct tg3 *tp = netdev_priv(dev);
12704 
12705 	if (!tg3_flag(tp, SUPPORT_MSIX))
12706 		return -EOPNOTSUPP;
12707 
12708 	if (channel->rx_count > tp->rxq_max ||
12709 	    channel->tx_count > tp->txq_max)
12710 		return -EINVAL;
12711 
12712 	tp->rxq_req = channel->rx_count;
12713 	tp->txq_req = channel->tx_count;
12714 
12715 	if (!netif_running(dev))
12716 		return 0;
12717 
12718 	tg3_stop(tp);
12719 
12720 	tg3_carrier_off(tp);
12721 
12722 	tg3_start(tp, true, false, false);
12723 
12724 	return 0;
12725 }
12726 
12727 static void tg3_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
12728 {
12729 	switch (stringset) {
12730 	case ETH_SS_STATS:
12731 		memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
12732 		break;
12733 	case ETH_SS_TEST:
12734 		memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
12735 		break;
12736 	default:
12737 		WARN_ON(1);	/* we need a WARN() */
12738 		break;
12739 	}
12740 }
12741 
12742 static int tg3_set_phys_id(struct net_device *dev,
12743 			    enum ethtool_phys_id_state state)
12744 {
12745 	struct tg3 *tp = netdev_priv(dev);
12746 
12747 	switch (state) {
12748 	case ETHTOOL_ID_ACTIVE:
12749 		return 1;	/* cycle on/off once per second */
12750 
12751 	case ETHTOOL_ID_ON:
12752 		tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
12753 		     LED_CTRL_1000MBPS_ON |
12754 		     LED_CTRL_100MBPS_ON |
12755 		     LED_CTRL_10MBPS_ON |
12756 		     LED_CTRL_TRAFFIC_OVERRIDE |
12757 		     LED_CTRL_TRAFFIC_BLINK |
12758 		     LED_CTRL_TRAFFIC_LED);
12759 		break;
12760 
12761 	case ETHTOOL_ID_OFF:
12762 		tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
12763 		     LED_CTRL_TRAFFIC_OVERRIDE);
12764 		break;
12765 
12766 	case ETHTOOL_ID_INACTIVE:
12767 		tw32(MAC_LED_CTRL, tp->led_ctrl);
12768 		break;
12769 	}
12770 
12771 	return 0;
12772 }
12773 
12774 static void tg3_get_ethtool_stats(struct net_device *dev,
12775 				   struct ethtool_stats *estats, u64 *tmp_stats)
12776 {
12777 	struct tg3 *tp = netdev_priv(dev);
12778 
12779 	if (tp->hw_stats)
12780 		tg3_get_estats(tp, (struct tg3_ethtool_stats *)tmp_stats);
12781 	else
12782 		memset(tmp_stats, 0, sizeof(struct tg3_ethtool_stats));
12783 }
12784 
12785 static __be32 *tg3_vpd_readblock(struct tg3 *tp, unsigned int *vpdlen)
12786 {
12787 	int i;
12788 	__be32 *buf;
12789 	u32 offset = 0, len = 0;
12790 	u32 magic, val;
12791 
12792 	if (tg3_flag(tp, NO_NVRAM) || tg3_nvram_read(tp, 0, &magic))
12793 		return NULL;
12794 
12795 	if (magic == TG3_EEPROM_MAGIC) {
12796 		for (offset = TG3_NVM_DIR_START;
12797 		     offset < TG3_NVM_DIR_END;
12798 		     offset += TG3_NVM_DIRENT_SIZE) {
12799 			if (tg3_nvram_read(tp, offset, &val))
12800 				return NULL;
12801 
12802 			if ((val >> TG3_NVM_DIRTYPE_SHIFT) ==
12803 			    TG3_NVM_DIRTYPE_EXTVPD)
12804 				break;
12805 		}
12806 
12807 		if (offset != TG3_NVM_DIR_END) {
12808 			len = (val & TG3_NVM_DIRTYPE_LENMSK) * 4;
12809 			if (tg3_nvram_read(tp, offset + 4, &offset))
12810 				return NULL;
12811 
12812 			offset = tg3_nvram_logical_addr(tp, offset);
12813 		}
12814 
12815 		if (!offset || !len) {
12816 			offset = TG3_NVM_VPD_OFF;
12817 			len = TG3_NVM_VPD_LEN;
12818 		}
12819 
12820 		buf = kmalloc(len, GFP_KERNEL);
12821 		if (!buf)
12822 			return NULL;
12823 
12824 		for (i = 0; i < len; i += 4) {
12825 			/* The data is in little-endian format in NVRAM.
12826 			 * Use the big-endian read routines to preserve
12827 			 * the byte order as it exists in NVRAM.
12828 			 */
12829 			if (tg3_nvram_read_be32(tp, offset + i, &buf[i/4]))
12830 				goto error;
12831 		}
12832 		*vpdlen = len;
12833 	} else {
12834 		buf = pci_vpd_alloc(tp->pdev, vpdlen);
12835 		if (IS_ERR(buf))
12836 			return NULL;
12837 	}
12838 
12839 	return buf;
12840 
12841 error:
12842 	kfree(buf);
12843 	return NULL;
12844 }
12845 
12846 #define NVRAM_TEST_SIZE 0x100
12847 #define NVRAM_SELFBOOT_FORMAT1_0_SIZE	0x14
12848 #define NVRAM_SELFBOOT_FORMAT1_2_SIZE	0x18
12849 #define NVRAM_SELFBOOT_FORMAT1_3_SIZE	0x1c
12850 #define NVRAM_SELFBOOT_FORMAT1_4_SIZE	0x20
12851 #define NVRAM_SELFBOOT_FORMAT1_5_SIZE	0x24
12852 #define NVRAM_SELFBOOT_FORMAT1_6_SIZE	0x50
12853 #define NVRAM_SELFBOOT_HW_SIZE 0x20
12854 #define NVRAM_SELFBOOT_DATA_SIZE 0x1c
12855 
12856 static int tg3_test_nvram(struct tg3 *tp)
12857 {
12858 	u32 csum, magic;
12859 	__be32 *buf;
12860 	int i, j, k, err = 0, size;
12861 	unsigned int len;
12862 
12863 	if (tg3_flag(tp, NO_NVRAM))
12864 		return 0;
12865 
12866 	if (tg3_nvram_read(tp, 0, &magic) != 0)
12867 		return -EIO;
12868 
12869 	if (magic == TG3_EEPROM_MAGIC)
12870 		size = NVRAM_TEST_SIZE;
12871 	else if ((magic & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW) {
12872 		if ((magic & TG3_EEPROM_SB_FORMAT_MASK) ==
12873 		    TG3_EEPROM_SB_FORMAT_1) {
12874 			switch (magic & TG3_EEPROM_SB_REVISION_MASK) {
12875 			case TG3_EEPROM_SB_REVISION_0:
12876 				size = NVRAM_SELFBOOT_FORMAT1_0_SIZE;
12877 				break;
12878 			case TG3_EEPROM_SB_REVISION_2:
12879 				size = NVRAM_SELFBOOT_FORMAT1_2_SIZE;
12880 				break;
12881 			case TG3_EEPROM_SB_REVISION_3:
12882 				size = NVRAM_SELFBOOT_FORMAT1_3_SIZE;
12883 				break;
12884 			case TG3_EEPROM_SB_REVISION_4:
12885 				size = NVRAM_SELFBOOT_FORMAT1_4_SIZE;
12886 				break;
12887 			case TG3_EEPROM_SB_REVISION_5:
12888 				size = NVRAM_SELFBOOT_FORMAT1_5_SIZE;
12889 				break;
12890 			case TG3_EEPROM_SB_REVISION_6:
12891 				size = NVRAM_SELFBOOT_FORMAT1_6_SIZE;
12892 				break;
12893 			default:
12894 				return -EIO;
12895 			}
12896 		} else
12897 			return 0;
12898 	} else if ((magic & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
12899 		size = NVRAM_SELFBOOT_HW_SIZE;
12900 	else
12901 		return -EIO;
12902 
12903 	buf = kmalloc(size, GFP_KERNEL);
12904 	if (buf == NULL)
12905 		return -ENOMEM;
12906 
12907 	err = -EIO;
12908 	for (i = 0, j = 0; i < size; i += 4, j++) {
12909 		err = tg3_nvram_read_be32(tp, i, &buf[j]);
12910 		if (err)
12911 			break;
12912 	}
12913 	if (i < size)
12914 		goto out;
12915 
12916 	/* Selfboot format */
12917 	magic = be32_to_cpu(buf[0]);
12918 	if ((magic & TG3_EEPROM_MAGIC_FW_MSK) ==
12919 	    TG3_EEPROM_MAGIC_FW) {
12920 		u8 *buf8 = (u8 *) buf, csum8 = 0;
12921 
12922 		if ((magic & TG3_EEPROM_SB_REVISION_MASK) ==
12923 		    TG3_EEPROM_SB_REVISION_2) {
12924 			/* For rev 2, the csum doesn't include the MBA. */
12925 			for (i = 0; i < TG3_EEPROM_SB_F1R2_MBA_OFF; i++)
12926 				csum8 += buf8[i];
12927 			for (i = TG3_EEPROM_SB_F1R2_MBA_OFF + 4; i < size; i++)
12928 				csum8 += buf8[i];
12929 		} else {
12930 			for (i = 0; i < size; i++)
12931 				csum8 += buf8[i];
12932 		}
12933 
12934 		if (csum8 == 0) {
12935 			err = 0;
12936 			goto out;
12937 		}
12938 
12939 		err = -EIO;
12940 		goto out;
12941 	}
12942 
12943 	if ((magic & TG3_EEPROM_MAGIC_HW_MSK) ==
12944 	    TG3_EEPROM_MAGIC_HW) {
12945 		u8 data[NVRAM_SELFBOOT_DATA_SIZE];
12946 		u8 parity[NVRAM_SELFBOOT_DATA_SIZE];
12947 		u8 *buf8 = (u8 *) buf;
12948 
12949 		/* Separate the parity bits and the data bytes.  */
12950 		for (i = 0, j = 0, k = 0; i < NVRAM_SELFBOOT_HW_SIZE; i++) {
12951 			if ((i == 0) || (i == 8)) {
12952 				int l;
12953 				u8 msk;
12954 
12955 				for (l = 0, msk = 0x80; l < 7; l++, msk >>= 1)
12956 					parity[k++] = buf8[i] & msk;
12957 				i++;
12958 			} else if (i == 16) {
12959 				int l;
12960 				u8 msk;
12961 
12962 				for (l = 0, msk = 0x20; l < 6; l++, msk >>= 1)
12963 					parity[k++] = buf8[i] & msk;
12964 				i++;
12965 
12966 				for (l = 0, msk = 0x80; l < 8; l++, msk >>= 1)
12967 					parity[k++] = buf8[i] & msk;
12968 				i++;
12969 			}
12970 			data[j++] = buf8[i];
12971 		}
12972 
12973 		err = -EIO;
12974 		for (i = 0; i < NVRAM_SELFBOOT_DATA_SIZE; i++) {
12975 			u8 hw8 = hweight8(data[i]);
12976 
12977 			if ((hw8 & 0x1) && parity[i])
12978 				goto out;
12979 			else if (!(hw8 & 0x1) && !parity[i])
12980 				goto out;
12981 		}
12982 		err = 0;
12983 		goto out;
12984 	}
12985 
12986 	err = -EIO;
12987 
12988 	/* Bootstrap checksum at offset 0x10 */
12989 	csum = calc_crc((unsigned char *) buf, 0x10);
12990 	if (csum != le32_to_cpu(buf[0x10/4]))
12991 		goto out;
12992 
12993 	/* Manufacturing block starts at offset 0x74, checksum at 0xfc */
12994 	csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
12995 	if (csum != le32_to_cpu(buf[0xfc/4]))
12996 		goto out;
12997 
12998 	kfree(buf);
12999 
13000 	buf = tg3_vpd_readblock(tp, &len);
13001 	if (!buf)
13002 		return -ENOMEM;
13003 
13004 	err = pci_vpd_check_csum(buf, len);
13005 	/* go on if no checksum found */
13006 	if (err == 1)
13007 		err = 0;
13008 out:
13009 	kfree(buf);
13010 	return err;
13011 }
13012 
13013 #define TG3_SERDES_TIMEOUT_SEC	2
13014 #define TG3_COPPER_TIMEOUT_SEC	6
13015 
13016 static int tg3_test_link(struct tg3 *tp)
13017 {
13018 	int i, max;
13019 
13020 	if (!netif_running(tp->dev))
13021 		return -ENODEV;
13022 
13023 	if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
13024 		max = TG3_SERDES_TIMEOUT_SEC;
13025 	else
13026 		max = TG3_COPPER_TIMEOUT_SEC;
13027 
13028 	for (i = 0; i < max; i++) {
13029 		if (tp->link_up)
13030 			return 0;
13031 
13032 		if (msleep_interruptible(1000))
13033 			break;
13034 	}
13035 
13036 	return -EIO;
13037 }
13038 
13039 /* Only test the commonly used registers */
13040 static int tg3_test_registers(struct tg3 *tp)
13041 {
13042 	int i, is_5705, is_5750;
13043 	u32 offset, read_mask, write_mask, val, save_val, read_val;
13044 	static struct {
13045 		u16 offset;
13046 		u16 flags;
13047 #define TG3_FL_5705	0x1
13048 #define TG3_FL_NOT_5705	0x2
13049 #define TG3_FL_NOT_5788	0x4
13050 #define TG3_FL_NOT_5750	0x8
13051 		u32 read_mask;
13052 		u32 write_mask;
13053 	} reg_tbl[] = {
13054 		/* MAC Control Registers */
13055 		{ MAC_MODE, TG3_FL_NOT_5705,
13056 			0x00000000, 0x00ef6f8c },
13057 		{ MAC_MODE, TG3_FL_5705,
13058 			0x00000000, 0x01ef6b8c },
13059 		{ MAC_STATUS, TG3_FL_NOT_5705,
13060 			0x03800107, 0x00000000 },
13061 		{ MAC_STATUS, TG3_FL_5705,
13062 			0x03800100, 0x00000000 },
13063 		{ MAC_ADDR_0_HIGH, 0x0000,
13064 			0x00000000, 0x0000ffff },
13065 		{ MAC_ADDR_0_LOW, 0x0000,
13066 			0x00000000, 0xffffffff },
13067 		{ MAC_RX_MTU_SIZE, 0x0000,
13068 			0x00000000, 0x0000ffff },
13069 		{ MAC_TX_MODE, 0x0000,
13070 			0x00000000, 0x00000070 },
13071 		{ MAC_TX_LENGTHS, 0x0000,
13072 			0x00000000, 0x00003fff },
13073 		{ MAC_RX_MODE, TG3_FL_NOT_5705,
13074 			0x00000000, 0x000007fc },
13075 		{ MAC_RX_MODE, TG3_FL_5705,
13076 			0x00000000, 0x000007dc },
13077 		{ MAC_HASH_REG_0, 0x0000,
13078 			0x00000000, 0xffffffff },
13079 		{ MAC_HASH_REG_1, 0x0000,
13080 			0x00000000, 0xffffffff },
13081 		{ MAC_HASH_REG_2, 0x0000,
13082 			0x00000000, 0xffffffff },
13083 		{ MAC_HASH_REG_3, 0x0000,
13084 			0x00000000, 0xffffffff },
13085 
13086 		/* Receive Data and Receive BD Initiator Control Registers. */
13087 		{ RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
13088 			0x00000000, 0xffffffff },
13089 		{ RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
13090 			0x00000000, 0xffffffff },
13091 		{ RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
13092 			0x00000000, 0x00000003 },
13093 		{ RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
13094 			0x00000000, 0xffffffff },
13095 		{ RCVDBDI_STD_BD+0, 0x0000,
13096 			0x00000000, 0xffffffff },
13097 		{ RCVDBDI_STD_BD+4, 0x0000,
13098 			0x00000000, 0xffffffff },
13099 		{ RCVDBDI_STD_BD+8, 0x0000,
13100 			0x00000000, 0xffff0002 },
13101 		{ RCVDBDI_STD_BD+0xc, 0x0000,
13102 			0x00000000, 0xffffffff },
13103 
13104 		/* Receive BD Initiator Control Registers. */
13105 		{ RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
13106 			0x00000000, 0xffffffff },
13107 		{ RCVBDI_STD_THRESH, TG3_FL_5705,
13108 			0x00000000, 0x000003ff },
13109 		{ RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
13110 			0x00000000, 0xffffffff },
13111 
13112 		/* Host Coalescing Control Registers. */
13113 		{ HOSTCC_MODE, TG3_FL_NOT_5705,
13114 			0x00000000, 0x00000004 },
13115 		{ HOSTCC_MODE, TG3_FL_5705,
13116 			0x00000000, 0x000000f6 },
13117 		{ HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
13118 			0x00000000, 0xffffffff },
13119 		{ HOSTCC_RXCOL_TICKS, TG3_FL_5705,
13120 			0x00000000, 0x000003ff },
13121 		{ HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
13122 			0x00000000, 0xffffffff },
13123 		{ HOSTCC_TXCOL_TICKS, TG3_FL_5705,
13124 			0x00000000, 0x000003ff },
13125 		{ HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
13126 			0x00000000, 0xffffffff },
13127 		{ HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
13128 			0x00000000, 0x000000ff },
13129 		{ HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
13130 			0x00000000, 0xffffffff },
13131 		{ HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
13132 			0x00000000, 0x000000ff },
13133 		{ HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
13134 			0x00000000, 0xffffffff },
13135 		{ HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
13136 			0x00000000, 0xffffffff },
13137 		{ HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
13138 			0x00000000, 0xffffffff },
13139 		{ HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
13140 			0x00000000, 0x000000ff },
13141 		{ HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
13142 			0x00000000, 0xffffffff },
13143 		{ HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
13144 			0x00000000, 0x000000ff },
13145 		{ HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
13146 			0x00000000, 0xffffffff },
13147 		{ HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
13148 			0x00000000, 0xffffffff },
13149 		{ HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
13150 			0x00000000, 0xffffffff },
13151 		{ HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
13152 			0x00000000, 0xffffffff },
13153 		{ HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
13154 			0x00000000, 0xffffffff },
13155 		{ HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
13156 			0xffffffff, 0x00000000 },
13157 		{ HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
13158 			0xffffffff, 0x00000000 },
13159 
13160 		/* Buffer Manager Control Registers. */
13161 		{ BUFMGR_MB_POOL_ADDR, TG3_FL_NOT_5750,
13162 			0x00000000, 0x007fff80 },
13163 		{ BUFMGR_MB_POOL_SIZE, TG3_FL_NOT_5750,
13164 			0x00000000, 0x007fffff },
13165 		{ BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
13166 			0x00000000, 0x0000003f },
13167 		{ BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
13168 			0x00000000, 0x000001ff },
13169 		{ BUFMGR_MB_HIGH_WATER, 0x0000,
13170 			0x00000000, 0x000001ff },
13171 		{ BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
13172 			0xffffffff, 0x00000000 },
13173 		{ BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
13174 			0xffffffff, 0x00000000 },
13175 
13176 		/* Mailbox Registers */
13177 		{ GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
13178 			0x00000000, 0x000001ff },
13179 		{ GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
13180 			0x00000000, 0x000001ff },
13181 		{ GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
13182 			0x00000000, 0x000007ff },
13183 		{ GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
13184 			0x00000000, 0x000001ff },
13185 
13186 		{ 0xffff, 0x0000, 0x00000000, 0x00000000 },
13187 	};
13188 
13189 	is_5705 = is_5750 = 0;
13190 	if (tg3_flag(tp, 5705_PLUS)) {
13191 		is_5705 = 1;
13192 		if (tg3_flag(tp, 5750_PLUS))
13193 			is_5750 = 1;
13194 	}
13195 
13196 	for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
13197 		if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
13198 			continue;
13199 
13200 		if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
13201 			continue;
13202 
13203 		if (tg3_flag(tp, IS_5788) &&
13204 		    (reg_tbl[i].flags & TG3_FL_NOT_5788))
13205 			continue;
13206 
13207 		if (is_5750 && (reg_tbl[i].flags & TG3_FL_NOT_5750))
13208 			continue;
13209 
13210 		offset = (u32) reg_tbl[i].offset;
13211 		read_mask = reg_tbl[i].read_mask;
13212 		write_mask = reg_tbl[i].write_mask;
13213 
13214 		/* Save the original register content */
13215 		save_val = tr32(offset);
13216 
13217 		/* Determine the read-only value. */
13218 		read_val = save_val & read_mask;
13219 
13220 		/* Write zero to the register, then make sure the read-only bits
13221 		 * are not changed and the read/write bits are all zeros.
13222 		 */
13223 		tw32(offset, 0);
13224 
13225 		val = tr32(offset);
13226 
13227 		/* Test the read-only and read/write bits. */
13228 		if (((val & read_mask) != read_val) || (val & write_mask))
13229 			goto out;
13230 
13231 		/* Write ones to all the bits defined by RdMask and WrMask, then
13232 		 * make sure the read-only bits are not changed and the
13233 		 * read/write bits are all ones.
13234 		 */
13235 		tw32(offset, read_mask | write_mask);
13236 
13237 		val = tr32(offset);
13238 
13239 		/* Test the read-only bits. */
13240 		if ((val & read_mask) != read_val)
13241 			goto out;
13242 
13243 		/* Test the read/write bits. */
13244 		if ((val & write_mask) != write_mask)
13245 			goto out;
13246 
13247 		tw32(offset, save_val);
13248 	}
13249 
13250 	return 0;
13251 
13252 out:
13253 	if (netif_msg_hw(tp))
13254 		netdev_err(tp->dev,
13255 			   "Register test failed at offset %x\n", offset);
13256 	tw32(offset, save_val);
13257 	return -EIO;
13258 }
13259 
13260 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
13261 {
13262 	static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
13263 	int i;
13264 	u32 j;
13265 
13266 	for (i = 0; i < ARRAY_SIZE(test_pattern); i++) {
13267 		for (j = 0; j < len; j += 4) {
13268 			u32 val;
13269 
13270 			tg3_write_mem(tp, offset + j, test_pattern[i]);
13271 			tg3_read_mem(tp, offset + j, &val);
13272 			if (val != test_pattern[i])
13273 				return -EIO;
13274 		}
13275 	}
13276 	return 0;
13277 }
13278 
13279 static int tg3_test_memory(struct tg3 *tp)
13280 {
13281 	static struct mem_entry {
13282 		u32 offset;
13283 		u32 len;
13284 	} mem_tbl_570x[] = {
13285 		{ 0x00000000, 0x00b50},
13286 		{ 0x00002000, 0x1c000},
13287 		{ 0xffffffff, 0x00000}
13288 	}, mem_tbl_5705[] = {
13289 		{ 0x00000100, 0x0000c},
13290 		{ 0x00000200, 0x00008},
13291 		{ 0x00004000, 0x00800},
13292 		{ 0x00006000, 0x01000},
13293 		{ 0x00008000, 0x02000},
13294 		{ 0x00010000, 0x0e000},
13295 		{ 0xffffffff, 0x00000}
13296 	}, mem_tbl_5755[] = {
13297 		{ 0x00000200, 0x00008},
13298 		{ 0x00004000, 0x00800},
13299 		{ 0x00006000, 0x00800},
13300 		{ 0x00008000, 0x02000},
13301 		{ 0x00010000, 0x0c000},
13302 		{ 0xffffffff, 0x00000}
13303 	}, mem_tbl_5906[] = {
13304 		{ 0x00000200, 0x00008},
13305 		{ 0x00004000, 0x00400},
13306 		{ 0x00006000, 0x00400},
13307 		{ 0x00008000, 0x01000},
13308 		{ 0x00010000, 0x01000},
13309 		{ 0xffffffff, 0x00000}
13310 	}, mem_tbl_5717[] = {
13311 		{ 0x00000200, 0x00008},
13312 		{ 0x00010000, 0x0a000},
13313 		{ 0x00020000, 0x13c00},
13314 		{ 0xffffffff, 0x00000}
13315 	}, mem_tbl_57765[] = {
13316 		{ 0x00000200, 0x00008},
13317 		{ 0x00004000, 0x00800},
13318 		{ 0x00006000, 0x09800},
13319 		{ 0x00010000, 0x0a000},
13320 		{ 0xffffffff, 0x00000}
13321 	};
13322 	struct mem_entry *mem_tbl;
13323 	int err = 0;
13324 	int i;
13325 
13326 	if (tg3_flag(tp, 5717_PLUS))
13327 		mem_tbl = mem_tbl_5717;
13328 	else if (tg3_flag(tp, 57765_CLASS) ||
13329 		 tg3_asic_rev(tp) == ASIC_REV_5762)
13330 		mem_tbl = mem_tbl_57765;
13331 	else if (tg3_flag(tp, 5755_PLUS))
13332 		mem_tbl = mem_tbl_5755;
13333 	else if (tg3_asic_rev(tp) == ASIC_REV_5906)
13334 		mem_tbl = mem_tbl_5906;
13335 	else if (tg3_flag(tp, 5705_PLUS))
13336 		mem_tbl = mem_tbl_5705;
13337 	else
13338 		mem_tbl = mem_tbl_570x;
13339 
13340 	for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
13341 		err = tg3_do_mem_test(tp, mem_tbl[i].offset, mem_tbl[i].len);
13342 		if (err)
13343 			break;
13344 	}
13345 
13346 	return err;
13347 }
13348 
13349 #define TG3_TSO_MSS		500
13350 
13351 #define TG3_TSO_IP_HDR_LEN	20
13352 #define TG3_TSO_TCP_HDR_LEN	20
13353 #define TG3_TSO_TCP_OPT_LEN	12
13354 
13355 static const u8 tg3_tso_header[] = {
13356 0x08, 0x00,
13357 0x45, 0x00, 0x00, 0x00,
13358 0x00, 0x00, 0x40, 0x00,
13359 0x40, 0x06, 0x00, 0x00,
13360 0x0a, 0x00, 0x00, 0x01,
13361 0x0a, 0x00, 0x00, 0x02,
13362 0x0d, 0x00, 0xe0, 0x00,
13363 0x00, 0x00, 0x01, 0x00,
13364 0x00, 0x00, 0x02, 0x00,
13365 0x80, 0x10, 0x10, 0x00,
13366 0x14, 0x09, 0x00, 0x00,
13367 0x01, 0x01, 0x08, 0x0a,
13368 0x11, 0x11, 0x11, 0x11,
13369 0x11, 0x11, 0x11, 0x11,
13370 };
13371 
13372 static int tg3_run_loopback(struct tg3 *tp, u32 pktsz, bool tso_loopback)
13373 {
13374 	u32 rx_start_idx, rx_idx, tx_idx, opaque_key;
13375 	u32 base_flags = 0, mss = 0, desc_idx, coal_now, data_off, val;
13376 	u32 budget;
13377 	struct sk_buff *skb;
13378 	u8 *tx_data, *rx_data;
13379 	dma_addr_t map;
13380 	int num_pkts, tx_len, rx_len, i, err;
13381 	struct tg3_rx_buffer_desc *desc;
13382 	struct tg3_napi *tnapi, *rnapi;
13383 	struct tg3_rx_prodring_set *tpr = &tp->napi[0].prodring;
13384 
13385 	tnapi = &tp->napi[0];
13386 	rnapi = &tp->napi[0];
13387 	if (tp->irq_cnt > 1) {
13388 		if (tg3_flag(tp, ENABLE_RSS))
13389 			rnapi = &tp->napi[1];
13390 		if (tg3_flag(tp, ENABLE_TSS))
13391 			tnapi = &tp->napi[1];
13392 	}
13393 	coal_now = tnapi->coal_now | rnapi->coal_now;
13394 
13395 	err = -EIO;
13396 
13397 	tx_len = pktsz;
13398 	skb = netdev_alloc_skb(tp->dev, tx_len);
13399 	if (!skb)
13400 		return -ENOMEM;
13401 
13402 	tx_data = skb_put(skb, tx_len);
13403 	memcpy(tx_data, tp->dev->dev_addr, ETH_ALEN);
13404 	memset(tx_data + ETH_ALEN, 0x0, 8);
13405 
13406 	tw32(MAC_RX_MTU_SIZE, tx_len + ETH_FCS_LEN);
13407 
13408 	if (tso_loopback) {
13409 		struct iphdr *iph = (struct iphdr *)&tx_data[ETH_HLEN];
13410 
13411 		u32 hdr_len = TG3_TSO_IP_HDR_LEN + TG3_TSO_TCP_HDR_LEN +
13412 			      TG3_TSO_TCP_OPT_LEN;
13413 
13414 		memcpy(tx_data + ETH_ALEN * 2, tg3_tso_header,
13415 		       sizeof(tg3_tso_header));
13416 		mss = TG3_TSO_MSS;
13417 
13418 		val = tx_len - ETH_ALEN * 2 - sizeof(tg3_tso_header);
13419 		num_pkts = DIV_ROUND_UP(val, TG3_TSO_MSS);
13420 
13421 		/* Set the total length field in the IP header */
13422 		iph->tot_len = htons((u16)(mss + hdr_len));
13423 
13424 		base_flags = (TXD_FLAG_CPU_PRE_DMA |
13425 			      TXD_FLAG_CPU_POST_DMA);
13426 
13427 		if (tg3_flag(tp, HW_TSO_1) ||
13428 		    tg3_flag(tp, HW_TSO_2) ||
13429 		    tg3_flag(tp, HW_TSO_3)) {
13430 			struct tcphdr *th;
13431 			val = ETH_HLEN + TG3_TSO_IP_HDR_LEN;
13432 			th = (struct tcphdr *)&tx_data[val];
13433 			th->check = 0;
13434 		} else
13435 			base_flags |= TXD_FLAG_TCPUDP_CSUM;
13436 
13437 		if (tg3_flag(tp, HW_TSO_3)) {
13438 			mss |= (hdr_len & 0xc) << 12;
13439 			if (hdr_len & 0x10)
13440 				base_flags |= 0x00000010;
13441 			base_flags |= (hdr_len & 0x3e0) << 5;
13442 		} else if (tg3_flag(tp, HW_TSO_2))
13443 			mss |= hdr_len << 9;
13444 		else if (tg3_flag(tp, HW_TSO_1) ||
13445 			 tg3_asic_rev(tp) == ASIC_REV_5705) {
13446 			mss |= (TG3_TSO_TCP_OPT_LEN << 9);
13447 		} else {
13448 			base_flags |= (TG3_TSO_TCP_OPT_LEN << 10);
13449 		}
13450 
13451 		data_off = ETH_ALEN * 2 + sizeof(tg3_tso_header);
13452 	} else {
13453 		num_pkts = 1;
13454 		data_off = ETH_HLEN;
13455 
13456 		if (tg3_flag(tp, USE_JUMBO_BDFLAG) &&
13457 		    tx_len > VLAN_ETH_FRAME_LEN)
13458 			base_flags |= TXD_FLAG_JMB_PKT;
13459 	}
13460 
13461 	for (i = data_off; i < tx_len; i++)
13462 		tx_data[i] = (u8) (i & 0xff);
13463 
13464 	map = dma_map_single(&tp->pdev->dev, skb->data, tx_len, DMA_TO_DEVICE);
13465 	if (dma_mapping_error(&tp->pdev->dev, map)) {
13466 		dev_kfree_skb(skb);
13467 		return -EIO;
13468 	}
13469 
13470 	val = tnapi->tx_prod;
13471 	tnapi->tx_buffers[val].skb = skb;
13472 	dma_unmap_addr_set(&tnapi->tx_buffers[val], mapping, map);
13473 
13474 	tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
13475 	       rnapi->coal_now);
13476 
13477 	udelay(10);
13478 
13479 	rx_start_idx = rnapi->hw_status->idx[0].rx_producer;
13480 
13481 	budget = tg3_tx_avail(tnapi);
13482 	if (tg3_tx_frag_set(tnapi, &val, &budget, map, tx_len,
13483 			    base_flags | TXD_FLAG_END, mss, 0)) {
13484 		tnapi->tx_buffers[val].skb = NULL;
13485 		dev_kfree_skb(skb);
13486 		return -EIO;
13487 	}
13488 
13489 	tnapi->tx_prod++;
13490 
13491 	/* Sync BD data before updating mailbox */
13492 	wmb();
13493 
13494 	tw32_tx_mbox(tnapi->prodmbox, tnapi->tx_prod);
13495 	tr32_mailbox(tnapi->prodmbox);
13496 
13497 	udelay(10);
13498 
13499 	/* 350 usec to allow enough time on some 10/100 Mbps devices.  */
13500 	for (i = 0; i < 35; i++) {
13501 		tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
13502 		       coal_now);
13503 
13504 		udelay(10);
13505 
13506 		tx_idx = tnapi->hw_status->idx[0].tx_consumer;
13507 		rx_idx = rnapi->hw_status->idx[0].rx_producer;
13508 		if ((tx_idx == tnapi->tx_prod) &&
13509 		    (rx_idx == (rx_start_idx + num_pkts)))
13510 			break;
13511 	}
13512 
13513 	tg3_tx_skb_unmap(tnapi, tnapi->tx_prod - 1, -1);
13514 	dev_kfree_skb(skb);
13515 
13516 	if (tx_idx != tnapi->tx_prod)
13517 		goto out;
13518 
13519 	if (rx_idx != rx_start_idx + num_pkts)
13520 		goto out;
13521 
13522 	val = data_off;
13523 	while (rx_idx != rx_start_idx) {
13524 		desc = &rnapi->rx_rcb[rx_start_idx++];
13525 		desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
13526 		opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
13527 
13528 		if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
13529 		    (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
13530 			goto out;
13531 
13532 		rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT)
13533 			 - ETH_FCS_LEN;
13534 
13535 		if (!tso_loopback) {
13536 			if (rx_len != tx_len)
13537 				goto out;
13538 
13539 			if (pktsz <= TG3_RX_STD_DMA_SZ - ETH_FCS_LEN) {
13540 				if (opaque_key != RXD_OPAQUE_RING_STD)
13541 					goto out;
13542 			} else {
13543 				if (opaque_key != RXD_OPAQUE_RING_JUMBO)
13544 					goto out;
13545 			}
13546 		} else if ((desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
13547 			   (desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
13548 			    >> RXD_TCPCSUM_SHIFT != 0xffff) {
13549 			goto out;
13550 		}
13551 
13552 		if (opaque_key == RXD_OPAQUE_RING_STD) {
13553 			rx_data = tpr->rx_std_buffers[desc_idx].data;
13554 			map = dma_unmap_addr(&tpr->rx_std_buffers[desc_idx],
13555 					     mapping);
13556 		} else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
13557 			rx_data = tpr->rx_jmb_buffers[desc_idx].data;
13558 			map = dma_unmap_addr(&tpr->rx_jmb_buffers[desc_idx],
13559 					     mapping);
13560 		} else
13561 			goto out;
13562 
13563 		dma_sync_single_for_cpu(&tp->pdev->dev, map, rx_len,
13564 					DMA_FROM_DEVICE);
13565 
13566 		rx_data += TG3_RX_OFFSET(tp);
13567 		for (i = data_off; i < rx_len; i++, val++) {
13568 			if (*(rx_data + i) != (u8) (val & 0xff))
13569 				goto out;
13570 		}
13571 	}
13572 
13573 	err = 0;
13574 
13575 	/* tg3_free_rings will unmap and free the rx_data */
13576 out:
13577 	return err;
13578 }
13579 
13580 #define TG3_STD_LOOPBACK_FAILED		1
13581 #define TG3_JMB_LOOPBACK_FAILED		2
13582 #define TG3_TSO_LOOPBACK_FAILED		4
13583 #define TG3_LOOPBACK_FAILED \
13584 	(TG3_STD_LOOPBACK_FAILED | \
13585 	 TG3_JMB_LOOPBACK_FAILED | \
13586 	 TG3_TSO_LOOPBACK_FAILED)
13587 
13588 static int tg3_test_loopback(struct tg3 *tp, u64 *data, bool do_extlpbk)
13589 {
13590 	int err = -EIO;
13591 	u32 eee_cap;
13592 	u32 jmb_pkt_sz = 9000;
13593 
13594 	if (tp->dma_limit)
13595 		jmb_pkt_sz = tp->dma_limit - ETH_HLEN;
13596 
13597 	eee_cap = tp->phy_flags & TG3_PHYFLG_EEE_CAP;
13598 	tp->phy_flags &= ~TG3_PHYFLG_EEE_CAP;
13599 
13600 	if (!netif_running(tp->dev)) {
13601 		data[TG3_MAC_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13602 		data[TG3_PHY_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13603 		if (do_extlpbk)
13604 			data[TG3_EXT_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13605 		goto done;
13606 	}
13607 
13608 	err = tg3_reset_hw(tp, true);
13609 	if (err) {
13610 		data[TG3_MAC_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13611 		data[TG3_PHY_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13612 		if (do_extlpbk)
13613 			data[TG3_EXT_LOOPB_TEST] = TG3_LOOPBACK_FAILED;
13614 		goto done;
13615 	}
13616 
13617 	if (tg3_flag(tp, ENABLE_RSS)) {
13618 		int i;
13619 
13620 		/* Reroute all rx packets to the 1st queue */
13621 		for (i = MAC_RSS_INDIR_TBL_0;
13622 		     i < MAC_RSS_INDIR_TBL_0 + TG3_RSS_INDIR_TBL_SIZE; i += 4)
13623 			tw32(i, 0x0);
13624 	}
13625 
13626 	/* HW errata - mac loopback fails in some cases on 5780.
13627 	 * Normal traffic and PHY loopback are not affected by
13628 	 * errata.  Also, the MAC loopback test is deprecated for
13629 	 * all newer ASIC revisions.
13630 	 */
13631 	if (tg3_asic_rev(tp) != ASIC_REV_5780 &&
13632 	    !tg3_flag(tp, CPMU_PRESENT)) {
13633 		tg3_mac_loopback(tp, true);
13634 
13635 		if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13636 			data[TG3_MAC_LOOPB_TEST] |= TG3_STD_LOOPBACK_FAILED;
13637 
13638 		if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13639 		    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13640 			data[TG3_MAC_LOOPB_TEST] |= TG3_JMB_LOOPBACK_FAILED;
13641 
13642 		tg3_mac_loopback(tp, false);
13643 	}
13644 
13645 	if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
13646 	    !tg3_flag(tp, USE_PHYLIB)) {
13647 		int i;
13648 
13649 		tg3_phy_lpbk_set(tp, 0, false);
13650 
13651 		/* Wait for link */
13652 		for (i = 0; i < 100; i++) {
13653 			if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
13654 				break;
13655 			mdelay(1);
13656 		}
13657 
13658 		if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13659 			data[TG3_PHY_LOOPB_TEST] |= TG3_STD_LOOPBACK_FAILED;
13660 		if (tg3_flag(tp, TSO_CAPABLE) &&
13661 		    tg3_run_loopback(tp, ETH_FRAME_LEN, true))
13662 			data[TG3_PHY_LOOPB_TEST] |= TG3_TSO_LOOPBACK_FAILED;
13663 		if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13664 		    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13665 			data[TG3_PHY_LOOPB_TEST] |= TG3_JMB_LOOPBACK_FAILED;
13666 
13667 		if (do_extlpbk) {
13668 			tg3_phy_lpbk_set(tp, 0, true);
13669 
13670 			/* All link indications report up, but the hardware
13671 			 * isn't really ready for about 20 msec.  Double it
13672 			 * to be sure.
13673 			 */
13674 			mdelay(40);
13675 
13676 			if (tg3_run_loopback(tp, ETH_FRAME_LEN, false))
13677 				data[TG3_EXT_LOOPB_TEST] |=
13678 							TG3_STD_LOOPBACK_FAILED;
13679 			if (tg3_flag(tp, TSO_CAPABLE) &&
13680 			    tg3_run_loopback(tp, ETH_FRAME_LEN, true))
13681 				data[TG3_EXT_LOOPB_TEST] |=
13682 							TG3_TSO_LOOPBACK_FAILED;
13683 			if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
13684 			    tg3_run_loopback(tp, jmb_pkt_sz + ETH_HLEN, false))
13685 				data[TG3_EXT_LOOPB_TEST] |=
13686 							TG3_JMB_LOOPBACK_FAILED;
13687 		}
13688 
13689 		/* Re-enable gphy autopowerdown. */
13690 		if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
13691 			tg3_phy_toggle_apd(tp, true);
13692 	}
13693 
13694 	err = (data[TG3_MAC_LOOPB_TEST] | data[TG3_PHY_LOOPB_TEST] |
13695 	       data[TG3_EXT_LOOPB_TEST]) ? -EIO : 0;
13696 
13697 done:
13698 	tp->phy_flags |= eee_cap;
13699 
13700 	return err;
13701 }
13702 
13703 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
13704 			  u64 *data)
13705 {
13706 	struct tg3 *tp = netdev_priv(dev);
13707 	bool doextlpbk = etest->flags & ETH_TEST_FL_EXTERNAL_LB;
13708 
13709 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
13710 		if (tg3_power_up(tp)) {
13711 			etest->flags |= ETH_TEST_FL_FAILED;
13712 			memset(data, 1, sizeof(u64) * TG3_NUM_TEST);
13713 			return;
13714 		}
13715 		tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
13716 	}
13717 
13718 	memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
13719 
13720 	if (tg3_test_nvram(tp) != 0) {
13721 		etest->flags |= ETH_TEST_FL_FAILED;
13722 		data[TG3_NVRAM_TEST] = 1;
13723 	}
13724 	if (!doextlpbk && tg3_test_link(tp)) {
13725 		etest->flags |= ETH_TEST_FL_FAILED;
13726 		data[TG3_LINK_TEST] = 1;
13727 	}
13728 	if (etest->flags & ETH_TEST_FL_OFFLINE) {
13729 		int err, err2 = 0, irq_sync = 0;
13730 
13731 		if (netif_running(dev)) {
13732 			tg3_phy_stop(tp);
13733 			tg3_netif_stop(tp);
13734 			irq_sync = 1;
13735 		}
13736 
13737 		tg3_full_lock(tp, irq_sync);
13738 		tg3_halt(tp, RESET_KIND_SUSPEND, 1);
13739 		err = tg3_nvram_lock(tp);
13740 		tg3_halt_cpu(tp, RX_CPU_BASE);
13741 		if (!tg3_flag(tp, 5705_PLUS))
13742 			tg3_halt_cpu(tp, TX_CPU_BASE);
13743 		if (!err)
13744 			tg3_nvram_unlock(tp);
13745 
13746 		if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
13747 			tg3_phy_reset(tp);
13748 
13749 		if (tg3_test_registers(tp) != 0) {
13750 			etest->flags |= ETH_TEST_FL_FAILED;
13751 			data[TG3_REGISTER_TEST] = 1;
13752 		}
13753 
13754 		if (tg3_test_memory(tp) != 0) {
13755 			etest->flags |= ETH_TEST_FL_FAILED;
13756 			data[TG3_MEMORY_TEST] = 1;
13757 		}
13758 
13759 		if (doextlpbk)
13760 			etest->flags |= ETH_TEST_FL_EXTERNAL_LB_DONE;
13761 
13762 		if (tg3_test_loopback(tp, data, doextlpbk))
13763 			etest->flags |= ETH_TEST_FL_FAILED;
13764 
13765 		tg3_full_unlock(tp);
13766 
13767 		if (tg3_test_interrupt(tp) != 0) {
13768 			etest->flags |= ETH_TEST_FL_FAILED;
13769 			data[TG3_INTERRUPT_TEST] = 1;
13770 		}
13771 
13772 		tg3_full_lock(tp, 0);
13773 
13774 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
13775 		if (netif_running(dev)) {
13776 			tg3_flag_set(tp, INIT_COMPLETE);
13777 			err2 = tg3_restart_hw(tp, true);
13778 			if (!err2)
13779 				tg3_netif_start(tp);
13780 		}
13781 
13782 		tg3_full_unlock(tp);
13783 
13784 		if (irq_sync && !err2)
13785 			tg3_phy_start(tp);
13786 	}
13787 	if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
13788 		tg3_power_down_prepare(tp);
13789 
13790 }
13791 
13792 static int tg3_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
13793 {
13794 	struct tg3 *tp = netdev_priv(dev);
13795 	struct hwtstamp_config stmpconf;
13796 
13797 	if (!tg3_flag(tp, PTP_CAPABLE))
13798 		return -EOPNOTSUPP;
13799 
13800 	if (copy_from_user(&stmpconf, ifr->ifr_data, sizeof(stmpconf)))
13801 		return -EFAULT;
13802 
13803 	if (stmpconf.flags)
13804 		return -EINVAL;
13805 
13806 	if (stmpconf.tx_type != HWTSTAMP_TX_ON &&
13807 	    stmpconf.tx_type != HWTSTAMP_TX_OFF)
13808 		return -ERANGE;
13809 
13810 	switch (stmpconf.rx_filter) {
13811 	case HWTSTAMP_FILTER_NONE:
13812 		tp->rxptpctl = 0;
13813 		break;
13814 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
13815 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13816 			       TG3_RX_PTP_CTL_ALL_V1_EVENTS;
13817 		break;
13818 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
13819 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13820 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13821 		break;
13822 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
13823 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V1_EN |
13824 			       TG3_RX_PTP_CTL_DELAY_REQ;
13825 		break;
13826 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
13827 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13828 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13829 		break;
13830 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
13831 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13832 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13833 		break;
13834 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
13835 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13836 			       TG3_RX_PTP_CTL_ALL_V2_EVENTS;
13837 		break;
13838 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
13839 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13840 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13841 		break;
13842 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
13843 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13844 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13845 		break;
13846 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
13847 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13848 			       TG3_RX_PTP_CTL_SYNC_EVNT;
13849 		break;
13850 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
13851 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_EN |
13852 			       TG3_RX_PTP_CTL_DELAY_REQ;
13853 		break;
13854 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
13855 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN |
13856 			       TG3_RX_PTP_CTL_DELAY_REQ;
13857 		break;
13858 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
13859 		tp->rxptpctl = TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN |
13860 			       TG3_RX_PTP_CTL_DELAY_REQ;
13861 		break;
13862 	default:
13863 		return -ERANGE;
13864 	}
13865 
13866 	if (netif_running(dev) && tp->rxptpctl)
13867 		tw32(TG3_RX_PTP_CTL,
13868 		     tp->rxptpctl | TG3_RX_PTP_CTL_HWTS_INTERLOCK);
13869 
13870 	if (stmpconf.tx_type == HWTSTAMP_TX_ON)
13871 		tg3_flag_set(tp, TX_TSTAMP_EN);
13872 	else
13873 		tg3_flag_clear(tp, TX_TSTAMP_EN);
13874 
13875 	return copy_to_user(ifr->ifr_data, &stmpconf, sizeof(stmpconf)) ?
13876 		-EFAULT : 0;
13877 }
13878 
13879 static int tg3_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
13880 {
13881 	struct tg3 *tp = netdev_priv(dev);
13882 	struct hwtstamp_config stmpconf;
13883 
13884 	if (!tg3_flag(tp, PTP_CAPABLE))
13885 		return -EOPNOTSUPP;
13886 
13887 	stmpconf.flags = 0;
13888 	stmpconf.tx_type = (tg3_flag(tp, TX_TSTAMP_EN) ?
13889 			    HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF);
13890 
13891 	switch (tp->rxptpctl) {
13892 	case 0:
13893 		stmpconf.rx_filter = HWTSTAMP_FILTER_NONE;
13894 		break;
13895 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_ALL_V1_EVENTS:
13896 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
13897 		break;
13898 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13899 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
13900 		break;
13901 	case TG3_RX_PTP_CTL_RX_PTP_V1_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13902 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
13903 		break;
13904 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13905 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
13906 		break;
13907 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13908 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
13909 		break;
13910 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_ALL_V2_EVENTS:
13911 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
13912 		break;
13913 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13914 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
13915 		break;
13916 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13917 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_SYNC;
13918 		break;
13919 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_SYNC_EVNT:
13920 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
13921 		break;
13922 	case TG3_RX_PTP_CTL_RX_PTP_V2_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13923 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
13924 		break;
13925 	case TG3_RX_PTP_CTL_RX_PTP_V2_L2_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13926 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ;
13927 		break;
13928 	case TG3_RX_PTP_CTL_RX_PTP_V2_L4_EN | TG3_RX_PTP_CTL_DELAY_REQ:
13929 		stmpconf.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
13930 		break;
13931 	default:
13932 		WARN_ON_ONCE(1);
13933 		return -ERANGE;
13934 	}
13935 
13936 	return copy_to_user(ifr->ifr_data, &stmpconf, sizeof(stmpconf)) ?
13937 		-EFAULT : 0;
13938 }
13939 
13940 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
13941 {
13942 	struct mii_ioctl_data *data = if_mii(ifr);
13943 	struct tg3 *tp = netdev_priv(dev);
13944 	int err;
13945 
13946 	if (tg3_flag(tp, USE_PHYLIB)) {
13947 		struct phy_device *phydev;
13948 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
13949 			return -EAGAIN;
13950 		phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
13951 		return phy_mii_ioctl(phydev, ifr, cmd);
13952 	}
13953 
13954 	switch (cmd) {
13955 	case SIOCGMIIPHY:
13956 		data->phy_id = tp->phy_addr;
13957 
13958 		fallthrough;
13959 	case SIOCGMIIREG: {
13960 		u32 mii_regval;
13961 
13962 		if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
13963 			break;			/* We have no PHY */
13964 
13965 		if (!netif_running(dev))
13966 			return -EAGAIN;
13967 
13968 		spin_lock_bh(&tp->lock);
13969 		err = __tg3_readphy(tp, data->phy_id & 0x1f,
13970 				    data->reg_num & 0x1f, &mii_regval);
13971 		spin_unlock_bh(&tp->lock);
13972 
13973 		data->val_out = mii_regval;
13974 
13975 		return err;
13976 	}
13977 
13978 	case SIOCSMIIREG:
13979 		if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
13980 			break;			/* We have no PHY */
13981 
13982 		if (!netif_running(dev))
13983 			return -EAGAIN;
13984 
13985 		spin_lock_bh(&tp->lock);
13986 		err = __tg3_writephy(tp, data->phy_id & 0x1f,
13987 				     data->reg_num & 0x1f, data->val_in);
13988 		spin_unlock_bh(&tp->lock);
13989 
13990 		return err;
13991 
13992 	case SIOCSHWTSTAMP:
13993 		return tg3_hwtstamp_set(dev, ifr);
13994 
13995 	case SIOCGHWTSTAMP:
13996 		return tg3_hwtstamp_get(dev, ifr);
13997 
13998 	default:
13999 		/* do nothing */
14000 		break;
14001 	}
14002 	return -EOPNOTSUPP;
14003 }
14004 
14005 static int tg3_get_coalesce(struct net_device *dev,
14006 			    struct ethtool_coalesce *ec,
14007 			    struct kernel_ethtool_coalesce *kernel_coal,
14008 			    struct netlink_ext_ack *extack)
14009 {
14010 	struct tg3 *tp = netdev_priv(dev);
14011 
14012 	memcpy(ec, &tp->coal, sizeof(*ec));
14013 	return 0;
14014 }
14015 
14016 static int tg3_set_coalesce(struct net_device *dev,
14017 			    struct ethtool_coalesce *ec,
14018 			    struct kernel_ethtool_coalesce *kernel_coal,
14019 			    struct netlink_ext_ack *extack)
14020 {
14021 	struct tg3 *tp = netdev_priv(dev);
14022 	u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
14023 	u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
14024 
14025 	if (!tg3_flag(tp, 5705_PLUS)) {
14026 		max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
14027 		max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
14028 		max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
14029 		min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
14030 	}
14031 
14032 	if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
14033 	    (!ec->rx_coalesce_usecs) ||
14034 	    (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
14035 	    (!ec->tx_coalesce_usecs) ||
14036 	    (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
14037 	    (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
14038 	    (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
14039 	    (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
14040 	    (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
14041 	    (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
14042 	    (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
14043 	    (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
14044 		return -EINVAL;
14045 
14046 	/* Only copy relevant parameters, ignore all others. */
14047 	tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
14048 	tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
14049 	tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
14050 	tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
14051 	tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
14052 	tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
14053 	tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
14054 	tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
14055 	tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
14056 
14057 	if (netif_running(dev)) {
14058 		tg3_full_lock(tp, 0);
14059 		__tg3_set_coalesce(tp, &tp->coal);
14060 		tg3_full_unlock(tp);
14061 	}
14062 	return 0;
14063 }
14064 
14065 static int tg3_set_eee(struct net_device *dev, struct ethtool_eee *edata)
14066 {
14067 	struct tg3 *tp = netdev_priv(dev);
14068 
14069 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
14070 		netdev_warn(tp->dev, "Board does not support EEE!\n");
14071 		return -EOPNOTSUPP;
14072 	}
14073 
14074 	if (edata->advertised != tp->eee.advertised) {
14075 		netdev_warn(tp->dev,
14076 			    "Direct manipulation of EEE advertisement is not supported\n");
14077 		return -EINVAL;
14078 	}
14079 
14080 	if (edata->tx_lpi_timer > TG3_CPMU_DBTMR1_LNKIDLE_MAX) {
14081 		netdev_warn(tp->dev,
14082 			    "Maximal Tx Lpi timer supported is %#x(u)\n",
14083 			    TG3_CPMU_DBTMR1_LNKIDLE_MAX);
14084 		return -EINVAL;
14085 	}
14086 
14087 	tp->eee = *edata;
14088 
14089 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
14090 	tg3_warn_mgmt_link_flap(tp);
14091 
14092 	if (netif_running(tp->dev)) {
14093 		tg3_full_lock(tp, 0);
14094 		tg3_setup_eee(tp);
14095 		tg3_phy_reset(tp);
14096 		tg3_full_unlock(tp);
14097 	}
14098 
14099 	return 0;
14100 }
14101 
14102 static int tg3_get_eee(struct net_device *dev, struct ethtool_eee *edata)
14103 {
14104 	struct tg3 *tp = netdev_priv(dev);
14105 
14106 	if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP)) {
14107 		netdev_warn(tp->dev,
14108 			    "Board does not support EEE!\n");
14109 		return -EOPNOTSUPP;
14110 	}
14111 
14112 	*edata = tp->eee;
14113 	return 0;
14114 }
14115 
14116 static const struct ethtool_ops tg3_ethtool_ops = {
14117 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
14118 				     ETHTOOL_COALESCE_MAX_FRAMES |
14119 				     ETHTOOL_COALESCE_USECS_IRQ |
14120 				     ETHTOOL_COALESCE_MAX_FRAMES_IRQ |
14121 				     ETHTOOL_COALESCE_STATS_BLOCK_USECS,
14122 	.get_drvinfo		= tg3_get_drvinfo,
14123 	.get_regs_len		= tg3_get_regs_len,
14124 	.get_regs		= tg3_get_regs,
14125 	.get_wol		= tg3_get_wol,
14126 	.set_wol		= tg3_set_wol,
14127 	.get_msglevel		= tg3_get_msglevel,
14128 	.set_msglevel		= tg3_set_msglevel,
14129 	.nway_reset		= tg3_nway_reset,
14130 	.get_link		= ethtool_op_get_link,
14131 	.get_eeprom_len		= tg3_get_eeprom_len,
14132 	.get_eeprom		= tg3_get_eeprom,
14133 	.set_eeprom		= tg3_set_eeprom,
14134 	.get_ringparam		= tg3_get_ringparam,
14135 	.set_ringparam		= tg3_set_ringparam,
14136 	.get_pauseparam		= tg3_get_pauseparam,
14137 	.set_pauseparam		= tg3_set_pauseparam,
14138 	.self_test		= tg3_self_test,
14139 	.get_strings		= tg3_get_strings,
14140 	.set_phys_id		= tg3_set_phys_id,
14141 	.get_ethtool_stats	= tg3_get_ethtool_stats,
14142 	.get_coalesce		= tg3_get_coalesce,
14143 	.set_coalesce		= tg3_set_coalesce,
14144 	.get_sset_count		= tg3_get_sset_count,
14145 	.get_rxnfc		= tg3_get_rxnfc,
14146 	.get_rxfh_indir_size    = tg3_get_rxfh_indir_size,
14147 	.get_rxfh		= tg3_get_rxfh,
14148 	.set_rxfh		= tg3_set_rxfh,
14149 	.get_channels		= tg3_get_channels,
14150 	.set_channels		= tg3_set_channels,
14151 	.get_ts_info		= tg3_get_ts_info,
14152 	.get_eee		= tg3_get_eee,
14153 	.set_eee		= tg3_set_eee,
14154 	.get_link_ksettings	= tg3_get_link_ksettings,
14155 	.set_link_ksettings	= tg3_set_link_ksettings,
14156 };
14157 
14158 static void tg3_get_stats64(struct net_device *dev,
14159 			    struct rtnl_link_stats64 *stats)
14160 {
14161 	struct tg3 *tp = netdev_priv(dev);
14162 
14163 	spin_lock_bh(&tp->lock);
14164 	if (!tp->hw_stats || !tg3_flag(tp, INIT_COMPLETE)) {
14165 		*stats = tp->net_stats_prev;
14166 		spin_unlock_bh(&tp->lock);
14167 		return;
14168 	}
14169 
14170 	tg3_get_nstats(tp, stats);
14171 	spin_unlock_bh(&tp->lock);
14172 }
14173 
14174 static void tg3_set_rx_mode(struct net_device *dev)
14175 {
14176 	struct tg3 *tp = netdev_priv(dev);
14177 
14178 	if (!netif_running(dev))
14179 		return;
14180 
14181 	tg3_full_lock(tp, 0);
14182 	__tg3_set_rx_mode(dev);
14183 	tg3_full_unlock(tp);
14184 }
14185 
14186 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
14187 			       int new_mtu)
14188 {
14189 	dev->mtu = new_mtu;
14190 
14191 	if (new_mtu > ETH_DATA_LEN) {
14192 		if (tg3_flag(tp, 5780_CLASS)) {
14193 			netdev_update_features(dev);
14194 			tg3_flag_clear(tp, TSO_CAPABLE);
14195 		} else {
14196 			tg3_flag_set(tp, JUMBO_RING_ENABLE);
14197 		}
14198 	} else {
14199 		if (tg3_flag(tp, 5780_CLASS)) {
14200 			tg3_flag_set(tp, TSO_CAPABLE);
14201 			netdev_update_features(dev);
14202 		}
14203 		tg3_flag_clear(tp, JUMBO_RING_ENABLE);
14204 	}
14205 }
14206 
14207 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
14208 {
14209 	struct tg3 *tp = netdev_priv(dev);
14210 	int err;
14211 	bool reset_phy = false;
14212 
14213 	if (!netif_running(dev)) {
14214 		/* We'll just catch it later when the
14215 		 * device is up'd.
14216 		 */
14217 		tg3_set_mtu(dev, tp, new_mtu);
14218 		return 0;
14219 	}
14220 
14221 	tg3_phy_stop(tp);
14222 
14223 	tg3_netif_stop(tp);
14224 
14225 	tg3_set_mtu(dev, tp, new_mtu);
14226 
14227 	tg3_full_lock(tp, 1);
14228 
14229 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
14230 
14231 	/* Reset PHY, otherwise the read DMA engine will be in a mode that
14232 	 * breaks all requests to 256 bytes.
14233 	 */
14234 	if (tg3_asic_rev(tp) == ASIC_REV_57766 ||
14235 	    tg3_asic_rev(tp) == ASIC_REV_5717 ||
14236 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
14237 	    tg3_asic_rev(tp) == ASIC_REV_5720)
14238 		reset_phy = true;
14239 
14240 	err = tg3_restart_hw(tp, reset_phy);
14241 
14242 	if (!err)
14243 		tg3_netif_start(tp);
14244 
14245 	tg3_full_unlock(tp);
14246 
14247 	if (!err)
14248 		tg3_phy_start(tp);
14249 
14250 	return err;
14251 }
14252 
14253 static const struct net_device_ops tg3_netdev_ops = {
14254 	.ndo_open		= tg3_open,
14255 	.ndo_stop		= tg3_close,
14256 	.ndo_start_xmit		= tg3_start_xmit,
14257 	.ndo_get_stats64	= tg3_get_stats64,
14258 	.ndo_validate_addr	= eth_validate_addr,
14259 	.ndo_set_rx_mode	= tg3_set_rx_mode,
14260 	.ndo_set_mac_address	= tg3_set_mac_addr,
14261 	.ndo_eth_ioctl		= tg3_ioctl,
14262 	.ndo_tx_timeout		= tg3_tx_timeout,
14263 	.ndo_change_mtu		= tg3_change_mtu,
14264 	.ndo_fix_features	= tg3_fix_features,
14265 	.ndo_set_features	= tg3_set_features,
14266 #ifdef CONFIG_NET_POLL_CONTROLLER
14267 	.ndo_poll_controller	= tg3_poll_controller,
14268 #endif
14269 };
14270 
14271 static void tg3_get_eeprom_size(struct tg3 *tp)
14272 {
14273 	u32 cursize, val, magic;
14274 
14275 	tp->nvram_size = EEPROM_CHIP_SIZE;
14276 
14277 	if (tg3_nvram_read(tp, 0, &magic) != 0)
14278 		return;
14279 
14280 	if ((magic != TG3_EEPROM_MAGIC) &&
14281 	    ((magic & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW) &&
14282 	    ((magic & TG3_EEPROM_MAGIC_HW_MSK) != TG3_EEPROM_MAGIC_HW))
14283 		return;
14284 
14285 	/*
14286 	 * Size the chip by reading offsets at increasing powers of two.
14287 	 * When we encounter our validation signature, we know the addressing
14288 	 * has wrapped around, and thus have our chip size.
14289 	 */
14290 	cursize = 0x10;
14291 
14292 	while (cursize < tp->nvram_size) {
14293 		if (tg3_nvram_read(tp, cursize, &val) != 0)
14294 			return;
14295 
14296 		if (val == magic)
14297 			break;
14298 
14299 		cursize <<= 1;
14300 	}
14301 
14302 	tp->nvram_size = cursize;
14303 }
14304 
14305 static void tg3_get_nvram_size(struct tg3 *tp)
14306 {
14307 	u32 val;
14308 
14309 	if (tg3_flag(tp, NO_NVRAM) || tg3_nvram_read(tp, 0, &val) != 0)
14310 		return;
14311 
14312 	/* Selfboot format */
14313 	if (val != TG3_EEPROM_MAGIC) {
14314 		tg3_get_eeprom_size(tp);
14315 		return;
14316 	}
14317 
14318 	if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
14319 		if (val != 0) {
14320 			/* This is confusing.  We want to operate on the
14321 			 * 16-bit value at offset 0xf2.  The tg3_nvram_read()
14322 			 * call will read from NVRAM and byteswap the data
14323 			 * according to the byteswapping settings for all
14324 			 * other register accesses.  This ensures the data we
14325 			 * want will always reside in the lower 16-bits.
14326 			 * However, the data in NVRAM is in LE format, which
14327 			 * means the data from the NVRAM read will always be
14328 			 * opposite the endianness of the CPU.  The 16-bit
14329 			 * byteswap then brings the data to CPU endianness.
14330 			 */
14331 			tp->nvram_size = swab16((u16)(val & 0x0000ffff)) * 1024;
14332 			return;
14333 		}
14334 	}
14335 	tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14336 }
14337 
14338 static void tg3_get_nvram_info(struct tg3 *tp)
14339 {
14340 	u32 nvcfg1;
14341 
14342 	nvcfg1 = tr32(NVRAM_CFG1);
14343 	if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
14344 		tg3_flag_set(tp, FLASH);
14345 	} else {
14346 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14347 		tw32(NVRAM_CFG1, nvcfg1);
14348 	}
14349 
14350 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
14351 	    tg3_flag(tp, 5780_CLASS)) {
14352 		switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
14353 		case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
14354 			tp->nvram_jedecnum = JEDEC_ATMEL;
14355 			tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
14356 			tg3_flag_set(tp, NVRAM_BUFFERED);
14357 			break;
14358 		case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
14359 			tp->nvram_jedecnum = JEDEC_ATMEL;
14360 			tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
14361 			break;
14362 		case FLASH_VENDOR_ATMEL_EEPROM:
14363 			tp->nvram_jedecnum = JEDEC_ATMEL;
14364 			tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14365 			tg3_flag_set(tp, NVRAM_BUFFERED);
14366 			break;
14367 		case FLASH_VENDOR_ST:
14368 			tp->nvram_jedecnum = JEDEC_ST;
14369 			tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
14370 			tg3_flag_set(tp, NVRAM_BUFFERED);
14371 			break;
14372 		case FLASH_VENDOR_SAIFUN:
14373 			tp->nvram_jedecnum = JEDEC_SAIFUN;
14374 			tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
14375 			break;
14376 		case FLASH_VENDOR_SST_SMALL:
14377 		case FLASH_VENDOR_SST_LARGE:
14378 			tp->nvram_jedecnum = JEDEC_SST;
14379 			tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
14380 			break;
14381 		}
14382 	} else {
14383 		tp->nvram_jedecnum = JEDEC_ATMEL;
14384 		tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
14385 		tg3_flag_set(tp, NVRAM_BUFFERED);
14386 	}
14387 }
14388 
14389 static void tg3_nvram_get_pagesize(struct tg3 *tp, u32 nvmcfg1)
14390 {
14391 	switch (nvmcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
14392 	case FLASH_5752PAGE_SIZE_256:
14393 		tp->nvram_pagesize = 256;
14394 		break;
14395 	case FLASH_5752PAGE_SIZE_512:
14396 		tp->nvram_pagesize = 512;
14397 		break;
14398 	case FLASH_5752PAGE_SIZE_1K:
14399 		tp->nvram_pagesize = 1024;
14400 		break;
14401 	case FLASH_5752PAGE_SIZE_2K:
14402 		tp->nvram_pagesize = 2048;
14403 		break;
14404 	case FLASH_5752PAGE_SIZE_4K:
14405 		tp->nvram_pagesize = 4096;
14406 		break;
14407 	case FLASH_5752PAGE_SIZE_264:
14408 		tp->nvram_pagesize = 264;
14409 		break;
14410 	case FLASH_5752PAGE_SIZE_528:
14411 		tp->nvram_pagesize = 528;
14412 		break;
14413 	}
14414 }
14415 
14416 static void tg3_get_5752_nvram_info(struct tg3 *tp)
14417 {
14418 	u32 nvcfg1;
14419 
14420 	nvcfg1 = tr32(NVRAM_CFG1);
14421 
14422 	/* NVRAM protection for TPM */
14423 	if (nvcfg1 & (1 << 27))
14424 		tg3_flag_set(tp, PROTECTED_NVRAM);
14425 
14426 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14427 	case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
14428 	case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
14429 		tp->nvram_jedecnum = JEDEC_ATMEL;
14430 		tg3_flag_set(tp, NVRAM_BUFFERED);
14431 		break;
14432 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14433 		tp->nvram_jedecnum = JEDEC_ATMEL;
14434 		tg3_flag_set(tp, NVRAM_BUFFERED);
14435 		tg3_flag_set(tp, FLASH);
14436 		break;
14437 	case FLASH_5752VENDOR_ST_M45PE10:
14438 	case FLASH_5752VENDOR_ST_M45PE20:
14439 	case FLASH_5752VENDOR_ST_M45PE40:
14440 		tp->nvram_jedecnum = JEDEC_ST;
14441 		tg3_flag_set(tp, NVRAM_BUFFERED);
14442 		tg3_flag_set(tp, FLASH);
14443 		break;
14444 	}
14445 
14446 	if (tg3_flag(tp, FLASH)) {
14447 		tg3_nvram_get_pagesize(tp, nvcfg1);
14448 	} else {
14449 		/* For eeprom, set pagesize to maximum eeprom size */
14450 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14451 
14452 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14453 		tw32(NVRAM_CFG1, nvcfg1);
14454 	}
14455 }
14456 
14457 static void tg3_get_5755_nvram_info(struct tg3 *tp)
14458 {
14459 	u32 nvcfg1, protect = 0;
14460 
14461 	nvcfg1 = tr32(NVRAM_CFG1);
14462 
14463 	/* NVRAM protection for TPM */
14464 	if (nvcfg1 & (1 << 27)) {
14465 		tg3_flag_set(tp, PROTECTED_NVRAM);
14466 		protect = 1;
14467 	}
14468 
14469 	nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
14470 	switch (nvcfg1) {
14471 	case FLASH_5755VENDOR_ATMEL_FLASH_1:
14472 	case FLASH_5755VENDOR_ATMEL_FLASH_2:
14473 	case FLASH_5755VENDOR_ATMEL_FLASH_3:
14474 	case FLASH_5755VENDOR_ATMEL_FLASH_5:
14475 		tp->nvram_jedecnum = JEDEC_ATMEL;
14476 		tg3_flag_set(tp, NVRAM_BUFFERED);
14477 		tg3_flag_set(tp, FLASH);
14478 		tp->nvram_pagesize = 264;
14479 		if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_1 ||
14480 		    nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_5)
14481 			tp->nvram_size = (protect ? 0x3e200 :
14482 					  TG3_NVRAM_SIZE_512KB);
14483 		else if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_2)
14484 			tp->nvram_size = (protect ? 0x1f200 :
14485 					  TG3_NVRAM_SIZE_256KB);
14486 		else
14487 			tp->nvram_size = (protect ? 0x1f200 :
14488 					  TG3_NVRAM_SIZE_128KB);
14489 		break;
14490 	case FLASH_5752VENDOR_ST_M45PE10:
14491 	case FLASH_5752VENDOR_ST_M45PE20:
14492 	case FLASH_5752VENDOR_ST_M45PE40:
14493 		tp->nvram_jedecnum = JEDEC_ST;
14494 		tg3_flag_set(tp, NVRAM_BUFFERED);
14495 		tg3_flag_set(tp, FLASH);
14496 		tp->nvram_pagesize = 256;
14497 		if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE10)
14498 			tp->nvram_size = (protect ?
14499 					  TG3_NVRAM_SIZE_64KB :
14500 					  TG3_NVRAM_SIZE_128KB);
14501 		else if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE20)
14502 			tp->nvram_size = (protect ?
14503 					  TG3_NVRAM_SIZE_64KB :
14504 					  TG3_NVRAM_SIZE_256KB);
14505 		else
14506 			tp->nvram_size = (protect ?
14507 					  TG3_NVRAM_SIZE_128KB :
14508 					  TG3_NVRAM_SIZE_512KB);
14509 		break;
14510 	}
14511 }
14512 
14513 static void tg3_get_5787_nvram_info(struct tg3 *tp)
14514 {
14515 	u32 nvcfg1;
14516 
14517 	nvcfg1 = tr32(NVRAM_CFG1);
14518 
14519 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14520 	case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
14521 	case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
14522 	case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
14523 	case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
14524 		tp->nvram_jedecnum = JEDEC_ATMEL;
14525 		tg3_flag_set(tp, NVRAM_BUFFERED);
14526 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14527 
14528 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14529 		tw32(NVRAM_CFG1, nvcfg1);
14530 		break;
14531 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14532 	case FLASH_5755VENDOR_ATMEL_FLASH_1:
14533 	case FLASH_5755VENDOR_ATMEL_FLASH_2:
14534 	case FLASH_5755VENDOR_ATMEL_FLASH_3:
14535 		tp->nvram_jedecnum = JEDEC_ATMEL;
14536 		tg3_flag_set(tp, NVRAM_BUFFERED);
14537 		tg3_flag_set(tp, FLASH);
14538 		tp->nvram_pagesize = 264;
14539 		break;
14540 	case FLASH_5752VENDOR_ST_M45PE10:
14541 	case FLASH_5752VENDOR_ST_M45PE20:
14542 	case FLASH_5752VENDOR_ST_M45PE40:
14543 		tp->nvram_jedecnum = JEDEC_ST;
14544 		tg3_flag_set(tp, NVRAM_BUFFERED);
14545 		tg3_flag_set(tp, FLASH);
14546 		tp->nvram_pagesize = 256;
14547 		break;
14548 	}
14549 }
14550 
14551 static void tg3_get_5761_nvram_info(struct tg3 *tp)
14552 {
14553 	u32 nvcfg1, protect = 0;
14554 
14555 	nvcfg1 = tr32(NVRAM_CFG1);
14556 
14557 	/* NVRAM protection for TPM */
14558 	if (nvcfg1 & (1 << 27)) {
14559 		tg3_flag_set(tp, PROTECTED_NVRAM);
14560 		protect = 1;
14561 	}
14562 
14563 	nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
14564 	switch (nvcfg1) {
14565 	case FLASH_5761VENDOR_ATMEL_ADB021D:
14566 	case FLASH_5761VENDOR_ATMEL_ADB041D:
14567 	case FLASH_5761VENDOR_ATMEL_ADB081D:
14568 	case FLASH_5761VENDOR_ATMEL_ADB161D:
14569 	case FLASH_5761VENDOR_ATMEL_MDB021D:
14570 	case FLASH_5761VENDOR_ATMEL_MDB041D:
14571 	case FLASH_5761VENDOR_ATMEL_MDB081D:
14572 	case FLASH_5761VENDOR_ATMEL_MDB161D:
14573 		tp->nvram_jedecnum = JEDEC_ATMEL;
14574 		tg3_flag_set(tp, NVRAM_BUFFERED);
14575 		tg3_flag_set(tp, FLASH);
14576 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14577 		tp->nvram_pagesize = 256;
14578 		break;
14579 	case FLASH_5761VENDOR_ST_A_M45PE20:
14580 	case FLASH_5761VENDOR_ST_A_M45PE40:
14581 	case FLASH_5761VENDOR_ST_A_M45PE80:
14582 	case FLASH_5761VENDOR_ST_A_M45PE16:
14583 	case FLASH_5761VENDOR_ST_M_M45PE20:
14584 	case FLASH_5761VENDOR_ST_M_M45PE40:
14585 	case FLASH_5761VENDOR_ST_M_M45PE80:
14586 	case FLASH_5761VENDOR_ST_M_M45PE16:
14587 		tp->nvram_jedecnum = JEDEC_ST;
14588 		tg3_flag_set(tp, NVRAM_BUFFERED);
14589 		tg3_flag_set(tp, FLASH);
14590 		tp->nvram_pagesize = 256;
14591 		break;
14592 	}
14593 
14594 	if (protect) {
14595 		tp->nvram_size = tr32(NVRAM_ADDR_LOCKOUT);
14596 	} else {
14597 		switch (nvcfg1) {
14598 		case FLASH_5761VENDOR_ATMEL_ADB161D:
14599 		case FLASH_5761VENDOR_ATMEL_MDB161D:
14600 		case FLASH_5761VENDOR_ST_A_M45PE16:
14601 		case FLASH_5761VENDOR_ST_M_M45PE16:
14602 			tp->nvram_size = TG3_NVRAM_SIZE_2MB;
14603 			break;
14604 		case FLASH_5761VENDOR_ATMEL_ADB081D:
14605 		case FLASH_5761VENDOR_ATMEL_MDB081D:
14606 		case FLASH_5761VENDOR_ST_A_M45PE80:
14607 		case FLASH_5761VENDOR_ST_M_M45PE80:
14608 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14609 			break;
14610 		case FLASH_5761VENDOR_ATMEL_ADB041D:
14611 		case FLASH_5761VENDOR_ATMEL_MDB041D:
14612 		case FLASH_5761VENDOR_ST_A_M45PE40:
14613 		case FLASH_5761VENDOR_ST_M_M45PE40:
14614 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14615 			break;
14616 		case FLASH_5761VENDOR_ATMEL_ADB021D:
14617 		case FLASH_5761VENDOR_ATMEL_MDB021D:
14618 		case FLASH_5761VENDOR_ST_A_M45PE20:
14619 		case FLASH_5761VENDOR_ST_M_M45PE20:
14620 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14621 			break;
14622 		}
14623 	}
14624 }
14625 
14626 static void tg3_get_5906_nvram_info(struct tg3 *tp)
14627 {
14628 	tp->nvram_jedecnum = JEDEC_ATMEL;
14629 	tg3_flag_set(tp, NVRAM_BUFFERED);
14630 	tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14631 }
14632 
14633 static void tg3_get_57780_nvram_info(struct tg3 *tp)
14634 {
14635 	u32 nvcfg1;
14636 
14637 	nvcfg1 = tr32(NVRAM_CFG1);
14638 
14639 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14640 	case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
14641 	case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
14642 		tp->nvram_jedecnum = JEDEC_ATMEL;
14643 		tg3_flag_set(tp, NVRAM_BUFFERED);
14644 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14645 
14646 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14647 		tw32(NVRAM_CFG1, nvcfg1);
14648 		return;
14649 	case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14650 	case FLASH_57780VENDOR_ATMEL_AT45DB011D:
14651 	case FLASH_57780VENDOR_ATMEL_AT45DB011B:
14652 	case FLASH_57780VENDOR_ATMEL_AT45DB021D:
14653 	case FLASH_57780VENDOR_ATMEL_AT45DB021B:
14654 	case FLASH_57780VENDOR_ATMEL_AT45DB041D:
14655 	case FLASH_57780VENDOR_ATMEL_AT45DB041B:
14656 		tp->nvram_jedecnum = JEDEC_ATMEL;
14657 		tg3_flag_set(tp, NVRAM_BUFFERED);
14658 		tg3_flag_set(tp, FLASH);
14659 
14660 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14661 		case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
14662 		case FLASH_57780VENDOR_ATMEL_AT45DB011D:
14663 		case FLASH_57780VENDOR_ATMEL_AT45DB011B:
14664 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14665 			break;
14666 		case FLASH_57780VENDOR_ATMEL_AT45DB021D:
14667 		case FLASH_57780VENDOR_ATMEL_AT45DB021B:
14668 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14669 			break;
14670 		case FLASH_57780VENDOR_ATMEL_AT45DB041D:
14671 		case FLASH_57780VENDOR_ATMEL_AT45DB041B:
14672 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14673 			break;
14674 		}
14675 		break;
14676 	case FLASH_5752VENDOR_ST_M45PE10:
14677 	case FLASH_5752VENDOR_ST_M45PE20:
14678 	case FLASH_5752VENDOR_ST_M45PE40:
14679 		tp->nvram_jedecnum = JEDEC_ST;
14680 		tg3_flag_set(tp, NVRAM_BUFFERED);
14681 		tg3_flag_set(tp, FLASH);
14682 
14683 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14684 		case FLASH_5752VENDOR_ST_M45PE10:
14685 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14686 			break;
14687 		case FLASH_5752VENDOR_ST_M45PE20:
14688 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14689 			break;
14690 		case FLASH_5752VENDOR_ST_M45PE40:
14691 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14692 			break;
14693 		}
14694 		break;
14695 	default:
14696 		tg3_flag_set(tp, NO_NVRAM);
14697 		return;
14698 	}
14699 
14700 	tg3_nvram_get_pagesize(tp, nvcfg1);
14701 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14702 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14703 }
14704 
14705 
14706 static void tg3_get_5717_nvram_info(struct tg3 *tp)
14707 {
14708 	u32 nvcfg1;
14709 
14710 	nvcfg1 = tr32(NVRAM_CFG1);
14711 
14712 	switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14713 	case FLASH_5717VENDOR_ATMEL_EEPROM:
14714 	case FLASH_5717VENDOR_MICRO_EEPROM:
14715 		tp->nvram_jedecnum = JEDEC_ATMEL;
14716 		tg3_flag_set(tp, NVRAM_BUFFERED);
14717 		tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14718 
14719 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14720 		tw32(NVRAM_CFG1, nvcfg1);
14721 		return;
14722 	case FLASH_5717VENDOR_ATMEL_MDB011D:
14723 	case FLASH_5717VENDOR_ATMEL_ADB011B:
14724 	case FLASH_5717VENDOR_ATMEL_ADB011D:
14725 	case FLASH_5717VENDOR_ATMEL_MDB021D:
14726 	case FLASH_5717VENDOR_ATMEL_ADB021B:
14727 	case FLASH_5717VENDOR_ATMEL_ADB021D:
14728 	case FLASH_5717VENDOR_ATMEL_45USPT:
14729 		tp->nvram_jedecnum = JEDEC_ATMEL;
14730 		tg3_flag_set(tp, NVRAM_BUFFERED);
14731 		tg3_flag_set(tp, FLASH);
14732 
14733 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14734 		case FLASH_5717VENDOR_ATMEL_MDB021D:
14735 			/* Detect size with tg3_nvram_get_size() */
14736 			break;
14737 		case FLASH_5717VENDOR_ATMEL_ADB021B:
14738 		case FLASH_5717VENDOR_ATMEL_ADB021D:
14739 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14740 			break;
14741 		default:
14742 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14743 			break;
14744 		}
14745 		break;
14746 	case FLASH_5717VENDOR_ST_M_M25PE10:
14747 	case FLASH_5717VENDOR_ST_A_M25PE10:
14748 	case FLASH_5717VENDOR_ST_M_M45PE10:
14749 	case FLASH_5717VENDOR_ST_A_M45PE10:
14750 	case FLASH_5717VENDOR_ST_M_M25PE20:
14751 	case FLASH_5717VENDOR_ST_A_M25PE20:
14752 	case FLASH_5717VENDOR_ST_M_M45PE20:
14753 	case FLASH_5717VENDOR_ST_A_M45PE20:
14754 	case FLASH_5717VENDOR_ST_25USPT:
14755 	case FLASH_5717VENDOR_ST_45USPT:
14756 		tp->nvram_jedecnum = JEDEC_ST;
14757 		tg3_flag_set(tp, NVRAM_BUFFERED);
14758 		tg3_flag_set(tp, FLASH);
14759 
14760 		switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
14761 		case FLASH_5717VENDOR_ST_M_M25PE20:
14762 		case FLASH_5717VENDOR_ST_M_M45PE20:
14763 			/* Detect size with tg3_nvram_get_size() */
14764 			break;
14765 		case FLASH_5717VENDOR_ST_A_M25PE20:
14766 		case FLASH_5717VENDOR_ST_A_M45PE20:
14767 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14768 			break;
14769 		default:
14770 			tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14771 			break;
14772 		}
14773 		break;
14774 	default:
14775 		tg3_flag_set(tp, NO_NVRAM);
14776 		return;
14777 	}
14778 
14779 	tg3_nvram_get_pagesize(tp, nvcfg1);
14780 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14781 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14782 }
14783 
14784 static void tg3_get_5720_nvram_info(struct tg3 *tp)
14785 {
14786 	u32 nvcfg1, nvmpinstrp, nv_status;
14787 
14788 	nvcfg1 = tr32(NVRAM_CFG1);
14789 	nvmpinstrp = nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK;
14790 
14791 	if (tg3_asic_rev(tp) == ASIC_REV_5762) {
14792 		if (!(nvcfg1 & NVRAM_CFG1_5762VENDOR_MASK)) {
14793 			tg3_flag_set(tp, NO_NVRAM);
14794 			return;
14795 		}
14796 
14797 		switch (nvmpinstrp) {
14798 		case FLASH_5762_MX25L_100:
14799 		case FLASH_5762_MX25L_200:
14800 		case FLASH_5762_MX25L_400:
14801 		case FLASH_5762_MX25L_800:
14802 		case FLASH_5762_MX25L_160_320:
14803 			tp->nvram_pagesize = 4096;
14804 			tp->nvram_jedecnum = JEDEC_MACRONIX;
14805 			tg3_flag_set(tp, NVRAM_BUFFERED);
14806 			tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14807 			tg3_flag_set(tp, FLASH);
14808 			nv_status = tr32(NVRAM_AUTOSENSE_STATUS);
14809 			tp->nvram_size =
14810 				(1 << (nv_status >> AUTOSENSE_DEVID &
14811 						AUTOSENSE_DEVID_MASK)
14812 					<< AUTOSENSE_SIZE_IN_MB);
14813 			return;
14814 
14815 		case FLASH_5762_EEPROM_HD:
14816 			nvmpinstrp = FLASH_5720_EEPROM_HD;
14817 			break;
14818 		case FLASH_5762_EEPROM_LD:
14819 			nvmpinstrp = FLASH_5720_EEPROM_LD;
14820 			break;
14821 		case FLASH_5720VENDOR_M_ST_M45PE20:
14822 			/* This pinstrap supports multiple sizes, so force it
14823 			 * to read the actual size from location 0xf0.
14824 			 */
14825 			nvmpinstrp = FLASH_5720VENDOR_ST_45USPT;
14826 			break;
14827 		}
14828 	}
14829 
14830 	switch (nvmpinstrp) {
14831 	case FLASH_5720_EEPROM_HD:
14832 	case FLASH_5720_EEPROM_LD:
14833 		tp->nvram_jedecnum = JEDEC_ATMEL;
14834 		tg3_flag_set(tp, NVRAM_BUFFERED);
14835 
14836 		nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
14837 		tw32(NVRAM_CFG1, nvcfg1);
14838 		if (nvmpinstrp == FLASH_5720_EEPROM_HD)
14839 			tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
14840 		else
14841 			tp->nvram_pagesize = ATMEL_AT24C02_CHIP_SIZE;
14842 		return;
14843 	case FLASH_5720VENDOR_M_ATMEL_DB011D:
14844 	case FLASH_5720VENDOR_A_ATMEL_DB011B:
14845 	case FLASH_5720VENDOR_A_ATMEL_DB011D:
14846 	case FLASH_5720VENDOR_M_ATMEL_DB021D:
14847 	case FLASH_5720VENDOR_A_ATMEL_DB021B:
14848 	case FLASH_5720VENDOR_A_ATMEL_DB021D:
14849 	case FLASH_5720VENDOR_M_ATMEL_DB041D:
14850 	case FLASH_5720VENDOR_A_ATMEL_DB041B:
14851 	case FLASH_5720VENDOR_A_ATMEL_DB041D:
14852 	case FLASH_5720VENDOR_M_ATMEL_DB081D:
14853 	case FLASH_5720VENDOR_A_ATMEL_DB081D:
14854 	case FLASH_5720VENDOR_ATMEL_45USPT:
14855 		tp->nvram_jedecnum = JEDEC_ATMEL;
14856 		tg3_flag_set(tp, NVRAM_BUFFERED);
14857 		tg3_flag_set(tp, FLASH);
14858 
14859 		switch (nvmpinstrp) {
14860 		case FLASH_5720VENDOR_M_ATMEL_DB021D:
14861 		case FLASH_5720VENDOR_A_ATMEL_DB021B:
14862 		case FLASH_5720VENDOR_A_ATMEL_DB021D:
14863 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14864 			break;
14865 		case FLASH_5720VENDOR_M_ATMEL_DB041D:
14866 		case FLASH_5720VENDOR_A_ATMEL_DB041B:
14867 		case FLASH_5720VENDOR_A_ATMEL_DB041D:
14868 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14869 			break;
14870 		case FLASH_5720VENDOR_M_ATMEL_DB081D:
14871 		case FLASH_5720VENDOR_A_ATMEL_DB081D:
14872 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14873 			break;
14874 		default:
14875 			if (tg3_asic_rev(tp) != ASIC_REV_5762)
14876 				tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14877 			break;
14878 		}
14879 		break;
14880 	case FLASH_5720VENDOR_M_ST_M25PE10:
14881 	case FLASH_5720VENDOR_M_ST_M45PE10:
14882 	case FLASH_5720VENDOR_A_ST_M25PE10:
14883 	case FLASH_5720VENDOR_A_ST_M45PE10:
14884 	case FLASH_5720VENDOR_M_ST_M25PE20:
14885 	case FLASH_5720VENDOR_M_ST_M45PE20:
14886 	case FLASH_5720VENDOR_A_ST_M25PE20:
14887 	case FLASH_5720VENDOR_A_ST_M45PE20:
14888 	case FLASH_5720VENDOR_M_ST_M25PE40:
14889 	case FLASH_5720VENDOR_M_ST_M45PE40:
14890 	case FLASH_5720VENDOR_A_ST_M25PE40:
14891 	case FLASH_5720VENDOR_A_ST_M45PE40:
14892 	case FLASH_5720VENDOR_M_ST_M25PE80:
14893 	case FLASH_5720VENDOR_M_ST_M45PE80:
14894 	case FLASH_5720VENDOR_A_ST_M25PE80:
14895 	case FLASH_5720VENDOR_A_ST_M45PE80:
14896 	case FLASH_5720VENDOR_ST_25USPT:
14897 	case FLASH_5720VENDOR_ST_45USPT:
14898 		tp->nvram_jedecnum = JEDEC_ST;
14899 		tg3_flag_set(tp, NVRAM_BUFFERED);
14900 		tg3_flag_set(tp, FLASH);
14901 
14902 		switch (nvmpinstrp) {
14903 		case FLASH_5720VENDOR_M_ST_M25PE20:
14904 		case FLASH_5720VENDOR_M_ST_M45PE20:
14905 		case FLASH_5720VENDOR_A_ST_M25PE20:
14906 		case FLASH_5720VENDOR_A_ST_M45PE20:
14907 			tp->nvram_size = TG3_NVRAM_SIZE_256KB;
14908 			break;
14909 		case FLASH_5720VENDOR_M_ST_M25PE40:
14910 		case FLASH_5720VENDOR_M_ST_M45PE40:
14911 		case FLASH_5720VENDOR_A_ST_M25PE40:
14912 		case FLASH_5720VENDOR_A_ST_M45PE40:
14913 			tp->nvram_size = TG3_NVRAM_SIZE_512KB;
14914 			break;
14915 		case FLASH_5720VENDOR_M_ST_M25PE80:
14916 		case FLASH_5720VENDOR_M_ST_M45PE80:
14917 		case FLASH_5720VENDOR_A_ST_M25PE80:
14918 		case FLASH_5720VENDOR_A_ST_M45PE80:
14919 			tp->nvram_size = TG3_NVRAM_SIZE_1MB;
14920 			break;
14921 		default:
14922 			if (tg3_asic_rev(tp) != ASIC_REV_5762)
14923 				tp->nvram_size = TG3_NVRAM_SIZE_128KB;
14924 			break;
14925 		}
14926 		break;
14927 	default:
14928 		tg3_flag_set(tp, NO_NVRAM);
14929 		return;
14930 	}
14931 
14932 	tg3_nvram_get_pagesize(tp, nvcfg1);
14933 	if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
14934 		tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
14935 
14936 	if (tg3_asic_rev(tp) == ASIC_REV_5762) {
14937 		u32 val;
14938 
14939 		if (tg3_nvram_read(tp, 0, &val))
14940 			return;
14941 
14942 		if (val != TG3_EEPROM_MAGIC &&
14943 		    (val & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW)
14944 			tg3_flag_set(tp, NO_NVRAM);
14945 	}
14946 }
14947 
14948 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
14949 static void tg3_nvram_init(struct tg3 *tp)
14950 {
14951 	if (tg3_flag(tp, IS_SSB_CORE)) {
14952 		/* No NVRAM and EEPROM on the SSB Broadcom GigE core. */
14953 		tg3_flag_clear(tp, NVRAM);
14954 		tg3_flag_clear(tp, NVRAM_BUFFERED);
14955 		tg3_flag_set(tp, NO_NVRAM);
14956 		return;
14957 	}
14958 
14959 	tw32_f(GRC_EEPROM_ADDR,
14960 	     (EEPROM_ADDR_FSM_RESET |
14961 	      (EEPROM_DEFAULT_CLOCK_PERIOD <<
14962 	       EEPROM_ADDR_CLKPERD_SHIFT)));
14963 
14964 	msleep(1);
14965 
14966 	/* Enable seeprom accesses. */
14967 	tw32_f(GRC_LOCAL_CTRL,
14968 	     tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
14969 	udelay(100);
14970 
14971 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
14972 	    tg3_asic_rev(tp) != ASIC_REV_5701) {
14973 		tg3_flag_set(tp, NVRAM);
14974 
14975 		if (tg3_nvram_lock(tp)) {
14976 			netdev_warn(tp->dev,
14977 				    "Cannot get nvram lock, %s failed\n",
14978 				    __func__);
14979 			return;
14980 		}
14981 		tg3_enable_nvram_access(tp);
14982 
14983 		tp->nvram_size = 0;
14984 
14985 		if (tg3_asic_rev(tp) == ASIC_REV_5752)
14986 			tg3_get_5752_nvram_info(tp);
14987 		else if (tg3_asic_rev(tp) == ASIC_REV_5755)
14988 			tg3_get_5755_nvram_info(tp);
14989 		else if (tg3_asic_rev(tp) == ASIC_REV_5787 ||
14990 			 tg3_asic_rev(tp) == ASIC_REV_5784 ||
14991 			 tg3_asic_rev(tp) == ASIC_REV_5785)
14992 			tg3_get_5787_nvram_info(tp);
14993 		else if (tg3_asic_rev(tp) == ASIC_REV_5761)
14994 			tg3_get_5761_nvram_info(tp);
14995 		else if (tg3_asic_rev(tp) == ASIC_REV_5906)
14996 			tg3_get_5906_nvram_info(tp);
14997 		else if (tg3_asic_rev(tp) == ASIC_REV_57780 ||
14998 			 tg3_flag(tp, 57765_CLASS))
14999 			tg3_get_57780_nvram_info(tp);
15000 		else if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
15001 			 tg3_asic_rev(tp) == ASIC_REV_5719)
15002 			tg3_get_5717_nvram_info(tp);
15003 		else if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
15004 			 tg3_asic_rev(tp) == ASIC_REV_5762)
15005 			tg3_get_5720_nvram_info(tp);
15006 		else
15007 			tg3_get_nvram_info(tp);
15008 
15009 		if (tp->nvram_size == 0)
15010 			tg3_get_nvram_size(tp);
15011 
15012 		tg3_disable_nvram_access(tp);
15013 		tg3_nvram_unlock(tp);
15014 
15015 	} else {
15016 		tg3_flag_clear(tp, NVRAM);
15017 		tg3_flag_clear(tp, NVRAM_BUFFERED);
15018 
15019 		tg3_get_eeprom_size(tp);
15020 	}
15021 }
15022 
15023 struct subsys_tbl_ent {
15024 	u16 subsys_vendor, subsys_devid;
15025 	u32 phy_id;
15026 };
15027 
15028 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
15029 	/* Broadcom boards. */
15030 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15031 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700A6, TG3_PHY_ID_BCM5401 },
15032 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15033 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A5, TG3_PHY_ID_BCM5701 },
15034 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15035 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700T6, TG3_PHY_ID_BCM8002 },
15036 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15037 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95700A9, 0 },
15038 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15039 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701T1, TG3_PHY_ID_BCM5701 },
15040 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15041 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701T8, TG3_PHY_ID_BCM5701 },
15042 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15043 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A7, 0 },
15044 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15045 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A10, TG3_PHY_ID_BCM5701 },
15046 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15047 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95701A12, TG3_PHY_ID_BCM5701 },
15048 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15049 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95703AX1, TG3_PHY_ID_BCM5703 },
15050 	{ TG3PCI_SUBVENDOR_ID_BROADCOM,
15051 	  TG3PCI_SUBDEVICE_ID_BROADCOM_95703AX2, TG3_PHY_ID_BCM5703 },
15052 
15053 	/* 3com boards. */
15054 	{ TG3PCI_SUBVENDOR_ID_3COM,
15055 	  TG3PCI_SUBDEVICE_ID_3COM_3C996T, TG3_PHY_ID_BCM5401 },
15056 	{ TG3PCI_SUBVENDOR_ID_3COM,
15057 	  TG3PCI_SUBDEVICE_ID_3COM_3C996BT, TG3_PHY_ID_BCM5701 },
15058 	{ TG3PCI_SUBVENDOR_ID_3COM,
15059 	  TG3PCI_SUBDEVICE_ID_3COM_3C996SX, 0 },
15060 	{ TG3PCI_SUBVENDOR_ID_3COM,
15061 	  TG3PCI_SUBDEVICE_ID_3COM_3C1000T, TG3_PHY_ID_BCM5701 },
15062 	{ TG3PCI_SUBVENDOR_ID_3COM,
15063 	  TG3PCI_SUBDEVICE_ID_3COM_3C940BR01, TG3_PHY_ID_BCM5701 },
15064 
15065 	/* DELL boards. */
15066 	{ TG3PCI_SUBVENDOR_ID_DELL,
15067 	  TG3PCI_SUBDEVICE_ID_DELL_VIPER, TG3_PHY_ID_BCM5401 },
15068 	{ TG3PCI_SUBVENDOR_ID_DELL,
15069 	  TG3PCI_SUBDEVICE_ID_DELL_JAGUAR, TG3_PHY_ID_BCM5401 },
15070 	{ TG3PCI_SUBVENDOR_ID_DELL,
15071 	  TG3PCI_SUBDEVICE_ID_DELL_MERLOT, TG3_PHY_ID_BCM5411 },
15072 	{ TG3PCI_SUBVENDOR_ID_DELL,
15073 	  TG3PCI_SUBDEVICE_ID_DELL_SLIM_MERLOT, TG3_PHY_ID_BCM5411 },
15074 
15075 	/* Compaq boards. */
15076 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15077 	  TG3PCI_SUBDEVICE_ID_COMPAQ_BANSHEE, TG3_PHY_ID_BCM5701 },
15078 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15079 	  TG3PCI_SUBDEVICE_ID_COMPAQ_BANSHEE_2, TG3_PHY_ID_BCM5701 },
15080 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15081 	  TG3PCI_SUBDEVICE_ID_COMPAQ_CHANGELING, 0 },
15082 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15083 	  TG3PCI_SUBDEVICE_ID_COMPAQ_NC7780, TG3_PHY_ID_BCM5701 },
15084 	{ TG3PCI_SUBVENDOR_ID_COMPAQ,
15085 	  TG3PCI_SUBDEVICE_ID_COMPAQ_NC7780_2, TG3_PHY_ID_BCM5701 },
15086 
15087 	/* IBM boards. */
15088 	{ TG3PCI_SUBVENDOR_ID_IBM,
15089 	  TG3PCI_SUBDEVICE_ID_IBM_5703SAX2, 0 }
15090 };
15091 
15092 static struct subsys_tbl_ent *tg3_lookup_by_subsys(struct tg3 *tp)
15093 {
15094 	int i;
15095 
15096 	for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
15097 		if ((subsys_id_to_phy_id[i].subsys_vendor ==
15098 		     tp->pdev->subsystem_vendor) &&
15099 		    (subsys_id_to_phy_id[i].subsys_devid ==
15100 		     tp->pdev->subsystem_device))
15101 			return &subsys_id_to_phy_id[i];
15102 	}
15103 	return NULL;
15104 }
15105 
15106 static void tg3_get_eeprom_hw_cfg(struct tg3 *tp)
15107 {
15108 	u32 val;
15109 
15110 	tp->phy_id = TG3_PHY_ID_INVALID;
15111 	tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15112 
15113 	/* Assume an onboard device and WOL capable by default.  */
15114 	tg3_flag_set(tp, EEPROM_WRITE_PROT);
15115 	tg3_flag_set(tp, WOL_CAP);
15116 
15117 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
15118 		if (!(tr32(PCIE_TRANSACTION_CFG) & PCIE_TRANS_CFG_LOM)) {
15119 			tg3_flag_clear(tp, EEPROM_WRITE_PROT);
15120 			tg3_flag_set(tp, IS_NIC);
15121 		}
15122 		val = tr32(VCPU_CFGSHDW);
15123 		if (val & VCPU_CFGSHDW_ASPM_DBNC)
15124 			tg3_flag_set(tp, ASPM_WORKAROUND);
15125 		if ((val & VCPU_CFGSHDW_WOL_ENABLE) &&
15126 		    (val & VCPU_CFGSHDW_WOL_MAGPKT)) {
15127 			tg3_flag_set(tp, WOL_ENABLE);
15128 			device_set_wakeup_enable(&tp->pdev->dev, true);
15129 		}
15130 		goto done;
15131 	}
15132 
15133 	tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
15134 	if (val == NIC_SRAM_DATA_SIG_MAGIC) {
15135 		u32 nic_cfg, led_cfg;
15136 		u32 cfg2 = 0, cfg4 = 0, cfg5 = 0;
15137 		u32 nic_phy_id, ver, eeprom_phy_id;
15138 		int eeprom_phy_serdes = 0;
15139 
15140 		tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
15141 		tp->nic_sram_data_cfg = nic_cfg;
15142 
15143 		tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
15144 		ver >>= NIC_SRAM_DATA_VER_SHIFT;
15145 		if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
15146 		    tg3_asic_rev(tp) != ASIC_REV_5701 &&
15147 		    tg3_asic_rev(tp) != ASIC_REV_5703 &&
15148 		    (ver > 0) && (ver < 0x100))
15149 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
15150 
15151 		if (tg3_asic_rev(tp) == ASIC_REV_5785)
15152 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_4, &cfg4);
15153 
15154 		if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
15155 		    tg3_asic_rev(tp) == ASIC_REV_5719 ||
15156 		    tg3_asic_rev(tp) == ASIC_REV_5720)
15157 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_5, &cfg5);
15158 
15159 		if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
15160 		    NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
15161 			eeprom_phy_serdes = 1;
15162 
15163 		tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
15164 		if (nic_phy_id != 0) {
15165 			u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
15166 			u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
15167 
15168 			eeprom_phy_id  = (id1 >> 16) << 10;
15169 			eeprom_phy_id |= (id2 & 0xfc00) << 16;
15170 			eeprom_phy_id |= (id2 & 0x03ff) <<  0;
15171 		} else
15172 			eeprom_phy_id = 0;
15173 
15174 		tp->phy_id = eeprom_phy_id;
15175 		if (eeprom_phy_serdes) {
15176 			if (!tg3_flag(tp, 5705_PLUS))
15177 				tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15178 			else
15179 				tp->phy_flags |= TG3_PHYFLG_MII_SERDES;
15180 		}
15181 
15182 		if (tg3_flag(tp, 5750_PLUS))
15183 			led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
15184 				    SHASTA_EXT_LED_MODE_MASK);
15185 		else
15186 			led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
15187 
15188 		switch (led_cfg) {
15189 		default:
15190 		case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
15191 			tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15192 			break;
15193 
15194 		case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
15195 			tp->led_ctrl = LED_CTRL_MODE_PHY_2;
15196 			break;
15197 
15198 		case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
15199 			tp->led_ctrl = LED_CTRL_MODE_MAC;
15200 
15201 			/* Default to PHY_1_MODE if 0 (MAC_MODE) is
15202 			 * read on some older 5700/5701 bootcode.
15203 			 */
15204 			if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
15205 			    tg3_asic_rev(tp) == ASIC_REV_5701)
15206 				tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15207 
15208 			break;
15209 
15210 		case SHASTA_EXT_LED_SHARED:
15211 			tp->led_ctrl = LED_CTRL_MODE_SHARED;
15212 			if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0 &&
15213 			    tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A1)
15214 				tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
15215 						 LED_CTRL_MODE_PHY_2);
15216 
15217 			if (tg3_flag(tp, 5717_PLUS) ||
15218 			    tg3_asic_rev(tp) == ASIC_REV_5762)
15219 				tp->led_ctrl |= LED_CTRL_BLINK_RATE_OVERRIDE |
15220 						LED_CTRL_BLINK_RATE_MASK;
15221 
15222 			break;
15223 
15224 		case SHASTA_EXT_LED_MAC:
15225 			tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
15226 			break;
15227 
15228 		case SHASTA_EXT_LED_COMBO:
15229 			tp->led_ctrl = LED_CTRL_MODE_COMBO;
15230 			if (tg3_chip_rev_id(tp) != CHIPREV_ID_5750_A0)
15231 				tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
15232 						 LED_CTRL_MODE_PHY_2);
15233 			break;
15234 
15235 		}
15236 
15237 		if ((tg3_asic_rev(tp) == ASIC_REV_5700 ||
15238 		     tg3_asic_rev(tp) == ASIC_REV_5701) &&
15239 		    tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
15240 			tp->led_ctrl = LED_CTRL_MODE_PHY_2;
15241 
15242 		if (tg3_chip_rev(tp) == CHIPREV_5784_AX)
15243 			tp->led_ctrl = LED_CTRL_MODE_PHY_1;
15244 
15245 		if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP) {
15246 			tg3_flag_set(tp, EEPROM_WRITE_PROT);
15247 			if ((tp->pdev->subsystem_vendor ==
15248 			     PCI_VENDOR_ID_ARIMA) &&
15249 			    (tp->pdev->subsystem_device == 0x205a ||
15250 			     tp->pdev->subsystem_device == 0x2063))
15251 				tg3_flag_clear(tp, EEPROM_WRITE_PROT);
15252 		} else {
15253 			tg3_flag_clear(tp, EEPROM_WRITE_PROT);
15254 			tg3_flag_set(tp, IS_NIC);
15255 		}
15256 
15257 		if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
15258 			tg3_flag_set(tp, ENABLE_ASF);
15259 			if (tg3_flag(tp, 5750_PLUS))
15260 				tg3_flag_set(tp, ASF_NEW_HANDSHAKE);
15261 		}
15262 
15263 		if ((nic_cfg & NIC_SRAM_DATA_CFG_APE_ENABLE) &&
15264 		    tg3_flag(tp, 5750_PLUS))
15265 			tg3_flag_set(tp, ENABLE_APE);
15266 
15267 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES &&
15268 		    !(nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL))
15269 			tg3_flag_clear(tp, WOL_CAP);
15270 
15271 		if (tg3_flag(tp, WOL_CAP) &&
15272 		    (nic_cfg & NIC_SRAM_DATA_CFG_WOL_ENABLE)) {
15273 			tg3_flag_set(tp, WOL_ENABLE);
15274 			device_set_wakeup_enable(&tp->pdev->dev, true);
15275 		}
15276 
15277 		if (cfg2 & (1 << 17))
15278 			tp->phy_flags |= TG3_PHYFLG_CAPACITIVE_COUPLING;
15279 
15280 		/* serdes signal pre-emphasis in register 0x590 set by */
15281 		/* bootcode if bit 18 is set */
15282 		if (cfg2 & (1 << 18))
15283 			tp->phy_flags |= TG3_PHYFLG_SERDES_PREEMPHASIS;
15284 
15285 		if ((tg3_flag(tp, 57765_PLUS) ||
15286 		     (tg3_asic_rev(tp) == ASIC_REV_5784 &&
15287 		      tg3_chip_rev(tp) != CHIPREV_5784_AX)) &&
15288 		    (cfg2 & NIC_SRAM_DATA_CFG_2_APD_EN))
15289 			tp->phy_flags |= TG3_PHYFLG_ENABLE_APD;
15290 
15291 		if (tg3_flag(tp, PCI_EXPRESS)) {
15292 			u32 cfg3;
15293 
15294 			tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &cfg3);
15295 			if (tg3_asic_rev(tp) != ASIC_REV_5785 &&
15296 			    !tg3_flag(tp, 57765_PLUS) &&
15297 			    (cfg3 & NIC_SRAM_ASPM_DEBOUNCE))
15298 				tg3_flag_set(tp, ASPM_WORKAROUND);
15299 			if (cfg3 & NIC_SRAM_LNK_FLAP_AVOID)
15300 				tp->phy_flags |= TG3_PHYFLG_KEEP_LINK_ON_PWRDN;
15301 			if (cfg3 & NIC_SRAM_1G_ON_VAUX_OK)
15302 				tp->phy_flags |= TG3_PHYFLG_1G_ON_VAUX_OK;
15303 		}
15304 
15305 		if (cfg4 & NIC_SRAM_RGMII_INBAND_DISABLE)
15306 			tg3_flag_set(tp, RGMII_INBAND_DISABLE);
15307 		if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_RX_EN)
15308 			tg3_flag_set(tp, RGMII_EXT_IBND_RX_EN);
15309 		if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_TX_EN)
15310 			tg3_flag_set(tp, RGMII_EXT_IBND_TX_EN);
15311 
15312 		if (cfg5 & NIC_SRAM_DISABLE_1G_HALF_ADV)
15313 			tp->phy_flags |= TG3_PHYFLG_DISABLE_1G_HD_ADV;
15314 	}
15315 done:
15316 	if (tg3_flag(tp, WOL_CAP))
15317 		device_set_wakeup_enable(&tp->pdev->dev,
15318 					 tg3_flag(tp, WOL_ENABLE));
15319 	else
15320 		device_set_wakeup_capable(&tp->pdev->dev, false);
15321 }
15322 
15323 static int tg3_ape_otp_read(struct tg3 *tp, u32 offset, u32 *val)
15324 {
15325 	int i, err;
15326 	u32 val2, off = offset * 8;
15327 
15328 	err = tg3_nvram_lock(tp);
15329 	if (err)
15330 		return err;
15331 
15332 	tg3_ape_write32(tp, TG3_APE_OTP_ADDR, off | APE_OTP_ADDR_CPU_ENABLE);
15333 	tg3_ape_write32(tp, TG3_APE_OTP_CTRL, APE_OTP_CTRL_PROG_EN |
15334 			APE_OTP_CTRL_CMD_RD | APE_OTP_CTRL_START);
15335 	tg3_ape_read32(tp, TG3_APE_OTP_CTRL);
15336 	udelay(10);
15337 
15338 	for (i = 0; i < 100; i++) {
15339 		val2 = tg3_ape_read32(tp, TG3_APE_OTP_STATUS);
15340 		if (val2 & APE_OTP_STATUS_CMD_DONE) {
15341 			*val = tg3_ape_read32(tp, TG3_APE_OTP_RD_DATA);
15342 			break;
15343 		}
15344 		udelay(10);
15345 	}
15346 
15347 	tg3_ape_write32(tp, TG3_APE_OTP_CTRL, 0);
15348 
15349 	tg3_nvram_unlock(tp);
15350 	if (val2 & APE_OTP_STATUS_CMD_DONE)
15351 		return 0;
15352 
15353 	return -EBUSY;
15354 }
15355 
15356 static int tg3_issue_otp_command(struct tg3 *tp, u32 cmd)
15357 {
15358 	int i;
15359 	u32 val;
15360 
15361 	tw32(OTP_CTRL, cmd | OTP_CTRL_OTP_CMD_START);
15362 	tw32(OTP_CTRL, cmd);
15363 
15364 	/* Wait for up to 1 ms for command to execute. */
15365 	for (i = 0; i < 100; i++) {
15366 		val = tr32(OTP_STATUS);
15367 		if (val & OTP_STATUS_CMD_DONE)
15368 			break;
15369 		udelay(10);
15370 	}
15371 
15372 	return (val & OTP_STATUS_CMD_DONE) ? 0 : -EBUSY;
15373 }
15374 
15375 /* Read the gphy configuration from the OTP region of the chip.  The gphy
15376  * configuration is a 32-bit value that straddles the alignment boundary.
15377  * We do two 32-bit reads and then shift and merge the results.
15378  */
15379 static u32 tg3_read_otp_phycfg(struct tg3 *tp)
15380 {
15381 	u32 bhalf_otp, thalf_otp;
15382 
15383 	tw32(OTP_MODE, OTP_MODE_OTP_THRU_GRC);
15384 
15385 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_INIT))
15386 		return 0;
15387 
15388 	tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC1);
15389 
15390 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
15391 		return 0;
15392 
15393 	thalf_otp = tr32(OTP_READ_DATA);
15394 
15395 	tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC2);
15396 
15397 	if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
15398 		return 0;
15399 
15400 	bhalf_otp = tr32(OTP_READ_DATA);
15401 
15402 	return ((thalf_otp & 0x0000ffff) << 16) | (bhalf_otp >> 16);
15403 }
15404 
15405 static void tg3_phy_init_link_config(struct tg3 *tp)
15406 {
15407 	u32 adv = ADVERTISED_Autoneg;
15408 
15409 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
15410 		if (!(tp->phy_flags & TG3_PHYFLG_DISABLE_1G_HD_ADV))
15411 			adv |= ADVERTISED_1000baseT_Half;
15412 		adv |= ADVERTISED_1000baseT_Full;
15413 	}
15414 
15415 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
15416 		adv |= ADVERTISED_100baseT_Half |
15417 		       ADVERTISED_100baseT_Full |
15418 		       ADVERTISED_10baseT_Half |
15419 		       ADVERTISED_10baseT_Full |
15420 		       ADVERTISED_TP;
15421 	else
15422 		adv |= ADVERTISED_FIBRE;
15423 
15424 	tp->link_config.advertising = adv;
15425 	tp->link_config.speed = SPEED_UNKNOWN;
15426 	tp->link_config.duplex = DUPLEX_UNKNOWN;
15427 	tp->link_config.autoneg = AUTONEG_ENABLE;
15428 	tp->link_config.active_speed = SPEED_UNKNOWN;
15429 	tp->link_config.active_duplex = DUPLEX_UNKNOWN;
15430 
15431 	tp->old_link = -1;
15432 }
15433 
15434 static int tg3_phy_probe(struct tg3 *tp)
15435 {
15436 	u32 hw_phy_id_1, hw_phy_id_2;
15437 	u32 hw_phy_id, hw_phy_id_masked;
15438 	int err;
15439 
15440 	/* flow control autonegotiation is default behavior */
15441 	tg3_flag_set(tp, PAUSE_AUTONEG);
15442 	tp->link_config.flowctrl = FLOW_CTRL_TX | FLOW_CTRL_RX;
15443 
15444 	if (tg3_flag(tp, ENABLE_APE)) {
15445 		switch (tp->pci_fn) {
15446 		case 0:
15447 			tp->phy_ape_lock = TG3_APE_LOCK_PHY0;
15448 			break;
15449 		case 1:
15450 			tp->phy_ape_lock = TG3_APE_LOCK_PHY1;
15451 			break;
15452 		case 2:
15453 			tp->phy_ape_lock = TG3_APE_LOCK_PHY2;
15454 			break;
15455 		case 3:
15456 			tp->phy_ape_lock = TG3_APE_LOCK_PHY3;
15457 			break;
15458 		}
15459 	}
15460 
15461 	if (!tg3_flag(tp, ENABLE_ASF) &&
15462 	    !(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15463 	    !(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
15464 		tp->phy_flags &= ~(TG3_PHYFLG_1G_ON_VAUX_OK |
15465 				   TG3_PHYFLG_KEEP_LINK_ON_PWRDN);
15466 
15467 	if (tg3_flag(tp, USE_PHYLIB))
15468 		return tg3_phy_init(tp);
15469 
15470 	/* Reading the PHY ID register can conflict with ASF
15471 	 * firmware access to the PHY hardware.
15472 	 */
15473 	err = 0;
15474 	if (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)) {
15475 		hw_phy_id = hw_phy_id_masked = TG3_PHY_ID_INVALID;
15476 	} else {
15477 		/* Now read the physical PHY_ID from the chip and verify
15478 		 * that it is sane.  If it doesn't look good, we fall back
15479 		 * to either the hard-coded table based PHY_ID and failing
15480 		 * that the value found in the eeprom area.
15481 		 */
15482 		err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
15483 		err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
15484 
15485 		hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
15486 		hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
15487 		hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
15488 
15489 		hw_phy_id_masked = hw_phy_id & TG3_PHY_ID_MASK;
15490 	}
15491 
15492 	if (!err && TG3_KNOWN_PHY_ID(hw_phy_id_masked)) {
15493 		tp->phy_id = hw_phy_id;
15494 		if (hw_phy_id_masked == TG3_PHY_ID_BCM8002)
15495 			tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15496 		else
15497 			tp->phy_flags &= ~TG3_PHYFLG_PHY_SERDES;
15498 	} else {
15499 		if (tp->phy_id != TG3_PHY_ID_INVALID) {
15500 			/* Do nothing, phy ID already set up in
15501 			 * tg3_get_eeprom_hw_cfg().
15502 			 */
15503 		} else {
15504 			struct subsys_tbl_ent *p;
15505 
15506 			/* No eeprom signature?  Try the hardcoded
15507 			 * subsys device table.
15508 			 */
15509 			p = tg3_lookup_by_subsys(tp);
15510 			if (p) {
15511 				tp->phy_id = p->phy_id;
15512 			} else if (!tg3_flag(tp, IS_SSB_CORE)) {
15513 				/* For now we saw the IDs 0xbc050cd0,
15514 				 * 0xbc050f80 and 0xbc050c30 on devices
15515 				 * connected to an BCM4785 and there are
15516 				 * probably more. Just assume that the phy is
15517 				 * supported when it is connected to a SSB core
15518 				 * for now.
15519 				 */
15520 				return -ENODEV;
15521 			}
15522 
15523 			if (!tp->phy_id ||
15524 			    tp->phy_id == TG3_PHY_ID_BCM8002)
15525 				tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
15526 		}
15527 	}
15528 
15529 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15530 	    (tg3_asic_rev(tp) == ASIC_REV_5719 ||
15531 	     tg3_asic_rev(tp) == ASIC_REV_5720 ||
15532 	     tg3_asic_rev(tp) == ASIC_REV_57766 ||
15533 	     tg3_asic_rev(tp) == ASIC_REV_5762 ||
15534 	     (tg3_asic_rev(tp) == ASIC_REV_5717 &&
15535 	      tg3_chip_rev_id(tp) != CHIPREV_ID_5717_A0) ||
15536 	     (tg3_asic_rev(tp) == ASIC_REV_57765 &&
15537 	      tg3_chip_rev_id(tp) != CHIPREV_ID_57765_A0))) {
15538 		tp->phy_flags |= TG3_PHYFLG_EEE_CAP;
15539 
15540 		tp->eee.supported = SUPPORTED_100baseT_Full |
15541 				    SUPPORTED_1000baseT_Full;
15542 		tp->eee.advertised = ADVERTISED_100baseT_Full |
15543 				     ADVERTISED_1000baseT_Full;
15544 		tp->eee.eee_enabled = 1;
15545 		tp->eee.tx_lpi_enabled = 1;
15546 		tp->eee.tx_lpi_timer = TG3_CPMU_DBTMR1_LNKIDLE_2047US;
15547 	}
15548 
15549 	tg3_phy_init_link_config(tp);
15550 
15551 	if (!(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN) &&
15552 	    !(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
15553 	    !tg3_flag(tp, ENABLE_APE) &&
15554 	    !tg3_flag(tp, ENABLE_ASF)) {
15555 		u32 bmsr, dummy;
15556 
15557 		tg3_readphy(tp, MII_BMSR, &bmsr);
15558 		if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
15559 		    (bmsr & BMSR_LSTATUS))
15560 			goto skip_phy_reset;
15561 
15562 		err = tg3_phy_reset(tp);
15563 		if (err)
15564 			return err;
15565 
15566 		tg3_phy_set_wirespeed(tp);
15567 
15568 		if (!tg3_phy_copper_an_config_ok(tp, &dummy)) {
15569 			tg3_phy_autoneg_cfg(tp, tp->link_config.advertising,
15570 					    tp->link_config.flowctrl);
15571 
15572 			tg3_writephy(tp, MII_BMCR,
15573 				     BMCR_ANENABLE | BMCR_ANRESTART);
15574 		}
15575 	}
15576 
15577 skip_phy_reset:
15578 	if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
15579 		err = tg3_init_5401phy_dsp(tp);
15580 		if (err)
15581 			return err;
15582 
15583 		err = tg3_init_5401phy_dsp(tp);
15584 	}
15585 
15586 	return err;
15587 }
15588 
15589 static void tg3_read_vpd(struct tg3 *tp)
15590 {
15591 	u8 *vpd_data;
15592 	unsigned int len, vpdlen;
15593 	int i;
15594 
15595 	vpd_data = (u8 *)tg3_vpd_readblock(tp, &vpdlen);
15596 	if (!vpd_data)
15597 		goto out_no_vpd;
15598 
15599 	i = pci_vpd_find_ro_info_keyword(vpd_data, vpdlen,
15600 					 PCI_VPD_RO_KEYWORD_MFR_ID, &len);
15601 	if (i < 0)
15602 		goto partno;
15603 
15604 	if (len != 4 || memcmp(vpd_data + i, "1028", 4))
15605 		goto partno;
15606 
15607 	i = pci_vpd_find_ro_info_keyword(vpd_data, vpdlen,
15608 					 PCI_VPD_RO_KEYWORD_VENDOR0, &len);
15609 	if (i < 0)
15610 		goto partno;
15611 
15612 	memset(tp->fw_ver, 0, sizeof(tp->fw_ver));
15613 	snprintf(tp->fw_ver, sizeof(tp->fw_ver), "%.*s bc ", len, vpd_data + i);
15614 
15615 partno:
15616 	i = pci_vpd_find_ro_info_keyword(vpd_data, vpdlen,
15617 					 PCI_VPD_RO_KEYWORD_PARTNO, &len);
15618 	if (i < 0)
15619 		goto out_not_found;
15620 
15621 	if (len > TG3_BPN_SIZE)
15622 		goto out_not_found;
15623 
15624 	memcpy(tp->board_part_number, &vpd_data[i], len);
15625 
15626 out_not_found:
15627 	kfree(vpd_data);
15628 	if (tp->board_part_number[0])
15629 		return;
15630 
15631 out_no_vpd:
15632 	if (tg3_asic_rev(tp) == ASIC_REV_5717) {
15633 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
15634 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C)
15635 			strcpy(tp->board_part_number, "BCM5717");
15636 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718)
15637 			strcpy(tp->board_part_number, "BCM5718");
15638 		else
15639 			goto nomatch;
15640 	} else if (tg3_asic_rev(tp) == ASIC_REV_57780) {
15641 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57780)
15642 			strcpy(tp->board_part_number, "BCM57780");
15643 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57760)
15644 			strcpy(tp->board_part_number, "BCM57760");
15645 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57790)
15646 			strcpy(tp->board_part_number, "BCM57790");
15647 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57788)
15648 			strcpy(tp->board_part_number, "BCM57788");
15649 		else
15650 			goto nomatch;
15651 	} else if (tg3_asic_rev(tp) == ASIC_REV_57765) {
15652 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761)
15653 			strcpy(tp->board_part_number, "BCM57761");
15654 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765)
15655 			strcpy(tp->board_part_number, "BCM57765");
15656 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57781)
15657 			strcpy(tp->board_part_number, "BCM57781");
15658 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57785)
15659 			strcpy(tp->board_part_number, "BCM57785");
15660 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791)
15661 			strcpy(tp->board_part_number, "BCM57791");
15662 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795)
15663 			strcpy(tp->board_part_number, "BCM57795");
15664 		else
15665 			goto nomatch;
15666 	} else if (tg3_asic_rev(tp) == ASIC_REV_57766) {
15667 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57762)
15668 			strcpy(tp->board_part_number, "BCM57762");
15669 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57766)
15670 			strcpy(tp->board_part_number, "BCM57766");
15671 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57782)
15672 			strcpy(tp->board_part_number, "BCM57782");
15673 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57786)
15674 			strcpy(tp->board_part_number, "BCM57786");
15675 		else
15676 			goto nomatch;
15677 	} else if (tg3_asic_rev(tp) == ASIC_REV_5906) {
15678 		strcpy(tp->board_part_number, "BCM95906");
15679 	} else {
15680 nomatch:
15681 		strcpy(tp->board_part_number, "none");
15682 	}
15683 }
15684 
15685 static int tg3_fw_img_is_valid(struct tg3 *tp, u32 offset)
15686 {
15687 	u32 val;
15688 
15689 	if (tg3_nvram_read(tp, offset, &val) ||
15690 	    (val & 0xfc000000) != 0x0c000000 ||
15691 	    tg3_nvram_read(tp, offset + 4, &val) ||
15692 	    val != 0)
15693 		return 0;
15694 
15695 	return 1;
15696 }
15697 
15698 static void tg3_read_bc_ver(struct tg3 *tp)
15699 {
15700 	u32 val, offset, start, ver_offset;
15701 	int i, dst_off;
15702 	bool newver = false;
15703 
15704 	if (tg3_nvram_read(tp, 0xc, &offset) ||
15705 	    tg3_nvram_read(tp, 0x4, &start))
15706 		return;
15707 
15708 	offset = tg3_nvram_logical_addr(tp, offset);
15709 
15710 	if (tg3_nvram_read(tp, offset, &val))
15711 		return;
15712 
15713 	if ((val & 0xfc000000) == 0x0c000000) {
15714 		if (tg3_nvram_read(tp, offset + 4, &val))
15715 			return;
15716 
15717 		if (val == 0)
15718 			newver = true;
15719 	}
15720 
15721 	dst_off = strlen(tp->fw_ver);
15722 
15723 	if (newver) {
15724 		if (TG3_VER_SIZE - dst_off < 16 ||
15725 		    tg3_nvram_read(tp, offset + 8, &ver_offset))
15726 			return;
15727 
15728 		offset = offset + ver_offset - start;
15729 		for (i = 0; i < 16; i += 4) {
15730 			__be32 v;
15731 			if (tg3_nvram_read_be32(tp, offset + i, &v))
15732 				return;
15733 
15734 			memcpy(tp->fw_ver + dst_off + i, &v, sizeof(v));
15735 		}
15736 	} else {
15737 		u32 major, minor;
15738 
15739 		if (tg3_nvram_read(tp, TG3_NVM_PTREV_BCVER, &ver_offset))
15740 			return;
15741 
15742 		major = (ver_offset & TG3_NVM_BCVER_MAJMSK) >>
15743 			TG3_NVM_BCVER_MAJSFT;
15744 		minor = ver_offset & TG3_NVM_BCVER_MINMSK;
15745 		snprintf(&tp->fw_ver[dst_off], TG3_VER_SIZE - dst_off,
15746 			 "v%d.%02d", major, minor);
15747 	}
15748 }
15749 
15750 static void tg3_read_hwsb_ver(struct tg3 *tp)
15751 {
15752 	u32 val, major, minor;
15753 
15754 	/* Use native endian representation */
15755 	if (tg3_nvram_read(tp, TG3_NVM_HWSB_CFG1, &val))
15756 		return;
15757 
15758 	major = (val & TG3_NVM_HWSB_CFG1_MAJMSK) >>
15759 		TG3_NVM_HWSB_CFG1_MAJSFT;
15760 	minor = (val & TG3_NVM_HWSB_CFG1_MINMSK) >>
15761 		TG3_NVM_HWSB_CFG1_MINSFT;
15762 
15763 	snprintf(&tp->fw_ver[0], 32, "sb v%d.%02d", major, minor);
15764 }
15765 
15766 static void tg3_read_sb_ver(struct tg3 *tp, u32 val)
15767 {
15768 	u32 offset, major, minor, build;
15769 
15770 	strncat(tp->fw_ver, "sb", TG3_VER_SIZE - strlen(tp->fw_ver) - 1);
15771 
15772 	if ((val & TG3_EEPROM_SB_FORMAT_MASK) != TG3_EEPROM_SB_FORMAT_1)
15773 		return;
15774 
15775 	switch (val & TG3_EEPROM_SB_REVISION_MASK) {
15776 	case TG3_EEPROM_SB_REVISION_0:
15777 		offset = TG3_EEPROM_SB_F1R0_EDH_OFF;
15778 		break;
15779 	case TG3_EEPROM_SB_REVISION_2:
15780 		offset = TG3_EEPROM_SB_F1R2_EDH_OFF;
15781 		break;
15782 	case TG3_EEPROM_SB_REVISION_3:
15783 		offset = TG3_EEPROM_SB_F1R3_EDH_OFF;
15784 		break;
15785 	case TG3_EEPROM_SB_REVISION_4:
15786 		offset = TG3_EEPROM_SB_F1R4_EDH_OFF;
15787 		break;
15788 	case TG3_EEPROM_SB_REVISION_5:
15789 		offset = TG3_EEPROM_SB_F1R5_EDH_OFF;
15790 		break;
15791 	case TG3_EEPROM_SB_REVISION_6:
15792 		offset = TG3_EEPROM_SB_F1R6_EDH_OFF;
15793 		break;
15794 	default:
15795 		return;
15796 	}
15797 
15798 	if (tg3_nvram_read(tp, offset, &val))
15799 		return;
15800 
15801 	build = (val & TG3_EEPROM_SB_EDH_BLD_MASK) >>
15802 		TG3_EEPROM_SB_EDH_BLD_SHFT;
15803 	major = (val & TG3_EEPROM_SB_EDH_MAJ_MASK) >>
15804 		TG3_EEPROM_SB_EDH_MAJ_SHFT;
15805 	minor =  val & TG3_EEPROM_SB_EDH_MIN_MASK;
15806 
15807 	if (minor > 99 || build > 26)
15808 		return;
15809 
15810 	offset = strlen(tp->fw_ver);
15811 	snprintf(&tp->fw_ver[offset], TG3_VER_SIZE - offset,
15812 		 " v%d.%02d", major, minor);
15813 
15814 	if (build > 0) {
15815 		offset = strlen(tp->fw_ver);
15816 		if (offset < TG3_VER_SIZE - 1)
15817 			tp->fw_ver[offset] = 'a' + build - 1;
15818 	}
15819 }
15820 
15821 static void tg3_read_mgmtfw_ver(struct tg3 *tp)
15822 {
15823 	u32 val, offset, start;
15824 	int i, vlen;
15825 
15826 	for (offset = TG3_NVM_DIR_START;
15827 	     offset < TG3_NVM_DIR_END;
15828 	     offset += TG3_NVM_DIRENT_SIZE) {
15829 		if (tg3_nvram_read(tp, offset, &val))
15830 			return;
15831 
15832 		if ((val >> TG3_NVM_DIRTYPE_SHIFT) == TG3_NVM_DIRTYPE_ASFINI)
15833 			break;
15834 	}
15835 
15836 	if (offset == TG3_NVM_DIR_END)
15837 		return;
15838 
15839 	if (!tg3_flag(tp, 5705_PLUS))
15840 		start = 0x08000000;
15841 	else if (tg3_nvram_read(tp, offset - 4, &start))
15842 		return;
15843 
15844 	if (tg3_nvram_read(tp, offset + 4, &offset) ||
15845 	    !tg3_fw_img_is_valid(tp, offset) ||
15846 	    tg3_nvram_read(tp, offset + 8, &val))
15847 		return;
15848 
15849 	offset += val - start;
15850 
15851 	vlen = strlen(tp->fw_ver);
15852 
15853 	tp->fw_ver[vlen++] = ',';
15854 	tp->fw_ver[vlen++] = ' ';
15855 
15856 	for (i = 0; i < 4; i++) {
15857 		__be32 v;
15858 		if (tg3_nvram_read_be32(tp, offset, &v))
15859 			return;
15860 
15861 		offset += sizeof(v);
15862 
15863 		if (vlen > TG3_VER_SIZE - sizeof(v)) {
15864 			memcpy(&tp->fw_ver[vlen], &v, TG3_VER_SIZE - vlen);
15865 			break;
15866 		}
15867 
15868 		memcpy(&tp->fw_ver[vlen], &v, sizeof(v));
15869 		vlen += sizeof(v);
15870 	}
15871 }
15872 
15873 static void tg3_probe_ncsi(struct tg3 *tp)
15874 {
15875 	u32 apedata;
15876 
15877 	apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
15878 	if (apedata != APE_SEG_SIG_MAGIC)
15879 		return;
15880 
15881 	apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
15882 	if (!(apedata & APE_FW_STATUS_READY))
15883 		return;
15884 
15885 	if (tg3_ape_read32(tp, TG3_APE_FW_FEATURES) & TG3_APE_FW_FEATURE_NCSI)
15886 		tg3_flag_set(tp, APE_HAS_NCSI);
15887 }
15888 
15889 static void tg3_read_dash_ver(struct tg3 *tp)
15890 {
15891 	int vlen;
15892 	u32 apedata;
15893 	char *fwtype;
15894 
15895 	apedata = tg3_ape_read32(tp, TG3_APE_FW_VERSION);
15896 
15897 	if (tg3_flag(tp, APE_HAS_NCSI))
15898 		fwtype = "NCSI";
15899 	else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725)
15900 		fwtype = "SMASH";
15901 	else
15902 		fwtype = "DASH";
15903 
15904 	vlen = strlen(tp->fw_ver);
15905 
15906 	snprintf(&tp->fw_ver[vlen], TG3_VER_SIZE - vlen, " %s v%d.%d.%d.%d",
15907 		 fwtype,
15908 		 (apedata & APE_FW_VERSION_MAJMSK) >> APE_FW_VERSION_MAJSFT,
15909 		 (apedata & APE_FW_VERSION_MINMSK) >> APE_FW_VERSION_MINSFT,
15910 		 (apedata & APE_FW_VERSION_REVMSK) >> APE_FW_VERSION_REVSFT,
15911 		 (apedata & APE_FW_VERSION_BLDMSK));
15912 }
15913 
15914 static void tg3_read_otp_ver(struct tg3 *tp)
15915 {
15916 	u32 val, val2;
15917 
15918 	if (tg3_asic_rev(tp) != ASIC_REV_5762)
15919 		return;
15920 
15921 	if (!tg3_ape_otp_read(tp, OTP_ADDRESS_MAGIC0, &val) &&
15922 	    !tg3_ape_otp_read(tp, OTP_ADDRESS_MAGIC0 + 4, &val2) &&
15923 	    TG3_OTP_MAGIC0_VALID(val)) {
15924 		u64 val64 = (u64) val << 32 | val2;
15925 		u32 ver = 0;
15926 		int i, vlen;
15927 
15928 		for (i = 0; i < 7; i++) {
15929 			if ((val64 & 0xff) == 0)
15930 				break;
15931 			ver = val64 & 0xff;
15932 			val64 >>= 8;
15933 		}
15934 		vlen = strlen(tp->fw_ver);
15935 		snprintf(&tp->fw_ver[vlen], TG3_VER_SIZE - vlen, " .%02d", ver);
15936 	}
15937 }
15938 
15939 static void tg3_read_fw_ver(struct tg3 *tp)
15940 {
15941 	u32 val;
15942 	bool vpd_vers = false;
15943 
15944 	if (tp->fw_ver[0] != 0)
15945 		vpd_vers = true;
15946 
15947 	if (tg3_flag(tp, NO_NVRAM)) {
15948 		strcat(tp->fw_ver, "sb");
15949 		tg3_read_otp_ver(tp);
15950 		return;
15951 	}
15952 
15953 	if (tg3_nvram_read(tp, 0, &val))
15954 		return;
15955 
15956 	if (val == TG3_EEPROM_MAGIC)
15957 		tg3_read_bc_ver(tp);
15958 	else if ((val & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW)
15959 		tg3_read_sb_ver(tp, val);
15960 	else if ((val & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
15961 		tg3_read_hwsb_ver(tp);
15962 
15963 	if (tg3_flag(tp, ENABLE_ASF)) {
15964 		if (tg3_flag(tp, ENABLE_APE)) {
15965 			tg3_probe_ncsi(tp);
15966 			if (!vpd_vers)
15967 				tg3_read_dash_ver(tp);
15968 		} else if (!vpd_vers) {
15969 			tg3_read_mgmtfw_ver(tp);
15970 		}
15971 	}
15972 
15973 	tp->fw_ver[TG3_VER_SIZE - 1] = 0;
15974 }
15975 
15976 static inline u32 tg3_rx_ret_ring_size(struct tg3 *tp)
15977 {
15978 	if (tg3_flag(tp, LRG_PROD_RING_CAP))
15979 		return TG3_RX_RET_MAX_SIZE_5717;
15980 	else if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS))
15981 		return TG3_RX_RET_MAX_SIZE_5700;
15982 	else
15983 		return TG3_RX_RET_MAX_SIZE_5705;
15984 }
15985 
15986 static const struct pci_device_id tg3_write_reorder_chipsets[] = {
15987 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_FE_GATE_700C) },
15988 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8131_BRIDGE) },
15989 	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8385_0) },
15990 	{ },
15991 };
15992 
15993 static struct pci_dev *tg3_find_peer(struct tg3 *tp)
15994 {
15995 	struct pci_dev *peer;
15996 	unsigned int func, devnr = tp->pdev->devfn & ~7;
15997 
15998 	for (func = 0; func < 8; func++) {
15999 		peer = pci_get_slot(tp->pdev->bus, devnr | func);
16000 		if (peer && peer != tp->pdev)
16001 			break;
16002 		pci_dev_put(peer);
16003 	}
16004 	/* 5704 can be configured in single-port mode, set peer to
16005 	 * tp->pdev in that case.
16006 	 */
16007 	if (!peer) {
16008 		peer = tp->pdev;
16009 		return peer;
16010 	}
16011 
16012 	/*
16013 	 * We don't need to keep the refcount elevated; there's no way
16014 	 * to remove one half of this device without removing the other
16015 	 */
16016 	pci_dev_put(peer);
16017 
16018 	return peer;
16019 }
16020 
16021 static void tg3_detect_asic_rev(struct tg3 *tp, u32 misc_ctrl_reg)
16022 {
16023 	tp->pci_chip_rev_id = misc_ctrl_reg >> MISC_HOST_CTRL_CHIPREV_SHIFT;
16024 	if (tg3_asic_rev(tp) == ASIC_REV_USE_PROD_ID_REG) {
16025 		u32 reg;
16026 
16027 		/* All devices that use the alternate
16028 		 * ASIC REV location have a CPMU.
16029 		 */
16030 		tg3_flag_set(tp, CPMU_PRESENT);
16031 
16032 		if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
16033 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C ||
16034 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718 ||
16035 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5719 ||
16036 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5720 ||
16037 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57767 ||
16038 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57764 ||
16039 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5762 ||
16040 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725 ||
16041 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5727 ||
16042 		    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57787)
16043 			reg = TG3PCI_GEN2_PRODID_ASICREV;
16044 		else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57781 ||
16045 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57785 ||
16046 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761 ||
16047 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765 ||
16048 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791 ||
16049 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795 ||
16050 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57762 ||
16051 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57766 ||
16052 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57782 ||
16053 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57786)
16054 			reg = TG3PCI_GEN15_PRODID_ASICREV;
16055 		else
16056 			reg = TG3PCI_PRODID_ASICREV;
16057 
16058 		pci_read_config_dword(tp->pdev, reg, &tp->pci_chip_rev_id);
16059 	}
16060 
16061 	/* Wrong chip ID in 5752 A0. This code can be removed later
16062 	 * as A0 is not in production.
16063 	 */
16064 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5752_A0_HW)
16065 		tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
16066 
16067 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5717_C0)
16068 		tp->pci_chip_rev_id = CHIPREV_ID_5720_A0;
16069 
16070 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16071 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
16072 	    tg3_asic_rev(tp) == ASIC_REV_5720)
16073 		tg3_flag_set(tp, 5717_PLUS);
16074 
16075 	if (tg3_asic_rev(tp) == ASIC_REV_57765 ||
16076 	    tg3_asic_rev(tp) == ASIC_REV_57766)
16077 		tg3_flag_set(tp, 57765_CLASS);
16078 
16079 	if (tg3_flag(tp, 57765_CLASS) || tg3_flag(tp, 5717_PLUS) ||
16080 	     tg3_asic_rev(tp) == ASIC_REV_5762)
16081 		tg3_flag_set(tp, 57765_PLUS);
16082 
16083 	/* Intentionally exclude ASIC_REV_5906 */
16084 	if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
16085 	    tg3_asic_rev(tp) == ASIC_REV_5787 ||
16086 	    tg3_asic_rev(tp) == ASIC_REV_5784 ||
16087 	    tg3_asic_rev(tp) == ASIC_REV_5761 ||
16088 	    tg3_asic_rev(tp) == ASIC_REV_5785 ||
16089 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
16090 	    tg3_flag(tp, 57765_PLUS))
16091 		tg3_flag_set(tp, 5755_PLUS);
16092 
16093 	if (tg3_asic_rev(tp) == ASIC_REV_5780 ||
16094 	    tg3_asic_rev(tp) == ASIC_REV_5714)
16095 		tg3_flag_set(tp, 5780_CLASS);
16096 
16097 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
16098 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
16099 	    tg3_asic_rev(tp) == ASIC_REV_5906 ||
16100 	    tg3_flag(tp, 5755_PLUS) ||
16101 	    tg3_flag(tp, 5780_CLASS))
16102 		tg3_flag_set(tp, 5750_PLUS);
16103 
16104 	if (tg3_asic_rev(tp) == ASIC_REV_5705 ||
16105 	    tg3_flag(tp, 5750_PLUS))
16106 		tg3_flag_set(tp, 5705_PLUS);
16107 }
16108 
16109 static bool tg3_10_100_only_device(struct tg3 *tp,
16110 				   const struct pci_device_id *ent)
16111 {
16112 	u32 grc_misc_cfg = tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK;
16113 
16114 	if ((tg3_asic_rev(tp) == ASIC_REV_5703 &&
16115 	     (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
16116 	    (tp->phy_flags & TG3_PHYFLG_IS_FET))
16117 		return true;
16118 
16119 	if (ent->driver_data & TG3_DRV_DATA_FLAG_10_100_ONLY) {
16120 		if (tg3_asic_rev(tp) == ASIC_REV_5705) {
16121 			if (ent->driver_data & TG3_DRV_DATA_FLAG_5705_10_100)
16122 				return true;
16123 		} else {
16124 			return true;
16125 		}
16126 	}
16127 
16128 	return false;
16129 }
16130 
16131 static int tg3_get_invariants(struct tg3 *tp, const struct pci_device_id *ent)
16132 {
16133 	u32 misc_ctrl_reg;
16134 	u32 pci_state_reg, grc_misc_cfg;
16135 	u32 val;
16136 	u16 pci_cmd;
16137 	int err;
16138 
16139 	/* Force memory write invalidate off.  If we leave it on,
16140 	 * then on 5700_BX chips we have to enable a workaround.
16141 	 * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
16142 	 * to match the cacheline size.  The Broadcom driver have this
16143 	 * workaround but turns MWI off all the times so never uses
16144 	 * it.  This seems to suggest that the workaround is insufficient.
16145 	 */
16146 	pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16147 	pci_cmd &= ~PCI_COMMAND_INVALIDATE;
16148 	pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16149 
16150 	/* Important! -- Make sure register accesses are byteswapped
16151 	 * correctly.  Also, for those chips that require it, make
16152 	 * sure that indirect register accesses are enabled before
16153 	 * the first operation.
16154 	 */
16155 	pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16156 			      &misc_ctrl_reg);
16157 	tp->misc_host_ctrl |= (misc_ctrl_reg &
16158 			       MISC_HOST_CTRL_CHIPREV);
16159 	pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16160 			       tp->misc_host_ctrl);
16161 
16162 	tg3_detect_asic_rev(tp, misc_ctrl_reg);
16163 
16164 	/* If we have 5702/03 A1 or A2 on certain ICH chipsets,
16165 	 * we need to disable memory and use config. cycles
16166 	 * only to access all registers. The 5702/03 chips
16167 	 * can mistakenly decode the special cycles from the
16168 	 * ICH chipsets as memory write cycles, causing corruption
16169 	 * of register and memory space. Only certain ICH bridges
16170 	 * will drive special cycles with non-zero data during the
16171 	 * address phase which can fall within the 5703's address
16172 	 * range. This is not an ICH bug as the PCI spec allows
16173 	 * non-zero address during special cycles. However, only
16174 	 * these ICH bridges are known to drive non-zero addresses
16175 	 * during special cycles.
16176 	 *
16177 	 * Since special cycles do not cross PCI bridges, we only
16178 	 * enable this workaround if the 5703 is on the secondary
16179 	 * bus of these ICH bridges.
16180 	 */
16181 	if ((tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A1) ||
16182 	    (tg3_chip_rev_id(tp) == CHIPREV_ID_5703_A2)) {
16183 		static struct tg3_dev_id {
16184 			u32	vendor;
16185 			u32	device;
16186 			u32	rev;
16187 		} ich_chipsets[] = {
16188 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
16189 			  PCI_ANY_ID },
16190 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
16191 			  PCI_ANY_ID },
16192 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
16193 			  0xa },
16194 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
16195 			  PCI_ANY_ID },
16196 			{ },
16197 		};
16198 		struct tg3_dev_id *pci_id = &ich_chipsets[0];
16199 		struct pci_dev *bridge = NULL;
16200 
16201 		while (pci_id->vendor != 0) {
16202 			bridge = pci_get_device(pci_id->vendor, pci_id->device,
16203 						bridge);
16204 			if (!bridge) {
16205 				pci_id++;
16206 				continue;
16207 			}
16208 			if (pci_id->rev != PCI_ANY_ID) {
16209 				if (bridge->revision > pci_id->rev)
16210 					continue;
16211 			}
16212 			if (bridge->subordinate &&
16213 			    (bridge->subordinate->number ==
16214 			     tp->pdev->bus->number)) {
16215 				tg3_flag_set(tp, ICH_WORKAROUND);
16216 				pci_dev_put(bridge);
16217 				break;
16218 			}
16219 		}
16220 	}
16221 
16222 	if (tg3_asic_rev(tp) == ASIC_REV_5701) {
16223 		static struct tg3_dev_id {
16224 			u32	vendor;
16225 			u32	device;
16226 		} bridge_chipsets[] = {
16227 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_0 },
16228 			{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_1 },
16229 			{ },
16230 		};
16231 		struct tg3_dev_id *pci_id = &bridge_chipsets[0];
16232 		struct pci_dev *bridge = NULL;
16233 
16234 		while (pci_id->vendor != 0) {
16235 			bridge = pci_get_device(pci_id->vendor,
16236 						pci_id->device,
16237 						bridge);
16238 			if (!bridge) {
16239 				pci_id++;
16240 				continue;
16241 			}
16242 			if (bridge->subordinate &&
16243 			    (bridge->subordinate->number <=
16244 			     tp->pdev->bus->number) &&
16245 			    (bridge->subordinate->busn_res.end >=
16246 			     tp->pdev->bus->number)) {
16247 				tg3_flag_set(tp, 5701_DMA_BUG);
16248 				pci_dev_put(bridge);
16249 				break;
16250 			}
16251 		}
16252 	}
16253 
16254 	/* The EPB bridge inside 5714, 5715, and 5780 cannot support
16255 	 * DMA addresses > 40-bit. This bridge may have other additional
16256 	 * 57xx devices behind it in some 4-port NIC designs for example.
16257 	 * Any tg3 device found behind the bridge will also need the 40-bit
16258 	 * DMA workaround.
16259 	 */
16260 	if (tg3_flag(tp, 5780_CLASS)) {
16261 		tg3_flag_set(tp, 40BIT_DMA_BUG);
16262 		tp->msi_cap = tp->pdev->msi_cap;
16263 	} else {
16264 		struct pci_dev *bridge = NULL;
16265 
16266 		do {
16267 			bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
16268 						PCI_DEVICE_ID_SERVERWORKS_EPB,
16269 						bridge);
16270 			if (bridge && bridge->subordinate &&
16271 			    (bridge->subordinate->number <=
16272 			     tp->pdev->bus->number) &&
16273 			    (bridge->subordinate->busn_res.end >=
16274 			     tp->pdev->bus->number)) {
16275 				tg3_flag_set(tp, 40BIT_DMA_BUG);
16276 				pci_dev_put(bridge);
16277 				break;
16278 			}
16279 		} while (bridge);
16280 	}
16281 
16282 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16283 	    tg3_asic_rev(tp) == ASIC_REV_5714)
16284 		tp->pdev_peer = tg3_find_peer(tp);
16285 
16286 	/* Determine TSO capabilities */
16287 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0)
16288 		; /* Do nothing. HW bug. */
16289 	else if (tg3_flag(tp, 57765_PLUS))
16290 		tg3_flag_set(tp, HW_TSO_3);
16291 	else if (tg3_flag(tp, 5755_PLUS) ||
16292 		 tg3_asic_rev(tp) == ASIC_REV_5906)
16293 		tg3_flag_set(tp, HW_TSO_2);
16294 	else if (tg3_flag(tp, 5750_PLUS)) {
16295 		tg3_flag_set(tp, HW_TSO_1);
16296 		tg3_flag_set(tp, TSO_BUG);
16297 		if (tg3_asic_rev(tp) == ASIC_REV_5750 &&
16298 		    tg3_chip_rev_id(tp) >= CHIPREV_ID_5750_C2)
16299 			tg3_flag_clear(tp, TSO_BUG);
16300 	} else if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
16301 		   tg3_asic_rev(tp) != ASIC_REV_5701 &&
16302 		   tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) {
16303 		tg3_flag_set(tp, FW_TSO);
16304 		tg3_flag_set(tp, TSO_BUG);
16305 		if (tg3_asic_rev(tp) == ASIC_REV_5705)
16306 			tp->fw_needed = FIRMWARE_TG3TSO5;
16307 		else
16308 			tp->fw_needed = FIRMWARE_TG3TSO;
16309 	}
16310 
16311 	/* Selectively allow TSO based on operating conditions */
16312 	if (tg3_flag(tp, HW_TSO_1) ||
16313 	    tg3_flag(tp, HW_TSO_2) ||
16314 	    tg3_flag(tp, HW_TSO_3) ||
16315 	    tg3_flag(tp, FW_TSO)) {
16316 		/* For firmware TSO, assume ASF is disabled.
16317 		 * We'll disable TSO later if we discover ASF
16318 		 * is enabled in tg3_get_eeprom_hw_cfg().
16319 		 */
16320 		tg3_flag_set(tp, TSO_CAPABLE);
16321 	} else {
16322 		tg3_flag_clear(tp, TSO_CAPABLE);
16323 		tg3_flag_clear(tp, TSO_BUG);
16324 		tp->fw_needed = NULL;
16325 	}
16326 
16327 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0)
16328 		tp->fw_needed = FIRMWARE_TG3;
16329 
16330 	if (tg3_asic_rev(tp) == ASIC_REV_57766)
16331 		tp->fw_needed = FIRMWARE_TG357766;
16332 
16333 	tp->irq_max = 1;
16334 
16335 	if (tg3_flag(tp, 5750_PLUS)) {
16336 		tg3_flag_set(tp, SUPPORT_MSI);
16337 		if (tg3_chip_rev(tp) == CHIPREV_5750_AX ||
16338 		    tg3_chip_rev(tp) == CHIPREV_5750_BX ||
16339 		    (tg3_asic_rev(tp) == ASIC_REV_5714 &&
16340 		     tg3_chip_rev_id(tp) <= CHIPREV_ID_5714_A2 &&
16341 		     tp->pdev_peer == tp->pdev))
16342 			tg3_flag_clear(tp, SUPPORT_MSI);
16343 
16344 		if (tg3_flag(tp, 5755_PLUS) ||
16345 		    tg3_asic_rev(tp) == ASIC_REV_5906) {
16346 			tg3_flag_set(tp, 1SHOT_MSI);
16347 		}
16348 
16349 		if (tg3_flag(tp, 57765_PLUS)) {
16350 			tg3_flag_set(tp, SUPPORT_MSIX);
16351 			tp->irq_max = TG3_IRQ_MAX_VECS;
16352 		}
16353 	}
16354 
16355 	tp->txq_max = 1;
16356 	tp->rxq_max = 1;
16357 	if (tp->irq_max > 1) {
16358 		tp->rxq_max = TG3_RSS_MAX_NUM_QS;
16359 		tg3_rss_init_dflt_indir_tbl(tp, TG3_RSS_MAX_NUM_QS);
16360 
16361 		if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
16362 		    tg3_asic_rev(tp) == ASIC_REV_5720)
16363 			tp->txq_max = tp->irq_max - 1;
16364 	}
16365 
16366 	if (tg3_flag(tp, 5755_PLUS) ||
16367 	    tg3_asic_rev(tp) == ASIC_REV_5906)
16368 		tg3_flag_set(tp, SHORT_DMA_BUG);
16369 
16370 	if (tg3_asic_rev(tp) == ASIC_REV_5719)
16371 		tp->dma_limit = TG3_TX_BD_DMA_MAX_4K;
16372 
16373 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16374 	    tg3_asic_rev(tp) == ASIC_REV_5719 ||
16375 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
16376 	    tg3_asic_rev(tp) == ASIC_REV_5762)
16377 		tg3_flag_set(tp, LRG_PROD_RING_CAP);
16378 
16379 	if (tg3_flag(tp, 57765_PLUS) &&
16380 	    tg3_chip_rev_id(tp) != CHIPREV_ID_5719_A0)
16381 		tg3_flag_set(tp, USE_JUMBO_BDFLAG);
16382 
16383 	if (!tg3_flag(tp, 5705_PLUS) ||
16384 	    tg3_flag(tp, 5780_CLASS) ||
16385 	    tg3_flag(tp, USE_JUMBO_BDFLAG))
16386 		tg3_flag_set(tp, JUMBO_CAPABLE);
16387 
16388 	pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
16389 			      &pci_state_reg);
16390 
16391 	if (pci_is_pcie(tp->pdev)) {
16392 		u16 lnkctl;
16393 
16394 		tg3_flag_set(tp, PCI_EXPRESS);
16395 
16396 		pcie_capability_read_word(tp->pdev, PCI_EXP_LNKCTL, &lnkctl);
16397 		if (lnkctl & PCI_EXP_LNKCTL_CLKREQ_EN) {
16398 			if (tg3_asic_rev(tp) == ASIC_REV_5906) {
16399 				tg3_flag_clear(tp, HW_TSO_2);
16400 				tg3_flag_clear(tp, TSO_CAPABLE);
16401 			}
16402 			if (tg3_asic_rev(tp) == ASIC_REV_5784 ||
16403 			    tg3_asic_rev(tp) == ASIC_REV_5761 ||
16404 			    tg3_chip_rev_id(tp) == CHIPREV_ID_57780_A0 ||
16405 			    tg3_chip_rev_id(tp) == CHIPREV_ID_57780_A1)
16406 				tg3_flag_set(tp, CLKREQ_BUG);
16407 		} else if (tg3_chip_rev_id(tp) == CHIPREV_ID_5717_A0) {
16408 			tg3_flag_set(tp, L1PLLPD_EN);
16409 		}
16410 	} else if (tg3_asic_rev(tp) == ASIC_REV_5785) {
16411 		/* BCM5785 devices are effectively PCIe devices, and should
16412 		 * follow PCIe codepaths, but do not have a PCIe capabilities
16413 		 * section.
16414 		 */
16415 		tg3_flag_set(tp, PCI_EXPRESS);
16416 	} else if (!tg3_flag(tp, 5705_PLUS) ||
16417 		   tg3_flag(tp, 5780_CLASS)) {
16418 		tp->pcix_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_PCIX);
16419 		if (!tp->pcix_cap) {
16420 			dev_err(&tp->pdev->dev,
16421 				"Cannot find PCI-X capability, aborting\n");
16422 			return -EIO;
16423 		}
16424 
16425 		if (!(pci_state_reg & PCISTATE_CONV_PCI_MODE))
16426 			tg3_flag_set(tp, PCIX_MODE);
16427 	}
16428 
16429 	/* If we have an AMD 762 or VIA K8T800 chipset, write
16430 	 * reordering to the mailbox registers done by the host
16431 	 * controller can cause major troubles.  We read back from
16432 	 * every mailbox register write to force the writes to be
16433 	 * posted to the chip in order.
16434 	 */
16435 	if (pci_dev_present(tg3_write_reorder_chipsets) &&
16436 	    !tg3_flag(tp, PCI_EXPRESS))
16437 		tg3_flag_set(tp, MBOX_WRITE_REORDER);
16438 
16439 	pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
16440 			     &tp->pci_cacheline_sz);
16441 	pci_read_config_byte(tp->pdev, PCI_LATENCY_TIMER,
16442 			     &tp->pci_lat_timer);
16443 	if (tg3_asic_rev(tp) == ASIC_REV_5703 &&
16444 	    tp->pci_lat_timer < 64) {
16445 		tp->pci_lat_timer = 64;
16446 		pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
16447 				      tp->pci_lat_timer);
16448 	}
16449 
16450 	/* Important! -- It is critical that the PCI-X hw workaround
16451 	 * situation is decided before the first MMIO register access.
16452 	 */
16453 	if (tg3_chip_rev(tp) == CHIPREV_5700_BX) {
16454 		/* 5700 BX chips need to have their TX producer index
16455 		 * mailboxes written twice to workaround a bug.
16456 		 */
16457 		tg3_flag_set(tp, TXD_MBOX_HWBUG);
16458 
16459 		/* If we are in PCI-X mode, enable register write workaround.
16460 		 *
16461 		 * The workaround is to use indirect register accesses
16462 		 * for all chip writes not to mailbox registers.
16463 		 */
16464 		if (tg3_flag(tp, PCIX_MODE)) {
16465 			u32 pm_reg;
16466 
16467 			tg3_flag_set(tp, PCIX_TARGET_HWBUG);
16468 
16469 			/* The chip can have it's power management PCI config
16470 			 * space registers clobbered due to this bug.
16471 			 * So explicitly force the chip into D0 here.
16472 			 */
16473 			pci_read_config_dword(tp->pdev,
16474 					      tp->pdev->pm_cap + PCI_PM_CTRL,
16475 					      &pm_reg);
16476 			pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
16477 			pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
16478 			pci_write_config_dword(tp->pdev,
16479 					       tp->pdev->pm_cap + PCI_PM_CTRL,
16480 					       pm_reg);
16481 
16482 			/* Also, force SERR#/PERR# in PCI command. */
16483 			pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16484 			pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
16485 			pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16486 		}
16487 	}
16488 
16489 	if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
16490 		tg3_flag_set(tp, PCI_HIGH_SPEED);
16491 	if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
16492 		tg3_flag_set(tp, PCI_32BIT);
16493 
16494 	/* Chip-specific fixup from Broadcom driver */
16495 	if ((tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0) &&
16496 	    (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
16497 		pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
16498 		pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
16499 	}
16500 
16501 	/* Default fast path register access methods */
16502 	tp->read32 = tg3_read32;
16503 	tp->write32 = tg3_write32;
16504 	tp->read32_mbox = tg3_read32;
16505 	tp->write32_mbox = tg3_write32;
16506 	tp->write32_tx_mbox = tg3_write32;
16507 	tp->write32_rx_mbox = tg3_write32;
16508 
16509 	/* Various workaround register access methods */
16510 	if (tg3_flag(tp, PCIX_TARGET_HWBUG))
16511 		tp->write32 = tg3_write_indirect_reg32;
16512 	else if (tg3_asic_rev(tp) == ASIC_REV_5701 ||
16513 		 (tg3_flag(tp, PCI_EXPRESS) &&
16514 		  tg3_chip_rev_id(tp) == CHIPREV_ID_5750_A0)) {
16515 		/*
16516 		 * Back to back register writes can cause problems on these
16517 		 * chips, the workaround is to read back all reg writes
16518 		 * except those to mailbox regs.
16519 		 *
16520 		 * See tg3_write_indirect_reg32().
16521 		 */
16522 		tp->write32 = tg3_write_flush_reg32;
16523 	}
16524 
16525 	if (tg3_flag(tp, TXD_MBOX_HWBUG) || tg3_flag(tp, MBOX_WRITE_REORDER)) {
16526 		tp->write32_tx_mbox = tg3_write32_tx_mbox;
16527 		if (tg3_flag(tp, MBOX_WRITE_REORDER))
16528 			tp->write32_rx_mbox = tg3_write_flush_reg32;
16529 	}
16530 
16531 	if (tg3_flag(tp, ICH_WORKAROUND)) {
16532 		tp->read32 = tg3_read_indirect_reg32;
16533 		tp->write32 = tg3_write_indirect_reg32;
16534 		tp->read32_mbox = tg3_read_indirect_mbox;
16535 		tp->write32_mbox = tg3_write_indirect_mbox;
16536 		tp->write32_tx_mbox = tg3_write_indirect_mbox;
16537 		tp->write32_rx_mbox = tg3_write_indirect_mbox;
16538 
16539 		iounmap(tp->regs);
16540 		tp->regs = NULL;
16541 
16542 		pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
16543 		pci_cmd &= ~PCI_COMMAND_MEMORY;
16544 		pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
16545 	}
16546 	if (tg3_asic_rev(tp) == ASIC_REV_5906) {
16547 		tp->read32_mbox = tg3_read32_mbox_5906;
16548 		tp->write32_mbox = tg3_write32_mbox_5906;
16549 		tp->write32_tx_mbox = tg3_write32_mbox_5906;
16550 		tp->write32_rx_mbox = tg3_write32_mbox_5906;
16551 	}
16552 
16553 	if (tp->write32 == tg3_write_indirect_reg32 ||
16554 	    (tg3_flag(tp, PCIX_MODE) &&
16555 	     (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16556 	      tg3_asic_rev(tp) == ASIC_REV_5701)))
16557 		tg3_flag_set(tp, SRAM_USE_CONFIG);
16558 
16559 	/* The memory arbiter has to be enabled in order for SRAM accesses
16560 	 * to succeed.  Normally on powerup the tg3 chip firmware will make
16561 	 * sure it is enabled, but other entities such as system netboot
16562 	 * code might disable it.
16563 	 */
16564 	val = tr32(MEMARB_MODE);
16565 	tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
16566 
16567 	tp->pci_fn = PCI_FUNC(tp->pdev->devfn) & 3;
16568 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16569 	    tg3_flag(tp, 5780_CLASS)) {
16570 		if (tg3_flag(tp, PCIX_MODE)) {
16571 			pci_read_config_dword(tp->pdev,
16572 					      tp->pcix_cap + PCI_X_STATUS,
16573 					      &val);
16574 			tp->pci_fn = val & 0x7;
16575 		}
16576 	} else if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16577 		   tg3_asic_rev(tp) == ASIC_REV_5719 ||
16578 		   tg3_asic_rev(tp) == ASIC_REV_5720) {
16579 		tg3_read_mem(tp, NIC_SRAM_CPMU_STATUS, &val);
16580 		if ((val & NIC_SRAM_CPMUSTAT_SIG_MSK) != NIC_SRAM_CPMUSTAT_SIG)
16581 			val = tr32(TG3_CPMU_STATUS);
16582 
16583 		if (tg3_asic_rev(tp) == ASIC_REV_5717)
16584 			tp->pci_fn = (val & TG3_CPMU_STATUS_FMSK_5717) ? 1 : 0;
16585 		else
16586 			tp->pci_fn = (val & TG3_CPMU_STATUS_FMSK_5719) >>
16587 				     TG3_CPMU_STATUS_FSHFT_5719;
16588 	}
16589 
16590 	if (tg3_flag(tp, FLUSH_POSTED_WRITES)) {
16591 		tp->write32_tx_mbox = tg3_write_flush_reg32;
16592 		tp->write32_rx_mbox = tg3_write_flush_reg32;
16593 	}
16594 
16595 	/* Get eeprom hw config before calling tg3_set_power_state().
16596 	 * In particular, the TG3_FLAG_IS_NIC flag must be
16597 	 * determined before calling tg3_set_power_state() so that
16598 	 * we know whether or not to switch out of Vaux power.
16599 	 * When the flag is set, it means that GPIO1 is used for eeprom
16600 	 * write protect and also implies that it is a LOM where GPIOs
16601 	 * are not used to switch power.
16602 	 */
16603 	tg3_get_eeprom_hw_cfg(tp);
16604 
16605 	if (tg3_flag(tp, FW_TSO) && tg3_flag(tp, ENABLE_ASF)) {
16606 		tg3_flag_clear(tp, TSO_CAPABLE);
16607 		tg3_flag_clear(tp, TSO_BUG);
16608 		tp->fw_needed = NULL;
16609 	}
16610 
16611 	if (tg3_flag(tp, ENABLE_APE)) {
16612 		/* Allow reads and writes to the
16613 		 * APE register and memory space.
16614 		 */
16615 		pci_state_reg |= PCISTATE_ALLOW_APE_CTLSPC_WR |
16616 				 PCISTATE_ALLOW_APE_SHMEM_WR |
16617 				 PCISTATE_ALLOW_APE_PSPACE_WR;
16618 		pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE,
16619 				       pci_state_reg);
16620 
16621 		tg3_ape_lock_init(tp);
16622 		tp->ape_hb_interval =
16623 			msecs_to_jiffies(APE_HOST_HEARTBEAT_INT_5SEC);
16624 	}
16625 
16626 	/* Set up tp->grc_local_ctrl before calling
16627 	 * tg3_pwrsrc_switch_to_vmain().  GPIO1 driven high
16628 	 * will bring 5700's external PHY out of reset.
16629 	 * It is also used as eeprom write protect on LOMs.
16630 	 */
16631 	tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
16632 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16633 	    tg3_flag(tp, EEPROM_WRITE_PROT))
16634 		tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
16635 				       GRC_LCLCTRL_GPIO_OUTPUT1);
16636 	/* Unused GPIO3 must be driven as output on 5752 because there
16637 	 * are no pull-up resistors on unused GPIO pins.
16638 	 */
16639 	else if (tg3_asic_rev(tp) == ASIC_REV_5752)
16640 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
16641 
16642 	if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
16643 	    tg3_asic_rev(tp) == ASIC_REV_57780 ||
16644 	    tg3_flag(tp, 57765_CLASS))
16645 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
16646 
16647 	if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
16648 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
16649 		/* Turn off the debug UART. */
16650 		tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
16651 		if (tg3_flag(tp, IS_NIC))
16652 			/* Keep VMain power. */
16653 			tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
16654 					      GRC_LCLCTRL_GPIO_OUTPUT0;
16655 	}
16656 
16657 	if (tg3_asic_rev(tp) == ASIC_REV_5762)
16658 		tp->grc_local_ctrl |=
16659 			tr32(GRC_LOCAL_CTRL) & GRC_LCLCTRL_GPIO_UART_SEL;
16660 
16661 	/* Switch out of Vaux if it is a NIC */
16662 	tg3_pwrsrc_switch_to_vmain(tp);
16663 
16664 	/* Derive initial jumbo mode from MTU assigned in
16665 	 * ether_setup() via the alloc_etherdev() call
16666 	 */
16667 	if (tp->dev->mtu > ETH_DATA_LEN && !tg3_flag(tp, 5780_CLASS))
16668 		tg3_flag_set(tp, JUMBO_RING_ENABLE);
16669 
16670 	/* Determine WakeOnLan speed to use. */
16671 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16672 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
16673 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0 ||
16674 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B2) {
16675 		tg3_flag_clear(tp, WOL_SPEED_100MB);
16676 	} else {
16677 		tg3_flag_set(tp, WOL_SPEED_100MB);
16678 	}
16679 
16680 	if (tg3_asic_rev(tp) == ASIC_REV_5906)
16681 		tp->phy_flags |= TG3_PHYFLG_IS_FET;
16682 
16683 	/* A few boards don't want Ethernet@WireSpeed phy feature */
16684 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
16685 	    (tg3_asic_rev(tp) == ASIC_REV_5705 &&
16686 	     (tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A0) &&
16687 	     (tg3_chip_rev_id(tp) != CHIPREV_ID_5705_A1)) ||
16688 	    (tp->phy_flags & TG3_PHYFLG_IS_FET) ||
16689 	    (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
16690 		tp->phy_flags |= TG3_PHYFLG_NO_ETH_WIRE_SPEED;
16691 
16692 	if (tg3_chip_rev(tp) == CHIPREV_5703_AX ||
16693 	    tg3_chip_rev(tp) == CHIPREV_5704_AX)
16694 		tp->phy_flags |= TG3_PHYFLG_ADC_BUG;
16695 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5704_A0)
16696 		tp->phy_flags |= TG3_PHYFLG_5704_A0_BUG;
16697 
16698 	if (tg3_flag(tp, 5705_PLUS) &&
16699 	    !(tp->phy_flags & TG3_PHYFLG_IS_FET) &&
16700 	    tg3_asic_rev(tp) != ASIC_REV_5785 &&
16701 	    tg3_asic_rev(tp) != ASIC_REV_57780 &&
16702 	    !tg3_flag(tp, 57765_PLUS)) {
16703 		if (tg3_asic_rev(tp) == ASIC_REV_5755 ||
16704 		    tg3_asic_rev(tp) == ASIC_REV_5787 ||
16705 		    tg3_asic_rev(tp) == ASIC_REV_5784 ||
16706 		    tg3_asic_rev(tp) == ASIC_REV_5761) {
16707 			if (tp->pdev->device != PCI_DEVICE_ID_TIGON3_5756 &&
16708 			    tp->pdev->device != PCI_DEVICE_ID_TIGON3_5722)
16709 				tp->phy_flags |= TG3_PHYFLG_JITTER_BUG;
16710 			if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5755M)
16711 				tp->phy_flags |= TG3_PHYFLG_ADJUST_TRIM;
16712 		} else
16713 			tp->phy_flags |= TG3_PHYFLG_BER_BUG;
16714 	}
16715 
16716 	if (tg3_asic_rev(tp) == ASIC_REV_5784 &&
16717 	    tg3_chip_rev(tp) != CHIPREV_5784_AX) {
16718 		tp->phy_otp = tg3_read_otp_phycfg(tp);
16719 		if (tp->phy_otp == 0)
16720 			tp->phy_otp = TG3_OTP_DEFAULT;
16721 	}
16722 
16723 	if (tg3_flag(tp, CPMU_PRESENT))
16724 		tp->mi_mode = MAC_MI_MODE_500KHZ_CONST;
16725 	else
16726 		tp->mi_mode = MAC_MI_MODE_BASE;
16727 
16728 	tp->coalesce_mode = 0;
16729 	if (tg3_chip_rev(tp) != CHIPREV_5700_AX &&
16730 	    tg3_chip_rev(tp) != CHIPREV_5700_BX)
16731 		tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
16732 
16733 	/* Set these bits to enable statistics workaround. */
16734 	if (tg3_asic_rev(tp) == ASIC_REV_5717 ||
16735 	    tg3_asic_rev(tp) == ASIC_REV_5762 ||
16736 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5719_A0 ||
16737 	    tg3_chip_rev_id(tp) == CHIPREV_ID_5720_A0) {
16738 		tp->coalesce_mode |= HOSTCC_MODE_ATTN;
16739 		tp->grc_mode |= GRC_MODE_IRQ_ON_FLOW_ATTN;
16740 	}
16741 
16742 	if (tg3_asic_rev(tp) == ASIC_REV_5785 ||
16743 	    tg3_asic_rev(tp) == ASIC_REV_57780)
16744 		tg3_flag_set(tp, USE_PHYLIB);
16745 
16746 	err = tg3_mdio_init(tp);
16747 	if (err)
16748 		return err;
16749 
16750 	/* Initialize data/descriptor byte/word swapping. */
16751 	val = tr32(GRC_MODE);
16752 	if (tg3_asic_rev(tp) == ASIC_REV_5720 ||
16753 	    tg3_asic_rev(tp) == ASIC_REV_5762)
16754 		val &= (GRC_MODE_BYTE_SWAP_B2HRX_DATA |
16755 			GRC_MODE_WORD_SWAP_B2HRX_DATA |
16756 			GRC_MODE_B2HRX_ENABLE |
16757 			GRC_MODE_HTX2B_ENABLE |
16758 			GRC_MODE_HOST_STACKUP);
16759 	else
16760 		val &= GRC_MODE_HOST_STACKUP;
16761 
16762 	tw32(GRC_MODE, val | tp->grc_mode);
16763 
16764 	tg3_switch_clocks(tp);
16765 
16766 	/* Clear this out for sanity. */
16767 	tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
16768 
16769 	/* Clear TG3PCI_REG_BASE_ADDR to prevent hangs. */
16770 	tw32(TG3PCI_REG_BASE_ADDR, 0);
16771 
16772 	pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
16773 			      &pci_state_reg);
16774 	if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
16775 	    !tg3_flag(tp, PCIX_TARGET_HWBUG)) {
16776 		if (tg3_chip_rev_id(tp) == CHIPREV_ID_5701_A0 ||
16777 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B0 ||
16778 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B2 ||
16779 		    tg3_chip_rev_id(tp) == CHIPREV_ID_5701_B5) {
16780 			void __iomem *sram_base;
16781 
16782 			/* Write some dummy words into the SRAM status block
16783 			 * area, see if it reads back correctly.  If the return
16784 			 * value is bad, force enable the PCIX workaround.
16785 			 */
16786 			sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
16787 
16788 			writel(0x00000000, sram_base);
16789 			writel(0x00000000, sram_base + 4);
16790 			writel(0xffffffff, sram_base + 4);
16791 			if (readl(sram_base) != 0x00000000)
16792 				tg3_flag_set(tp, PCIX_TARGET_HWBUG);
16793 		}
16794 	}
16795 
16796 	udelay(50);
16797 	tg3_nvram_init(tp);
16798 
16799 	/* If the device has an NVRAM, no need to load patch firmware */
16800 	if (tg3_asic_rev(tp) == ASIC_REV_57766 &&
16801 	    !tg3_flag(tp, NO_NVRAM))
16802 		tp->fw_needed = NULL;
16803 
16804 	grc_misc_cfg = tr32(GRC_MISC_CFG);
16805 	grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
16806 
16807 	if (tg3_asic_rev(tp) == ASIC_REV_5705 &&
16808 	    (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
16809 	     grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
16810 		tg3_flag_set(tp, IS_5788);
16811 
16812 	if (!tg3_flag(tp, IS_5788) &&
16813 	    tg3_asic_rev(tp) != ASIC_REV_5700)
16814 		tg3_flag_set(tp, TAGGED_STATUS);
16815 	if (tg3_flag(tp, TAGGED_STATUS)) {
16816 		tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
16817 				      HOSTCC_MODE_CLRTICK_TXBD);
16818 
16819 		tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
16820 		pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
16821 				       tp->misc_host_ctrl);
16822 	}
16823 
16824 	/* Preserve the APE MAC_MODE bits */
16825 	if (tg3_flag(tp, ENABLE_APE))
16826 		tp->mac_mode = MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
16827 	else
16828 		tp->mac_mode = 0;
16829 
16830 	if (tg3_10_100_only_device(tp, ent))
16831 		tp->phy_flags |= TG3_PHYFLG_10_100_ONLY;
16832 
16833 	err = tg3_phy_probe(tp);
16834 	if (err) {
16835 		dev_err(&tp->pdev->dev, "phy probe failed, err %d\n", err);
16836 		/* ... but do not return immediately ... */
16837 		tg3_mdio_fini(tp);
16838 	}
16839 
16840 	tg3_read_vpd(tp);
16841 	tg3_read_fw_ver(tp);
16842 
16843 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
16844 		tp->phy_flags &= ~TG3_PHYFLG_USE_MI_INTERRUPT;
16845 	} else {
16846 		if (tg3_asic_rev(tp) == ASIC_REV_5700)
16847 			tp->phy_flags |= TG3_PHYFLG_USE_MI_INTERRUPT;
16848 		else
16849 			tp->phy_flags &= ~TG3_PHYFLG_USE_MI_INTERRUPT;
16850 	}
16851 
16852 	/* 5700 {AX,BX} chips have a broken status block link
16853 	 * change bit implementation, so we must use the
16854 	 * status register in those cases.
16855 	 */
16856 	if (tg3_asic_rev(tp) == ASIC_REV_5700)
16857 		tg3_flag_set(tp, USE_LINKCHG_REG);
16858 	else
16859 		tg3_flag_clear(tp, USE_LINKCHG_REG);
16860 
16861 	/* The led_ctrl is set during tg3_phy_probe, here we might
16862 	 * have to force the link status polling mechanism based
16863 	 * upon subsystem IDs.
16864 	 */
16865 	if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
16866 	    tg3_asic_rev(tp) == ASIC_REV_5701 &&
16867 	    !(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
16868 		tp->phy_flags |= TG3_PHYFLG_USE_MI_INTERRUPT;
16869 		tg3_flag_set(tp, USE_LINKCHG_REG);
16870 	}
16871 
16872 	/* For all SERDES we poll the MAC status register. */
16873 	if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
16874 		tg3_flag_set(tp, POLL_SERDES);
16875 	else
16876 		tg3_flag_clear(tp, POLL_SERDES);
16877 
16878 	if (tg3_flag(tp, ENABLE_APE) && tg3_flag(tp, ENABLE_ASF))
16879 		tg3_flag_set(tp, POLL_CPMU_LINK);
16880 
16881 	tp->rx_offset = NET_SKB_PAD + NET_IP_ALIGN;
16882 	tp->rx_copy_thresh = TG3_RX_COPY_THRESHOLD;
16883 	if (tg3_asic_rev(tp) == ASIC_REV_5701 &&
16884 	    tg3_flag(tp, PCIX_MODE)) {
16885 		tp->rx_offset = NET_SKB_PAD;
16886 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
16887 		tp->rx_copy_thresh = ~(u16)0;
16888 #endif
16889 	}
16890 
16891 	tp->rx_std_ring_mask = TG3_RX_STD_RING_SIZE(tp) - 1;
16892 	tp->rx_jmb_ring_mask = TG3_RX_JMB_RING_SIZE(tp) - 1;
16893 	tp->rx_ret_ring_mask = tg3_rx_ret_ring_size(tp) - 1;
16894 
16895 	tp->rx_std_max_post = tp->rx_std_ring_mask + 1;
16896 
16897 	/* Increment the rx prod index on the rx std ring by at most
16898 	 * 8 for these chips to workaround hw errata.
16899 	 */
16900 	if (tg3_asic_rev(tp) == ASIC_REV_5750 ||
16901 	    tg3_asic_rev(tp) == ASIC_REV_5752 ||
16902 	    tg3_asic_rev(tp) == ASIC_REV_5755)
16903 		tp->rx_std_max_post = 8;
16904 
16905 	if (tg3_flag(tp, ASPM_WORKAROUND))
16906 		tp->pwrmgmt_thresh = tr32(PCIE_PWR_MGMT_THRESH) &
16907 				     PCIE_PWR_MGMT_L1_THRESH_MSK;
16908 
16909 	return err;
16910 }
16911 
16912 static int tg3_get_device_address(struct tg3 *tp, u8 *addr)
16913 {
16914 	u32 hi, lo, mac_offset;
16915 	int addr_ok = 0;
16916 	int err;
16917 
16918 	if (!eth_platform_get_mac_address(&tp->pdev->dev, addr))
16919 		return 0;
16920 
16921 	if (tg3_flag(tp, IS_SSB_CORE)) {
16922 		err = ssb_gige_get_macaddr(tp->pdev, addr);
16923 		if (!err && is_valid_ether_addr(addr))
16924 			return 0;
16925 	}
16926 
16927 	mac_offset = 0x7c;
16928 	if (tg3_asic_rev(tp) == ASIC_REV_5704 ||
16929 	    tg3_flag(tp, 5780_CLASS)) {
16930 		if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
16931 			mac_offset = 0xcc;
16932 		if (tg3_nvram_lock(tp))
16933 			tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
16934 		else
16935 			tg3_nvram_unlock(tp);
16936 	} else if (tg3_flag(tp, 5717_PLUS)) {
16937 		if (tp->pci_fn & 1)
16938 			mac_offset = 0xcc;
16939 		if (tp->pci_fn > 1)
16940 			mac_offset += 0x18c;
16941 	} else if (tg3_asic_rev(tp) == ASIC_REV_5906)
16942 		mac_offset = 0x10;
16943 
16944 	/* First try to get it from MAC address mailbox. */
16945 	tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
16946 	if ((hi >> 16) == 0x484b) {
16947 		addr[0] = (hi >>  8) & 0xff;
16948 		addr[1] = (hi >>  0) & 0xff;
16949 
16950 		tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
16951 		addr[2] = (lo >> 24) & 0xff;
16952 		addr[3] = (lo >> 16) & 0xff;
16953 		addr[4] = (lo >>  8) & 0xff;
16954 		addr[5] = (lo >>  0) & 0xff;
16955 
16956 		/* Some old bootcode may report a 0 MAC address in SRAM */
16957 		addr_ok = is_valid_ether_addr(addr);
16958 	}
16959 	if (!addr_ok) {
16960 		/* Next, try NVRAM. */
16961 		if (!tg3_flag(tp, NO_NVRAM) &&
16962 		    !tg3_nvram_read_be32(tp, mac_offset + 0, &hi) &&
16963 		    !tg3_nvram_read_be32(tp, mac_offset + 4, &lo)) {
16964 			memcpy(&addr[0], ((char *)&hi) + 2, 2);
16965 			memcpy(&addr[2], (char *)&lo, sizeof(lo));
16966 		}
16967 		/* Finally just fetch it out of the MAC control regs. */
16968 		else {
16969 			hi = tr32(MAC_ADDR_0_HIGH);
16970 			lo = tr32(MAC_ADDR_0_LOW);
16971 
16972 			addr[5] = lo & 0xff;
16973 			addr[4] = (lo >> 8) & 0xff;
16974 			addr[3] = (lo >> 16) & 0xff;
16975 			addr[2] = (lo >> 24) & 0xff;
16976 			addr[1] = hi & 0xff;
16977 			addr[0] = (hi >> 8) & 0xff;
16978 		}
16979 	}
16980 
16981 	if (!is_valid_ether_addr(addr))
16982 		return -EINVAL;
16983 	return 0;
16984 }
16985 
16986 #define BOUNDARY_SINGLE_CACHELINE	1
16987 #define BOUNDARY_MULTI_CACHELINE	2
16988 
16989 static u32 tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
16990 {
16991 	int cacheline_size;
16992 	u8 byte;
16993 	int goal;
16994 
16995 	pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
16996 	if (byte == 0)
16997 		cacheline_size = 1024;
16998 	else
16999 		cacheline_size = (int) byte * 4;
17000 
17001 	/* On 5703 and later chips, the boundary bits have no
17002 	 * effect.
17003 	 */
17004 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
17005 	    tg3_asic_rev(tp) != ASIC_REV_5701 &&
17006 	    !tg3_flag(tp, PCI_EXPRESS))
17007 		goto out;
17008 
17009 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
17010 	goal = BOUNDARY_MULTI_CACHELINE;
17011 #else
17012 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
17013 	goal = BOUNDARY_SINGLE_CACHELINE;
17014 #else
17015 	goal = 0;
17016 #endif
17017 #endif
17018 
17019 	if (tg3_flag(tp, 57765_PLUS)) {
17020 		val = goal ? 0 : DMA_RWCTRL_DIS_CACHE_ALIGNMENT;
17021 		goto out;
17022 	}
17023 
17024 	if (!goal)
17025 		goto out;
17026 
17027 	/* PCI controllers on most RISC systems tend to disconnect
17028 	 * when a device tries to burst across a cache-line boundary.
17029 	 * Therefore, letting tg3 do so just wastes PCI bandwidth.
17030 	 *
17031 	 * Unfortunately, for PCI-E there are only limited
17032 	 * write-side controls for this, and thus for reads
17033 	 * we will still get the disconnects.  We'll also waste
17034 	 * these PCI cycles for both read and write for chips
17035 	 * other than 5700 and 5701 which do not implement the
17036 	 * boundary bits.
17037 	 */
17038 	if (tg3_flag(tp, PCIX_MODE) && !tg3_flag(tp, PCI_EXPRESS)) {
17039 		switch (cacheline_size) {
17040 		case 16:
17041 		case 32:
17042 		case 64:
17043 		case 128:
17044 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17045 				val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
17046 					DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
17047 			} else {
17048 				val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
17049 					DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
17050 			}
17051 			break;
17052 
17053 		case 256:
17054 			val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
17055 				DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
17056 			break;
17057 
17058 		default:
17059 			val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
17060 				DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
17061 			break;
17062 		}
17063 	} else if (tg3_flag(tp, PCI_EXPRESS)) {
17064 		switch (cacheline_size) {
17065 		case 16:
17066 		case 32:
17067 		case 64:
17068 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17069 				val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
17070 				val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
17071 				break;
17072 			}
17073 			fallthrough;
17074 		case 128:
17075 		default:
17076 			val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
17077 			val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
17078 			break;
17079 		}
17080 	} else {
17081 		switch (cacheline_size) {
17082 		case 16:
17083 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17084 				val |= (DMA_RWCTRL_READ_BNDRY_16 |
17085 					DMA_RWCTRL_WRITE_BNDRY_16);
17086 				break;
17087 			}
17088 			fallthrough;
17089 		case 32:
17090 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17091 				val |= (DMA_RWCTRL_READ_BNDRY_32 |
17092 					DMA_RWCTRL_WRITE_BNDRY_32);
17093 				break;
17094 			}
17095 			fallthrough;
17096 		case 64:
17097 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17098 				val |= (DMA_RWCTRL_READ_BNDRY_64 |
17099 					DMA_RWCTRL_WRITE_BNDRY_64);
17100 				break;
17101 			}
17102 			fallthrough;
17103 		case 128:
17104 			if (goal == BOUNDARY_SINGLE_CACHELINE) {
17105 				val |= (DMA_RWCTRL_READ_BNDRY_128 |
17106 					DMA_RWCTRL_WRITE_BNDRY_128);
17107 				break;
17108 			}
17109 			fallthrough;
17110 		case 256:
17111 			val |= (DMA_RWCTRL_READ_BNDRY_256 |
17112 				DMA_RWCTRL_WRITE_BNDRY_256);
17113 			break;
17114 		case 512:
17115 			val |= (DMA_RWCTRL_READ_BNDRY_512 |
17116 				DMA_RWCTRL_WRITE_BNDRY_512);
17117 			break;
17118 		case 1024:
17119 		default:
17120 			val |= (DMA_RWCTRL_READ_BNDRY_1024 |
17121 				DMA_RWCTRL_WRITE_BNDRY_1024);
17122 			break;
17123 		}
17124 	}
17125 
17126 out:
17127 	return val;
17128 }
17129 
17130 static int tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma,
17131 			   int size, bool to_device)
17132 {
17133 	struct tg3_internal_buffer_desc test_desc;
17134 	u32 sram_dma_descs;
17135 	int i, ret;
17136 
17137 	sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
17138 
17139 	tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
17140 	tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
17141 	tw32(RDMAC_STATUS, 0);
17142 	tw32(WDMAC_STATUS, 0);
17143 
17144 	tw32(BUFMGR_MODE, 0);
17145 	tw32(FTQ_RESET, 0);
17146 
17147 	test_desc.addr_hi = ((u64) buf_dma) >> 32;
17148 	test_desc.addr_lo = buf_dma & 0xffffffff;
17149 	test_desc.nic_mbuf = 0x00002100;
17150 	test_desc.len = size;
17151 
17152 	/*
17153 	 * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
17154 	 * the *second* time the tg3 driver was getting loaded after an
17155 	 * initial scan.
17156 	 *
17157 	 * Broadcom tells me:
17158 	 *   ...the DMA engine is connected to the GRC block and a DMA
17159 	 *   reset may affect the GRC block in some unpredictable way...
17160 	 *   The behavior of resets to individual blocks has not been tested.
17161 	 *
17162 	 * Broadcom noted the GRC reset will also reset all sub-components.
17163 	 */
17164 	if (to_device) {
17165 		test_desc.cqid_sqid = (13 << 8) | 2;
17166 
17167 		tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
17168 		udelay(40);
17169 	} else {
17170 		test_desc.cqid_sqid = (16 << 8) | 7;
17171 
17172 		tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
17173 		udelay(40);
17174 	}
17175 	test_desc.flags = 0x00000005;
17176 
17177 	for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
17178 		u32 val;
17179 
17180 		val = *(((u32 *)&test_desc) + i);
17181 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
17182 				       sram_dma_descs + (i * sizeof(u32)));
17183 		pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
17184 	}
17185 	pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
17186 
17187 	if (to_device)
17188 		tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
17189 	else
17190 		tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
17191 
17192 	ret = -ENODEV;
17193 	for (i = 0; i < 40; i++) {
17194 		u32 val;
17195 
17196 		if (to_device)
17197 			val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
17198 		else
17199 			val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
17200 		if ((val & 0xffff) == sram_dma_descs) {
17201 			ret = 0;
17202 			break;
17203 		}
17204 
17205 		udelay(100);
17206 	}
17207 
17208 	return ret;
17209 }
17210 
17211 #define TEST_BUFFER_SIZE	0x2000
17212 
17213 static const struct pci_device_id tg3_dma_wait_state_chipsets[] = {
17214 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
17215 	{ },
17216 };
17217 
17218 static int tg3_test_dma(struct tg3 *tp)
17219 {
17220 	dma_addr_t buf_dma;
17221 	u32 *buf, saved_dma_rwctrl;
17222 	int ret = 0;
17223 
17224 	buf = dma_alloc_coherent(&tp->pdev->dev, TEST_BUFFER_SIZE,
17225 				 &buf_dma, GFP_KERNEL);
17226 	if (!buf) {
17227 		ret = -ENOMEM;
17228 		goto out_nofree;
17229 	}
17230 
17231 	tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
17232 			  (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
17233 
17234 	tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
17235 
17236 	if (tg3_flag(tp, 57765_PLUS))
17237 		goto out;
17238 
17239 	if (tg3_flag(tp, PCI_EXPRESS)) {
17240 		/* DMA read watermark not used on PCIE */
17241 		tp->dma_rwctrl |= 0x00180000;
17242 	} else if (!tg3_flag(tp, PCIX_MODE)) {
17243 		if (tg3_asic_rev(tp) == ASIC_REV_5705 ||
17244 		    tg3_asic_rev(tp) == ASIC_REV_5750)
17245 			tp->dma_rwctrl |= 0x003f0000;
17246 		else
17247 			tp->dma_rwctrl |= 0x003f000f;
17248 	} else {
17249 		if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
17250 		    tg3_asic_rev(tp) == ASIC_REV_5704) {
17251 			u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
17252 			u32 read_water = 0x7;
17253 
17254 			/* If the 5704 is behind the EPB bridge, we can
17255 			 * do the less restrictive ONE_DMA workaround for
17256 			 * better performance.
17257 			 */
17258 			if (tg3_flag(tp, 40BIT_DMA_BUG) &&
17259 			    tg3_asic_rev(tp) == ASIC_REV_5704)
17260 				tp->dma_rwctrl |= 0x8000;
17261 			else if (ccval == 0x6 || ccval == 0x7)
17262 				tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
17263 
17264 			if (tg3_asic_rev(tp) == ASIC_REV_5703)
17265 				read_water = 4;
17266 			/* Set bit 23 to enable PCIX hw bug fix */
17267 			tp->dma_rwctrl |=
17268 				(read_water << DMA_RWCTRL_READ_WATER_SHIFT) |
17269 				(0x3 << DMA_RWCTRL_WRITE_WATER_SHIFT) |
17270 				(1 << 23);
17271 		} else if (tg3_asic_rev(tp) == ASIC_REV_5780) {
17272 			/* 5780 always in PCIX mode */
17273 			tp->dma_rwctrl |= 0x00144000;
17274 		} else if (tg3_asic_rev(tp) == ASIC_REV_5714) {
17275 			/* 5714 always in PCIX mode */
17276 			tp->dma_rwctrl |= 0x00148000;
17277 		} else {
17278 			tp->dma_rwctrl |= 0x001b000f;
17279 		}
17280 	}
17281 	if (tg3_flag(tp, ONE_DMA_AT_ONCE))
17282 		tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
17283 
17284 	if (tg3_asic_rev(tp) == ASIC_REV_5703 ||
17285 	    tg3_asic_rev(tp) == ASIC_REV_5704)
17286 		tp->dma_rwctrl &= 0xfffffff0;
17287 
17288 	if (tg3_asic_rev(tp) == ASIC_REV_5700 ||
17289 	    tg3_asic_rev(tp) == ASIC_REV_5701) {
17290 		/* Remove this if it causes problems for some boards. */
17291 		tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
17292 
17293 		/* On 5700/5701 chips, we need to set this bit.
17294 		 * Otherwise the chip will issue cacheline transactions
17295 		 * to streamable DMA memory with not all the byte
17296 		 * enables turned on.  This is an error on several
17297 		 * RISC PCI controllers, in particular sparc64.
17298 		 *
17299 		 * On 5703/5704 chips, this bit has been reassigned
17300 		 * a different meaning.  In particular, it is used
17301 		 * on those chips to enable a PCI-X workaround.
17302 		 */
17303 		tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
17304 	}
17305 
17306 	tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17307 
17308 
17309 	if (tg3_asic_rev(tp) != ASIC_REV_5700 &&
17310 	    tg3_asic_rev(tp) != ASIC_REV_5701)
17311 		goto out;
17312 
17313 	/* It is best to perform DMA test with maximum write burst size
17314 	 * to expose the 5700/5701 write DMA bug.
17315 	 */
17316 	saved_dma_rwctrl = tp->dma_rwctrl;
17317 	tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17318 	tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17319 
17320 	while (1) {
17321 		u32 *p = buf, i;
17322 
17323 		for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
17324 			p[i] = i;
17325 
17326 		/* Send the buffer to the chip. */
17327 		ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, true);
17328 		if (ret) {
17329 			dev_err(&tp->pdev->dev,
17330 				"%s: Buffer write failed. err = %d\n",
17331 				__func__, ret);
17332 			break;
17333 		}
17334 
17335 		/* Now read it back. */
17336 		ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, false);
17337 		if (ret) {
17338 			dev_err(&tp->pdev->dev, "%s: Buffer read failed. "
17339 				"err = %d\n", __func__, ret);
17340 			break;
17341 		}
17342 
17343 		/* Verify it. */
17344 		for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
17345 			if (p[i] == i)
17346 				continue;
17347 
17348 			if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
17349 			    DMA_RWCTRL_WRITE_BNDRY_16) {
17350 				tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17351 				tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
17352 				tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17353 				break;
17354 			} else {
17355 				dev_err(&tp->pdev->dev,
17356 					"%s: Buffer corrupted on read back! "
17357 					"(%d != %d)\n", __func__, p[i], i);
17358 				ret = -ENODEV;
17359 				goto out;
17360 			}
17361 		}
17362 
17363 		if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
17364 			/* Success. */
17365 			ret = 0;
17366 			break;
17367 		}
17368 	}
17369 	if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
17370 	    DMA_RWCTRL_WRITE_BNDRY_16) {
17371 		/* DMA test passed without adjusting DMA boundary,
17372 		 * now look for chipsets that are known to expose the
17373 		 * DMA bug without failing the test.
17374 		 */
17375 		if (pci_dev_present(tg3_dma_wait_state_chipsets)) {
17376 			tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
17377 			tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
17378 		} else {
17379 			/* Safe to use the calculated DMA boundary. */
17380 			tp->dma_rwctrl = saved_dma_rwctrl;
17381 		}
17382 
17383 		tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
17384 	}
17385 
17386 out:
17387 	dma_free_coherent(&tp->pdev->dev, TEST_BUFFER_SIZE, buf, buf_dma);
17388 out_nofree:
17389 	return ret;
17390 }
17391 
17392 static void tg3_init_bufmgr_config(struct tg3 *tp)
17393 {
17394 	if (tg3_flag(tp, 57765_PLUS)) {
17395 		tp->bufmgr_config.mbuf_read_dma_low_water =
17396 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17397 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17398 			DEFAULT_MB_MACRX_LOW_WATER_57765;
17399 		tp->bufmgr_config.mbuf_high_water =
17400 			DEFAULT_MB_HIGH_WATER_57765;
17401 
17402 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17403 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17404 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17405 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO_57765;
17406 		tp->bufmgr_config.mbuf_high_water_jumbo =
17407 			DEFAULT_MB_HIGH_WATER_JUMBO_57765;
17408 	} else if (tg3_flag(tp, 5705_PLUS)) {
17409 		tp->bufmgr_config.mbuf_read_dma_low_water =
17410 			DEFAULT_MB_RDMA_LOW_WATER_5705;
17411 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17412 			DEFAULT_MB_MACRX_LOW_WATER_5705;
17413 		tp->bufmgr_config.mbuf_high_water =
17414 			DEFAULT_MB_HIGH_WATER_5705;
17415 		if (tg3_asic_rev(tp) == ASIC_REV_5906) {
17416 			tp->bufmgr_config.mbuf_mac_rx_low_water =
17417 				DEFAULT_MB_MACRX_LOW_WATER_5906;
17418 			tp->bufmgr_config.mbuf_high_water =
17419 				DEFAULT_MB_HIGH_WATER_5906;
17420 		}
17421 
17422 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17423 			DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
17424 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17425 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
17426 		tp->bufmgr_config.mbuf_high_water_jumbo =
17427 			DEFAULT_MB_HIGH_WATER_JUMBO_5780;
17428 	} else {
17429 		tp->bufmgr_config.mbuf_read_dma_low_water =
17430 			DEFAULT_MB_RDMA_LOW_WATER;
17431 		tp->bufmgr_config.mbuf_mac_rx_low_water =
17432 			DEFAULT_MB_MACRX_LOW_WATER;
17433 		tp->bufmgr_config.mbuf_high_water =
17434 			DEFAULT_MB_HIGH_WATER;
17435 
17436 		tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
17437 			DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
17438 		tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
17439 			DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
17440 		tp->bufmgr_config.mbuf_high_water_jumbo =
17441 			DEFAULT_MB_HIGH_WATER_JUMBO;
17442 	}
17443 
17444 	tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
17445 	tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
17446 }
17447 
17448 static char *tg3_phy_string(struct tg3 *tp)
17449 {
17450 	switch (tp->phy_id & TG3_PHY_ID_MASK) {
17451 	case TG3_PHY_ID_BCM5400:	return "5400";
17452 	case TG3_PHY_ID_BCM5401:	return "5401";
17453 	case TG3_PHY_ID_BCM5411:	return "5411";
17454 	case TG3_PHY_ID_BCM5701:	return "5701";
17455 	case TG3_PHY_ID_BCM5703:	return "5703";
17456 	case TG3_PHY_ID_BCM5704:	return "5704";
17457 	case TG3_PHY_ID_BCM5705:	return "5705";
17458 	case TG3_PHY_ID_BCM5750:	return "5750";
17459 	case TG3_PHY_ID_BCM5752:	return "5752";
17460 	case TG3_PHY_ID_BCM5714:	return "5714";
17461 	case TG3_PHY_ID_BCM5780:	return "5780";
17462 	case TG3_PHY_ID_BCM5755:	return "5755";
17463 	case TG3_PHY_ID_BCM5787:	return "5787";
17464 	case TG3_PHY_ID_BCM5784:	return "5784";
17465 	case TG3_PHY_ID_BCM5756:	return "5722/5756";
17466 	case TG3_PHY_ID_BCM5906:	return "5906";
17467 	case TG3_PHY_ID_BCM5761:	return "5761";
17468 	case TG3_PHY_ID_BCM5718C:	return "5718C";
17469 	case TG3_PHY_ID_BCM5718S:	return "5718S";
17470 	case TG3_PHY_ID_BCM57765:	return "57765";
17471 	case TG3_PHY_ID_BCM5719C:	return "5719C";
17472 	case TG3_PHY_ID_BCM5720C:	return "5720C";
17473 	case TG3_PHY_ID_BCM5762:	return "5762C";
17474 	case TG3_PHY_ID_BCM8002:	return "8002/serdes";
17475 	case 0:			return "serdes";
17476 	default:		return "unknown";
17477 	}
17478 }
17479 
17480 static char *tg3_bus_string(struct tg3 *tp, char *str)
17481 {
17482 	if (tg3_flag(tp, PCI_EXPRESS)) {
17483 		strcpy(str, "PCI Express");
17484 		return str;
17485 	} else if (tg3_flag(tp, PCIX_MODE)) {
17486 		u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
17487 
17488 		strcpy(str, "PCIX:");
17489 
17490 		if ((clock_ctrl == 7) ||
17491 		    ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
17492 		     GRC_MISC_CFG_BOARD_ID_5704CIOBE))
17493 			strcat(str, "133MHz");
17494 		else if (clock_ctrl == 0)
17495 			strcat(str, "33MHz");
17496 		else if (clock_ctrl == 2)
17497 			strcat(str, "50MHz");
17498 		else if (clock_ctrl == 4)
17499 			strcat(str, "66MHz");
17500 		else if (clock_ctrl == 6)
17501 			strcat(str, "100MHz");
17502 	} else {
17503 		strcpy(str, "PCI:");
17504 		if (tg3_flag(tp, PCI_HIGH_SPEED))
17505 			strcat(str, "66MHz");
17506 		else
17507 			strcat(str, "33MHz");
17508 	}
17509 	if (tg3_flag(tp, PCI_32BIT))
17510 		strcat(str, ":32-bit");
17511 	else
17512 		strcat(str, ":64-bit");
17513 	return str;
17514 }
17515 
17516 static void tg3_init_coal(struct tg3 *tp)
17517 {
17518 	struct ethtool_coalesce *ec = &tp->coal;
17519 
17520 	memset(ec, 0, sizeof(*ec));
17521 	ec->cmd = ETHTOOL_GCOALESCE;
17522 	ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
17523 	ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
17524 	ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
17525 	ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
17526 	ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
17527 	ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
17528 	ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
17529 	ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
17530 	ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
17531 
17532 	if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
17533 				 HOSTCC_MODE_CLRTICK_TXBD)) {
17534 		ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
17535 		ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
17536 		ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
17537 		ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
17538 	}
17539 
17540 	if (tg3_flag(tp, 5705_PLUS)) {
17541 		ec->rx_coalesce_usecs_irq = 0;
17542 		ec->tx_coalesce_usecs_irq = 0;
17543 		ec->stats_block_coalesce_usecs = 0;
17544 	}
17545 }
17546 
17547 static int tg3_init_one(struct pci_dev *pdev,
17548 				  const struct pci_device_id *ent)
17549 {
17550 	struct net_device *dev;
17551 	struct tg3 *tp;
17552 	int i, err;
17553 	u32 sndmbx, rcvmbx, intmbx;
17554 	char str[40];
17555 	u64 dma_mask, persist_dma_mask;
17556 	netdev_features_t features = 0;
17557 	u8 addr[ETH_ALEN] __aligned(2);
17558 
17559 	err = pci_enable_device(pdev);
17560 	if (err) {
17561 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
17562 		return err;
17563 	}
17564 
17565 	err = pci_request_regions(pdev, DRV_MODULE_NAME);
17566 	if (err) {
17567 		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
17568 		goto err_out_disable_pdev;
17569 	}
17570 
17571 	pci_set_master(pdev);
17572 
17573 	dev = alloc_etherdev_mq(sizeof(*tp), TG3_IRQ_MAX_VECS);
17574 	if (!dev) {
17575 		err = -ENOMEM;
17576 		goto err_out_free_res;
17577 	}
17578 
17579 	SET_NETDEV_DEV(dev, &pdev->dev);
17580 
17581 	tp = netdev_priv(dev);
17582 	tp->pdev = pdev;
17583 	tp->dev = dev;
17584 	tp->rx_mode = TG3_DEF_RX_MODE;
17585 	tp->tx_mode = TG3_DEF_TX_MODE;
17586 	tp->irq_sync = 1;
17587 	tp->pcierr_recovery = false;
17588 
17589 	if (tg3_debug > 0)
17590 		tp->msg_enable = tg3_debug;
17591 	else
17592 		tp->msg_enable = TG3_DEF_MSG_ENABLE;
17593 
17594 	if (pdev_is_ssb_gige_core(pdev)) {
17595 		tg3_flag_set(tp, IS_SSB_CORE);
17596 		if (ssb_gige_must_flush_posted_writes(pdev))
17597 			tg3_flag_set(tp, FLUSH_POSTED_WRITES);
17598 		if (ssb_gige_one_dma_at_once(pdev))
17599 			tg3_flag_set(tp, ONE_DMA_AT_ONCE);
17600 		if (ssb_gige_have_roboswitch(pdev)) {
17601 			tg3_flag_set(tp, USE_PHYLIB);
17602 			tg3_flag_set(tp, ROBOSWITCH);
17603 		}
17604 		if (ssb_gige_is_rgmii(pdev))
17605 			tg3_flag_set(tp, RGMII_MODE);
17606 	}
17607 
17608 	/* The word/byte swap controls here control register access byte
17609 	 * swapping.  DMA data byte swapping is controlled in the GRC_MODE
17610 	 * setting below.
17611 	 */
17612 	tp->misc_host_ctrl =
17613 		MISC_HOST_CTRL_MASK_PCI_INT |
17614 		MISC_HOST_CTRL_WORD_SWAP |
17615 		MISC_HOST_CTRL_INDIR_ACCESS |
17616 		MISC_HOST_CTRL_PCISTATE_RW;
17617 
17618 	/* The NONFRM (non-frame) byte/word swap controls take effect
17619 	 * on descriptor entries, anything which isn't packet data.
17620 	 *
17621 	 * The StrongARM chips on the board (one for tx, one for rx)
17622 	 * are running in big-endian mode.
17623 	 */
17624 	tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
17625 			GRC_MODE_WSWAP_NONFRM_DATA);
17626 #ifdef __BIG_ENDIAN
17627 	tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
17628 #endif
17629 	spin_lock_init(&tp->lock);
17630 	spin_lock_init(&tp->indirect_lock);
17631 	INIT_WORK(&tp->reset_task, tg3_reset_task);
17632 
17633 	tp->regs = pci_ioremap_bar(pdev, BAR_0);
17634 	if (!tp->regs) {
17635 		dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
17636 		err = -ENOMEM;
17637 		goto err_out_free_dev;
17638 	}
17639 
17640 	if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
17641 	    tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761E ||
17642 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S ||
17643 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761SE ||
17644 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
17645 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717_C ||
17646 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718 ||
17647 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5719 ||
17648 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5720 ||
17649 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57767 ||
17650 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57764 ||
17651 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5762 ||
17652 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5725 ||
17653 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_5727 ||
17654 	    tp->pdev->device == TG3PCI_DEVICE_TIGON3_57787) {
17655 		tg3_flag_set(tp, ENABLE_APE);
17656 		tp->aperegs = pci_ioremap_bar(pdev, BAR_2);
17657 		if (!tp->aperegs) {
17658 			dev_err(&pdev->dev,
17659 				"Cannot map APE registers, aborting\n");
17660 			err = -ENOMEM;
17661 			goto err_out_iounmap;
17662 		}
17663 	}
17664 
17665 	tp->rx_pending = TG3_DEF_RX_RING_PENDING;
17666 	tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
17667 
17668 	dev->ethtool_ops = &tg3_ethtool_ops;
17669 	dev->watchdog_timeo = TG3_TX_TIMEOUT;
17670 	dev->netdev_ops = &tg3_netdev_ops;
17671 	dev->irq = pdev->irq;
17672 
17673 	err = tg3_get_invariants(tp, ent);
17674 	if (err) {
17675 		dev_err(&pdev->dev,
17676 			"Problem fetching invariants of chip, aborting\n");
17677 		goto err_out_apeunmap;
17678 	}
17679 
17680 	/* The EPB bridge inside 5714, 5715, and 5780 and any
17681 	 * device behind the EPB cannot support DMA addresses > 40-bit.
17682 	 * On 64-bit systems with IOMMU, use 40-bit dma_mask.
17683 	 * On 64-bit systems without IOMMU, use 64-bit dma_mask and
17684 	 * do DMA address check in tg3_start_xmit().
17685 	 */
17686 	if (tg3_flag(tp, IS_5788))
17687 		persist_dma_mask = dma_mask = DMA_BIT_MASK(32);
17688 	else if (tg3_flag(tp, 40BIT_DMA_BUG)) {
17689 		persist_dma_mask = dma_mask = DMA_BIT_MASK(40);
17690 #ifdef CONFIG_HIGHMEM
17691 		dma_mask = DMA_BIT_MASK(64);
17692 #endif
17693 	} else
17694 		persist_dma_mask = dma_mask = DMA_BIT_MASK(64);
17695 
17696 	/* Configure DMA attributes. */
17697 	if (dma_mask > DMA_BIT_MASK(32)) {
17698 		err = dma_set_mask(&pdev->dev, dma_mask);
17699 		if (!err) {
17700 			features |= NETIF_F_HIGHDMA;
17701 			err = dma_set_coherent_mask(&pdev->dev,
17702 						    persist_dma_mask);
17703 			if (err < 0) {
17704 				dev_err(&pdev->dev, "Unable to obtain 64 bit "
17705 					"DMA for consistent allocations\n");
17706 				goto err_out_apeunmap;
17707 			}
17708 		}
17709 	}
17710 	if (err || dma_mask == DMA_BIT_MASK(32)) {
17711 		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
17712 		if (err) {
17713 			dev_err(&pdev->dev,
17714 				"No usable DMA configuration, aborting\n");
17715 			goto err_out_apeunmap;
17716 		}
17717 	}
17718 
17719 	tg3_init_bufmgr_config(tp);
17720 
17721 	/* 5700 B0 chips do not support checksumming correctly due
17722 	 * to hardware bugs.
17723 	 */
17724 	if (tg3_chip_rev_id(tp) != CHIPREV_ID_5700_B0) {
17725 		features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
17726 
17727 		if (tg3_flag(tp, 5755_PLUS))
17728 			features |= NETIF_F_IPV6_CSUM;
17729 	}
17730 
17731 	/* TSO is on by default on chips that support hardware TSO.
17732 	 * Firmware TSO on older chips gives lower performance, so it
17733 	 * is off by default, but can be enabled using ethtool.
17734 	 */
17735 	if ((tg3_flag(tp, HW_TSO_1) ||
17736 	     tg3_flag(tp, HW_TSO_2) ||
17737 	     tg3_flag(tp, HW_TSO_3)) &&
17738 	    (features & NETIF_F_IP_CSUM))
17739 		features |= NETIF_F_TSO;
17740 	if (tg3_flag(tp, HW_TSO_2) || tg3_flag(tp, HW_TSO_3)) {
17741 		if (features & NETIF_F_IPV6_CSUM)
17742 			features |= NETIF_F_TSO6;
17743 		if (tg3_flag(tp, HW_TSO_3) ||
17744 		    tg3_asic_rev(tp) == ASIC_REV_5761 ||
17745 		    (tg3_asic_rev(tp) == ASIC_REV_5784 &&
17746 		     tg3_chip_rev(tp) != CHIPREV_5784_AX) ||
17747 		    tg3_asic_rev(tp) == ASIC_REV_5785 ||
17748 		    tg3_asic_rev(tp) == ASIC_REV_57780)
17749 			features |= NETIF_F_TSO_ECN;
17750 	}
17751 
17752 	dev->features |= features | NETIF_F_HW_VLAN_CTAG_TX |
17753 			 NETIF_F_HW_VLAN_CTAG_RX;
17754 	dev->vlan_features |= features;
17755 
17756 	/*
17757 	 * Add loopback capability only for a subset of devices that support
17758 	 * MAC-LOOPBACK. Eventually this need to be enhanced to allow INT-PHY
17759 	 * loopback for the remaining devices.
17760 	 */
17761 	if (tg3_asic_rev(tp) != ASIC_REV_5780 &&
17762 	    !tg3_flag(tp, CPMU_PRESENT))
17763 		/* Add the loopback capability */
17764 		features |= NETIF_F_LOOPBACK;
17765 
17766 	dev->hw_features |= features;
17767 	dev->priv_flags |= IFF_UNICAST_FLT;
17768 
17769 	/* MTU range: 60 - 9000 or 1500, depending on hardware */
17770 	dev->min_mtu = TG3_MIN_MTU;
17771 	dev->max_mtu = TG3_MAX_MTU(tp);
17772 
17773 	if (tg3_chip_rev_id(tp) == CHIPREV_ID_5705_A1 &&
17774 	    !tg3_flag(tp, TSO_CAPABLE) &&
17775 	    !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
17776 		tg3_flag_set(tp, MAX_RXPEND_64);
17777 		tp->rx_pending = 63;
17778 	}
17779 
17780 	err = tg3_get_device_address(tp, addr);
17781 	if (err) {
17782 		dev_err(&pdev->dev,
17783 			"Could not obtain valid ethernet address, aborting\n");
17784 		goto err_out_apeunmap;
17785 	}
17786 	eth_hw_addr_set(dev, addr);
17787 
17788 	intmbx = MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW;
17789 	rcvmbx = MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW;
17790 	sndmbx = MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW;
17791 	for (i = 0; i < tp->irq_max; i++) {
17792 		struct tg3_napi *tnapi = &tp->napi[i];
17793 
17794 		tnapi->tp = tp;
17795 		tnapi->tx_pending = TG3_DEF_TX_RING_PENDING;
17796 
17797 		tnapi->int_mbox = intmbx;
17798 		if (i <= 4)
17799 			intmbx += 0x8;
17800 		else
17801 			intmbx += 0x4;
17802 
17803 		tnapi->consmbox = rcvmbx;
17804 		tnapi->prodmbox = sndmbx;
17805 
17806 		if (i)
17807 			tnapi->coal_now = HOSTCC_MODE_COAL_VEC1_NOW << (i - 1);
17808 		else
17809 			tnapi->coal_now = HOSTCC_MODE_NOW;
17810 
17811 		if (!tg3_flag(tp, SUPPORT_MSIX))
17812 			break;
17813 
17814 		/*
17815 		 * If we support MSIX, we'll be using RSS.  If we're using
17816 		 * RSS, the first vector only handles link interrupts and the
17817 		 * remaining vectors handle rx and tx interrupts.  Reuse the
17818 		 * mailbox values for the next iteration.  The values we setup
17819 		 * above are still useful for the single vectored mode.
17820 		 */
17821 		if (!i)
17822 			continue;
17823 
17824 		rcvmbx += 0x8;
17825 
17826 		if (sndmbx & 0x4)
17827 			sndmbx -= 0x4;
17828 		else
17829 			sndmbx += 0xc;
17830 	}
17831 
17832 	/*
17833 	 * Reset chip in case UNDI or EFI driver did not shutdown
17834 	 * DMA self test will enable WDMAC and we'll see (spurious)
17835 	 * pending DMA on the PCI bus at that point.
17836 	 */
17837 	if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
17838 	    (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
17839 		tg3_full_lock(tp, 0);
17840 		tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
17841 		tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
17842 		tg3_full_unlock(tp);
17843 	}
17844 
17845 	err = tg3_test_dma(tp);
17846 	if (err) {
17847 		dev_err(&pdev->dev, "DMA engine test failed, aborting\n");
17848 		goto err_out_apeunmap;
17849 	}
17850 
17851 	tg3_init_coal(tp);
17852 
17853 	pci_set_drvdata(pdev, dev);
17854 
17855 	if (tg3_asic_rev(tp) == ASIC_REV_5719 ||
17856 	    tg3_asic_rev(tp) == ASIC_REV_5720 ||
17857 	    tg3_asic_rev(tp) == ASIC_REV_5762)
17858 		tg3_flag_set(tp, PTP_CAPABLE);
17859 
17860 	tg3_timer_init(tp);
17861 
17862 	tg3_carrier_off(tp);
17863 
17864 	err = register_netdev(dev);
17865 	if (err) {
17866 		dev_err(&pdev->dev, "Cannot register net device, aborting\n");
17867 		goto err_out_apeunmap;
17868 	}
17869 
17870 	if (tg3_flag(tp, PTP_CAPABLE)) {
17871 		tg3_ptp_init(tp);
17872 		tp->ptp_clock = ptp_clock_register(&tp->ptp_info,
17873 						   &tp->pdev->dev);
17874 		if (IS_ERR(tp->ptp_clock))
17875 			tp->ptp_clock = NULL;
17876 	}
17877 
17878 	netdev_info(dev, "Tigon3 [partno(%s) rev %04x] (%s) MAC address %pM\n",
17879 		    tp->board_part_number,
17880 		    tg3_chip_rev_id(tp),
17881 		    tg3_bus_string(tp, str),
17882 		    dev->dev_addr);
17883 
17884 	if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED)) {
17885 		char *ethtype;
17886 
17887 		if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
17888 			ethtype = "10/100Base-TX";
17889 		else if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
17890 			ethtype = "1000Base-SX";
17891 		else
17892 			ethtype = "10/100/1000Base-T";
17893 
17894 		netdev_info(dev, "attached PHY is %s (%s Ethernet) "
17895 			    "(WireSpeed[%d], EEE[%d])\n",
17896 			    tg3_phy_string(tp), ethtype,
17897 			    (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED) == 0,
17898 			    (tp->phy_flags & TG3_PHYFLG_EEE_CAP) != 0);
17899 	}
17900 
17901 	netdev_info(dev, "RXcsums[%d] LinkChgREG[%d] MIirq[%d] ASF[%d] TSOcap[%d]\n",
17902 		    (dev->features & NETIF_F_RXCSUM) != 0,
17903 		    tg3_flag(tp, USE_LINKCHG_REG) != 0,
17904 		    (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT) != 0,
17905 		    tg3_flag(tp, ENABLE_ASF) != 0,
17906 		    tg3_flag(tp, TSO_CAPABLE) != 0);
17907 	netdev_info(dev, "dma_rwctrl[%08x] dma_mask[%d-bit]\n",
17908 		    tp->dma_rwctrl,
17909 		    pdev->dma_mask == DMA_BIT_MASK(32) ? 32 :
17910 		    ((u64)pdev->dma_mask) == DMA_BIT_MASK(40) ? 40 : 64);
17911 
17912 	pci_save_state(pdev);
17913 
17914 	return 0;
17915 
17916 err_out_apeunmap:
17917 	if (tp->aperegs) {
17918 		iounmap(tp->aperegs);
17919 		tp->aperegs = NULL;
17920 	}
17921 
17922 err_out_iounmap:
17923 	if (tp->regs) {
17924 		iounmap(tp->regs);
17925 		tp->regs = NULL;
17926 	}
17927 
17928 err_out_free_dev:
17929 	free_netdev(dev);
17930 
17931 err_out_free_res:
17932 	pci_release_regions(pdev);
17933 
17934 err_out_disable_pdev:
17935 	if (pci_is_enabled(pdev))
17936 		pci_disable_device(pdev);
17937 	return err;
17938 }
17939 
17940 static void tg3_remove_one(struct pci_dev *pdev)
17941 {
17942 	struct net_device *dev = pci_get_drvdata(pdev);
17943 
17944 	if (dev) {
17945 		struct tg3 *tp = netdev_priv(dev);
17946 
17947 		tg3_ptp_fini(tp);
17948 
17949 		release_firmware(tp->fw);
17950 
17951 		tg3_reset_task_cancel(tp);
17952 
17953 		if (tg3_flag(tp, USE_PHYLIB)) {
17954 			tg3_phy_fini(tp);
17955 			tg3_mdio_fini(tp);
17956 		}
17957 
17958 		unregister_netdev(dev);
17959 		if (tp->aperegs) {
17960 			iounmap(tp->aperegs);
17961 			tp->aperegs = NULL;
17962 		}
17963 		if (tp->regs) {
17964 			iounmap(tp->regs);
17965 			tp->regs = NULL;
17966 		}
17967 		free_netdev(dev);
17968 		pci_release_regions(pdev);
17969 		pci_disable_device(pdev);
17970 	}
17971 }
17972 
17973 #ifdef CONFIG_PM_SLEEP
17974 static int tg3_suspend(struct device *device)
17975 {
17976 	struct net_device *dev = dev_get_drvdata(device);
17977 	struct tg3 *tp = netdev_priv(dev);
17978 	int err = 0;
17979 
17980 	rtnl_lock();
17981 
17982 	if (!netif_running(dev))
17983 		goto unlock;
17984 
17985 	tg3_reset_task_cancel(tp);
17986 	tg3_phy_stop(tp);
17987 	tg3_netif_stop(tp);
17988 
17989 	tg3_timer_stop(tp);
17990 
17991 	tg3_full_lock(tp, 1);
17992 	tg3_disable_ints(tp);
17993 	tg3_full_unlock(tp);
17994 
17995 	netif_device_detach(dev);
17996 
17997 	tg3_full_lock(tp, 0);
17998 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
17999 	tg3_flag_clear(tp, INIT_COMPLETE);
18000 	tg3_full_unlock(tp);
18001 
18002 	err = tg3_power_down_prepare(tp);
18003 	if (err) {
18004 		int err2;
18005 
18006 		tg3_full_lock(tp, 0);
18007 
18008 		tg3_flag_set(tp, INIT_COMPLETE);
18009 		err2 = tg3_restart_hw(tp, true);
18010 		if (err2)
18011 			goto out;
18012 
18013 		tg3_timer_start(tp);
18014 
18015 		netif_device_attach(dev);
18016 		tg3_netif_start(tp);
18017 
18018 out:
18019 		tg3_full_unlock(tp);
18020 
18021 		if (!err2)
18022 			tg3_phy_start(tp);
18023 	}
18024 
18025 unlock:
18026 	rtnl_unlock();
18027 	return err;
18028 }
18029 
18030 static int tg3_resume(struct device *device)
18031 {
18032 	struct net_device *dev = dev_get_drvdata(device);
18033 	struct tg3 *tp = netdev_priv(dev);
18034 	int err = 0;
18035 
18036 	rtnl_lock();
18037 
18038 	if (!netif_running(dev))
18039 		goto unlock;
18040 
18041 	netif_device_attach(dev);
18042 
18043 	tg3_full_lock(tp, 0);
18044 
18045 	tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
18046 
18047 	tg3_flag_set(tp, INIT_COMPLETE);
18048 	err = tg3_restart_hw(tp,
18049 			     !(tp->phy_flags & TG3_PHYFLG_KEEP_LINK_ON_PWRDN));
18050 	if (err)
18051 		goto out;
18052 
18053 	tg3_timer_start(tp);
18054 
18055 	tg3_netif_start(tp);
18056 
18057 out:
18058 	tg3_full_unlock(tp);
18059 
18060 	if (!err)
18061 		tg3_phy_start(tp);
18062 
18063 unlock:
18064 	rtnl_unlock();
18065 	return err;
18066 }
18067 #endif /* CONFIG_PM_SLEEP */
18068 
18069 static SIMPLE_DEV_PM_OPS(tg3_pm_ops, tg3_suspend, tg3_resume);
18070 
18071 static void tg3_shutdown(struct pci_dev *pdev)
18072 {
18073 	struct net_device *dev = pci_get_drvdata(pdev);
18074 	struct tg3 *tp = netdev_priv(dev);
18075 
18076 	rtnl_lock();
18077 	netif_device_detach(dev);
18078 
18079 	if (netif_running(dev))
18080 		dev_close(dev);
18081 
18082 	if (system_state == SYSTEM_POWER_OFF)
18083 		tg3_power_down(tp);
18084 
18085 	rtnl_unlock();
18086 }
18087 
18088 /**
18089  * tg3_io_error_detected - called when PCI error is detected
18090  * @pdev: Pointer to PCI device
18091  * @state: The current pci connection state
18092  *
18093  * This function is called after a PCI bus error affecting
18094  * this device has been detected.
18095  */
18096 static pci_ers_result_t tg3_io_error_detected(struct pci_dev *pdev,
18097 					      pci_channel_state_t state)
18098 {
18099 	struct net_device *netdev = pci_get_drvdata(pdev);
18100 	struct tg3 *tp = netdev_priv(netdev);
18101 	pci_ers_result_t err = PCI_ERS_RESULT_NEED_RESET;
18102 
18103 	netdev_info(netdev, "PCI I/O error detected\n");
18104 
18105 	rtnl_lock();
18106 
18107 	/* Could be second call or maybe we don't have netdev yet */
18108 	if (!netdev || tp->pcierr_recovery || !netif_running(netdev))
18109 		goto done;
18110 
18111 	/* We needn't recover from permanent error */
18112 	if (state == pci_channel_io_frozen)
18113 		tp->pcierr_recovery = true;
18114 
18115 	tg3_phy_stop(tp);
18116 
18117 	tg3_netif_stop(tp);
18118 
18119 	tg3_timer_stop(tp);
18120 
18121 	/* Want to make sure that the reset task doesn't run */
18122 	tg3_reset_task_cancel(tp);
18123 
18124 	netif_device_detach(netdev);
18125 
18126 	/* Clean up software state, even if MMIO is blocked */
18127 	tg3_full_lock(tp, 0);
18128 	tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
18129 	tg3_full_unlock(tp);
18130 
18131 done:
18132 	if (state == pci_channel_io_perm_failure) {
18133 		if (netdev) {
18134 			tg3_napi_enable(tp);
18135 			dev_close(netdev);
18136 		}
18137 		err = PCI_ERS_RESULT_DISCONNECT;
18138 	} else {
18139 		pci_disable_device(pdev);
18140 	}
18141 
18142 	rtnl_unlock();
18143 
18144 	return err;
18145 }
18146 
18147 /**
18148  * tg3_io_slot_reset - called after the pci bus has been reset.
18149  * @pdev: Pointer to PCI device
18150  *
18151  * Restart the card from scratch, as if from a cold-boot.
18152  * At this point, the card has exprienced a hard reset,
18153  * followed by fixups by BIOS, and has its config space
18154  * set up identically to what it was at cold boot.
18155  */
18156 static pci_ers_result_t tg3_io_slot_reset(struct pci_dev *pdev)
18157 {
18158 	struct net_device *netdev = pci_get_drvdata(pdev);
18159 	struct tg3 *tp = netdev_priv(netdev);
18160 	pci_ers_result_t rc = PCI_ERS_RESULT_DISCONNECT;
18161 	int err;
18162 
18163 	rtnl_lock();
18164 
18165 	if (pci_enable_device(pdev)) {
18166 		dev_err(&pdev->dev,
18167 			"Cannot re-enable PCI device after reset.\n");
18168 		goto done;
18169 	}
18170 
18171 	pci_set_master(pdev);
18172 	pci_restore_state(pdev);
18173 	pci_save_state(pdev);
18174 
18175 	if (!netdev || !netif_running(netdev)) {
18176 		rc = PCI_ERS_RESULT_RECOVERED;
18177 		goto done;
18178 	}
18179 
18180 	err = tg3_power_up(tp);
18181 	if (err)
18182 		goto done;
18183 
18184 	rc = PCI_ERS_RESULT_RECOVERED;
18185 
18186 done:
18187 	if (rc != PCI_ERS_RESULT_RECOVERED && netdev && netif_running(netdev)) {
18188 		tg3_napi_enable(tp);
18189 		dev_close(netdev);
18190 	}
18191 	rtnl_unlock();
18192 
18193 	return rc;
18194 }
18195 
18196 /**
18197  * tg3_io_resume - called when traffic can start flowing again.
18198  * @pdev: Pointer to PCI device
18199  *
18200  * This callback is called when the error recovery driver tells
18201  * us that its OK to resume normal operation.
18202  */
18203 static void tg3_io_resume(struct pci_dev *pdev)
18204 {
18205 	struct net_device *netdev = pci_get_drvdata(pdev);
18206 	struct tg3 *tp = netdev_priv(netdev);
18207 	int err;
18208 
18209 	rtnl_lock();
18210 
18211 	if (!netdev || !netif_running(netdev))
18212 		goto done;
18213 
18214 	tg3_full_lock(tp, 0);
18215 	tg3_ape_driver_state_change(tp, RESET_KIND_INIT);
18216 	tg3_flag_set(tp, INIT_COMPLETE);
18217 	err = tg3_restart_hw(tp, true);
18218 	if (err) {
18219 		tg3_full_unlock(tp);
18220 		netdev_err(netdev, "Cannot restart hardware after reset.\n");
18221 		goto done;
18222 	}
18223 
18224 	netif_device_attach(netdev);
18225 
18226 	tg3_timer_start(tp);
18227 
18228 	tg3_netif_start(tp);
18229 
18230 	tg3_full_unlock(tp);
18231 
18232 	tg3_phy_start(tp);
18233 
18234 done:
18235 	tp->pcierr_recovery = false;
18236 	rtnl_unlock();
18237 }
18238 
18239 static const struct pci_error_handlers tg3_err_handler = {
18240 	.error_detected	= tg3_io_error_detected,
18241 	.slot_reset	= tg3_io_slot_reset,
18242 	.resume		= tg3_io_resume
18243 };
18244 
18245 static struct pci_driver tg3_driver = {
18246 	.name		= DRV_MODULE_NAME,
18247 	.id_table	= tg3_pci_tbl,
18248 	.probe		= tg3_init_one,
18249 	.remove		= tg3_remove_one,
18250 	.err_handler	= &tg3_err_handler,
18251 	.driver.pm	= &tg3_pm_ops,
18252 	.shutdown	= tg3_shutdown,
18253 };
18254 
18255 module_pci_driver(tg3_driver);
18256