1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * r8169.c: RealTek 8169/8168/8101 ethernet driver.
4  *
5  * Copyright (c) 2002 ShuChen <shuchen@realtek.com.tw>
6  * Copyright (c) 2003 - 2007 Francois Romieu <romieu@fr.zoreil.com>
7  * Copyright (c) a lot of people too. Please respect their work.
8  *
9  * See MAINTAINERS file for support contact information.
10  */
11 
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/netdevice.h>
15 #include <linux/etherdevice.h>
16 #include <linux/clk.h>
17 #include <linux/delay.h>
18 #include <linux/ethtool.h>
19 #include <linux/phy.h>
20 #include <linux/if_vlan.h>
21 #include <linux/in.h>
22 #include <linux/io.h>
23 #include <linux/ip.h>
24 #include <linux/tcp.h>
25 #include <linux/interrupt.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/bitfield.h>
29 #include <linux/prefetch.h>
30 #include <linux/ipv6.h>
31 #include <asm/unaligned.h>
32 #include <net/ip6_checksum.h>
33 #include <net/netdev_queues.h>
34 
35 #include "r8169.h"
36 #include "r8169_firmware.h"
37 
38 #define FIRMWARE_8168D_1	"rtl_nic/rtl8168d-1.fw"
39 #define FIRMWARE_8168D_2	"rtl_nic/rtl8168d-2.fw"
40 #define FIRMWARE_8168E_1	"rtl_nic/rtl8168e-1.fw"
41 #define FIRMWARE_8168E_2	"rtl_nic/rtl8168e-2.fw"
42 #define FIRMWARE_8168E_3	"rtl_nic/rtl8168e-3.fw"
43 #define FIRMWARE_8168F_1	"rtl_nic/rtl8168f-1.fw"
44 #define FIRMWARE_8168F_2	"rtl_nic/rtl8168f-2.fw"
45 #define FIRMWARE_8105E_1	"rtl_nic/rtl8105e-1.fw"
46 #define FIRMWARE_8402_1		"rtl_nic/rtl8402-1.fw"
47 #define FIRMWARE_8411_1		"rtl_nic/rtl8411-1.fw"
48 #define FIRMWARE_8411_2		"rtl_nic/rtl8411-2.fw"
49 #define FIRMWARE_8106E_1	"rtl_nic/rtl8106e-1.fw"
50 #define FIRMWARE_8106E_2	"rtl_nic/rtl8106e-2.fw"
51 #define FIRMWARE_8168G_2	"rtl_nic/rtl8168g-2.fw"
52 #define FIRMWARE_8168G_3	"rtl_nic/rtl8168g-3.fw"
53 #define FIRMWARE_8168H_2	"rtl_nic/rtl8168h-2.fw"
54 #define FIRMWARE_8168FP_3	"rtl_nic/rtl8168fp-3.fw"
55 #define FIRMWARE_8107E_2	"rtl_nic/rtl8107e-2.fw"
56 #define FIRMWARE_8125A_3	"rtl_nic/rtl8125a-3.fw"
57 #define FIRMWARE_8125B_2	"rtl_nic/rtl8125b-2.fw"
58 
59 /* Maximum number of multicast addresses to filter (vs. Rx-all-multicast).
60    The RTL chips use a 64 element hash table based on the Ethernet CRC. */
61 #define	MC_FILTER_LIMIT	32
62 
63 #define TX_DMA_BURST	7	/* Maximum PCI burst, '7' is unlimited */
64 #define InterFrameGap	0x03	/* 3 means InterFrameGap = the shortest one */
65 
66 #define R8169_REGS_SIZE		256
67 #define R8169_RX_BUF_SIZE	(SZ_16K - 1)
68 #define NUM_TX_DESC	256	/* Number of Tx descriptor registers */
69 #define NUM_RX_DESC	256	/* Number of Rx descriptor registers */
70 #define R8169_TX_RING_BYTES	(NUM_TX_DESC * sizeof(struct TxDesc))
71 #define R8169_RX_RING_BYTES	(NUM_RX_DESC * sizeof(struct RxDesc))
72 #define R8169_TX_STOP_THRS	(MAX_SKB_FRAGS + 1)
73 #define R8169_TX_START_THRS	(2 * R8169_TX_STOP_THRS)
74 
75 #define OCP_STD_PHY_BASE	0xa400
76 
77 #define RTL_CFG_NO_GBIT	1
78 
79 /* write/read MMIO register */
80 #define RTL_W8(tp, reg, val8)	writeb((val8), tp->mmio_addr + (reg))
81 #define RTL_W16(tp, reg, val16)	writew((val16), tp->mmio_addr + (reg))
82 #define RTL_W32(tp, reg, val32)	writel((val32), tp->mmio_addr + (reg))
83 #define RTL_R8(tp, reg)		readb(tp->mmio_addr + (reg))
84 #define RTL_R16(tp, reg)		readw(tp->mmio_addr + (reg))
85 #define RTL_R32(tp, reg)		readl(tp->mmio_addr + (reg))
86 
87 #define JUMBO_4K	(4 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
88 #define JUMBO_6K	(6 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
89 #define JUMBO_7K	(7 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
90 #define JUMBO_9K	(9 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
91 
92 static const struct {
93 	const char *name;
94 	const char *fw_name;
95 } rtl_chip_infos[] = {
96 	/* PCI devices. */
97 	[RTL_GIGA_MAC_VER_02] = {"RTL8169s"				},
98 	[RTL_GIGA_MAC_VER_03] = {"RTL8110s"				},
99 	[RTL_GIGA_MAC_VER_04] = {"RTL8169sb/8110sb"			},
100 	[RTL_GIGA_MAC_VER_05] = {"RTL8169sc/8110sc"			},
101 	[RTL_GIGA_MAC_VER_06] = {"RTL8169sc/8110sc"			},
102 	/* PCI-E devices. */
103 	[RTL_GIGA_MAC_VER_07] = {"RTL8102e"				},
104 	[RTL_GIGA_MAC_VER_08] = {"RTL8102e"				},
105 	[RTL_GIGA_MAC_VER_09] = {"RTL8102e/RTL8103e"			},
106 	[RTL_GIGA_MAC_VER_10] = {"RTL8101e/RTL8100e"			},
107 	[RTL_GIGA_MAC_VER_11] = {"RTL8168b/8111b"			},
108 	[RTL_GIGA_MAC_VER_14] = {"RTL8401"				},
109 	[RTL_GIGA_MAC_VER_17] = {"RTL8168b/8111b"			},
110 	[RTL_GIGA_MAC_VER_18] = {"RTL8168cp/8111cp"			},
111 	[RTL_GIGA_MAC_VER_19] = {"RTL8168c/8111c"			},
112 	[RTL_GIGA_MAC_VER_20] = {"RTL8168c/8111c"			},
113 	[RTL_GIGA_MAC_VER_21] = {"RTL8168c/8111c"			},
114 	[RTL_GIGA_MAC_VER_22] = {"RTL8168c/8111c"			},
115 	[RTL_GIGA_MAC_VER_23] = {"RTL8168cp/8111cp"			},
116 	[RTL_GIGA_MAC_VER_24] = {"RTL8168cp/8111cp"			},
117 	[RTL_GIGA_MAC_VER_25] = {"RTL8168d/8111d",	FIRMWARE_8168D_1},
118 	[RTL_GIGA_MAC_VER_26] = {"RTL8168d/8111d",	FIRMWARE_8168D_2},
119 	[RTL_GIGA_MAC_VER_28] = {"RTL8168dp/8111dp"			},
120 	[RTL_GIGA_MAC_VER_29] = {"RTL8105e",		FIRMWARE_8105E_1},
121 	[RTL_GIGA_MAC_VER_30] = {"RTL8105e",		FIRMWARE_8105E_1},
122 	[RTL_GIGA_MAC_VER_31] = {"RTL8168dp/8111dp"			},
123 	[RTL_GIGA_MAC_VER_32] = {"RTL8168e/8111e",	FIRMWARE_8168E_1},
124 	[RTL_GIGA_MAC_VER_33] = {"RTL8168e/8111e",	FIRMWARE_8168E_2},
125 	[RTL_GIGA_MAC_VER_34] = {"RTL8168evl/8111evl",	FIRMWARE_8168E_3},
126 	[RTL_GIGA_MAC_VER_35] = {"RTL8168f/8111f",	FIRMWARE_8168F_1},
127 	[RTL_GIGA_MAC_VER_36] = {"RTL8168f/8111f",	FIRMWARE_8168F_2},
128 	[RTL_GIGA_MAC_VER_37] = {"RTL8402",		FIRMWARE_8402_1 },
129 	[RTL_GIGA_MAC_VER_38] = {"RTL8411",		FIRMWARE_8411_1 },
130 	[RTL_GIGA_MAC_VER_39] = {"RTL8106e",		FIRMWARE_8106E_1},
131 	[RTL_GIGA_MAC_VER_40] = {"RTL8168g/8111g",	FIRMWARE_8168G_2},
132 	[RTL_GIGA_MAC_VER_42] = {"RTL8168gu/8111gu",	FIRMWARE_8168G_3},
133 	[RTL_GIGA_MAC_VER_43] = {"RTL8106eus",		FIRMWARE_8106E_2},
134 	[RTL_GIGA_MAC_VER_44] = {"RTL8411b",		FIRMWARE_8411_2 },
135 	[RTL_GIGA_MAC_VER_46] = {"RTL8168h/8111h",	FIRMWARE_8168H_2},
136 	[RTL_GIGA_MAC_VER_48] = {"RTL8107e",		FIRMWARE_8107E_2},
137 	[RTL_GIGA_MAC_VER_51] = {"RTL8168ep/8111ep"			},
138 	[RTL_GIGA_MAC_VER_52] = {"RTL8168fp/RTL8117",  FIRMWARE_8168FP_3},
139 	[RTL_GIGA_MAC_VER_53] = {"RTL8168fp/RTL8117",			},
140 	[RTL_GIGA_MAC_VER_61] = {"RTL8125A",		FIRMWARE_8125A_3},
141 	/* reserve 62 for CFG_METHOD_4 in the vendor driver */
142 	[RTL_GIGA_MAC_VER_63] = {"RTL8125B",		FIRMWARE_8125B_2},
143 };
144 
145 static const struct pci_device_id rtl8169_pci_tbl[] = {
146 	{ PCI_VDEVICE(REALTEK,	0x2502) },
147 	{ PCI_VDEVICE(REALTEK,	0x2600) },
148 	{ PCI_VDEVICE(REALTEK,	0x8129) },
149 	{ PCI_VDEVICE(REALTEK,	0x8136), RTL_CFG_NO_GBIT },
150 	{ PCI_VDEVICE(REALTEK,	0x8161) },
151 	{ PCI_VDEVICE(REALTEK,	0x8162) },
152 	{ PCI_VDEVICE(REALTEK,	0x8167) },
153 	{ PCI_VDEVICE(REALTEK,	0x8168) },
154 	{ PCI_VDEVICE(NCUBE,	0x8168) },
155 	{ PCI_VDEVICE(REALTEK,	0x8169) },
156 	{ PCI_VENDOR_ID_DLINK,	0x4300,
157 		PCI_VENDOR_ID_DLINK, 0x4b10, 0, 0 },
158 	{ PCI_VDEVICE(DLINK,	0x4300) },
159 	{ PCI_VDEVICE(DLINK,	0x4302) },
160 	{ PCI_VDEVICE(AT,	0xc107) },
161 	{ PCI_VDEVICE(USR,	0x0116) },
162 	{ PCI_VENDOR_ID_LINKSYS, 0x1032, PCI_ANY_ID, 0x0024 },
163 	{ 0x0001, 0x8168, PCI_ANY_ID, 0x2410 },
164 	{ PCI_VDEVICE(REALTEK,	0x8125) },
165 	{ PCI_VDEVICE(REALTEK,	0x3000) },
166 	{}
167 };
168 
169 MODULE_DEVICE_TABLE(pci, rtl8169_pci_tbl);
170 
171 enum rtl_registers {
172 	MAC0		= 0,	/* Ethernet hardware address. */
173 	MAC4		= 4,
174 	MAR0		= 8,	/* Multicast filter. */
175 	CounterAddrLow		= 0x10,
176 	CounterAddrHigh		= 0x14,
177 	TxDescStartAddrLow	= 0x20,
178 	TxDescStartAddrHigh	= 0x24,
179 	TxHDescStartAddrLow	= 0x28,
180 	TxHDescStartAddrHigh	= 0x2c,
181 	FLASH		= 0x30,
182 	ERSR		= 0x36,
183 	ChipCmd		= 0x37,
184 	TxPoll		= 0x38,
185 	IntrMask	= 0x3c,
186 	IntrStatus	= 0x3e,
187 
188 	TxConfig	= 0x40,
189 #define	TXCFG_AUTO_FIFO			(1 << 7)	/* 8111e-vl */
190 #define	TXCFG_EMPTY			(1 << 11)	/* 8111e-vl */
191 
192 	RxConfig	= 0x44,
193 #define	RX128_INT_EN			(1 << 15)	/* 8111c and later */
194 #define	RX_MULTI_EN			(1 << 14)	/* 8111c only */
195 #define	RXCFG_FIFO_SHIFT		13
196 					/* No threshold before first PCI xfer */
197 #define	RX_FIFO_THRESH			(7 << RXCFG_FIFO_SHIFT)
198 #define	RX_EARLY_OFF			(1 << 11)
199 #define	RXCFG_DMA_SHIFT			8
200 					/* Unlimited maximum PCI burst. */
201 #define	RX_DMA_BURST			(7 << RXCFG_DMA_SHIFT)
202 
203 	Cfg9346		= 0x50,
204 	Config0		= 0x51,
205 	Config1		= 0x52,
206 	Config2		= 0x53,
207 #define PME_SIGNAL			(1 << 5)	/* 8168c and later */
208 
209 	Config3		= 0x54,
210 	Config4		= 0x55,
211 	Config5		= 0x56,
212 	PHYAR		= 0x60,
213 	PHYstatus	= 0x6c,
214 	RxMaxSize	= 0xda,
215 	CPlusCmd	= 0xe0,
216 	IntrMitigate	= 0xe2,
217 
218 #define RTL_COALESCE_TX_USECS	GENMASK(15, 12)
219 #define RTL_COALESCE_TX_FRAMES	GENMASK(11, 8)
220 #define RTL_COALESCE_RX_USECS	GENMASK(7, 4)
221 #define RTL_COALESCE_RX_FRAMES	GENMASK(3, 0)
222 
223 #define RTL_COALESCE_T_MAX	0x0fU
224 #define RTL_COALESCE_FRAME_MAX	(RTL_COALESCE_T_MAX * 4)
225 
226 	RxDescAddrLow	= 0xe4,
227 	RxDescAddrHigh	= 0xe8,
228 	EarlyTxThres	= 0xec,	/* 8169. Unit of 32 bytes. */
229 
230 #define NoEarlyTx	0x3f	/* Max value : no early transmit. */
231 
232 	MaxTxPacketSize	= 0xec,	/* 8101/8168. Unit of 128 bytes. */
233 
234 #define TxPacketMax	(8064 >> 7)
235 #define EarlySize	0x27
236 
237 	FuncEvent	= 0xf0,
238 	FuncEventMask	= 0xf4,
239 	FuncPresetState	= 0xf8,
240 	IBCR0           = 0xf8,
241 	IBCR2           = 0xf9,
242 	IBIMR0          = 0xfa,
243 	IBISR0          = 0xfb,
244 	FuncForceEvent	= 0xfc,
245 };
246 
247 enum rtl8168_8101_registers {
248 	CSIDR			= 0x64,
249 	CSIAR			= 0x68,
250 #define	CSIAR_FLAG			0x80000000
251 #define	CSIAR_WRITE_CMD			0x80000000
252 #define	CSIAR_BYTE_ENABLE		0x0000f000
253 #define	CSIAR_ADDR_MASK			0x00000fff
254 	PMCH			= 0x6f,
255 #define D3COLD_NO_PLL_DOWN		BIT(7)
256 #define D3HOT_NO_PLL_DOWN		BIT(6)
257 #define D3_NO_PLL_DOWN			(BIT(7) | BIT(6))
258 	EPHYAR			= 0x80,
259 #define	EPHYAR_FLAG			0x80000000
260 #define	EPHYAR_WRITE_CMD		0x80000000
261 #define	EPHYAR_REG_MASK			0x1f
262 #define	EPHYAR_REG_SHIFT		16
263 #define	EPHYAR_DATA_MASK		0xffff
264 	DLLPR			= 0xd0,
265 #define	PFM_EN				(1 << 6)
266 #define	TX_10M_PS_EN			(1 << 7)
267 	DBG_REG			= 0xd1,
268 #define	FIX_NAK_1			(1 << 4)
269 #define	FIX_NAK_2			(1 << 3)
270 	TWSI			= 0xd2,
271 	MCU			= 0xd3,
272 #define	NOW_IS_OOB			(1 << 7)
273 #define	TX_EMPTY			(1 << 5)
274 #define	RX_EMPTY			(1 << 4)
275 #define	RXTX_EMPTY			(TX_EMPTY | RX_EMPTY)
276 #define	EN_NDP				(1 << 3)
277 #define	EN_OOB_RESET			(1 << 2)
278 #define	LINK_LIST_RDY			(1 << 1)
279 	EFUSEAR			= 0xdc,
280 #define	EFUSEAR_FLAG			0x80000000
281 #define	EFUSEAR_WRITE_CMD		0x80000000
282 #define	EFUSEAR_READ_CMD		0x00000000
283 #define	EFUSEAR_REG_MASK		0x03ff
284 #define	EFUSEAR_REG_SHIFT		8
285 #define	EFUSEAR_DATA_MASK		0xff
286 	MISC_1			= 0xf2,
287 #define	PFM_D3COLD_EN			(1 << 6)
288 };
289 
290 enum rtl8168_registers {
291 	LED_FREQ		= 0x1a,
292 	EEE_LED			= 0x1b,
293 	ERIDR			= 0x70,
294 	ERIAR			= 0x74,
295 #define ERIAR_FLAG			0x80000000
296 #define ERIAR_WRITE_CMD			0x80000000
297 #define ERIAR_READ_CMD			0x00000000
298 #define ERIAR_ADDR_BYTE_ALIGN		4
299 #define ERIAR_TYPE_SHIFT		16
300 #define ERIAR_EXGMAC			(0x00 << ERIAR_TYPE_SHIFT)
301 #define ERIAR_MSIX			(0x01 << ERIAR_TYPE_SHIFT)
302 #define ERIAR_ASF			(0x02 << ERIAR_TYPE_SHIFT)
303 #define ERIAR_OOB			(0x02 << ERIAR_TYPE_SHIFT)
304 #define ERIAR_MASK_SHIFT		12
305 #define ERIAR_MASK_0001			(0x1 << ERIAR_MASK_SHIFT)
306 #define ERIAR_MASK_0011			(0x3 << ERIAR_MASK_SHIFT)
307 #define ERIAR_MASK_0100			(0x4 << ERIAR_MASK_SHIFT)
308 #define ERIAR_MASK_0101			(0x5 << ERIAR_MASK_SHIFT)
309 #define ERIAR_MASK_1111			(0xf << ERIAR_MASK_SHIFT)
310 	EPHY_RXER_NUM		= 0x7c,
311 	OCPDR			= 0xb0,	/* OCP GPHY access */
312 #define OCPDR_WRITE_CMD			0x80000000
313 #define OCPDR_READ_CMD			0x00000000
314 #define OCPDR_REG_MASK			0x7f
315 #define OCPDR_GPHY_REG_SHIFT		16
316 #define OCPDR_DATA_MASK			0xffff
317 	OCPAR			= 0xb4,
318 #define OCPAR_FLAG			0x80000000
319 #define OCPAR_GPHY_WRITE_CMD		0x8000f060
320 #define OCPAR_GPHY_READ_CMD		0x0000f060
321 	GPHY_OCP		= 0xb8,
322 	RDSAR1			= 0xd0,	/* 8168c only. Undocumented on 8168dp */
323 	MISC			= 0xf0,	/* 8168e only. */
324 #define TXPLA_RST			(1 << 29)
325 #define DISABLE_LAN_EN			(1 << 23) /* Enable GPIO pin */
326 #define PWM_EN				(1 << 22)
327 #define RXDV_GATED_EN			(1 << 19)
328 #define EARLY_TALLY_EN			(1 << 16)
329 };
330 
331 enum rtl8125_registers {
332 	IntrMask_8125		= 0x38,
333 	IntrStatus_8125		= 0x3c,
334 	TxPoll_8125		= 0x90,
335 	MAC0_BKP		= 0x19e0,
336 	EEE_TXIDLE_TIMER_8125	= 0x6048,
337 };
338 
339 #define RX_VLAN_INNER_8125	BIT(22)
340 #define RX_VLAN_OUTER_8125	BIT(23)
341 #define RX_VLAN_8125		(RX_VLAN_INNER_8125 | RX_VLAN_OUTER_8125)
342 
343 #define RX_FETCH_DFLT_8125	(8 << 27)
344 
345 enum rtl_register_content {
346 	/* InterruptStatusBits */
347 	SYSErr		= 0x8000,
348 	PCSTimeout	= 0x4000,
349 	SWInt		= 0x0100,
350 	TxDescUnavail	= 0x0080,
351 	RxFIFOOver	= 0x0040,
352 	LinkChg		= 0x0020,
353 	RxOverflow	= 0x0010,
354 	TxErr		= 0x0008,
355 	TxOK		= 0x0004,
356 	RxErr		= 0x0002,
357 	RxOK		= 0x0001,
358 
359 	/* RxStatusDesc */
360 	RxRWT	= (1 << 22),
361 	RxRES	= (1 << 21),
362 	RxRUNT	= (1 << 20),
363 	RxCRC	= (1 << 19),
364 
365 	/* ChipCmdBits */
366 	StopReq		= 0x80,
367 	CmdReset	= 0x10,
368 	CmdRxEnb	= 0x08,
369 	CmdTxEnb	= 0x04,
370 	RxBufEmpty	= 0x01,
371 
372 	/* TXPoll register p.5 */
373 	HPQ		= 0x80,		/* Poll cmd on the high prio queue */
374 	NPQ		= 0x40,		/* Poll cmd on the low prio queue */
375 	FSWInt		= 0x01,		/* Forced software interrupt */
376 
377 	/* Cfg9346Bits */
378 	Cfg9346_Lock	= 0x00,
379 	Cfg9346_Unlock	= 0xc0,
380 
381 	/* rx_mode_bits */
382 	AcceptErr	= 0x20,
383 	AcceptRunt	= 0x10,
384 #define RX_CONFIG_ACCEPT_ERR_MASK	0x30
385 	AcceptBroadcast	= 0x08,
386 	AcceptMulticast	= 0x04,
387 	AcceptMyPhys	= 0x02,
388 	AcceptAllPhys	= 0x01,
389 #define RX_CONFIG_ACCEPT_OK_MASK	0x0f
390 #define RX_CONFIG_ACCEPT_MASK		0x3f
391 
392 	/* TxConfigBits */
393 	TxInterFrameGapShift = 24,
394 	TxDMAShift = 8,	/* DMA burst value (0-7) is shift this many bits */
395 
396 	/* Config1 register p.24 */
397 	LEDS1		= (1 << 7),
398 	LEDS0		= (1 << 6),
399 	Speed_down	= (1 << 4),
400 	MEMMAP		= (1 << 3),
401 	IOMAP		= (1 << 2),
402 	VPD		= (1 << 1),
403 	PMEnable	= (1 << 0),	/* Power Management Enable */
404 
405 	/* Config2 register p. 25 */
406 	ClkReqEn	= (1 << 7),	/* Clock Request Enable */
407 	MSIEnable	= (1 << 5),	/* 8169 only. Reserved in the 8168. */
408 	PCI_Clock_66MHz = 0x01,
409 	PCI_Clock_33MHz = 0x00,
410 
411 	/* Config3 register p.25 */
412 	MagicPacket	= (1 << 5),	/* Wake up when receives a Magic Packet */
413 	LinkUp		= (1 << 4),	/* Wake up when the cable connection is re-established */
414 	Jumbo_En0	= (1 << 2),	/* 8168 only. Reserved in the 8168b */
415 	Rdy_to_L23	= (1 << 1),	/* L23 Enable */
416 	Beacon_en	= (1 << 0),	/* 8168 only. Reserved in the 8168b */
417 
418 	/* Config4 register */
419 	Jumbo_En1	= (1 << 1),	/* 8168 only. Reserved in the 8168b */
420 
421 	/* Config5 register p.27 */
422 	BWF		= (1 << 6),	/* Accept Broadcast wakeup frame */
423 	MWF		= (1 << 5),	/* Accept Multicast wakeup frame */
424 	UWF		= (1 << 4),	/* Accept Unicast wakeup frame */
425 	Spi_en		= (1 << 3),
426 	LanWake		= (1 << 1),	/* LanWake enable/disable */
427 	PMEStatus	= (1 << 0),	/* PME status can be reset by PCI RST# */
428 	ASPM_en		= (1 << 0),	/* ASPM enable */
429 
430 	/* CPlusCmd p.31 */
431 	EnableBist	= (1 << 15),	// 8168 8101
432 	Mac_dbgo_oe	= (1 << 14),	// 8168 8101
433 	EnAnaPLL	= (1 << 14),	// 8169
434 	Normal_mode	= (1 << 13),	// unused
435 	Force_half_dup	= (1 << 12),	// 8168 8101
436 	Force_rxflow_en	= (1 << 11),	// 8168 8101
437 	Force_txflow_en	= (1 << 10),	// 8168 8101
438 	Cxpl_dbg_sel	= (1 << 9),	// 8168 8101
439 	ASF		= (1 << 8),	// 8168 8101
440 	PktCntrDisable	= (1 << 7),	// 8168 8101
441 	Mac_dbgo_sel	= 0x001c,	// 8168
442 	RxVlan		= (1 << 6),
443 	RxChkSum	= (1 << 5),
444 	PCIDAC		= (1 << 4),
445 	PCIMulRW	= (1 << 3),
446 #define INTT_MASK	GENMASK(1, 0)
447 #define CPCMD_MASK	(Normal_mode | RxVlan | RxChkSum | INTT_MASK)
448 
449 	/* rtl8169_PHYstatus */
450 	TBI_Enable	= 0x80,
451 	TxFlowCtrl	= 0x40,
452 	RxFlowCtrl	= 0x20,
453 	_1000bpsF	= 0x10,
454 	_100bps		= 0x08,
455 	_10bps		= 0x04,
456 	LinkStatus	= 0x02,
457 	FullDup		= 0x01,
458 
459 	/* ResetCounterCommand */
460 	CounterReset	= 0x1,
461 
462 	/* DumpCounterCommand */
463 	CounterDump	= 0x8,
464 
465 	/* magic enable v2 */
466 	MagicPacket_v2	= (1 << 16),	/* Wake up when receives a Magic Packet */
467 };
468 
469 enum rtl_desc_bit {
470 	/* First doubleword. */
471 	DescOwn		= (1 << 31), /* Descriptor is owned by NIC */
472 	RingEnd		= (1 << 30), /* End of descriptor ring */
473 	FirstFrag	= (1 << 29), /* First segment of a packet */
474 	LastFrag	= (1 << 28), /* Final segment of a packet */
475 };
476 
477 /* Generic case. */
478 enum rtl_tx_desc_bit {
479 	/* First doubleword. */
480 	TD_LSO		= (1 << 27),		/* Large Send Offload */
481 #define TD_MSS_MAX			0x07ffu	/* MSS value */
482 
483 	/* Second doubleword. */
484 	TxVlanTag	= (1 << 17),		/* Add VLAN tag */
485 };
486 
487 /* 8169, 8168b and 810x except 8102e. */
488 enum rtl_tx_desc_bit_0 {
489 	/* First doubleword. */
490 #define TD0_MSS_SHIFT			16	/* MSS position (11 bits) */
491 	TD0_TCP_CS	= (1 << 16),		/* Calculate TCP/IP checksum */
492 	TD0_UDP_CS	= (1 << 17),		/* Calculate UDP/IP checksum */
493 	TD0_IP_CS	= (1 << 18),		/* Calculate IP checksum */
494 };
495 
496 /* 8102e, 8168c and beyond. */
497 enum rtl_tx_desc_bit_1 {
498 	/* First doubleword. */
499 	TD1_GTSENV4	= (1 << 26),		/* Giant Send for IPv4 */
500 	TD1_GTSENV6	= (1 << 25),		/* Giant Send for IPv6 */
501 #define GTTCPHO_SHIFT			18
502 #define GTTCPHO_MAX			0x7f
503 
504 	/* Second doubleword. */
505 #define TCPHO_SHIFT			18
506 #define TCPHO_MAX			0x3ff
507 #define TD1_MSS_SHIFT			18	/* MSS position (11 bits) */
508 	TD1_IPv6_CS	= (1 << 28),		/* Calculate IPv6 checksum */
509 	TD1_IPv4_CS	= (1 << 29),		/* Calculate IPv4 checksum */
510 	TD1_TCP_CS	= (1 << 30),		/* Calculate TCP/IP checksum */
511 	TD1_UDP_CS	= (1 << 31),		/* Calculate UDP/IP checksum */
512 };
513 
514 enum rtl_rx_desc_bit {
515 	/* Rx private */
516 	PID1		= (1 << 18), /* Protocol ID bit 1/2 */
517 	PID0		= (1 << 17), /* Protocol ID bit 0/2 */
518 
519 #define RxProtoUDP	(PID1)
520 #define RxProtoTCP	(PID0)
521 #define RxProtoIP	(PID1 | PID0)
522 #define RxProtoMask	RxProtoIP
523 
524 	IPFail		= (1 << 16), /* IP checksum failed */
525 	UDPFail		= (1 << 15), /* UDP/IP checksum failed */
526 	TCPFail		= (1 << 14), /* TCP/IP checksum failed */
527 
528 #define RxCSFailMask	(IPFail | UDPFail | TCPFail)
529 
530 	RxVlanTag	= (1 << 16), /* VLAN tag available */
531 };
532 
533 #define RTL_GSO_MAX_SIZE_V1	32000
534 #define RTL_GSO_MAX_SEGS_V1	24
535 #define RTL_GSO_MAX_SIZE_V2	64000
536 #define RTL_GSO_MAX_SEGS_V2	64
537 
538 struct TxDesc {
539 	__le32 opts1;
540 	__le32 opts2;
541 	__le64 addr;
542 };
543 
544 struct RxDesc {
545 	__le32 opts1;
546 	__le32 opts2;
547 	__le64 addr;
548 };
549 
550 struct ring_info {
551 	struct sk_buff	*skb;
552 	u32		len;
553 };
554 
555 struct rtl8169_counters {
556 	__le64	tx_packets;
557 	__le64	rx_packets;
558 	__le64	tx_errors;
559 	__le32	rx_errors;
560 	__le16	rx_missed;
561 	__le16	align_errors;
562 	__le32	tx_one_collision;
563 	__le32	tx_multi_collision;
564 	__le64	rx_unicast;
565 	__le64	rx_broadcast;
566 	__le32	rx_multicast;
567 	__le16	tx_aborted;
568 	__le16	tx_underun;
569 };
570 
571 struct rtl8169_tc_offsets {
572 	bool	inited;
573 	__le64	tx_errors;
574 	__le32	tx_multi_collision;
575 	__le16	tx_aborted;
576 	__le16	rx_missed;
577 };
578 
579 enum rtl_flag {
580 	RTL_FLAG_TASK_ENABLED = 0,
581 	RTL_FLAG_TASK_RESET_PENDING,
582 	RTL_FLAG_TASK_TX_TIMEOUT,
583 	RTL_FLAG_MAX
584 };
585 
586 enum rtl_dash_type {
587 	RTL_DASH_NONE,
588 	RTL_DASH_DP,
589 	RTL_DASH_EP,
590 };
591 
592 struct rtl8169_private {
593 	void __iomem *mmio_addr;	/* memory map physical address */
594 	struct pci_dev *pci_dev;
595 	struct net_device *dev;
596 	struct phy_device *phydev;
597 	struct napi_struct napi;
598 	enum mac_version mac_version;
599 	enum rtl_dash_type dash_type;
600 	u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */
601 	u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */
602 	u32 dirty_tx;
603 	struct TxDesc *TxDescArray;	/* 256-aligned Tx descriptor ring */
604 	struct RxDesc *RxDescArray;	/* 256-aligned Rx descriptor ring */
605 	dma_addr_t TxPhyAddr;
606 	dma_addr_t RxPhyAddr;
607 	struct page *Rx_databuff[NUM_RX_DESC];	/* Rx data buffers */
608 	struct ring_info tx_skb[NUM_TX_DESC];	/* Tx data buffers */
609 	u16 cp_cmd;
610 	u32 irq_mask;
611 	int irq;
612 	struct clk *clk;
613 
614 	struct {
615 		DECLARE_BITMAP(flags, RTL_FLAG_MAX);
616 		struct work_struct work;
617 	} wk;
618 
619 	raw_spinlock_t config25_lock;
620 	raw_spinlock_t mac_ocp_lock;
621 
622 	raw_spinlock_t cfg9346_usage_lock;
623 	int cfg9346_usage_count;
624 
625 	unsigned supports_gmii:1;
626 	dma_addr_t counters_phys_addr;
627 	struct rtl8169_counters *counters;
628 	struct rtl8169_tc_offsets tc_offset;
629 	u32 saved_wolopts;
630 	int eee_adv;
631 
632 	const char *fw_name;
633 	struct rtl_fw *rtl_fw;
634 
635 	u32 ocp_base;
636 };
637 
638 typedef void (*rtl_generic_fct)(struct rtl8169_private *tp);
639 
640 MODULE_AUTHOR("Realtek and the Linux r8169 crew <netdev@vger.kernel.org>");
641 MODULE_DESCRIPTION("RealTek RTL-8169 Gigabit Ethernet driver");
642 MODULE_SOFTDEP("pre: realtek");
643 MODULE_LICENSE("GPL");
644 MODULE_FIRMWARE(FIRMWARE_8168D_1);
645 MODULE_FIRMWARE(FIRMWARE_8168D_2);
646 MODULE_FIRMWARE(FIRMWARE_8168E_1);
647 MODULE_FIRMWARE(FIRMWARE_8168E_2);
648 MODULE_FIRMWARE(FIRMWARE_8168E_3);
649 MODULE_FIRMWARE(FIRMWARE_8105E_1);
650 MODULE_FIRMWARE(FIRMWARE_8168F_1);
651 MODULE_FIRMWARE(FIRMWARE_8168F_2);
652 MODULE_FIRMWARE(FIRMWARE_8402_1);
653 MODULE_FIRMWARE(FIRMWARE_8411_1);
654 MODULE_FIRMWARE(FIRMWARE_8411_2);
655 MODULE_FIRMWARE(FIRMWARE_8106E_1);
656 MODULE_FIRMWARE(FIRMWARE_8106E_2);
657 MODULE_FIRMWARE(FIRMWARE_8168G_2);
658 MODULE_FIRMWARE(FIRMWARE_8168G_3);
659 MODULE_FIRMWARE(FIRMWARE_8168H_2);
660 MODULE_FIRMWARE(FIRMWARE_8168FP_3);
661 MODULE_FIRMWARE(FIRMWARE_8107E_2);
662 MODULE_FIRMWARE(FIRMWARE_8125A_3);
663 MODULE_FIRMWARE(FIRMWARE_8125B_2);
664 
665 static inline struct device *tp_to_dev(struct rtl8169_private *tp)
666 {
667 	return &tp->pci_dev->dev;
668 }
669 
670 static void rtl_lock_config_regs(struct rtl8169_private *tp)
671 {
672 	unsigned long flags;
673 
674 	raw_spin_lock_irqsave(&tp->cfg9346_usage_lock, flags);
675 	if (!--tp->cfg9346_usage_count)
676 		RTL_W8(tp, Cfg9346, Cfg9346_Lock);
677 	raw_spin_unlock_irqrestore(&tp->cfg9346_usage_lock, flags);
678 }
679 
680 static void rtl_unlock_config_regs(struct rtl8169_private *tp)
681 {
682 	unsigned long flags;
683 
684 	raw_spin_lock_irqsave(&tp->cfg9346_usage_lock, flags);
685 	if (!tp->cfg9346_usage_count++)
686 		RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
687 	raw_spin_unlock_irqrestore(&tp->cfg9346_usage_lock, flags);
688 }
689 
690 static void rtl_pci_commit(struct rtl8169_private *tp)
691 {
692 	/* Read an arbitrary register to commit a preceding PCI write */
693 	RTL_R8(tp, ChipCmd);
694 }
695 
696 static void rtl_mod_config2(struct rtl8169_private *tp, u8 clear, u8 set)
697 {
698 	unsigned long flags;
699 	u8 val;
700 
701 	raw_spin_lock_irqsave(&tp->config25_lock, flags);
702 	val = RTL_R8(tp, Config2);
703 	RTL_W8(tp, Config2, (val & ~clear) | set);
704 	raw_spin_unlock_irqrestore(&tp->config25_lock, flags);
705 }
706 
707 static void rtl_mod_config5(struct rtl8169_private *tp, u8 clear, u8 set)
708 {
709 	unsigned long flags;
710 	u8 val;
711 
712 	raw_spin_lock_irqsave(&tp->config25_lock, flags);
713 	val = RTL_R8(tp, Config5);
714 	RTL_W8(tp, Config5, (val & ~clear) | set);
715 	raw_spin_unlock_irqrestore(&tp->config25_lock, flags);
716 }
717 
718 static bool rtl_is_8125(struct rtl8169_private *tp)
719 {
720 	return tp->mac_version >= RTL_GIGA_MAC_VER_61;
721 }
722 
723 static bool rtl_is_8168evl_up(struct rtl8169_private *tp)
724 {
725 	return tp->mac_version >= RTL_GIGA_MAC_VER_34 &&
726 	       tp->mac_version != RTL_GIGA_MAC_VER_39 &&
727 	       tp->mac_version <= RTL_GIGA_MAC_VER_53;
728 }
729 
730 static bool rtl_supports_eee(struct rtl8169_private *tp)
731 {
732 	return tp->mac_version >= RTL_GIGA_MAC_VER_34 &&
733 	       tp->mac_version != RTL_GIGA_MAC_VER_37 &&
734 	       tp->mac_version != RTL_GIGA_MAC_VER_39;
735 }
736 
737 static void rtl_read_mac_from_reg(struct rtl8169_private *tp, u8 *mac, int reg)
738 {
739 	int i;
740 
741 	for (i = 0; i < ETH_ALEN; i++)
742 		mac[i] = RTL_R8(tp, reg + i);
743 }
744 
745 struct rtl_cond {
746 	bool (*check)(struct rtl8169_private *);
747 	const char *msg;
748 };
749 
750 static bool rtl_loop_wait(struct rtl8169_private *tp, const struct rtl_cond *c,
751 			  unsigned long usecs, int n, bool high)
752 {
753 	int i;
754 
755 	for (i = 0; i < n; i++) {
756 		if (c->check(tp) == high)
757 			return true;
758 		fsleep(usecs);
759 	}
760 
761 	if (net_ratelimit())
762 		netdev_err(tp->dev, "%s == %d (loop: %d, delay: %lu).\n",
763 			   c->msg, !high, n, usecs);
764 	return false;
765 }
766 
767 static bool rtl_loop_wait_high(struct rtl8169_private *tp,
768 			       const struct rtl_cond *c,
769 			       unsigned long d, int n)
770 {
771 	return rtl_loop_wait(tp, c, d, n, true);
772 }
773 
774 static bool rtl_loop_wait_low(struct rtl8169_private *tp,
775 			      const struct rtl_cond *c,
776 			      unsigned long d, int n)
777 {
778 	return rtl_loop_wait(tp, c, d, n, false);
779 }
780 
781 #define DECLARE_RTL_COND(name)				\
782 static bool name ## _check(struct rtl8169_private *);	\
783 							\
784 static const struct rtl_cond name = {			\
785 	.check	= name ## _check,			\
786 	.msg	= #name					\
787 };							\
788 							\
789 static bool name ## _check(struct rtl8169_private *tp)
790 
791 static void r8168fp_adjust_ocp_cmd(struct rtl8169_private *tp, u32 *cmd, int type)
792 {
793 	/* based on RTL8168FP_OOBMAC_BASE in vendor driver */
794 	if (type == ERIAR_OOB &&
795 	    (tp->mac_version == RTL_GIGA_MAC_VER_52 ||
796 	     tp->mac_version == RTL_GIGA_MAC_VER_53))
797 		*cmd |= 0xf70 << 18;
798 }
799 
800 DECLARE_RTL_COND(rtl_eriar_cond)
801 {
802 	return RTL_R32(tp, ERIAR) & ERIAR_FLAG;
803 }
804 
805 static void _rtl_eri_write(struct rtl8169_private *tp, int addr, u32 mask,
806 			   u32 val, int type)
807 {
808 	u32 cmd = ERIAR_WRITE_CMD | type | mask | addr;
809 
810 	if (WARN(addr & 3 || !mask, "addr: 0x%x, mask: 0x%08x\n", addr, mask))
811 		return;
812 
813 	RTL_W32(tp, ERIDR, val);
814 	r8168fp_adjust_ocp_cmd(tp, &cmd, type);
815 	RTL_W32(tp, ERIAR, cmd);
816 
817 	rtl_loop_wait_low(tp, &rtl_eriar_cond, 100, 100);
818 }
819 
820 static void rtl_eri_write(struct rtl8169_private *tp, int addr, u32 mask,
821 			  u32 val)
822 {
823 	_rtl_eri_write(tp, addr, mask, val, ERIAR_EXGMAC);
824 }
825 
826 static u32 _rtl_eri_read(struct rtl8169_private *tp, int addr, int type)
827 {
828 	u32 cmd = ERIAR_READ_CMD | type | ERIAR_MASK_1111 | addr;
829 
830 	r8168fp_adjust_ocp_cmd(tp, &cmd, type);
831 	RTL_W32(tp, ERIAR, cmd);
832 
833 	return rtl_loop_wait_high(tp, &rtl_eriar_cond, 100, 100) ?
834 		RTL_R32(tp, ERIDR) : ~0;
835 }
836 
837 static u32 rtl_eri_read(struct rtl8169_private *tp, int addr)
838 {
839 	return _rtl_eri_read(tp, addr, ERIAR_EXGMAC);
840 }
841 
842 static void rtl_w0w1_eri(struct rtl8169_private *tp, int addr, u32 p, u32 m)
843 {
844 	u32 val = rtl_eri_read(tp, addr);
845 
846 	rtl_eri_write(tp, addr, ERIAR_MASK_1111, (val & ~m) | p);
847 }
848 
849 static void rtl_eri_set_bits(struct rtl8169_private *tp, int addr, u32 p)
850 {
851 	rtl_w0w1_eri(tp, addr, p, 0);
852 }
853 
854 static void rtl_eri_clear_bits(struct rtl8169_private *tp, int addr, u32 m)
855 {
856 	rtl_w0w1_eri(tp, addr, 0, m);
857 }
858 
859 static bool rtl_ocp_reg_failure(u32 reg)
860 {
861 	return WARN_ONCE(reg & 0xffff0001, "Invalid ocp reg %x!\n", reg);
862 }
863 
864 DECLARE_RTL_COND(rtl_ocp_gphy_cond)
865 {
866 	return RTL_R32(tp, GPHY_OCP) & OCPAR_FLAG;
867 }
868 
869 static void r8168_phy_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
870 {
871 	if (rtl_ocp_reg_failure(reg))
872 		return;
873 
874 	RTL_W32(tp, GPHY_OCP, OCPAR_FLAG | (reg << 15) | data);
875 
876 	rtl_loop_wait_low(tp, &rtl_ocp_gphy_cond, 25, 10);
877 }
878 
879 static int r8168_phy_ocp_read(struct rtl8169_private *tp, u32 reg)
880 {
881 	if (rtl_ocp_reg_failure(reg))
882 		return 0;
883 
884 	RTL_W32(tp, GPHY_OCP, reg << 15);
885 
886 	return rtl_loop_wait_high(tp, &rtl_ocp_gphy_cond, 25, 10) ?
887 		(RTL_R32(tp, GPHY_OCP) & 0xffff) : -ETIMEDOUT;
888 }
889 
890 static void __r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
891 {
892 	if (rtl_ocp_reg_failure(reg))
893 		return;
894 
895 	RTL_W32(tp, OCPDR, OCPAR_FLAG | (reg << 15) | data);
896 }
897 
898 static void r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
899 {
900 	unsigned long flags;
901 
902 	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
903 	__r8168_mac_ocp_write(tp, reg, data);
904 	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
905 }
906 
907 static u16 __r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
908 {
909 	if (rtl_ocp_reg_failure(reg))
910 		return 0;
911 
912 	RTL_W32(tp, OCPDR, reg << 15);
913 
914 	return RTL_R32(tp, OCPDR);
915 }
916 
917 static u16 r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
918 {
919 	unsigned long flags;
920 	u16 val;
921 
922 	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
923 	val = __r8168_mac_ocp_read(tp, reg);
924 	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
925 
926 	return val;
927 }
928 
929 static void r8168_mac_ocp_modify(struct rtl8169_private *tp, u32 reg, u16 mask,
930 				 u16 set)
931 {
932 	unsigned long flags;
933 	u16 data;
934 
935 	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
936 	data = __r8168_mac_ocp_read(tp, reg);
937 	__r8168_mac_ocp_write(tp, reg, (data & ~mask) | set);
938 	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
939 }
940 
941 /* Work around a hw issue with RTL8168g PHY, the quirk disables
942  * PHY MCU interrupts before PHY power-down.
943  */
944 static void rtl8168g_phy_suspend_quirk(struct rtl8169_private *tp, int value)
945 {
946 	switch (tp->mac_version) {
947 	case RTL_GIGA_MAC_VER_40:
948 		if (value & BMCR_RESET || !(value & BMCR_PDOWN))
949 			rtl_eri_set_bits(tp, 0x1a8, 0xfc000000);
950 		else
951 			rtl_eri_clear_bits(tp, 0x1a8, 0xfc000000);
952 		break;
953 	default:
954 		break;
955 	}
956 };
957 
958 static void r8168g_mdio_write(struct rtl8169_private *tp, int reg, int value)
959 {
960 	if (reg == 0x1f) {
961 		tp->ocp_base = value ? value << 4 : OCP_STD_PHY_BASE;
962 		return;
963 	}
964 
965 	if (tp->ocp_base != OCP_STD_PHY_BASE)
966 		reg -= 0x10;
967 
968 	if (tp->ocp_base == OCP_STD_PHY_BASE && reg == MII_BMCR)
969 		rtl8168g_phy_suspend_quirk(tp, value);
970 
971 	r8168_phy_ocp_write(tp, tp->ocp_base + reg * 2, value);
972 }
973 
974 static int r8168g_mdio_read(struct rtl8169_private *tp, int reg)
975 {
976 	if (reg == 0x1f)
977 		return tp->ocp_base == OCP_STD_PHY_BASE ? 0 : tp->ocp_base >> 4;
978 
979 	if (tp->ocp_base != OCP_STD_PHY_BASE)
980 		reg -= 0x10;
981 
982 	return r8168_phy_ocp_read(tp, tp->ocp_base + reg * 2);
983 }
984 
985 static void mac_mcu_write(struct rtl8169_private *tp, int reg, int value)
986 {
987 	if (reg == 0x1f) {
988 		tp->ocp_base = value << 4;
989 		return;
990 	}
991 
992 	r8168_mac_ocp_write(tp, tp->ocp_base + reg, value);
993 }
994 
995 static int mac_mcu_read(struct rtl8169_private *tp, int reg)
996 {
997 	return r8168_mac_ocp_read(tp, tp->ocp_base + reg);
998 }
999 
1000 DECLARE_RTL_COND(rtl_phyar_cond)
1001 {
1002 	return RTL_R32(tp, PHYAR) & 0x80000000;
1003 }
1004 
1005 static void r8169_mdio_write(struct rtl8169_private *tp, int reg, int value)
1006 {
1007 	RTL_W32(tp, PHYAR, 0x80000000 | (reg & 0x1f) << 16 | (value & 0xffff));
1008 
1009 	rtl_loop_wait_low(tp, &rtl_phyar_cond, 25, 20);
1010 	/*
1011 	 * According to hardware specs a 20us delay is required after write
1012 	 * complete indication, but before sending next command.
1013 	 */
1014 	udelay(20);
1015 }
1016 
1017 static int r8169_mdio_read(struct rtl8169_private *tp, int reg)
1018 {
1019 	int value;
1020 
1021 	RTL_W32(tp, PHYAR, 0x0 | (reg & 0x1f) << 16);
1022 
1023 	value = rtl_loop_wait_high(tp, &rtl_phyar_cond, 25, 20) ?
1024 		RTL_R32(tp, PHYAR) & 0xffff : -ETIMEDOUT;
1025 
1026 	/*
1027 	 * According to hardware specs a 20us delay is required after read
1028 	 * complete indication, but before sending next command.
1029 	 */
1030 	udelay(20);
1031 
1032 	return value;
1033 }
1034 
1035 DECLARE_RTL_COND(rtl_ocpar_cond)
1036 {
1037 	return RTL_R32(tp, OCPAR) & OCPAR_FLAG;
1038 }
1039 
1040 #define R8168DP_1_MDIO_ACCESS_BIT	0x00020000
1041 
1042 static void r8168dp_2_mdio_start(struct rtl8169_private *tp)
1043 {
1044 	RTL_W32(tp, 0xd0, RTL_R32(tp, 0xd0) & ~R8168DP_1_MDIO_ACCESS_BIT);
1045 }
1046 
1047 static void r8168dp_2_mdio_stop(struct rtl8169_private *tp)
1048 {
1049 	RTL_W32(tp, 0xd0, RTL_R32(tp, 0xd0) | R8168DP_1_MDIO_ACCESS_BIT);
1050 }
1051 
1052 static void r8168dp_2_mdio_write(struct rtl8169_private *tp, int reg, int value)
1053 {
1054 	r8168dp_2_mdio_start(tp);
1055 
1056 	r8169_mdio_write(tp, reg, value);
1057 
1058 	r8168dp_2_mdio_stop(tp);
1059 }
1060 
1061 static int r8168dp_2_mdio_read(struct rtl8169_private *tp, int reg)
1062 {
1063 	int value;
1064 
1065 	/* Work around issue with chip reporting wrong PHY ID */
1066 	if (reg == MII_PHYSID2)
1067 		return 0xc912;
1068 
1069 	r8168dp_2_mdio_start(tp);
1070 
1071 	value = r8169_mdio_read(tp, reg);
1072 
1073 	r8168dp_2_mdio_stop(tp);
1074 
1075 	return value;
1076 }
1077 
1078 static void rtl_writephy(struct rtl8169_private *tp, int location, int val)
1079 {
1080 	switch (tp->mac_version) {
1081 	case RTL_GIGA_MAC_VER_28:
1082 	case RTL_GIGA_MAC_VER_31:
1083 		r8168dp_2_mdio_write(tp, location, val);
1084 		break;
1085 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
1086 		r8168g_mdio_write(tp, location, val);
1087 		break;
1088 	default:
1089 		r8169_mdio_write(tp, location, val);
1090 		break;
1091 	}
1092 }
1093 
1094 static int rtl_readphy(struct rtl8169_private *tp, int location)
1095 {
1096 	switch (tp->mac_version) {
1097 	case RTL_GIGA_MAC_VER_28:
1098 	case RTL_GIGA_MAC_VER_31:
1099 		return r8168dp_2_mdio_read(tp, location);
1100 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
1101 		return r8168g_mdio_read(tp, location);
1102 	default:
1103 		return r8169_mdio_read(tp, location);
1104 	}
1105 }
1106 
1107 DECLARE_RTL_COND(rtl_ephyar_cond)
1108 {
1109 	return RTL_R32(tp, EPHYAR) & EPHYAR_FLAG;
1110 }
1111 
1112 static void rtl_ephy_write(struct rtl8169_private *tp, int reg_addr, int value)
1113 {
1114 	RTL_W32(tp, EPHYAR, EPHYAR_WRITE_CMD | (value & EPHYAR_DATA_MASK) |
1115 		(reg_addr & EPHYAR_REG_MASK) << EPHYAR_REG_SHIFT);
1116 
1117 	rtl_loop_wait_low(tp, &rtl_ephyar_cond, 10, 100);
1118 
1119 	udelay(10);
1120 }
1121 
1122 static u16 rtl_ephy_read(struct rtl8169_private *tp, int reg_addr)
1123 {
1124 	RTL_W32(tp, EPHYAR, (reg_addr & EPHYAR_REG_MASK) << EPHYAR_REG_SHIFT);
1125 
1126 	return rtl_loop_wait_high(tp, &rtl_ephyar_cond, 10, 100) ?
1127 		RTL_R32(tp, EPHYAR) & EPHYAR_DATA_MASK : ~0;
1128 }
1129 
1130 static u32 r8168dp_ocp_read(struct rtl8169_private *tp, u16 reg)
1131 {
1132 	RTL_W32(tp, OCPAR, 0x0fu << 12 | (reg & 0x0fff));
1133 	return rtl_loop_wait_high(tp, &rtl_ocpar_cond, 100, 20) ?
1134 		RTL_R32(tp, OCPDR) : ~0;
1135 }
1136 
1137 static u32 r8168ep_ocp_read(struct rtl8169_private *tp, u16 reg)
1138 {
1139 	return _rtl_eri_read(tp, reg, ERIAR_OOB);
1140 }
1141 
1142 static void r8168dp_ocp_write(struct rtl8169_private *tp, u8 mask, u16 reg,
1143 			      u32 data)
1144 {
1145 	RTL_W32(tp, OCPDR, data);
1146 	RTL_W32(tp, OCPAR, OCPAR_FLAG | ((u32)mask & 0x0f) << 12 | (reg & 0x0fff));
1147 	rtl_loop_wait_low(tp, &rtl_ocpar_cond, 100, 20);
1148 }
1149 
1150 static void r8168ep_ocp_write(struct rtl8169_private *tp, u8 mask, u16 reg,
1151 			      u32 data)
1152 {
1153 	_rtl_eri_write(tp, reg, ((u32)mask & 0x0f) << ERIAR_MASK_SHIFT,
1154 		       data, ERIAR_OOB);
1155 }
1156 
1157 static void r8168dp_oob_notify(struct rtl8169_private *tp, u8 cmd)
1158 {
1159 	rtl_eri_write(tp, 0xe8, ERIAR_MASK_0001, cmd);
1160 
1161 	r8168dp_ocp_write(tp, 0x1, 0x30, 0x00000001);
1162 }
1163 
1164 #define OOB_CMD_RESET		0x00
1165 #define OOB_CMD_DRIVER_START	0x05
1166 #define OOB_CMD_DRIVER_STOP	0x06
1167 
1168 static u16 rtl8168_get_ocp_reg(struct rtl8169_private *tp)
1169 {
1170 	return (tp->mac_version == RTL_GIGA_MAC_VER_31) ? 0xb8 : 0x10;
1171 }
1172 
1173 DECLARE_RTL_COND(rtl_dp_ocp_read_cond)
1174 {
1175 	u16 reg;
1176 
1177 	reg = rtl8168_get_ocp_reg(tp);
1178 
1179 	return r8168dp_ocp_read(tp, reg) & 0x00000800;
1180 }
1181 
1182 DECLARE_RTL_COND(rtl_ep_ocp_read_cond)
1183 {
1184 	return r8168ep_ocp_read(tp, 0x124) & 0x00000001;
1185 }
1186 
1187 DECLARE_RTL_COND(rtl_ocp_tx_cond)
1188 {
1189 	return RTL_R8(tp, IBISR0) & 0x20;
1190 }
1191 
1192 static void rtl8168ep_stop_cmac(struct rtl8169_private *tp)
1193 {
1194 	RTL_W8(tp, IBCR2, RTL_R8(tp, IBCR2) & ~0x01);
1195 	rtl_loop_wait_high(tp, &rtl_ocp_tx_cond, 50000, 2000);
1196 	RTL_W8(tp, IBISR0, RTL_R8(tp, IBISR0) | 0x20);
1197 	RTL_W8(tp, IBCR0, RTL_R8(tp, IBCR0) & ~0x01);
1198 }
1199 
1200 static void rtl8168dp_driver_start(struct rtl8169_private *tp)
1201 {
1202 	r8168dp_oob_notify(tp, OOB_CMD_DRIVER_START);
1203 	rtl_loop_wait_high(tp, &rtl_dp_ocp_read_cond, 10000, 10);
1204 }
1205 
1206 static void rtl8168ep_driver_start(struct rtl8169_private *tp)
1207 {
1208 	r8168ep_ocp_write(tp, 0x01, 0x180, OOB_CMD_DRIVER_START);
1209 	r8168ep_ocp_write(tp, 0x01, 0x30, r8168ep_ocp_read(tp, 0x30) | 0x01);
1210 	rtl_loop_wait_high(tp, &rtl_ep_ocp_read_cond, 10000, 10);
1211 }
1212 
1213 static void rtl8168_driver_start(struct rtl8169_private *tp)
1214 {
1215 	if (tp->dash_type == RTL_DASH_DP)
1216 		rtl8168dp_driver_start(tp);
1217 	else
1218 		rtl8168ep_driver_start(tp);
1219 }
1220 
1221 static void rtl8168dp_driver_stop(struct rtl8169_private *tp)
1222 {
1223 	r8168dp_oob_notify(tp, OOB_CMD_DRIVER_STOP);
1224 	rtl_loop_wait_low(tp, &rtl_dp_ocp_read_cond, 10000, 10);
1225 }
1226 
1227 static void rtl8168ep_driver_stop(struct rtl8169_private *tp)
1228 {
1229 	rtl8168ep_stop_cmac(tp);
1230 	r8168ep_ocp_write(tp, 0x01, 0x180, OOB_CMD_DRIVER_STOP);
1231 	r8168ep_ocp_write(tp, 0x01, 0x30, r8168ep_ocp_read(tp, 0x30) | 0x01);
1232 	rtl_loop_wait_low(tp, &rtl_ep_ocp_read_cond, 10000, 10);
1233 }
1234 
1235 static void rtl8168_driver_stop(struct rtl8169_private *tp)
1236 {
1237 	if (tp->dash_type == RTL_DASH_DP)
1238 		rtl8168dp_driver_stop(tp);
1239 	else
1240 		rtl8168ep_driver_stop(tp);
1241 }
1242 
1243 static bool r8168dp_check_dash(struct rtl8169_private *tp)
1244 {
1245 	u16 reg = rtl8168_get_ocp_reg(tp);
1246 
1247 	return r8168dp_ocp_read(tp, reg) & BIT(15);
1248 }
1249 
1250 static bool r8168ep_check_dash(struct rtl8169_private *tp)
1251 {
1252 	return r8168ep_ocp_read(tp, 0x128) & BIT(0);
1253 }
1254 
1255 static enum rtl_dash_type rtl_check_dash(struct rtl8169_private *tp)
1256 {
1257 	switch (tp->mac_version) {
1258 	case RTL_GIGA_MAC_VER_28:
1259 	case RTL_GIGA_MAC_VER_31:
1260 		return r8168dp_check_dash(tp) ? RTL_DASH_DP : RTL_DASH_NONE;
1261 	case RTL_GIGA_MAC_VER_51 ... RTL_GIGA_MAC_VER_53:
1262 		return r8168ep_check_dash(tp) ? RTL_DASH_EP : RTL_DASH_NONE;
1263 	default:
1264 		return RTL_DASH_NONE;
1265 	}
1266 }
1267 
1268 static void rtl_set_d3_pll_down(struct rtl8169_private *tp, bool enable)
1269 {
1270 	switch (tp->mac_version) {
1271 	case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_26:
1272 	case RTL_GIGA_MAC_VER_29 ... RTL_GIGA_MAC_VER_30:
1273 	case RTL_GIGA_MAC_VER_32 ... RTL_GIGA_MAC_VER_37:
1274 	case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_63:
1275 		if (enable)
1276 			RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) & ~D3_NO_PLL_DOWN);
1277 		else
1278 			RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | D3_NO_PLL_DOWN);
1279 		break;
1280 	default:
1281 		break;
1282 	}
1283 }
1284 
1285 static void rtl_reset_packet_filter(struct rtl8169_private *tp)
1286 {
1287 	rtl_eri_clear_bits(tp, 0xdc, BIT(0));
1288 	rtl_eri_set_bits(tp, 0xdc, BIT(0));
1289 }
1290 
1291 DECLARE_RTL_COND(rtl_efusear_cond)
1292 {
1293 	return RTL_R32(tp, EFUSEAR) & EFUSEAR_FLAG;
1294 }
1295 
1296 u8 rtl8168d_efuse_read(struct rtl8169_private *tp, int reg_addr)
1297 {
1298 	RTL_W32(tp, EFUSEAR, (reg_addr & EFUSEAR_REG_MASK) << EFUSEAR_REG_SHIFT);
1299 
1300 	return rtl_loop_wait_high(tp, &rtl_efusear_cond, 100, 300) ?
1301 		RTL_R32(tp, EFUSEAR) & EFUSEAR_DATA_MASK : ~0;
1302 }
1303 
1304 static u32 rtl_get_events(struct rtl8169_private *tp)
1305 {
1306 	if (rtl_is_8125(tp))
1307 		return RTL_R32(tp, IntrStatus_8125);
1308 	else
1309 		return RTL_R16(tp, IntrStatus);
1310 }
1311 
1312 static void rtl_ack_events(struct rtl8169_private *tp, u32 bits)
1313 {
1314 	if (rtl_is_8125(tp))
1315 		RTL_W32(tp, IntrStatus_8125, bits);
1316 	else
1317 		RTL_W16(tp, IntrStatus, bits);
1318 }
1319 
1320 static void rtl_irq_disable(struct rtl8169_private *tp)
1321 {
1322 	if (rtl_is_8125(tp))
1323 		RTL_W32(tp, IntrMask_8125, 0);
1324 	else
1325 		RTL_W16(tp, IntrMask, 0);
1326 }
1327 
1328 static void rtl_irq_enable(struct rtl8169_private *tp)
1329 {
1330 	if (rtl_is_8125(tp))
1331 		RTL_W32(tp, IntrMask_8125, tp->irq_mask);
1332 	else
1333 		RTL_W16(tp, IntrMask, tp->irq_mask);
1334 }
1335 
1336 static void rtl8169_irq_mask_and_ack(struct rtl8169_private *tp)
1337 {
1338 	rtl_irq_disable(tp);
1339 	rtl_ack_events(tp, 0xffffffff);
1340 	rtl_pci_commit(tp);
1341 }
1342 
1343 static void rtl_link_chg_patch(struct rtl8169_private *tp)
1344 {
1345 	struct phy_device *phydev = tp->phydev;
1346 
1347 	if (tp->mac_version == RTL_GIGA_MAC_VER_34 ||
1348 	    tp->mac_version == RTL_GIGA_MAC_VER_38) {
1349 		if (phydev->speed == SPEED_1000) {
1350 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x00000011);
1351 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1352 		} else if (phydev->speed == SPEED_100) {
1353 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1354 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1355 		} else {
1356 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1357 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x0000003f);
1358 		}
1359 		rtl_reset_packet_filter(tp);
1360 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_35 ||
1361 		   tp->mac_version == RTL_GIGA_MAC_VER_36) {
1362 		if (phydev->speed == SPEED_1000) {
1363 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x00000011);
1364 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1365 		} else {
1366 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1367 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x0000003f);
1368 		}
1369 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_37) {
1370 		if (phydev->speed == SPEED_10) {
1371 			rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x4d02);
1372 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_0011, 0x0060a);
1373 		} else {
1374 			rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x0000);
1375 		}
1376 	}
1377 }
1378 
1379 #define WAKE_ANY (WAKE_PHY | WAKE_MAGIC | WAKE_UCAST | WAKE_BCAST | WAKE_MCAST)
1380 
1381 static void rtl8169_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1382 {
1383 	struct rtl8169_private *tp = netdev_priv(dev);
1384 
1385 	wol->supported = WAKE_ANY;
1386 	wol->wolopts = tp->saved_wolopts;
1387 }
1388 
1389 static void __rtl8169_set_wol(struct rtl8169_private *tp, u32 wolopts)
1390 {
1391 	static const struct {
1392 		u32 opt;
1393 		u16 reg;
1394 		u8  mask;
1395 	} cfg[] = {
1396 		{ WAKE_PHY,   Config3, LinkUp },
1397 		{ WAKE_UCAST, Config5, UWF },
1398 		{ WAKE_BCAST, Config5, BWF },
1399 		{ WAKE_MCAST, Config5, MWF },
1400 		{ WAKE_ANY,   Config5, LanWake },
1401 		{ WAKE_MAGIC, Config3, MagicPacket }
1402 	};
1403 	unsigned int i, tmp = ARRAY_SIZE(cfg);
1404 	unsigned long flags;
1405 	u8 options;
1406 
1407 	rtl_unlock_config_regs(tp);
1408 
1409 	if (rtl_is_8168evl_up(tp)) {
1410 		tmp--;
1411 		if (wolopts & WAKE_MAGIC)
1412 			rtl_eri_set_bits(tp, 0x0dc, MagicPacket_v2);
1413 		else
1414 			rtl_eri_clear_bits(tp, 0x0dc, MagicPacket_v2);
1415 	} else if (rtl_is_8125(tp)) {
1416 		tmp--;
1417 		if (wolopts & WAKE_MAGIC)
1418 			r8168_mac_ocp_modify(tp, 0xc0b6, 0, BIT(0));
1419 		else
1420 			r8168_mac_ocp_modify(tp, 0xc0b6, BIT(0), 0);
1421 	}
1422 
1423 	raw_spin_lock_irqsave(&tp->config25_lock, flags);
1424 	for (i = 0; i < tmp; i++) {
1425 		options = RTL_R8(tp, cfg[i].reg) & ~cfg[i].mask;
1426 		if (wolopts & cfg[i].opt)
1427 			options |= cfg[i].mask;
1428 		RTL_W8(tp, cfg[i].reg, options);
1429 	}
1430 	raw_spin_unlock_irqrestore(&tp->config25_lock, flags);
1431 
1432 	switch (tp->mac_version) {
1433 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
1434 		options = RTL_R8(tp, Config1) & ~PMEnable;
1435 		if (wolopts)
1436 			options |= PMEnable;
1437 		RTL_W8(tp, Config1, options);
1438 		break;
1439 	case RTL_GIGA_MAC_VER_34:
1440 	case RTL_GIGA_MAC_VER_37:
1441 	case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_63:
1442 		if (wolopts)
1443 			rtl_mod_config2(tp, 0, PME_SIGNAL);
1444 		else
1445 			rtl_mod_config2(tp, PME_SIGNAL, 0);
1446 		break;
1447 	default:
1448 		break;
1449 	}
1450 
1451 	rtl_lock_config_regs(tp);
1452 
1453 	device_set_wakeup_enable(tp_to_dev(tp), wolopts);
1454 
1455 	if (tp->dash_type == RTL_DASH_NONE) {
1456 		rtl_set_d3_pll_down(tp, !wolopts);
1457 		tp->dev->wol_enabled = wolopts ? 1 : 0;
1458 	}
1459 }
1460 
1461 static int rtl8169_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1462 {
1463 	struct rtl8169_private *tp = netdev_priv(dev);
1464 
1465 	if (wol->wolopts & ~WAKE_ANY)
1466 		return -EINVAL;
1467 
1468 	tp->saved_wolopts = wol->wolopts;
1469 	__rtl8169_set_wol(tp, tp->saved_wolopts);
1470 
1471 	return 0;
1472 }
1473 
1474 static void rtl8169_get_drvinfo(struct net_device *dev,
1475 				struct ethtool_drvinfo *info)
1476 {
1477 	struct rtl8169_private *tp = netdev_priv(dev);
1478 	struct rtl_fw *rtl_fw = tp->rtl_fw;
1479 
1480 	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1481 	strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
1482 	BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version));
1483 	if (rtl_fw)
1484 		strscpy(info->fw_version, rtl_fw->version,
1485 			sizeof(info->fw_version));
1486 }
1487 
1488 static int rtl8169_get_regs_len(struct net_device *dev)
1489 {
1490 	return R8169_REGS_SIZE;
1491 }
1492 
1493 static netdev_features_t rtl8169_fix_features(struct net_device *dev,
1494 	netdev_features_t features)
1495 {
1496 	struct rtl8169_private *tp = netdev_priv(dev);
1497 
1498 	if (dev->mtu > TD_MSS_MAX)
1499 		features &= ~NETIF_F_ALL_TSO;
1500 
1501 	if (dev->mtu > ETH_DATA_LEN &&
1502 	    tp->mac_version > RTL_GIGA_MAC_VER_06)
1503 		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_ALL_TSO);
1504 
1505 	return features;
1506 }
1507 
1508 static void rtl_set_rx_config_features(struct rtl8169_private *tp,
1509 				       netdev_features_t features)
1510 {
1511 	u32 rx_config = RTL_R32(tp, RxConfig);
1512 
1513 	if (features & NETIF_F_RXALL)
1514 		rx_config |= RX_CONFIG_ACCEPT_ERR_MASK;
1515 	else
1516 		rx_config &= ~RX_CONFIG_ACCEPT_ERR_MASK;
1517 
1518 	if (rtl_is_8125(tp)) {
1519 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
1520 			rx_config |= RX_VLAN_8125;
1521 		else
1522 			rx_config &= ~RX_VLAN_8125;
1523 	}
1524 
1525 	RTL_W32(tp, RxConfig, rx_config);
1526 }
1527 
1528 static int rtl8169_set_features(struct net_device *dev,
1529 				netdev_features_t features)
1530 {
1531 	struct rtl8169_private *tp = netdev_priv(dev);
1532 
1533 	rtl_set_rx_config_features(tp, features);
1534 
1535 	if (features & NETIF_F_RXCSUM)
1536 		tp->cp_cmd |= RxChkSum;
1537 	else
1538 		tp->cp_cmd &= ~RxChkSum;
1539 
1540 	if (!rtl_is_8125(tp)) {
1541 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
1542 			tp->cp_cmd |= RxVlan;
1543 		else
1544 			tp->cp_cmd &= ~RxVlan;
1545 	}
1546 
1547 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
1548 	rtl_pci_commit(tp);
1549 
1550 	return 0;
1551 }
1552 
1553 static inline u32 rtl8169_tx_vlan_tag(struct sk_buff *skb)
1554 {
1555 	return (skb_vlan_tag_present(skb)) ?
1556 		TxVlanTag | swab16(skb_vlan_tag_get(skb)) : 0x00;
1557 }
1558 
1559 static void rtl8169_rx_vlan_tag(struct RxDesc *desc, struct sk_buff *skb)
1560 {
1561 	u32 opts2 = le32_to_cpu(desc->opts2);
1562 
1563 	if (opts2 & RxVlanTag)
1564 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), swab16(opts2 & 0xffff));
1565 }
1566 
1567 static void rtl8169_get_regs(struct net_device *dev, struct ethtool_regs *regs,
1568 			     void *p)
1569 {
1570 	struct rtl8169_private *tp = netdev_priv(dev);
1571 	u32 __iomem *data = tp->mmio_addr;
1572 	u32 *dw = p;
1573 	int i;
1574 
1575 	for (i = 0; i < R8169_REGS_SIZE; i += 4)
1576 		memcpy_fromio(dw++, data++, 4);
1577 }
1578 
1579 static const char rtl8169_gstrings[][ETH_GSTRING_LEN] = {
1580 	"tx_packets",
1581 	"rx_packets",
1582 	"tx_errors",
1583 	"rx_errors",
1584 	"rx_missed",
1585 	"align_errors",
1586 	"tx_single_collisions",
1587 	"tx_multi_collisions",
1588 	"unicast",
1589 	"broadcast",
1590 	"multicast",
1591 	"tx_aborted",
1592 	"tx_underrun",
1593 };
1594 
1595 static int rtl8169_get_sset_count(struct net_device *dev, int sset)
1596 {
1597 	switch (sset) {
1598 	case ETH_SS_STATS:
1599 		return ARRAY_SIZE(rtl8169_gstrings);
1600 	default:
1601 		return -EOPNOTSUPP;
1602 	}
1603 }
1604 
1605 DECLARE_RTL_COND(rtl_counters_cond)
1606 {
1607 	return RTL_R32(tp, CounterAddrLow) & (CounterReset | CounterDump);
1608 }
1609 
1610 static void rtl8169_do_counters(struct rtl8169_private *tp, u32 counter_cmd)
1611 {
1612 	u32 cmd = lower_32_bits(tp->counters_phys_addr);
1613 
1614 	RTL_W32(tp, CounterAddrHigh, upper_32_bits(tp->counters_phys_addr));
1615 	rtl_pci_commit(tp);
1616 	RTL_W32(tp, CounterAddrLow, cmd);
1617 	RTL_W32(tp, CounterAddrLow, cmd | counter_cmd);
1618 
1619 	rtl_loop_wait_low(tp, &rtl_counters_cond, 10, 1000);
1620 }
1621 
1622 static void rtl8169_update_counters(struct rtl8169_private *tp)
1623 {
1624 	u8 val = RTL_R8(tp, ChipCmd);
1625 
1626 	/*
1627 	 * Some chips are unable to dump tally counters when the receiver
1628 	 * is disabled. If 0xff chip may be in a PCI power-save state.
1629 	 */
1630 	if (val & CmdRxEnb && val != 0xff)
1631 		rtl8169_do_counters(tp, CounterDump);
1632 }
1633 
1634 static void rtl8169_init_counter_offsets(struct rtl8169_private *tp)
1635 {
1636 	struct rtl8169_counters *counters = tp->counters;
1637 
1638 	/*
1639 	 * rtl8169_init_counter_offsets is called from rtl_open.  On chip
1640 	 * versions prior to RTL_GIGA_MAC_VER_19 the tally counters are only
1641 	 * reset by a power cycle, while the counter values collected by the
1642 	 * driver are reset at every driver unload/load cycle.
1643 	 *
1644 	 * To make sure the HW values returned by @get_stats64 match the SW
1645 	 * values, we collect the initial values at first open(*) and use them
1646 	 * as offsets to normalize the values returned by @get_stats64.
1647 	 *
1648 	 * (*) We can't call rtl8169_init_counter_offsets from rtl_init_one
1649 	 * for the reason stated in rtl8169_update_counters; CmdRxEnb is only
1650 	 * set at open time by rtl_hw_start.
1651 	 */
1652 
1653 	if (tp->tc_offset.inited)
1654 		return;
1655 
1656 	if (tp->mac_version >= RTL_GIGA_MAC_VER_19) {
1657 		rtl8169_do_counters(tp, CounterReset);
1658 	} else {
1659 		rtl8169_update_counters(tp);
1660 		tp->tc_offset.tx_errors = counters->tx_errors;
1661 		tp->tc_offset.tx_multi_collision = counters->tx_multi_collision;
1662 		tp->tc_offset.tx_aborted = counters->tx_aborted;
1663 		tp->tc_offset.rx_missed = counters->rx_missed;
1664 	}
1665 
1666 	tp->tc_offset.inited = true;
1667 }
1668 
1669 static void rtl8169_get_ethtool_stats(struct net_device *dev,
1670 				      struct ethtool_stats *stats, u64 *data)
1671 {
1672 	struct rtl8169_private *tp = netdev_priv(dev);
1673 	struct rtl8169_counters *counters;
1674 
1675 	counters = tp->counters;
1676 	rtl8169_update_counters(tp);
1677 
1678 	data[0] = le64_to_cpu(counters->tx_packets);
1679 	data[1] = le64_to_cpu(counters->rx_packets);
1680 	data[2] = le64_to_cpu(counters->tx_errors);
1681 	data[3] = le32_to_cpu(counters->rx_errors);
1682 	data[4] = le16_to_cpu(counters->rx_missed);
1683 	data[5] = le16_to_cpu(counters->align_errors);
1684 	data[6] = le32_to_cpu(counters->tx_one_collision);
1685 	data[7] = le32_to_cpu(counters->tx_multi_collision);
1686 	data[8] = le64_to_cpu(counters->rx_unicast);
1687 	data[9] = le64_to_cpu(counters->rx_broadcast);
1688 	data[10] = le32_to_cpu(counters->rx_multicast);
1689 	data[11] = le16_to_cpu(counters->tx_aborted);
1690 	data[12] = le16_to_cpu(counters->tx_underun);
1691 }
1692 
1693 static void rtl8169_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1694 {
1695 	switch(stringset) {
1696 	case ETH_SS_STATS:
1697 		memcpy(data, rtl8169_gstrings, sizeof(rtl8169_gstrings));
1698 		break;
1699 	}
1700 }
1701 
1702 /*
1703  * Interrupt coalescing
1704  *
1705  * > 1 - the availability of the IntrMitigate (0xe2) register through the
1706  * >     8169, 8168 and 810x line of chipsets
1707  *
1708  * 8169, 8168, and 8136(810x) serial chipsets support it.
1709  *
1710  * > 2 - the Tx timer unit at gigabit speed
1711  *
1712  * The unit of the timer depends on both the speed and the setting of CPlusCmd
1713  * (0xe0) bit 1 and bit 0.
1714  *
1715  * For 8169
1716  * bit[1:0] \ speed        1000M           100M            10M
1717  * 0 0                     320ns           2.56us          40.96us
1718  * 0 1                     2.56us          20.48us         327.7us
1719  * 1 0                     5.12us          40.96us         655.4us
1720  * 1 1                     10.24us         81.92us         1.31ms
1721  *
1722  * For the other
1723  * bit[1:0] \ speed        1000M           100M            10M
1724  * 0 0                     5us             2.56us          40.96us
1725  * 0 1                     40us            20.48us         327.7us
1726  * 1 0                     80us            40.96us         655.4us
1727  * 1 1                     160us           81.92us         1.31ms
1728  */
1729 
1730 /* rx/tx scale factors for all CPlusCmd[0:1] cases */
1731 struct rtl_coalesce_info {
1732 	u32 speed;
1733 	u32 scale_nsecs[4];
1734 };
1735 
1736 /* produce array with base delay *1, *8, *8*2, *8*2*2 */
1737 #define COALESCE_DELAY(d) { (d), 8 * (d), 16 * (d), 32 * (d) }
1738 
1739 static const struct rtl_coalesce_info rtl_coalesce_info_8169[] = {
1740 	{ SPEED_1000,	COALESCE_DELAY(320) },
1741 	{ SPEED_100,	COALESCE_DELAY(2560) },
1742 	{ SPEED_10,	COALESCE_DELAY(40960) },
1743 	{ 0 },
1744 };
1745 
1746 static const struct rtl_coalesce_info rtl_coalesce_info_8168_8136[] = {
1747 	{ SPEED_1000,	COALESCE_DELAY(5000) },
1748 	{ SPEED_100,	COALESCE_DELAY(2560) },
1749 	{ SPEED_10,	COALESCE_DELAY(40960) },
1750 	{ 0 },
1751 };
1752 #undef COALESCE_DELAY
1753 
1754 /* get rx/tx scale vector corresponding to current speed */
1755 static const struct rtl_coalesce_info *
1756 rtl_coalesce_info(struct rtl8169_private *tp)
1757 {
1758 	const struct rtl_coalesce_info *ci;
1759 
1760 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
1761 		ci = rtl_coalesce_info_8169;
1762 	else
1763 		ci = rtl_coalesce_info_8168_8136;
1764 
1765 	/* if speed is unknown assume highest one */
1766 	if (tp->phydev->speed == SPEED_UNKNOWN)
1767 		return ci;
1768 
1769 	for (; ci->speed; ci++) {
1770 		if (tp->phydev->speed == ci->speed)
1771 			return ci;
1772 	}
1773 
1774 	return ERR_PTR(-ELNRNG);
1775 }
1776 
1777 static int rtl_get_coalesce(struct net_device *dev,
1778 			    struct ethtool_coalesce *ec,
1779 			    struct kernel_ethtool_coalesce *kernel_coal,
1780 			    struct netlink_ext_ack *extack)
1781 {
1782 	struct rtl8169_private *tp = netdev_priv(dev);
1783 	const struct rtl_coalesce_info *ci;
1784 	u32 scale, c_us, c_fr;
1785 	u16 intrmit;
1786 
1787 	if (rtl_is_8125(tp))
1788 		return -EOPNOTSUPP;
1789 
1790 	memset(ec, 0, sizeof(*ec));
1791 
1792 	/* get rx/tx scale corresponding to current speed and CPlusCmd[0:1] */
1793 	ci = rtl_coalesce_info(tp);
1794 	if (IS_ERR(ci))
1795 		return PTR_ERR(ci);
1796 
1797 	scale = ci->scale_nsecs[tp->cp_cmd & INTT_MASK];
1798 
1799 	intrmit = RTL_R16(tp, IntrMitigate);
1800 
1801 	c_us = FIELD_GET(RTL_COALESCE_TX_USECS, intrmit);
1802 	ec->tx_coalesce_usecs = DIV_ROUND_UP(c_us * scale, 1000);
1803 
1804 	c_fr = FIELD_GET(RTL_COALESCE_TX_FRAMES, intrmit);
1805 	/* ethtool_coalesce states usecs and max_frames must not both be 0 */
1806 	ec->tx_max_coalesced_frames = (c_us || c_fr) ? c_fr * 4 : 1;
1807 
1808 	c_us = FIELD_GET(RTL_COALESCE_RX_USECS, intrmit);
1809 	ec->rx_coalesce_usecs = DIV_ROUND_UP(c_us * scale, 1000);
1810 
1811 	c_fr = FIELD_GET(RTL_COALESCE_RX_FRAMES, intrmit);
1812 	ec->rx_max_coalesced_frames = (c_us || c_fr) ? c_fr * 4 : 1;
1813 
1814 	return 0;
1815 }
1816 
1817 /* choose appropriate scale factor and CPlusCmd[0:1] for (speed, usec) */
1818 static int rtl_coalesce_choose_scale(struct rtl8169_private *tp, u32 usec,
1819 				     u16 *cp01)
1820 {
1821 	const struct rtl_coalesce_info *ci;
1822 	u16 i;
1823 
1824 	ci = rtl_coalesce_info(tp);
1825 	if (IS_ERR(ci))
1826 		return PTR_ERR(ci);
1827 
1828 	for (i = 0; i < 4; i++) {
1829 		if (usec <= ci->scale_nsecs[i] * RTL_COALESCE_T_MAX / 1000U) {
1830 			*cp01 = i;
1831 			return ci->scale_nsecs[i];
1832 		}
1833 	}
1834 
1835 	return -ERANGE;
1836 }
1837 
1838 static int rtl_set_coalesce(struct net_device *dev,
1839 			    struct ethtool_coalesce *ec,
1840 			    struct kernel_ethtool_coalesce *kernel_coal,
1841 			    struct netlink_ext_ack *extack)
1842 {
1843 	struct rtl8169_private *tp = netdev_priv(dev);
1844 	u32 tx_fr = ec->tx_max_coalesced_frames;
1845 	u32 rx_fr = ec->rx_max_coalesced_frames;
1846 	u32 coal_usec_max, units;
1847 	u16 w = 0, cp01 = 0;
1848 	int scale;
1849 
1850 	if (rtl_is_8125(tp))
1851 		return -EOPNOTSUPP;
1852 
1853 	if (rx_fr > RTL_COALESCE_FRAME_MAX || tx_fr > RTL_COALESCE_FRAME_MAX)
1854 		return -ERANGE;
1855 
1856 	coal_usec_max = max(ec->rx_coalesce_usecs, ec->tx_coalesce_usecs);
1857 	scale = rtl_coalesce_choose_scale(tp, coal_usec_max, &cp01);
1858 	if (scale < 0)
1859 		return scale;
1860 
1861 	/* Accept max_frames=1 we returned in rtl_get_coalesce. Accept it
1862 	 * not only when usecs=0 because of e.g. the following scenario:
1863 	 *
1864 	 * - both rx_usecs=0 & rx_frames=0 in hardware (no delay on RX)
1865 	 * - rtl_get_coalesce returns rx_usecs=0, rx_frames=1
1866 	 * - then user does `ethtool -C eth0 rx-usecs 100`
1867 	 *
1868 	 * Since ethtool sends to kernel whole ethtool_coalesce settings,
1869 	 * if we want to ignore rx_frames then it has to be set to 0.
1870 	 */
1871 	if (rx_fr == 1)
1872 		rx_fr = 0;
1873 	if (tx_fr == 1)
1874 		tx_fr = 0;
1875 
1876 	/* HW requires time limit to be set if frame limit is set */
1877 	if ((tx_fr && !ec->tx_coalesce_usecs) ||
1878 	    (rx_fr && !ec->rx_coalesce_usecs))
1879 		return -EINVAL;
1880 
1881 	w |= FIELD_PREP(RTL_COALESCE_TX_FRAMES, DIV_ROUND_UP(tx_fr, 4));
1882 	w |= FIELD_PREP(RTL_COALESCE_RX_FRAMES, DIV_ROUND_UP(rx_fr, 4));
1883 
1884 	units = DIV_ROUND_UP(ec->tx_coalesce_usecs * 1000U, scale);
1885 	w |= FIELD_PREP(RTL_COALESCE_TX_USECS, units);
1886 	units = DIV_ROUND_UP(ec->rx_coalesce_usecs * 1000U, scale);
1887 	w |= FIELD_PREP(RTL_COALESCE_RX_USECS, units);
1888 
1889 	RTL_W16(tp, IntrMitigate, w);
1890 
1891 	/* Meaning of PktCntrDisable bit changed from RTL8168e-vl */
1892 	if (rtl_is_8168evl_up(tp)) {
1893 		if (!rx_fr && !tx_fr)
1894 			/* disable packet counter */
1895 			tp->cp_cmd |= PktCntrDisable;
1896 		else
1897 			tp->cp_cmd &= ~PktCntrDisable;
1898 	}
1899 
1900 	tp->cp_cmd = (tp->cp_cmd & ~INTT_MASK) | cp01;
1901 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
1902 	rtl_pci_commit(tp);
1903 
1904 	return 0;
1905 }
1906 
1907 static int rtl8169_get_eee(struct net_device *dev, struct ethtool_eee *data)
1908 {
1909 	struct rtl8169_private *tp = netdev_priv(dev);
1910 
1911 	if (!rtl_supports_eee(tp))
1912 		return -EOPNOTSUPP;
1913 
1914 	return phy_ethtool_get_eee(tp->phydev, data);
1915 }
1916 
1917 static int rtl8169_set_eee(struct net_device *dev, struct ethtool_eee *data)
1918 {
1919 	struct rtl8169_private *tp = netdev_priv(dev);
1920 	int ret;
1921 
1922 	if (!rtl_supports_eee(tp))
1923 		return -EOPNOTSUPP;
1924 
1925 	ret = phy_ethtool_set_eee(tp->phydev, data);
1926 
1927 	if (!ret)
1928 		tp->eee_adv = phy_read_mmd(dev->phydev, MDIO_MMD_AN,
1929 					   MDIO_AN_EEE_ADV);
1930 	return ret;
1931 }
1932 
1933 static void rtl8169_get_ringparam(struct net_device *dev,
1934 				  struct ethtool_ringparam *data,
1935 				  struct kernel_ethtool_ringparam *kernel_data,
1936 				  struct netlink_ext_ack *extack)
1937 {
1938 	data->rx_max_pending = NUM_RX_DESC;
1939 	data->rx_pending = NUM_RX_DESC;
1940 	data->tx_max_pending = NUM_TX_DESC;
1941 	data->tx_pending = NUM_TX_DESC;
1942 }
1943 
1944 static void rtl8169_get_pauseparam(struct net_device *dev,
1945 				   struct ethtool_pauseparam *data)
1946 {
1947 	struct rtl8169_private *tp = netdev_priv(dev);
1948 	bool tx_pause, rx_pause;
1949 
1950 	phy_get_pause(tp->phydev, &tx_pause, &rx_pause);
1951 
1952 	data->autoneg = tp->phydev->autoneg;
1953 	data->tx_pause = tx_pause ? 1 : 0;
1954 	data->rx_pause = rx_pause ? 1 : 0;
1955 }
1956 
1957 static int rtl8169_set_pauseparam(struct net_device *dev,
1958 				  struct ethtool_pauseparam *data)
1959 {
1960 	struct rtl8169_private *tp = netdev_priv(dev);
1961 
1962 	if (dev->mtu > ETH_DATA_LEN)
1963 		return -EOPNOTSUPP;
1964 
1965 	phy_set_asym_pause(tp->phydev, data->rx_pause, data->tx_pause);
1966 
1967 	return 0;
1968 }
1969 
1970 static const struct ethtool_ops rtl8169_ethtool_ops = {
1971 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
1972 				     ETHTOOL_COALESCE_MAX_FRAMES,
1973 	.get_drvinfo		= rtl8169_get_drvinfo,
1974 	.get_regs_len		= rtl8169_get_regs_len,
1975 	.get_link		= ethtool_op_get_link,
1976 	.get_coalesce		= rtl_get_coalesce,
1977 	.set_coalesce		= rtl_set_coalesce,
1978 	.get_regs		= rtl8169_get_regs,
1979 	.get_wol		= rtl8169_get_wol,
1980 	.set_wol		= rtl8169_set_wol,
1981 	.get_strings		= rtl8169_get_strings,
1982 	.get_sset_count		= rtl8169_get_sset_count,
1983 	.get_ethtool_stats	= rtl8169_get_ethtool_stats,
1984 	.get_ts_info		= ethtool_op_get_ts_info,
1985 	.nway_reset		= phy_ethtool_nway_reset,
1986 	.get_eee		= rtl8169_get_eee,
1987 	.set_eee		= rtl8169_set_eee,
1988 	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
1989 	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
1990 	.get_ringparam		= rtl8169_get_ringparam,
1991 	.get_pauseparam		= rtl8169_get_pauseparam,
1992 	.set_pauseparam		= rtl8169_set_pauseparam,
1993 };
1994 
1995 static void rtl_enable_eee(struct rtl8169_private *tp)
1996 {
1997 	struct phy_device *phydev = tp->phydev;
1998 	int adv;
1999 
2000 	/* respect EEE advertisement the user may have set */
2001 	if (tp->eee_adv >= 0)
2002 		adv = tp->eee_adv;
2003 	else
2004 		adv = phy_read_mmd(phydev, MDIO_MMD_PCS, MDIO_PCS_EEE_ABLE);
2005 
2006 	if (adv >= 0)
2007 		phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_EEE_ADV, adv);
2008 }
2009 
2010 static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii)
2011 {
2012 	/*
2013 	 * The driver currently handles the 8168Bf and the 8168Be identically
2014 	 * but they can be identified more specifically through the test below
2015 	 * if needed:
2016 	 *
2017 	 * (RTL_R32(tp, TxConfig) & 0x700000) == 0x500000 ? 8168Bf : 8168Be
2018 	 *
2019 	 * Same thing for the 8101Eb and the 8101Ec:
2020 	 *
2021 	 * (RTL_R32(tp, TxConfig) & 0x700000) == 0x200000 ? 8101Eb : 8101Ec
2022 	 */
2023 	static const struct rtl_mac_info {
2024 		u16 mask;
2025 		u16 val;
2026 		enum mac_version ver;
2027 	} mac_info[] = {
2028 		/* 8125B family. */
2029 		{ 0x7cf, 0x641,	RTL_GIGA_MAC_VER_63 },
2030 
2031 		/* 8125A family. */
2032 		{ 0x7cf, 0x609,	RTL_GIGA_MAC_VER_61 },
2033 		/* It seems only XID 609 made it to the mass market.
2034 		 * { 0x7cf, 0x608,	RTL_GIGA_MAC_VER_60 },
2035 		 * { 0x7c8, 0x608,	RTL_GIGA_MAC_VER_61 },
2036 		 */
2037 
2038 		/* RTL8117 */
2039 		{ 0x7cf, 0x54b,	RTL_GIGA_MAC_VER_53 },
2040 		{ 0x7cf, 0x54a,	RTL_GIGA_MAC_VER_52 },
2041 
2042 		/* 8168EP family. */
2043 		{ 0x7cf, 0x502,	RTL_GIGA_MAC_VER_51 },
2044 		/* It seems this chip version never made it to
2045 		 * the wild. Let's disable detection.
2046 		 * { 0x7cf, 0x501,      RTL_GIGA_MAC_VER_50 },
2047 		 * { 0x7cf, 0x500,      RTL_GIGA_MAC_VER_49 },
2048 		 */
2049 
2050 		/* 8168H family. */
2051 		{ 0x7cf, 0x541,	RTL_GIGA_MAC_VER_46 },
2052 		/* It seems this chip version never made it to
2053 		 * the wild. Let's disable detection.
2054 		 * { 0x7cf, 0x540,	RTL_GIGA_MAC_VER_45 },
2055 		 */
2056 
2057 		/* 8168G family. */
2058 		{ 0x7cf, 0x5c8,	RTL_GIGA_MAC_VER_44 },
2059 		{ 0x7cf, 0x509,	RTL_GIGA_MAC_VER_42 },
2060 		/* It seems this chip version never made it to
2061 		 * the wild. Let's disable detection.
2062 		 * { 0x7cf, 0x4c1,	RTL_GIGA_MAC_VER_41 },
2063 		 */
2064 		{ 0x7cf, 0x4c0,	RTL_GIGA_MAC_VER_40 },
2065 
2066 		/* 8168F family. */
2067 		{ 0x7c8, 0x488,	RTL_GIGA_MAC_VER_38 },
2068 		{ 0x7cf, 0x481,	RTL_GIGA_MAC_VER_36 },
2069 		{ 0x7cf, 0x480,	RTL_GIGA_MAC_VER_35 },
2070 
2071 		/* 8168E family. */
2072 		{ 0x7c8, 0x2c8,	RTL_GIGA_MAC_VER_34 },
2073 		{ 0x7cf, 0x2c1,	RTL_GIGA_MAC_VER_32 },
2074 		{ 0x7c8, 0x2c0,	RTL_GIGA_MAC_VER_33 },
2075 
2076 		/* 8168D family. */
2077 		{ 0x7cf, 0x281,	RTL_GIGA_MAC_VER_25 },
2078 		{ 0x7c8, 0x280,	RTL_GIGA_MAC_VER_26 },
2079 
2080 		/* 8168DP family. */
2081 		/* It seems this early RTL8168dp version never made it to
2082 		 * the wild. Support has been removed.
2083 		 * { 0x7cf, 0x288,      RTL_GIGA_MAC_VER_27 },
2084 		 */
2085 		{ 0x7cf, 0x28a,	RTL_GIGA_MAC_VER_28 },
2086 		{ 0x7cf, 0x28b,	RTL_GIGA_MAC_VER_31 },
2087 
2088 		/* 8168C family. */
2089 		{ 0x7cf, 0x3c9,	RTL_GIGA_MAC_VER_23 },
2090 		{ 0x7cf, 0x3c8,	RTL_GIGA_MAC_VER_18 },
2091 		{ 0x7c8, 0x3c8,	RTL_GIGA_MAC_VER_24 },
2092 		{ 0x7cf, 0x3c0,	RTL_GIGA_MAC_VER_19 },
2093 		{ 0x7cf, 0x3c2,	RTL_GIGA_MAC_VER_20 },
2094 		{ 0x7cf, 0x3c3,	RTL_GIGA_MAC_VER_21 },
2095 		{ 0x7c8, 0x3c0,	RTL_GIGA_MAC_VER_22 },
2096 
2097 		/* 8168B family. */
2098 		{ 0x7c8, 0x380,	RTL_GIGA_MAC_VER_17 },
2099 		{ 0x7c8, 0x300,	RTL_GIGA_MAC_VER_11 },
2100 
2101 		/* 8101 family. */
2102 		{ 0x7c8, 0x448,	RTL_GIGA_MAC_VER_39 },
2103 		{ 0x7c8, 0x440,	RTL_GIGA_MAC_VER_37 },
2104 		{ 0x7cf, 0x409,	RTL_GIGA_MAC_VER_29 },
2105 		{ 0x7c8, 0x408,	RTL_GIGA_MAC_VER_30 },
2106 		{ 0x7cf, 0x349,	RTL_GIGA_MAC_VER_08 },
2107 		{ 0x7cf, 0x249,	RTL_GIGA_MAC_VER_08 },
2108 		{ 0x7cf, 0x348,	RTL_GIGA_MAC_VER_07 },
2109 		{ 0x7cf, 0x248,	RTL_GIGA_MAC_VER_07 },
2110 		{ 0x7cf, 0x240,	RTL_GIGA_MAC_VER_14 },
2111 		{ 0x7c8, 0x348,	RTL_GIGA_MAC_VER_09 },
2112 		{ 0x7c8, 0x248,	RTL_GIGA_MAC_VER_09 },
2113 		{ 0x7c8, 0x340,	RTL_GIGA_MAC_VER_10 },
2114 
2115 		/* 8110 family. */
2116 		{ 0xfc8, 0x980,	RTL_GIGA_MAC_VER_06 },
2117 		{ 0xfc8, 0x180,	RTL_GIGA_MAC_VER_05 },
2118 		{ 0xfc8, 0x100,	RTL_GIGA_MAC_VER_04 },
2119 		{ 0xfc8, 0x040,	RTL_GIGA_MAC_VER_03 },
2120 		{ 0xfc8, 0x008,	RTL_GIGA_MAC_VER_02 },
2121 
2122 		/* Catch-all */
2123 		{ 0x000, 0x000,	RTL_GIGA_MAC_NONE   }
2124 	};
2125 	const struct rtl_mac_info *p = mac_info;
2126 	enum mac_version ver;
2127 
2128 	while ((xid & p->mask) != p->val)
2129 		p++;
2130 	ver = p->ver;
2131 
2132 	if (ver != RTL_GIGA_MAC_NONE && !gmii) {
2133 		if (ver == RTL_GIGA_MAC_VER_42)
2134 			ver = RTL_GIGA_MAC_VER_43;
2135 		else if (ver == RTL_GIGA_MAC_VER_46)
2136 			ver = RTL_GIGA_MAC_VER_48;
2137 	}
2138 
2139 	return ver;
2140 }
2141 
2142 static void rtl_release_firmware(struct rtl8169_private *tp)
2143 {
2144 	if (tp->rtl_fw) {
2145 		rtl_fw_release_firmware(tp->rtl_fw);
2146 		kfree(tp->rtl_fw);
2147 		tp->rtl_fw = NULL;
2148 	}
2149 }
2150 
2151 void r8169_apply_firmware(struct rtl8169_private *tp)
2152 {
2153 	int val;
2154 
2155 	/* TODO: release firmware if rtl_fw_write_firmware signals failure. */
2156 	if (tp->rtl_fw) {
2157 		rtl_fw_write_firmware(tp, tp->rtl_fw);
2158 		/* At least one firmware doesn't reset tp->ocp_base. */
2159 		tp->ocp_base = OCP_STD_PHY_BASE;
2160 
2161 		/* PHY soft reset may still be in progress */
2162 		phy_read_poll_timeout(tp->phydev, MII_BMCR, val,
2163 				      !(val & BMCR_RESET),
2164 				      50000, 600000, true);
2165 	}
2166 }
2167 
2168 static void rtl8168_config_eee_mac(struct rtl8169_private *tp)
2169 {
2170 	/* Adjust EEE LED frequency */
2171 	if (tp->mac_version != RTL_GIGA_MAC_VER_38)
2172 		RTL_W8(tp, EEE_LED, RTL_R8(tp, EEE_LED) & ~0x07);
2173 
2174 	rtl_eri_set_bits(tp, 0x1b0, 0x0003);
2175 }
2176 
2177 static void rtl8125a_config_eee_mac(struct rtl8169_private *tp)
2178 {
2179 	r8168_mac_ocp_modify(tp, 0xe040, 0, BIT(1) | BIT(0));
2180 	r8168_mac_ocp_modify(tp, 0xeb62, 0, BIT(2) | BIT(1));
2181 }
2182 
2183 static void rtl8125_set_eee_txidle_timer(struct rtl8169_private *tp)
2184 {
2185 	RTL_W16(tp, EEE_TXIDLE_TIMER_8125, tp->dev->mtu + ETH_HLEN + 0x20);
2186 }
2187 
2188 static void rtl8125b_config_eee_mac(struct rtl8169_private *tp)
2189 {
2190 	rtl8125_set_eee_txidle_timer(tp);
2191 	r8168_mac_ocp_modify(tp, 0xe040, 0, BIT(1) | BIT(0));
2192 }
2193 
2194 static void rtl_rar_exgmac_set(struct rtl8169_private *tp, const u8 *addr)
2195 {
2196 	rtl_eri_write(tp, 0xe0, ERIAR_MASK_1111, get_unaligned_le32(addr));
2197 	rtl_eri_write(tp, 0xe4, ERIAR_MASK_1111, get_unaligned_le16(addr + 4));
2198 	rtl_eri_write(tp, 0xf0, ERIAR_MASK_1111, get_unaligned_le16(addr) << 16);
2199 	rtl_eri_write(tp, 0xf4, ERIAR_MASK_1111, get_unaligned_le32(addr + 2));
2200 }
2201 
2202 u16 rtl8168h_2_get_adc_bias_ioffset(struct rtl8169_private *tp)
2203 {
2204 	u16 data1, data2, ioffset;
2205 
2206 	r8168_mac_ocp_write(tp, 0xdd02, 0x807d);
2207 	data1 = r8168_mac_ocp_read(tp, 0xdd02);
2208 	data2 = r8168_mac_ocp_read(tp, 0xdd00);
2209 
2210 	ioffset = (data2 >> 1) & 0x7ff8;
2211 	ioffset |= data2 & 0x0007;
2212 	if (data1 & BIT(7))
2213 		ioffset |= BIT(15);
2214 
2215 	return ioffset;
2216 }
2217 
2218 static void rtl_schedule_task(struct rtl8169_private *tp, enum rtl_flag flag)
2219 {
2220 	set_bit(flag, tp->wk.flags);
2221 	schedule_work(&tp->wk.work);
2222 }
2223 
2224 static void rtl8169_init_phy(struct rtl8169_private *tp)
2225 {
2226 	r8169_hw_phy_config(tp, tp->phydev, tp->mac_version);
2227 
2228 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06) {
2229 		pci_write_config_byte(tp->pci_dev, PCI_LATENCY_TIMER, 0x40);
2230 		pci_write_config_byte(tp->pci_dev, PCI_CACHE_LINE_SIZE, 0x08);
2231 		/* set undocumented MAC Reg C+CR Offset 0x82h */
2232 		RTL_W8(tp, 0x82, 0x01);
2233 	}
2234 
2235 	if (tp->mac_version == RTL_GIGA_MAC_VER_05 &&
2236 	    tp->pci_dev->subsystem_vendor == PCI_VENDOR_ID_GIGABYTE &&
2237 	    tp->pci_dev->subsystem_device == 0xe000)
2238 		phy_write_paged(tp->phydev, 0x0001, 0x10, 0xf01b);
2239 
2240 	/* We may have called phy_speed_down before */
2241 	phy_speed_up(tp->phydev);
2242 
2243 	if (rtl_supports_eee(tp))
2244 		rtl_enable_eee(tp);
2245 
2246 	genphy_soft_reset(tp->phydev);
2247 }
2248 
2249 static void rtl_rar_set(struct rtl8169_private *tp, const u8 *addr)
2250 {
2251 	rtl_unlock_config_regs(tp);
2252 
2253 	RTL_W32(tp, MAC4, get_unaligned_le16(addr + 4));
2254 	rtl_pci_commit(tp);
2255 
2256 	RTL_W32(tp, MAC0, get_unaligned_le32(addr));
2257 	rtl_pci_commit(tp);
2258 
2259 	if (tp->mac_version == RTL_GIGA_MAC_VER_34)
2260 		rtl_rar_exgmac_set(tp, addr);
2261 
2262 	rtl_lock_config_regs(tp);
2263 }
2264 
2265 static int rtl_set_mac_address(struct net_device *dev, void *p)
2266 {
2267 	struct rtl8169_private *tp = netdev_priv(dev);
2268 	int ret;
2269 
2270 	ret = eth_mac_addr(dev, p);
2271 	if (ret)
2272 		return ret;
2273 
2274 	rtl_rar_set(tp, dev->dev_addr);
2275 
2276 	return 0;
2277 }
2278 
2279 static void rtl_init_rxcfg(struct rtl8169_private *tp)
2280 {
2281 	switch (tp->mac_version) {
2282 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
2283 	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
2284 		RTL_W32(tp, RxConfig, RX_FIFO_THRESH | RX_DMA_BURST);
2285 		break;
2286 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
2287 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_36:
2288 	case RTL_GIGA_MAC_VER_38:
2289 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST);
2290 		break;
2291 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_53:
2292 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST | RX_EARLY_OFF);
2293 		break;
2294 	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63:
2295 		RTL_W32(tp, RxConfig, RX_FETCH_DFLT_8125 | RX_DMA_BURST);
2296 		break;
2297 	default:
2298 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_DMA_BURST);
2299 		break;
2300 	}
2301 }
2302 
2303 static void rtl8169_init_ring_indexes(struct rtl8169_private *tp)
2304 {
2305 	tp->dirty_tx = tp->cur_tx = tp->cur_rx = 0;
2306 }
2307 
2308 static void r8168c_hw_jumbo_enable(struct rtl8169_private *tp)
2309 {
2310 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2311 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | Jumbo_En1);
2312 }
2313 
2314 static void r8168c_hw_jumbo_disable(struct rtl8169_private *tp)
2315 {
2316 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2317 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~Jumbo_En1);
2318 }
2319 
2320 static void r8168dp_hw_jumbo_enable(struct rtl8169_private *tp)
2321 {
2322 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2323 }
2324 
2325 static void r8168dp_hw_jumbo_disable(struct rtl8169_private *tp)
2326 {
2327 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2328 }
2329 
2330 static void r8168e_hw_jumbo_enable(struct rtl8169_private *tp)
2331 {
2332 	RTL_W8(tp, MaxTxPacketSize, 0x24);
2333 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2334 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | 0x01);
2335 }
2336 
2337 static void r8168e_hw_jumbo_disable(struct rtl8169_private *tp)
2338 {
2339 	RTL_W8(tp, MaxTxPacketSize, 0x3f);
2340 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2341 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~0x01);
2342 }
2343 
2344 static void r8168b_1_hw_jumbo_enable(struct rtl8169_private *tp)
2345 {
2346 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | (1 << 0));
2347 }
2348 
2349 static void r8168b_1_hw_jumbo_disable(struct rtl8169_private *tp)
2350 {
2351 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~(1 << 0));
2352 }
2353 
2354 static void rtl_jumbo_config(struct rtl8169_private *tp)
2355 {
2356 	bool jumbo = tp->dev->mtu > ETH_DATA_LEN;
2357 	int readrq = 4096;
2358 
2359 	rtl_unlock_config_regs(tp);
2360 	switch (tp->mac_version) {
2361 	case RTL_GIGA_MAC_VER_17:
2362 		if (jumbo) {
2363 			readrq = 512;
2364 			r8168b_1_hw_jumbo_enable(tp);
2365 		} else {
2366 			r8168b_1_hw_jumbo_disable(tp);
2367 		}
2368 		break;
2369 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_26:
2370 		if (jumbo) {
2371 			readrq = 512;
2372 			r8168c_hw_jumbo_enable(tp);
2373 		} else {
2374 			r8168c_hw_jumbo_disable(tp);
2375 		}
2376 		break;
2377 	case RTL_GIGA_MAC_VER_28:
2378 		if (jumbo)
2379 			r8168dp_hw_jumbo_enable(tp);
2380 		else
2381 			r8168dp_hw_jumbo_disable(tp);
2382 		break;
2383 	case RTL_GIGA_MAC_VER_31 ... RTL_GIGA_MAC_VER_33:
2384 		if (jumbo)
2385 			r8168e_hw_jumbo_enable(tp);
2386 		else
2387 			r8168e_hw_jumbo_disable(tp);
2388 		break;
2389 	default:
2390 		break;
2391 	}
2392 	rtl_lock_config_regs(tp);
2393 
2394 	if (pci_is_pcie(tp->pci_dev) && tp->supports_gmii)
2395 		pcie_set_readrq(tp->pci_dev, readrq);
2396 
2397 	/* Chip doesn't support pause in jumbo mode */
2398 	if (jumbo) {
2399 		linkmode_clear_bit(ETHTOOL_LINK_MODE_Pause_BIT,
2400 				   tp->phydev->advertising);
2401 		linkmode_clear_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT,
2402 				   tp->phydev->advertising);
2403 		phy_start_aneg(tp->phydev);
2404 	}
2405 }
2406 
2407 DECLARE_RTL_COND(rtl_chipcmd_cond)
2408 {
2409 	return RTL_R8(tp, ChipCmd) & CmdReset;
2410 }
2411 
2412 static void rtl_hw_reset(struct rtl8169_private *tp)
2413 {
2414 	RTL_W8(tp, ChipCmd, CmdReset);
2415 
2416 	rtl_loop_wait_low(tp, &rtl_chipcmd_cond, 100, 100);
2417 }
2418 
2419 static void rtl_request_firmware(struct rtl8169_private *tp)
2420 {
2421 	struct rtl_fw *rtl_fw;
2422 
2423 	/* firmware loaded already or no firmware available */
2424 	if (tp->rtl_fw || !tp->fw_name)
2425 		return;
2426 
2427 	rtl_fw = kzalloc(sizeof(*rtl_fw), GFP_KERNEL);
2428 	if (!rtl_fw)
2429 		return;
2430 
2431 	rtl_fw->phy_write = rtl_writephy;
2432 	rtl_fw->phy_read = rtl_readphy;
2433 	rtl_fw->mac_mcu_write = mac_mcu_write;
2434 	rtl_fw->mac_mcu_read = mac_mcu_read;
2435 	rtl_fw->fw_name = tp->fw_name;
2436 	rtl_fw->dev = tp_to_dev(tp);
2437 
2438 	if (rtl_fw_request_firmware(rtl_fw))
2439 		kfree(rtl_fw);
2440 	else
2441 		tp->rtl_fw = rtl_fw;
2442 }
2443 
2444 static void rtl_rx_close(struct rtl8169_private *tp)
2445 {
2446 	RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & ~RX_CONFIG_ACCEPT_MASK);
2447 }
2448 
2449 DECLARE_RTL_COND(rtl_npq_cond)
2450 {
2451 	return RTL_R8(tp, TxPoll) & NPQ;
2452 }
2453 
2454 DECLARE_RTL_COND(rtl_txcfg_empty_cond)
2455 {
2456 	return RTL_R32(tp, TxConfig) & TXCFG_EMPTY;
2457 }
2458 
2459 DECLARE_RTL_COND(rtl_rxtx_empty_cond)
2460 {
2461 	return (RTL_R8(tp, MCU) & RXTX_EMPTY) == RXTX_EMPTY;
2462 }
2463 
2464 DECLARE_RTL_COND(rtl_rxtx_empty_cond_2)
2465 {
2466 	/* IntrMitigate has new functionality on RTL8125 */
2467 	return (RTL_R16(tp, IntrMitigate) & 0x0103) == 0x0103;
2468 }
2469 
2470 static void rtl_wait_txrx_fifo_empty(struct rtl8169_private *tp)
2471 {
2472 	switch (tp->mac_version) {
2473 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_53:
2474 		rtl_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 42);
2475 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2476 		break;
2477 	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_61:
2478 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2479 		break;
2480 	case RTL_GIGA_MAC_VER_63:
2481 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
2482 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2483 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond_2, 100, 42);
2484 		break;
2485 	default:
2486 		break;
2487 	}
2488 }
2489 
2490 static void rtl_disable_rxdvgate(struct rtl8169_private *tp)
2491 {
2492 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
2493 }
2494 
2495 static void rtl_enable_rxdvgate(struct rtl8169_private *tp)
2496 {
2497 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | RXDV_GATED_EN);
2498 	fsleep(2000);
2499 	rtl_wait_txrx_fifo_empty(tp);
2500 }
2501 
2502 static void rtl_wol_enable_rx(struct rtl8169_private *tp)
2503 {
2504 	if (tp->mac_version >= RTL_GIGA_MAC_VER_25)
2505 		RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) |
2506 			AcceptBroadcast | AcceptMulticast | AcceptMyPhys);
2507 
2508 	if (tp->mac_version >= RTL_GIGA_MAC_VER_40)
2509 		rtl_disable_rxdvgate(tp);
2510 }
2511 
2512 static void rtl_prepare_power_down(struct rtl8169_private *tp)
2513 {
2514 	if (tp->dash_type != RTL_DASH_NONE)
2515 		return;
2516 
2517 	if (tp->mac_version == RTL_GIGA_MAC_VER_32 ||
2518 	    tp->mac_version == RTL_GIGA_MAC_VER_33)
2519 		rtl_ephy_write(tp, 0x19, 0xff64);
2520 
2521 	if (device_may_wakeup(tp_to_dev(tp))) {
2522 		phy_speed_down(tp->phydev, false);
2523 		rtl_wol_enable_rx(tp);
2524 	}
2525 }
2526 
2527 static void rtl_set_tx_config_registers(struct rtl8169_private *tp)
2528 {
2529 	u32 val = TX_DMA_BURST << TxDMAShift |
2530 		  InterFrameGap << TxInterFrameGapShift;
2531 
2532 	if (rtl_is_8168evl_up(tp))
2533 		val |= TXCFG_AUTO_FIFO;
2534 
2535 	RTL_W32(tp, TxConfig, val);
2536 }
2537 
2538 static void rtl_set_rx_max_size(struct rtl8169_private *tp)
2539 {
2540 	/* Low hurts. Let's disable the filtering. */
2541 	RTL_W16(tp, RxMaxSize, R8169_RX_BUF_SIZE + 1);
2542 }
2543 
2544 static void rtl_set_rx_tx_desc_registers(struct rtl8169_private *tp)
2545 {
2546 	/*
2547 	 * Magic spell: some iop3xx ARM board needs the TxDescAddrHigh
2548 	 * register to be written before TxDescAddrLow to work.
2549 	 * Switching from MMIO to I/O access fixes the issue as well.
2550 	 */
2551 	RTL_W32(tp, TxDescStartAddrHigh, ((u64) tp->TxPhyAddr) >> 32);
2552 	RTL_W32(tp, TxDescStartAddrLow, ((u64) tp->TxPhyAddr) & DMA_BIT_MASK(32));
2553 	RTL_W32(tp, RxDescAddrHigh, ((u64) tp->RxPhyAddr) >> 32);
2554 	RTL_W32(tp, RxDescAddrLow, ((u64) tp->RxPhyAddr) & DMA_BIT_MASK(32));
2555 }
2556 
2557 static void rtl8169_set_magic_reg(struct rtl8169_private *tp)
2558 {
2559 	u32 val;
2560 
2561 	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
2562 		val = 0x000fff00;
2563 	else if (tp->mac_version == RTL_GIGA_MAC_VER_06)
2564 		val = 0x00ffff00;
2565 	else
2566 		return;
2567 
2568 	if (RTL_R8(tp, Config2) & PCI_Clock_66MHz)
2569 		val |= 0xff;
2570 
2571 	RTL_W32(tp, 0x7c, val);
2572 }
2573 
2574 static void rtl_set_rx_mode(struct net_device *dev)
2575 {
2576 	u32 rx_mode = AcceptBroadcast | AcceptMyPhys | AcceptMulticast;
2577 	/* Multicast hash filter */
2578 	u32 mc_filter[2] = { 0xffffffff, 0xffffffff };
2579 	struct rtl8169_private *tp = netdev_priv(dev);
2580 	u32 tmp;
2581 
2582 	if (dev->flags & IFF_PROMISC) {
2583 		rx_mode |= AcceptAllPhys;
2584 	} else if (netdev_mc_count(dev) > MC_FILTER_LIMIT ||
2585 		   dev->flags & IFF_ALLMULTI ||
2586 		   tp->mac_version == RTL_GIGA_MAC_VER_35) {
2587 		/* accept all multicasts */
2588 	} else if (netdev_mc_empty(dev)) {
2589 		rx_mode &= ~AcceptMulticast;
2590 	} else {
2591 		struct netdev_hw_addr *ha;
2592 
2593 		mc_filter[1] = mc_filter[0] = 0;
2594 		netdev_for_each_mc_addr(ha, dev) {
2595 			u32 bit_nr = eth_hw_addr_crc(ha) >> 26;
2596 			mc_filter[bit_nr >> 5] |= BIT(bit_nr & 31);
2597 		}
2598 
2599 		if (tp->mac_version > RTL_GIGA_MAC_VER_06) {
2600 			tmp = mc_filter[0];
2601 			mc_filter[0] = swab32(mc_filter[1]);
2602 			mc_filter[1] = swab32(tmp);
2603 		}
2604 	}
2605 
2606 	RTL_W32(tp, MAR0 + 4, mc_filter[1]);
2607 	RTL_W32(tp, MAR0 + 0, mc_filter[0]);
2608 
2609 	tmp = RTL_R32(tp, RxConfig);
2610 	RTL_W32(tp, RxConfig, (tmp & ~RX_CONFIG_ACCEPT_OK_MASK) | rx_mode);
2611 }
2612 
2613 DECLARE_RTL_COND(rtl_csiar_cond)
2614 {
2615 	return RTL_R32(tp, CSIAR) & CSIAR_FLAG;
2616 }
2617 
2618 static void rtl_csi_write(struct rtl8169_private *tp, int addr, int value)
2619 {
2620 	u32 func = PCI_FUNC(tp->pci_dev->devfn);
2621 
2622 	RTL_W32(tp, CSIDR, value);
2623 	RTL_W32(tp, CSIAR, CSIAR_WRITE_CMD | (addr & CSIAR_ADDR_MASK) |
2624 		CSIAR_BYTE_ENABLE | func << 16);
2625 
2626 	rtl_loop_wait_low(tp, &rtl_csiar_cond, 10, 100);
2627 }
2628 
2629 static u32 rtl_csi_read(struct rtl8169_private *tp, int addr)
2630 {
2631 	u32 func = PCI_FUNC(tp->pci_dev->devfn);
2632 
2633 	RTL_W32(tp, CSIAR, (addr & CSIAR_ADDR_MASK) | func << 16 |
2634 		CSIAR_BYTE_ENABLE);
2635 
2636 	return rtl_loop_wait_high(tp, &rtl_csiar_cond, 10, 100) ?
2637 		RTL_R32(tp, CSIDR) : ~0;
2638 }
2639 
2640 static void rtl_set_aspm_entry_latency(struct rtl8169_private *tp, u8 val)
2641 {
2642 	struct pci_dev *pdev = tp->pci_dev;
2643 	u32 csi;
2644 
2645 	/* According to Realtek the value at config space address 0x070f
2646 	 * controls the L0s/L1 entrance latency. We try standard ECAM access
2647 	 * first and if it fails fall back to CSI.
2648 	 * bit 0..2: L0: 0 = 1us, 1 = 2us .. 6 = 7us, 7 = 7us (no typo)
2649 	 * bit 3..5: L1: 0 = 1us, 1 = 2us .. 6 = 64us, 7 = 64us
2650 	 */
2651 	if (pdev->cfg_size > 0x070f &&
2652 	    pci_write_config_byte(pdev, 0x070f, val) == PCIBIOS_SUCCESSFUL)
2653 		return;
2654 
2655 	netdev_notice_once(tp->dev,
2656 		"No native access to PCI extended config space, falling back to CSI\n");
2657 	csi = rtl_csi_read(tp, 0x070c) & 0x00ffffff;
2658 	rtl_csi_write(tp, 0x070c, csi | val << 24);
2659 }
2660 
2661 static void rtl_set_def_aspm_entry_latency(struct rtl8169_private *tp)
2662 {
2663 	/* L0 7us, L1 16us */
2664 	rtl_set_aspm_entry_latency(tp, 0x27);
2665 }
2666 
2667 struct ephy_info {
2668 	unsigned int offset;
2669 	u16 mask;
2670 	u16 bits;
2671 };
2672 
2673 static void __rtl_ephy_init(struct rtl8169_private *tp,
2674 			    const struct ephy_info *e, int len)
2675 {
2676 	u16 w;
2677 
2678 	while (len-- > 0) {
2679 		w = (rtl_ephy_read(tp, e->offset) & ~e->mask) | e->bits;
2680 		rtl_ephy_write(tp, e->offset, w);
2681 		e++;
2682 	}
2683 }
2684 
2685 #define rtl_ephy_init(tp, a) __rtl_ephy_init(tp, a, ARRAY_SIZE(a))
2686 
2687 static void rtl_disable_clock_request(struct rtl8169_private *tp)
2688 {
2689 	pcie_capability_clear_word(tp->pci_dev, PCI_EXP_LNKCTL,
2690 				   PCI_EXP_LNKCTL_CLKREQ_EN);
2691 }
2692 
2693 static void rtl_enable_clock_request(struct rtl8169_private *tp)
2694 {
2695 	pcie_capability_set_word(tp->pci_dev, PCI_EXP_LNKCTL,
2696 				 PCI_EXP_LNKCTL_CLKREQ_EN);
2697 }
2698 
2699 static void rtl_pcie_state_l2l3_disable(struct rtl8169_private *tp)
2700 {
2701 	/* work around an issue when PCI reset occurs during L2/L3 state */
2702 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Rdy_to_L23);
2703 }
2704 
2705 static void rtl_enable_exit_l1(struct rtl8169_private *tp)
2706 {
2707 	/* Bits control which events trigger ASPM L1 exit:
2708 	 * Bit 12: rxdv
2709 	 * Bit 11: ltr_msg
2710 	 * Bit 10: txdma_poll
2711 	 * Bit  9: xadm
2712 	 * Bit  8: pktavi
2713 	 * Bit  7: txpla
2714 	 */
2715 	switch (tp->mac_version) {
2716 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_36:
2717 		rtl_eri_set_bits(tp, 0xd4, 0x1f00);
2718 		break;
2719 	case RTL_GIGA_MAC_VER_37 ... RTL_GIGA_MAC_VER_38:
2720 		rtl_eri_set_bits(tp, 0xd4, 0x0c00);
2721 		break;
2722 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
2723 		r8168_mac_ocp_modify(tp, 0xc0ac, 0, 0x1f80);
2724 		break;
2725 	default:
2726 		break;
2727 	}
2728 }
2729 
2730 static void rtl_disable_exit_l1(struct rtl8169_private *tp)
2731 {
2732 	switch (tp->mac_version) {
2733 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
2734 		rtl_eri_clear_bits(tp, 0xd4, 0x1f00);
2735 		break;
2736 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
2737 		r8168_mac_ocp_modify(tp, 0xc0ac, 0x1f80, 0);
2738 		break;
2739 	default:
2740 		break;
2741 	}
2742 }
2743 
2744 static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
2745 {
2746 	if (tp->mac_version < RTL_GIGA_MAC_VER_32)
2747 		return;
2748 
2749 	if (enable) {
2750 		rtl_mod_config5(tp, 0, ASPM_en);
2751 		rtl_mod_config2(tp, 0, ClkReqEn);
2752 
2753 		switch (tp->mac_version) {
2754 		case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
2755 		case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63:
2756 			/* reset ephy tx/rx disable timer */
2757 			r8168_mac_ocp_modify(tp, 0xe094, 0xff00, 0);
2758 			/* chip can trigger L1.2 */
2759 			r8168_mac_ocp_modify(tp, 0xe092, 0x00ff, BIT(2));
2760 			break;
2761 		default:
2762 			break;
2763 		}
2764 	} else {
2765 		switch (tp->mac_version) {
2766 		case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
2767 		case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63:
2768 			r8168_mac_ocp_modify(tp, 0xe092, 0x00ff, 0);
2769 			break;
2770 		default:
2771 			break;
2772 		}
2773 
2774 		rtl_mod_config2(tp, ClkReqEn, 0);
2775 		rtl_mod_config5(tp, ASPM_en, 0);
2776 	}
2777 }
2778 
2779 static void rtl_set_fifo_size(struct rtl8169_private *tp, u16 rx_stat,
2780 			      u16 tx_stat, u16 rx_dyn, u16 tx_dyn)
2781 {
2782 	/* Usage of dynamic vs. static FIFO is controlled by bit
2783 	 * TXCFG_AUTO_FIFO. Exact meaning of FIFO values isn't known.
2784 	 */
2785 	rtl_eri_write(tp, 0xc8, ERIAR_MASK_1111, (rx_stat << 16) | rx_dyn);
2786 	rtl_eri_write(tp, 0xe8, ERIAR_MASK_1111, (tx_stat << 16) | tx_dyn);
2787 }
2788 
2789 static void rtl8168g_set_pause_thresholds(struct rtl8169_private *tp,
2790 					  u8 low, u8 high)
2791 {
2792 	/* FIFO thresholds for pause flow control */
2793 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_0001, low);
2794 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_0001, high);
2795 }
2796 
2797 static void rtl_hw_start_8168b(struct rtl8169_private *tp)
2798 {
2799 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2800 }
2801 
2802 static void __rtl_hw_start_8168cp(struct rtl8169_private *tp)
2803 {
2804 	RTL_W8(tp, Config1, RTL_R8(tp, Config1) | Speed_down);
2805 
2806 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2807 
2808 	rtl_disable_clock_request(tp);
2809 }
2810 
2811 static void rtl_hw_start_8168cp_1(struct rtl8169_private *tp)
2812 {
2813 	static const struct ephy_info e_info_8168cp[] = {
2814 		{ 0x01, 0,	0x0001 },
2815 		{ 0x02, 0x0800,	0x1000 },
2816 		{ 0x03, 0,	0x0042 },
2817 		{ 0x06, 0x0080,	0x0000 },
2818 		{ 0x07, 0,	0x2000 }
2819 	};
2820 
2821 	rtl_set_def_aspm_entry_latency(tp);
2822 
2823 	rtl_ephy_init(tp, e_info_8168cp);
2824 
2825 	__rtl_hw_start_8168cp(tp);
2826 }
2827 
2828 static void rtl_hw_start_8168cp_2(struct rtl8169_private *tp)
2829 {
2830 	rtl_set_def_aspm_entry_latency(tp);
2831 
2832 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2833 }
2834 
2835 static void rtl_hw_start_8168cp_3(struct rtl8169_private *tp)
2836 {
2837 	rtl_set_def_aspm_entry_latency(tp);
2838 
2839 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2840 
2841 	/* Magic. */
2842 	RTL_W8(tp, DBG_REG, 0x20);
2843 }
2844 
2845 static void rtl_hw_start_8168c_1(struct rtl8169_private *tp)
2846 {
2847 	static const struct ephy_info e_info_8168c_1[] = {
2848 		{ 0x02, 0x0800,	0x1000 },
2849 		{ 0x03, 0,	0x0002 },
2850 		{ 0x06, 0x0080,	0x0000 }
2851 	};
2852 
2853 	rtl_set_def_aspm_entry_latency(tp);
2854 
2855 	RTL_W8(tp, DBG_REG, 0x06 | FIX_NAK_1 | FIX_NAK_2);
2856 
2857 	rtl_ephy_init(tp, e_info_8168c_1);
2858 
2859 	__rtl_hw_start_8168cp(tp);
2860 }
2861 
2862 static void rtl_hw_start_8168c_2(struct rtl8169_private *tp)
2863 {
2864 	static const struct ephy_info e_info_8168c_2[] = {
2865 		{ 0x01, 0,	0x0001 },
2866 		{ 0x03, 0x0400,	0x0020 }
2867 	};
2868 
2869 	rtl_set_def_aspm_entry_latency(tp);
2870 
2871 	rtl_ephy_init(tp, e_info_8168c_2);
2872 
2873 	__rtl_hw_start_8168cp(tp);
2874 }
2875 
2876 static void rtl_hw_start_8168c_4(struct rtl8169_private *tp)
2877 {
2878 	rtl_set_def_aspm_entry_latency(tp);
2879 
2880 	__rtl_hw_start_8168cp(tp);
2881 }
2882 
2883 static void rtl_hw_start_8168d(struct rtl8169_private *tp)
2884 {
2885 	rtl_set_def_aspm_entry_latency(tp);
2886 
2887 	rtl_disable_clock_request(tp);
2888 }
2889 
2890 static void rtl_hw_start_8168d_4(struct rtl8169_private *tp)
2891 {
2892 	static const struct ephy_info e_info_8168d_4[] = {
2893 		{ 0x0b, 0x0000,	0x0048 },
2894 		{ 0x19, 0x0020,	0x0050 },
2895 		{ 0x0c, 0x0100,	0x0020 },
2896 		{ 0x10, 0x0004,	0x0000 },
2897 	};
2898 
2899 	rtl_set_def_aspm_entry_latency(tp);
2900 
2901 	rtl_ephy_init(tp, e_info_8168d_4);
2902 
2903 	rtl_enable_clock_request(tp);
2904 }
2905 
2906 static void rtl_hw_start_8168e_1(struct rtl8169_private *tp)
2907 {
2908 	static const struct ephy_info e_info_8168e_1[] = {
2909 		{ 0x00, 0x0200,	0x0100 },
2910 		{ 0x00, 0x0000,	0x0004 },
2911 		{ 0x06, 0x0002,	0x0001 },
2912 		{ 0x06, 0x0000,	0x0030 },
2913 		{ 0x07, 0x0000,	0x2000 },
2914 		{ 0x00, 0x0000,	0x0020 },
2915 		{ 0x03, 0x5800,	0x2000 },
2916 		{ 0x03, 0x0000,	0x0001 },
2917 		{ 0x01, 0x0800,	0x1000 },
2918 		{ 0x07, 0x0000,	0x4000 },
2919 		{ 0x1e, 0x0000,	0x2000 },
2920 		{ 0x19, 0xffff,	0xfe6c },
2921 		{ 0x0a, 0x0000,	0x0040 }
2922 	};
2923 
2924 	rtl_set_def_aspm_entry_latency(tp);
2925 
2926 	rtl_ephy_init(tp, e_info_8168e_1);
2927 
2928 	rtl_disable_clock_request(tp);
2929 
2930 	/* Reset tx FIFO pointer */
2931 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | TXPLA_RST);
2932 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~TXPLA_RST);
2933 
2934 	rtl_mod_config5(tp, Spi_en, 0);
2935 }
2936 
2937 static void rtl_hw_start_8168e_2(struct rtl8169_private *tp)
2938 {
2939 	static const struct ephy_info e_info_8168e_2[] = {
2940 		{ 0x09, 0x0000,	0x0080 },
2941 		{ 0x19, 0x0000,	0x0224 },
2942 		{ 0x00, 0x0000,	0x0004 },
2943 		{ 0x0c, 0x3df0,	0x0200 },
2944 	};
2945 
2946 	rtl_set_def_aspm_entry_latency(tp);
2947 
2948 	rtl_ephy_init(tp, e_info_8168e_2);
2949 
2950 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
2951 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_1111, 0x0000);
2952 	rtl_set_fifo_size(tp, 0x10, 0x10, 0x02, 0x06);
2953 	rtl_eri_set_bits(tp, 0x1d0, BIT(1));
2954 	rtl_reset_packet_filter(tp);
2955 	rtl_eri_set_bits(tp, 0x1b0, BIT(4));
2956 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_1111, 0x00000050);
2957 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_1111, 0x07ff0060);
2958 
2959 	rtl_disable_clock_request(tp);
2960 
2961 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
2962 
2963 	rtl8168_config_eee_mac(tp);
2964 
2965 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
2966 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
2967 	rtl_mod_config5(tp, Spi_en, 0);
2968 }
2969 
2970 static void rtl_hw_start_8168f(struct rtl8169_private *tp)
2971 {
2972 	rtl_set_def_aspm_entry_latency(tp);
2973 
2974 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
2975 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_1111, 0x0000);
2976 	rtl_set_fifo_size(tp, 0x10, 0x10, 0x02, 0x06);
2977 	rtl_reset_packet_filter(tp);
2978 	rtl_eri_set_bits(tp, 0x1b0, BIT(4));
2979 	rtl_eri_set_bits(tp, 0x1d0, BIT(4) | BIT(1));
2980 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_1111, 0x00000050);
2981 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_1111, 0x00000060);
2982 
2983 	rtl_disable_clock_request(tp);
2984 
2985 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
2986 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
2987 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
2988 	rtl_mod_config5(tp, Spi_en, 0);
2989 
2990 	rtl8168_config_eee_mac(tp);
2991 }
2992 
2993 static void rtl_hw_start_8168f_1(struct rtl8169_private *tp)
2994 {
2995 	static const struct ephy_info e_info_8168f_1[] = {
2996 		{ 0x06, 0x00c0,	0x0020 },
2997 		{ 0x08, 0x0001,	0x0002 },
2998 		{ 0x09, 0x0000,	0x0080 },
2999 		{ 0x19, 0x0000,	0x0224 },
3000 		{ 0x00, 0x0000,	0x0008 },
3001 		{ 0x0c, 0x3df0,	0x0200 },
3002 	};
3003 
3004 	rtl_hw_start_8168f(tp);
3005 
3006 	rtl_ephy_init(tp, e_info_8168f_1);
3007 }
3008 
3009 static void rtl_hw_start_8411(struct rtl8169_private *tp)
3010 {
3011 	static const struct ephy_info e_info_8168f_1[] = {
3012 		{ 0x06, 0x00c0,	0x0020 },
3013 		{ 0x0f, 0xffff,	0x5200 },
3014 		{ 0x19, 0x0000,	0x0224 },
3015 		{ 0x00, 0x0000,	0x0008 },
3016 		{ 0x0c, 0x3df0,	0x0200 },
3017 	};
3018 
3019 	rtl_hw_start_8168f(tp);
3020 	rtl_pcie_state_l2l3_disable(tp);
3021 
3022 	rtl_ephy_init(tp, e_info_8168f_1);
3023 }
3024 
3025 static void rtl_hw_start_8168g(struct rtl8169_private *tp)
3026 {
3027 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3028 	rtl8168g_set_pause_thresholds(tp, 0x38, 0x48);
3029 
3030 	rtl_set_def_aspm_entry_latency(tp);
3031 
3032 	rtl_reset_packet_filter(tp);
3033 	rtl_eri_write(tp, 0x2f8, ERIAR_MASK_0011, 0x1d8f);
3034 
3035 	rtl_disable_rxdvgate(tp);
3036 
3037 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3038 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3039 
3040 	rtl8168_config_eee_mac(tp);
3041 
3042 	rtl_w0w1_eri(tp, 0x2fc, 0x01, 0x06);
3043 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
3044 
3045 	rtl_pcie_state_l2l3_disable(tp);
3046 }
3047 
3048 static void rtl_hw_start_8168g_1(struct rtl8169_private *tp)
3049 {
3050 	static const struct ephy_info e_info_8168g_1[] = {
3051 		{ 0x00, 0x0008,	0x0000 },
3052 		{ 0x0c, 0x3ff0,	0x0820 },
3053 		{ 0x1e, 0x0000,	0x0001 },
3054 		{ 0x19, 0x8000,	0x0000 }
3055 	};
3056 
3057 	rtl_hw_start_8168g(tp);
3058 	rtl_ephy_init(tp, e_info_8168g_1);
3059 }
3060 
3061 static void rtl_hw_start_8168g_2(struct rtl8169_private *tp)
3062 {
3063 	static const struct ephy_info e_info_8168g_2[] = {
3064 		{ 0x00, 0x0008,	0x0000 },
3065 		{ 0x0c, 0x3ff0,	0x0820 },
3066 		{ 0x19, 0xffff,	0x7c00 },
3067 		{ 0x1e, 0xffff,	0x20eb },
3068 		{ 0x0d, 0xffff,	0x1666 },
3069 		{ 0x00, 0xffff,	0x10a3 },
3070 		{ 0x06, 0xffff,	0xf050 },
3071 		{ 0x04, 0x0000,	0x0010 },
3072 		{ 0x1d, 0x4000,	0x0000 },
3073 	};
3074 
3075 	rtl_hw_start_8168g(tp);
3076 	rtl_ephy_init(tp, e_info_8168g_2);
3077 }
3078 
3079 static void rtl_hw_start_8411_2(struct rtl8169_private *tp)
3080 {
3081 	static const struct ephy_info e_info_8411_2[] = {
3082 		{ 0x00, 0x0008,	0x0000 },
3083 		{ 0x0c, 0x37d0,	0x0820 },
3084 		{ 0x1e, 0x0000,	0x0001 },
3085 		{ 0x19, 0x8021,	0x0000 },
3086 		{ 0x1e, 0x0000,	0x2000 },
3087 		{ 0x0d, 0x0100,	0x0200 },
3088 		{ 0x00, 0x0000,	0x0080 },
3089 		{ 0x06, 0x0000,	0x0010 },
3090 		{ 0x04, 0x0000,	0x0010 },
3091 		{ 0x1d, 0x0000,	0x4000 },
3092 	};
3093 
3094 	rtl_hw_start_8168g(tp);
3095 
3096 	rtl_ephy_init(tp, e_info_8411_2);
3097 
3098 	/* The following Realtek-provided magic fixes an issue with the RX unit
3099 	 * getting confused after the PHY having been powered-down.
3100 	 */
3101 	r8168_mac_ocp_write(tp, 0xFC28, 0x0000);
3102 	r8168_mac_ocp_write(tp, 0xFC2A, 0x0000);
3103 	r8168_mac_ocp_write(tp, 0xFC2C, 0x0000);
3104 	r8168_mac_ocp_write(tp, 0xFC2E, 0x0000);
3105 	r8168_mac_ocp_write(tp, 0xFC30, 0x0000);
3106 	r8168_mac_ocp_write(tp, 0xFC32, 0x0000);
3107 	r8168_mac_ocp_write(tp, 0xFC34, 0x0000);
3108 	r8168_mac_ocp_write(tp, 0xFC36, 0x0000);
3109 	mdelay(3);
3110 	r8168_mac_ocp_write(tp, 0xFC26, 0x0000);
3111 
3112 	r8168_mac_ocp_write(tp, 0xF800, 0xE008);
3113 	r8168_mac_ocp_write(tp, 0xF802, 0xE00A);
3114 	r8168_mac_ocp_write(tp, 0xF804, 0xE00C);
3115 	r8168_mac_ocp_write(tp, 0xF806, 0xE00E);
3116 	r8168_mac_ocp_write(tp, 0xF808, 0xE027);
3117 	r8168_mac_ocp_write(tp, 0xF80A, 0xE04F);
3118 	r8168_mac_ocp_write(tp, 0xF80C, 0xE05E);
3119 	r8168_mac_ocp_write(tp, 0xF80E, 0xE065);
3120 	r8168_mac_ocp_write(tp, 0xF810, 0xC602);
3121 	r8168_mac_ocp_write(tp, 0xF812, 0xBE00);
3122 	r8168_mac_ocp_write(tp, 0xF814, 0x0000);
3123 	r8168_mac_ocp_write(tp, 0xF816, 0xC502);
3124 	r8168_mac_ocp_write(tp, 0xF818, 0xBD00);
3125 	r8168_mac_ocp_write(tp, 0xF81A, 0x074C);
3126 	r8168_mac_ocp_write(tp, 0xF81C, 0xC302);
3127 	r8168_mac_ocp_write(tp, 0xF81E, 0xBB00);
3128 	r8168_mac_ocp_write(tp, 0xF820, 0x080A);
3129 	r8168_mac_ocp_write(tp, 0xF822, 0x6420);
3130 	r8168_mac_ocp_write(tp, 0xF824, 0x48C2);
3131 	r8168_mac_ocp_write(tp, 0xF826, 0x8C20);
3132 	r8168_mac_ocp_write(tp, 0xF828, 0xC516);
3133 	r8168_mac_ocp_write(tp, 0xF82A, 0x64A4);
3134 	r8168_mac_ocp_write(tp, 0xF82C, 0x49C0);
3135 	r8168_mac_ocp_write(tp, 0xF82E, 0xF009);
3136 	r8168_mac_ocp_write(tp, 0xF830, 0x74A2);
3137 	r8168_mac_ocp_write(tp, 0xF832, 0x8CA5);
3138 	r8168_mac_ocp_write(tp, 0xF834, 0x74A0);
3139 	r8168_mac_ocp_write(tp, 0xF836, 0xC50E);
3140 	r8168_mac_ocp_write(tp, 0xF838, 0x9CA2);
3141 	r8168_mac_ocp_write(tp, 0xF83A, 0x1C11);
3142 	r8168_mac_ocp_write(tp, 0xF83C, 0x9CA0);
3143 	r8168_mac_ocp_write(tp, 0xF83E, 0xE006);
3144 	r8168_mac_ocp_write(tp, 0xF840, 0x74F8);
3145 	r8168_mac_ocp_write(tp, 0xF842, 0x48C4);
3146 	r8168_mac_ocp_write(tp, 0xF844, 0x8CF8);
3147 	r8168_mac_ocp_write(tp, 0xF846, 0xC404);
3148 	r8168_mac_ocp_write(tp, 0xF848, 0xBC00);
3149 	r8168_mac_ocp_write(tp, 0xF84A, 0xC403);
3150 	r8168_mac_ocp_write(tp, 0xF84C, 0xBC00);
3151 	r8168_mac_ocp_write(tp, 0xF84E, 0x0BF2);
3152 	r8168_mac_ocp_write(tp, 0xF850, 0x0C0A);
3153 	r8168_mac_ocp_write(tp, 0xF852, 0xE434);
3154 	r8168_mac_ocp_write(tp, 0xF854, 0xD3C0);
3155 	r8168_mac_ocp_write(tp, 0xF856, 0x49D9);
3156 	r8168_mac_ocp_write(tp, 0xF858, 0xF01F);
3157 	r8168_mac_ocp_write(tp, 0xF85A, 0xC526);
3158 	r8168_mac_ocp_write(tp, 0xF85C, 0x64A5);
3159 	r8168_mac_ocp_write(tp, 0xF85E, 0x1400);
3160 	r8168_mac_ocp_write(tp, 0xF860, 0xF007);
3161 	r8168_mac_ocp_write(tp, 0xF862, 0x0C01);
3162 	r8168_mac_ocp_write(tp, 0xF864, 0x8CA5);
3163 	r8168_mac_ocp_write(tp, 0xF866, 0x1C15);
3164 	r8168_mac_ocp_write(tp, 0xF868, 0xC51B);
3165 	r8168_mac_ocp_write(tp, 0xF86A, 0x9CA0);
3166 	r8168_mac_ocp_write(tp, 0xF86C, 0xE013);
3167 	r8168_mac_ocp_write(tp, 0xF86E, 0xC519);
3168 	r8168_mac_ocp_write(tp, 0xF870, 0x74A0);
3169 	r8168_mac_ocp_write(tp, 0xF872, 0x48C4);
3170 	r8168_mac_ocp_write(tp, 0xF874, 0x8CA0);
3171 	r8168_mac_ocp_write(tp, 0xF876, 0xC516);
3172 	r8168_mac_ocp_write(tp, 0xF878, 0x74A4);
3173 	r8168_mac_ocp_write(tp, 0xF87A, 0x48C8);
3174 	r8168_mac_ocp_write(tp, 0xF87C, 0x48CA);
3175 	r8168_mac_ocp_write(tp, 0xF87E, 0x9CA4);
3176 	r8168_mac_ocp_write(tp, 0xF880, 0xC512);
3177 	r8168_mac_ocp_write(tp, 0xF882, 0x1B00);
3178 	r8168_mac_ocp_write(tp, 0xF884, 0x9BA0);
3179 	r8168_mac_ocp_write(tp, 0xF886, 0x1B1C);
3180 	r8168_mac_ocp_write(tp, 0xF888, 0x483F);
3181 	r8168_mac_ocp_write(tp, 0xF88A, 0x9BA2);
3182 	r8168_mac_ocp_write(tp, 0xF88C, 0x1B04);
3183 	r8168_mac_ocp_write(tp, 0xF88E, 0xC508);
3184 	r8168_mac_ocp_write(tp, 0xF890, 0x9BA0);
3185 	r8168_mac_ocp_write(tp, 0xF892, 0xC505);
3186 	r8168_mac_ocp_write(tp, 0xF894, 0xBD00);
3187 	r8168_mac_ocp_write(tp, 0xF896, 0xC502);
3188 	r8168_mac_ocp_write(tp, 0xF898, 0xBD00);
3189 	r8168_mac_ocp_write(tp, 0xF89A, 0x0300);
3190 	r8168_mac_ocp_write(tp, 0xF89C, 0x051E);
3191 	r8168_mac_ocp_write(tp, 0xF89E, 0xE434);
3192 	r8168_mac_ocp_write(tp, 0xF8A0, 0xE018);
3193 	r8168_mac_ocp_write(tp, 0xF8A2, 0xE092);
3194 	r8168_mac_ocp_write(tp, 0xF8A4, 0xDE20);
3195 	r8168_mac_ocp_write(tp, 0xF8A6, 0xD3C0);
3196 	r8168_mac_ocp_write(tp, 0xF8A8, 0xC50F);
3197 	r8168_mac_ocp_write(tp, 0xF8AA, 0x76A4);
3198 	r8168_mac_ocp_write(tp, 0xF8AC, 0x49E3);
3199 	r8168_mac_ocp_write(tp, 0xF8AE, 0xF007);
3200 	r8168_mac_ocp_write(tp, 0xF8B0, 0x49C0);
3201 	r8168_mac_ocp_write(tp, 0xF8B2, 0xF103);
3202 	r8168_mac_ocp_write(tp, 0xF8B4, 0xC607);
3203 	r8168_mac_ocp_write(tp, 0xF8B6, 0xBE00);
3204 	r8168_mac_ocp_write(tp, 0xF8B8, 0xC606);
3205 	r8168_mac_ocp_write(tp, 0xF8BA, 0xBE00);
3206 	r8168_mac_ocp_write(tp, 0xF8BC, 0xC602);
3207 	r8168_mac_ocp_write(tp, 0xF8BE, 0xBE00);
3208 	r8168_mac_ocp_write(tp, 0xF8C0, 0x0C4C);
3209 	r8168_mac_ocp_write(tp, 0xF8C2, 0x0C28);
3210 	r8168_mac_ocp_write(tp, 0xF8C4, 0x0C2C);
3211 	r8168_mac_ocp_write(tp, 0xF8C6, 0xDC00);
3212 	r8168_mac_ocp_write(tp, 0xF8C8, 0xC707);
3213 	r8168_mac_ocp_write(tp, 0xF8CA, 0x1D00);
3214 	r8168_mac_ocp_write(tp, 0xF8CC, 0x8DE2);
3215 	r8168_mac_ocp_write(tp, 0xF8CE, 0x48C1);
3216 	r8168_mac_ocp_write(tp, 0xF8D0, 0xC502);
3217 	r8168_mac_ocp_write(tp, 0xF8D2, 0xBD00);
3218 	r8168_mac_ocp_write(tp, 0xF8D4, 0x00AA);
3219 	r8168_mac_ocp_write(tp, 0xF8D6, 0xE0C0);
3220 	r8168_mac_ocp_write(tp, 0xF8D8, 0xC502);
3221 	r8168_mac_ocp_write(tp, 0xF8DA, 0xBD00);
3222 	r8168_mac_ocp_write(tp, 0xF8DC, 0x0132);
3223 
3224 	r8168_mac_ocp_write(tp, 0xFC26, 0x8000);
3225 
3226 	r8168_mac_ocp_write(tp, 0xFC2A, 0x0743);
3227 	r8168_mac_ocp_write(tp, 0xFC2C, 0x0801);
3228 	r8168_mac_ocp_write(tp, 0xFC2E, 0x0BE9);
3229 	r8168_mac_ocp_write(tp, 0xFC30, 0x02FD);
3230 	r8168_mac_ocp_write(tp, 0xFC32, 0x0C25);
3231 	r8168_mac_ocp_write(tp, 0xFC34, 0x00A9);
3232 	r8168_mac_ocp_write(tp, 0xFC36, 0x012D);
3233 }
3234 
3235 static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
3236 {
3237 	static const struct ephy_info e_info_8168h_1[] = {
3238 		{ 0x1e, 0x0800,	0x0001 },
3239 		{ 0x1d, 0x0000,	0x0800 },
3240 		{ 0x05, 0xffff,	0x2089 },
3241 		{ 0x06, 0xffff,	0x5881 },
3242 		{ 0x04, 0xffff,	0x854a },
3243 		{ 0x01, 0xffff,	0x068b }
3244 	};
3245 	int rg_saw_cnt;
3246 
3247 	rtl_ephy_init(tp, e_info_8168h_1);
3248 
3249 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3250 	rtl8168g_set_pause_thresholds(tp, 0x38, 0x48);
3251 
3252 	rtl_set_def_aspm_entry_latency(tp);
3253 
3254 	rtl_reset_packet_filter(tp);
3255 
3256 	rtl_eri_set_bits(tp, 0xdc, 0x001c);
3257 
3258 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3259 
3260 	rtl_disable_rxdvgate(tp);
3261 
3262 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3263 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3264 
3265 	rtl8168_config_eee_mac(tp);
3266 
3267 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3268 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3269 
3270 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3271 
3272 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
3273 
3274 	rtl_pcie_state_l2l3_disable(tp);
3275 
3276 	rg_saw_cnt = phy_read_paged(tp->phydev, 0x0c42, 0x13) & 0x3fff;
3277 	if (rg_saw_cnt > 0) {
3278 		u16 sw_cnt_1ms_ini;
3279 
3280 		sw_cnt_1ms_ini = 16000000/rg_saw_cnt;
3281 		sw_cnt_1ms_ini &= 0x0fff;
3282 		r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini);
3283 	}
3284 
3285 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0070);
3286 	r8168_mac_ocp_modify(tp, 0xe052, 0x6000, 0x8008);
3287 	r8168_mac_ocp_modify(tp, 0xe0d6, 0x01ff, 0x017f);
3288 	r8168_mac_ocp_modify(tp, 0xd420, 0x0fff, 0x047f);
3289 
3290 	r8168_mac_ocp_write(tp, 0xe63e, 0x0001);
3291 	r8168_mac_ocp_write(tp, 0xe63e, 0x0000);
3292 	r8168_mac_ocp_write(tp, 0xc094, 0x0000);
3293 	r8168_mac_ocp_write(tp, 0xc09e, 0x0000);
3294 }
3295 
3296 static void rtl_hw_start_8168ep(struct rtl8169_private *tp)
3297 {
3298 	rtl8168ep_stop_cmac(tp);
3299 
3300 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3301 	rtl8168g_set_pause_thresholds(tp, 0x2f, 0x5f);
3302 
3303 	rtl_set_def_aspm_entry_latency(tp);
3304 
3305 	rtl_reset_packet_filter(tp);
3306 
3307 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3308 
3309 	rtl_disable_rxdvgate(tp);
3310 
3311 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3312 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3313 
3314 	rtl8168_config_eee_mac(tp);
3315 
3316 	rtl_w0w1_eri(tp, 0x2fc, 0x01, 0x06);
3317 
3318 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3319 
3320 	rtl_pcie_state_l2l3_disable(tp);
3321 }
3322 
3323 static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
3324 {
3325 	static const struct ephy_info e_info_8168ep_3[] = {
3326 		{ 0x00, 0x0000,	0x0080 },
3327 		{ 0x0d, 0x0100,	0x0200 },
3328 		{ 0x19, 0x8021,	0x0000 },
3329 		{ 0x1e, 0x0000,	0x2000 },
3330 	};
3331 
3332 	rtl_ephy_init(tp, e_info_8168ep_3);
3333 
3334 	rtl_hw_start_8168ep(tp);
3335 
3336 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3337 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3338 
3339 	r8168_mac_ocp_modify(tp, 0xd3e2, 0x0fff, 0x0271);
3340 	r8168_mac_ocp_modify(tp, 0xd3e4, 0x00ff, 0x0000);
3341 	r8168_mac_ocp_modify(tp, 0xe860, 0x0000, 0x0080);
3342 }
3343 
3344 static void rtl_hw_start_8117(struct rtl8169_private *tp)
3345 {
3346 	static const struct ephy_info e_info_8117[] = {
3347 		{ 0x19, 0x0040,	0x1100 },
3348 		{ 0x59, 0x0040,	0x1100 },
3349 	};
3350 	int rg_saw_cnt;
3351 
3352 	rtl8168ep_stop_cmac(tp);
3353 	rtl_ephy_init(tp, e_info_8117);
3354 
3355 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3356 	rtl8168g_set_pause_thresholds(tp, 0x2f, 0x5f);
3357 
3358 	rtl_set_def_aspm_entry_latency(tp);
3359 
3360 	rtl_reset_packet_filter(tp);
3361 
3362 	rtl_eri_set_bits(tp, 0xd4, 0x0010);
3363 
3364 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3365 
3366 	rtl_disable_rxdvgate(tp);
3367 
3368 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3369 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3370 
3371 	rtl8168_config_eee_mac(tp);
3372 
3373 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3374 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3375 
3376 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3377 
3378 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
3379 
3380 	rtl_pcie_state_l2l3_disable(tp);
3381 
3382 	rg_saw_cnt = phy_read_paged(tp->phydev, 0x0c42, 0x13) & 0x3fff;
3383 	if (rg_saw_cnt > 0) {
3384 		u16 sw_cnt_1ms_ini;
3385 
3386 		sw_cnt_1ms_ini = (16000000 / rg_saw_cnt) & 0x0fff;
3387 		r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini);
3388 	}
3389 
3390 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0070);
3391 	r8168_mac_ocp_write(tp, 0xea80, 0x0003);
3392 	r8168_mac_ocp_modify(tp, 0xe052, 0x0000, 0x0009);
3393 	r8168_mac_ocp_modify(tp, 0xd420, 0x0fff, 0x047f);
3394 
3395 	r8168_mac_ocp_write(tp, 0xe63e, 0x0001);
3396 	r8168_mac_ocp_write(tp, 0xe63e, 0x0000);
3397 	r8168_mac_ocp_write(tp, 0xc094, 0x0000);
3398 	r8168_mac_ocp_write(tp, 0xc09e, 0x0000);
3399 
3400 	/* firmware is for MAC only */
3401 	r8169_apply_firmware(tp);
3402 }
3403 
3404 static void rtl_hw_start_8102e_1(struct rtl8169_private *tp)
3405 {
3406 	static const struct ephy_info e_info_8102e_1[] = {
3407 		{ 0x01,	0, 0x6e65 },
3408 		{ 0x02,	0, 0x091f },
3409 		{ 0x03,	0, 0xc2f9 },
3410 		{ 0x06,	0, 0xafb5 },
3411 		{ 0x07,	0, 0x0e00 },
3412 		{ 0x19,	0, 0xec80 },
3413 		{ 0x01,	0, 0x2e65 },
3414 		{ 0x01,	0, 0x6e65 }
3415 	};
3416 	u8 cfg1;
3417 
3418 	rtl_set_def_aspm_entry_latency(tp);
3419 
3420 	RTL_W8(tp, DBG_REG, FIX_NAK_1);
3421 
3422 	RTL_W8(tp, Config1,
3423 	       LEDS1 | LEDS0 | Speed_down | MEMMAP | IOMAP | VPD | PMEnable);
3424 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3425 
3426 	cfg1 = RTL_R8(tp, Config1);
3427 	if ((cfg1 & LEDS0) && (cfg1 & LEDS1))
3428 		RTL_W8(tp, Config1, cfg1 & ~LEDS0);
3429 
3430 	rtl_ephy_init(tp, e_info_8102e_1);
3431 }
3432 
3433 static void rtl_hw_start_8102e_2(struct rtl8169_private *tp)
3434 {
3435 	rtl_set_def_aspm_entry_latency(tp);
3436 
3437 	RTL_W8(tp, Config1, MEMMAP | IOMAP | VPD | PMEnable);
3438 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3439 }
3440 
3441 static void rtl_hw_start_8102e_3(struct rtl8169_private *tp)
3442 {
3443 	rtl_hw_start_8102e_2(tp);
3444 
3445 	rtl_ephy_write(tp, 0x03, 0xc2f9);
3446 }
3447 
3448 static void rtl_hw_start_8401(struct rtl8169_private *tp)
3449 {
3450 	static const struct ephy_info e_info_8401[] = {
3451 		{ 0x01,	0xffff, 0x6fe5 },
3452 		{ 0x03,	0xffff, 0x0599 },
3453 		{ 0x06,	0xffff, 0xaf25 },
3454 		{ 0x07,	0xffff, 0x8e68 },
3455 	};
3456 
3457 	rtl_ephy_init(tp, e_info_8401);
3458 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3459 }
3460 
3461 static void rtl_hw_start_8105e_1(struct rtl8169_private *tp)
3462 {
3463 	static const struct ephy_info e_info_8105e_1[] = {
3464 		{ 0x07,	0, 0x4000 },
3465 		{ 0x19,	0, 0x0200 },
3466 		{ 0x19,	0, 0x0020 },
3467 		{ 0x1e,	0, 0x2000 },
3468 		{ 0x03,	0, 0x0001 },
3469 		{ 0x19,	0, 0x0100 },
3470 		{ 0x19,	0, 0x0004 },
3471 		{ 0x0a,	0, 0x0020 }
3472 	};
3473 
3474 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3475 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3476 
3477 	/* Disable Early Tally Counter */
3478 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) & ~0x010000);
3479 
3480 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) | EN_NDP | EN_OOB_RESET);
3481 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
3482 
3483 	rtl_ephy_init(tp, e_info_8105e_1);
3484 
3485 	rtl_pcie_state_l2l3_disable(tp);
3486 }
3487 
3488 static void rtl_hw_start_8105e_2(struct rtl8169_private *tp)
3489 {
3490 	rtl_hw_start_8105e_1(tp);
3491 	rtl_ephy_write(tp, 0x1e, rtl_ephy_read(tp, 0x1e) | 0x8000);
3492 }
3493 
3494 static void rtl_hw_start_8402(struct rtl8169_private *tp)
3495 {
3496 	static const struct ephy_info e_info_8402[] = {
3497 		{ 0x19,	0xffff, 0xff64 },
3498 		{ 0x1e,	0, 0x4000 }
3499 	};
3500 
3501 	rtl_set_def_aspm_entry_latency(tp);
3502 
3503 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3504 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3505 
3506 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
3507 
3508 	rtl_ephy_init(tp, e_info_8402);
3509 
3510 	rtl_set_fifo_size(tp, 0x00, 0x00, 0x02, 0x06);
3511 	rtl_reset_packet_filter(tp);
3512 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3513 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3514 	rtl_w0w1_eri(tp, 0x0d4, 0x0e00, 0xff00);
3515 
3516 	/* disable EEE */
3517 	rtl_eri_write(tp, 0x1b0, ERIAR_MASK_0011, 0x0000);
3518 
3519 	rtl_pcie_state_l2l3_disable(tp);
3520 }
3521 
3522 static void rtl_hw_start_8106(struct rtl8169_private *tp)
3523 {
3524 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3525 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3526 
3527 	RTL_W32(tp, MISC, (RTL_R32(tp, MISC) | DISABLE_LAN_EN) & ~EARLY_TALLY_EN);
3528 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) | EN_NDP | EN_OOB_RESET);
3529 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3530 
3531 	/* L0 7us, L1 32us - needed to avoid issues with link-up detection */
3532 	rtl_set_aspm_entry_latency(tp, 0x2f);
3533 
3534 	rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x0000);
3535 
3536 	/* disable EEE */
3537 	rtl_eri_write(tp, 0x1b0, ERIAR_MASK_0011, 0x0000);
3538 
3539 	rtl_pcie_state_l2l3_disable(tp);
3540 }
3541 
3542 DECLARE_RTL_COND(rtl_mac_ocp_e00e_cond)
3543 {
3544 	return r8168_mac_ocp_read(tp, 0xe00e) & BIT(13);
3545 }
3546 
3547 static void rtl_hw_start_8125_common(struct rtl8169_private *tp)
3548 {
3549 	rtl_pcie_state_l2l3_disable(tp);
3550 
3551 	RTL_W16(tp, 0x382, 0x221b);
3552 	RTL_W8(tp, 0x4500, 0);
3553 	RTL_W16(tp, 0x4800, 0);
3554 
3555 	/* disable UPS */
3556 	r8168_mac_ocp_modify(tp, 0xd40a, 0x0010, 0x0000);
3557 
3558 	RTL_W8(tp, Config1, RTL_R8(tp, Config1) & ~0x10);
3559 
3560 	r8168_mac_ocp_write(tp, 0xc140, 0xffff);
3561 	r8168_mac_ocp_write(tp, 0xc142, 0xffff);
3562 
3563 	r8168_mac_ocp_modify(tp, 0xd3e2, 0x0fff, 0x03a9);
3564 	r8168_mac_ocp_modify(tp, 0xd3e4, 0x00ff, 0x0000);
3565 	r8168_mac_ocp_modify(tp, 0xe860, 0x0000, 0x0080);
3566 
3567 	/* disable new tx descriptor format */
3568 	r8168_mac_ocp_modify(tp, 0xeb58, 0x0001, 0x0000);
3569 
3570 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3571 		r8168_mac_ocp_modify(tp, 0xe614, 0x0700, 0x0200);
3572 	else
3573 		r8168_mac_ocp_modify(tp, 0xe614, 0x0700, 0x0400);
3574 
3575 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3576 		r8168_mac_ocp_modify(tp, 0xe63e, 0x0c30, 0x0000);
3577 	else
3578 		r8168_mac_ocp_modify(tp, 0xe63e, 0x0c30, 0x0020);
3579 
3580 	r8168_mac_ocp_modify(tp, 0xc0b4, 0x0000, 0x000c);
3581 	r8168_mac_ocp_modify(tp, 0xeb6a, 0x00ff, 0x0033);
3582 	r8168_mac_ocp_modify(tp, 0xeb50, 0x03e0, 0x0040);
3583 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0030);
3584 	r8168_mac_ocp_modify(tp, 0xe040, 0x1000, 0x0000);
3585 	r8168_mac_ocp_modify(tp, 0xea1c, 0x0003, 0x0001);
3586 	r8168_mac_ocp_modify(tp, 0xe0c0, 0x4f0f, 0x4403);
3587 	r8168_mac_ocp_modify(tp, 0xe052, 0x0080, 0x0068);
3588 	r8168_mac_ocp_modify(tp, 0xd430, 0x0fff, 0x047f);
3589 
3590 	r8168_mac_ocp_modify(tp, 0xea1c, 0x0004, 0x0000);
3591 	r8168_mac_ocp_modify(tp, 0xeb54, 0x0000, 0x0001);
3592 	udelay(1);
3593 	r8168_mac_ocp_modify(tp, 0xeb54, 0x0001, 0x0000);
3594 	RTL_W16(tp, 0x1880, RTL_R16(tp, 0x1880) & ~0x0030);
3595 
3596 	r8168_mac_ocp_write(tp, 0xe098, 0xc302);
3597 
3598 	rtl_loop_wait_low(tp, &rtl_mac_ocp_e00e_cond, 1000, 10);
3599 
3600 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3601 		rtl8125b_config_eee_mac(tp);
3602 	else
3603 		rtl8125a_config_eee_mac(tp);
3604 
3605 	rtl_disable_rxdvgate(tp);
3606 }
3607 
3608 static void rtl_hw_start_8125a_2(struct rtl8169_private *tp)
3609 {
3610 	static const struct ephy_info e_info_8125a_2[] = {
3611 		{ 0x04, 0xffff, 0xd000 },
3612 		{ 0x0a, 0xffff, 0x8653 },
3613 		{ 0x23, 0xffff, 0xab66 },
3614 		{ 0x20, 0xffff, 0x9455 },
3615 		{ 0x21, 0xffff, 0x99ff },
3616 		{ 0x29, 0xffff, 0xfe04 },
3617 
3618 		{ 0x44, 0xffff, 0xd000 },
3619 		{ 0x4a, 0xffff, 0x8653 },
3620 		{ 0x63, 0xffff, 0xab66 },
3621 		{ 0x60, 0xffff, 0x9455 },
3622 		{ 0x61, 0xffff, 0x99ff },
3623 		{ 0x69, 0xffff, 0xfe04 },
3624 	};
3625 
3626 	rtl_set_def_aspm_entry_latency(tp);
3627 	rtl_ephy_init(tp, e_info_8125a_2);
3628 	rtl_hw_start_8125_common(tp);
3629 }
3630 
3631 static void rtl_hw_start_8125b(struct rtl8169_private *tp)
3632 {
3633 	static const struct ephy_info e_info_8125b[] = {
3634 		{ 0x0b, 0xffff, 0xa908 },
3635 		{ 0x1e, 0xffff, 0x20eb },
3636 		{ 0x4b, 0xffff, 0xa908 },
3637 		{ 0x5e, 0xffff, 0x20eb },
3638 		{ 0x22, 0x0030, 0x0020 },
3639 		{ 0x62, 0x0030, 0x0020 },
3640 	};
3641 
3642 	rtl_set_def_aspm_entry_latency(tp);
3643 	rtl_ephy_init(tp, e_info_8125b);
3644 	rtl_hw_start_8125_common(tp);
3645 }
3646 
3647 static void rtl_hw_config(struct rtl8169_private *tp)
3648 {
3649 	static const rtl_generic_fct hw_configs[] = {
3650 		[RTL_GIGA_MAC_VER_07] = rtl_hw_start_8102e_1,
3651 		[RTL_GIGA_MAC_VER_08] = rtl_hw_start_8102e_3,
3652 		[RTL_GIGA_MAC_VER_09] = rtl_hw_start_8102e_2,
3653 		[RTL_GIGA_MAC_VER_10] = NULL,
3654 		[RTL_GIGA_MAC_VER_11] = rtl_hw_start_8168b,
3655 		[RTL_GIGA_MAC_VER_14] = rtl_hw_start_8401,
3656 		[RTL_GIGA_MAC_VER_17] = rtl_hw_start_8168b,
3657 		[RTL_GIGA_MAC_VER_18] = rtl_hw_start_8168cp_1,
3658 		[RTL_GIGA_MAC_VER_19] = rtl_hw_start_8168c_1,
3659 		[RTL_GIGA_MAC_VER_20] = rtl_hw_start_8168c_2,
3660 		[RTL_GIGA_MAC_VER_21] = rtl_hw_start_8168c_2,
3661 		[RTL_GIGA_MAC_VER_22] = rtl_hw_start_8168c_4,
3662 		[RTL_GIGA_MAC_VER_23] = rtl_hw_start_8168cp_2,
3663 		[RTL_GIGA_MAC_VER_24] = rtl_hw_start_8168cp_3,
3664 		[RTL_GIGA_MAC_VER_25] = rtl_hw_start_8168d,
3665 		[RTL_GIGA_MAC_VER_26] = rtl_hw_start_8168d,
3666 		[RTL_GIGA_MAC_VER_28] = rtl_hw_start_8168d_4,
3667 		[RTL_GIGA_MAC_VER_29] = rtl_hw_start_8105e_1,
3668 		[RTL_GIGA_MAC_VER_30] = rtl_hw_start_8105e_2,
3669 		[RTL_GIGA_MAC_VER_31] = rtl_hw_start_8168d,
3670 		[RTL_GIGA_MAC_VER_32] = rtl_hw_start_8168e_1,
3671 		[RTL_GIGA_MAC_VER_33] = rtl_hw_start_8168e_1,
3672 		[RTL_GIGA_MAC_VER_34] = rtl_hw_start_8168e_2,
3673 		[RTL_GIGA_MAC_VER_35] = rtl_hw_start_8168f_1,
3674 		[RTL_GIGA_MAC_VER_36] = rtl_hw_start_8168f_1,
3675 		[RTL_GIGA_MAC_VER_37] = rtl_hw_start_8402,
3676 		[RTL_GIGA_MAC_VER_38] = rtl_hw_start_8411,
3677 		[RTL_GIGA_MAC_VER_39] = rtl_hw_start_8106,
3678 		[RTL_GIGA_MAC_VER_40] = rtl_hw_start_8168g_1,
3679 		[RTL_GIGA_MAC_VER_42] = rtl_hw_start_8168g_2,
3680 		[RTL_GIGA_MAC_VER_43] = rtl_hw_start_8168g_2,
3681 		[RTL_GIGA_MAC_VER_44] = rtl_hw_start_8411_2,
3682 		[RTL_GIGA_MAC_VER_46] = rtl_hw_start_8168h_1,
3683 		[RTL_GIGA_MAC_VER_48] = rtl_hw_start_8168h_1,
3684 		[RTL_GIGA_MAC_VER_51] = rtl_hw_start_8168ep_3,
3685 		[RTL_GIGA_MAC_VER_52] = rtl_hw_start_8117,
3686 		[RTL_GIGA_MAC_VER_53] = rtl_hw_start_8117,
3687 		[RTL_GIGA_MAC_VER_61] = rtl_hw_start_8125a_2,
3688 		[RTL_GIGA_MAC_VER_63] = rtl_hw_start_8125b,
3689 	};
3690 
3691 	if (hw_configs[tp->mac_version])
3692 		hw_configs[tp->mac_version](tp);
3693 }
3694 
3695 static void rtl_hw_start_8125(struct rtl8169_private *tp)
3696 {
3697 	int i;
3698 
3699 	/* disable interrupt coalescing */
3700 	for (i = 0xa00; i < 0xb00; i += 4)
3701 		RTL_W32(tp, i, 0);
3702 
3703 	rtl_hw_config(tp);
3704 }
3705 
3706 static void rtl_hw_start_8168(struct rtl8169_private *tp)
3707 {
3708 	if (rtl_is_8168evl_up(tp))
3709 		RTL_W8(tp, MaxTxPacketSize, EarlySize);
3710 	else
3711 		RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
3712 
3713 	rtl_hw_config(tp);
3714 
3715 	/* disable interrupt coalescing */
3716 	RTL_W16(tp, IntrMitigate, 0x0000);
3717 }
3718 
3719 static void rtl_hw_start_8169(struct rtl8169_private *tp)
3720 {
3721 	RTL_W8(tp, EarlyTxThres, NoEarlyTx);
3722 
3723 	tp->cp_cmd |= PCIMulRW;
3724 
3725 	if (tp->mac_version == RTL_GIGA_MAC_VER_02 ||
3726 	    tp->mac_version == RTL_GIGA_MAC_VER_03)
3727 		tp->cp_cmd |= EnAnaPLL;
3728 
3729 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
3730 
3731 	rtl8169_set_magic_reg(tp);
3732 
3733 	/* disable interrupt coalescing */
3734 	RTL_W16(tp, IntrMitigate, 0x0000);
3735 }
3736 
3737 static void rtl_hw_start(struct  rtl8169_private *tp)
3738 {
3739 	rtl_unlock_config_regs(tp);
3740 	/* disable aspm and clock request before ephy access */
3741 	rtl_hw_aspm_clkreq_enable(tp, false);
3742 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
3743 
3744 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
3745 		rtl_hw_start_8169(tp);
3746 	else if (rtl_is_8125(tp))
3747 		rtl_hw_start_8125(tp);
3748 	else
3749 		rtl_hw_start_8168(tp);
3750 
3751 	rtl_enable_exit_l1(tp);
3752 	rtl_hw_aspm_clkreq_enable(tp, true);
3753 	rtl_set_rx_max_size(tp);
3754 	rtl_set_rx_tx_desc_registers(tp);
3755 	rtl_lock_config_regs(tp);
3756 
3757 	rtl_jumbo_config(tp);
3758 
3759 	/* Initially a 10 us delay. Turned it into a PCI commit. - FR */
3760 	rtl_pci_commit(tp);
3761 
3762 	RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
3763 	rtl_init_rxcfg(tp);
3764 	rtl_set_tx_config_registers(tp);
3765 	rtl_set_rx_config_features(tp, tp->dev->features);
3766 	rtl_set_rx_mode(tp->dev);
3767 	rtl_irq_enable(tp);
3768 }
3769 
3770 static int rtl8169_change_mtu(struct net_device *dev, int new_mtu)
3771 {
3772 	struct rtl8169_private *tp = netdev_priv(dev);
3773 
3774 	dev->mtu = new_mtu;
3775 	netdev_update_features(dev);
3776 	rtl_jumbo_config(tp);
3777 
3778 	switch (tp->mac_version) {
3779 	case RTL_GIGA_MAC_VER_61:
3780 	case RTL_GIGA_MAC_VER_63:
3781 		rtl8125_set_eee_txidle_timer(tp);
3782 		break;
3783 	default:
3784 		break;
3785 	}
3786 
3787 	return 0;
3788 }
3789 
3790 static void rtl8169_mark_to_asic(struct RxDesc *desc)
3791 {
3792 	u32 eor = le32_to_cpu(desc->opts1) & RingEnd;
3793 
3794 	desc->opts2 = 0;
3795 	/* Force memory writes to complete before releasing descriptor */
3796 	dma_wmb();
3797 	WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | R8169_RX_BUF_SIZE));
3798 }
3799 
3800 static struct page *rtl8169_alloc_rx_data(struct rtl8169_private *tp,
3801 					  struct RxDesc *desc)
3802 {
3803 	struct device *d = tp_to_dev(tp);
3804 	int node = dev_to_node(d);
3805 	dma_addr_t mapping;
3806 	struct page *data;
3807 
3808 	data = alloc_pages_node(node, GFP_KERNEL, get_order(R8169_RX_BUF_SIZE));
3809 	if (!data)
3810 		return NULL;
3811 
3812 	mapping = dma_map_page(d, data, 0, R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
3813 	if (unlikely(dma_mapping_error(d, mapping))) {
3814 		netdev_err(tp->dev, "Failed to map RX DMA!\n");
3815 		__free_pages(data, get_order(R8169_RX_BUF_SIZE));
3816 		return NULL;
3817 	}
3818 
3819 	desc->addr = cpu_to_le64(mapping);
3820 	rtl8169_mark_to_asic(desc);
3821 
3822 	return data;
3823 }
3824 
3825 static void rtl8169_rx_clear(struct rtl8169_private *tp)
3826 {
3827 	int i;
3828 
3829 	for (i = 0; i < NUM_RX_DESC && tp->Rx_databuff[i]; i++) {
3830 		dma_unmap_page(tp_to_dev(tp),
3831 			       le64_to_cpu(tp->RxDescArray[i].addr),
3832 			       R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
3833 		__free_pages(tp->Rx_databuff[i], get_order(R8169_RX_BUF_SIZE));
3834 		tp->Rx_databuff[i] = NULL;
3835 		tp->RxDescArray[i].addr = 0;
3836 		tp->RxDescArray[i].opts1 = 0;
3837 	}
3838 }
3839 
3840 static int rtl8169_rx_fill(struct rtl8169_private *tp)
3841 {
3842 	int i;
3843 
3844 	for (i = 0; i < NUM_RX_DESC; i++) {
3845 		struct page *data;
3846 
3847 		data = rtl8169_alloc_rx_data(tp, tp->RxDescArray + i);
3848 		if (!data) {
3849 			rtl8169_rx_clear(tp);
3850 			return -ENOMEM;
3851 		}
3852 		tp->Rx_databuff[i] = data;
3853 	}
3854 
3855 	/* mark as last descriptor in the ring */
3856 	tp->RxDescArray[NUM_RX_DESC - 1].opts1 |= cpu_to_le32(RingEnd);
3857 
3858 	return 0;
3859 }
3860 
3861 static int rtl8169_init_ring(struct rtl8169_private *tp)
3862 {
3863 	rtl8169_init_ring_indexes(tp);
3864 
3865 	memset(tp->tx_skb, 0, sizeof(tp->tx_skb));
3866 	memset(tp->Rx_databuff, 0, sizeof(tp->Rx_databuff));
3867 
3868 	return rtl8169_rx_fill(tp);
3869 }
3870 
3871 static void rtl8169_unmap_tx_skb(struct rtl8169_private *tp, unsigned int entry)
3872 {
3873 	struct ring_info *tx_skb = tp->tx_skb + entry;
3874 	struct TxDesc *desc = tp->TxDescArray + entry;
3875 
3876 	dma_unmap_single(tp_to_dev(tp), le64_to_cpu(desc->addr), tx_skb->len,
3877 			 DMA_TO_DEVICE);
3878 	memset(desc, 0, sizeof(*desc));
3879 	memset(tx_skb, 0, sizeof(*tx_skb));
3880 }
3881 
3882 static void rtl8169_tx_clear_range(struct rtl8169_private *tp, u32 start,
3883 				   unsigned int n)
3884 {
3885 	unsigned int i;
3886 
3887 	for (i = 0; i < n; i++) {
3888 		unsigned int entry = (start + i) % NUM_TX_DESC;
3889 		struct ring_info *tx_skb = tp->tx_skb + entry;
3890 		unsigned int len = tx_skb->len;
3891 
3892 		if (len) {
3893 			struct sk_buff *skb = tx_skb->skb;
3894 
3895 			rtl8169_unmap_tx_skb(tp, entry);
3896 			if (skb)
3897 				dev_consume_skb_any(skb);
3898 		}
3899 	}
3900 }
3901 
3902 static void rtl8169_tx_clear(struct rtl8169_private *tp)
3903 {
3904 	rtl8169_tx_clear_range(tp, tp->dirty_tx, NUM_TX_DESC);
3905 	netdev_reset_queue(tp->dev);
3906 }
3907 
3908 static void rtl8169_cleanup(struct rtl8169_private *tp)
3909 {
3910 	napi_disable(&tp->napi);
3911 
3912 	/* Give a racing hard_start_xmit a few cycles to complete. */
3913 	synchronize_net();
3914 
3915 	/* Disable interrupts */
3916 	rtl8169_irq_mask_and_ack(tp);
3917 
3918 	rtl_rx_close(tp);
3919 
3920 	switch (tp->mac_version) {
3921 	case RTL_GIGA_MAC_VER_28:
3922 	case RTL_GIGA_MAC_VER_31:
3923 		rtl_loop_wait_low(tp, &rtl_npq_cond, 20, 2000);
3924 		break;
3925 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
3926 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
3927 		rtl_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 666);
3928 		break;
3929 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
3930 		rtl_enable_rxdvgate(tp);
3931 		fsleep(2000);
3932 		break;
3933 	default:
3934 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
3935 		fsleep(100);
3936 		break;
3937 	}
3938 
3939 	rtl_hw_reset(tp);
3940 
3941 	rtl8169_tx_clear(tp);
3942 	rtl8169_init_ring_indexes(tp);
3943 }
3944 
3945 static void rtl_reset_work(struct rtl8169_private *tp)
3946 {
3947 	int i;
3948 
3949 	netif_stop_queue(tp->dev);
3950 
3951 	rtl8169_cleanup(tp);
3952 
3953 	for (i = 0; i < NUM_RX_DESC; i++)
3954 		rtl8169_mark_to_asic(tp->RxDescArray + i);
3955 
3956 	napi_enable(&tp->napi);
3957 	rtl_hw_start(tp);
3958 }
3959 
3960 static void rtl8169_tx_timeout(struct net_device *dev, unsigned int txqueue)
3961 {
3962 	struct rtl8169_private *tp = netdev_priv(dev);
3963 
3964 	rtl_schedule_task(tp, RTL_FLAG_TASK_TX_TIMEOUT);
3965 }
3966 
3967 static int rtl8169_tx_map(struct rtl8169_private *tp, const u32 *opts, u32 len,
3968 			  void *addr, unsigned int entry, bool desc_own)
3969 {
3970 	struct TxDesc *txd = tp->TxDescArray + entry;
3971 	struct device *d = tp_to_dev(tp);
3972 	dma_addr_t mapping;
3973 	u32 opts1;
3974 	int ret;
3975 
3976 	mapping = dma_map_single(d, addr, len, DMA_TO_DEVICE);
3977 	ret = dma_mapping_error(d, mapping);
3978 	if (unlikely(ret)) {
3979 		if (net_ratelimit())
3980 			netdev_err(tp->dev, "Failed to map TX data!\n");
3981 		return ret;
3982 	}
3983 
3984 	txd->addr = cpu_to_le64(mapping);
3985 	txd->opts2 = cpu_to_le32(opts[1]);
3986 
3987 	opts1 = opts[0] | len;
3988 	if (entry == NUM_TX_DESC - 1)
3989 		opts1 |= RingEnd;
3990 	if (desc_own)
3991 		opts1 |= DescOwn;
3992 	txd->opts1 = cpu_to_le32(opts1);
3993 
3994 	tp->tx_skb[entry].len = len;
3995 
3996 	return 0;
3997 }
3998 
3999 static int rtl8169_xmit_frags(struct rtl8169_private *tp, struct sk_buff *skb,
4000 			      const u32 *opts, unsigned int entry)
4001 {
4002 	struct skb_shared_info *info = skb_shinfo(skb);
4003 	unsigned int cur_frag;
4004 
4005 	for (cur_frag = 0; cur_frag < info->nr_frags; cur_frag++) {
4006 		const skb_frag_t *frag = info->frags + cur_frag;
4007 		void *addr = skb_frag_address(frag);
4008 		u32 len = skb_frag_size(frag);
4009 
4010 		entry = (entry + 1) % NUM_TX_DESC;
4011 
4012 		if (unlikely(rtl8169_tx_map(tp, opts, len, addr, entry, true)))
4013 			goto err_out;
4014 	}
4015 
4016 	return 0;
4017 
4018 err_out:
4019 	rtl8169_tx_clear_range(tp, tp->cur_tx + 1, cur_frag);
4020 	return -EIO;
4021 }
4022 
4023 static bool rtl_skb_is_udp(struct sk_buff *skb)
4024 {
4025 	int no = skb_network_offset(skb);
4026 	struct ipv6hdr *i6h, _i6h;
4027 	struct iphdr *ih, _ih;
4028 
4029 	switch (vlan_get_protocol(skb)) {
4030 	case htons(ETH_P_IP):
4031 		ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih);
4032 		return ih && ih->protocol == IPPROTO_UDP;
4033 	case htons(ETH_P_IPV6):
4034 		i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h);
4035 		return i6h && i6h->nexthdr == IPPROTO_UDP;
4036 	default:
4037 		return false;
4038 	}
4039 }
4040 
4041 #define RTL_MIN_PATCH_LEN	47
4042 
4043 /* see rtl8125_get_patch_pad_len() in r8125 vendor driver */
4044 static unsigned int rtl8125_quirk_udp_padto(struct rtl8169_private *tp,
4045 					    struct sk_buff *skb)
4046 {
4047 	unsigned int padto = 0, len = skb->len;
4048 
4049 	if (rtl_is_8125(tp) && len < 128 + RTL_MIN_PATCH_LEN &&
4050 	    rtl_skb_is_udp(skb) && skb_transport_header_was_set(skb)) {
4051 		unsigned int trans_data_len = skb_tail_pointer(skb) -
4052 					      skb_transport_header(skb);
4053 
4054 		if (trans_data_len >= offsetof(struct udphdr, len) &&
4055 		    trans_data_len < RTL_MIN_PATCH_LEN) {
4056 			u16 dest = ntohs(udp_hdr(skb)->dest);
4057 
4058 			/* dest is a standard PTP port */
4059 			if (dest == 319 || dest == 320)
4060 				padto = len + RTL_MIN_PATCH_LEN - trans_data_len;
4061 		}
4062 
4063 		if (trans_data_len < sizeof(struct udphdr))
4064 			padto = max_t(unsigned int, padto,
4065 				      len + sizeof(struct udphdr) - trans_data_len);
4066 	}
4067 
4068 	return padto;
4069 }
4070 
4071 static unsigned int rtl_quirk_packet_padto(struct rtl8169_private *tp,
4072 					   struct sk_buff *skb)
4073 {
4074 	unsigned int padto;
4075 
4076 	padto = rtl8125_quirk_udp_padto(tp, skb);
4077 
4078 	switch (tp->mac_version) {
4079 	case RTL_GIGA_MAC_VER_34:
4080 	case RTL_GIGA_MAC_VER_61:
4081 	case RTL_GIGA_MAC_VER_63:
4082 		padto = max_t(unsigned int, padto, ETH_ZLEN);
4083 		break;
4084 	default:
4085 		break;
4086 	}
4087 
4088 	return padto;
4089 }
4090 
4091 static void rtl8169_tso_csum_v1(struct sk_buff *skb, u32 *opts)
4092 {
4093 	u32 mss = skb_shinfo(skb)->gso_size;
4094 
4095 	if (mss) {
4096 		opts[0] |= TD_LSO;
4097 		opts[0] |= mss << TD0_MSS_SHIFT;
4098 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4099 		const struct iphdr *ip = ip_hdr(skb);
4100 
4101 		if (ip->protocol == IPPROTO_TCP)
4102 			opts[0] |= TD0_IP_CS | TD0_TCP_CS;
4103 		else if (ip->protocol == IPPROTO_UDP)
4104 			opts[0] |= TD0_IP_CS | TD0_UDP_CS;
4105 		else
4106 			WARN_ON_ONCE(1);
4107 	}
4108 }
4109 
4110 static bool rtl8169_tso_csum_v2(struct rtl8169_private *tp,
4111 				struct sk_buff *skb, u32 *opts)
4112 {
4113 	struct skb_shared_info *shinfo = skb_shinfo(skb);
4114 	u32 mss = shinfo->gso_size;
4115 
4116 	if (mss) {
4117 		if (shinfo->gso_type & SKB_GSO_TCPV4) {
4118 			opts[0] |= TD1_GTSENV4;
4119 		} else if (shinfo->gso_type & SKB_GSO_TCPV6) {
4120 			if (skb_cow_head(skb, 0))
4121 				return false;
4122 
4123 			tcp_v6_gso_csum_prep(skb);
4124 			opts[0] |= TD1_GTSENV6;
4125 		} else {
4126 			WARN_ON_ONCE(1);
4127 		}
4128 
4129 		opts[0] |= skb_transport_offset(skb) << GTTCPHO_SHIFT;
4130 		opts[1] |= mss << TD1_MSS_SHIFT;
4131 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4132 		u8 ip_protocol;
4133 
4134 		switch (vlan_get_protocol(skb)) {
4135 		case htons(ETH_P_IP):
4136 			opts[1] |= TD1_IPv4_CS;
4137 			ip_protocol = ip_hdr(skb)->protocol;
4138 			break;
4139 
4140 		case htons(ETH_P_IPV6):
4141 			opts[1] |= TD1_IPv6_CS;
4142 			ip_protocol = ipv6_hdr(skb)->nexthdr;
4143 			break;
4144 
4145 		default:
4146 			ip_protocol = IPPROTO_RAW;
4147 			break;
4148 		}
4149 
4150 		if (ip_protocol == IPPROTO_TCP)
4151 			opts[1] |= TD1_TCP_CS;
4152 		else if (ip_protocol == IPPROTO_UDP)
4153 			opts[1] |= TD1_UDP_CS;
4154 		else
4155 			WARN_ON_ONCE(1);
4156 
4157 		opts[1] |= skb_transport_offset(skb) << TCPHO_SHIFT;
4158 	} else {
4159 		unsigned int padto = rtl_quirk_packet_padto(tp, skb);
4160 
4161 		/* skb_padto would free the skb on error */
4162 		return !__skb_put_padto(skb, padto, false);
4163 	}
4164 
4165 	return true;
4166 }
4167 
4168 static unsigned int rtl_tx_slots_avail(struct rtl8169_private *tp)
4169 {
4170 	return READ_ONCE(tp->dirty_tx) + NUM_TX_DESC - READ_ONCE(tp->cur_tx);
4171 }
4172 
4173 /* Versions RTL8102e and from RTL8168c onwards support csum_v2 */
4174 static bool rtl_chip_supports_csum_v2(struct rtl8169_private *tp)
4175 {
4176 	switch (tp->mac_version) {
4177 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
4178 	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
4179 		return false;
4180 	default:
4181 		return true;
4182 	}
4183 }
4184 
4185 static void rtl8169_doorbell(struct rtl8169_private *tp)
4186 {
4187 	if (rtl_is_8125(tp))
4188 		RTL_W16(tp, TxPoll_8125, BIT(0));
4189 	else
4190 		RTL_W8(tp, TxPoll, NPQ);
4191 }
4192 
4193 static netdev_tx_t rtl8169_start_xmit(struct sk_buff *skb,
4194 				      struct net_device *dev)
4195 {
4196 	unsigned int frags = skb_shinfo(skb)->nr_frags;
4197 	struct rtl8169_private *tp = netdev_priv(dev);
4198 	unsigned int entry = tp->cur_tx % NUM_TX_DESC;
4199 	struct TxDesc *txd_first, *txd_last;
4200 	bool stop_queue, door_bell;
4201 	u32 opts[2];
4202 
4203 	if (unlikely(!rtl_tx_slots_avail(tp))) {
4204 		if (net_ratelimit())
4205 			netdev_err(dev, "BUG! Tx Ring full when queue awake!\n");
4206 		goto err_stop_0;
4207 	}
4208 
4209 	opts[1] = rtl8169_tx_vlan_tag(skb);
4210 	opts[0] = 0;
4211 
4212 	if (!rtl_chip_supports_csum_v2(tp))
4213 		rtl8169_tso_csum_v1(skb, opts);
4214 	else if (!rtl8169_tso_csum_v2(tp, skb, opts))
4215 		goto err_dma_0;
4216 
4217 	if (unlikely(rtl8169_tx_map(tp, opts, skb_headlen(skb), skb->data,
4218 				    entry, false)))
4219 		goto err_dma_0;
4220 
4221 	txd_first = tp->TxDescArray + entry;
4222 
4223 	if (frags) {
4224 		if (rtl8169_xmit_frags(tp, skb, opts, entry))
4225 			goto err_dma_1;
4226 		entry = (entry + frags) % NUM_TX_DESC;
4227 	}
4228 
4229 	txd_last = tp->TxDescArray + entry;
4230 	txd_last->opts1 |= cpu_to_le32(LastFrag);
4231 	tp->tx_skb[entry].skb = skb;
4232 
4233 	skb_tx_timestamp(skb);
4234 
4235 	/* Force memory writes to complete before releasing descriptor */
4236 	dma_wmb();
4237 
4238 	door_bell = __netdev_sent_queue(dev, skb->len, netdev_xmit_more());
4239 
4240 	txd_first->opts1 |= cpu_to_le32(DescOwn | FirstFrag);
4241 
4242 	/* rtl_tx needs to see descriptor changes before updated tp->cur_tx */
4243 	smp_wmb();
4244 
4245 	WRITE_ONCE(tp->cur_tx, tp->cur_tx + frags + 1);
4246 
4247 	stop_queue = !netif_subqueue_maybe_stop(dev, 0, rtl_tx_slots_avail(tp),
4248 						R8169_TX_STOP_THRS,
4249 						R8169_TX_START_THRS);
4250 	if (door_bell || stop_queue)
4251 		rtl8169_doorbell(tp);
4252 
4253 	return NETDEV_TX_OK;
4254 
4255 err_dma_1:
4256 	rtl8169_unmap_tx_skb(tp, entry);
4257 err_dma_0:
4258 	dev_kfree_skb_any(skb);
4259 	dev->stats.tx_dropped++;
4260 	return NETDEV_TX_OK;
4261 
4262 err_stop_0:
4263 	netif_stop_queue(dev);
4264 	dev->stats.tx_dropped++;
4265 	return NETDEV_TX_BUSY;
4266 }
4267 
4268 static unsigned int rtl_last_frag_len(struct sk_buff *skb)
4269 {
4270 	struct skb_shared_info *info = skb_shinfo(skb);
4271 	unsigned int nr_frags = info->nr_frags;
4272 
4273 	if (!nr_frags)
4274 		return UINT_MAX;
4275 
4276 	return skb_frag_size(info->frags + nr_frags - 1);
4277 }
4278 
4279 /* Workaround for hw issues with TSO on RTL8168evl */
4280 static netdev_features_t rtl8168evl_fix_tso(struct sk_buff *skb,
4281 					    netdev_features_t features)
4282 {
4283 	/* IPv4 header has options field */
4284 	if (vlan_get_protocol(skb) == htons(ETH_P_IP) &&
4285 	    ip_hdrlen(skb) > sizeof(struct iphdr))
4286 		features &= ~NETIF_F_ALL_TSO;
4287 
4288 	/* IPv4 TCP header has options field */
4289 	else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4 &&
4290 		 tcp_hdrlen(skb) > sizeof(struct tcphdr))
4291 		features &= ~NETIF_F_ALL_TSO;
4292 
4293 	else if (rtl_last_frag_len(skb) <= 6)
4294 		features &= ~NETIF_F_ALL_TSO;
4295 
4296 	return features;
4297 }
4298 
4299 static netdev_features_t rtl8169_features_check(struct sk_buff *skb,
4300 						struct net_device *dev,
4301 						netdev_features_t features)
4302 {
4303 	struct rtl8169_private *tp = netdev_priv(dev);
4304 
4305 	if (skb_is_gso(skb)) {
4306 		if (tp->mac_version == RTL_GIGA_MAC_VER_34)
4307 			features = rtl8168evl_fix_tso(skb, features);
4308 
4309 		if (skb_transport_offset(skb) > GTTCPHO_MAX &&
4310 		    rtl_chip_supports_csum_v2(tp))
4311 			features &= ~NETIF_F_ALL_TSO;
4312 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4313 		/* work around hw bug on some chip versions */
4314 		if (skb->len < ETH_ZLEN)
4315 			features &= ~NETIF_F_CSUM_MASK;
4316 
4317 		if (rtl_quirk_packet_padto(tp, skb))
4318 			features &= ~NETIF_F_CSUM_MASK;
4319 
4320 		if (skb_transport_offset(skb) > TCPHO_MAX &&
4321 		    rtl_chip_supports_csum_v2(tp))
4322 			features &= ~NETIF_F_CSUM_MASK;
4323 	}
4324 
4325 	return vlan_features_check(skb, features);
4326 }
4327 
4328 static void rtl8169_pcierr_interrupt(struct net_device *dev)
4329 {
4330 	struct rtl8169_private *tp = netdev_priv(dev);
4331 	struct pci_dev *pdev = tp->pci_dev;
4332 	int pci_status_errs;
4333 	u16 pci_cmd;
4334 
4335 	pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
4336 
4337 	pci_status_errs = pci_status_get_and_clear_errors(pdev);
4338 
4339 	if (net_ratelimit())
4340 		netdev_err(dev, "PCI error (cmd = 0x%04x, status_errs = 0x%04x)\n",
4341 			   pci_cmd, pci_status_errs);
4342 
4343 	rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
4344 }
4345 
4346 static void rtl_tx(struct net_device *dev, struct rtl8169_private *tp,
4347 		   int budget)
4348 {
4349 	unsigned int dirty_tx, bytes_compl = 0, pkts_compl = 0;
4350 	struct sk_buff *skb;
4351 
4352 	dirty_tx = tp->dirty_tx;
4353 
4354 	while (READ_ONCE(tp->cur_tx) != dirty_tx) {
4355 		unsigned int entry = dirty_tx % NUM_TX_DESC;
4356 		u32 status;
4357 
4358 		status = le32_to_cpu(tp->TxDescArray[entry].opts1);
4359 		if (status & DescOwn)
4360 			break;
4361 
4362 		skb = tp->tx_skb[entry].skb;
4363 		rtl8169_unmap_tx_skb(tp, entry);
4364 
4365 		if (skb) {
4366 			pkts_compl++;
4367 			bytes_compl += skb->len;
4368 			napi_consume_skb(skb, budget);
4369 		}
4370 		dirty_tx++;
4371 	}
4372 
4373 	if (tp->dirty_tx != dirty_tx) {
4374 		dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl);
4375 		WRITE_ONCE(tp->dirty_tx, dirty_tx);
4376 
4377 		netif_subqueue_completed_wake(dev, 0, pkts_compl, bytes_compl,
4378 					      rtl_tx_slots_avail(tp),
4379 					      R8169_TX_START_THRS);
4380 		/*
4381 		 * 8168 hack: TxPoll requests are lost when the Tx packets are
4382 		 * too close. Let's kick an extra TxPoll request when a burst
4383 		 * of start_xmit activity is detected (if it is not detected,
4384 		 * it is slow enough). -- FR
4385 		 * If skb is NULL then we come here again once a tx irq is
4386 		 * triggered after the last fragment is marked transmitted.
4387 		 */
4388 		if (tp->cur_tx != dirty_tx && skb)
4389 			rtl8169_doorbell(tp);
4390 	}
4391 }
4392 
4393 static inline int rtl8169_fragmented_frame(u32 status)
4394 {
4395 	return (status & (FirstFrag | LastFrag)) != (FirstFrag | LastFrag);
4396 }
4397 
4398 static inline void rtl8169_rx_csum(struct sk_buff *skb, u32 opts1)
4399 {
4400 	u32 status = opts1 & (RxProtoMask | RxCSFailMask);
4401 
4402 	if (status == RxProtoTCP || status == RxProtoUDP)
4403 		skb->ip_summed = CHECKSUM_UNNECESSARY;
4404 	else
4405 		skb_checksum_none_assert(skb);
4406 }
4407 
4408 static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget)
4409 {
4410 	struct device *d = tp_to_dev(tp);
4411 	int count;
4412 
4413 	for (count = 0; count < budget; count++, tp->cur_rx++) {
4414 		unsigned int pkt_size, entry = tp->cur_rx % NUM_RX_DESC;
4415 		struct RxDesc *desc = tp->RxDescArray + entry;
4416 		struct sk_buff *skb;
4417 		const void *rx_buf;
4418 		dma_addr_t addr;
4419 		u32 status;
4420 
4421 		status = le32_to_cpu(desc->opts1);
4422 		if (status & DescOwn)
4423 			break;
4424 
4425 		/* This barrier is needed to keep us from reading
4426 		 * any other fields out of the Rx descriptor until
4427 		 * we know the status of DescOwn
4428 		 */
4429 		dma_rmb();
4430 
4431 		if (unlikely(status & RxRES)) {
4432 			if (net_ratelimit())
4433 				netdev_warn(dev, "Rx ERROR. status = %08x\n",
4434 					    status);
4435 			dev->stats.rx_errors++;
4436 			if (status & (RxRWT | RxRUNT))
4437 				dev->stats.rx_length_errors++;
4438 			if (status & RxCRC)
4439 				dev->stats.rx_crc_errors++;
4440 
4441 			if (!(dev->features & NETIF_F_RXALL))
4442 				goto release_descriptor;
4443 			else if (status & RxRWT || !(status & (RxRUNT | RxCRC)))
4444 				goto release_descriptor;
4445 		}
4446 
4447 		pkt_size = status & GENMASK(13, 0);
4448 		if (likely(!(dev->features & NETIF_F_RXFCS)))
4449 			pkt_size -= ETH_FCS_LEN;
4450 
4451 		/* The driver does not support incoming fragmented frames.
4452 		 * They are seen as a symptom of over-mtu sized frames.
4453 		 */
4454 		if (unlikely(rtl8169_fragmented_frame(status))) {
4455 			dev->stats.rx_dropped++;
4456 			dev->stats.rx_length_errors++;
4457 			goto release_descriptor;
4458 		}
4459 
4460 		skb = napi_alloc_skb(&tp->napi, pkt_size);
4461 		if (unlikely(!skb)) {
4462 			dev->stats.rx_dropped++;
4463 			goto release_descriptor;
4464 		}
4465 
4466 		addr = le64_to_cpu(desc->addr);
4467 		rx_buf = page_address(tp->Rx_databuff[entry]);
4468 
4469 		dma_sync_single_for_cpu(d, addr, pkt_size, DMA_FROM_DEVICE);
4470 		prefetch(rx_buf);
4471 		skb_copy_to_linear_data(skb, rx_buf, pkt_size);
4472 		skb->tail += pkt_size;
4473 		skb->len = pkt_size;
4474 		dma_sync_single_for_device(d, addr, pkt_size, DMA_FROM_DEVICE);
4475 
4476 		rtl8169_rx_csum(skb, status);
4477 		skb->protocol = eth_type_trans(skb, dev);
4478 
4479 		rtl8169_rx_vlan_tag(desc, skb);
4480 
4481 		if (skb->pkt_type == PACKET_MULTICAST)
4482 			dev->stats.multicast++;
4483 
4484 		napi_gro_receive(&tp->napi, skb);
4485 
4486 		dev_sw_netstats_rx_add(dev, pkt_size);
4487 release_descriptor:
4488 		rtl8169_mark_to_asic(desc);
4489 	}
4490 
4491 	return count;
4492 }
4493 
4494 static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance)
4495 {
4496 	struct rtl8169_private *tp = dev_instance;
4497 	u32 status = rtl_get_events(tp);
4498 
4499 	if ((status & 0xffff) == 0xffff || !(status & tp->irq_mask))
4500 		return IRQ_NONE;
4501 
4502 	if (unlikely(status & SYSErr)) {
4503 		rtl8169_pcierr_interrupt(tp->dev);
4504 		goto out;
4505 	}
4506 
4507 	if (status & LinkChg)
4508 		phy_mac_interrupt(tp->phydev);
4509 
4510 	if (unlikely(status & RxFIFOOver &&
4511 	    tp->mac_version == RTL_GIGA_MAC_VER_11)) {
4512 		netif_stop_queue(tp->dev);
4513 		rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
4514 	}
4515 
4516 	if (napi_schedule_prep(&tp->napi)) {
4517 		rtl_unlock_config_regs(tp);
4518 		rtl_hw_aspm_clkreq_enable(tp, false);
4519 		rtl_lock_config_regs(tp);
4520 
4521 		rtl_irq_disable(tp);
4522 		__napi_schedule(&tp->napi);
4523 	}
4524 out:
4525 	rtl_ack_events(tp, status);
4526 
4527 	return IRQ_HANDLED;
4528 }
4529 
4530 static void rtl_task(struct work_struct *work)
4531 {
4532 	struct rtl8169_private *tp =
4533 		container_of(work, struct rtl8169_private, wk.work);
4534 	int ret;
4535 
4536 	rtnl_lock();
4537 
4538 	if (!netif_running(tp->dev) ||
4539 	    !test_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags))
4540 		goto out_unlock;
4541 
4542 	if (test_and_clear_bit(RTL_FLAG_TASK_TX_TIMEOUT, tp->wk.flags)) {
4543 		/* if chip isn't accessible, reset bus to revive it */
4544 		if (RTL_R32(tp, TxConfig) == ~0) {
4545 			ret = pci_reset_bus(tp->pci_dev);
4546 			if (ret < 0) {
4547 				netdev_err(tp->dev, "Can't reset secondary PCI bus, detach NIC\n");
4548 				netif_device_detach(tp->dev);
4549 				goto out_unlock;
4550 			}
4551 		}
4552 
4553 		/* ASPM compatibility issues are a typical reason for tx timeouts */
4554 		ret = pci_disable_link_state(tp->pci_dev, PCIE_LINK_STATE_L1 |
4555 							  PCIE_LINK_STATE_L0S);
4556 		if (!ret)
4557 			netdev_warn_once(tp->dev, "ASPM disabled on Tx timeout\n");
4558 		goto reset;
4559 	}
4560 
4561 	if (test_and_clear_bit(RTL_FLAG_TASK_RESET_PENDING, tp->wk.flags)) {
4562 reset:
4563 		rtl_reset_work(tp);
4564 		netif_wake_queue(tp->dev);
4565 	}
4566 out_unlock:
4567 	rtnl_unlock();
4568 }
4569 
4570 static int rtl8169_poll(struct napi_struct *napi, int budget)
4571 {
4572 	struct rtl8169_private *tp = container_of(napi, struct rtl8169_private, napi);
4573 	struct net_device *dev = tp->dev;
4574 	int work_done;
4575 
4576 	rtl_tx(dev, tp, budget);
4577 
4578 	work_done = rtl_rx(dev, tp, budget);
4579 
4580 	if (work_done < budget && napi_complete_done(napi, work_done)) {
4581 		rtl_irq_enable(tp);
4582 
4583 		rtl_unlock_config_regs(tp);
4584 		rtl_hw_aspm_clkreq_enable(tp, true);
4585 		rtl_lock_config_regs(tp);
4586 	}
4587 
4588 	return work_done;
4589 }
4590 
4591 static void r8169_phylink_handler(struct net_device *ndev)
4592 {
4593 	struct rtl8169_private *tp = netdev_priv(ndev);
4594 	struct device *d = tp_to_dev(tp);
4595 
4596 	if (netif_carrier_ok(ndev)) {
4597 		rtl_link_chg_patch(tp);
4598 		pm_request_resume(d);
4599 	} else {
4600 		pm_runtime_idle(d);
4601 	}
4602 
4603 	phy_print_status(tp->phydev);
4604 }
4605 
4606 static int r8169_phy_connect(struct rtl8169_private *tp)
4607 {
4608 	struct phy_device *phydev = tp->phydev;
4609 	phy_interface_t phy_mode;
4610 	int ret;
4611 
4612 	phy_mode = tp->supports_gmii ? PHY_INTERFACE_MODE_GMII :
4613 		   PHY_INTERFACE_MODE_MII;
4614 
4615 	ret = phy_connect_direct(tp->dev, phydev, r8169_phylink_handler,
4616 				 phy_mode);
4617 	if (ret)
4618 		return ret;
4619 
4620 	if (!tp->supports_gmii)
4621 		phy_set_max_speed(phydev, SPEED_100);
4622 
4623 	phy_attached_info(phydev);
4624 
4625 	return 0;
4626 }
4627 
4628 static void rtl8169_down(struct rtl8169_private *tp)
4629 {
4630 	/* Clear all task flags */
4631 	bitmap_zero(tp->wk.flags, RTL_FLAG_MAX);
4632 
4633 	phy_stop(tp->phydev);
4634 
4635 	rtl8169_update_counters(tp);
4636 
4637 	pci_clear_master(tp->pci_dev);
4638 	rtl_pci_commit(tp);
4639 
4640 	rtl8169_cleanup(tp);
4641 	rtl_disable_exit_l1(tp);
4642 	rtl_prepare_power_down(tp);
4643 }
4644 
4645 static void rtl8169_up(struct rtl8169_private *tp)
4646 {
4647 	pci_set_master(tp->pci_dev);
4648 	phy_init_hw(tp->phydev);
4649 	phy_resume(tp->phydev);
4650 	rtl8169_init_phy(tp);
4651 	napi_enable(&tp->napi);
4652 	set_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags);
4653 	rtl_reset_work(tp);
4654 
4655 	phy_start(tp->phydev);
4656 }
4657 
4658 static int rtl8169_close(struct net_device *dev)
4659 {
4660 	struct rtl8169_private *tp = netdev_priv(dev);
4661 	struct pci_dev *pdev = tp->pci_dev;
4662 
4663 	pm_runtime_get_sync(&pdev->dev);
4664 
4665 	netif_stop_queue(dev);
4666 	rtl8169_down(tp);
4667 	rtl8169_rx_clear(tp);
4668 
4669 	cancel_work_sync(&tp->wk.work);
4670 
4671 	free_irq(tp->irq, tp);
4672 
4673 	phy_disconnect(tp->phydev);
4674 
4675 	dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray,
4676 			  tp->RxPhyAddr);
4677 	dma_free_coherent(&pdev->dev, R8169_TX_RING_BYTES, tp->TxDescArray,
4678 			  tp->TxPhyAddr);
4679 	tp->TxDescArray = NULL;
4680 	tp->RxDescArray = NULL;
4681 
4682 	pm_runtime_put_sync(&pdev->dev);
4683 
4684 	return 0;
4685 }
4686 
4687 #ifdef CONFIG_NET_POLL_CONTROLLER
4688 static void rtl8169_netpoll(struct net_device *dev)
4689 {
4690 	struct rtl8169_private *tp = netdev_priv(dev);
4691 
4692 	rtl8169_interrupt(tp->irq, tp);
4693 }
4694 #endif
4695 
4696 static int rtl_open(struct net_device *dev)
4697 {
4698 	struct rtl8169_private *tp = netdev_priv(dev);
4699 	struct pci_dev *pdev = tp->pci_dev;
4700 	unsigned long irqflags;
4701 	int retval = -ENOMEM;
4702 
4703 	pm_runtime_get_sync(&pdev->dev);
4704 
4705 	/*
4706 	 * Rx and Tx descriptors needs 256 bytes alignment.
4707 	 * dma_alloc_coherent provides more.
4708 	 */
4709 	tp->TxDescArray = dma_alloc_coherent(&pdev->dev, R8169_TX_RING_BYTES,
4710 					     &tp->TxPhyAddr, GFP_KERNEL);
4711 	if (!tp->TxDescArray)
4712 		goto out;
4713 
4714 	tp->RxDescArray = dma_alloc_coherent(&pdev->dev, R8169_RX_RING_BYTES,
4715 					     &tp->RxPhyAddr, GFP_KERNEL);
4716 	if (!tp->RxDescArray)
4717 		goto err_free_tx_0;
4718 
4719 	retval = rtl8169_init_ring(tp);
4720 	if (retval < 0)
4721 		goto err_free_rx_1;
4722 
4723 	rtl_request_firmware(tp);
4724 
4725 	irqflags = pci_dev_msi_enabled(pdev) ? IRQF_NO_THREAD : IRQF_SHARED;
4726 	retval = request_irq(tp->irq, rtl8169_interrupt, irqflags, dev->name, tp);
4727 	if (retval < 0)
4728 		goto err_release_fw_2;
4729 
4730 	retval = r8169_phy_connect(tp);
4731 	if (retval)
4732 		goto err_free_irq;
4733 
4734 	rtl8169_up(tp);
4735 	rtl8169_init_counter_offsets(tp);
4736 	netif_start_queue(dev);
4737 out:
4738 	pm_runtime_put_sync(&pdev->dev);
4739 
4740 	return retval;
4741 
4742 err_free_irq:
4743 	free_irq(tp->irq, tp);
4744 err_release_fw_2:
4745 	rtl_release_firmware(tp);
4746 	rtl8169_rx_clear(tp);
4747 err_free_rx_1:
4748 	dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray,
4749 			  tp->RxPhyAddr);
4750 	tp->RxDescArray = NULL;
4751 err_free_tx_0:
4752 	dma_free_coherent(&pdev->dev, R8169_TX_RING_BYTES, tp->TxDescArray,
4753 			  tp->TxPhyAddr);
4754 	tp->TxDescArray = NULL;
4755 	goto out;
4756 }
4757 
4758 static void
4759 rtl8169_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
4760 {
4761 	struct rtl8169_private *tp = netdev_priv(dev);
4762 	struct pci_dev *pdev = tp->pci_dev;
4763 	struct rtl8169_counters *counters = tp->counters;
4764 
4765 	pm_runtime_get_noresume(&pdev->dev);
4766 
4767 	netdev_stats_to_stats64(stats, &dev->stats);
4768 	dev_fetch_sw_netstats(stats, dev->tstats);
4769 
4770 	/*
4771 	 * Fetch additional counter values missing in stats collected by driver
4772 	 * from tally counters.
4773 	 */
4774 	if (pm_runtime_active(&pdev->dev))
4775 		rtl8169_update_counters(tp);
4776 
4777 	/*
4778 	 * Subtract values fetched during initalization.
4779 	 * See rtl8169_init_counter_offsets for a description why we do that.
4780 	 */
4781 	stats->tx_errors = le64_to_cpu(counters->tx_errors) -
4782 		le64_to_cpu(tp->tc_offset.tx_errors);
4783 	stats->collisions = le32_to_cpu(counters->tx_multi_collision) -
4784 		le32_to_cpu(tp->tc_offset.tx_multi_collision);
4785 	stats->tx_aborted_errors = le16_to_cpu(counters->tx_aborted) -
4786 		le16_to_cpu(tp->tc_offset.tx_aborted);
4787 	stats->rx_missed_errors = le16_to_cpu(counters->rx_missed) -
4788 		le16_to_cpu(tp->tc_offset.rx_missed);
4789 
4790 	pm_runtime_put_noidle(&pdev->dev);
4791 }
4792 
4793 static void rtl8169_net_suspend(struct rtl8169_private *tp)
4794 {
4795 	netif_device_detach(tp->dev);
4796 
4797 	if (netif_running(tp->dev))
4798 		rtl8169_down(tp);
4799 }
4800 
4801 static int rtl8169_runtime_resume(struct device *dev)
4802 {
4803 	struct rtl8169_private *tp = dev_get_drvdata(dev);
4804 
4805 	rtl_rar_set(tp, tp->dev->dev_addr);
4806 	__rtl8169_set_wol(tp, tp->saved_wolopts);
4807 
4808 	if (tp->TxDescArray)
4809 		rtl8169_up(tp);
4810 
4811 	netif_device_attach(tp->dev);
4812 
4813 	return 0;
4814 }
4815 
4816 static int rtl8169_suspend(struct device *device)
4817 {
4818 	struct rtl8169_private *tp = dev_get_drvdata(device);
4819 
4820 	rtnl_lock();
4821 	rtl8169_net_suspend(tp);
4822 	if (!device_may_wakeup(tp_to_dev(tp)))
4823 		clk_disable_unprepare(tp->clk);
4824 	rtnl_unlock();
4825 
4826 	return 0;
4827 }
4828 
4829 static int rtl8169_resume(struct device *device)
4830 {
4831 	struct rtl8169_private *tp = dev_get_drvdata(device);
4832 
4833 	if (!device_may_wakeup(tp_to_dev(tp)))
4834 		clk_prepare_enable(tp->clk);
4835 
4836 	/* Reportedly at least Asus X453MA truncates packets otherwise */
4837 	if (tp->mac_version == RTL_GIGA_MAC_VER_37)
4838 		rtl_init_rxcfg(tp);
4839 
4840 	return rtl8169_runtime_resume(device);
4841 }
4842 
4843 static int rtl8169_runtime_suspend(struct device *device)
4844 {
4845 	struct rtl8169_private *tp = dev_get_drvdata(device);
4846 
4847 	if (!tp->TxDescArray) {
4848 		netif_device_detach(tp->dev);
4849 		return 0;
4850 	}
4851 
4852 	rtnl_lock();
4853 	__rtl8169_set_wol(tp, WAKE_PHY);
4854 	rtl8169_net_suspend(tp);
4855 	rtnl_unlock();
4856 
4857 	return 0;
4858 }
4859 
4860 static int rtl8169_runtime_idle(struct device *device)
4861 {
4862 	struct rtl8169_private *tp = dev_get_drvdata(device);
4863 
4864 	if (tp->dash_type != RTL_DASH_NONE)
4865 		return -EBUSY;
4866 
4867 	if (!netif_running(tp->dev) || !netif_carrier_ok(tp->dev))
4868 		pm_schedule_suspend(device, 10000);
4869 
4870 	return -EBUSY;
4871 }
4872 
4873 static const struct dev_pm_ops rtl8169_pm_ops = {
4874 	SYSTEM_SLEEP_PM_OPS(rtl8169_suspend, rtl8169_resume)
4875 	RUNTIME_PM_OPS(rtl8169_runtime_suspend, rtl8169_runtime_resume,
4876 		       rtl8169_runtime_idle)
4877 };
4878 
4879 static void rtl_shutdown(struct pci_dev *pdev)
4880 {
4881 	struct rtl8169_private *tp = pci_get_drvdata(pdev);
4882 
4883 	rtnl_lock();
4884 	rtl8169_net_suspend(tp);
4885 	rtnl_unlock();
4886 
4887 	/* Restore original MAC address */
4888 	rtl_rar_set(tp, tp->dev->perm_addr);
4889 
4890 	if (system_state == SYSTEM_POWER_OFF &&
4891 	    tp->dash_type == RTL_DASH_NONE) {
4892 		pci_wake_from_d3(pdev, tp->saved_wolopts);
4893 		pci_set_power_state(pdev, PCI_D3hot);
4894 	}
4895 }
4896 
4897 static void rtl_remove_one(struct pci_dev *pdev)
4898 {
4899 	struct rtl8169_private *tp = pci_get_drvdata(pdev);
4900 
4901 	if (pci_dev_run_wake(pdev))
4902 		pm_runtime_get_noresume(&pdev->dev);
4903 
4904 	unregister_netdev(tp->dev);
4905 
4906 	if (tp->dash_type != RTL_DASH_NONE)
4907 		rtl8168_driver_stop(tp);
4908 
4909 	rtl_release_firmware(tp);
4910 
4911 	/* restore original MAC address */
4912 	rtl_rar_set(tp, tp->dev->perm_addr);
4913 }
4914 
4915 static const struct net_device_ops rtl_netdev_ops = {
4916 	.ndo_open		= rtl_open,
4917 	.ndo_stop		= rtl8169_close,
4918 	.ndo_get_stats64	= rtl8169_get_stats64,
4919 	.ndo_start_xmit		= rtl8169_start_xmit,
4920 	.ndo_features_check	= rtl8169_features_check,
4921 	.ndo_tx_timeout		= rtl8169_tx_timeout,
4922 	.ndo_validate_addr	= eth_validate_addr,
4923 	.ndo_change_mtu		= rtl8169_change_mtu,
4924 	.ndo_fix_features	= rtl8169_fix_features,
4925 	.ndo_set_features	= rtl8169_set_features,
4926 	.ndo_set_mac_address	= rtl_set_mac_address,
4927 	.ndo_eth_ioctl		= phy_do_ioctl_running,
4928 	.ndo_set_rx_mode	= rtl_set_rx_mode,
4929 #ifdef CONFIG_NET_POLL_CONTROLLER
4930 	.ndo_poll_controller	= rtl8169_netpoll,
4931 #endif
4932 
4933 };
4934 
4935 static void rtl_set_irq_mask(struct rtl8169_private *tp)
4936 {
4937 	tp->irq_mask = RxOK | RxErr | TxOK | TxErr | LinkChg;
4938 
4939 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
4940 		tp->irq_mask |= SYSErr | RxOverflow | RxFIFOOver;
4941 	else if (tp->mac_version == RTL_GIGA_MAC_VER_11)
4942 		/* special workaround needed */
4943 		tp->irq_mask |= RxFIFOOver;
4944 	else
4945 		tp->irq_mask |= RxOverflow;
4946 }
4947 
4948 static int rtl_alloc_irq(struct rtl8169_private *tp)
4949 {
4950 	unsigned int flags;
4951 
4952 	switch (tp->mac_version) {
4953 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
4954 		rtl_unlock_config_regs(tp);
4955 		RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~MSIEnable);
4956 		rtl_lock_config_regs(tp);
4957 		fallthrough;
4958 	case RTL_GIGA_MAC_VER_07 ... RTL_GIGA_MAC_VER_17:
4959 		flags = PCI_IRQ_LEGACY;
4960 		break;
4961 	default:
4962 		flags = PCI_IRQ_ALL_TYPES;
4963 		break;
4964 	}
4965 
4966 	return pci_alloc_irq_vectors(tp->pci_dev, 1, 1, flags);
4967 }
4968 
4969 static void rtl_read_mac_address(struct rtl8169_private *tp,
4970 				 u8 mac_addr[ETH_ALEN])
4971 {
4972 	/* Get MAC address */
4973 	if (rtl_is_8168evl_up(tp) && tp->mac_version != RTL_GIGA_MAC_VER_34) {
4974 		u32 value;
4975 
4976 		value = rtl_eri_read(tp, 0xe0);
4977 		put_unaligned_le32(value, mac_addr);
4978 		value = rtl_eri_read(tp, 0xe4);
4979 		put_unaligned_le16(value, mac_addr + 4);
4980 	} else if (rtl_is_8125(tp)) {
4981 		rtl_read_mac_from_reg(tp, mac_addr, MAC0_BKP);
4982 	}
4983 }
4984 
4985 DECLARE_RTL_COND(rtl_link_list_ready_cond)
4986 {
4987 	return RTL_R8(tp, MCU) & LINK_LIST_RDY;
4988 }
4989 
4990 static void r8168g_wait_ll_share_fifo_ready(struct rtl8169_private *tp)
4991 {
4992 	rtl_loop_wait_high(tp, &rtl_link_list_ready_cond, 100, 42);
4993 }
4994 
4995 static int r8169_mdio_read_reg(struct mii_bus *mii_bus, int phyaddr, int phyreg)
4996 {
4997 	struct rtl8169_private *tp = mii_bus->priv;
4998 
4999 	if (phyaddr > 0)
5000 		return -ENODEV;
5001 
5002 	return rtl_readphy(tp, phyreg);
5003 }
5004 
5005 static int r8169_mdio_write_reg(struct mii_bus *mii_bus, int phyaddr,
5006 				int phyreg, u16 val)
5007 {
5008 	struct rtl8169_private *tp = mii_bus->priv;
5009 
5010 	if (phyaddr > 0)
5011 		return -ENODEV;
5012 
5013 	rtl_writephy(tp, phyreg, val);
5014 
5015 	return 0;
5016 }
5017 
5018 static int r8169_mdio_register(struct rtl8169_private *tp)
5019 {
5020 	struct pci_dev *pdev = tp->pci_dev;
5021 	struct mii_bus *new_bus;
5022 	int ret;
5023 
5024 	new_bus = devm_mdiobus_alloc(&pdev->dev);
5025 	if (!new_bus)
5026 		return -ENOMEM;
5027 
5028 	new_bus->name = "r8169";
5029 	new_bus->priv = tp;
5030 	new_bus->parent = &pdev->dev;
5031 	new_bus->irq[0] = PHY_MAC_INTERRUPT;
5032 	snprintf(new_bus->id, MII_BUS_ID_SIZE, "r8169-%x-%x",
5033 		 pci_domain_nr(pdev->bus), pci_dev_id(pdev));
5034 
5035 	new_bus->read = r8169_mdio_read_reg;
5036 	new_bus->write = r8169_mdio_write_reg;
5037 
5038 	ret = devm_mdiobus_register(&pdev->dev, new_bus);
5039 	if (ret)
5040 		return ret;
5041 
5042 	tp->phydev = mdiobus_get_phy(new_bus, 0);
5043 	if (!tp->phydev) {
5044 		return -ENODEV;
5045 	} else if (!tp->phydev->drv) {
5046 		/* Most chip versions fail with the genphy driver.
5047 		 * Therefore ensure that the dedicated PHY driver is loaded.
5048 		 */
5049 		dev_err(&pdev->dev, "no dedicated PHY driver found for PHY ID 0x%08x, maybe realtek.ko needs to be added to initramfs?\n",
5050 			tp->phydev->phy_id);
5051 		return -EUNATCH;
5052 	}
5053 
5054 	tp->phydev->mac_managed_pm = true;
5055 
5056 	phy_support_asym_pause(tp->phydev);
5057 
5058 	/* PHY will be woken up in rtl_open() */
5059 	phy_suspend(tp->phydev);
5060 
5061 	return 0;
5062 }
5063 
5064 static void rtl_hw_init_8168g(struct rtl8169_private *tp)
5065 {
5066 	rtl_enable_rxdvgate(tp);
5067 
5068 	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & ~(CmdTxEnb | CmdRxEnb));
5069 	msleep(1);
5070 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
5071 
5072 	r8168_mac_ocp_modify(tp, 0xe8de, BIT(14), 0);
5073 	r8168g_wait_ll_share_fifo_ready(tp);
5074 
5075 	r8168_mac_ocp_modify(tp, 0xe8de, 0, BIT(15));
5076 	r8168g_wait_ll_share_fifo_ready(tp);
5077 }
5078 
5079 static void rtl_hw_init_8125(struct rtl8169_private *tp)
5080 {
5081 	rtl_enable_rxdvgate(tp);
5082 
5083 	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & ~(CmdTxEnb | CmdRxEnb));
5084 	msleep(1);
5085 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
5086 
5087 	r8168_mac_ocp_modify(tp, 0xe8de, BIT(14), 0);
5088 	r8168g_wait_ll_share_fifo_ready(tp);
5089 
5090 	r8168_mac_ocp_write(tp, 0xc0aa, 0x07d0);
5091 	r8168_mac_ocp_write(tp, 0xc0a6, 0x0150);
5092 	r8168_mac_ocp_write(tp, 0xc01e, 0x5555);
5093 	r8168g_wait_ll_share_fifo_ready(tp);
5094 }
5095 
5096 static void rtl_hw_initialize(struct rtl8169_private *tp)
5097 {
5098 	switch (tp->mac_version) {
5099 	case RTL_GIGA_MAC_VER_51 ... RTL_GIGA_MAC_VER_53:
5100 		rtl8168ep_stop_cmac(tp);
5101 		fallthrough;
5102 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_48:
5103 		rtl_hw_init_8168g(tp);
5104 		break;
5105 	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63:
5106 		rtl_hw_init_8125(tp);
5107 		break;
5108 	default:
5109 		break;
5110 	}
5111 }
5112 
5113 static int rtl_jumbo_max(struct rtl8169_private *tp)
5114 {
5115 	/* Non-GBit versions don't support jumbo frames */
5116 	if (!tp->supports_gmii)
5117 		return 0;
5118 
5119 	switch (tp->mac_version) {
5120 	/* RTL8169 */
5121 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
5122 		return JUMBO_7K;
5123 	/* RTL8168b */
5124 	case RTL_GIGA_MAC_VER_11:
5125 	case RTL_GIGA_MAC_VER_17:
5126 		return JUMBO_4K;
5127 	/* RTL8168c */
5128 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
5129 		return JUMBO_6K;
5130 	default:
5131 		return JUMBO_9K;
5132 	}
5133 }
5134 
5135 static void rtl_init_mac_address(struct rtl8169_private *tp)
5136 {
5137 	u8 mac_addr[ETH_ALEN] __aligned(2) = {};
5138 	struct net_device *dev = tp->dev;
5139 	int rc;
5140 
5141 	rc = eth_platform_get_mac_address(tp_to_dev(tp), mac_addr);
5142 	if (!rc)
5143 		goto done;
5144 
5145 	rtl_read_mac_address(tp, mac_addr);
5146 	if (is_valid_ether_addr(mac_addr))
5147 		goto done;
5148 
5149 	rtl_read_mac_from_reg(tp, mac_addr, MAC0);
5150 	if (is_valid_ether_addr(mac_addr))
5151 		goto done;
5152 
5153 	eth_random_addr(mac_addr);
5154 	dev->addr_assign_type = NET_ADDR_RANDOM;
5155 	dev_warn(tp_to_dev(tp), "can't read MAC address, setting random one\n");
5156 done:
5157 	eth_hw_addr_set(dev, mac_addr);
5158 	rtl_rar_set(tp, mac_addr);
5159 }
5160 
5161 static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5162 {
5163 	struct rtl8169_private *tp;
5164 	int jumbo_max, region, rc;
5165 	enum mac_version chipset;
5166 	struct net_device *dev;
5167 	u32 txconfig;
5168 	u16 xid;
5169 
5170 	dev = devm_alloc_etherdev(&pdev->dev, sizeof (*tp));
5171 	if (!dev)
5172 		return -ENOMEM;
5173 
5174 	SET_NETDEV_DEV(dev, &pdev->dev);
5175 	dev->netdev_ops = &rtl_netdev_ops;
5176 	tp = netdev_priv(dev);
5177 	tp->dev = dev;
5178 	tp->pci_dev = pdev;
5179 	tp->supports_gmii = ent->driver_data == RTL_CFG_NO_GBIT ? 0 : 1;
5180 	tp->eee_adv = -1;
5181 	tp->ocp_base = OCP_STD_PHY_BASE;
5182 
5183 	raw_spin_lock_init(&tp->cfg9346_usage_lock);
5184 	raw_spin_lock_init(&tp->config25_lock);
5185 	raw_spin_lock_init(&tp->mac_ocp_lock);
5186 
5187 	dev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev,
5188 						   struct pcpu_sw_netstats);
5189 	if (!dev->tstats)
5190 		return -ENOMEM;
5191 
5192 	/* Get the *optional* external "ether_clk" used on some boards */
5193 	tp->clk = devm_clk_get_optional_enabled(&pdev->dev, "ether_clk");
5194 	if (IS_ERR(tp->clk))
5195 		return dev_err_probe(&pdev->dev, PTR_ERR(tp->clk), "failed to get ether_clk\n");
5196 
5197 	/* enable device (incl. PCI PM wakeup and hotplug setup) */
5198 	rc = pcim_enable_device(pdev);
5199 	if (rc < 0)
5200 		return dev_err_probe(&pdev->dev, rc, "enable failure\n");
5201 
5202 	if (pcim_set_mwi(pdev) < 0)
5203 		dev_info(&pdev->dev, "Mem-Wr-Inval unavailable\n");
5204 
5205 	/* use first MMIO region */
5206 	region = ffs(pci_select_bars(pdev, IORESOURCE_MEM)) - 1;
5207 	if (region < 0)
5208 		return dev_err_probe(&pdev->dev, -ENODEV, "no MMIO resource found\n");
5209 
5210 	rc = pcim_iomap_regions(pdev, BIT(region), KBUILD_MODNAME);
5211 	if (rc < 0)
5212 		return dev_err_probe(&pdev->dev, rc, "cannot remap MMIO, aborting\n");
5213 
5214 	tp->mmio_addr = pcim_iomap_table(pdev)[region];
5215 
5216 	txconfig = RTL_R32(tp, TxConfig);
5217 	if (txconfig == ~0U)
5218 		return dev_err_probe(&pdev->dev, -EIO, "PCI read failed\n");
5219 
5220 	xid = (txconfig >> 20) & 0xfcf;
5221 
5222 	/* Identify chip attached to board */
5223 	chipset = rtl8169_get_mac_version(xid, tp->supports_gmii);
5224 	if (chipset == RTL_GIGA_MAC_NONE)
5225 		return dev_err_probe(&pdev->dev, -ENODEV,
5226 				     "unknown chip XID %03x, contact r8169 maintainers (see MAINTAINERS file)\n",
5227 				     xid);
5228 	tp->mac_version = chipset;
5229 
5230 	tp->dash_type = rtl_check_dash(tp);
5231 
5232 	tp->cp_cmd = RTL_R16(tp, CPlusCmd) & CPCMD_MASK;
5233 
5234 	if (sizeof(dma_addr_t) > 4 && tp->mac_version >= RTL_GIGA_MAC_VER_18 &&
5235 	    !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)))
5236 		dev->features |= NETIF_F_HIGHDMA;
5237 
5238 	rtl_init_rxcfg(tp);
5239 
5240 	rtl8169_irq_mask_and_ack(tp);
5241 
5242 	rtl_hw_initialize(tp);
5243 
5244 	rtl_hw_reset(tp);
5245 
5246 	rc = rtl_alloc_irq(tp);
5247 	if (rc < 0)
5248 		return dev_err_probe(&pdev->dev, rc, "Can't allocate interrupt\n");
5249 
5250 	tp->irq = pci_irq_vector(pdev, 0);
5251 
5252 	INIT_WORK(&tp->wk.work, rtl_task);
5253 
5254 	rtl_init_mac_address(tp);
5255 
5256 	dev->ethtool_ops = &rtl8169_ethtool_ops;
5257 
5258 	netif_napi_add(dev, &tp->napi, rtl8169_poll);
5259 
5260 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
5261 			   NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
5262 	dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
5263 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
5264 
5265 	/*
5266 	 * Pretend we are using VLANs; This bypasses a nasty bug where
5267 	 * Interrupts stop flowing on high load on 8110SCd controllers.
5268 	 */
5269 	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
5270 		/* Disallow toggling */
5271 		dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
5272 
5273 	if (rtl_chip_supports_csum_v2(tp))
5274 		dev->hw_features |= NETIF_F_IPV6_CSUM;
5275 
5276 	dev->features |= dev->hw_features;
5277 
5278 	/* There has been a number of reports that using SG/TSO results in
5279 	 * tx timeouts. However for a lot of people SG/TSO works fine.
5280 	 * Therefore disable both features by default, but allow users to
5281 	 * enable them. Use at own risk!
5282 	 */
5283 	if (rtl_chip_supports_csum_v2(tp)) {
5284 		dev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6;
5285 		netif_set_tso_max_size(dev, RTL_GSO_MAX_SIZE_V2);
5286 		netif_set_tso_max_segs(dev, RTL_GSO_MAX_SEGS_V2);
5287 	} else {
5288 		dev->hw_features |= NETIF_F_SG | NETIF_F_TSO;
5289 		netif_set_tso_max_size(dev, RTL_GSO_MAX_SIZE_V1);
5290 		netif_set_tso_max_segs(dev, RTL_GSO_MAX_SEGS_V1);
5291 	}
5292 
5293 	dev->hw_features |= NETIF_F_RXALL;
5294 	dev->hw_features |= NETIF_F_RXFCS;
5295 
5296 	netdev_sw_irq_coalesce_default_on(dev);
5297 
5298 	/* configure chip for default features */
5299 	rtl8169_set_features(dev, dev->features);
5300 
5301 	if (tp->dash_type == RTL_DASH_NONE) {
5302 		rtl_set_d3_pll_down(tp, true);
5303 	} else {
5304 		rtl_set_d3_pll_down(tp, false);
5305 		dev->wol_enabled = 1;
5306 	}
5307 
5308 	jumbo_max = rtl_jumbo_max(tp);
5309 	if (jumbo_max)
5310 		dev->max_mtu = jumbo_max;
5311 
5312 	rtl_set_irq_mask(tp);
5313 
5314 	tp->fw_name = rtl_chip_infos[chipset].fw_name;
5315 
5316 	tp->counters = dmam_alloc_coherent (&pdev->dev, sizeof(*tp->counters),
5317 					    &tp->counters_phys_addr,
5318 					    GFP_KERNEL);
5319 	if (!tp->counters)
5320 		return -ENOMEM;
5321 
5322 	pci_set_drvdata(pdev, tp);
5323 
5324 	rc = r8169_mdio_register(tp);
5325 	if (rc)
5326 		return rc;
5327 
5328 	rc = register_netdev(dev);
5329 	if (rc)
5330 		return rc;
5331 
5332 	netdev_info(dev, "%s, %pM, XID %03x, IRQ %d\n",
5333 		    rtl_chip_infos[chipset].name, dev->dev_addr, xid, tp->irq);
5334 
5335 	if (jumbo_max)
5336 		netdev_info(dev, "jumbo features [frames: %d bytes, tx checksumming: %s]\n",
5337 			    jumbo_max, tp->mac_version <= RTL_GIGA_MAC_VER_06 ?
5338 			    "ok" : "ko");
5339 
5340 	if (tp->dash_type != RTL_DASH_NONE) {
5341 		netdev_info(dev, "DASH enabled\n");
5342 		rtl8168_driver_start(tp);
5343 	}
5344 
5345 	if (pci_dev_run_wake(pdev))
5346 		pm_runtime_put_sync(&pdev->dev);
5347 
5348 	return 0;
5349 }
5350 
5351 static struct pci_driver rtl8169_pci_driver = {
5352 	.name		= KBUILD_MODNAME,
5353 	.id_table	= rtl8169_pci_tbl,
5354 	.probe		= rtl_init_one,
5355 	.remove		= rtl_remove_one,
5356 	.shutdown	= rtl_shutdown,
5357 	.driver.pm	= pm_ptr(&rtl8169_pm_ops),
5358 };
5359 
5360 module_pci_driver(rtl8169_pci_driver);
5361