1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * r8169.c: RealTek 8169/8168/8101 ethernet driver.
4  *
5  * Copyright (c) 2002 ShuChen <shuchen@realtek.com.tw>
6  * Copyright (c) 2003 - 2007 Francois Romieu <romieu@fr.zoreil.com>
7  * Copyright (c) a lot of people too. Please respect their work.
8  *
9  * See MAINTAINERS file for support contact information.
10  */
11 
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/netdevice.h>
15 #include <linux/etherdevice.h>
16 #include <linux/clk.h>
17 #include <linux/delay.h>
18 #include <linux/ethtool.h>
19 #include <linux/phy.h>
20 #include <linux/if_vlan.h>
21 #include <linux/in.h>
22 #include <linux/io.h>
23 #include <linux/ip.h>
24 #include <linux/tcp.h>
25 #include <linux/interrupt.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/bitfield.h>
29 #include <linux/prefetch.h>
30 #include <linux/ipv6.h>
31 #include <asm/unaligned.h>
32 #include <net/ip6_checksum.h>
33 
34 #include "r8169.h"
35 #include "r8169_firmware.h"
36 
37 #define FIRMWARE_8168D_1	"rtl_nic/rtl8168d-1.fw"
38 #define FIRMWARE_8168D_2	"rtl_nic/rtl8168d-2.fw"
39 #define FIRMWARE_8168E_1	"rtl_nic/rtl8168e-1.fw"
40 #define FIRMWARE_8168E_2	"rtl_nic/rtl8168e-2.fw"
41 #define FIRMWARE_8168E_3	"rtl_nic/rtl8168e-3.fw"
42 #define FIRMWARE_8168F_1	"rtl_nic/rtl8168f-1.fw"
43 #define FIRMWARE_8168F_2	"rtl_nic/rtl8168f-2.fw"
44 #define FIRMWARE_8105E_1	"rtl_nic/rtl8105e-1.fw"
45 #define FIRMWARE_8402_1		"rtl_nic/rtl8402-1.fw"
46 #define FIRMWARE_8411_1		"rtl_nic/rtl8411-1.fw"
47 #define FIRMWARE_8411_2		"rtl_nic/rtl8411-2.fw"
48 #define FIRMWARE_8106E_1	"rtl_nic/rtl8106e-1.fw"
49 #define FIRMWARE_8106E_2	"rtl_nic/rtl8106e-2.fw"
50 #define FIRMWARE_8168G_2	"rtl_nic/rtl8168g-2.fw"
51 #define FIRMWARE_8168G_3	"rtl_nic/rtl8168g-3.fw"
52 #define FIRMWARE_8168H_1	"rtl_nic/rtl8168h-1.fw"
53 #define FIRMWARE_8168H_2	"rtl_nic/rtl8168h-2.fw"
54 #define FIRMWARE_8168FP_3	"rtl_nic/rtl8168fp-3.fw"
55 #define FIRMWARE_8107E_1	"rtl_nic/rtl8107e-1.fw"
56 #define FIRMWARE_8107E_2	"rtl_nic/rtl8107e-2.fw"
57 #define FIRMWARE_8125A_3	"rtl_nic/rtl8125a-3.fw"
58 #define FIRMWARE_8125B_2	"rtl_nic/rtl8125b-2.fw"
59 
60 /* Maximum number of multicast addresses to filter (vs. Rx-all-multicast).
61    The RTL chips use a 64 element hash table based on the Ethernet CRC. */
62 #define	MC_FILTER_LIMIT	32
63 
64 #define TX_DMA_BURST	7	/* Maximum PCI burst, '7' is unlimited */
65 #define InterFrameGap	0x03	/* 3 means InterFrameGap = the shortest one */
66 
67 #define R8169_REGS_SIZE		256
68 #define R8169_RX_BUF_SIZE	(SZ_16K - 1)
69 #define NUM_TX_DESC	256	/* Number of Tx descriptor registers */
70 #define NUM_RX_DESC	256	/* Number of Rx descriptor registers */
71 #define R8169_TX_RING_BYTES	(NUM_TX_DESC * sizeof(struct TxDesc))
72 #define R8169_RX_RING_BYTES	(NUM_RX_DESC * sizeof(struct RxDesc))
73 
74 #define OCP_STD_PHY_BASE	0xa400
75 
76 #define RTL_CFG_NO_GBIT	1
77 
78 /* write/read MMIO register */
79 #define RTL_W8(tp, reg, val8)	writeb((val8), tp->mmio_addr + (reg))
80 #define RTL_W16(tp, reg, val16)	writew((val16), tp->mmio_addr + (reg))
81 #define RTL_W32(tp, reg, val32)	writel((val32), tp->mmio_addr + (reg))
82 #define RTL_R8(tp, reg)		readb(tp->mmio_addr + (reg))
83 #define RTL_R16(tp, reg)		readw(tp->mmio_addr + (reg))
84 #define RTL_R32(tp, reg)		readl(tp->mmio_addr + (reg))
85 
86 #define JUMBO_4K	(4 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
87 #define JUMBO_6K	(6 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
88 #define JUMBO_7K	(7 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
89 #define JUMBO_9K	(9 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
90 
91 static const struct {
92 	const char *name;
93 	const char *fw_name;
94 } rtl_chip_infos[] = {
95 	/* PCI devices. */
96 	[RTL_GIGA_MAC_VER_02] = {"RTL8169s"				},
97 	[RTL_GIGA_MAC_VER_03] = {"RTL8110s"				},
98 	[RTL_GIGA_MAC_VER_04] = {"RTL8169sb/8110sb"			},
99 	[RTL_GIGA_MAC_VER_05] = {"RTL8169sc/8110sc"			},
100 	[RTL_GIGA_MAC_VER_06] = {"RTL8169sc/8110sc"			},
101 	/* PCI-E devices. */
102 	[RTL_GIGA_MAC_VER_07] = {"RTL8102e"				},
103 	[RTL_GIGA_MAC_VER_08] = {"RTL8102e"				},
104 	[RTL_GIGA_MAC_VER_09] = {"RTL8102e/RTL8103e"			},
105 	[RTL_GIGA_MAC_VER_10] = {"RTL8101e"				},
106 	[RTL_GIGA_MAC_VER_11] = {"RTL8168b/8111b"			},
107 	[RTL_GIGA_MAC_VER_12] = {"RTL8168b/8111b"			},
108 	[RTL_GIGA_MAC_VER_13] = {"RTL8101e/RTL8100e"			},
109 	[RTL_GIGA_MAC_VER_14] = {"RTL8401"				},
110 	[RTL_GIGA_MAC_VER_16] = {"RTL8101e"				},
111 	[RTL_GIGA_MAC_VER_17] = {"RTL8168b/8111b"			},
112 	[RTL_GIGA_MAC_VER_18] = {"RTL8168cp/8111cp"			},
113 	[RTL_GIGA_MAC_VER_19] = {"RTL8168c/8111c"			},
114 	[RTL_GIGA_MAC_VER_20] = {"RTL8168c/8111c"			},
115 	[RTL_GIGA_MAC_VER_21] = {"RTL8168c/8111c"			},
116 	[RTL_GIGA_MAC_VER_22] = {"RTL8168c/8111c"			},
117 	[RTL_GIGA_MAC_VER_23] = {"RTL8168cp/8111cp"			},
118 	[RTL_GIGA_MAC_VER_24] = {"RTL8168cp/8111cp"			},
119 	[RTL_GIGA_MAC_VER_25] = {"RTL8168d/8111d",	FIRMWARE_8168D_1},
120 	[RTL_GIGA_MAC_VER_26] = {"RTL8168d/8111d",	FIRMWARE_8168D_2},
121 	[RTL_GIGA_MAC_VER_28] = {"RTL8168dp/8111dp"			},
122 	[RTL_GIGA_MAC_VER_29] = {"RTL8105e",		FIRMWARE_8105E_1},
123 	[RTL_GIGA_MAC_VER_30] = {"RTL8105e",		FIRMWARE_8105E_1},
124 	[RTL_GIGA_MAC_VER_31] = {"RTL8168dp/8111dp"			},
125 	[RTL_GIGA_MAC_VER_32] = {"RTL8168e/8111e",	FIRMWARE_8168E_1},
126 	[RTL_GIGA_MAC_VER_33] = {"RTL8168e/8111e",	FIRMWARE_8168E_2},
127 	[RTL_GIGA_MAC_VER_34] = {"RTL8168evl/8111evl",	FIRMWARE_8168E_3},
128 	[RTL_GIGA_MAC_VER_35] = {"RTL8168f/8111f",	FIRMWARE_8168F_1},
129 	[RTL_GIGA_MAC_VER_36] = {"RTL8168f/8111f",	FIRMWARE_8168F_2},
130 	[RTL_GIGA_MAC_VER_37] = {"RTL8402",		FIRMWARE_8402_1 },
131 	[RTL_GIGA_MAC_VER_38] = {"RTL8411",		FIRMWARE_8411_1 },
132 	[RTL_GIGA_MAC_VER_39] = {"RTL8106e",		FIRMWARE_8106E_1},
133 	[RTL_GIGA_MAC_VER_40] = {"RTL8168g/8111g",	FIRMWARE_8168G_2},
134 	[RTL_GIGA_MAC_VER_41] = {"RTL8168g/8111g"			},
135 	[RTL_GIGA_MAC_VER_42] = {"RTL8168gu/8111gu",	FIRMWARE_8168G_3},
136 	[RTL_GIGA_MAC_VER_43] = {"RTL8106eus",		FIRMWARE_8106E_2},
137 	[RTL_GIGA_MAC_VER_44] = {"RTL8411b",		FIRMWARE_8411_2 },
138 	[RTL_GIGA_MAC_VER_45] = {"RTL8168h/8111h",	FIRMWARE_8168H_1},
139 	[RTL_GIGA_MAC_VER_46] = {"RTL8168h/8111h",	FIRMWARE_8168H_2},
140 	[RTL_GIGA_MAC_VER_47] = {"RTL8107e",		FIRMWARE_8107E_1},
141 	[RTL_GIGA_MAC_VER_48] = {"RTL8107e",		FIRMWARE_8107E_2},
142 	[RTL_GIGA_MAC_VER_49] = {"RTL8168ep/8111ep"			},
143 	[RTL_GIGA_MAC_VER_50] = {"RTL8168ep/8111ep"			},
144 	[RTL_GIGA_MAC_VER_51] = {"RTL8168ep/8111ep"			},
145 	[RTL_GIGA_MAC_VER_52] = {"RTL8168fp/RTL8117",  FIRMWARE_8168FP_3},
146 	[RTL_GIGA_MAC_VER_53] = {"RTL8168fp/RTL8117",			},
147 	[RTL_GIGA_MAC_VER_60] = {"RTL8125A"				},
148 	[RTL_GIGA_MAC_VER_61] = {"RTL8125A",		FIRMWARE_8125A_3},
149 	/* reserve 62 for CFG_METHOD_4 in the vendor driver */
150 	[RTL_GIGA_MAC_VER_63] = {"RTL8125B",		FIRMWARE_8125B_2},
151 };
152 
153 static const struct pci_device_id rtl8169_pci_tbl[] = {
154 	{ PCI_VDEVICE(REALTEK,	0x2502) },
155 	{ PCI_VDEVICE(REALTEK,	0x2600) },
156 	{ PCI_VDEVICE(REALTEK,	0x8129) },
157 	{ PCI_VDEVICE(REALTEK,	0x8136), RTL_CFG_NO_GBIT },
158 	{ PCI_VDEVICE(REALTEK,	0x8161) },
159 	{ PCI_VDEVICE(REALTEK,	0x8162) },
160 	{ PCI_VDEVICE(REALTEK,	0x8167) },
161 	{ PCI_VDEVICE(REALTEK,	0x8168) },
162 	{ PCI_VDEVICE(NCUBE,	0x8168) },
163 	{ PCI_VDEVICE(REALTEK,	0x8169) },
164 	{ PCI_VENDOR_ID_DLINK,	0x4300,
165 		PCI_VENDOR_ID_DLINK, 0x4b10, 0, 0 },
166 	{ PCI_VDEVICE(DLINK,	0x4300) },
167 	{ PCI_VDEVICE(DLINK,	0x4302) },
168 	{ PCI_VDEVICE(AT,	0xc107) },
169 	{ PCI_VDEVICE(USR,	0x0116) },
170 	{ PCI_VENDOR_ID_LINKSYS, 0x1032, PCI_ANY_ID, 0x0024 },
171 	{ 0x0001, 0x8168, PCI_ANY_ID, 0x2410 },
172 	{ PCI_VDEVICE(REALTEK,	0x8125) },
173 	{ PCI_VDEVICE(REALTEK,	0x3000) },
174 	{}
175 };
176 
177 MODULE_DEVICE_TABLE(pci, rtl8169_pci_tbl);
178 
179 enum rtl_registers {
180 	MAC0		= 0,	/* Ethernet hardware address. */
181 	MAC4		= 4,
182 	MAR0		= 8,	/* Multicast filter. */
183 	CounterAddrLow		= 0x10,
184 	CounterAddrHigh		= 0x14,
185 	TxDescStartAddrLow	= 0x20,
186 	TxDescStartAddrHigh	= 0x24,
187 	TxHDescStartAddrLow	= 0x28,
188 	TxHDescStartAddrHigh	= 0x2c,
189 	FLASH		= 0x30,
190 	ERSR		= 0x36,
191 	ChipCmd		= 0x37,
192 	TxPoll		= 0x38,
193 	IntrMask	= 0x3c,
194 	IntrStatus	= 0x3e,
195 
196 	TxConfig	= 0x40,
197 #define	TXCFG_AUTO_FIFO			(1 << 7)	/* 8111e-vl */
198 #define	TXCFG_EMPTY			(1 << 11)	/* 8111e-vl */
199 
200 	RxConfig	= 0x44,
201 #define	RX128_INT_EN			(1 << 15)	/* 8111c and later */
202 #define	RX_MULTI_EN			(1 << 14)	/* 8111c only */
203 #define	RXCFG_FIFO_SHIFT		13
204 					/* No threshold before first PCI xfer */
205 #define	RX_FIFO_THRESH			(7 << RXCFG_FIFO_SHIFT)
206 #define	RX_EARLY_OFF			(1 << 11)
207 #define	RXCFG_DMA_SHIFT			8
208 					/* Unlimited maximum PCI burst. */
209 #define	RX_DMA_BURST			(7 << RXCFG_DMA_SHIFT)
210 
211 	Cfg9346		= 0x50,
212 	Config0		= 0x51,
213 	Config1		= 0x52,
214 	Config2		= 0x53,
215 #define PME_SIGNAL			(1 << 5)	/* 8168c and later */
216 
217 	Config3		= 0x54,
218 	Config4		= 0x55,
219 	Config5		= 0x56,
220 	PHYAR		= 0x60,
221 	PHYstatus	= 0x6c,
222 	RxMaxSize	= 0xda,
223 	CPlusCmd	= 0xe0,
224 	IntrMitigate	= 0xe2,
225 
226 #define RTL_COALESCE_TX_USECS	GENMASK(15, 12)
227 #define RTL_COALESCE_TX_FRAMES	GENMASK(11, 8)
228 #define RTL_COALESCE_RX_USECS	GENMASK(7, 4)
229 #define RTL_COALESCE_RX_FRAMES	GENMASK(3, 0)
230 
231 #define RTL_COALESCE_T_MAX	0x0fU
232 #define RTL_COALESCE_FRAME_MAX	(RTL_COALESCE_T_MAX * 4)
233 
234 	RxDescAddrLow	= 0xe4,
235 	RxDescAddrHigh	= 0xe8,
236 	EarlyTxThres	= 0xec,	/* 8169. Unit of 32 bytes. */
237 
238 #define NoEarlyTx	0x3f	/* Max value : no early transmit. */
239 
240 	MaxTxPacketSize	= 0xec,	/* 8101/8168. Unit of 128 bytes. */
241 
242 #define TxPacketMax	(8064 >> 7)
243 #define EarlySize	0x27
244 
245 	FuncEvent	= 0xf0,
246 	FuncEventMask	= 0xf4,
247 	FuncPresetState	= 0xf8,
248 	IBCR0           = 0xf8,
249 	IBCR2           = 0xf9,
250 	IBIMR0          = 0xfa,
251 	IBISR0          = 0xfb,
252 	FuncForceEvent	= 0xfc,
253 };
254 
255 enum rtl8168_8101_registers {
256 	CSIDR			= 0x64,
257 	CSIAR			= 0x68,
258 #define	CSIAR_FLAG			0x80000000
259 #define	CSIAR_WRITE_CMD			0x80000000
260 #define	CSIAR_BYTE_ENABLE		0x0000f000
261 #define	CSIAR_ADDR_MASK			0x00000fff
262 	PMCH			= 0x6f,
263 #define D3COLD_NO_PLL_DOWN		BIT(7)
264 #define D3HOT_NO_PLL_DOWN		BIT(6)
265 #define D3_NO_PLL_DOWN			(BIT(7) | BIT(6))
266 	EPHYAR			= 0x80,
267 #define	EPHYAR_FLAG			0x80000000
268 #define	EPHYAR_WRITE_CMD		0x80000000
269 #define	EPHYAR_REG_MASK			0x1f
270 #define	EPHYAR_REG_SHIFT		16
271 #define	EPHYAR_DATA_MASK		0xffff
272 	DLLPR			= 0xd0,
273 #define	PFM_EN				(1 << 6)
274 #define	TX_10M_PS_EN			(1 << 7)
275 	DBG_REG			= 0xd1,
276 #define	FIX_NAK_1			(1 << 4)
277 #define	FIX_NAK_2			(1 << 3)
278 	TWSI			= 0xd2,
279 	MCU			= 0xd3,
280 #define	NOW_IS_OOB			(1 << 7)
281 #define	TX_EMPTY			(1 << 5)
282 #define	RX_EMPTY			(1 << 4)
283 #define	RXTX_EMPTY			(TX_EMPTY | RX_EMPTY)
284 #define	EN_NDP				(1 << 3)
285 #define	EN_OOB_RESET			(1 << 2)
286 #define	LINK_LIST_RDY			(1 << 1)
287 	EFUSEAR			= 0xdc,
288 #define	EFUSEAR_FLAG			0x80000000
289 #define	EFUSEAR_WRITE_CMD		0x80000000
290 #define	EFUSEAR_READ_CMD		0x00000000
291 #define	EFUSEAR_REG_MASK		0x03ff
292 #define	EFUSEAR_REG_SHIFT		8
293 #define	EFUSEAR_DATA_MASK		0xff
294 	MISC_1			= 0xf2,
295 #define	PFM_D3COLD_EN			(1 << 6)
296 };
297 
298 enum rtl8168_registers {
299 	LED_FREQ		= 0x1a,
300 	EEE_LED			= 0x1b,
301 	ERIDR			= 0x70,
302 	ERIAR			= 0x74,
303 #define ERIAR_FLAG			0x80000000
304 #define ERIAR_WRITE_CMD			0x80000000
305 #define ERIAR_READ_CMD			0x00000000
306 #define ERIAR_ADDR_BYTE_ALIGN		4
307 #define ERIAR_TYPE_SHIFT		16
308 #define ERIAR_EXGMAC			(0x00 << ERIAR_TYPE_SHIFT)
309 #define ERIAR_MSIX			(0x01 << ERIAR_TYPE_SHIFT)
310 #define ERIAR_ASF			(0x02 << ERIAR_TYPE_SHIFT)
311 #define ERIAR_OOB			(0x02 << ERIAR_TYPE_SHIFT)
312 #define ERIAR_MASK_SHIFT		12
313 #define ERIAR_MASK_0001			(0x1 << ERIAR_MASK_SHIFT)
314 #define ERIAR_MASK_0011			(0x3 << ERIAR_MASK_SHIFT)
315 #define ERIAR_MASK_0100			(0x4 << ERIAR_MASK_SHIFT)
316 #define ERIAR_MASK_0101			(0x5 << ERIAR_MASK_SHIFT)
317 #define ERIAR_MASK_1111			(0xf << ERIAR_MASK_SHIFT)
318 	EPHY_RXER_NUM		= 0x7c,
319 	OCPDR			= 0xb0,	/* OCP GPHY access */
320 #define OCPDR_WRITE_CMD			0x80000000
321 #define OCPDR_READ_CMD			0x00000000
322 #define OCPDR_REG_MASK			0x7f
323 #define OCPDR_GPHY_REG_SHIFT		16
324 #define OCPDR_DATA_MASK			0xffff
325 	OCPAR			= 0xb4,
326 #define OCPAR_FLAG			0x80000000
327 #define OCPAR_GPHY_WRITE_CMD		0x8000f060
328 #define OCPAR_GPHY_READ_CMD		0x0000f060
329 	GPHY_OCP		= 0xb8,
330 	RDSAR1			= 0xd0,	/* 8168c only. Undocumented on 8168dp */
331 	MISC			= 0xf0,	/* 8168e only. */
332 #define TXPLA_RST			(1 << 29)
333 #define DISABLE_LAN_EN			(1 << 23) /* Enable GPIO pin */
334 #define PWM_EN				(1 << 22)
335 #define RXDV_GATED_EN			(1 << 19)
336 #define EARLY_TALLY_EN			(1 << 16)
337 };
338 
339 enum rtl8125_registers {
340 	IntrMask_8125		= 0x38,
341 	IntrStatus_8125		= 0x3c,
342 	TxPoll_8125		= 0x90,
343 	MAC0_BKP		= 0x19e0,
344 	EEE_TXIDLE_TIMER_8125	= 0x6048,
345 };
346 
347 #define RX_VLAN_INNER_8125	BIT(22)
348 #define RX_VLAN_OUTER_8125	BIT(23)
349 #define RX_VLAN_8125		(RX_VLAN_INNER_8125 | RX_VLAN_OUTER_8125)
350 
351 #define RX_FETCH_DFLT_8125	(8 << 27)
352 
353 enum rtl_register_content {
354 	/* InterruptStatusBits */
355 	SYSErr		= 0x8000,
356 	PCSTimeout	= 0x4000,
357 	SWInt		= 0x0100,
358 	TxDescUnavail	= 0x0080,
359 	RxFIFOOver	= 0x0040,
360 	LinkChg		= 0x0020,
361 	RxOverflow	= 0x0010,
362 	TxErr		= 0x0008,
363 	TxOK		= 0x0004,
364 	RxErr		= 0x0002,
365 	RxOK		= 0x0001,
366 
367 	/* RxStatusDesc */
368 	RxRWT	= (1 << 22),
369 	RxRES	= (1 << 21),
370 	RxRUNT	= (1 << 20),
371 	RxCRC	= (1 << 19),
372 
373 	/* ChipCmdBits */
374 	StopReq		= 0x80,
375 	CmdReset	= 0x10,
376 	CmdRxEnb	= 0x08,
377 	CmdTxEnb	= 0x04,
378 	RxBufEmpty	= 0x01,
379 
380 	/* TXPoll register p.5 */
381 	HPQ		= 0x80,		/* Poll cmd on the high prio queue */
382 	NPQ		= 0x40,		/* Poll cmd on the low prio queue */
383 	FSWInt		= 0x01,		/* Forced software interrupt */
384 
385 	/* Cfg9346Bits */
386 	Cfg9346_Lock	= 0x00,
387 	Cfg9346_Unlock	= 0xc0,
388 
389 	/* rx_mode_bits */
390 	AcceptErr	= 0x20,
391 	AcceptRunt	= 0x10,
392 #define RX_CONFIG_ACCEPT_ERR_MASK	0x30
393 	AcceptBroadcast	= 0x08,
394 	AcceptMulticast	= 0x04,
395 	AcceptMyPhys	= 0x02,
396 	AcceptAllPhys	= 0x01,
397 #define RX_CONFIG_ACCEPT_OK_MASK	0x0f
398 #define RX_CONFIG_ACCEPT_MASK		0x3f
399 
400 	/* TxConfigBits */
401 	TxInterFrameGapShift = 24,
402 	TxDMAShift = 8,	/* DMA burst value (0-7) is shift this many bits */
403 
404 	/* Config1 register p.24 */
405 	LEDS1		= (1 << 7),
406 	LEDS0		= (1 << 6),
407 	Speed_down	= (1 << 4),
408 	MEMMAP		= (1 << 3),
409 	IOMAP		= (1 << 2),
410 	VPD		= (1 << 1),
411 	PMEnable	= (1 << 0),	/* Power Management Enable */
412 
413 	/* Config2 register p. 25 */
414 	ClkReqEn	= (1 << 7),	/* Clock Request Enable */
415 	MSIEnable	= (1 << 5),	/* 8169 only. Reserved in the 8168. */
416 	PCI_Clock_66MHz = 0x01,
417 	PCI_Clock_33MHz = 0x00,
418 
419 	/* Config3 register p.25 */
420 	MagicPacket	= (1 << 5),	/* Wake up when receives a Magic Packet */
421 	LinkUp		= (1 << 4),	/* Wake up when the cable connection is re-established */
422 	Jumbo_En0	= (1 << 2),	/* 8168 only. Reserved in the 8168b */
423 	Rdy_to_L23	= (1 << 1),	/* L23 Enable */
424 	Beacon_en	= (1 << 0),	/* 8168 only. Reserved in the 8168b */
425 
426 	/* Config4 register */
427 	Jumbo_En1	= (1 << 1),	/* 8168 only. Reserved in the 8168b */
428 
429 	/* Config5 register p.27 */
430 	BWF		= (1 << 6),	/* Accept Broadcast wakeup frame */
431 	MWF		= (1 << 5),	/* Accept Multicast wakeup frame */
432 	UWF		= (1 << 4),	/* Accept Unicast wakeup frame */
433 	Spi_en		= (1 << 3),
434 	LanWake		= (1 << 1),	/* LanWake enable/disable */
435 	PMEStatus	= (1 << 0),	/* PME status can be reset by PCI RST# */
436 	ASPM_en		= (1 << 0),	/* ASPM enable */
437 
438 	/* CPlusCmd p.31 */
439 	EnableBist	= (1 << 15),	// 8168 8101
440 	Mac_dbgo_oe	= (1 << 14),	// 8168 8101
441 	EnAnaPLL	= (1 << 14),	// 8169
442 	Normal_mode	= (1 << 13),	// unused
443 	Force_half_dup	= (1 << 12),	// 8168 8101
444 	Force_rxflow_en	= (1 << 11),	// 8168 8101
445 	Force_txflow_en	= (1 << 10),	// 8168 8101
446 	Cxpl_dbg_sel	= (1 << 9),	// 8168 8101
447 	ASF		= (1 << 8),	// 8168 8101
448 	PktCntrDisable	= (1 << 7),	// 8168 8101
449 	Mac_dbgo_sel	= 0x001c,	// 8168
450 	RxVlan		= (1 << 6),
451 	RxChkSum	= (1 << 5),
452 	PCIDAC		= (1 << 4),
453 	PCIMulRW	= (1 << 3),
454 #define INTT_MASK	GENMASK(1, 0)
455 #define CPCMD_MASK	(Normal_mode | RxVlan | RxChkSum | INTT_MASK)
456 
457 	/* rtl8169_PHYstatus */
458 	TBI_Enable	= 0x80,
459 	TxFlowCtrl	= 0x40,
460 	RxFlowCtrl	= 0x20,
461 	_1000bpsF	= 0x10,
462 	_100bps		= 0x08,
463 	_10bps		= 0x04,
464 	LinkStatus	= 0x02,
465 	FullDup		= 0x01,
466 
467 	/* ResetCounterCommand */
468 	CounterReset	= 0x1,
469 
470 	/* DumpCounterCommand */
471 	CounterDump	= 0x8,
472 
473 	/* magic enable v2 */
474 	MagicPacket_v2	= (1 << 16),	/* Wake up when receives a Magic Packet */
475 };
476 
477 enum rtl_desc_bit {
478 	/* First doubleword. */
479 	DescOwn		= (1 << 31), /* Descriptor is owned by NIC */
480 	RingEnd		= (1 << 30), /* End of descriptor ring */
481 	FirstFrag	= (1 << 29), /* First segment of a packet */
482 	LastFrag	= (1 << 28), /* Final segment of a packet */
483 };
484 
485 /* Generic case. */
486 enum rtl_tx_desc_bit {
487 	/* First doubleword. */
488 	TD_LSO		= (1 << 27),		/* Large Send Offload */
489 #define TD_MSS_MAX			0x07ffu	/* MSS value */
490 
491 	/* Second doubleword. */
492 	TxVlanTag	= (1 << 17),		/* Add VLAN tag */
493 };
494 
495 /* 8169, 8168b and 810x except 8102e. */
496 enum rtl_tx_desc_bit_0 {
497 	/* First doubleword. */
498 #define TD0_MSS_SHIFT			16	/* MSS position (11 bits) */
499 	TD0_TCP_CS	= (1 << 16),		/* Calculate TCP/IP checksum */
500 	TD0_UDP_CS	= (1 << 17),		/* Calculate UDP/IP checksum */
501 	TD0_IP_CS	= (1 << 18),		/* Calculate IP checksum */
502 };
503 
504 /* 8102e, 8168c and beyond. */
505 enum rtl_tx_desc_bit_1 {
506 	/* First doubleword. */
507 	TD1_GTSENV4	= (1 << 26),		/* Giant Send for IPv4 */
508 	TD1_GTSENV6	= (1 << 25),		/* Giant Send for IPv6 */
509 #define GTTCPHO_SHIFT			18
510 #define GTTCPHO_MAX			0x7f
511 
512 	/* Second doubleword. */
513 #define TCPHO_SHIFT			18
514 #define TCPHO_MAX			0x3ff
515 #define TD1_MSS_SHIFT			18	/* MSS position (11 bits) */
516 	TD1_IPv6_CS	= (1 << 28),		/* Calculate IPv6 checksum */
517 	TD1_IPv4_CS	= (1 << 29),		/* Calculate IPv4 checksum */
518 	TD1_TCP_CS	= (1 << 30),		/* Calculate TCP/IP checksum */
519 	TD1_UDP_CS	= (1 << 31),		/* Calculate UDP/IP checksum */
520 };
521 
522 enum rtl_rx_desc_bit {
523 	/* Rx private */
524 	PID1		= (1 << 18), /* Protocol ID bit 1/2 */
525 	PID0		= (1 << 17), /* Protocol ID bit 0/2 */
526 
527 #define RxProtoUDP	(PID1)
528 #define RxProtoTCP	(PID0)
529 #define RxProtoIP	(PID1 | PID0)
530 #define RxProtoMask	RxProtoIP
531 
532 	IPFail		= (1 << 16), /* IP checksum failed */
533 	UDPFail		= (1 << 15), /* UDP/IP checksum failed */
534 	TCPFail		= (1 << 14), /* TCP/IP checksum failed */
535 
536 #define RxCSFailMask	(IPFail | UDPFail | TCPFail)
537 
538 	RxVlanTag	= (1 << 16), /* VLAN tag available */
539 };
540 
541 #define RTL_GSO_MAX_SIZE_V1	32000
542 #define RTL_GSO_MAX_SEGS_V1	24
543 #define RTL_GSO_MAX_SIZE_V2	64000
544 #define RTL_GSO_MAX_SEGS_V2	64
545 
546 struct TxDesc {
547 	__le32 opts1;
548 	__le32 opts2;
549 	__le64 addr;
550 };
551 
552 struct RxDesc {
553 	__le32 opts1;
554 	__le32 opts2;
555 	__le64 addr;
556 };
557 
558 struct ring_info {
559 	struct sk_buff	*skb;
560 	u32		len;
561 };
562 
563 struct rtl8169_counters {
564 	__le64	tx_packets;
565 	__le64	rx_packets;
566 	__le64	tx_errors;
567 	__le32	rx_errors;
568 	__le16	rx_missed;
569 	__le16	align_errors;
570 	__le32	tx_one_collision;
571 	__le32	tx_multi_collision;
572 	__le64	rx_unicast;
573 	__le64	rx_broadcast;
574 	__le32	rx_multicast;
575 	__le16	tx_aborted;
576 	__le16	tx_underun;
577 };
578 
579 struct rtl8169_tc_offsets {
580 	bool	inited;
581 	__le64	tx_errors;
582 	__le32	tx_multi_collision;
583 	__le16	tx_aborted;
584 	__le16	rx_missed;
585 };
586 
587 enum rtl_flag {
588 	RTL_FLAG_TASK_ENABLED = 0,
589 	RTL_FLAG_TASK_RESET_PENDING,
590 	RTL_FLAG_MAX
591 };
592 
593 enum rtl_dash_type {
594 	RTL_DASH_NONE,
595 	RTL_DASH_DP,
596 	RTL_DASH_EP,
597 };
598 
599 struct rtl8169_private {
600 	void __iomem *mmio_addr;	/* memory map physical address */
601 	struct pci_dev *pci_dev;
602 	struct net_device *dev;
603 	struct phy_device *phydev;
604 	struct napi_struct napi;
605 	enum mac_version mac_version;
606 	enum rtl_dash_type dash_type;
607 	u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */
608 	u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */
609 	u32 dirty_tx;
610 	struct TxDesc *TxDescArray;	/* 256-aligned Tx descriptor ring */
611 	struct RxDesc *RxDescArray;	/* 256-aligned Rx descriptor ring */
612 	dma_addr_t TxPhyAddr;
613 	dma_addr_t RxPhyAddr;
614 	struct page *Rx_databuff[NUM_RX_DESC];	/* Rx data buffers */
615 	struct ring_info tx_skb[NUM_TX_DESC];	/* Tx data buffers */
616 	u16 cp_cmd;
617 	u32 irq_mask;
618 	int irq;
619 	struct clk *clk;
620 
621 	struct {
622 		DECLARE_BITMAP(flags, RTL_FLAG_MAX);
623 		struct work_struct work;
624 	} wk;
625 
626 	unsigned supports_gmii:1;
627 	unsigned aspm_manageable:1;
628 	dma_addr_t counters_phys_addr;
629 	struct rtl8169_counters *counters;
630 	struct rtl8169_tc_offsets tc_offset;
631 	u32 saved_wolopts;
632 	int eee_adv;
633 
634 	const char *fw_name;
635 	struct rtl_fw *rtl_fw;
636 
637 	u32 ocp_base;
638 };
639 
640 typedef void (*rtl_generic_fct)(struct rtl8169_private *tp);
641 
642 MODULE_AUTHOR("Realtek and the Linux r8169 crew <netdev@vger.kernel.org>");
643 MODULE_DESCRIPTION("RealTek RTL-8169 Gigabit Ethernet driver");
644 MODULE_SOFTDEP("pre: realtek");
645 MODULE_LICENSE("GPL");
646 MODULE_FIRMWARE(FIRMWARE_8168D_1);
647 MODULE_FIRMWARE(FIRMWARE_8168D_2);
648 MODULE_FIRMWARE(FIRMWARE_8168E_1);
649 MODULE_FIRMWARE(FIRMWARE_8168E_2);
650 MODULE_FIRMWARE(FIRMWARE_8168E_3);
651 MODULE_FIRMWARE(FIRMWARE_8105E_1);
652 MODULE_FIRMWARE(FIRMWARE_8168F_1);
653 MODULE_FIRMWARE(FIRMWARE_8168F_2);
654 MODULE_FIRMWARE(FIRMWARE_8402_1);
655 MODULE_FIRMWARE(FIRMWARE_8411_1);
656 MODULE_FIRMWARE(FIRMWARE_8411_2);
657 MODULE_FIRMWARE(FIRMWARE_8106E_1);
658 MODULE_FIRMWARE(FIRMWARE_8106E_2);
659 MODULE_FIRMWARE(FIRMWARE_8168G_2);
660 MODULE_FIRMWARE(FIRMWARE_8168G_3);
661 MODULE_FIRMWARE(FIRMWARE_8168H_1);
662 MODULE_FIRMWARE(FIRMWARE_8168H_2);
663 MODULE_FIRMWARE(FIRMWARE_8168FP_3);
664 MODULE_FIRMWARE(FIRMWARE_8107E_1);
665 MODULE_FIRMWARE(FIRMWARE_8107E_2);
666 MODULE_FIRMWARE(FIRMWARE_8125A_3);
667 MODULE_FIRMWARE(FIRMWARE_8125B_2);
668 
669 static inline struct device *tp_to_dev(struct rtl8169_private *tp)
670 {
671 	return &tp->pci_dev->dev;
672 }
673 
674 static void rtl_lock_config_regs(struct rtl8169_private *tp)
675 {
676 	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
677 }
678 
679 static void rtl_unlock_config_regs(struct rtl8169_private *tp)
680 {
681 	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
682 }
683 
684 static void rtl_pci_commit(struct rtl8169_private *tp)
685 {
686 	/* Read an arbitrary register to commit a preceding PCI write */
687 	RTL_R8(tp, ChipCmd);
688 }
689 
690 static bool rtl_is_8125(struct rtl8169_private *tp)
691 {
692 	return tp->mac_version >= RTL_GIGA_MAC_VER_60;
693 }
694 
695 static bool rtl_is_8168evl_up(struct rtl8169_private *tp)
696 {
697 	return tp->mac_version >= RTL_GIGA_MAC_VER_34 &&
698 	       tp->mac_version != RTL_GIGA_MAC_VER_39 &&
699 	       tp->mac_version <= RTL_GIGA_MAC_VER_53;
700 }
701 
702 static bool rtl_supports_eee(struct rtl8169_private *tp)
703 {
704 	return tp->mac_version >= RTL_GIGA_MAC_VER_34 &&
705 	       tp->mac_version != RTL_GIGA_MAC_VER_37 &&
706 	       tp->mac_version != RTL_GIGA_MAC_VER_39;
707 }
708 
709 static void rtl_read_mac_from_reg(struct rtl8169_private *tp, u8 *mac, int reg)
710 {
711 	int i;
712 
713 	for (i = 0; i < ETH_ALEN; i++)
714 		mac[i] = RTL_R8(tp, reg + i);
715 }
716 
717 struct rtl_cond {
718 	bool (*check)(struct rtl8169_private *);
719 	const char *msg;
720 };
721 
722 static bool rtl_loop_wait(struct rtl8169_private *tp, const struct rtl_cond *c,
723 			  unsigned long usecs, int n, bool high)
724 {
725 	int i;
726 
727 	for (i = 0; i < n; i++) {
728 		if (c->check(tp) == high)
729 			return true;
730 		fsleep(usecs);
731 	}
732 
733 	if (net_ratelimit())
734 		netdev_err(tp->dev, "%s == %d (loop: %d, delay: %lu).\n",
735 			   c->msg, !high, n, usecs);
736 	return false;
737 }
738 
739 static bool rtl_loop_wait_high(struct rtl8169_private *tp,
740 			       const struct rtl_cond *c,
741 			       unsigned long d, int n)
742 {
743 	return rtl_loop_wait(tp, c, d, n, true);
744 }
745 
746 static bool rtl_loop_wait_low(struct rtl8169_private *tp,
747 			      const struct rtl_cond *c,
748 			      unsigned long d, int n)
749 {
750 	return rtl_loop_wait(tp, c, d, n, false);
751 }
752 
753 #define DECLARE_RTL_COND(name)				\
754 static bool name ## _check(struct rtl8169_private *);	\
755 							\
756 static const struct rtl_cond name = {			\
757 	.check	= name ## _check,			\
758 	.msg	= #name					\
759 };							\
760 							\
761 static bool name ## _check(struct rtl8169_private *tp)
762 
763 static void r8168fp_adjust_ocp_cmd(struct rtl8169_private *tp, u32 *cmd, int type)
764 {
765 	/* based on RTL8168FP_OOBMAC_BASE in vendor driver */
766 	if (type == ERIAR_OOB &&
767 	    (tp->mac_version == RTL_GIGA_MAC_VER_52 ||
768 	     tp->mac_version == RTL_GIGA_MAC_VER_53))
769 		*cmd |= 0xf70 << 18;
770 }
771 
772 DECLARE_RTL_COND(rtl_eriar_cond)
773 {
774 	return RTL_R32(tp, ERIAR) & ERIAR_FLAG;
775 }
776 
777 static void _rtl_eri_write(struct rtl8169_private *tp, int addr, u32 mask,
778 			   u32 val, int type)
779 {
780 	u32 cmd = ERIAR_WRITE_CMD | type | mask | addr;
781 
782 	if (WARN(addr & 3 || !mask, "addr: 0x%x, mask: 0x%08x\n", addr, mask))
783 		return;
784 
785 	RTL_W32(tp, ERIDR, val);
786 	r8168fp_adjust_ocp_cmd(tp, &cmd, type);
787 	RTL_W32(tp, ERIAR, cmd);
788 
789 	rtl_loop_wait_low(tp, &rtl_eriar_cond, 100, 100);
790 }
791 
792 static void rtl_eri_write(struct rtl8169_private *tp, int addr, u32 mask,
793 			  u32 val)
794 {
795 	_rtl_eri_write(tp, addr, mask, val, ERIAR_EXGMAC);
796 }
797 
798 static u32 _rtl_eri_read(struct rtl8169_private *tp, int addr, int type)
799 {
800 	u32 cmd = ERIAR_READ_CMD | type | ERIAR_MASK_1111 | addr;
801 
802 	r8168fp_adjust_ocp_cmd(tp, &cmd, type);
803 	RTL_W32(tp, ERIAR, cmd);
804 
805 	return rtl_loop_wait_high(tp, &rtl_eriar_cond, 100, 100) ?
806 		RTL_R32(tp, ERIDR) : ~0;
807 }
808 
809 static u32 rtl_eri_read(struct rtl8169_private *tp, int addr)
810 {
811 	return _rtl_eri_read(tp, addr, ERIAR_EXGMAC);
812 }
813 
814 static void rtl_w0w1_eri(struct rtl8169_private *tp, int addr, u32 p, u32 m)
815 {
816 	u32 val = rtl_eri_read(tp, addr);
817 
818 	rtl_eri_write(tp, addr, ERIAR_MASK_1111, (val & ~m) | p);
819 }
820 
821 static void rtl_eri_set_bits(struct rtl8169_private *tp, int addr, u32 p)
822 {
823 	rtl_w0w1_eri(tp, addr, p, 0);
824 }
825 
826 static void rtl_eri_clear_bits(struct rtl8169_private *tp, int addr, u32 m)
827 {
828 	rtl_w0w1_eri(tp, addr, 0, m);
829 }
830 
831 static bool rtl_ocp_reg_failure(u32 reg)
832 {
833 	return WARN_ONCE(reg & 0xffff0001, "Invalid ocp reg %x!\n", reg);
834 }
835 
836 DECLARE_RTL_COND(rtl_ocp_gphy_cond)
837 {
838 	return RTL_R32(tp, GPHY_OCP) & OCPAR_FLAG;
839 }
840 
841 static void r8168_phy_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
842 {
843 	if (rtl_ocp_reg_failure(reg))
844 		return;
845 
846 	RTL_W32(tp, GPHY_OCP, OCPAR_FLAG | (reg << 15) | data);
847 
848 	rtl_loop_wait_low(tp, &rtl_ocp_gphy_cond, 25, 10);
849 }
850 
851 static int r8168_phy_ocp_read(struct rtl8169_private *tp, u32 reg)
852 {
853 	if (rtl_ocp_reg_failure(reg))
854 		return 0;
855 
856 	RTL_W32(tp, GPHY_OCP, reg << 15);
857 
858 	return rtl_loop_wait_high(tp, &rtl_ocp_gphy_cond, 25, 10) ?
859 		(RTL_R32(tp, GPHY_OCP) & 0xffff) : -ETIMEDOUT;
860 }
861 
862 static void r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
863 {
864 	if (rtl_ocp_reg_failure(reg))
865 		return;
866 
867 	RTL_W32(tp, OCPDR, OCPAR_FLAG | (reg << 15) | data);
868 }
869 
870 static u16 r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
871 {
872 	if (rtl_ocp_reg_failure(reg))
873 		return 0;
874 
875 	RTL_W32(tp, OCPDR, reg << 15);
876 
877 	return RTL_R32(tp, OCPDR);
878 }
879 
880 static void r8168_mac_ocp_modify(struct rtl8169_private *tp, u32 reg, u16 mask,
881 				 u16 set)
882 {
883 	u16 data = r8168_mac_ocp_read(tp, reg);
884 
885 	r8168_mac_ocp_write(tp, reg, (data & ~mask) | set);
886 }
887 
888 /* Work around a hw issue with RTL8168g PHY, the quirk disables
889  * PHY MCU interrupts before PHY power-down.
890  */
891 static void rtl8168g_phy_suspend_quirk(struct rtl8169_private *tp, int value)
892 {
893 	switch (tp->mac_version) {
894 	case RTL_GIGA_MAC_VER_40:
895 	case RTL_GIGA_MAC_VER_41:
896 	case RTL_GIGA_MAC_VER_49:
897 		if (value & BMCR_RESET || !(value & BMCR_PDOWN))
898 			rtl_eri_set_bits(tp, 0x1a8, 0xfc000000);
899 		else
900 			rtl_eri_clear_bits(tp, 0x1a8, 0xfc000000);
901 		break;
902 	default:
903 		break;
904 	}
905 };
906 
907 static void r8168g_mdio_write(struct rtl8169_private *tp, int reg, int value)
908 {
909 	if (reg == 0x1f) {
910 		tp->ocp_base = value ? value << 4 : OCP_STD_PHY_BASE;
911 		return;
912 	}
913 
914 	if (tp->ocp_base != OCP_STD_PHY_BASE)
915 		reg -= 0x10;
916 
917 	if (tp->ocp_base == OCP_STD_PHY_BASE && reg == MII_BMCR)
918 		rtl8168g_phy_suspend_quirk(tp, value);
919 
920 	r8168_phy_ocp_write(tp, tp->ocp_base + reg * 2, value);
921 }
922 
923 static int r8168g_mdio_read(struct rtl8169_private *tp, int reg)
924 {
925 	if (reg == 0x1f)
926 		return tp->ocp_base == OCP_STD_PHY_BASE ? 0 : tp->ocp_base >> 4;
927 
928 	if (tp->ocp_base != OCP_STD_PHY_BASE)
929 		reg -= 0x10;
930 
931 	return r8168_phy_ocp_read(tp, tp->ocp_base + reg * 2);
932 }
933 
934 static void mac_mcu_write(struct rtl8169_private *tp, int reg, int value)
935 {
936 	if (reg == 0x1f) {
937 		tp->ocp_base = value << 4;
938 		return;
939 	}
940 
941 	r8168_mac_ocp_write(tp, tp->ocp_base + reg, value);
942 }
943 
944 static int mac_mcu_read(struct rtl8169_private *tp, int reg)
945 {
946 	return r8168_mac_ocp_read(tp, tp->ocp_base + reg);
947 }
948 
949 DECLARE_RTL_COND(rtl_phyar_cond)
950 {
951 	return RTL_R32(tp, PHYAR) & 0x80000000;
952 }
953 
954 static void r8169_mdio_write(struct rtl8169_private *tp, int reg, int value)
955 {
956 	RTL_W32(tp, PHYAR, 0x80000000 | (reg & 0x1f) << 16 | (value & 0xffff));
957 
958 	rtl_loop_wait_low(tp, &rtl_phyar_cond, 25, 20);
959 	/*
960 	 * According to hardware specs a 20us delay is required after write
961 	 * complete indication, but before sending next command.
962 	 */
963 	udelay(20);
964 }
965 
966 static int r8169_mdio_read(struct rtl8169_private *tp, int reg)
967 {
968 	int value;
969 
970 	RTL_W32(tp, PHYAR, 0x0 | (reg & 0x1f) << 16);
971 
972 	value = rtl_loop_wait_high(tp, &rtl_phyar_cond, 25, 20) ?
973 		RTL_R32(tp, PHYAR) & 0xffff : -ETIMEDOUT;
974 
975 	/*
976 	 * According to hardware specs a 20us delay is required after read
977 	 * complete indication, but before sending next command.
978 	 */
979 	udelay(20);
980 
981 	return value;
982 }
983 
984 DECLARE_RTL_COND(rtl_ocpar_cond)
985 {
986 	return RTL_R32(tp, OCPAR) & OCPAR_FLAG;
987 }
988 
989 #define R8168DP_1_MDIO_ACCESS_BIT	0x00020000
990 
991 static void r8168dp_2_mdio_start(struct rtl8169_private *tp)
992 {
993 	RTL_W32(tp, 0xd0, RTL_R32(tp, 0xd0) & ~R8168DP_1_MDIO_ACCESS_BIT);
994 }
995 
996 static void r8168dp_2_mdio_stop(struct rtl8169_private *tp)
997 {
998 	RTL_W32(tp, 0xd0, RTL_R32(tp, 0xd0) | R8168DP_1_MDIO_ACCESS_BIT);
999 }
1000 
1001 static void r8168dp_2_mdio_write(struct rtl8169_private *tp, int reg, int value)
1002 {
1003 	r8168dp_2_mdio_start(tp);
1004 
1005 	r8169_mdio_write(tp, reg, value);
1006 
1007 	r8168dp_2_mdio_stop(tp);
1008 }
1009 
1010 static int r8168dp_2_mdio_read(struct rtl8169_private *tp, int reg)
1011 {
1012 	int value;
1013 
1014 	/* Work around issue with chip reporting wrong PHY ID */
1015 	if (reg == MII_PHYSID2)
1016 		return 0xc912;
1017 
1018 	r8168dp_2_mdio_start(tp);
1019 
1020 	value = r8169_mdio_read(tp, reg);
1021 
1022 	r8168dp_2_mdio_stop(tp);
1023 
1024 	return value;
1025 }
1026 
1027 static void rtl_writephy(struct rtl8169_private *tp, int location, int val)
1028 {
1029 	switch (tp->mac_version) {
1030 	case RTL_GIGA_MAC_VER_28:
1031 	case RTL_GIGA_MAC_VER_31:
1032 		r8168dp_2_mdio_write(tp, location, val);
1033 		break;
1034 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
1035 		r8168g_mdio_write(tp, location, val);
1036 		break;
1037 	default:
1038 		r8169_mdio_write(tp, location, val);
1039 		break;
1040 	}
1041 }
1042 
1043 static int rtl_readphy(struct rtl8169_private *tp, int location)
1044 {
1045 	switch (tp->mac_version) {
1046 	case RTL_GIGA_MAC_VER_28:
1047 	case RTL_GIGA_MAC_VER_31:
1048 		return r8168dp_2_mdio_read(tp, location);
1049 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
1050 		return r8168g_mdio_read(tp, location);
1051 	default:
1052 		return r8169_mdio_read(tp, location);
1053 	}
1054 }
1055 
1056 DECLARE_RTL_COND(rtl_ephyar_cond)
1057 {
1058 	return RTL_R32(tp, EPHYAR) & EPHYAR_FLAG;
1059 }
1060 
1061 static void rtl_ephy_write(struct rtl8169_private *tp, int reg_addr, int value)
1062 {
1063 	RTL_W32(tp, EPHYAR, EPHYAR_WRITE_CMD | (value & EPHYAR_DATA_MASK) |
1064 		(reg_addr & EPHYAR_REG_MASK) << EPHYAR_REG_SHIFT);
1065 
1066 	rtl_loop_wait_low(tp, &rtl_ephyar_cond, 10, 100);
1067 
1068 	udelay(10);
1069 }
1070 
1071 static u16 rtl_ephy_read(struct rtl8169_private *tp, int reg_addr)
1072 {
1073 	RTL_W32(tp, EPHYAR, (reg_addr & EPHYAR_REG_MASK) << EPHYAR_REG_SHIFT);
1074 
1075 	return rtl_loop_wait_high(tp, &rtl_ephyar_cond, 10, 100) ?
1076 		RTL_R32(tp, EPHYAR) & EPHYAR_DATA_MASK : ~0;
1077 }
1078 
1079 static u32 r8168dp_ocp_read(struct rtl8169_private *tp, u16 reg)
1080 {
1081 	RTL_W32(tp, OCPAR, 0x0fu << 12 | (reg & 0x0fff));
1082 	return rtl_loop_wait_high(tp, &rtl_ocpar_cond, 100, 20) ?
1083 		RTL_R32(tp, OCPDR) : ~0;
1084 }
1085 
1086 static u32 r8168ep_ocp_read(struct rtl8169_private *tp, u16 reg)
1087 {
1088 	return _rtl_eri_read(tp, reg, ERIAR_OOB);
1089 }
1090 
1091 static void r8168dp_ocp_write(struct rtl8169_private *tp, u8 mask, u16 reg,
1092 			      u32 data)
1093 {
1094 	RTL_W32(tp, OCPDR, data);
1095 	RTL_W32(tp, OCPAR, OCPAR_FLAG | ((u32)mask & 0x0f) << 12 | (reg & 0x0fff));
1096 	rtl_loop_wait_low(tp, &rtl_ocpar_cond, 100, 20);
1097 }
1098 
1099 static void r8168ep_ocp_write(struct rtl8169_private *tp, u8 mask, u16 reg,
1100 			      u32 data)
1101 {
1102 	_rtl_eri_write(tp, reg, ((u32)mask & 0x0f) << ERIAR_MASK_SHIFT,
1103 		       data, ERIAR_OOB);
1104 }
1105 
1106 static void r8168dp_oob_notify(struct rtl8169_private *tp, u8 cmd)
1107 {
1108 	rtl_eri_write(tp, 0xe8, ERIAR_MASK_0001, cmd);
1109 
1110 	r8168dp_ocp_write(tp, 0x1, 0x30, 0x00000001);
1111 }
1112 
1113 #define OOB_CMD_RESET		0x00
1114 #define OOB_CMD_DRIVER_START	0x05
1115 #define OOB_CMD_DRIVER_STOP	0x06
1116 
1117 static u16 rtl8168_get_ocp_reg(struct rtl8169_private *tp)
1118 {
1119 	return (tp->mac_version == RTL_GIGA_MAC_VER_31) ? 0xb8 : 0x10;
1120 }
1121 
1122 DECLARE_RTL_COND(rtl_dp_ocp_read_cond)
1123 {
1124 	u16 reg;
1125 
1126 	reg = rtl8168_get_ocp_reg(tp);
1127 
1128 	return r8168dp_ocp_read(tp, reg) & 0x00000800;
1129 }
1130 
1131 DECLARE_RTL_COND(rtl_ep_ocp_read_cond)
1132 {
1133 	return r8168ep_ocp_read(tp, 0x124) & 0x00000001;
1134 }
1135 
1136 DECLARE_RTL_COND(rtl_ocp_tx_cond)
1137 {
1138 	return RTL_R8(tp, IBISR0) & 0x20;
1139 }
1140 
1141 static void rtl8168ep_stop_cmac(struct rtl8169_private *tp)
1142 {
1143 	RTL_W8(tp, IBCR2, RTL_R8(tp, IBCR2) & ~0x01);
1144 	rtl_loop_wait_high(tp, &rtl_ocp_tx_cond, 50000, 2000);
1145 	RTL_W8(tp, IBISR0, RTL_R8(tp, IBISR0) | 0x20);
1146 	RTL_W8(tp, IBCR0, RTL_R8(tp, IBCR0) & ~0x01);
1147 }
1148 
1149 static void rtl8168dp_driver_start(struct rtl8169_private *tp)
1150 {
1151 	r8168dp_oob_notify(tp, OOB_CMD_DRIVER_START);
1152 	rtl_loop_wait_high(tp, &rtl_dp_ocp_read_cond, 10000, 10);
1153 }
1154 
1155 static void rtl8168ep_driver_start(struct rtl8169_private *tp)
1156 {
1157 	r8168ep_ocp_write(tp, 0x01, 0x180, OOB_CMD_DRIVER_START);
1158 	r8168ep_ocp_write(tp, 0x01, 0x30, r8168ep_ocp_read(tp, 0x30) | 0x01);
1159 	rtl_loop_wait_high(tp, &rtl_ep_ocp_read_cond, 10000, 10);
1160 }
1161 
1162 static void rtl8168_driver_start(struct rtl8169_private *tp)
1163 {
1164 	if (tp->dash_type == RTL_DASH_DP)
1165 		rtl8168dp_driver_start(tp);
1166 	else
1167 		rtl8168ep_driver_start(tp);
1168 }
1169 
1170 static void rtl8168dp_driver_stop(struct rtl8169_private *tp)
1171 {
1172 	r8168dp_oob_notify(tp, OOB_CMD_DRIVER_STOP);
1173 	rtl_loop_wait_low(tp, &rtl_dp_ocp_read_cond, 10000, 10);
1174 }
1175 
1176 static void rtl8168ep_driver_stop(struct rtl8169_private *tp)
1177 {
1178 	rtl8168ep_stop_cmac(tp);
1179 	r8168ep_ocp_write(tp, 0x01, 0x180, OOB_CMD_DRIVER_STOP);
1180 	r8168ep_ocp_write(tp, 0x01, 0x30, r8168ep_ocp_read(tp, 0x30) | 0x01);
1181 	rtl_loop_wait_low(tp, &rtl_ep_ocp_read_cond, 10000, 10);
1182 }
1183 
1184 static void rtl8168_driver_stop(struct rtl8169_private *tp)
1185 {
1186 	if (tp->dash_type == RTL_DASH_DP)
1187 		rtl8168dp_driver_stop(tp);
1188 	else
1189 		rtl8168ep_driver_stop(tp);
1190 }
1191 
1192 static bool r8168dp_check_dash(struct rtl8169_private *tp)
1193 {
1194 	u16 reg = rtl8168_get_ocp_reg(tp);
1195 
1196 	return r8168dp_ocp_read(tp, reg) & BIT(15);
1197 }
1198 
1199 static bool r8168ep_check_dash(struct rtl8169_private *tp)
1200 {
1201 	return r8168ep_ocp_read(tp, 0x128) & BIT(0);
1202 }
1203 
1204 static enum rtl_dash_type rtl_check_dash(struct rtl8169_private *tp)
1205 {
1206 	switch (tp->mac_version) {
1207 	case RTL_GIGA_MAC_VER_28:
1208 	case RTL_GIGA_MAC_VER_31:
1209 		return r8168dp_check_dash(tp) ? RTL_DASH_DP : RTL_DASH_NONE;
1210 	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_53:
1211 		return r8168ep_check_dash(tp) ? RTL_DASH_EP : RTL_DASH_NONE;
1212 	default:
1213 		return RTL_DASH_NONE;
1214 	}
1215 }
1216 
1217 static void rtl_set_d3_pll_down(struct rtl8169_private *tp, bool enable)
1218 {
1219 	switch (tp->mac_version) {
1220 	case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_26:
1221 	case RTL_GIGA_MAC_VER_29 ... RTL_GIGA_MAC_VER_30:
1222 	case RTL_GIGA_MAC_VER_32 ... RTL_GIGA_MAC_VER_37:
1223 	case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_63:
1224 		if (enable)
1225 			RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) & ~D3_NO_PLL_DOWN);
1226 		else
1227 			RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | D3_NO_PLL_DOWN);
1228 		break;
1229 	default:
1230 		break;
1231 	}
1232 }
1233 
1234 static void rtl_reset_packet_filter(struct rtl8169_private *tp)
1235 {
1236 	rtl_eri_clear_bits(tp, 0xdc, BIT(0));
1237 	rtl_eri_set_bits(tp, 0xdc, BIT(0));
1238 }
1239 
1240 DECLARE_RTL_COND(rtl_efusear_cond)
1241 {
1242 	return RTL_R32(tp, EFUSEAR) & EFUSEAR_FLAG;
1243 }
1244 
1245 u8 rtl8168d_efuse_read(struct rtl8169_private *tp, int reg_addr)
1246 {
1247 	RTL_W32(tp, EFUSEAR, (reg_addr & EFUSEAR_REG_MASK) << EFUSEAR_REG_SHIFT);
1248 
1249 	return rtl_loop_wait_high(tp, &rtl_efusear_cond, 100, 300) ?
1250 		RTL_R32(tp, EFUSEAR) & EFUSEAR_DATA_MASK : ~0;
1251 }
1252 
1253 static u32 rtl_get_events(struct rtl8169_private *tp)
1254 {
1255 	if (rtl_is_8125(tp))
1256 		return RTL_R32(tp, IntrStatus_8125);
1257 	else
1258 		return RTL_R16(tp, IntrStatus);
1259 }
1260 
1261 static void rtl_ack_events(struct rtl8169_private *tp, u32 bits)
1262 {
1263 	if (rtl_is_8125(tp))
1264 		RTL_W32(tp, IntrStatus_8125, bits);
1265 	else
1266 		RTL_W16(tp, IntrStatus, bits);
1267 }
1268 
1269 static void rtl_irq_disable(struct rtl8169_private *tp)
1270 {
1271 	if (rtl_is_8125(tp))
1272 		RTL_W32(tp, IntrMask_8125, 0);
1273 	else
1274 		RTL_W16(tp, IntrMask, 0);
1275 }
1276 
1277 static void rtl_irq_enable(struct rtl8169_private *tp)
1278 {
1279 	if (rtl_is_8125(tp))
1280 		RTL_W32(tp, IntrMask_8125, tp->irq_mask);
1281 	else
1282 		RTL_W16(tp, IntrMask, tp->irq_mask);
1283 }
1284 
1285 static void rtl8169_irq_mask_and_ack(struct rtl8169_private *tp)
1286 {
1287 	rtl_irq_disable(tp);
1288 	rtl_ack_events(tp, 0xffffffff);
1289 	rtl_pci_commit(tp);
1290 }
1291 
1292 static void rtl_link_chg_patch(struct rtl8169_private *tp)
1293 {
1294 	struct phy_device *phydev = tp->phydev;
1295 
1296 	if (tp->mac_version == RTL_GIGA_MAC_VER_34 ||
1297 	    tp->mac_version == RTL_GIGA_MAC_VER_38) {
1298 		if (phydev->speed == SPEED_1000) {
1299 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x00000011);
1300 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1301 		} else if (phydev->speed == SPEED_100) {
1302 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1303 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1304 		} else {
1305 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1306 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x0000003f);
1307 		}
1308 		rtl_reset_packet_filter(tp);
1309 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_35 ||
1310 		   tp->mac_version == RTL_GIGA_MAC_VER_36) {
1311 		if (phydev->speed == SPEED_1000) {
1312 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x00000011);
1313 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1314 		} else {
1315 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1316 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x0000003f);
1317 		}
1318 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_37) {
1319 		if (phydev->speed == SPEED_10) {
1320 			rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x4d02);
1321 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_0011, 0x0060a);
1322 		} else {
1323 			rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x0000);
1324 		}
1325 	}
1326 }
1327 
1328 #define WAKE_ANY (WAKE_PHY | WAKE_MAGIC | WAKE_UCAST | WAKE_BCAST | WAKE_MCAST)
1329 
1330 static void rtl8169_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1331 {
1332 	struct rtl8169_private *tp = netdev_priv(dev);
1333 
1334 	wol->supported = WAKE_ANY;
1335 	wol->wolopts = tp->saved_wolopts;
1336 }
1337 
1338 static void __rtl8169_set_wol(struct rtl8169_private *tp, u32 wolopts)
1339 {
1340 	static const struct {
1341 		u32 opt;
1342 		u16 reg;
1343 		u8  mask;
1344 	} cfg[] = {
1345 		{ WAKE_PHY,   Config3, LinkUp },
1346 		{ WAKE_UCAST, Config5, UWF },
1347 		{ WAKE_BCAST, Config5, BWF },
1348 		{ WAKE_MCAST, Config5, MWF },
1349 		{ WAKE_ANY,   Config5, LanWake },
1350 		{ WAKE_MAGIC, Config3, MagicPacket }
1351 	};
1352 	unsigned int i, tmp = ARRAY_SIZE(cfg);
1353 	u8 options;
1354 
1355 	rtl_unlock_config_regs(tp);
1356 
1357 	if (rtl_is_8168evl_up(tp)) {
1358 		tmp--;
1359 		if (wolopts & WAKE_MAGIC)
1360 			rtl_eri_set_bits(tp, 0x0dc, MagicPacket_v2);
1361 		else
1362 			rtl_eri_clear_bits(tp, 0x0dc, MagicPacket_v2);
1363 	} else if (rtl_is_8125(tp)) {
1364 		tmp--;
1365 		if (wolopts & WAKE_MAGIC)
1366 			r8168_mac_ocp_modify(tp, 0xc0b6, 0, BIT(0));
1367 		else
1368 			r8168_mac_ocp_modify(tp, 0xc0b6, BIT(0), 0);
1369 	}
1370 
1371 	for (i = 0; i < tmp; i++) {
1372 		options = RTL_R8(tp, cfg[i].reg) & ~cfg[i].mask;
1373 		if (wolopts & cfg[i].opt)
1374 			options |= cfg[i].mask;
1375 		RTL_W8(tp, cfg[i].reg, options);
1376 	}
1377 
1378 	switch (tp->mac_version) {
1379 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
1380 		options = RTL_R8(tp, Config1) & ~PMEnable;
1381 		if (wolopts)
1382 			options |= PMEnable;
1383 		RTL_W8(tp, Config1, options);
1384 		break;
1385 	case RTL_GIGA_MAC_VER_34:
1386 	case RTL_GIGA_MAC_VER_37:
1387 	case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_63:
1388 		options = RTL_R8(tp, Config2) & ~PME_SIGNAL;
1389 		if (wolopts)
1390 			options |= PME_SIGNAL;
1391 		RTL_W8(tp, Config2, options);
1392 		break;
1393 	default:
1394 		break;
1395 	}
1396 
1397 	rtl_lock_config_regs(tp);
1398 
1399 	device_set_wakeup_enable(tp_to_dev(tp), wolopts);
1400 
1401 	if (tp->dash_type == RTL_DASH_NONE) {
1402 		rtl_set_d3_pll_down(tp, !wolopts);
1403 		tp->dev->wol_enabled = wolopts ? 1 : 0;
1404 	}
1405 }
1406 
1407 static int rtl8169_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1408 {
1409 	struct rtl8169_private *tp = netdev_priv(dev);
1410 
1411 	if (wol->wolopts & ~WAKE_ANY)
1412 		return -EINVAL;
1413 
1414 	tp->saved_wolopts = wol->wolopts;
1415 	__rtl8169_set_wol(tp, tp->saved_wolopts);
1416 
1417 	return 0;
1418 }
1419 
1420 static void rtl8169_get_drvinfo(struct net_device *dev,
1421 				struct ethtool_drvinfo *info)
1422 {
1423 	struct rtl8169_private *tp = netdev_priv(dev);
1424 	struct rtl_fw *rtl_fw = tp->rtl_fw;
1425 
1426 	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1427 	strlcpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
1428 	BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version));
1429 	if (rtl_fw)
1430 		strlcpy(info->fw_version, rtl_fw->version,
1431 			sizeof(info->fw_version));
1432 }
1433 
1434 static int rtl8169_get_regs_len(struct net_device *dev)
1435 {
1436 	return R8169_REGS_SIZE;
1437 }
1438 
1439 static netdev_features_t rtl8169_fix_features(struct net_device *dev,
1440 	netdev_features_t features)
1441 {
1442 	struct rtl8169_private *tp = netdev_priv(dev);
1443 
1444 	if (dev->mtu > TD_MSS_MAX)
1445 		features &= ~NETIF_F_ALL_TSO;
1446 
1447 	if (dev->mtu > ETH_DATA_LEN &&
1448 	    tp->mac_version > RTL_GIGA_MAC_VER_06)
1449 		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_ALL_TSO);
1450 
1451 	return features;
1452 }
1453 
1454 static void rtl_set_rx_config_features(struct rtl8169_private *tp,
1455 				       netdev_features_t features)
1456 {
1457 	u32 rx_config = RTL_R32(tp, RxConfig);
1458 
1459 	if (features & NETIF_F_RXALL)
1460 		rx_config |= RX_CONFIG_ACCEPT_ERR_MASK;
1461 	else
1462 		rx_config &= ~RX_CONFIG_ACCEPT_ERR_MASK;
1463 
1464 	if (rtl_is_8125(tp)) {
1465 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
1466 			rx_config |= RX_VLAN_8125;
1467 		else
1468 			rx_config &= ~RX_VLAN_8125;
1469 	}
1470 
1471 	RTL_W32(tp, RxConfig, rx_config);
1472 }
1473 
1474 static int rtl8169_set_features(struct net_device *dev,
1475 				netdev_features_t features)
1476 {
1477 	struct rtl8169_private *tp = netdev_priv(dev);
1478 
1479 	rtl_set_rx_config_features(tp, features);
1480 
1481 	if (features & NETIF_F_RXCSUM)
1482 		tp->cp_cmd |= RxChkSum;
1483 	else
1484 		tp->cp_cmd &= ~RxChkSum;
1485 
1486 	if (!rtl_is_8125(tp)) {
1487 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
1488 			tp->cp_cmd |= RxVlan;
1489 		else
1490 			tp->cp_cmd &= ~RxVlan;
1491 	}
1492 
1493 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
1494 	rtl_pci_commit(tp);
1495 
1496 	return 0;
1497 }
1498 
1499 static inline u32 rtl8169_tx_vlan_tag(struct sk_buff *skb)
1500 {
1501 	return (skb_vlan_tag_present(skb)) ?
1502 		TxVlanTag | swab16(skb_vlan_tag_get(skb)) : 0x00;
1503 }
1504 
1505 static void rtl8169_rx_vlan_tag(struct RxDesc *desc, struct sk_buff *skb)
1506 {
1507 	u32 opts2 = le32_to_cpu(desc->opts2);
1508 
1509 	if (opts2 & RxVlanTag)
1510 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), swab16(opts2 & 0xffff));
1511 }
1512 
1513 static void rtl8169_get_regs(struct net_device *dev, struct ethtool_regs *regs,
1514 			     void *p)
1515 {
1516 	struct rtl8169_private *tp = netdev_priv(dev);
1517 	u32 __iomem *data = tp->mmio_addr;
1518 	u32 *dw = p;
1519 	int i;
1520 
1521 	for (i = 0; i < R8169_REGS_SIZE; i += 4)
1522 		memcpy_fromio(dw++, data++, 4);
1523 }
1524 
1525 static const char rtl8169_gstrings[][ETH_GSTRING_LEN] = {
1526 	"tx_packets",
1527 	"rx_packets",
1528 	"tx_errors",
1529 	"rx_errors",
1530 	"rx_missed",
1531 	"align_errors",
1532 	"tx_single_collisions",
1533 	"tx_multi_collisions",
1534 	"unicast",
1535 	"broadcast",
1536 	"multicast",
1537 	"tx_aborted",
1538 	"tx_underrun",
1539 };
1540 
1541 static int rtl8169_get_sset_count(struct net_device *dev, int sset)
1542 {
1543 	switch (sset) {
1544 	case ETH_SS_STATS:
1545 		return ARRAY_SIZE(rtl8169_gstrings);
1546 	default:
1547 		return -EOPNOTSUPP;
1548 	}
1549 }
1550 
1551 DECLARE_RTL_COND(rtl_counters_cond)
1552 {
1553 	return RTL_R32(tp, CounterAddrLow) & (CounterReset | CounterDump);
1554 }
1555 
1556 static void rtl8169_do_counters(struct rtl8169_private *tp, u32 counter_cmd)
1557 {
1558 	u32 cmd = lower_32_bits(tp->counters_phys_addr);
1559 
1560 	RTL_W32(tp, CounterAddrHigh, upper_32_bits(tp->counters_phys_addr));
1561 	rtl_pci_commit(tp);
1562 	RTL_W32(tp, CounterAddrLow, cmd);
1563 	RTL_W32(tp, CounterAddrLow, cmd | counter_cmd);
1564 
1565 	rtl_loop_wait_low(tp, &rtl_counters_cond, 10, 1000);
1566 }
1567 
1568 static void rtl8169_update_counters(struct rtl8169_private *tp)
1569 {
1570 	u8 val = RTL_R8(tp, ChipCmd);
1571 
1572 	/*
1573 	 * Some chips are unable to dump tally counters when the receiver
1574 	 * is disabled. If 0xff chip may be in a PCI power-save state.
1575 	 */
1576 	if (val & CmdRxEnb && val != 0xff)
1577 		rtl8169_do_counters(tp, CounterDump);
1578 }
1579 
1580 static void rtl8169_init_counter_offsets(struct rtl8169_private *tp)
1581 {
1582 	struct rtl8169_counters *counters = tp->counters;
1583 
1584 	/*
1585 	 * rtl8169_init_counter_offsets is called from rtl_open.  On chip
1586 	 * versions prior to RTL_GIGA_MAC_VER_19 the tally counters are only
1587 	 * reset by a power cycle, while the counter values collected by the
1588 	 * driver are reset at every driver unload/load cycle.
1589 	 *
1590 	 * To make sure the HW values returned by @get_stats64 match the SW
1591 	 * values, we collect the initial values at first open(*) and use them
1592 	 * as offsets to normalize the values returned by @get_stats64.
1593 	 *
1594 	 * (*) We can't call rtl8169_init_counter_offsets from rtl_init_one
1595 	 * for the reason stated in rtl8169_update_counters; CmdRxEnb is only
1596 	 * set at open time by rtl_hw_start.
1597 	 */
1598 
1599 	if (tp->tc_offset.inited)
1600 		return;
1601 
1602 	if (tp->mac_version >= RTL_GIGA_MAC_VER_19) {
1603 		rtl8169_do_counters(tp, CounterReset);
1604 	} else {
1605 		rtl8169_update_counters(tp);
1606 		tp->tc_offset.tx_errors = counters->tx_errors;
1607 		tp->tc_offset.tx_multi_collision = counters->tx_multi_collision;
1608 		tp->tc_offset.tx_aborted = counters->tx_aborted;
1609 		tp->tc_offset.rx_missed = counters->rx_missed;
1610 	}
1611 
1612 	tp->tc_offset.inited = true;
1613 }
1614 
1615 static void rtl8169_get_ethtool_stats(struct net_device *dev,
1616 				      struct ethtool_stats *stats, u64 *data)
1617 {
1618 	struct rtl8169_private *tp = netdev_priv(dev);
1619 	struct rtl8169_counters *counters;
1620 
1621 	counters = tp->counters;
1622 	rtl8169_update_counters(tp);
1623 
1624 	data[0] = le64_to_cpu(counters->tx_packets);
1625 	data[1] = le64_to_cpu(counters->rx_packets);
1626 	data[2] = le64_to_cpu(counters->tx_errors);
1627 	data[3] = le32_to_cpu(counters->rx_errors);
1628 	data[4] = le16_to_cpu(counters->rx_missed);
1629 	data[5] = le16_to_cpu(counters->align_errors);
1630 	data[6] = le32_to_cpu(counters->tx_one_collision);
1631 	data[7] = le32_to_cpu(counters->tx_multi_collision);
1632 	data[8] = le64_to_cpu(counters->rx_unicast);
1633 	data[9] = le64_to_cpu(counters->rx_broadcast);
1634 	data[10] = le32_to_cpu(counters->rx_multicast);
1635 	data[11] = le16_to_cpu(counters->tx_aborted);
1636 	data[12] = le16_to_cpu(counters->tx_underun);
1637 }
1638 
1639 static void rtl8169_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1640 {
1641 	switch(stringset) {
1642 	case ETH_SS_STATS:
1643 		memcpy(data, rtl8169_gstrings, sizeof(rtl8169_gstrings));
1644 		break;
1645 	}
1646 }
1647 
1648 /*
1649  * Interrupt coalescing
1650  *
1651  * > 1 - the availability of the IntrMitigate (0xe2) register through the
1652  * >     8169, 8168 and 810x line of chipsets
1653  *
1654  * 8169, 8168, and 8136(810x) serial chipsets support it.
1655  *
1656  * > 2 - the Tx timer unit at gigabit speed
1657  *
1658  * The unit of the timer depends on both the speed and the setting of CPlusCmd
1659  * (0xe0) bit 1 and bit 0.
1660  *
1661  * For 8169
1662  * bit[1:0] \ speed        1000M           100M            10M
1663  * 0 0                     320ns           2.56us          40.96us
1664  * 0 1                     2.56us          20.48us         327.7us
1665  * 1 0                     5.12us          40.96us         655.4us
1666  * 1 1                     10.24us         81.92us         1.31ms
1667  *
1668  * For the other
1669  * bit[1:0] \ speed        1000M           100M            10M
1670  * 0 0                     5us             2.56us          40.96us
1671  * 0 1                     40us            20.48us         327.7us
1672  * 1 0                     80us            40.96us         655.4us
1673  * 1 1                     160us           81.92us         1.31ms
1674  */
1675 
1676 /* rx/tx scale factors for all CPlusCmd[0:1] cases */
1677 struct rtl_coalesce_info {
1678 	u32 speed;
1679 	u32 scale_nsecs[4];
1680 };
1681 
1682 /* produce array with base delay *1, *8, *8*2, *8*2*2 */
1683 #define COALESCE_DELAY(d) { (d), 8 * (d), 16 * (d), 32 * (d) }
1684 
1685 static const struct rtl_coalesce_info rtl_coalesce_info_8169[] = {
1686 	{ SPEED_1000,	COALESCE_DELAY(320) },
1687 	{ SPEED_100,	COALESCE_DELAY(2560) },
1688 	{ SPEED_10,	COALESCE_DELAY(40960) },
1689 	{ 0 },
1690 };
1691 
1692 static const struct rtl_coalesce_info rtl_coalesce_info_8168_8136[] = {
1693 	{ SPEED_1000,	COALESCE_DELAY(5000) },
1694 	{ SPEED_100,	COALESCE_DELAY(2560) },
1695 	{ SPEED_10,	COALESCE_DELAY(40960) },
1696 	{ 0 },
1697 };
1698 #undef COALESCE_DELAY
1699 
1700 /* get rx/tx scale vector corresponding to current speed */
1701 static const struct rtl_coalesce_info *
1702 rtl_coalesce_info(struct rtl8169_private *tp)
1703 {
1704 	const struct rtl_coalesce_info *ci;
1705 
1706 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
1707 		ci = rtl_coalesce_info_8169;
1708 	else
1709 		ci = rtl_coalesce_info_8168_8136;
1710 
1711 	/* if speed is unknown assume highest one */
1712 	if (tp->phydev->speed == SPEED_UNKNOWN)
1713 		return ci;
1714 
1715 	for (; ci->speed; ci++) {
1716 		if (tp->phydev->speed == ci->speed)
1717 			return ci;
1718 	}
1719 
1720 	return ERR_PTR(-ELNRNG);
1721 }
1722 
1723 static int rtl_get_coalesce(struct net_device *dev,
1724 			    struct ethtool_coalesce *ec,
1725 			    struct kernel_ethtool_coalesce *kernel_coal,
1726 			    struct netlink_ext_ack *extack)
1727 {
1728 	struct rtl8169_private *tp = netdev_priv(dev);
1729 	const struct rtl_coalesce_info *ci;
1730 	u32 scale, c_us, c_fr;
1731 	u16 intrmit;
1732 
1733 	if (rtl_is_8125(tp))
1734 		return -EOPNOTSUPP;
1735 
1736 	memset(ec, 0, sizeof(*ec));
1737 
1738 	/* get rx/tx scale corresponding to current speed and CPlusCmd[0:1] */
1739 	ci = rtl_coalesce_info(tp);
1740 	if (IS_ERR(ci))
1741 		return PTR_ERR(ci);
1742 
1743 	scale = ci->scale_nsecs[tp->cp_cmd & INTT_MASK];
1744 
1745 	intrmit = RTL_R16(tp, IntrMitigate);
1746 
1747 	c_us = FIELD_GET(RTL_COALESCE_TX_USECS, intrmit);
1748 	ec->tx_coalesce_usecs = DIV_ROUND_UP(c_us * scale, 1000);
1749 
1750 	c_fr = FIELD_GET(RTL_COALESCE_TX_FRAMES, intrmit);
1751 	/* ethtool_coalesce states usecs and max_frames must not both be 0 */
1752 	ec->tx_max_coalesced_frames = (c_us || c_fr) ? c_fr * 4 : 1;
1753 
1754 	c_us = FIELD_GET(RTL_COALESCE_RX_USECS, intrmit);
1755 	ec->rx_coalesce_usecs = DIV_ROUND_UP(c_us * scale, 1000);
1756 
1757 	c_fr = FIELD_GET(RTL_COALESCE_RX_FRAMES, intrmit);
1758 	ec->rx_max_coalesced_frames = (c_us || c_fr) ? c_fr * 4 : 1;
1759 
1760 	return 0;
1761 }
1762 
1763 /* choose appropriate scale factor and CPlusCmd[0:1] for (speed, usec) */
1764 static int rtl_coalesce_choose_scale(struct rtl8169_private *tp, u32 usec,
1765 				     u16 *cp01)
1766 {
1767 	const struct rtl_coalesce_info *ci;
1768 	u16 i;
1769 
1770 	ci = rtl_coalesce_info(tp);
1771 	if (IS_ERR(ci))
1772 		return PTR_ERR(ci);
1773 
1774 	for (i = 0; i < 4; i++) {
1775 		if (usec <= ci->scale_nsecs[i] * RTL_COALESCE_T_MAX / 1000U) {
1776 			*cp01 = i;
1777 			return ci->scale_nsecs[i];
1778 		}
1779 	}
1780 
1781 	return -ERANGE;
1782 }
1783 
1784 static int rtl_set_coalesce(struct net_device *dev,
1785 			    struct ethtool_coalesce *ec,
1786 			    struct kernel_ethtool_coalesce *kernel_coal,
1787 			    struct netlink_ext_ack *extack)
1788 {
1789 	struct rtl8169_private *tp = netdev_priv(dev);
1790 	u32 tx_fr = ec->tx_max_coalesced_frames;
1791 	u32 rx_fr = ec->rx_max_coalesced_frames;
1792 	u32 coal_usec_max, units;
1793 	u16 w = 0, cp01 = 0;
1794 	int scale;
1795 
1796 	if (rtl_is_8125(tp))
1797 		return -EOPNOTSUPP;
1798 
1799 	if (rx_fr > RTL_COALESCE_FRAME_MAX || tx_fr > RTL_COALESCE_FRAME_MAX)
1800 		return -ERANGE;
1801 
1802 	coal_usec_max = max(ec->rx_coalesce_usecs, ec->tx_coalesce_usecs);
1803 	scale = rtl_coalesce_choose_scale(tp, coal_usec_max, &cp01);
1804 	if (scale < 0)
1805 		return scale;
1806 
1807 	/* Accept max_frames=1 we returned in rtl_get_coalesce. Accept it
1808 	 * not only when usecs=0 because of e.g. the following scenario:
1809 	 *
1810 	 * - both rx_usecs=0 & rx_frames=0 in hardware (no delay on RX)
1811 	 * - rtl_get_coalesce returns rx_usecs=0, rx_frames=1
1812 	 * - then user does `ethtool -C eth0 rx-usecs 100`
1813 	 *
1814 	 * Since ethtool sends to kernel whole ethtool_coalesce settings,
1815 	 * if we want to ignore rx_frames then it has to be set to 0.
1816 	 */
1817 	if (rx_fr == 1)
1818 		rx_fr = 0;
1819 	if (tx_fr == 1)
1820 		tx_fr = 0;
1821 
1822 	/* HW requires time limit to be set if frame limit is set */
1823 	if ((tx_fr && !ec->tx_coalesce_usecs) ||
1824 	    (rx_fr && !ec->rx_coalesce_usecs))
1825 		return -EINVAL;
1826 
1827 	w |= FIELD_PREP(RTL_COALESCE_TX_FRAMES, DIV_ROUND_UP(tx_fr, 4));
1828 	w |= FIELD_PREP(RTL_COALESCE_RX_FRAMES, DIV_ROUND_UP(rx_fr, 4));
1829 
1830 	units = DIV_ROUND_UP(ec->tx_coalesce_usecs * 1000U, scale);
1831 	w |= FIELD_PREP(RTL_COALESCE_TX_USECS, units);
1832 	units = DIV_ROUND_UP(ec->rx_coalesce_usecs * 1000U, scale);
1833 	w |= FIELD_PREP(RTL_COALESCE_RX_USECS, units);
1834 
1835 	RTL_W16(tp, IntrMitigate, w);
1836 
1837 	/* Meaning of PktCntrDisable bit changed from RTL8168e-vl */
1838 	if (rtl_is_8168evl_up(tp)) {
1839 		if (!rx_fr && !tx_fr)
1840 			/* disable packet counter */
1841 			tp->cp_cmd |= PktCntrDisable;
1842 		else
1843 			tp->cp_cmd &= ~PktCntrDisable;
1844 	}
1845 
1846 	tp->cp_cmd = (tp->cp_cmd & ~INTT_MASK) | cp01;
1847 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
1848 	rtl_pci_commit(tp);
1849 
1850 	return 0;
1851 }
1852 
1853 static int rtl8169_get_eee(struct net_device *dev, struct ethtool_eee *data)
1854 {
1855 	struct rtl8169_private *tp = netdev_priv(dev);
1856 
1857 	if (!rtl_supports_eee(tp))
1858 		return -EOPNOTSUPP;
1859 
1860 	return phy_ethtool_get_eee(tp->phydev, data);
1861 }
1862 
1863 static int rtl8169_set_eee(struct net_device *dev, struct ethtool_eee *data)
1864 {
1865 	struct rtl8169_private *tp = netdev_priv(dev);
1866 	int ret;
1867 
1868 	if (!rtl_supports_eee(tp))
1869 		return -EOPNOTSUPP;
1870 
1871 	ret = phy_ethtool_set_eee(tp->phydev, data);
1872 
1873 	if (!ret)
1874 		tp->eee_adv = phy_read_mmd(dev->phydev, MDIO_MMD_AN,
1875 					   MDIO_AN_EEE_ADV);
1876 	return ret;
1877 }
1878 
1879 static void rtl8169_get_ringparam(struct net_device *dev,
1880 				  struct ethtool_ringparam *data,
1881 				  struct kernel_ethtool_ringparam *kernel_data,
1882 				  struct netlink_ext_ack *extack)
1883 {
1884 	data->rx_max_pending = NUM_RX_DESC;
1885 	data->rx_pending = NUM_RX_DESC;
1886 	data->tx_max_pending = NUM_TX_DESC;
1887 	data->tx_pending = NUM_TX_DESC;
1888 }
1889 
1890 static void rtl8169_get_pauseparam(struct net_device *dev,
1891 				   struct ethtool_pauseparam *data)
1892 {
1893 	struct rtl8169_private *tp = netdev_priv(dev);
1894 	bool tx_pause, rx_pause;
1895 
1896 	phy_get_pause(tp->phydev, &tx_pause, &rx_pause);
1897 
1898 	data->autoneg = tp->phydev->autoneg;
1899 	data->tx_pause = tx_pause ? 1 : 0;
1900 	data->rx_pause = rx_pause ? 1 : 0;
1901 }
1902 
1903 static int rtl8169_set_pauseparam(struct net_device *dev,
1904 				  struct ethtool_pauseparam *data)
1905 {
1906 	struct rtl8169_private *tp = netdev_priv(dev);
1907 
1908 	if (dev->mtu > ETH_DATA_LEN)
1909 		return -EOPNOTSUPP;
1910 
1911 	phy_set_asym_pause(tp->phydev, data->rx_pause, data->tx_pause);
1912 
1913 	return 0;
1914 }
1915 
1916 static const struct ethtool_ops rtl8169_ethtool_ops = {
1917 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
1918 				     ETHTOOL_COALESCE_MAX_FRAMES,
1919 	.get_drvinfo		= rtl8169_get_drvinfo,
1920 	.get_regs_len		= rtl8169_get_regs_len,
1921 	.get_link		= ethtool_op_get_link,
1922 	.get_coalesce		= rtl_get_coalesce,
1923 	.set_coalesce		= rtl_set_coalesce,
1924 	.get_regs		= rtl8169_get_regs,
1925 	.get_wol		= rtl8169_get_wol,
1926 	.set_wol		= rtl8169_set_wol,
1927 	.get_strings		= rtl8169_get_strings,
1928 	.get_sset_count		= rtl8169_get_sset_count,
1929 	.get_ethtool_stats	= rtl8169_get_ethtool_stats,
1930 	.get_ts_info		= ethtool_op_get_ts_info,
1931 	.nway_reset		= phy_ethtool_nway_reset,
1932 	.get_eee		= rtl8169_get_eee,
1933 	.set_eee		= rtl8169_set_eee,
1934 	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
1935 	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
1936 	.get_ringparam		= rtl8169_get_ringparam,
1937 	.get_pauseparam		= rtl8169_get_pauseparam,
1938 	.set_pauseparam		= rtl8169_set_pauseparam,
1939 };
1940 
1941 static void rtl_enable_eee(struct rtl8169_private *tp)
1942 {
1943 	struct phy_device *phydev = tp->phydev;
1944 	int adv;
1945 
1946 	/* respect EEE advertisement the user may have set */
1947 	if (tp->eee_adv >= 0)
1948 		adv = tp->eee_adv;
1949 	else
1950 		adv = phy_read_mmd(phydev, MDIO_MMD_PCS, MDIO_PCS_EEE_ABLE);
1951 
1952 	if (adv >= 0)
1953 		phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_EEE_ADV, adv);
1954 }
1955 
1956 static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii)
1957 {
1958 	/*
1959 	 * The driver currently handles the 8168Bf and the 8168Be identically
1960 	 * but they can be identified more specifically through the test below
1961 	 * if needed:
1962 	 *
1963 	 * (RTL_R32(tp, TxConfig) & 0x700000) == 0x500000 ? 8168Bf : 8168Be
1964 	 *
1965 	 * Same thing for the 8101Eb and the 8101Ec:
1966 	 *
1967 	 * (RTL_R32(tp, TxConfig) & 0x700000) == 0x200000 ? 8101Eb : 8101Ec
1968 	 */
1969 	static const struct rtl_mac_info {
1970 		u16 mask;
1971 		u16 val;
1972 		enum mac_version ver;
1973 	} mac_info[] = {
1974 		/* 8125B family. */
1975 		{ 0x7cf, 0x641,	RTL_GIGA_MAC_VER_63 },
1976 
1977 		/* 8125A family. */
1978 		{ 0x7cf, 0x609,	RTL_GIGA_MAC_VER_61 },
1979 		/* It seems only XID 609 made it to the mass market.
1980 		 * { 0x7cf, 0x608,	RTL_GIGA_MAC_VER_60 },
1981 		 * { 0x7c8, 0x608,	RTL_GIGA_MAC_VER_61 },
1982 		 */
1983 
1984 		/* RTL8117 */
1985 		{ 0x7cf, 0x54b,	RTL_GIGA_MAC_VER_53 },
1986 		{ 0x7cf, 0x54a,	RTL_GIGA_MAC_VER_52 },
1987 
1988 		/* 8168EP family. */
1989 		{ 0x7cf, 0x502,	RTL_GIGA_MAC_VER_51 },
1990 		/* It seems this chip version never made it to
1991 		 * the wild. Let's disable detection.
1992 		 * { 0x7cf, 0x501,      RTL_GIGA_MAC_VER_50 },
1993 		 * { 0x7cf, 0x500,      RTL_GIGA_MAC_VER_49 },
1994 		 */
1995 
1996 		/* 8168H family. */
1997 		{ 0x7cf, 0x541,	RTL_GIGA_MAC_VER_46 },
1998 		/* It seems this chip version never made it to
1999 		 * the wild. Let's disable detection.
2000 		 * { 0x7cf, 0x540,	RTL_GIGA_MAC_VER_45 },
2001 		 */
2002 
2003 		/* 8168G family. */
2004 		{ 0x7cf, 0x5c8,	RTL_GIGA_MAC_VER_44 },
2005 		{ 0x7cf, 0x509,	RTL_GIGA_MAC_VER_42 },
2006 		/* It seems this chip version never made it to
2007 		 * the wild. Let's disable detection.
2008 		 * { 0x7cf, 0x4c1,	RTL_GIGA_MAC_VER_41 },
2009 		 */
2010 		{ 0x7cf, 0x4c0,	RTL_GIGA_MAC_VER_40 },
2011 
2012 		/* 8168F family. */
2013 		{ 0x7c8, 0x488,	RTL_GIGA_MAC_VER_38 },
2014 		{ 0x7cf, 0x481,	RTL_GIGA_MAC_VER_36 },
2015 		{ 0x7cf, 0x480,	RTL_GIGA_MAC_VER_35 },
2016 
2017 		/* 8168E family. */
2018 		{ 0x7c8, 0x2c8,	RTL_GIGA_MAC_VER_34 },
2019 		{ 0x7cf, 0x2c1,	RTL_GIGA_MAC_VER_32 },
2020 		{ 0x7c8, 0x2c0,	RTL_GIGA_MAC_VER_33 },
2021 
2022 		/* 8168D family. */
2023 		{ 0x7cf, 0x281,	RTL_GIGA_MAC_VER_25 },
2024 		{ 0x7c8, 0x280,	RTL_GIGA_MAC_VER_26 },
2025 
2026 		/* 8168DP family. */
2027 		/* It seems this early RTL8168dp version never made it to
2028 		 * the wild. Support has been removed.
2029 		 * { 0x7cf, 0x288,      RTL_GIGA_MAC_VER_27 },
2030 		 */
2031 		{ 0x7cf, 0x28a,	RTL_GIGA_MAC_VER_28 },
2032 		{ 0x7cf, 0x28b,	RTL_GIGA_MAC_VER_31 },
2033 
2034 		/* 8168C family. */
2035 		{ 0x7cf, 0x3c9,	RTL_GIGA_MAC_VER_23 },
2036 		{ 0x7cf, 0x3c8,	RTL_GIGA_MAC_VER_18 },
2037 		{ 0x7c8, 0x3c8,	RTL_GIGA_MAC_VER_24 },
2038 		{ 0x7cf, 0x3c0,	RTL_GIGA_MAC_VER_19 },
2039 		{ 0x7cf, 0x3c2,	RTL_GIGA_MAC_VER_20 },
2040 		{ 0x7cf, 0x3c3,	RTL_GIGA_MAC_VER_21 },
2041 		{ 0x7c8, 0x3c0,	RTL_GIGA_MAC_VER_22 },
2042 
2043 		/* 8168B family. */
2044 		{ 0x7cf, 0x380,	RTL_GIGA_MAC_VER_12 },
2045 		{ 0x7c8, 0x380,	RTL_GIGA_MAC_VER_17 },
2046 		{ 0x7c8, 0x300,	RTL_GIGA_MAC_VER_11 },
2047 
2048 		/* 8101 family. */
2049 		{ 0x7c8, 0x448,	RTL_GIGA_MAC_VER_39 },
2050 		{ 0x7c8, 0x440,	RTL_GIGA_MAC_VER_37 },
2051 		{ 0x7cf, 0x409,	RTL_GIGA_MAC_VER_29 },
2052 		{ 0x7c8, 0x408,	RTL_GIGA_MAC_VER_30 },
2053 		{ 0x7cf, 0x349,	RTL_GIGA_MAC_VER_08 },
2054 		{ 0x7cf, 0x249,	RTL_GIGA_MAC_VER_08 },
2055 		{ 0x7cf, 0x348,	RTL_GIGA_MAC_VER_07 },
2056 		{ 0x7cf, 0x248,	RTL_GIGA_MAC_VER_07 },
2057 		{ 0x7cf, 0x340,	RTL_GIGA_MAC_VER_13 },
2058 		{ 0x7cf, 0x240,	RTL_GIGA_MAC_VER_14 },
2059 		{ 0x7cf, 0x343,	RTL_GIGA_MAC_VER_10 },
2060 		{ 0x7cf, 0x342,	RTL_GIGA_MAC_VER_16 },
2061 		{ 0x7c8, 0x348,	RTL_GIGA_MAC_VER_09 },
2062 		{ 0x7c8, 0x248,	RTL_GIGA_MAC_VER_09 },
2063 		{ 0x7c8, 0x340,	RTL_GIGA_MAC_VER_16 },
2064 		/* FIXME: where did these entries come from ? -- FR
2065 		 * Not even r8101 vendor driver knows these id's,
2066 		 * so let's disable detection for now. -- HK
2067 		 * { 0xfc8, 0x388,	RTL_GIGA_MAC_VER_13 },
2068 		 * { 0xfc8, 0x308,	RTL_GIGA_MAC_VER_13 },
2069 		 */
2070 
2071 		/* 8110 family. */
2072 		{ 0xfc8, 0x980,	RTL_GIGA_MAC_VER_06 },
2073 		{ 0xfc8, 0x180,	RTL_GIGA_MAC_VER_05 },
2074 		{ 0xfc8, 0x100,	RTL_GIGA_MAC_VER_04 },
2075 		{ 0xfc8, 0x040,	RTL_GIGA_MAC_VER_03 },
2076 		{ 0xfc8, 0x008,	RTL_GIGA_MAC_VER_02 },
2077 
2078 		/* Catch-all */
2079 		{ 0x000, 0x000,	RTL_GIGA_MAC_NONE   }
2080 	};
2081 	const struct rtl_mac_info *p = mac_info;
2082 	enum mac_version ver;
2083 
2084 	while ((xid & p->mask) != p->val)
2085 		p++;
2086 	ver = p->ver;
2087 
2088 	if (ver != RTL_GIGA_MAC_NONE && !gmii) {
2089 		if (ver == RTL_GIGA_MAC_VER_42)
2090 			ver = RTL_GIGA_MAC_VER_43;
2091 		else if (ver == RTL_GIGA_MAC_VER_45)
2092 			ver = RTL_GIGA_MAC_VER_47;
2093 		else if (ver == RTL_GIGA_MAC_VER_46)
2094 			ver = RTL_GIGA_MAC_VER_48;
2095 	}
2096 
2097 	return ver;
2098 }
2099 
2100 static void rtl_release_firmware(struct rtl8169_private *tp)
2101 {
2102 	if (tp->rtl_fw) {
2103 		rtl_fw_release_firmware(tp->rtl_fw);
2104 		kfree(tp->rtl_fw);
2105 		tp->rtl_fw = NULL;
2106 	}
2107 }
2108 
2109 void r8169_apply_firmware(struct rtl8169_private *tp)
2110 {
2111 	int val;
2112 
2113 	/* TODO: release firmware if rtl_fw_write_firmware signals failure. */
2114 	if (tp->rtl_fw) {
2115 		rtl_fw_write_firmware(tp, tp->rtl_fw);
2116 		/* At least one firmware doesn't reset tp->ocp_base. */
2117 		tp->ocp_base = OCP_STD_PHY_BASE;
2118 
2119 		/* PHY soft reset may still be in progress */
2120 		phy_read_poll_timeout(tp->phydev, MII_BMCR, val,
2121 				      !(val & BMCR_RESET),
2122 				      50000, 600000, true);
2123 	}
2124 }
2125 
2126 static void rtl8168_config_eee_mac(struct rtl8169_private *tp)
2127 {
2128 	/* Adjust EEE LED frequency */
2129 	if (tp->mac_version != RTL_GIGA_MAC_VER_38)
2130 		RTL_W8(tp, EEE_LED, RTL_R8(tp, EEE_LED) & ~0x07);
2131 
2132 	rtl_eri_set_bits(tp, 0x1b0, 0x0003);
2133 }
2134 
2135 static void rtl8125a_config_eee_mac(struct rtl8169_private *tp)
2136 {
2137 	r8168_mac_ocp_modify(tp, 0xe040, 0, BIT(1) | BIT(0));
2138 	r8168_mac_ocp_modify(tp, 0xeb62, 0, BIT(2) | BIT(1));
2139 }
2140 
2141 static void rtl8125_set_eee_txidle_timer(struct rtl8169_private *tp)
2142 {
2143 	RTL_W16(tp, EEE_TXIDLE_TIMER_8125, tp->dev->mtu + ETH_HLEN + 0x20);
2144 }
2145 
2146 static void rtl8125b_config_eee_mac(struct rtl8169_private *tp)
2147 {
2148 	rtl8125_set_eee_txidle_timer(tp);
2149 	r8168_mac_ocp_modify(tp, 0xe040, 0, BIT(1) | BIT(0));
2150 }
2151 
2152 static void rtl_rar_exgmac_set(struct rtl8169_private *tp, const u8 *addr)
2153 {
2154 	rtl_eri_write(tp, 0xe0, ERIAR_MASK_1111, get_unaligned_le32(addr));
2155 	rtl_eri_write(tp, 0xe4, ERIAR_MASK_1111, get_unaligned_le16(addr + 4));
2156 	rtl_eri_write(tp, 0xf0, ERIAR_MASK_1111, get_unaligned_le16(addr) << 16);
2157 	rtl_eri_write(tp, 0xf4, ERIAR_MASK_1111, get_unaligned_le32(addr + 2));
2158 }
2159 
2160 u16 rtl8168h_2_get_adc_bias_ioffset(struct rtl8169_private *tp)
2161 {
2162 	u16 data1, data2, ioffset;
2163 
2164 	r8168_mac_ocp_write(tp, 0xdd02, 0x807d);
2165 	data1 = r8168_mac_ocp_read(tp, 0xdd02);
2166 	data2 = r8168_mac_ocp_read(tp, 0xdd00);
2167 
2168 	ioffset = (data2 >> 1) & 0x7ff8;
2169 	ioffset |= data2 & 0x0007;
2170 	if (data1 & BIT(7))
2171 		ioffset |= BIT(15);
2172 
2173 	return ioffset;
2174 }
2175 
2176 static void rtl_schedule_task(struct rtl8169_private *tp, enum rtl_flag flag)
2177 {
2178 	set_bit(flag, tp->wk.flags);
2179 	schedule_work(&tp->wk.work);
2180 }
2181 
2182 static void rtl8169_init_phy(struct rtl8169_private *tp)
2183 {
2184 	r8169_hw_phy_config(tp, tp->phydev, tp->mac_version);
2185 
2186 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06) {
2187 		pci_write_config_byte(tp->pci_dev, PCI_LATENCY_TIMER, 0x40);
2188 		pci_write_config_byte(tp->pci_dev, PCI_CACHE_LINE_SIZE, 0x08);
2189 		/* set undocumented MAC Reg C+CR Offset 0x82h */
2190 		RTL_W8(tp, 0x82, 0x01);
2191 	}
2192 
2193 	if (tp->mac_version == RTL_GIGA_MAC_VER_05 &&
2194 	    tp->pci_dev->subsystem_vendor == PCI_VENDOR_ID_GIGABYTE &&
2195 	    tp->pci_dev->subsystem_device == 0xe000)
2196 		phy_write_paged(tp->phydev, 0x0001, 0x10, 0xf01b);
2197 
2198 	/* We may have called phy_speed_down before */
2199 	phy_speed_up(tp->phydev);
2200 
2201 	if (rtl_supports_eee(tp))
2202 		rtl_enable_eee(tp);
2203 
2204 	genphy_soft_reset(tp->phydev);
2205 }
2206 
2207 static void rtl_rar_set(struct rtl8169_private *tp, const u8 *addr)
2208 {
2209 	rtl_unlock_config_regs(tp);
2210 
2211 	RTL_W32(tp, MAC4, get_unaligned_le16(addr + 4));
2212 	rtl_pci_commit(tp);
2213 
2214 	RTL_W32(tp, MAC0, get_unaligned_le32(addr));
2215 	rtl_pci_commit(tp);
2216 
2217 	if (tp->mac_version == RTL_GIGA_MAC_VER_34)
2218 		rtl_rar_exgmac_set(tp, addr);
2219 
2220 	rtl_lock_config_regs(tp);
2221 }
2222 
2223 static int rtl_set_mac_address(struct net_device *dev, void *p)
2224 {
2225 	struct rtl8169_private *tp = netdev_priv(dev);
2226 	int ret;
2227 
2228 	ret = eth_mac_addr(dev, p);
2229 	if (ret)
2230 		return ret;
2231 
2232 	rtl_rar_set(tp, dev->dev_addr);
2233 
2234 	return 0;
2235 }
2236 
2237 static void rtl_wol_enable_rx(struct rtl8169_private *tp)
2238 {
2239 	if (tp->mac_version >= RTL_GIGA_MAC_VER_25)
2240 		RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) |
2241 			AcceptBroadcast | AcceptMulticast | AcceptMyPhys);
2242 }
2243 
2244 static void rtl_prepare_power_down(struct rtl8169_private *tp)
2245 {
2246 	if (tp->dash_type != RTL_DASH_NONE)
2247 		return;
2248 
2249 	if (tp->mac_version == RTL_GIGA_MAC_VER_32 ||
2250 	    tp->mac_version == RTL_GIGA_MAC_VER_33)
2251 		rtl_ephy_write(tp, 0x19, 0xff64);
2252 
2253 	if (device_may_wakeup(tp_to_dev(tp))) {
2254 		phy_speed_down(tp->phydev, false);
2255 		rtl_wol_enable_rx(tp);
2256 	}
2257 }
2258 
2259 static void rtl_init_rxcfg(struct rtl8169_private *tp)
2260 {
2261 	switch (tp->mac_version) {
2262 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
2263 	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
2264 		RTL_W32(tp, RxConfig, RX_FIFO_THRESH | RX_DMA_BURST);
2265 		break;
2266 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
2267 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_36:
2268 	case RTL_GIGA_MAC_VER_38:
2269 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST);
2270 		break;
2271 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_53:
2272 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST | RX_EARLY_OFF);
2273 		break;
2274 	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
2275 		RTL_W32(tp, RxConfig, RX_FETCH_DFLT_8125 | RX_DMA_BURST);
2276 		break;
2277 	default:
2278 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_DMA_BURST);
2279 		break;
2280 	}
2281 }
2282 
2283 static void rtl8169_init_ring_indexes(struct rtl8169_private *tp)
2284 {
2285 	tp->dirty_tx = tp->cur_tx = tp->cur_rx = 0;
2286 }
2287 
2288 static void r8168c_hw_jumbo_enable(struct rtl8169_private *tp)
2289 {
2290 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2291 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | Jumbo_En1);
2292 }
2293 
2294 static void r8168c_hw_jumbo_disable(struct rtl8169_private *tp)
2295 {
2296 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2297 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~Jumbo_En1);
2298 }
2299 
2300 static void r8168dp_hw_jumbo_enable(struct rtl8169_private *tp)
2301 {
2302 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2303 }
2304 
2305 static void r8168dp_hw_jumbo_disable(struct rtl8169_private *tp)
2306 {
2307 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2308 }
2309 
2310 static void r8168e_hw_jumbo_enable(struct rtl8169_private *tp)
2311 {
2312 	RTL_W8(tp, MaxTxPacketSize, 0x24);
2313 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2314 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | 0x01);
2315 }
2316 
2317 static void r8168e_hw_jumbo_disable(struct rtl8169_private *tp)
2318 {
2319 	RTL_W8(tp, MaxTxPacketSize, 0x3f);
2320 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2321 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~0x01);
2322 }
2323 
2324 static void r8168b_1_hw_jumbo_enable(struct rtl8169_private *tp)
2325 {
2326 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | (1 << 0));
2327 }
2328 
2329 static void r8168b_1_hw_jumbo_disable(struct rtl8169_private *tp)
2330 {
2331 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~(1 << 0));
2332 }
2333 
2334 static void rtl_jumbo_config(struct rtl8169_private *tp)
2335 {
2336 	bool jumbo = tp->dev->mtu > ETH_DATA_LEN;
2337 	int readrq = 4096;
2338 
2339 	rtl_unlock_config_regs(tp);
2340 	switch (tp->mac_version) {
2341 	case RTL_GIGA_MAC_VER_12:
2342 	case RTL_GIGA_MAC_VER_17:
2343 		if (jumbo) {
2344 			readrq = 512;
2345 			r8168b_1_hw_jumbo_enable(tp);
2346 		} else {
2347 			r8168b_1_hw_jumbo_disable(tp);
2348 		}
2349 		break;
2350 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_26:
2351 		if (jumbo) {
2352 			readrq = 512;
2353 			r8168c_hw_jumbo_enable(tp);
2354 		} else {
2355 			r8168c_hw_jumbo_disable(tp);
2356 		}
2357 		break;
2358 	case RTL_GIGA_MAC_VER_28:
2359 		if (jumbo)
2360 			r8168dp_hw_jumbo_enable(tp);
2361 		else
2362 			r8168dp_hw_jumbo_disable(tp);
2363 		break;
2364 	case RTL_GIGA_MAC_VER_31 ... RTL_GIGA_MAC_VER_33:
2365 		if (jumbo)
2366 			r8168e_hw_jumbo_enable(tp);
2367 		else
2368 			r8168e_hw_jumbo_disable(tp);
2369 		break;
2370 	default:
2371 		break;
2372 	}
2373 	rtl_lock_config_regs(tp);
2374 
2375 	if (pci_is_pcie(tp->pci_dev) && tp->supports_gmii)
2376 		pcie_set_readrq(tp->pci_dev, readrq);
2377 
2378 	/* Chip doesn't support pause in jumbo mode */
2379 	if (jumbo) {
2380 		linkmode_clear_bit(ETHTOOL_LINK_MODE_Pause_BIT,
2381 				   tp->phydev->advertising);
2382 		linkmode_clear_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT,
2383 				   tp->phydev->advertising);
2384 		phy_start_aneg(tp->phydev);
2385 	}
2386 }
2387 
2388 DECLARE_RTL_COND(rtl_chipcmd_cond)
2389 {
2390 	return RTL_R8(tp, ChipCmd) & CmdReset;
2391 }
2392 
2393 static void rtl_hw_reset(struct rtl8169_private *tp)
2394 {
2395 	RTL_W8(tp, ChipCmd, CmdReset);
2396 
2397 	rtl_loop_wait_low(tp, &rtl_chipcmd_cond, 100, 100);
2398 }
2399 
2400 static void rtl_request_firmware(struct rtl8169_private *tp)
2401 {
2402 	struct rtl_fw *rtl_fw;
2403 
2404 	/* firmware loaded already or no firmware available */
2405 	if (tp->rtl_fw || !tp->fw_name)
2406 		return;
2407 
2408 	rtl_fw = kzalloc(sizeof(*rtl_fw), GFP_KERNEL);
2409 	if (!rtl_fw)
2410 		return;
2411 
2412 	rtl_fw->phy_write = rtl_writephy;
2413 	rtl_fw->phy_read = rtl_readphy;
2414 	rtl_fw->mac_mcu_write = mac_mcu_write;
2415 	rtl_fw->mac_mcu_read = mac_mcu_read;
2416 	rtl_fw->fw_name = tp->fw_name;
2417 	rtl_fw->dev = tp_to_dev(tp);
2418 
2419 	if (rtl_fw_request_firmware(rtl_fw))
2420 		kfree(rtl_fw);
2421 	else
2422 		tp->rtl_fw = rtl_fw;
2423 }
2424 
2425 static void rtl_rx_close(struct rtl8169_private *tp)
2426 {
2427 	RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & ~RX_CONFIG_ACCEPT_MASK);
2428 }
2429 
2430 DECLARE_RTL_COND(rtl_npq_cond)
2431 {
2432 	return RTL_R8(tp, TxPoll) & NPQ;
2433 }
2434 
2435 DECLARE_RTL_COND(rtl_txcfg_empty_cond)
2436 {
2437 	return RTL_R32(tp, TxConfig) & TXCFG_EMPTY;
2438 }
2439 
2440 DECLARE_RTL_COND(rtl_rxtx_empty_cond)
2441 {
2442 	return (RTL_R8(tp, MCU) & RXTX_EMPTY) == RXTX_EMPTY;
2443 }
2444 
2445 DECLARE_RTL_COND(rtl_rxtx_empty_cond_2)
2446 {
2447 	/* IntrMitigate has new functionality on RTL8125 */
2448 	return (RTL_R16(tp, IntrMitigate) & 0x0103) == 0x0103;
2449 }
2450 
2451 static void rtl_wait_txrx_fifo_empty(struct rtl8169_private *tp)
2452 {
2453 	switch (tp->mac_version) {
2454 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_53:
2455 		rtl_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 42);
2456 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2457 		break;
2458 	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_61:
2459 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2460 		break;
2461 	case RTL_GIGA_MAC_VER_63:
2462 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
2463 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2464 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond_2, 100, 42);
2465 		break;
2466 	default:
2467 		break;
2468 	}
2469 }
2470 
2471 static void rtl_enable_rxdvgate(struct rtl8169_private *tp)
2472 {
2473 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | RXDV_GATED_EN);
2474 	fsleep(2000);
2475 	rtl_wait_txrx_fifo_empty(tp);
2476 }
2477 
2478 static void rtl_set_tx_config_registers(struct rtl8169_private *tp)
2479 {
2480 	u32 val = TX_DMA_BURST << TxDMAShift |
2481 		  InterFrameGap << TxInterFrameGapShift;
2482 
2483 	if (rtl_is_8168evl_up(tp))
2484 		val |= TXCFG_AUTO_FIFO;
2485 
2486 	RTL_W32(tp, TxConfig, val);
2487 }
2488 
2489 static void rtl_set_rx_max_size(struct rtl8169_private *tp)
2490 {
2491 	/* Low hurts. Let's disable the filtering. */
2492 	RTL_W16(tp, RxMaxSize, R8169_RX_BUF_SIZE + 1);
2493 }
2494 
2495 static void rtl_set_rx_tx_desc_registers(struct rtl8169_private *tp)
2496 {
2497 	/*
2498 	 * Magic spell: some iop3xx ARM board needs the TxDescAddrHigh
2499 	 * register to be written before TxDescAddrLow to work.
2500 	 * Switching from MMIO to I/O access fixes the issue as well.
2501 	 */
2502 	RTL_W32(tp, TxDescStartAddrHigh, ((u64) tp->TxPhyAddr) >> 32);
2503 	RTL_W32(tp, TxDescStartAddrLow, ((u64) tp->TxPhyAddr) & DMA_BIT_MASK(32));
2504 	RTL_W32(tp, RxDescAddrHigh, ((u64) tp->RxPhyAddr) >> 32);
2505 	RTL_W32(tp, RxDescAddrLow, ((u64) tp->RxPhyAddr) & DMA_BIT_MASK(32));
2506 }
2507 
2508 static void rtl8169_set_magic_reg(struct rtl8169_private *tp)
2509 {
2510 	u32 val;
2511 
2512 	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
2513 		val = 0x000fff00;
2514 	else if (tp->mac_version == RTL_GIGA_MAC_VER_06)
2515 		val = 0x00ffff00;
2516 	else
2517 		return;
2518 
2519 	if (RTL_R8(tp, Config2) & PCI_Clock_66MHz)
2520 		val |= 0xff;
2521 
2522 	RTL_W32(tp, 0x7c, val);
2523 }
2524 
2525 static void rtl_set_rx_mode(struct net_device *dev)
2526 {
2527 	u32 rx_mode = AcceptBroadcast | AcceptMyPhys | AcceptMulticast;
2528 	/* Multicast hash filter */
2529 	u32 mc_filter[2] = { 0xffffffff, 0xffffffff };
2530 	struct rtl8169_private *tp = netdev_priv(dev);
2531 	u32 tmp;
2532 
2533 	if (dev->flags & IFF_PROMISC) {
2534 		rx_mode |= AcceptAllPhys;
2535 	} else if (netdev_mc_count(dev) > MC_FILTER_LIMIT ||
2536 		   dev->flags & IFF_ALLMULTI ||
2537 		   tp->mac_version == RTL_GIGA_MAC_VER_35) {
2538 		/* accept all multicasts */
2539 	} else if (netdev_mc_empty(dev)) {
2540 		rx_mode &= ~AcceptMulticast;
2541 	} else {
2542 		struct netdev_hw_addr *ha;
2543 
2544 		mc_filter[1] = mc_filter[0] = 0;
2545 		netdev_for_each_mc_addr(ha, dev) {
2546 			u32 bit_nr = eth_hw_addr_crc(ha) >> 26;
2547 			mc_filter[bit_nr >> 5] |= BIT(bit_nr & 31);
2548 		}
2549 
2550 		if (tp->mac_version > RTL_GIGA_MAC_VER_06) {
2551 			tmp = mc_filter[0];
2552 			mc_filter[0] = swab32(mc_filter[1]);
2553 			mc_filter[1] = swab32(tmp);
2554 		}
2555 	}
2556 
2557 	RTL_W32(tp, MAR0 + 4, mc_filter[1]);
2558 	RTL_W32(tp, MAR0 + 0, mc_filter[0]);
2559 
2560 	tmp = RTL_R32(tp, RxConfig);
2561 	RTL_W32(tp, RxConfig, (tmp & ~RX_CONFIG_ACCEPT_OK_MASK) | rx_mode);
2562 }
2563 
2564 DECLARE_RTL_COND(rtl_csiar_cond)
2565 {
2566 	return RTL_R32(tp, CSIAR) & CSIAR_FLAG;
2567 }
2568 
2569 static void rtl_csi_write(struct rtl8169_private *tp, int addr, int value)
2570 {
2571 	u32 func = PCI_FUNC(tp->pci_dev->devfn);
2572 
2573 	RTL_W32(tp, CSIDR, value);
2574 	RTL_W32(tp, CSIAR, CSIAR_WRITE_CMD | (addr & CSIAR_ADDR_MASK) |
2575 		CSIAR_BYTE_ENABLE | func << 16);
2576 
2577 	rtl_loop_wait_low(tp, &rtl_csiar_cond, 10, 100);
2578 }
2579 
2580 static u32 rtl_csi_read(struct rtl8169_private *tp, int addr)
2581 {
2582 	u32 func = PCI_FUNC(tp->pci_dev->devfn);
2583 
2584 	RTL_W32(tp, CSIAR, (addr & CSIAR_ADDR_MASK) | func << 16 |
2585 		CSIAR_BYTE_ENABLE);
2586 
2587 	return rtl_loop_wait_high(tp, &rtl_csiar_cond, 10, 100) ?
2588 		RTL_R32(tp, CSIDR) : ~0;
2589 }
2590 
2591 static void rtl_set_aspm_entry_latency(struct rtl8169_private *tp, u8 val)
2592 {
2593 	struct pci_dev *pdev = tp->pci_dev;
2594 	u32 csi;
2595 
2596 	/* According to Realtek the value at config space address 0x070f
2597 	 * controls the L0s/L1 entrance latency. We try standard ECAM access
2598 	 * first and if it fails fall back to CSI.
2599 	 * bit 0..2: L0: 0 = 1us, 1 = 2us .. 6 = 7us, 7 = 7us (no typo)
2600 	 * bit 3..5: L1: 0 = 1us, 1 = 2us .. 6 = 64us, 7 = 64us
2601 	 */
2602 	if (pdev->cfg_size > 0x070f &&
2603 	    pci_write_config_byte(pdev, 0x070f, val) == PCIBIOS_SUCCESSFUL)
2604 		return;
2605 
2606 	netdev_notice_once(tp->dev,
2607 		"No native access to PCI extended config space, falling back to CSI\n");
2608 	csi = rtl_csi_read(tp, 0x070c) & 0x00ffffff;
2609 	rtl_csi_write(tp, 0x070c, csi | val << 24);
2610 }
2611 
2612 static void rtl_set_def_aspm_entry_latency(struct rtl8169_private *tp)
2613 {
2614 	/* L0 7us, L1 16us */
2615 	rtl_set_aspm_entry_latency(tp, 0x27);
2616 }
2617 
2618 struct ephy_info {
2619 	unsigned int offset;
2620 	u16 mask;
2621 	u16 bits;
2622 };
2623 
2624 static void __rtl_ephy_init(struct rtl8169_private *tp,
2625 			    const struct ephy_info *e, int len)
2626 {
2627 	u16 w;
2628 
2629 	while (len-- > 0) {
2630 		w = (rtl_ephy_read(tp, e->offset) & ~e->mask) | e->bits;
2631 		rtl_ephy_write(tp, e->offset, w);
2632 		e++;
2633 	}
2634 }
2635 
2636 #define rtl_ephy_init(tp, a) __rtl_ephy_init(tp, a, ARRAY_SIZE(a))
2637 
2638 static void rtl_disable_clock_request(struct rtl8169_private *tp)
2639 {
2640 	pcie_capability_clear_word(tp->pci_dev, PCI_EXP_LNKCTL,
2641 				   PCI_EXP_LNKCTL_CLKREQ_EN);
2642 }
2643 
2644 static void rtl_enable_clock_request(struct rtl8169_private *tp)
2645 {
2646 	pcie_capability_set_word(tp->pci_dev, PCI_EXP_LNKCTL,
2647 				 PCI_EXP_LNKCTL_CLKREQ_EN);
2648 }
2649 
2650 static void rtl_pcie_state_l2l3_disable(struct rtl8169_private *tp)
2651 {
2652 	/* work around an issue when PCI reset occurs during L2/L3 state */
2653 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Rdy_to_L23);
2654 }
2655 
2656 static void rtl_enable_exit_l1(struct rtl8169_private *tp)
2657 {
2658 	/* Bits control which events trigger ASPM L1 exit:
2659 	 * Bit 12: rxdv
2660 	 * Bit 11: ltr_msg
2661 	 * Bit 10: txdma_poll
2662 	 * Bit  9: xadm
2663 	 * Bit  8: pktavi
2664 	 * Bit  7: txpla
2665 	 */
2666 	switch (tp->mac_version) {
2667 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_36:
2668 		rtl_eri_set_bits(tp, 0xd4, 0x1f00);
2669 		break;
2670 	case RTL_GIGA_MAC_VER_37 ... RTL_GIGA_MAC_VER_38:
2671 		rtl_eri_set_bits(tp, 0xd4, 0x0c00);
2672 		break;
2673 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
2674 		r8168_mac_ocp_modify(tp, 0xc0ac, 0, 0x1f80);
2675 		break;
2676 	default:
2677 		break;
2678 	}
2679 }
2680 
2681 static void rtl_disable_exit_l1(struct rtl8169_private *tp)
2682 {
2683 	switch (tp->mac_version) {
2684 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
2685 		rtl_eri_clear_bits(tp, 0xd4, 0x1f00);
2686 		break;
2687 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
2688 		r8168_mac_ocp_modify(tp, 0xc0ac, 0x1f80, 0);
2689 		break;
2690 	default:
2691 		break;
2692 	}
2693 }
2694 
2695 static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
2696 {
2697 	/* Don't enable ASPM in the chip if OS can't control ASPM */
2698 	if (enable && tp->aspm_manageable) {
2699 		RTL_W8(tp, Config5, RTL_R8(tp, Config5) | ASPM_en);
2700 		RTL_W8(tp, Config2, RTL_R8(tp, Config2) | ClkReqEn);
2701 
2702 		switch (tp->mac_version) {
2703 		case RTL_GIGA_MAC_VER_45 ... RTL_GIGA_MAC_VER_48:
2704 		case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
2705 			/* reset ephy tx/rx disable timer */
2706 			r8168_mac_ocp_modify(tp, 0xe094, 0xff00, 0);
2707 			/* chip can trigger L1.2 */
2708 			r8168_mac_ocp_modify(tp, 0xe092, 0x00ff, BIT(2));
2709 			break;
2710 		default:
2711 			break;
2712 		}
2713 	} else {
2714 		switch (tp->mac_version) {
2715 		case RTL_GIGA_MAC_VER_45 ... RTL_GIGA_MAC_VER_48:
2716 		case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
2717 			r8168_mac_ocp_modify(tp, 0xe092, 0x00ff, 0);
2718 			break;
2719 		default:
2720 			break;
2721 		}
2722 
2723 		RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~ClkReqEn);
2724 		RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~ASPM_en);
2725 	}
2726 
2727 	udelay(10);
2728 }
2729 
2730 static void rtl_set_fifo_size(struct rtl8169_private *tp, u16 rx_stat,
2731 			      u16 tx_stat, u16 rx_dyn, u16 tx_dyn)
2732 {
2733 	/* Usage of dynamic vs. static FIFO is controlled by bit
2734 	 * TXCFG_AUTO_FIFO. Exact meaning of FIFO values isn't known.
2735 	 */
2736 	rtl_eri_write(tp, 0xc8, ERIAR_MASK_1111, (rx_stat << 16) | rx_dyn);
2737 	rtl_eri_write(tp, 0xe8, ERIAR_MASK_1111, (tx_stat << 16) | tx_dyn);
2738 }
2739 
2740 static void rtl8168g_set_pause_thresholds(struct rtl8169_private *tp,
2741 					  u8 low, u8 high)
2742 {
2743 	/* FIFO thresholds for pause flow control */
2744 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_0001, low);
2745 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_0001, high);
2746 }
2747 
2748 static void rtl_hw_start_8168b(struct rtl8169_private *tp)
2749 {
2750 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2751 }
2752 
2753 static void __rtl_hw_start_8168cp(struct rtl8169_private *tp)
2754 {
2755 	RTL_W8(tp, Config1, RTL_R8(tp, Config1) | Speed_down);
2756 
2757 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2758 
2759 	rtl_disable_clock_request(tp);
2760 }
2761 
2762 static void rtl_hw_start_8168cp_1(struct rtl8169_private *tp)
2763 {
2764 	static const struct ephy_info e_info_8168cp[] = {
2765 		{ 0x01, 0,	0x0001 },
2766 		{ 0x02, 0x0800,	0x1000 },
2767 		{ 0x03, 0,	0x0042 },
2768 		{ 0x06, 0x0080,	0x0000 },
2769 		{ 0x07, 0,	0x2000 }
2770 	};
2771 
2772 	rtl_set_def_aspm_entry_latency(tp);
2773 
2774 	rtl_ephy_init(tp, e_info_8168cp);
2775 
2776 	__rtl_hw_start_8168cp(tp);
2777 }
2778 
2779 static void rtl_hw_start_8168cp_2(struct rtl8169_private *tp)
2780 {
2781 	rtl_set_def_aspm_entry_latency(tp);
2782 
2783 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2784 }
2785 
2786 static void rtl_hw_start_8168cp_3(struct rtl8169_private *tp)
2787 {
2788 	rtl_set_def_aspm_entry_latency(tp);
2789 
2790 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2791 
2792 	/* Magic. */
2793 	RTL_W8(tp, DBG_REG, 0x20);
2794 }
2795 
2796 static void rtl_hw_start_8168c_1(struct rtl8169_private *tp)
2797 {
2798 	static const struct ephy_info e_info_8168c_1[] = {
2799 		{ 0x02, 0x0800,	0x1000 },
2800 		{ 0x03, 0,	0x0002 },
2801 		{ 0x06, 0x0080,	0x0000 }
2802 	};
2803 
2804 	rtl_set_def_aspm_entry_latency(tp);
2805 
2806 	RTL_W8(tp, DBG_REG, 0x06 | FIX_NAK_1 | FIX_NAK_2);
2807 
2808 	rtl_ephy_init(tp, e_info_8168c_1);
2809 
2810 	__rtl_hw_start_8168cp(tp);
2811 }
2812 
2813 static void rtl_hw_start_8168c_2(struct rtl8169_private *tp)
2814 {
2815 	static const struct ephy_info e_info_8168c_2[] = {
2816 		{ 0x01, 0,	0x0001 },
2817 		{ 0x03, 0x0400,	0x0020 }
2818 	};
2819 
2820 	rtl_set_def_aspm_entry_latency(tp);
2821 
2822 	rtl_ephy_init(tp, e_info_8168c_2);
2823 
2824 	__rtl_hw_start_8168cp(tp);
2825 }
2826 
2827 static void rtl_hw_start_8168c_4(struct rtl8169_private *tp)
2828 {
2829 	rtl_set_def_aspm_entry_latency(tp);
2830 
2831 	__rtl_hw_start_8168cp(tp);
2832 }
2833 
2834 static void rtl_hw_start_8168d(struct rtl8169_private *tp)
2835 {
2836 	rtl_set_def_aspm_entry_latency(tp);
2837 
2838 	rtl_disable_clock_request(tp);
2839 }
2840 
2841 static void rtl_hw_start_8168d_4(struct rtl8169_private *tp)
2842 {
2843 	static const struct ephy_info e_info_8168d_4[] = {
2844 		{ 0x0b, 0x0000,	0x0048 },
2845 		{ 0x19, 0x0020,	0x0050 },
2846 		{ 0x0c, 0x0100,	0x0020 },
2847 		{ 0x10, 0x0004,	0x0000 },
2848 	};
2849 
2850 	rtl_set_def_aspm_entry_latency(tp);
2851 
2852 	rtl_ephy_init(tp, e_info_8168d_4);
2853 
2854 	rtl_enable_clock_request(tp);
2855 }
2856 
2857 static void rtl_hw_start_8168e_1(struct rtl8169_private *tp)
2858 {
2859 	static const struct ephy_info e_info_8168e_1[] = {
2860 		{ 0x00, 0x0200,	0x0100 },
2861 		{ 0x00, 0x0000,	0x0004 },
2862 		{ 0x06, 0x0002,	0x0001 },
2863 		{ 0x06, 0x0000,	0x0030 },
2864 		{ 0x07, 0x0000,	0x2000 },
2865 		{ 0x00, 0x0000,	0x0020 },
2866 		{ 0x03, 0x5800,	0x2000 },
2867 		{ 0x03, 0x0000,	0x0001 },
2868 		{ 0x01, 0x0800,	0x1000 },
2869 		{ 0x07, 0x0000,	0x4000 },
2870 		{ 0x1e, 0x0000,	0x2000 },
2871 		{ 0x19, 0xffff,	0xfe6c },
2872 		{ 0x0a, 0x0000,	0x0040 }
2873 	};
2874 
2875 	rtl_set_def_aspm_entry_latency(tp);
2876 
2877 	rtl_ephy_init(tp, e_info_8168e_1);
2878 
2879 	rtl_disable_clock_request(tp);
2880 
2881 	/* Reset tx FIFO pointer */
2882 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | TXPLA_RST);
2883 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~TXPLA_RST);
2884 
2885 	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
2886 }
2887 
2888 static void rtl_hw_start_8168e_2(struct rtl8169_private *tp)
2889 {
2890 	static const struct ephy_info e_info_8168e_2[] = {
2891 		{ 0x09, 0x0000,	0x0080 },
2892 		{ 0x19, 0x0000,	0x0224 },
2893 		{ 0x00, 0x0000,	0x0004 },
2894 		{ 0x0c, 0x3df0,	0x0200 },
2895 	};
2896 
2897 	rtl_set_def_aspm_entry_latency(tp);
2898 
2899 	rtl_ephy_init(tp, e_info_8168e_2);
2900 
2901 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
2902 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_1111, 0x0000);
2903 	rtl_set_fifo_size(tp, 0x10, 0x10, 0x02, 0x06);
2904 	rtl_eri_set_bits(tp, 0x1d0, BIT(1));
2905 	rtl_reset_packet_filter(tp);
2906 	rtl_eri_set_bits(tp, 0x1b0, BIT(4));
2907 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_1111, 0x00000050);
2908 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_1111, 0x07ff0060);
2909 
2910 	rtl_disable_clock_request(tp);
2911 
2912 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
2913 
2914 	rtl8168_config_eee_mac(tp);
2915 
2916 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
2917 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
2918 	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
2919 
2920 	rtl_hw_aspm_clkreq_enable(tp, true);
2921 }
2922 
2923 static void rtl_hw_start_8168f(struct rtl8169_private *tp)
2924 {
2925 	rtl_set_def_aspm_entry_latency(tp);
2926 
2927 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
2928 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_1111, 0x0000);
2929 	rtl_set_fifo_size(tp, 0x10, 0x10, 0x02, 0x06);
2930 	rtl_reset_packet_filter(tp);
2931 	rtl_eri_set_bits(tp, 0x1b0, BIT(4));
2932 	rtl_eri_set_bits(tp, 0x1d0, BIT(4) | BIT(1));
2933 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_1111, 0x00000050);
2934 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_1111, 0x00000060);
2935 
2936 	rtl_disable_clock_request(tp);
2937 
2938 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
2939 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
2940 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
2941 	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
2942 
2943 	rtl8168_config_eee_mac(tp);
2944 }
2945 
2946 static void rtl_hw_start_8168f_1(struct rtl8169_private *tp)
2947 {
2948 	static const struct ephy_info e_info_8168f_1[] = {
2949 		{ 0x06, 0x00c0,	0x0020 },
2950 		{ 0x08, 0x0001,	0x0002 },
2951 		{ 0x09, 0x0000,	0x0080 },
2952 		{ 0x19, 0x0000,	0x0224 },
2953 		{ 0x00, 0x0000,	0x0008 },
2954 		{ 0x0c, 0x3df0,	0x0200 },
2955 	};
2956 
2957 	rtl_hw_start_8168f(tp);
2958 
2959 	rtl_ephy_init(tp, e_info_8168f_1);
2960 }
2961 
2962 static void rtl_hw_start_8411(struct rtl8169_private *tp)
2963 {
2964 	static const struct ephy_info e_info_8168f_1[] = {
2965 		{ 0x06, 0x00c0,	0x0020 },
2966 		{ 0x0f, 0xffff,	0x5200 },
2967 		{ 0x19, 0x0000,	0x0224 },
2968 		{ 0x00, 0x0000,	0x0008 },
2969 		{ 0x0c, 0x3df0,	0x0200 },
2970 	};
2971 
2972 	rtl_hw_start_8168f(tp);
2973 	rtl_pcie_state_l2l3_disable(tp);
2974 
2975 	rtl_ephy_init(tp, e_info_8168f_1);
2976 }
2977 
2978 static void rtl_hw_start_8168g(struct rtl8169_private *tp)
2979 {
2980 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
2981 	rtl8168g_set_pause_thresholds(tp, 0x38, 0x48);
2982 
2983 	rtl_set_def_aspm_entry_latency(tp);
2984 
2985 	rtl_reset_packet_filter(tp);
2986 	rtl_eri_write(tp, 0x2f8, ERIAR_MASK_0011, 0x1d8f);
2987 
2988 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
2989 
2990 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
2991 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
2992 
2993 	rtl8168_config_eee_mac(tp);
2994 
2995 	rtl_w0w1_eri(tp, 0x2fc, 0x01, 0x06);
2996 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
2997 
2998 	rtl_pcie_state_l2l3_disable(tp);
2999 }
3000 
3001 static void rtl_hw_start_8168g_1(struct rtl8169_private *tp)
3002 {
3003 	static const struct ephy_info e_info_8168g_1[] = {
3004 		{ 0x00, 0x0008,	0x0000 },
3005 		{ 0x0c, 0x3ff0,	0x0820 },
3006 		{ 0x1e, 0x0000,	0x0001 },
3007 		{ 0x19, 0x8000,	0x0000 }
3008 	};
3009 
3010 	rtl_hw_start_8168g(tp);
3011 
3012 	/* disable aspm and clock request before access ephy */
3013 	rtl_hw_aspm_clkreq_enable(tp, false);
3014 	rtl_ephy_init(tp, e_info_8168g_1);
3015 	rtl_hw_aspm_clkreq_enable(tp, true);
3016 }
3017 
3018 static void rtl_hw_start_8168g_2(struct rtl8169_private *tp)
3019 {
3020 	static const struct ephy_info e_info_8168g_2[] = {
3021 		{ 0x00, 0x0008,	0x0000 },
3022 		{ 0x0c, 0x3ff0,	0x0820 },
3023 		{ 0x19, 0xffff,	0x7c00 },
3024 		{ 0x1e, 0xffff,	0x20eb },
3025 		{ 0x0d, 0xffff,	0x1666 },
3026 		{ 0x00, 0xffff,	0x10a3 },
3027 		{ 0x06, 0xffff,	0xf050 },
3028 		{ 0x04, 0x0000,	0x0010 },
3029 		{ 0x1d, 0x4000,	0x0000 },
3030 	};
3031 
3032 	rtl_hw_start_8168g(tp);
3033 
3034 	/* disable aspm and clock request before access ephy */
3035 	rtl_hw_aspm_clkreq_enable(tp, false);
3036 	rtl_ephy_init(tp, e_info_8168g_2);
3037 }
3038 
3039 static void rtl_hw_start_8411_2(struct rtl8169_private *tp)
3040 {
3041 	static const struct ephy_info e_info_8411_2[] = {
3042 		{ 0x00, 0x0008,	0x0000 },
3043 		{ 0x0c, 0x37d0,	0x0820 },
3044 		{ 0x1e, 0x0000,	0x0001 },
3045 		{ 0x19, 0x8021,	0x0000 },
3046 		{ 0x1e, 0x0000,	0x2000 },
3047 		{ 0x0d, 0x0100,	0x0200 },
3048 		{ 0x00, 0x0000,	0x0080 },
3049 		{ 0x06, 0x0000,	0x0010 },
3050 		{ 0x04, 0x0000,	0x0010 },
3051 		{ 0x1d, 0x0000,	0x4000 },
3052 	};
3053 
3054 	rtl_hw_start_8168g(tp);
3055 
3056 	/* disable aspm and clock request before access ephy */
3057 	rtl_hw_aspm_clkreq_enable(tp, false);
3058 	rtl_ephy_init(tp, e_info_8411_2);
3059 
3060 	/* The following Realtek-provided magic fixes an issue with the RX unit
3061 	 * getting confused after the PHY having been powered-down.
3062 	 */
3063 	r8168_mac_ocp_write(tp, 0xFC28, 0x0000);
3064 	r8168_mac_ocp_write(tp, 0xFC2A, 0x0000);
3065 	r8168_mac_ocp_write(tp, 0xFC2C, 0x0000);
3066 	r8168_mac_ocp_write(tp, 0xFC2E, 0x0000);
3067 	r8168_mac_ocp_write(tp, 0xFC30, 0x0000);
3068 	r8168_mac_ocp_write(tp, 0xFC32, 0x0000);
3069 	r8168_mac_ocp_write(tp, 0xFC34, 0x0000);
3070 	r8168_mac_ocp_write(tp, 0xFC36, 0x0000);
3071 	mdelay(3);
3072 	r8168_mac_ocp_write(tp, 0xFC26, 0x0000);
3073 
3074 	r8168_mac_ocp_write(tp, 0xF800, 0xE008);
3075 	r8168_mac_ocp_write(tp, 0xF802, 0xE00A);
3076 	r8168_mac_ocp_write(tp, 0xF804, 0xE00C);
3077 	r8168_mac_ocp_write(tp, 0xF806, 0xE00E);
3078 	r8168_mac_ocp_write(tp, 0xF808, 0xE027);
3079 	r8168_mac_ocp_write(tp, 0xF80A, 0xE04F);
3080 	r8168_mac_ocp_write(tp, 0xF80C, 0xE05E);
3081 	r8168_mac_ocp_write(tp, 0xF80E, 0xE065);
3082 	r8168_mac_ocp_write(tp, 0xF810, 0xC602);
3083 	r8168_mac_ocp_write(tp, 0xF812, 0xBE00);
3084 	r8168_mac_ocp_write(tp, 0xF814, 0x0000);
3085 	r8168_mac_ocp_write(tp, 0xF816, 0xC502);
3086 	r8168_mac_ocp_write(tp, 0xF818, 0xBD00);
3087 	r8168_mac_ocp_write(tp, 0xF81A, 0x074C);
3088 	r8168_mac_ocp_write(tp, 0xF81C, 0xC302);
3089 	r8168_mac_ocp_write(tp, 0xF81E, 0xBB00);
3090 	r8168_mac_ocp_write(tp, 0xF820, 0x080A);
3091 	r8168_mac_ocp_write(tp, 0xF822, 0x6420);
3092 	r8168_mac_ocp_write(tp, 0xF824, 0x48C2);
3093 	r8168_mac_ocp_write(tp, 0xF826, 0x8C20);
3094 	r8168_mac_ocp_write(tp, 0xF828, 0xC516);
3095 	r8168_mac_ocp_write(tp, 0xF82A, 0x64A4);
3096 	r8168_mac_ocp_write(tp, 0xF82C, 0x49C0);
3097 	r8168_mac_ocp_write(tp, 0xF82E, 0xF009);
3098 	r8168_mac_ocp_write(tp, 0xF830, 0x74A2);
3099 	r8168_mac_ocp_write(tp, 0xF832, 0x8CA5);
3100 	r8168_mac_ocp_write(tp, 0xF834, 0x74A0);
3101 	r8168_mac_ocp_write(tp, 0xF836, 0xC50E);
3102 	r8168_mac_ocp_write(tp, 0xF838, 0x9CA2);
3103 	r8168_mac_ocp_write(tp, 0xF83A, 0x1C11);
3104 	r8168_mac_ocp_write(tp, 0xF83C, 0x9CA0);
3105 	r8168_mac_ocp_write(tp, 0xF83E, 0xE006);
3106 	r8168_mac_ocp_write(tp, 0xF840, 0x74F8);
3107 	r8168_mac_ocp_write(tp, 0xF842, 0x48C4);
3108 	r8168_mac_ocp_write(tp, 0xF844, 0x8CF8);
3109 	r8168_mac_ocp_write(tp, 0xF846, 0xC404);
3110 	r8168_mac_ocp_write(tp, 0xF848, 0xBC00);
3111 	r8168_mac_ocp_write(tp, 0xF84A, 0xC403);
3112 	r8168_mac_ocp_write(tp, 0xF84C, 0xBC00);
3113 	r8168_mac_ocp_write(tp, 0xF84E, 0x0BF2);
3114 	r8168_mac_ocp_write(tp, 0xF850, 0x0C0A);
3115 	r8168_mac_ocp_write(tp, 0xF852, 0xE434);
3116 	r8168_mac_ocp_write(tp, 0xF854, 0xD3C0);
3117 	r8168_mac_ocp_write(tp, 0xF856, 0x49D9);
3118 	r8168_mac_ocp_write(tp, 0xF858, 0xF01F);
3119 	r8168_mac_ocp_write(tp, 0xF85A, 0xC526);
3120 	r8168_mac_ocp_write(tp, 0xF85C, 0x64A5);
3121 	r8168_mac_ocp_write(tp, 0xF85E, 0x1400);
3122 	r8168_mac_ocp_write(tp, 0xF860, 0xF007);
3123 	r8168_mac_ocp_write(tp, 0xF862, 0x0C01);
3124 	r8168_mac_ocp_write(tp, 0xF864, 0x8CA5);
3125 	r8168_mac_ocp_write(tp, 0xF866, 0x1C15);
3126 	r8168_mac_ocp_write(tp, 0xF868, 0xC51B);
3127 	r8168_mac_ocp_write(tp, 0xF86A, 0x9CA0);
3128 	r8168_mac_ocp_write(tp, 0xF86C, 0xE013);
3129 	r8168_mac_ocp_write(tp, 0xF86E, 0xC519);
3130 	r8168_mac_ocp_write(tp, 0xF870, 0x74A0);
3131 	r8168_mac_ocp_write(tp, 0xF872, 0x48C4);
3132 	r8168_mac_ocp_write(tp, 0xF874, 0x8CA0);
3133 	r8168_mac_ocp_write(tp, 0xF876, 0xC516);
3134 	r8168_mac_ocp_write(tp, 0xF878, 0x74A4);
3135 	r8168_mac_ocp_write(tp, 0xF87A, 0x48C8);
3136 	r8168_mac_ocp_write(tp, 0xF87C, 0x48CA);
3137 	r8168_mac_ocp_write(tp, 0xF87E, 0x9CA4);
3138 	r8168_mac_ocp_write(tp, 0xF880, 0xC512);
3139 	r8168_mac_ocp_write(tp, 0xF882, 0x1B00);
3140 	r8168_mac_ocp_write(tp, 0xF884, 0x9BA0);
3141 	r8168_mac_ocp_write(tp, 0xF886, 0x1B1C);
3142 	r8168_mac_ocp_write(tp, 0xF888, 0x483F);
3143 	r8168_mac_ocp_write(tp, 0xF88A, 0x9BA2);
3144 	r8168_mac_ocp_write(tp, 0xF88C, 0x1B04);
3145 	r8168_mac_ocp_write(tp, 0xF88E, 0xC508);
3146 	r8168_mac_ocp_write(tp, 0xF890, 0x9BA0);
3147 	r8168_mac_ocp_write(tp, 0xF892, 0xC505);
3148 	r8168_mac_ocp_write(tp, 0xF894, 0xBD00);
3149 	r8168_mac_ocp_write(tp, 0xF896, 0xC502);
3150 	r8168_mac_ocp_write(tp, 0xF898, 0xBD00);
3151 	r8168_mac_ocp_write(tp, 0xF89A, 0x0300);
3152 	r8168_mac_ocp_write(tp, 0xF89C, 0x051E);
3153 	r8168_mac_ocp_write(tp, 0xF89E, 0xE434);
3154 	r8168_mac_ocp_write(tp, 0xF8A0, 0xE018);
3155 	r8168_mac_ocp_write(tp, 0xF8A2, 0xE092);
3156 	r8168_mac_ocp_write(tp, 0xF8A4, 0xDE20);
3157 	r8168_mac_ocp_write(tp, 0xF8A6, 0xD3C0);
3158 	r8168_mac_ocp_write(tp, 0xF8A8, 0xC50F);
3159 	r8168_mac_ocp_write(tp, 0xF8AA, 0x76A4);
3160 	r8168_mac_ocp_write(tp, 0xF8AC, 0x49E3);
3161 	r8168_mac_ocp_write(tp, 0xF8AE, 0xF007);
3162 	r8168_mac_ocp_write(tp, 0xF8B0, 0x49C0);
3163 	r8168_mac_ocp_write(tp, 0xF8B2, 0xF103);
3164 	r8168_mac_ocp_write(tp, 0xF8B4, 0xC607);
3165 	r8168_mac_ocp_write(tp, 0xF8B6, 0xBE00);
3166 	r8168_mac_ocp_write(tp, 0xF8B8, 0xC606);
3167 	r8168_mac_ocp_write(tp, 0xF8BA, 0xBE00);
3168 	r8168_mac_ocp_write(tp, 0xF8BC, 0xC602);
3169 	r8168_mac_ocp_write(tp, 0xF8BE, 0xBE00);
3170 	r8168_mac_ocp_write(tp, 0xF8C0, 0x0C4C);
3171 	r8168_mac_ocp_write(tp, 0xF8C2, 0x0C28);
3172 	r8168_mac_ocp_write(tp, 0xF8C4, 0x0C2C);
3173 	r8168_mac_ocp_write(tp, 0xF8C6, 0xDC00);
3174 	r8168_mac_ocp_write(tp, 0xF8C8, 0xC707);
3175 	r8168_mac_ocp_write(tp, 0xF8CA, 0x1D00);
3176 	r8168_mac_ocp_write(tp, 0xF8CC, 0x8DE2);
3177 	r8168_mac_ocp_write(tp, 0xF8CE, 0x48C1);
3178 	r8168_mac_ocp_write(tp, 0xF8D0, 0xC502);
3179 	r8168_mac_ocp_write(tp, 0xF8D2, 0xBD00);
3180 	r8168_mac_ocp_write(tp, 0xF8D4, 0x00AA);
3181 	r8168_mac_ocp_write(tp, 0xF8D6, 0xE0C0);
3182 	r8168_mac_ocp_write(tp, 0xF8D8, 0xC502);
3183 	r8168_mac_ocp_write(tp, 0xF8DA, 0xBD00);
3184 	r8168_mac_ocp_write(tp, 0xF8DC, 0x0132);
3185 
3186 	r8168_mac_ocp_write(tp, 0xFC26, 0x8000);
3187 
3188 	r8168_mac_ocp_write(tp, 0xFC2A, 0x0743);
3189 	r8168_mac_ocp_write(tp, 0xFC2C, 0x0801);
3190 	r8168_mac_ocp_write(tp, 0xFC2E, 0x0BE9);
3191 	r8168_mac_ocp_write(tp, 0xFC30, 0x02FD);
3192 	r8168_mac_ocp_write(tp, 0xFC32, 0x0C25);
3193 	r8168_mac_ocp_write(tp, 0xFC34, 0x00A9);
3194 	r8168_mac_ocp_write(tp, 0xFC36, 0x012D);
3195 
3196 	rtl_hw_aspm_clkreq_enable(tp, true);
3197 }
3198 
3199 static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
3200 {
3201 	static const struct ephy_info e_info_8168h_1[] = {
3202 		{ 0x1e, 0x0800,	0x0001 },
3203 		{ 0x1d, 0x0000,	0x0800 },
3204 		{ 0x05, 0xffff,	0x2089 },
3205 		{ 0x06, 0xffff,	0x5881 },
3206 		{ 0x04, 0xffff,	0x854a },
3207 		{ 0x01, 0xffff,	0x068b }
3208 	};
3209 	int rg_saw_cnt;
3210 
3211 	/* disable aspm and clock request before access ephy */
3212 	rtl_hw_aspm_clkreq_enable(tp, false);
3213 	rtl_ephy_init(tp, e_info_8168h_1);
3214 
3215 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3216 	rtl8168g_set_pause_thresholds(tp, 0x38, 0x48);
3217 
3218 	rtl_set_def_aspm_entry_latency(tp);
3219 
3220 	rtl_reset_packet_filter(tp);
3221 
3222 	rtl_eri_set_bits(tp, 0xdc, 0x001c);
3223 
3224 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3225 
3226 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
3227 
3228 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3229 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3230 
3231 	rtl8168_config_eee_mac(tp);
3232 
3233 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3234 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3235 
3236 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3237 
3238 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
3239 
3240 	rtl_pcie_state_l2l3_disable(tp);
3241 
3242 	rg_saw_cnt = phy_read_paged(tp->phydev, 0x0c42, 0x13) & 0x3fff;
3243 	if (rg_saw_cnt > 0) {
3244 		u16 sw_cnt_1ms_ini;
3245 
3246 		sw_cnt_1ms_ini = 16000000/rg_saw_cnt;
3247 		sw_cnt_1ms_ini &= 0x0fff;
3248 		r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini);
3249 	}
3250 
3251 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0070);
3252 	r8168_mac_ocp_modify(tp, 0xe052, 0x6000, 0x8008);
3253 	r8168_mac_ocp_modify(tp, 0xe0d6, 0x01ff, 0x017f);
3254 	r8168_mac_ocp_modify(tp, 0xd420, 0x0fff, 0x047f);
3255 
3256 	r8168_mac_ocp_write(tp, 0xe63e, 0x0001);
3257 	r8168_mac_ocp_write(tp, 0xe63e, 0x0000);
3258 	r8168_mac_ocp_write(tp, 0xc094, 0x0000);
3259 	r8168_mac_ocp_write(tp, 0xc09e, 0x0000);
3260 
3261 	rtl_hw_aspm_clkreq_enable(tp, true);
3262 }
3263 
3264 static void rtl_hw_start_8168ep(struct rtl8169_private *tp)
3265 {
3266 	rtl8168ep_stop_cmac(tp);
3267 
3268 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3269 	rtl8168g_set_pause_thresholds(tp, 0x2f, 0x5f);
3270 
3271 	rtl_set_def_aspm_entry_latency(tp);
3272 
3273 	rtl_reset_packet_filter(tp);
3274 
3275 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3276 
3277 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
3278 
3279 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3280 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3281 
3282 	rtl8168_config_eee_mac(tp);
3283 
3284 	rtl_w0w1_eri(tp, 0x2fc, 0x01, 0x06);
3285 
3286 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3287 
3288 	rtl_pcie_state_l2l3_disable(tp);
3289 }
3290 
3291 static void rtl_hw_start_8168ep_1(struct rtl8169_private *tp)
3292 {
3293 	static const struct ephy_info e_info_8168ep_1[] = {
3294 		{ 0x00, 0xffff,	0x10ab },
3295 		{ 0x06, 0xffff,	0xf030 },
3296 		{ 0x08, 0xffff,	0x2006 },
3297 		{ 0x0d, 0xffff,	0x1666 },
3298 		{ 0x0c, 0x3ff0,	0x0000 }
3299 	};
3300 
3301 	/* disable aspm and clock request before access ephy */
3302 	rtl_hw_aspm_clkreq_enable(tp, false);
3303 	rtl_ephy_init(tp, e_info_8168ep_1);
3304 
3305 	rtl_hw_start_8168ep(tp);
3306 
3307 	rtl_hw_aspm_clkreq_enable(tp, true);
3308 }
3309 
3310 static void rtl_hw_start_8168ep_2(struct rtl8169_private *tp)
3311 {
3312 	static const struct ephy_info e_info_8168ep_2[] = {
3313 		{ 0x00, 0xffff,	0x10a3 },
3314 		{ 0x19, 0xffff,	0xfc00 },
3315 		{ 0x1e, 0xffff,	0x20ea }
3316 	};
3317 
3318 	/* disable aspm and clock request before access ephy */
3319 	rtl_hw_aspm_clkreq_enable(tp, false);
3320 	rtl_ephy_init(tp, e_info_8168ep_2);
3321 
3322 	rtl_hw_start_8168ep(tp);
3323 
3324 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3325 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3326 
3327 	rtl_hw_aspm_clkreq_enable(tp, true);
3328 }
3329 
3330 static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
3331 {
3332 	static const struct ephy_info e_info_8168ep_3[] = {
3333 		{ 0x00, 0x0000,	0x0080 },
3334 		{ 0x0d, 0x0100,	0x0200 },
3335 		{ 0x19, 0x8021,	0x0000 },
3336 		{ 0x1e, 0x0000,	0x2000 },
3337 	};
3338 
3339 	/* disable aspm and clock request before access ephy */
3340 	rtl_hw_aspm_clkreq_enable(tp, false);
3341 	rtl_ephy_init(tp, e_info_8168ep_3);
3342 
3343 	rtl_hw_start_8168ep(tp);
3344 
3345 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3346 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3347 
3348 	r8168_mac_ocp_modify(tp, 0xd3e2, 0x0fff, 0x0271);
3349 	r8168_mac_ocp_modify(tp, 0xd3e4, 0x00ff, 0x0000);
3350 	r8168_mac_ocp_modify(tp, 0xe860, 0x0000, 0x0080);
3351 
3352 	rtl_hw_aspm_clkreq_enable(tp, true);
3353 }
3354 
3355 static void rtl_hw_start_8117(struct rtl8169_private *tp)
3356 {
3357 	static const struct ephy_info e_info_8117[] = {
3358 		{ 0x19, 0x0040,	0x1100 },
3359 		{ 0x59, 0x0040,	0x1100 },
3360 	};
3361 	int rg_saw_cnt;
3362 
3363 	rtl8168ep_stop_cmac(tp);
3364 
3365 	/* disable aspm and clock request before access ephy */
3366 	rtl_hw_aspm_clkreq_enable(tp, false);
3367 	rtl_ephy_init(tp, e_info_8117);
3368 
3369 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3370 	rtl8168g_set_pause_thresholds(tp, 0x2f, 0x5f);
3371 
3372 	rtl_set_def_aspm_entry_latency(tp);
3373 
3374 	rtl_reset_packet_filter(tp);
3375 
3376 	rtl_eri_set_bits(tp, 0xd4, 0x0010);
3377 
3378 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3379 
3380 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
3381 
3382 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3383 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3384 
3385 	rtl8168_config_eee_mac(tp);
3386 
3387 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3388 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3389 
3390 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3391 
3392 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
3393 
3394 	rtl_pcie_state_l2l3_disable(tp);
3395 
3396 	rg_saw_cnt = phy_read_paged(tp->phydev, 0x0c42, 0x13) & 0x3fff;
3397 	if (rg_saw_cnt > 0) {
3398 		u16 sw_cnt_1ms_ini;
3399 
3400 		sw_cnt_1ms_ini = (16000000 / rg_saw_cnt) & 0x0fff;
3401 		r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini);
3402 	}
3403 
3404 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0070);
3405 	r8168_mac_ocp_write(tp, 0xea80, 0x0003);
3406 	r8168_mac_ocp_modify(tp, 0xe052, 0x0000, 0x0009);
3407 	r8168_mac_ocp_modify(tp, 0xd420, 0x0fff, 0x047f);
3408 
3409 	r8168_mac_ocp_write(tp, 0xe63e, 0x0001);
3410 	r8168_mac_ocp_write(tp, 0xe63e, 0x0000);
3411 	r8168_mac_ocp_write(tp, 0xc094, 0x0000);
3412 	r8168_mac_ocp_write(tp, 0xc09e, 0x0000);
3413 
3414 	/* firmware is for MAC only */
3415 	r8169_apply_firmware(tp);
3416 
3417 	rtl_hw_aspm_clkreq_enable(tp, true);
3418 }
3419 
3420 static void rtl_hw_start_8102e_1(struct rtl8169_private *tp)
3421 {
3422 	static const struct ephy_info e_info_8102e_1[] = {
3423 		{ 0x01,	0, 0x6e65 },
3424 		{ 0x02,	0, 0x091f },
3425 		{ 0x03,	0, 0xc2f9 },
3426 		{ 0x06,	0, 0xafb5 },
3427 		{ 0x07,	0, 0x0e00 },
3428 		{ 0x19,	0, 0xec80 },
3429 		{ 0x01,	0, 0x2e65 },
3430 		{ 0x01,	0, 0x6e65 }
3431 	};
3432 	u8 cfg1;
3433 
3434 	rtl_set_def_aspm_entry_latency(tp);
3435 
3436 	RTL_W8(tp, DBG_REG, FIX_NAK_1);
3437 
3438 	RTL_W8(tp, Config1,
3439 	       LEDS1 | LEDS0 | Speed_down | MEMMAP | IOMAP | VPD | PMEnable);
3440 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3441 
3442 	cfg1 = RTL_R8(tp, Config1);
3443 	if ((cfg1 & LEDS0) && (cfg1 & LEDS1))
3444 		RTL_W8(tp, Config1, cfg1 & ~LEDS0);
3445 
3446 	rtl_ephy_init(tp, e_info_8102e_1);
3447 }
3448 
3449 static void rtl_hw_start_8102e_2(struct rtl8169_private *tp)
3450 {
3451 	rtl_set_def_aspm_entry_latency(tp);
3452 
3453 	RTL_W8(tp, Config1, MEMMAP | IOMAP | VPD | PMEnable);
3454 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3455 }
3456 
3457 static void rtl_hw_start_8102e_3(struct rtl8169_private *tp)
3458 {
3459 	rtl_hw_start_8102e_2(tp);
3460 
3461 	rtl_ephy_write(tp, 0x03, 0xc2f9);
3462 }
3463 
3464 static void rtl_hw_start_8401(struct rtl8169_private *tp)
3465 {
3466 	static const struct ephy_info e_info_8401[] = {
3467 		{ 0x01,	0xffff, 0x6fe5 },
3468 		{ 0x03,	0xffff, 0x0599 },
3469 		{ 0x06,	0xffff, 0xaf25 },
3470 		{ 0x07,	0xffff, 0x8e68 },
3471 	};
3472 
3473 	rtl_ephy_init(tp, e_info_8401);
3474 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3475 }
3476 
3477 static void rtl_hw_start_8105e_1(struct rtl8169_private *tp)
3478 {
3479 	static const struct ephy_info e_info_8105e_1[] = {
3480 		{ 0x07,	0, 0x4000 },
3481 		{ 0x19,	0, 0x0200 },
3482 		{ 0x19,	0, 0x0020 },
3483 		{ 0x1e,	0, 0x2000 },
3484 		{ 0x03,	0, 0x0001 },
3485 		{ 0x19,	0, 0x0100 },
3486 		{ 0x19,	0, 0x0004 },
3487 		{ 0x0a,	0, 0x0020 }
3488 	};
3489 
3490 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3491 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3492 
3493 	/* Disable Early Tally Counter */
3494 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) & ~0x010000);
3495 
3496 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) | EN_NDP | EN_OOB_RESET);
3497 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
3498 
3499 	rtl_ephy_init(tp, e_info_8105e_1);
3500 
3501 	rtl_pcie_state_l2l3_disable(tp);
3502 }
3503 
3504 static void rtl_hw_start_8105e_2(struct rtl8169_private *tp)
3505 {
3506 	rtl_hw_start_8105e_1(tp);
3507 	rtl_ephy_write(tp, 0x1e, rtl_ephy_read(tp, 0x1e) | 0x8000);
3508 }
3509 
3510 static void rtl_hw_start_8402(struct rtl8169_private *tp)
3511 {
3512 	static const struct ephy_info e_info_8402[] = {
3513 		{ 0x19,	0xffff, 0xff64 },
3514 		{ 0x1e,	0, 0x4000 }
3515 	};
3516 
3517 	rtl_set_def_aspm_entry_latency(tp);
3518 
3519 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3520 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3521 
3522 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
3523 
3524 	rtl_ephy_init(tp, e_info_8402);
3525 
3526 	rtl_set_fifo_size(tp, 0x00, 0x00, 0x02, 0x06);
3527 	rtl_reset_packet_filter(tp);
3528 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3529 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3530 	rtl_w0w1_eri(tp, 0x0d4, 0x0e00, 0xff00);
3531 
3532 	/* disable EEE */
3533 	rtl_eri_write(tp, 0x1b0, ERIAR_MASK_0011, 0x0000);
3534 
3535 	rtl_pcie_state_l2l3_disable(tp);
3536 }
3537 
3538 static void rtl_hw_start_8106(struct rtl8169_private *tp)
3539 {
3540 	rtl_hw_aspm_clkreq_enable(tp, false);
3541 
3542 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3543 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3544 
3545 	RTL_W32(tp, MISC, (RTL_R32(tp, MISC) | DISABLE_LAN_EN) & ~EARLY_TALLY_EN);
3546 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) | EN_NDP | EN_OOB_RESET);
3547 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3548 
3549 	/* L0 7us, L1 32us - needed to avoid issues with link-up detection */
3550 	rtl_set_aspm_entry_latency(tp, 0x2f);
3551 
3552 	rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x0000);
3553 
3554 	/* disable EEE */
3555 	rtl_eri_write(tp, 0x1b0, ERIAR_MASK_0011, 0x0000);
3556 
3557 	rtl_pcie_state_l2l3_disable(tp);
3558 	rtl_hw_aspm_clkreq_enable(tp, true);
3559 }
3560 
3561 DECLARE_RTL_COND(rtl_mac_ocp_e00e_cond)
3562 {
3563 	return r8168_mac_ocp_read(tp, 0xe00e) & BIT(13);
3564 }
3565 
3566 static void rtl_hw_start_8125_common(struct rtl8169_private *tp)
3567 {
3568 	rtl_pcie_state_l2l3_disable(tp);
3569 
3570 	RTL_W16(tp, 0x382, 0x221b);
3571 	RTL_W8(tp, 0x4500, 0);
3572 	RTL_W16(tp, 0x4800, 0);
3573 
3574 	/* disable UPS */
3575 	r8168_mac_ocp_modify(tp, 0xd40a, 0x0010, 0x0000);
3576 
3577 	RTL_W8(tp, Config1, RTL_R8(tp, Config1) & ~0x10);
3578 
3579 	r8168_mac_ocp_write(tp, 0xc140, 0xffff);
3580 	r8168_mac_ocp_write(tp, 0xc142, 0xffff);
3581 
3582 	r8168_mac_ocp_modify(tp, 0xd3e2, 0x0fff, 0x03a9);
3583 	r8168_mac_ocp_modify(tp, 0xd3e4, 0x00ff, 0x0000);
3584 	r8168_mac_ocp_modify(tp, 0xe860, 0x0000, 0x0080);
3585 
3586 	/* disable new tx descriptor format */
3587 	r8168_mac_ocp_modify(tp, 0xeb58, 0x0001, 0x0000);
3588 
3589 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3590 		r8168_mac_ocp_modify(tp, 0xe614, 0x0700, 0x0200);
3591 	else
3592 		r8168_mac_ocp_modify(tp, 0xe614, 0x0700, 0x0400);
3593 
3594 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3595 		r8168_mac_ocp_modify(tp, 0xe63e, 0x0c30, 0x0000);
3596 	else
3597 		r8168_mac_ocp_modify(tp, 0xe63e, 0x0c30, 0x0020);
3598 
3599 	r8168_mac_ocp_modify(tp, 0xc0b4, 0x0000, 0x000c);
3600 	r8168_mac_ocp_modify(tp, 0xeb6a, 0x00ff, 0x0033);
3601 	r8168_mac_ocp_modify(tp, 0xeb50, 0x03e0, 0x0040);
3602 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0030);
3603 	r8168_mac_ocp_modify(tp, 0xe040, 0x1000, 0x0000);
3604 	r8168_mac_ocp_modify(tp, 0xea1c, 0x0003, 0x0001);
3605 	r8168_mac_ocp_modify(tp, 0xe0c0, 0x4f0f, 0x4403);
3606 	r8168_mac_ocp_modify(tp, 0xe052, 0x0080, 0x0068);
3607 	r8168_mac_ocp_modify(tp, 0xd430, 0x0fff, 0x047f);
3608 
3609 	r8168_mac_ocp_modify(tp, 0xea1c, 0x0004, 0x0000);
3610 	r8168_mac_ocp_modify(tp, 0xeb54, 0x0000, 0x0001);
3611 	udelay(1);
3612 	r8168_mac_ocp_modify(tp, 0xeb54, 0x0001, 0x0000);
3613 	RTL_W16(tp, 0x1880, RTL_R16(tp, 0x1880) & ~0x0030);
3614 
3615 	r8168_mac_ocp_write(tp, 0xe098, 0xc302);
3616 
3617 	rtl_loop_wait_low(tp, &rtl_mac_ocp_e00e_cond, 1000, 10);
3618 
3619 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3620 		rtl8125b_config_eee_mac(tp);
3621 	else
3622 		rtl8125a_config_eee_mac(tp);
3623 
3624 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
3625 	udelay(10);
3626 }
3627 
3628 static void rtl_hw_start_8125a_1(struct rtl8169_private *tp)
3629 {
3630 	static const struct ephy_info e_info_8125a_1[] = {
3631 		{ 0x01, 0xffff, 0xa812 },
3632 		{ 0x09, 0xffff, 0x520c },
3633 		{ 0x04, 0xffff, 0xd000 },
3634 		{ 0x0d, 0xffff, 0xf702 },
3635 		{ 0x0a, 0xffff, 0x8653 },
3636 		{ 0x06, 0xffff, 0x001e },
3637 		{ 0x08, 0xffff, 0x3595 },
3638 		{ 0x20, 0xffff, 0x9455 },
3639 		{ 0x21, 0xffff, 0x99ff },
3640 		{ 0x02, 0xffff, 0x6046 },
3641 		{ 0x29, 0xffff, 0xfe00 },
3642 		{ 0x23, 0xffff, 0xab62 },
3643 
3644 		{ 0x41, 0xffff, 0xa80c },
3645 		{ 0x49, 0xffff, 0x520c },
3646 		{ 0x44, 0xffff, 0xd000 },
3647 		{ 0x4d, 0xffff, 0xf702 },
3648 		{ 0x4a, 0xffff, 0x8653 },
3649 		{ 0x46, 0xffff, 0x001e },
3650 		{ 0x48, 0xffff, 0x3595 },
3651 		{ 0x60, 0xffff, 0x9455 },
3652 		{ 0x61, 0xffff, 0x99ff },
3653 		{ 0x42, 0xffff, 0x6046 },
3654 		{ 0x69, 0xffff, 0xfe00 },
3655 		{ 0x63, 0xffff, 0xab62 },
3656 	};
3657 
3658 	rtl_set_def_aspm_entry_latency(tp);
3659 
3660 	/* disable aspm and clock request before access ephy */
3661 	rtl_hw_aspm_clkreq_enable(tp, false);
3662 	rtl_ephy_init(tp, e_info_8125a_1);
3663 
3664 	rtl_hw_start_8125_common(tp);
3665 	rtl_hw_aspm_clkreq_enable(tp, true);
3666 }
3667 
3668 static void rtl_hw_start_8125a_2(struct rtl8169_private *tp)
3669 {
3670 	static const struct ephy_info e_info_8125a_2[] = {
3671 		{ 0x04, 0xffff, 0xd000 },
3672 		{ 0x0a, 0xffff, 0x8653 },
3673 		{ 0x23, 0xffff, 0xab66 },
3674 		{ 0x20, 0xffff, 0x9455 },
3675 		{ 0x21, 0xffff, 0x99ff },
3676 		{ 0x29, 0xffff, 0xfe04 },
3677 
3678 		{ 0x44, 0xffff, 0xd000 },
3679 		{ 0x4a, 0xffff, 0x8653 },
3680 		{ 0x63, 0xffff, 0xab66 },
3681 		{ 0x60, 0xffff, 0x9455 },
3682 		{ 0x61, 0xffff, 0x99ff },
3683 		{ 0x69, 0xffff, 0xfe04 },
3684 	};
3685 
3686 	rtl_set_def_aspm_entry_latency(tp);
3687 
3688 	/* disable aspm and clock request before access ephy */
3689 	rtl_hw_aspm_clkreq_enable(tp, false);
3690 	rtl_ephy_init(tp, e_info_8125a_2);
3691 
3692 	rtl_hw_start_8125_common(tp);
3693 	rtl_hw_aspm_clkreq_enable(tp, true);
3694 }
3695 
3696 static void rtl_hw_start_8125b(struct rtl8169_private *tp)
3697 {
3698 	static const struct ephy_info e_info_8125b[] = {
3699 		{ 0x0b, 0xffff, 0xa908 },
3700 		{ 0x1e, 0xffff, 0x20eb },
3701 		{ 0x4b, 0xffff, 0xa908 },
3702 		{ 0x5e, 0xffff, 0x20eb },
3703 		{ 0x22, 0x0030, 0x0020 },
3704 		{ 0x62, 0x0030, 0x0020 },
3705 	};
3706 
3707 	rtl_set_def_aspm_entry_latency(tp);
3708 	rtl_hw_aspm_clkreq_enable(tp, false);
3709 
3710 	rtl_ephy_init(tp, e_info_8125b);
3711 	rtl_hw_start_8125_common(tp);
3712 
3713 	rtl_hw_aspm_clkreq_enable(tp, true);
3714 }
3715 
3716 static void rtl_hw_config(struct rtl8169_private *tp)
3717 {
3718 	static const rtl_generic_fct hw_configs[] = {
3719 		[RTL_GIGA_MAC_VER_07] = rtl_hw_start_8102e_1,
3720 		[RTL_GIGA_MAC_VER_08] = rtl_hw_start_8102e_3,
3721 		[RTL_GIGA_MAC_VER_09] = rtl_hw_start_8102e_2,
3722 		[RTL_GIGA_MAC_VER_10] = NULL,
3723 		[RTL_GIGA_MAC_VER_11] = rtl_hw_start_8168b,
3724 		[RTL_GIGA_MAC_VER_12] = rtl_hw_start_8168b,
3725 		[RTL_GIGA_MAC_VER_13] = NULL,
3726 		[RTL_GIGA_MAC_VER_14] = rtl_hw_start_8401,
3727 		[RTL_GIGA_MAC_VER_16] = NULL,
3728 		[RTL_GIGA_MAC_VER_17] = rtl_hw_start_8168b,
3729 		[RTL_GIGA_MAC_VER_18] = rtl_hw_start_8168cp_1,
3730 		[RTL_GIGA_MAC_VER_19] = rtl_hw_start_8168c_1,
3731 		[RTL_GIGA_MAC_VER_20] = rtl_hw_start_8168c_2,
3732 		[RTL_GIGA_MAC_VER_21] = rtl_hw_start_8168c_2,
3733 		[RTL_GIGA_MAC_VER_22] = rtl_hw_start_8168c_4,
3734 		[RTL_GIGA_MAC_VER_23] = rtl_hw_start_8168cp_2,
3735 		[RTL_GIGA_MAC_VER_24] = rtl_hw_start_8168cp_3,
3736 		[RTL_GIGA_MAC_VER_25] = rtl_hw_start_8168d,
3737 		[RTL_GIGA_MAC_VER_26] = rtl_hw_start_8168d,
3738 		[RTL_GIGA_MAC_VER_28] = rtl_hw_start_8168d_4,
3739 		[RTL_GIGA_MAC_VER_29] = rtl_hw_start_8105e_1,
3740 		[RTL_GIGA_MAC_VER_30] = rtl_hw_start_8105e_2,
3741 		[RTL_GIGA_MAC_VER_31] = rtl_hw_start_8168d,
3742 		[RTL_GIGA_MAC_VER_32] = rtl_hw_start_8168e_1,
3743 		[RTL_GIGA_MAC_VER_33] = rtl_hw_start_8168e_1,
3744 		[RTL_GIGA_MAC_VER_34] = rtl_hw_start_8168e_2,
3745 		[RTL_GIGA_MAC_VER_35] = rtl_hw_start_8168f_1,
3746 		[RTL_GIGA_MAC_VER_36] = rtl_hw_start_8168f_1,
3747 		[RTL_GIGA_MAC_VER_37] = rtl_hw_start_8402,
3748 		[RTL_GIGA_MAC_VER_38] = rtl_hw_start_8411,
3749 		[RTL_GIGA_MAC_VER_39] = rtl_hw_start_8106,
3750 		[RTL_GIGA_MAC_VER_40] = rtl_hw_start_8168g_1,
3751 		[RTL_GIGA_MAC_VER_41] = rtl_hw_start_8168g_1,
3752 		[RTL_GIGA_MAC_VER_42] = rtl_hw_start_8168g_2,
3753 		[RTL_GIGA_MAC_VER_43] = rtl_hw_start_8168g_2,
3754 		[RTL_GIGA_MAC_VER_44] = rtl_hw_start_8411_2,
3755 		[RTL_GIGA_MAC_VER_45] = rtl_hw_start_8168h_1,
3756 		[RTL_GIGA_MAC_VER_46] = rtl_hw_start_8168h_1,
3757 		[RTL_GIGA_MAC_VER_47] = rtl_hw_start_8168h_1,
3758 		[RTL_GIGA_MAC_VER_48] = rtl_hw_start_8168h_1,
3759 		[RTL_GIGA_MAC_VER_49] = rtl_hw_start_8168ep_1,
3760 		[RTL_GIGA_MAC_VER_50] = rtl_hw_start_8168ep_2,
3761 		[RTL_GIGA_MAC_VER_51] = rtl_hw_start_8168ep_3,
3762 		[RTL_GIGA_MAC_VER_52] = rtl_hw_start_8117,
3763 		[RTL_GIGA_MAC_VER_53] = rtl_hw_start_8117,
3764 		[RTL_GIGA_MAC_VER_60] = rtl_hw_start_8125a_1,
3765 		[RTL_GIGA_MAC_VER_61] = rtl_hw_start_8125a_2,
3766 		[RTL_GIGA_MAC_VER_63] = rtl_hw_start_8125b,
3767 	};
3768 
3769 	if (hw_configs[tp->mac_version])
3770 		hw_configs[tp->mac_version](tp);
3771 }
3772 
3773 static void rtl_hw_start_8125(struct rtl8169_private *tp)
3774 {
3775 	int i;
3776 
3777 	/* disable interrupt coalescing */
3778 	for (i = 0xa00; i < 0xb00; i += 4)
3779 		RTL_W32(tp, i, 0);
3780 
3781 	rtl_hw_config(tp);
3782 }
3783 
3784 static void rtl_hw_start_8168(struct rtl8169_private *tp)
3785 {
3786 	if (rtl_is_8168evl_up(tp))
3787 		RTL_W8(tp, MaxTxPacketSize, EarlySize);
3788 	else
3789 		RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
3790 
3791 	rtl_hw_config(tp);
3792 
3793 	/* disable interrupt coalescing */
3794 	RTL_W16(tp, IntrMitigate, 0x0000);
3795 }
3796 
3797 static void rtl_hw_start_8169(struct rtl8169_private *tp)
3798 {
3799 	RTL_W8(tp, EarlyTxThres, NoEarlyTx);
3800 
3801 	tp->cp_cmd |= PCIMulRW;
3802 
3803 	if (tp->mac_version == RTL_GIGA_MAC_VER_02 ||
3804 	    tp->mac_version == RTL_GIGA_MAC_VER_03)
3805 		tp->cp_cmd |= EnAnaPLL;
3806 
3807 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
3808 
3809 	rtl8169_set_magic_reg(tp);
3810 
3811 	/* disable interrupt coalescing */
3812 	RTL_W16(tp, IntrMitigate, 0x0000);
3813 }
3814 
3815 static void rtl_hw_start(struct  rtl8169_private *tp)
3816 {
3817 	rtl_unlock_config_regs(tp);
3818 
3819 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
3820 
3821 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
3822 		rtl_hw_start_8169(tp);
3823 	else if (rtl_is_8125(tp))
3824 		rtl_hw_start_8125(tp);
3825 	else
3826 		rtl_hw_start_8168(tp);
3827 
3828 	rtl_enable_exit_l1(tp);
3829 	rtl_set_rx_max_size(tp);
3830 	rtl_set_rx_tx_desc_registers(tp);
3831 	rtl_lock_config_regs(tp);
3832 
3833 	rtl_jumbo_config(tp);
3834 
3835 	/* Initially a 10 us delay. Turned it into a PCI commit. - FR */
3836 	rtl_pci_commit(tp);
3837 
3838 	RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
3839 	rtl_init_rxcfg(tp);
3840 	rtl_set_tx_config_registers(tp);
3841 	rtl_set_rx_config_features(tp, tp->dev->features);
3842 	rtl_set_rx_mode(tp->dev);
3843 	rtl_irq_enable(tp);
3844 }
3845 
3846 static int rtl8169_change_mtu(struct net_device *dev, int new_mtu)
3847 {
3848 	struct rtl8169_private *tp = netdev_priv(dev);
3849 
3850 	dev->mtu = new_mtu;
3851 	netdev_update_features(dev);
3852 	rtl_jumbo_config(tp);
3853 
3854 	switch (tp->mac_version) {
3855 	case RTL_GIGA_MAC_VER_61:
3856 	case RTL_GIGA_MAC_VER_63:
3857 		rtl8125_set_eee_txidle_timer(tp);
3858 		break;
3859 	default:
3860 		break;
3861 	}
3862 
3863 	return 0;
3864 }
3865 
3866 static void rtl8169_mark_to_asic(struct RxDesc *desc)
3867 {
3868 	u32 eor = le32_to_cpu(desc->opts1) & RingEnd;
3869 
3870 	desc->opts2 = 0;
3871 	/* Force memory writes to complete before releasing descriptor */
3872 	dma_wmb();
3873 	WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | R8169_RX_BUF_SIZE));
3874 }
3875 
3876 static struct page *rtl8169_alloc_rx_data(struct rtl8169_private *tp,
3877 					  struct RxDesc *desc)
3878 {
3879 	struct device *d = tp_to_dev(tp);
3880 	int node = dev_to_node(d);
3881 	dma_addr_t mapping;
3882 	struct page *data;
3883 
3884 	data = alloc_pages_node(node, GFP_KERNEL, get_order(R8169_RX_BUF_SIZE));
3885 	if (!data)
3886 		return NULL;
3887 
3888 	mapping = dma_map_page(d, data, 0, R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
3889 	if (unlikely(dma_mapping_error(d, mapping))) {
3890 		netdev_err(tp->dev, "Failed to map RX DMA!\n");
3891 		__free_pages(data, get_order(R8169_RX_BUF_SIZE));
3892 		return NULL;
3893 	}
3894 
3895 	desc->addr = cpu_to_le64(mapping);
3896 	rtl8169_mark_to_asic(desc);
3897 
3898 	return data;
3899 }
3900 
3901 static void rtl8169_rx_clear(struct rtl8169_private *tp)
3902 {
3903 	int i;
3904 
3905 	for (i = 0; i < NUM_RX_DESC && tp->Rx_databuff[i]; i++) {
3906 		dma_unmap_page(tp_to_dev(tp),
3907 			       le64_to_cpu(tp->RxDescArray[i].addr),
3908 			       R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
3909 		__free_pages(tp->Rx_databuff[i], get_order(R8169_RX_BUF_SIZE));
3910 		tp->Rx_databuff[i] = NULL;
3911 		tp->RxDescArray[i].addr = 0;
3912 		tp->RxDescArray[i].opts1 = 0;
3913 	}
3914 }
3915 
3916 static int rtl8169_rx_fill(struct rtl8169_private *tp)
3917 {
3918 	int i;
3919 
3920 	for (i = 0; i < NUM_RX_DESC; i++) {
3921 		struct page *data;
3922 
3923 		data = rtl8169_alloc_rx_data(tp, tp->RxDescArray + i);
3924 		if (!data) {
3925 			rtl8169_rx_clear(tp);
3926 			return -ENOMEM;
3927 		}
3928 		tp->Rx_databuff[i] = data;
3929 	}
3930 
3931 	/* mark as last descriptor in the ring */
3932 	tp->RxDescArray[NUM_RX_DESC - 1].opts1 |= cpu_to_le32(RingEnd);
3933 
3934 	return 0;
3935 }
3936 
3937 static int rtl8169_init_ring(struct rtl8169_private *tp)
3938 {
3939 	rtl8169_init_ring_indexes(tp);
3940 
3941 	memset(tp->tx_skb, 0, sizeof(tp->tx_skb));
3942 	memset(tp->Rx_databuff, 0, sizeof(tp->Rx_databuff));
3943 
3944 	return rtl8169_rx_fill(tp);
3945 }
3946 
3947 static void rtl8169_unmap_tx_skb(struct rtl8169_private *tp, unsigned int entry)
3948 {
3949 	struct ring_info *tx_skb = tp->tx_skb + entry;
3950 	struct TxDesc *desc = tp->TxDescArray + entry;
3951 
3952 	dma_unmap_single(tp_to_dev(tp), le64_to_cpu(desc->addr), tx_skb->len,
3953 			 DMA_TO_DEVICE);
3954 	memset(desc, 0, sizeof(*desc));
3955 	memset(tx_skb, 0, sizeof(*tx_skb));
3956 }
3957 
3958 static void rtl8169_tx_clear_range(struct rtl8169_private *tp, u32 start,
3959 				   unsigned int n)
3960 {
3961 	unsigned int i;
3962 
3963 	for (i = 0; i < n; i++) {
3964 		unsigned int entry = (start + i) % NUM_TX_DESC;
3965 		struct ring_info *tx_skb = tp->tx_skb + entry;
3966 		unsigned int len = tx_skb->len;
3967 
3968 		if (len) {
3969 			struct sk_buff *skb = tx_skb->skb;
3970 
3971 			rtl8169_unmap_tx_skb(tp, entry);
3972 			if (skb)
3973 				dev_consume_skb_any(skb);
3974 		}
3975 	}
3976 }
3977 
3978 static void rtl8169_tx_clear(struct rtl8169_private *tp)
3979 {
3980 	rtl8169_tx_clear_range(tp, tp->dirty_tx, NUM_TX_DESC);
3981 	netdev_reset_queue(tp->dev);
3982 }
3983 
3984 static void rtl8169_cleanup(struct rtl8169_private *tp, bool going_down)
3985 {
3986 	napi_disable(&tp->napi);
3987 
3988 	/* Give a racing hard_start_xmit a few cycles to complete. */
3989 	synchronize_net();
3990 
3991 	/* Disable interrupts */
3992 	rtl8169_irq_mask_and_ack(tp);
3993 
3994 	rtl_rx_close(tp);
3995 
3996 	if (going_down && tp->dev->wol_enabled)
3997 		goto no_reset;
3998 
3999 	switch (tp->mac_version) {
4000 	case RTL_GIGA_MAC_VER_28:
4001 	case RTL_GIGA_MAC_VER_31:
4002 		rtl_loop_wait_low(tp, &rtl_npq_cond, 20, 2000);
4003 		break;
4004 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
4005 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
4006 		rtl_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 666);
4007 		break;
4008 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
4009 		rtl_enable_rxdvgate(tp);
4010 		fsleep(2000);
4011 		break;
4012 	default:
4013 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
4014 		fsleep(100);
4015 		break;
4016 	}
4017 
4018 	rtl_hw_reset(tp);
4019 no_reset:
4020 	rtl8169_tx_clear(tp);
4021 	rtl8169_init_ring_indexes(tp);
4022 }
4023 
4024 static void rtl_reset_work(struct rtl8169_private *tp)
4025 {
4026 	int i;
4027 
4028 	netif_stop_queue(tp->dev);
4029 
4030 	rtl8169_cleanup(tp, false);
4031 
4032 	for (i = 0; i < NUM_RX_DESC; i++)
4033 		rtl8169_mark_to_asic(tp->RxDescArray + i);
4034 
4035 	napi_enable(&tp->napi);
4036 	rtl_hw_start(tp);
4037 }
4038 
4039 static void rtl8169_tx_timeout(struct net_device *dev, unsigned int txqueue)
4040 {
4041 	struct rtl8169_private *tp = netdev_priv(dev);
4042 
4043 	rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
4044 }
4045 
4046 static int rtl8169_tx_map(struct rtl8169_private *tp, const u32 *opts, u32 len,
4047 			  void *addr, unsigned int entry, bool desc_own)
4048 {
4049 	struct TxDesc *txd = tp->TxDescArray + entry;
4050 	struct device *d = tp_to_dev(tp);
4051 	dma_addr_t mapping;
4052 	u32 opts1;
4053 	int ret;
4054 
4055 	mapping = dma_map_single(d, addr, len, DMA_TO_DEVICE);
4056 	ret = dma_mapping_error(d, mapping);
4057 	if (unlikely(ret)) {
4058 		if (net_ratelimit())
4059 			netdev_err(tp->dev, "Failed to map TX data!\n");
4060 		return ret;
4061 	}
4062 
4063 	txd->addr = cpu_to_le64(mapping);
4064 	txd->opts2 = cpu_to_le32(opts[1]);
4065 
4066 	opts1 = opts[0] | len;
4067 	if (entry == NUM_TX_DESC - 1)
4068 		opts1 |= RingEnd;
4069 	if (desc_own)
4070 		opts1 |= DescOwn;
4071 	txd->opts1 = cpu_to_le32(opts1);
4072 
4073 	tp->tx_skb[entry].len = len;
4074 
4075 	return 0;
4076 }
4077 
4078 static int rtl8169_xmit_frags(struct rtl8169_private *tp, struct sk_buff *skb,
4079 			      const u32 *opts, unsigned int entry)
4080 {
4081 	struct skb_shared_info *info = skb_shinfo(skb);
4082 	unsigned int cur_frag;
4083 
4084 	for (cur_frag = 0; cur_frag < info->nr_frags; cur_frag++) {
4085 		const skb_frag_t *frag = info->frags + cur_frag;
4086 		void *addr = skb_frag_address(frag);
4087 		u32 len = skb_frag_size(frag);
4088 
4089 		entry = (entry + 1) % NUM_TX_DESC;
4090 
4091 		if (unlikely(rtl8169_tx_map(tp, opts, len, addr, entry, true)))
4092 			goto err_out;
4093 	}
4094 
4095 	return 0;
4096 
4097 err_out:
4098 	rtl8169_tx_clear_range(tp, tp->cur_tx + 1, cur_frag);
4099 	return -EIO;
4100 }
4101 
4102 static bool rtl_skb_is_udp(struct sk_buff *skb)
4103 {
4104 	int no = skb_network_offset(skb);
4105 	struct ipv6hdr *i6h, _i6h;
4106 	struct iphdr *ih, _ih;
4107 
4108 	switch (vlan_get_protocol(skb)) {
4109 	case htons(ETH_P_IP):
4110 		ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih);
4111 		return ih && ih->protocol == IPPROTO_UDP;
4112 	case htons(ETH_P_IPV6):
4113 		i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h);
4114 		return i6h && i6h->nexthdr == IPPROTO_UDP;
4115 	default:
4116 		return false;
4117 	}
4118 }
4119 
4120 #define RTL_MIN_PATCH_LEN	47
4121 
4122 /* see rtl8125_get_patch_pad_len() in r8125 vendor driver */
4123 static unsigned int rtl8125_quirk_udp_padto(struct rtl8169_private *tp,
4124 					    struct sk_buff *skb)
4125 {
4126 	unsigned int padto = 0, len = skb->len;
4127 
4128 	if (rtl_is_8125(tp) && len < 128 + RTL_MIN_PATCH_LEN &&
4129 	    rtl_skb_is_udp(skb) && skb_transport_header_was_set(skb)) {
4130 		unsigned int trans_data_len = skb_tail_pointer(skb) -
4131 					      skb_transport_header(skb);
4132 
4133 		if (trans_data_len >= offsetof(struct udphdr, len) &&
4134 		    trans_data_len < RTL_MIN_PATCH_LEN) {
4135 			u16 dest = ntohs(udp_hdr(skb)->dest);
4136 
4137 			/* dest is a standard PTP port */
4138 			if (dest == 319 || dest == 320)
4139 				padto = len + RTL_MIN_PATCH_LEN - trans_data_len;
4140 		}
4141 
4142 		if (trans_data_len < sizeof(struct udphdr))
4143 			padto = max_t(unsigned int, padto,
4144 				      len + sizeof(struct udphdr) - trans_data_len);
4145 	}
4146 
4147 	return padto;
4148 }
4149 
4150 static unsigned int rtl_quirk_packet_padto(struct rtl8169_private *tp,
4151 					   struct sk_buff *skb)
4152 {
4153 	unsigned int padto;
4154 
4155 	padto = rtl8125_quirk_udp_padto(tp, skb);
4156 
4157 	switch (tp->mac_version) {
4158 	case RTL_GIGA_MAC_VER_34:
4159 	case RTL_GIGA_MAC_VER_60:
4160 	case RTL_GIGA_MAC_VER_61:
4161 	case RTL_GIGA_MAC_VER_63:
4162 		padto = max_t(unsigned int, padto, ETH_ZLEN);
4163 		break;
4164 	default:
4165 		break;
4166 	}
4167 
4168 	return padto;
4169 }
4170 
4171 static void rtl8169_tso_csum_v1(struct sk_buff *skb, u32 *opts)
4172 {
4173 	u32 mss = skb_shinfo(skb)->gso_size;
4174 
4175 	if (mss) {
4176 		opts[0] |= TD_LSO;
4177 		opts[0] |= mss << TD0_MSS_SHIFT;
4178 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4179 		const struct iphdr *ip = ip_hdr(skb);
4180 
4181 		if (ip->protocol == IPPROTO_TCP)
4182 			opts[0] |= TD0_IP_CS | TD0_TCP_CS;
4183 		else if (ip->protocol == IPPROTO_UDP)
4184 			opts[0] |= TD0_IP_CS | TD0_UDP_CS;
4185 		else
4186 			WARN_ON_ONCE(1);
4187 	}
4188 }
4189 
4190 static bool rtl8169_tso_csum_v2(struct rtl8169_private *tp,
4191 				struct sk_buff *skb, u32 *opts)
4192 {
4193 	u32 transport_offset = (u32)skb_transport_offset(skb);
4194 	struct skb_shared_info *shinfo = skb_shinfo(skb);
4195 	u32 mss = shinfo->gso_size;
4196 
4197 	if (mss) {
4198 		if (shinfo->gso_type & SKB_GSO_TCPV4) {
4199 			opts[0] |= TD1_GTSENV4;
4200 		} else if (shinfo->gso_type & SKB_GSO_TCPV6) {
4201 			if (skb_cow_head(skb, 0))
4202 				return false;
4203 
4204 			tcp_v6_gso_csum_prep(skb);
4205 			opts[0] |= TD1_GTSENV6;
4206 		} else {
4207 			WARN_ON_ONCE(1);
4208 		}
4209 
4210 		opts[0] |= transport_offset << GTTCPHO_SHIFT;
4211 		opts[1] |= mss << TD1_MSS_SHIFT;
4212 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4213 		u8 ip_protocol;
4214 
4215 		switch (vlan_get_protocol(skb)) {
4216 		case htons(ETH_P_IP):
4217 			opts[1] |= TD1_IPv4_CS;
4218 			ip_protocol = ip_hdr(skb)->protocol;
4219 			break;
4220 
4221 		case htons(ETH_P_IPV6):
4222 			opts[1] |= TD1_IPv6_CS;
4223 			ip_protocol = ipv6_hdr(skb)->nexthdr;
4224 			break;
4225 
4226 		default:
4227 			ip_protocol = IPPROTO_RAW;
4228 			break;
4229 		}
4230 
4231 		if (ip_protocol == IPPROTO_TCP)
4232 			opts[1] |= TD1_TCP_CS;
4233 		else if (ip_protocol == IPPROTO_UDP)
4234 			opts[1] |= TD1_UDP_CS;
4235 		else
4236 			WARN_ON_ONCE(1);
4237 
4238 		opts[1] |= transport_offset << TCPHO_SHIFT;
4239 	} else {
4240 		unsigned int padto = rtl_quirk_packet_padto(tp, skb);
4241 
4242 		/* skb_padto would free the skb on error */
4243 		return !__skb_put_padto(skb, padto, false);
4244 	}
4245 
4246 	return true;
4247 }
4248 
4249 static bool rtl_tx_slots_avail(struct rtl8169_private *tp)
4250 {
4251 	unsigned int slots_avail = READ_ONCE(tp->dirty_tx) + NUM_TX_DESC
4252 					- READ_ONCE(tp->cur_tx);
4253 
4254 	/* A skbuff with nr_frags needs nr_frags+1 entries in the tx queue */
4255 	return slots_avail > MAX_SKB_FRAGS;
4256 }
4257 
4258 /* Versions RTL8102e and from RTL8168c onwards support csum_v2 */
4259 static bool rtl_chip_supports_csum_v2(struct rtl8169_private *tp)
4260 {
4261 	switch (tp->mac_version) {
4262 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
4263 	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
4264 		return false;
4265 	default:
4266 		return true;
4267 	}
4268 }
4269 
4270 static void rtl8169_doorbell(struct rtl8169_private *tp)
4271 {
4272 	if (rtl_is_8125(tp))
4273 		RTL_W16(tp, TxPoll_8125, BIT(0));
4274 	else
4275 		RTL_W8(tp, TxPoll, NPQ);
4276 }
4277 
4278 static netdev_tx_t rtl8169_start_xmit(struct sk_buff *skb,
4279 				      struct net_device *dev)
4280 {
4281 	unsigned int frags = skb_shinfo(skb)->nr_frags;
4282 	struct rtl8169_private *tp = netdev_priv(dev);
4283 	unsigned int entry = tp->cur_tx % NUM_TX_DESC;
4284 	struct TxDesc *txd_first, *txd_last;
4285 	bool stop_queue, door_bell;
4286 	u32 opts[2];
4287 
4288 	if (unlikely(!rtl_tx_slots_avail(tp))) {
4289 		if (net_ratelimit())
4290 			netdev_err(dev, "BUG! Tx Ring full when queue awake!\n");
4291 		goto err_stop_0;
4292 	}
4293 
4294 	opts[1] = rtl8169_tx_vlan_tag(skb);
4295 	opts[0] = 0;
4296 
4297 	if (!rtl_chip_supports_csum_v2(tp))
4298 		rtl8169_tso_csum_v1(skb, opts);
4299 	else if (!rtl8169_tso_csum_v2(tp, skb, opts))
4300 		goto err_dma_0;
4301 
4302 	if (unlikely(rtl8169_tx_map(tp, opts, skb_headlen(skb), skb->data,
4303 				    entry, false)))
4304 		goto err_dma_0;
4305 
4306 	txd_first = tp->TxDescArray + entry;
4307 
4308 	if (frags) {
4309 		if (rtl8169_xmit_frags(tp, skb, opts, entry))
4310 			goto err_dma_1;
4311 		entry = (entry + frags) % NUM_TX_DESC;
4312 	}
4313 
4314 	txd_last = tp->TxDescArray + entry;
4315 	txd_last->opts1 |= cpu_to_le32(LastFrag);
4316 	tp->tx_skb[entry].skb = skb;
4317 
4318 	skb_tx_timestamp(skb);
4319 
4320 	/* Force memory writes to complete before releasing descriptor */
4321 	dma_wmb();
4322 
4323 	door_bell = __netdev_sent_queue(dev, skb->len, netdev_xmit_more());
4324 
4325 	txd_first->opts1 |= cpu_to_le32(DescOwn | FirstFrag);
4326 
4327 	/* rtl_tx needs to see descriptor changes before updated tp->cur_tx */
4328 	smp_wmb();
4329 
4330 	WRITE_ONCE(tp->cur_tx, tp->cur_tx + frags + 1);
4331 
4332 	stop_queue = !rtl_tx_slots_avail(tp);
4333 	if (unlikely(stop_queue)) {
4334 		/* Avoid wrongly optimistic queue wake-up: rtl_tx thread must
4335 		 * not miss a ring update when it notices a stopped queue.
4336 		 */
4337 		smp_wmb();
4338 		netif_stop_queue(dev);
4339 		/* Sync with rtl_tx:
4340 		 * - publish queue status and cur_tx ring index (write barrier)
4341 		 * - refresh dirty_tx ring index (read barrier).
4342 		 * May the current thread have a pessimistic view of the ring
4343 		 * status and forget to wake up queue, a racing rtl_tx thread
4344 		 * can't.
4345 		 */
4346 		smp_mb__after_atomic();
4347 		if (rtl_tx_slots_avail(tp))
4348 			netif_start_queue(dev);
4349 		door_bell = true;
4350 	}
4351 
4352 	if (door_bell)
4353 		rtl8169_doorbell(tp);
4354 
4355 	return NETDEV_TX_OK;
4356 
4357 err_dma_1:
4358 	rtl8169_unmap_tx_skb(tp, entry);
4359 err_dma_0:
4360 	dev_kfree_skb_any(skb);
4361 	dev->stats.tx_dropped++;
4362 	return NETDEV_TX_OK;
4363 
4364 err_stop_0:
4365 	netif_stop_queue(dev);
4366 	dev->stats.tx_dropped++;
4367 	return NETDEV_TX_BUSY;
4368 }
4369 
4370 static unsigned int rtl_last_frag_len(struct sk_buff *skb)
4371 {
4372 	struct skb_shared_info *info = skb_shinfo(skb);
4373 	unsigned int nr_frags = info->nr_frags;
4374 
4375 	if (!nr_frags)
4376 		return UINT_MAX;
4377 
4378 	return skb_frag_size(info->frags + nr_frags - 1);
4379 }
4380 
4381 /* Workaround for hw issues with TSO on RTL8168evl */
4382 static netdev_features_t rtl8168evl_fix_tso(struct sk_buff *skb,
4383 					    netdev_features_t features)
4384 {
4385 	/* IPv4 header has options field */
4386 	if (vlan_get_protocol(skb) == htons(ETH_P_IP) &&
4387 	    ip_hdrlen(skb) > sizeof(struct iphdr))
4388 		features &= ~NETIF_F_ALL_TSO;
4389 
4390 	/* IPv4 TCP header has options field */
4391 	else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4 &&
4392 		 tcp_hdrlen(skb) > sizeof(struct tcphdr))
4393 		features &= ~NETIF_F_ALL_TSO;
4394 
4395 	else if (rtl_last_frag_len(skb) <= 6)
4396 		features &= ~NETIF_F_ALL_TSO;
4397 
4398 	return features;
4399 }
4400 
4401 static netdev_features_t rtl8169_features_check(struct sk_buff *skb,
4402 						struct net_device *dev,
4403 						netdev_features_t features)
4404 {
4405 	int transport_offset = skb_transport_offset(skb);
4406 	struct rtl8169_private *tp = netdev_priv(dev);
4407 
4408 	if (skb_is_gso(skb)) {
4409 		if (tp->mac_version == RTL_GIGA_MAC_VER_34)
4410 			features = rtl8168evl_fix_tso(skb, features);
4411 
4412 		if (transport_offset > GTTCPHO_MAX &&
4413 		    rtl_chip_supports_csum_v2(tp))
4414 			features &= ~NETIF_F_ALL_TSO;
4415 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4416 		/* work around hw bug on some chip versions */
4417 		if (skb->len < ETH_ZLEN)
4418 			features &= ~NETIF_F_CSUM_MASK;
4419 
4420 		if (rtl_quirk_packet_padto(tp, skb))
4421 			features &= ~NETIF_F_CSUM_MASK;
4422 
4423 		if (transport_offset > TCPHO_MAX &&
4424 		    rtl_chip_supports_csum_v2(tp))
4425 			features &= ~NETIF_F_CSUM_MASK;
4426 	}
4427 
4428 	return vlan_features_check(skb, features);
4429 }
4430 
4431 static void rtl8169_pcierr_interrupt(struct net_device *dev)
4432 {
4433 	struct rtl8169_private *tp = netdev_priv(dev);
4434 	struct pci_dev *pdev = tp->pci_dev;
4435 	int pci_status_errs;
4436 	u16 pci_cmd;
4437 
4438 	pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
4439 
4440 	pci_status_errs = pci_status_get_and_clear_errors(pdev);
4441 
4442 	if (net_ratelimit())
4443 		netdev_err(dev, "PCI error (cmd = 0x%04x, status_errs = 0x%04x)\n",
4444 			   pci_cmd, pci_status_errs);
4445 
4446 	rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
4447 }
4448 
4449 static void rtl_tx(struct net_device *dev, struct rtl8169_private *tp,
4450 		   int budget)
4451 {
4452 	unsigned int dirty_tx, bytes_compl = 0, pkts_compl = 0;
4453 	struct sk_buff *skb;
4454 
4455 	dirty_tx = tp->dirty_tx;
4456 
4457 	while (READ_ONCE(tp->cur_tx) != dirty_tx) {
4458 		unsigned int entry = dirty_tx % NUM_TX_DESC;
4459 		u32 status;
4460 
4461 		status = le32_to_cpu(tp->TxDescArray[entry].opts1);
4462 		if (status & DescOwn)
4463 			break;
4464 
4465 		skb = tp->tx_skb[entry].skb;
4466 		rtl8169_unmap_tx_skb(tp, entry);
4467 
4468 		if (skb) {
4469 			pkts_compl++;
4470 			bytes_compl += skb->len;
4471 			napi_consume_skb(skb, budget);
4472 		}
4473 		dirty_tx++;
4474 	}
4475 
4476 	if (tp->dirty_tx != dirty_tx) {
4477 		netdev_completed_queue(dev, pkts_compl, bytes_compl);
4478 		dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl);
4479 
4480 		/* Sync with rtl8169_start_xmit:
4481 		 * - publish dirty_tx ring index (write barrier)
4482 		 * - refresh cur_tx ring index and queue status (read barrier)
4483 		 * May the current thread miss the stopped queue condition,
4484 		 * a racing xmit thread can only have a right view of the
4485 		 * ring status.
4486 		 */
4487 		smp_store_mb(tp->dirty_tx, dirty_tx);
4488 		if (netif_queue_stopped(dev) && rtl_tx_slots_avail(tp))
4489 			netif_wake_queue(dev);
4490 		/*
4491 		 * 8168 hack: TxPoll requests are lost when the Tx packets are
4492 		 * too close. Let's kick an extra TxPoll request when a burst
4493 		 * of start_xmit activity is detected (if it is not detected,
4494 		 * it is slow enough). -- FR
4495 		 * If skb is NULL then we come here again once a tx irq is
4496 		 * triggered after the last fragment is marked transmitted.
4497 		 */
4498 		if (tp->cur_tx != dirty_tx && skb)
4499 			rtl8169_doorbell(tp);
4500 	}
4501 }
4502 
4503 static inline int rtl8169_fragmented_frame(u32 status)
4504 {
4505 	return (status & (FirstFrag | LastFrag)) != (FirstFrag | LastFrag);
4506 }
4507 
4508 static inline void rtl8169_rx_csum(struct sk_buff *skb, u32 opts1)
4509 {
4510 	u32 status = opts1 & (RxProtoMask | RxCSFailMask);
4511 
4512 	if (status == RxProtoTCP || status == RxProtoUDP)
4513 		skb->ip_summed = CHECKSUM_UNNECESSARY;
4514 	else
4515 		skb_checksum_none_assert(skb);
4516 }
4517 
4518 static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget)
4519 {
4520 	struct device *d = tp_to_dev(tp);
4521 	int count;
4522 
4523 	for (count = 0; count < budget; count++, tp->cur_rx++) {
4524 		unsigned int pkt_size, entry = tp->cur_rx % NUM_RX_DESC;
4525 		struct RxDesc *desc = tp->RxDescArray + entry;
4526 		struct sk_buff *skb;
4527 		const void *rx_buf;
4528 		dma_addr_t addr;
4529 		u32 status;
4530 
4531 		status = le32_to_cpu(desc->opts1);
4532 		if (status & DescOwn)
4533 			break;
4534 
4535 		/* This barrier is needed to keep us from reading
4536 		 * any other fields out of the Rx descriptor until
4537 		 * we know the status of DescOwn
4538 		 */
4539 		dma_rmb();
4540 
4541 		if (unlikely(status & RxRES)) {
4542 			if (net_ratelimit())
4543 				netdev_warn(dev, "Rx ERROR. status = %08x\n",
4544 					    status);
4545 			dev->stats.rx_errors++;
4546 			if (status & (RxRWT | RxRUNT))
4547 				dev->stats.rx_length_errors++;
4548 			if (status & RxCRC)
4549 				dev->stats.rx_crc_errors++;
4550 
4551 			if (!(dev->features & NETIF_F_RXALL))
4552 				goto release_descriptor;
4553 			else if (status & RxRWT || !(status & (RxRUNT | RxCRC)))
4554 				goto release_descriptor;
4555 		}
4556 
4557 		pkt_size = status & GENMASK(13, 0);
4558 		if (likely(!(dev->features & NETIF_F_RXFCS)))
4559 			pkt_size -= ETH_FCS_LEN;
4560 
4561 		/* The driver does not support incoming fragmented frames.
4562 		 * They are seen as a symptom of over-mtu sized frames.
4563 		 */
4564 		if (unlikely(rtl8169_fragmented_frame(status))) {
4565 			dev->stats.rx_dropped++;
4566 			dev->stats.rx_length_errors++;
4567 			goto release_descriptor;
4568 		}
4569 
4570 		skb = napi_alloc_skb(&tp->napi, pkt_size);
4571 		if (unlikely(!skb)) {
4572 			dev->stats.rx_dropped++;
4573 			goto release_descriptor;
4574 		}
4575 
4576 		addr = le64_to_cpu(desc->addr);
4577 		rx_buf = page_address(tp->Rx_databuff[entry]);
4578 
4579 		dma_sync_single_for_cpu(d, addr, pkt_size, DMA_FROM_DEVICE);
4580 		prefetch(rx_buf);
4581 		skb_copy_to_linear_data(skb, rx_buf, pkt_size);
4582 		skb->tail += pkt_size;
4583 		skb->len = pkt_size;
4584 		dma_sync_single_for_device(d, addr, pkt_size, DMA_FROM_DEVICE);
4585 
4586 		rtl8169_rx_csum(skb, status);
4587 		skb->protocol = eth_type_trans(skb, dev);
4588 
4589 		rtl8169_rx_vlan_tag(desc, skb);
4590 
4591 		if (skb->pkt_type == PACKET_MULTICAST)
4592 			dev->stats.multicast++;
4593 
4594 		napi_gro_receive(&tp->napi, skb);
4595 
4596 		dev_sw_netstats_rx_add(dev, pkt_size);
4597 release_descriptor:
4598 		rtl8169_mark_to_asic(desc);
4599 	}
4600 
4601 	return count;
4602 }
4603 
4604 static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance)
4605 {
4606 	struct rtl8169_private *tp = dev_instance;
4607 	u32 status = rtl_get_events(tp);
4608 
4609 	if ((status & 0xffff) == 0xffff || !(status & tp->irq_mask))
4610 		return IRQ_NONE;
4611 
4612 	if (unlikely(status & SYSErr)) {
4613 		rtl8169_pcierr_interrupt(tp->dev);
4614 		goto out;
4615 	}
4616 
4617 	if (status & LinkChg)
4618 		phy_mac_interrupt(tp->phydev);
4619 
4620 	if (unlikely(status & RxFIFOOver &&
4621 	    tp->mac_version == RTL_GIGA_MAC_VER_11)) {
4622 		netif_stop_queue(tp->dev);
4623 		rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
4624 	}
4625 
4626 	if (napi_schedule_prep(&tp->napi)) {
4627 		rtl_irq_disable(tp);
4628 		__napi_schedule(&tp->napi);
4629 	}
4630 out:
4631 	rtl_ack_events(tp, status);
4632 
4633 	return IRQ_HANDLED;
4634 }
4635 
4636 static void rtl_task(struct work_struct *work)
4637 {
4638 	struct rtl8169_private *tp =
4639 		container_of(work, struct rtl8169_private, wk.work);
4640 
4641 	rtnl_lock();
4642 
4643 	if (!netif_running(tp->dev) ||
4644 	    !test_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags))
4645 		goto out_unlock;
4646 
4647 	if (test_and_clear_bit(RTL_FLAG_TASK_RESET_PENDING, tp->wk.flags)) {
4648 		rtl_reset_work(tp);
4649 		netif_wake_queue(tp->dev);
4650 	}
4651 out_unlock:
4652 	rtnl_unlock();
4653 }
4654 
4655 static int rtl8169_poll(struct napi_struct *napi, int budget)
4656 {
4657 	struct rtl8169_private *tp = container_of(napi, struct rtl8169_private, napi);
4658 	struct net_device *dev = tp->dev;
4659 	int work_done;
4660 
4661 	rtl_tx(dev, tp, budget);
4662 
4663 	work_done = rtl_rx(dev, tp, budget);
4664 
4665 	if (work_done < budget && napi_complete_done(napi, work_done))
4666 		rtl_irq_enable(tp);
4667 
4668 	return work_done;
4669 }
4670 
4671 static void r8169_phylink_handler(struct net_device *ndev)
4672 {
4673 	struct rtl8169_private *tp = netdev_priv(ndev);
4674 
4675 	if (netif_carrier_ok(ndev)) {
4676 		rtl_link_chg_patch(tp);
4677 		pm_request_resume(&tp->pci_dev->dev);
4678 	} else {
4679 		pm_runtime_idle(&tp->pci_dev->dev);
4680 	}
4681 
4682 	if (net_ratelimit())
4683 		phy_print_status(tp->phydev);
4684 }
4685 
4686 static int r8169_phy_connect(struct rtl8169_private *tp)
4687 {
4688 	struct phy_device *phydev = tp->phydev;
4689 	phy_interface_t phy_mode;
4690 	int ret;
4691 
4692 	phy_mode = tp->supports_gmii ? PHY_INTERFACE_MODE_GMII :
4693 		   PHY_INTERFACE_MODE_MII;
4694 
4695 	ret = phy_connect_direct(tp->dev, phydev, r8169_phylink_handler,
4696 				 phy_mode);
4697 	if (ret)
4698 		return ret;
4699 
4700 	if (!tp->supports_gmii)
4701 		phy_set_max_speed(phydev, SPEED_100);
4702 
4703 	phy_attached_info(phydev);
4704 
4705 	return 0;
4706 }
4707 
4708 static void rtl8169_down(struct rtl8169_private *tp)
4709 {
4710 	/* Clear all task flags */
4711 	bitmap_zero(tp->wk.flags, RTL_FLAG_MAX);
4712 
4713 	phy_stop(tp->phydev);
4714 
4715 	rtl8169_update_counters(tp);
4716 
4717 	pci_clear_master(tp->pci_dev);
4718 	rtl_pci_commit(tp);
4719 
4720 	rtl8169_cleanup(tp, true);
4721 	rtl_disable_exit_l1(tp);
4722 	rtl_prepare_power_down(tp);
4723 }
4724 
4725 static void rtl8169_up(struct rtl8169_private *tp)
4726 {
4727 	pci_set_master(tp->pci_dev);
4728 	phy_init_hw(tp->phydev);
4729 	phy_resume(tp->phydev);
4730 	rtl8169_init_phy(tp);
4731 	napi_enable(&tp->napi);
4732 	set_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags);
4733 	rtl_reset_work(tp);
4734 
4735 	phy_start(tp->phydev);
4736 }
4737 
4738 static int rtl8169_close(struct net_device *dev)
4739 {
4740 	struct rtl8169_private *tp = netdev_priv(dev);
4741 	struct pci_dev *pdev = tp->pci_dev;
4742 
4743 	pm_runtime_get_sync(&pdev->dev);
4744 
4745 	netif_stop_queue(dev);
4746 	rtl8169_down(tp);
4747 	rtl8169_rx_clear(tp);
4748 
4749 	cancel_work_sync(&tp->wk.work);
4750 
4751 	free_irq(tp->irq, tp);
4752 
4753 	phy_disconnect(tp->phydev);
4754 
4755 	dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray,
4756 			  tp->RxPhyAddr);
4757 	dma_free_coherent(&pdev->dev, R8169_TX_RING_BYTES, tp->TxDescArray,
4758 			  tp->TxPhyAddr);
4759 	tp->TxDescArray = NULL;
4760 	tp->RxDescArray = NULL;
4761 
4762 	pm_runtime_put_sync(&pdev->dev);
4763 
4764 	return 0;
4765 }
4766 
4767 #ifdef CONFIG_NET_POLL_CONTROLLER
4768 static void rtl8169_netpoll(struct net_device *dev)
4769 {
4770 	struct rtl8169_private *tp = netdev_priv(dev);
4771 
4772 	rtl8169_interrupt(tp->irq, tp);
4773 }
4774 #endif
4775 
4776 static int rtl_open(struct net_device *dev)
4777 {
4778 	struct rtl8169_private *tp = netdev_priv(dev);
4779 	struct pci_dev *pdev = tp->pci_dev;
4780 	unsigned long irqflags;
4781 	int retval = -ENOMEM;
4782 
4783 	pm_runtime_get_sync(&pdev->dev);
4784 
4785 	/*
4786 	 * Rx and Tx descriptors needs 256 bytes alignment.
4787 	 * dma_alloc_coherent provides more.
4788 	 */
4789 	tp->TxDescArray = dma_alloc_coherent(&pdev->dev, R8169_TX_RING_BYTES,
4790 					     &tp->TxPhyAddr, GFP_KERNEL);
4791 	if (!tp->TxDescArray)
4792 		goto out;
4793 
4794 	tp->RxDescArray = dma_alloc_coherent(&pdev->dev, R8169_RX_RING_BYTES,
4795 					     &tp->RxPhyAddr, GFP_KERNEL);
4796 	if (!tp->RxDescArray)
4797 		goto err_free_tx_0;
4798 
4799 	retval = rtl8169_init_ring(tp);
4800 	if (retval < 0)
4801 		goto err_free_rx_1;
4802 
4803 	rtl_request_firmware(tp);
4804 
4805 	irqflags = pci_dev_msi_enabled(pdev) ? IRQF_NO_THREAD : IRQF_SHARED;
4806 	retval = request_irq(tp->irq, rtl8169_interrupt, irqflags, dev->name, tp);
4807 	if (retval < 0)
4808 		goto err_release_fw_2;
4809 
4810 	retval = r8169_phy_connect(tp);
4811 	if (retval)
4812 		goto err_free_irq;
4813 
4814 	rtl8169_up(tp);
4815 	rtl8169_init_counter_offsets(tp);
4816 	netif_start_queue(dev);
4817 out:
4818 	pm_runtime_put_sync(&pdev->dev);
4819 
4820 	return retval;
4821 
4822 err_free_irq:
4823 	free_irq(tp->irq, tp);
4824 err_release_fw_2:
4825 	rtl_release_firmware(tp);
4826 	rtl8169_rx_clear(tp);
4827 err_free_rx_1:
4828 	dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray,
4829 			  tp->RxPhyAddr);
4830 	tp->RxDescArray = NULL;
4831 err_free_tx_0:
4832 	dma_free_coherent(&pdev->dev, R8169_TX_RING_BYTES, tp->TxDescArray,
4833 			  tp->TxPhyAddr);
4834 	tp->TxDescArray = NULL;
4835 	goto out;
4836 }
4837 
4838 static void
4839 rtl8169_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
4840 {
4841 	struct rtl8169_private *tp = netdev_priv(dev);
4842 	struct pci_dev *pdev = tp->pci_dev;
4843 	struct rtl8169_counters *counters = tp->counters;
4844 
4845 	pm_runtime_get_noresume(&pdev->dev);
4846 
4847 	netdev_stats_to_stats64(stats, &dev->stats);
4848 	dev_fetch_sw_netstats(stats, dev->tstats);
4849 
4850 	/*
4851 	 * Fetch additional counter values missing in stats collected by driver
4852 	 * from tally counters.
4853 	 */
4854 	if (pm_runtime_active(&pdev->dev))
4855 		rtl8169_update_counters(tp);
4856 
4857 	/*
4858 	 * Subtract values fetched during initalization.
4859 	 * See rtl8169_init_counter_offsets for a description why we do that.
4860 	 */
4861 	stats->tx_errors = le64_to_cpu(counters->tx_errors) -
4862 		le64_to_cpu(tp->tc_offset.tx_errors);
4863 	stats->collisions = le32_to_cpu(counters->tx_multi_collision) -
4864 		le32_to_cpu(tp->tc_offset.tx_multi_collision);
4865 	stats->tx_aborted_errors = le16_to_cpu(counters->tx_aborted) -
4866 		le16_to_cpu(tp->tc_offset.tx_aborted);
4867 	stats->rx_missed_errors = le16_to_cpu(counters->rx_missed) -
4868 		le16_to_cpu(tp->tc_offset.rx_missed);
4869 
4870 	pm_runtime_put_noidle(&pdev->dev);
4871 }
4872 
4873 static void rtl8169_net_suspend(struct rtl8169_private *tp)
4874 {
4875 	netif_device_detach(tp->dev);
4876 
4877 	if (netif_running(tp->dev))
4878 		rtl8169_down(tp);
4879 }
4880 
4881 static int rtl8169_runtime_resume(struct device *dev)
4882 {
4883 	struct rtl8169_private *tp = dev_get_drvdata(dev);
4884 
4885 	rtl_rar_set(tp, tp->dev->dev_addr);
4886 	__rtl8169_set_wol(tp, tp->saved_wolopts);
4887 
4888 	if (tp->TxDescArray)
4889 		rtl8169_up(tp);
4890 
4891 	netif_device_attach(tp->dev);
4892 
4893 	return 0;
4894 }
4895 
4896 static int rtl8169_suspend(struct device *device)
4897 {
4898 	struct rtl8169_private *tp = dev_get_drvdata(device);
4899 
4900 	rtnl_lock();
4901 	rtl8169_net_suspend(tp);
4902 	if (!device_may_wakeup(tp_to_dev(tp)))
4903 		clk_disable_unprepare(tp->clk);
4904 	rtnl_unlock();
4905 
4906 	return 0;
4907 }
4908 
4909 static int rtl8169_resume(struct device *device)
4910 {
4911 	struct rtl8169_private *tp = dev_get_drvdata(device);
4912 
4913 	if (!device_may_wakeup(tp_to_dev(tp)))
4914 		clk_prepare_enable(tp->clk);
4915 
4916 	/* Reportedly at least Asus X453MA truncates packets otherwise */
4917 	if (tp->mac_version == RTL_GIGA_MAC_VER_37)
4918 		rtl_init_rxcfg(tp);
4919 
4920 	return rtl8169_runtime_resume(device);
4921 }
4922 
4923 static int rtl8169_runtime_suspend(struct device *device)
4924 {
4925 	struct rtl8169_private *tp = dev_get_drvdata(device);
4926 
4927 	if (!tp->TxDescArray) {
4928 		netif_device_detach(tp->dev);
4929 		return 0;
4930 	}
4931 
4932 	rtnl_lock();
4933 	__rtl8169_set_wol(tp, WAKE_PHY);
4934 	rtl8169_net_suspend(tp);
4935 	rtnl_unlock();
4936 
4937 	return 0;
4938 }
4939 
4940 static int rtl8169_runtime_idle(struct device *device)
4941 {
4942 	struct rtl8169_private *tp = dev_get_drvdata(device);
4943 
4944 	if (tp->dash_type != RTL_DASH_NONE)
4945 		return -EBUSY;
4946 
4947 	if (!netif_running(tp->dev) || !netif_carrier_ok(tp->dev))
4948 		pm_schedule_suspend(device, 10000);
4949 
4950 	return -EBUSY;
4951 }
4952 
4953 static const struct dev_pm_ops rtl8169_pm_ops = {
4954 	SYSTEM_SLEEP_PM_OPS(rtl8169_suspend, rtl8169_resume)
4955 	RUNTIME_PM_OPS(rtl8169_runtime_suspend, rtl8169_runtime_resume,
4956 		       rtl8169_runtime_idle)
4957 };
4958 
4959 static void rtl_wol_shutdown_quirk(struct rtl8169_private *tp)
4960 {
4961 	/* WoL fails with 8168b when the receiver is disabled. */
4962 	switch (tp->mac_version) {
4963 	case RTL_GIGA_MAC_VER_11:
4964 	case RTL_GIGA_MAC_VER_12:
4965 	case RTL_GIGA_MAC_VER_17:
4966 		pci_clear_master(tp->pci_dev);
4967 
4968 		RTL_W8(tp, ChipCmd, CmdRxEnb);
4969 		rtl_pci_commit(tp);
4970 		break;
4971 	default:
4972 		break;
4973 	}
4974 }
4975 
4976 static void rtl_shutdown(struct pci_dev *pdev)
4977 {
4978 	struct rtl8169_private *tp = pci_get_drvdata(pdev);
4979 
4980 	rtnl_lock();
4981 	rtl8169_net_suspend(tp);
4982 	rtnl_unlock();
4983 
4984 	/* Restore original MAC address */
4985 	rtl_rar_set(tp, tp->dev->perm_addr);
4986 
4987 	if (system_state == SYSTEM_POWER_OFF &&
4988 	    tp->dash_type == RTL_DASH_NONE) {
4989 		if (tp->saved_wolopts)
4990 			rtl_wol_shutdown_quirk(tp);
4991 
4992 		pci_wake_from_d3(pdev, tp->saved_wolopts);
4993 		pci_set_power_state(pdev, PCI_D3hot);
4994 	}
4995 }
4996 
4997 static void rtl_remove_one(struct pci_dev *pdev)
4998 {
4999 	struct rtl8169_private *tp = pci_get_drvdata(pdev);
5000 
5001 	if (pci_dev_run_wake(pdev))
5002 		pm_runtime_get_noresume(&pdev->dev);
5003 
5004 	unregister_netdev(tp->dev);
5005 
5006 	if (tp->dash_type != RTL_DASH_NONE)
5007 		rtl8168_driver_stop(tp);
5008 
5009 	rtl_release_firmware(tp);
5010 
5011 	/* restore original MAC address */
5012 	rtl_rar_set(tp, tp->dev->perm_addr);
5013 }
5014 
5015 static const struct net_device_ops rtl_netdev_ops = {
5016 	.ndo_open		= rtl_open,
5017 	.ndo_stop		= rtl8169_close,
5018 	.ndo_get_stats64	= rtl8169_get_stats64,
5019 	.ndo_start_xmit		= rtl8169_start_xmit,
5020 	.ndo_features_check	= rtl8169_features_check,
5021 	.ndo_tx_timeout		= rtl8169_tx_timeout,
5022 	.ndo_validate_addr	= eth_validate_addr,
5023 	.ndo_change_mtu		= rtl8169_change_mtu,
5024 	.ndo_fix_features	= rtl8169_fix_features,
5025 	.ndo_set_features	= rtl8169_set_features,
5026 	.ndo_set_mac_address	= rtl_set_mac_address,
5027 	.ndo_eth_ioctl		= phy_do_ioctl_running,
5028 	.ndo_set_rx_mode	= rtl_set_rx_mode,
5029 #ifdef CONFIG_NET_POLL_CONTROLLER
5030 	.ndo_poll_controller	= rtl8169_netpoll,
5031 #endif
5032 
5033 };
5034 
5035 static void rtl_set_irq_mask(struct rtl8169_private *tp)
5036 {
5037 	tp->irq_mask = RxOK | RxErr | TxOK | TxErr | LinkChg;
5038 
5039 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
5040 		tp->irq_mask |= SYSErr | RxOverflow | RxFIFOOver;
5041 	else if (tp->mac_version == RTL_GIGA_MAC_VER_11)
5042 		/* special workaround needed */
5043 		tp->irq_mask |= RxFIFOOver;
5044 	else
5045 		tp->irq_mask |= RxOverflow;
5046 }
5047 
5048 static int rtl_alloc_irq(struct rtl8169_private *tp)
5049 {
5050 	unsigned int flags;
5051 
5052 	switch (tp->mac_version) {
5053 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
5054 		rtl_unlock_config_regs(tp);
5055 		RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~MSIEnable);
5056 		rtl_lock_config_regs(tp);
5057 		fallthrough;
5058 	case RTL_GIGA_MAC_VER_07 ... RTL_GIGA_MAC_VER_17:
5059 		flags = PCI_IRQ_LEGACY;
5060 		break;
5061 	default:
5062 		flags = PCI_IRQ_ALL_TYPES;
5063 		break;
5064 	}
5065 
5066 	return pci_alloc_irq_vectors(tp->pci_dev, 1, 1, flags);
5067 }
5068 
5069 static void rtl_read_mac_address(struct rtl8169_private *tp,
5070 				 u8 mac_addr[ETH_ALEN])
5071 {
5072 	/* Get MAC address */
5073 	if (rtl_is_8168evl_up(tp) && tp->mac_version != RTL_GIGA_MAC_VER_34) {
5074 		u32 value;
5075 
5076 		value = rtl_eri_read(tp, 0xe0);
5077 		put_unaligned_le32(value, mac_addr);
5078 		value = rtl_eri_read(tp, 0xe4);
5079 		put_unaligned_le16(value, mac_addr + 4);
5080 	} else if (rtl_is_8125(tp)) {
5081 		rtl_read_mac_from_reg(tp, mac_addr, MAC0_BKP);
5082 	}
5083 }
5084 
5085 DECLARE_RTL_COND(rtl_link_list_ready_cond)
5086 {
5087 	return RTL_R8(tp, MCU) & LINK_LIST_RDY;
5088 }
5089 
5090 static void r8168g_wait_ll_share_fifo_ready(struct rtl8169_private *tp)
5091 {
5092 	rtl_loop_wait_high(tp, &rtl_link_list_ready_cond, 100, 42);
5093 }
5094 
5095 static int r8169_mdio_read_reg(struct mii_bus *mii_bus, int phyaddr, int phyreg)
5096 {
5097 	struct rtl8169_private *tp = mii_bus->priv;
5098 
5099 	if (phyaddr > 0)
5100 		return -ENODEV;
5101 
5102 	return rtl_readphy(tp, phyreg);
5103 }
5104 
5105 static int r8169_mdio_write_reg(struct mii_bus *mii_bus, int phyaddr,
5106 				int phyreg, u16 val)
5107 {
5108 	struct rtl8169_private *tp = mii_bus->priv;
5109 
5110 	if (phyaddr > 0)
5111 		return -ENODEV;
5112 
5113 	rtl_writephy(tp, phyreg, val);
5114 
5115 	return 0;
5116 }
5117 
5118 static int r8169_mdio_register(struct rtl8169_private *tp)
5119 {
5120 	struct pci_dev *pdev = tp->pci_dev;
5121 	struct mii_bus *new_bus;
5122 	int ret;
5123 
5124 	new_bus = devm_mdiobus_alloc(&pdev->dev);
5125 	if (!new_bus)
5126 		return -ENOMEM;
5127 
5128 	new_bus->name = "r8169";
5129 	new_bus->priv = tp;
5130 	new_bus->parent = &pdev->dev;
5131 	new_bus->irq[0] = PHY_MAC_INTERRUPT;
5132 	snprintf(new_bus->id, MII_BUS_ID_SIZE, "r8169-%x-%x",
5133 		 pci_domain_nr(pdev->bus), pci_dev_id(pdev));
5134 
5135 	new_bus->read = r8169_mdio_read_reg;
5136 	new_bus->write = r8169_mdio_write_reg;
5137 
5138 	ret = devm_mdiobus_register(&pdev->dev, new_bus);
5139 	if (ret)
5140 		return ret;
5141 
5142 	tp->phydev = mdiobus_get_phy(new_bus, 0);
5143 	if (!tp->phydev) {
5144 		return -ENODEV;
5145 	} else if (!tp->phydev->drv) {
5146 		/* Most chip versions fail with the genphy driver.
5147 		 * Therefore ensure that the dedicated PHY driver is loaded.
5148 		 */
5149 		dev_err(&pdev->dev, "no dedicated PHY driver found for PHY ID 0x%08x, maybe realtek.ko needs to be added to initramfs?\n",
5150 			tp->phydev->phy_id);
5151 		return -EUNATCH;
5152 	}
5153 
5154 	tp->phydev->mac_managed_pm = 1;
5155 
5156 	phy_support_asym_pause(tp->phydev);
5157 
5158 	/* PHY will be woken up in rtl_open() */
5159 	phy_suspend(tp->phydev);
5160 
5161 	return 0;
5162 }
5163 
5164 static void rtl_hw_init_8168g(struct rtl8169_private *tp)
5165 {
5166 	rtl_enable_rxdvgate(tp);
5167 
5168 	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & ~(CmdTxEnb | CmdRxEnb));
5169 	msleep(1);
5170 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
5171 
5172 	r8168_mac_ocp_modify(tp, 0xe8de, BIT(14), 0);
5173 	r8168g_wait_ll_share_fifo_ready(tp);
5174 
5175 	r8168_mac_ocp_modify(tp, 0xe8de, 0, BIT(15));
5176 	r8168g_wait_ll_share_fifo_ready(tp);
5177 }
5178 
5179 static void rtl_hw_init_8125(struct rtl8169_private *tp)
5180 {
5181 	rtl_enable_rxdvgate(tp);
5182 
5183 	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & ~(CmdTxEnb | CmdRxEnb));
5184 	msleep(1);
5185 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
5186 
5187 	r8168_mac_ocp_modify(tp, 0xe8de, BIT(14), 0);
5188 	r8168g_wait_ll_share_fifo_ready(tp);
5189 
5190 	r8168_mac_ocp_write(tp, 0xc0aa, 0x07d0);
5191 	r8168_mac_ocp_write(tp, 0xc0a6, 0x0150);
5192 	r8168_mac_ocp_write(tp, 0xc01e, 0x5555);
5193 	r8168g_wait_ll_share_fifo_ready(tp);
5194 }
5195 
5196 static void rtl_hw_initialize(struct rtl8169_private *tp)
5197 {
5198 	switch (tp->mac_version) {
5199 	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_53:
5200 		rtl8168ep_stop_cmac(tp);
5201 		fallthrough;
5202 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_48:
5203 		rtl_hw_init_8168g(tp);
5204 		break;
5205 	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
5206 		rtl_hw_init_8125(tp);
5207 		break;
5208 	default:
5209 		break;
5210 	}
5211 }
5212 
5213 static int rtl_jumbo_max(struct rtl8169_private *tp)
5214 {
5215 	/* Non-GBit versions don't support jumbo frames */
5216 	if (!tp->supports_gmii)
5217 		return 0;
5218 
5219 	switch (tp->mac_version) {
5220 	/* RTL8169 */
5221 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
5222 		return JUMBO_7K;
5223 	/* RTL8168b */
5224 	case RTL_GIGA_MAC_VER_11:
5225 	case RTL_GIGA_MAC_VER_12:
5226 	case RTL_GIGA_MAC_VER_17:
5227 		return JUMBO_4K;
5228 	/* RTL8168c */
5229 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
5230 		return JUMBO_6K;
5231 	default:
5232 		return JUMBO_9K;
5233 	}
5234 }
5235 
5236 static void rtl_disable_clk(void *data)
5237 {
5238 	clk_disable_unprepare(data);
5239 }
5240 
5241 static int rtl_get_ether_clk(struct rtl8169_private *tp)
5242 {
5243 	struct device *d = tp_to_dev(tp);
5244 	struct clk *clk;
5245 	int rc;
5246 
5247 	clk = devm_clk_get(d, "ether_clk");
5248 	if (IS_ERR(clk)) {
5249 		rc = PTR_ERR(clk);
5250 		if (rc == -ENOENT)
5251 			/* clk-core allows NULL (for suspend / resume) */
5252 			rc = 0;
5253 		else
5254 			dev_err_probe(d, rc, "failed to get clk\n");
5255 	} else {
5256 		tp->clk = clk;
5257 		rc = clk_prepare_enable(clk);
5258 		if (rc)
5259 			dev_err(d, "failed to enable clk: %d\n", rc);
5260 		else
5261 			rc = devm_add_action_or_reset(d, rtl_disable_clk, clk);
5262 	}
5263 
5264 	return rc;
5265 }
5266 
5267 static void rtl_init_mac_address(struct rtl8169_private *tp)
5268 {
5269 	u8 mac_addr[ETH_ALEN] __aligned(2) = {};
5270 	struct net_device *dev = tp->dev;
5271 	int rc;
5272 
5273 	rc = eth_platform_get_mac_address(tp_to_dev(tp), mac_addr);
5274 	if (!rc)
5275 		goto done;
5276 
5277 	rtl_read_mac_address(tp, mac_addr);
5278 	if (is_valid_ether_addr(mac_addr))
5279 		goto done;
5280 
5281 	rtl_read_mac_from_reg(tp, mac_addr, MAC0);
5282 	if (is_valid_ether_addr(mac_addr))
5283 		goto done;
5284 
5285 	eth_random_addr(mac_addr);
5286 	dev->addr_assign_type = NET_ADDR_RANDOM;
5287 	dev_warn(tp_to_dev(tp), "can't read MAC address, setting random one\n");
5288 done:
5289 	eth_hw_addr_set(dev, mac_addr);
5290 	rtl_rar_set(tp, mac_addr);
5291 }
5292 
5293 /* register is set if system vendor successfully tested ASPM 1.2 */
5294 static bool rtl_aspm_is_safe(struct rtl8169_private *tp)
5295 {
5296 	if (tp->mac_version >= RTL_GIGA_MAC_VER_60 &&
5297 	    r8168_mac_ocp_read(tp, 0xc0b2) & 0xf)
5298 		return true;
5299 
5300 	return false;
5301 }
5302 
5303 static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5304 {
5305 	struct rtl8169_private *tp;
5306 	int jumbo_max, region, rc;
5307 	enum mac_version chipset;
5308 	struct net_device *dev;
5309 	u16 xid;
5310 
5311 	dev = devm_alloc_etherdev(&pdev->dev, sizeof (*tp));
5312 	if (!dev)
5313 		return -ENOMEM;
5314 
5315 	SET_NETDEV_DEV(dev, &pdev->dev);
5316 	dev->netdev_ops = &rtl_netdev_ops;
5317 	tp = netdev_priv(dev);
5318 	tp->dev = dev;
5319 	tp->pci_dev = pdev;
5320 	tp->supports_gmii = ent->driver_data == RTL_CFG_NO_GBIT ? 0 : 1;
5321 	tp->eee_adv = -1;
5322 	tp->ocp_base = OCP_STD_PHY_BASE;
5323 
5324 	dev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev,
5325 						   struct pcpu_sw_netstats);
5326 	if (!dev->tstats)
5327 		return -ENOMEM;
5328 
5329 	/* Get the *optional* external "ether_clk" used on some boards */
5330 	rc = rtl_get_ether_clk(tp);
5331 	if (rc)
5332 		return rc;
5333 
5334 	/* enable device (incl. PCI PM wakeup and hotplug setup) */
5335 	rc = pcim_enable_device(pdev);
5336 	if (rc < 0) {
5337 		dev_err(&pdev->dev, "enable failure\n");
5338 		return rc;
5339 	}
5340 
5341 	if (pcim_set_mwi(pdev) < 0)
5342 		dev_info(&pdev->dev, "Mem-Wr-Inval unavailable\n");
5343 
5344 	/* use first MMIO region */
5345 	region = ffs(pci_select_bars(pdev, IORESOURCE_MEM)) - 1;
5346 	if (region < 0) {
5347 		dev_err(&pdev->dev, "no MMIO resource found\n");
5348 		return -ENODEV;
5349 	}
5350 
5351 	/* check for weird/broken PCI region reporting */
5352 	if (pci_resource_len(pdev, region) < R8169_REGS_SIZE) {
5353 		dev_err(&pdev->dev, "Invalid PCI region size(s), aborting\n");
5354 		return -ENODEV;
5355 	}
5356 
5357 	rc = pcim_iomap_regions(pdev, BIT(region), KBUILD_MODNAME);
5358 	if (rc < 0) {
5359 		dev_err(&pdev->dev, "cannot remap MMIO, aborting\n");
5360 		return rc;
5361 	}
5362 
5363 	tp->mmio_addr = pcim_iomap_table(pdev)[region];
5364 
5365 	xid = (RTL_R32(tp, TxConfig) >> 20) & 0xfcf;
5366 
5367 	/* Identify chip attached to board */
5368 	chipset = rtl8169_get_mac_version(xid, tp->supports_gmii);
5369 	if (chipset == RTL_GIGA_MAC_NONE) {
5370 		dev_err(&pdev->dev, "unknown chip XID %03x, contact r8169 maintainers (see MAINTAINERS file)\n", xid);
5371 		return -ENODEV;
5372 	}
5373 
5374 	tp->mac_version = chipset;
5375 
5376 	/* Disable ASPM L1 as that cause random device stop working
5377 	 * problems as well as full system hangs for some PCIe devices users.
5378 	 * Chips from RTL8168h partially have issues with L1.2, but seem
5379 	 * to work fine with L1 and L1.1.
5380 	 */
5381 	if (rtl_aspm_is_safe(tp))
5382 		rc = 0;
5383 	else if (tp->mac_version >= RTL_GIGA_MAC_VER_45)
5384 		rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2);
5385 	else
5386 		rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1);
5387 	tp->aspm_manageable = !rc;
5388 
5389 	tp->dash_type = rtl_check_dash(tp);
5390 
5391 	tp->cp_cmd = RTL_R16(tp, CPlusCmd) & CPCMD_MASK;
5392 
5393 	if (sizeof(dma_addr_t) > 4 && tp->mac_version >= RTL_GIGA_MAC_VER_18 &&
5394 	    !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)))
5395 		dev->features |= NETIF_F_HIGHDMA;
5396 
5397 	rtl_init_rxcfg(tp);
5398 
5399 	rtl8169_irq_mask_and_ack(tp);
5400 
5401 	rtl_hw_initialize(tp);
5402 
5403 	rtl_hw_reset(tp);
5404 
5405 	rc = rtl_alloc_irq(tp);
5406 	if (rc < 0) {
5407 		dev_err(&pdev->dev, "Can't allocate interrupt\n");
5408 		return rc;
5409 	}
5410 	tp->irq = pci_irq_vector(pdev, 0);
5411 
5412 	INIT_WORK(&tp->wk.work, rtl_task);
5413 
5414 	rtl_init_mac_address(tp);
5415 
5416 	dev->ethtool_ops = &rtl8169_ethtool_ops;
5417 
5418 	netif_napi_add(dev, &tp->napi, rtl8169_poll, NAPI_POLL_WEIGHT);
5419 
5420 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
5421 			   NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
5422 	dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
5423 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
5424 
5425 	/*
5426 	 * Pretend we are using VLANs; This bypasses a nasty bug where
5427 	 * Interrupts stop flowing on high load on 8110SCd controllers.
5428 	 */
5429 	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
5430 		/* Disallow toggling */
5431 		dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
5432 
5433 	if (rtl_chip_supports_csum_v2(tp))
5434 		dev->hw_features |= NETIF_F_IPV6_CSUM;
5435 
5436 	dev->features |= dev->hw_features;
5437 
5438 	/* There has been a number of reports that using SG/TSO results in
5439 	 * tx timeouts. However for a lot of people SG/TSO works fine.
5440 	 * Therefore disable both features by default, but allow users to
5441 	 * enable them. Use at own risk!
5442 	 */
5443 	if (rtl_chip_supports_csum_v2(tp)) {
5444 		dev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6;
5445 		netif_set_gso_max_size(dev, RTL_GSO_MAX_SIZE_V2);
5446 		netif_set_gso_max_segs(dev, RTL_GSO_MAX_SEGS_V2);
5447 	} else {
5448 		dev->hw_features |= NETIF_F_SG | NETIF_F_TSO;
5449 		netif_set_gso_max_size(dev, RTL_GSO_MAX_SIZE_V1);
5450 		netif_set_gso_max_segs(dev, RTL_GSO_MAX_SEGS_V1);
5451 	}
5452 
5453 	dev->hw_features |= NETIF_F_RXALL;
5454 	dev->hw_features |= NETIF_F_RXFCS;
5455 
5456 	/* configure chip for default features */
5457 	rtl8169_set_features(dev, dev->features);
5458 
5459 	if (tp->dash_type == RTL_DASH_NONE) {
5460 		rtl_set_d3_pll_down(tp, true);
5461 	} else {
5462 		rtl_set_d3_pll_down(tp, false);
5463 		dev->wol_enabled = 1;
5464 	}
5465 
5466 	jumbo_max = rtl_jumbo_max(tp);
5467 	if (jumbo_max)
5468 		dev->max_mtu = jumbo_max;
5469 
5470 	rtl_set_irq_mask(tp);
5471 
5472 	tp->fw_name = rtl_chip_infos[chipset].fw_name;
5473 
5474 	tp->counters = dmam_alloc_coherent (&pdev->dev, sizeof(*tp->counters),
5475 					    &tp->counters_phys_addr,
5476 					    GFP_KERNEL);
5477 	if (!tp->counters)
5478 		return -ENOMEM;
5479 
5480 	pci_set_drvdata(pdev, tp);
5481 
5482 	rc = r8169_mdio_register(tp);
5483 	if (rc)
5484 		return rc;
5485 
5486 	rc = register_netdev(dev);
5487 	if (rc)
5488 		return rc;
5489 
5490 	netdev_info(dev, "%s, %pM, XID %03x, IRQ %d\n",
5491 		    rtl_chip_infos[chipset].name, dev->dev_addr, xid, tp->irq);
5492 
5493 	if (jumbo_max)
5494 		netdev_info(dev, "jumbo features [frames: %d bytes, tx checksumming: %s]\n",
5495 			    jumbo_max, tp->mac_version <= RTL_GIGA_MAC_VER_06 ?
5496 			    "ok" : "ko");
5497 
5498 	if (tp->dash_type != RTL_DASH_NONE) {
5499 		netdev_info(dev, "DASH enabled\n");
5500 		rtl8168_driver_start(tp);
5501 	}
5502 
5503 	if (pci_dev_run_wake(pdev))
5504 		pm_runtime_put_sync(&pdev->dev);
5505 
5506 	return 0;
5507 }
5508 
5509 static struct pci_driver rtl8169_pci_driver = {
5510 	.name		= KBUILD_MODNAME,
5511 	.id_table	= rtl8169_pci_tbl,
5512 	.probe		= rtl_init_one,
5513 	.remove		= rtl_remove_one,
5514 	.shutdown	= rtl_shutdown,
5515 	.driver.pm	= pm_ptr(&rtl8169_pm_ops),
5516 };
5517 
5518 module_pci_driver(rtl8169_pci_driver);
5519