1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *
4  *   Copyright (C) 2009-2016 John Crispin <blogic@openwrt.org>
5  *   Copyright (C) 2009-2016 Felix Fietkau <nbd@openwrt.org>
6  *   Copyright (C) 2013-2016 Michael Lee <igvtee@gmail.com>
7  */
8 
9 #include <linux/of_device.h>
10 #include <linux/of_mdio.h>
11 #include <linux/of_net.h>
12 #include <linux/mfd/syscon.h>
13 #include <linux/regmap.h>
14 #include <linux/clk.h>
15 #include <linux/pm_runtime.h>
16 #include <linux/if_vlan.h>
17 #include <linux/reset.h>
18 #include <linux/tcp.h>
19 #include <linux/interrupt.h>
20 #include <linux/pinctrl/devinfo.h>
21 #include <linux/phylink.h>
22 
23 #include "mtk_eth_soc.h"
24 
25 static int mtk_msg_level = -1;
26 module_param_named(msg_level, mtk_msg_level, int, 0);
27 MODULE_PARM_DESC(msg_level, "Message level (-1=defaults,0=none,...,16=all)");
28 
29 #define MTK_ETHTOOL_STAT(x) { #x, \
30 			      offsetof(struct mtk_hw_stats, x) / sizeof(u64) }
31 
32 /* strings used by ethtool */
33 static const struct mtk_ethtool_stats {
34 	char str[ETH_GSTRING_LEN];
35 	u32 offset;
36 } mtk_ethtool_stats[] = {
37 	MTK_ETHTOOL_STAT(tx_bytes),
38 	MTK_ETHTOOL_STAT(tx_packets),
39 	MTK_ETHTOOL_STAT(tx_skip),
40 	MTK_ETHTOOL_STAT(tx_collisions),
41 	MTK_ETHTOOL_STAT(rx_bytes),
42 	MTK_ETHTOOL_STAT(rx_packets),
43 	MTK_ETHTOOL_STAT(rx_overflow),
44 	MTK_ETHTOOL_STAT(rx_fcs_errors),
45 	MTK_ETHTOOL_STAT(rx_short_errors),
46 	MTK_ETHTOOL_STAT(rx_long_errors),
47 	MTK_ETHTOOL_STAT(rx_checksum_errors),
48 	MTK_ETHTOOL_STAT(rx_flow_control_packets),
49 };
50 
51 static const char * const mtk_clks_source_name[] = {
52 	"ethif", "sgmiitop", "esw", "gp0", "gp1", "gp2", "fe", "trgpll",
53 	"sgmii_tx250m", "sgmii_rx250m", "sgmii_cdr_ref", "sgmii_cdr_fb",
54 	"sgmii2_tx250m", "sgmii2_rx250m", "sgmii2_cdr_ref", "sgmii2_cdr_fb",
55 	"sgmii_ck", "eth2pll",
56 };
57 
58 void mtk_w32(struct mtk_eth *eth, u32 val, unsigned reg)
59 {
60 	__raw_writel(val, eth->base + reg);
61 }
62 
63 u32 mtk_r32(struct mtk_eth *eth, unsigned reg)
64 {
65 	return __raw_readl(eth->base + reg);
66 }
67 
68 static int mtk_mdio_busy_wait(struct mtk_eth *eth)
69 {
70 	unsigned long t_start = jiffies;
71 
72 	while (1) {
73 		if (!(mtk_r32(eth, MTK_PHY_IAC) & PHY_IAC_ACCESS))
74 			return 0;
75 		if (time_after(jiffies, t_start + PHY_IAC_TIMEOUT))
76 			break;
77 		usleep_range(10, 20);
78 	}
79 
80 	dev_err(eth->dev, "mdio: MDIO timeout\n");
81 	return -1;
82 }
83 
84 static u32 _mtk_mdio_write(struct mtk_eth *eth, u32 phy_addr,
85 			   u32 phy_register, u32 write_data)
86 {
87 	if (mtk_mdio_busy_wait(eth))
88 		return -1;
89 
90 	write_data &= 0xffff;
91 
92 	mtk_w32(eth, PHY_IAC_ACCESS | PHY_IAC_START | PHY_IAC_WRITE |
93 		(phy_register << PHY_IAC_REG_SHIFT) |
94 		(phy_addr << PHY_IAC_ADDR_SHIFT) | write_data,
95 		MTK_PHY_IAC);
96 
97 	if (mtk_mdio_busy_wait(eth))
98 		return -1;
99 
100 	return 0;
101 }
102 
103 static u32 _mtk_mdio_read(struct mtk_eth *eth, int phy_addr, int phy_reg)
104 {
105 	u32 d;
106 
107 	if (mtk_mdio_busy_wait(eth))
108 		return 0xffff;
109 
110 	mtk_w32(eth, PHY_IAC_ACCESS | PHY_IAC_START | PHY_IAC_READ |
111 		(phy_reg << PHY_IAC_REG_SHIFT) |
112 		(phy_addr << PHY_IAC_ADDR_SHIFT),
113 		MTK_PHY_IAC);
114 
115 	if (mtk_mdio_busy_wait(eth))
116 		return 0xffff;
117 
118 	d = mtk_r32(eth, MTK_PHY_IAC) & 0xffff;
119 
120 	return d;
121 }
122 
123 static int mtk_mdio_write(struct mii_bus *bus, int phy_addr,
124 			  int phy_reg, u16 val)
125 {
126 	struct mtk_eth *eth = bus->priv;
127 
128 	return _mtk_mdio_write(eth, phy_addr, phy_reg, val);
129 }
130 
131 static int mtk_mdio_read(struct mii_bus *bus, int phy_addr, int phy_reg)
132 {
133 	struct mtk_eth *eth = bus->priv;
134 
135 	return _mtk_mdio_read(eth, phy_addr, phy_reg);
136 }
137 
138 static int mt7621_gmac0_rgmii_adjust(struct mtk_eth *eth,
139 				     phy_interface_t interface)
140 {
141 	u32 val;
142 
143 	/* Check DDR memory type.
144 	 * Currently TRGMII mode with DDR2 memory is not supported.
145 	 */
146 	regmap_read(eth->ethsys, ETHSYS_SYSCFG, &val);
147 	if (interface == PHY_INTERFACE_MODE_TRGMII &&
148 	    val & SYSCFG_DRAM_TYPE_DDR2) {
149 		dev_err(eth->dev,
150 			"TRGMII mode with DDR2 memory is not supported!\n");
151 		return -EOPNOTSUPP;
152 	}
153 
154 	val = (interface == PHY_INTERFACE_MODE_TRGMII) ?
155 		ETHSYS_TRGMII_MT7621_DDR_PLL : 0;
156 
157 	regmap_update_bits(eth->ethsys, ETHSYS_CLKCFG0,
158 			   ETHSYS_TRGMII_MT7621_MASK, val);
159 
160 	return 0;
161 }
162 
163 static void mtk_gmac0_rgmii_adjust(struct mtk_eth *eth, int speed)
164 {
165 	u32 val;
166 	int ret;
167 
168 	val = (speed == SPEED_1000) ?
169 		INTF_MODE_RGMII_1000 : INTF_MODE_RGMII_10_100;
170 	mtk_w32(eth, val, INTF_MODE);
171 
172 	regmap_update_bits(eth->ethsys, ETHSYS_CLKCFG0,
173 			   ETHSYS_TRGMII_CLK_SEL362_5,
174 			   ETHSYS_TRGMII_CLK_SEL362_5);
175 
176 	val = (speed == SPEED_1000) ? 250000000 : 500000000;
177 	ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], val);
178 	if (ret)
179 		dev_err(eth->dev, "Failed to set trgmii pll: %d\n", ret);
180 
181 	val = (speed == SPEED_1000) ?
182 		RCK_CTRL_RGMII_1000 : RCK_CTRL_RGMII_10_100;
183 	mtk_w32(eth, val, TRGMII_RCK_CTRL);
184 
185 	val = (speed == SPEED_1000) ?
186 		TCK_CTRL_RGMII_1000 : TCK_CTRL_RGMII_10_100;
187 	mtk_w32(eth, val, TRGMII_TCK_CTRL);
188 }
189 
190 static void mtk_mac_config(struct phylink_config *config, unsigned int mode,
191 			   const struct phylink_link_state *state)
192 {
193 	struct mtk_mac *mac = container_of(config, struct mtk_mac,
194 					   phylink_config);
195 	struct mtk_eth *eth = mac->hw;
196 	u32 mcr_cur, mcr_new, sid;
197 	int val, ge_mode, err;
198 
199 	/* MT76x8 has no hardware settings between for the MAC */
200 	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) &&
201 	    mac->interface != state->interface) {
202 		/* Setup soc pin functions */
203 		switch (state->interface) {
204 		case PHY_INTERFACE_MODE_TRGMII:
205 			if (mac->id)
206 				goto err_phy;
207 			if (!MTK_HAS_CAPS(mac->hw->soc->caps,
208 					  MTK_GMAC1_TRGMII))
209 				goto err_phy;
210 			/* fall through */
211 		case PHY_INTERFACE_MODE_RGMII_TXID:
212 		case PHY_INTERFACE_MODE_RGMII_RXID:
213 		case PHY_INTERFACE_MODE_RGMII_ID:
214 		case PHY_INTERFACE_MODE_RGMII:
215 		case PHY_INTERFACE_MODE_MII:
216 		case PHY_INTERFACE_MODE_REVMII:
217 		case PHY_INTERFACE_MODE_RMII:
218 			if (MTK_HAS_CAPS(eth->soc->caps, MTK_RGMII)) {
219 				err = mtk_gmac_rgmii_path_setup(eth, mac->id);
220 				if (err)
221 					goto init_err;
222 			}
223 			break;
224 		case PHY_INTERFACE_MODE_1000BASEX:
225 		case PHY_INTERFACE_MODE_2500BASEX:
226 		case PHY_INTERFACE_MODE_SGMII:
227 			if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) {
228 				err = mtk_gmac_sgmii_path_setup(eth, mac->id);
229 				if (err)
230 					goto init_err;
231 			}
232 			break;
233 		case PHY_INTERFACE_MODE_GMII:
234 			if (MTK_HAS_CAPS(eth->soc->caps, MTK_GEPHY)) {
235 				err = mtk_gmac_gephy_path_setup(eth, mac->id);
236 				if (err)
237 					goto init_err;
238 			}
239 			break;
240 		default:
241 			goto err_phy;
242 		}
243 
244 		/* Setup clock for 1st gmac */
245 		if (!mac->id && state->interface != PHY_INTERFACE_MODE_SGMII &&
246 		    !phy_interface_mode_is_8023z(state->interface) &&
247 		    MTK_HAS_CAPS(mac->hw->soc->caps, MTK_GMAC1_TRGMII)) {
248 			if (MTK_HAS_CAPS(mac->hw->soc->caps,
249 					 MTK_TRGMII_MT7621_CLK)) {
250 				if (mt7621_gmac0_rgmii_adjust(mac->hw,
251 							      state->interface))
252 					goto err_phy;
253 			} else {
254 				if (state->interface !=
255 				    PHY_INTERFACE_MODE_TRGMII)
256 					mtk_gmac0_rgmii_adjust(mac->hw,
257 							       state->speed);
258 			}
259 		}
260 
261 		ge_mode = 0;
262 		switch (state->interface) {
263 		case PHY_INTERFACE_MODE_MII:
264 		case PHY_INTERFACE_MODE_GMII:
265 			ge_mode = 1;
266 			break;
267 		case PHY_INTERFACE_MODE_REVMII:
268 			ge_mode = 2;
269 			break;
270 		case PHY_INTERFACE_MODE_RMII:
271 			if (mac->id)
272 				goto err_phy;
273 			ge_mode = 3;
274 			break;
275 		default:
276 			break;
277 		}
278 
279 		/* put the gmac into the right mode */
280 		regmap_read(eth->ethsys, ETHSYS_SYSCFG0, &val);
281 		val &= ~SYSCFG0_GE_MODE(SYSCFG0_GE_MASK, mac->id);
282 		val |= SYSCFG0_GE_MODE(ge_mode, mac->id);
283 		regmap_write(eth->ethsys, ETHSYS_SYSCFG0, val);
284 
285 		mac->interface = state->interface;
286 	}
287 
288 	/* SGMII */
289 	if (state->interface == PHY_INTERFACE_MODE_SGMII ||
290 	    phy_interface_mode_is_8023z(state->interface)) {
291 		/* The path GMAC to SGMII will be enabled once the SGMIISYS is
292 		 * being setup done.
293 		 */
294 		regmap_read(eth->ethsys, ETHSYS_SYSCFG0, &val);
295 
296 		regmap_update_bits(eth->ethsys, ETHSYS_SYSCFG0,
297 				   SYSCFG0_SGMII_MASK,
298 				   ~(u32)SYSCFG0_SGMII_MASK);
299 
300 		/* Decide how GMAC and SGMIISYS be mapped */
301 		sid = (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_SGMII)) ?
302 		       0 : mac->id;
303 
304 		/* Setup SGMIISYS with the determined property */
305 		if (state->interface != PHY_INTERFACE_MODE_SGMII)
306 			err = mtk_sgmii_setup_mode_force(eth->sgmii, sid,
307 							 state);
308 		else if (phylink_autoneg_inband(mode))
309 			err = mtk_sgmii_setup_mode_an(eth->sgmii, sid);
310 
311 		if (err)
312 			goto init_err;
313 
314 		regmap_update_bits(eth->ethsys, ETHSYS_SYSCFG0,
315 				   SYSCFG0_SGMII_MASK, val);
316 	} else if (phylink_autoneg_inband(mode)) {
317 		dev_err(eth->dev,
318 			"In-band mode not supported in non SGMII mode!\n");
319 		return;
320 	}
321 
322 	/* Setup gmac */
323 	mcr_cur = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
324 	mcr_new = mcr_cur;
325 	mcr_new &= ~(MAC_MCR_SPEED_100 | MAC_MCR_SPEED_1000 |
326 		     MAC_MCR_FORCE_DPX | MAC_MCR_FORCE_TX_FC |
327 		     MAC_MCR_FORCE_RX_FC);
328 	mcr_new |= MAC_MCR_MAX_RX_1536 | MAC_MCR_IPG_CFG | MAC_MCR_FORCE_MODE |
329 		   MAC_MCR_BACKOFF_EN | MAC_MCR_BACKPR_EN | MAC_MCR_FORCE_LINK;
330 
331 	switch (state->speed) {
332 	case SPEED_2500:
333 	case SPEED_1000:
334 		mcr_new |= MAC_MCR_SPEED_1000;
335 		break;
336 	case SPEED_100:
337 		mcr_new |= MAC_MCR_SPEED_100;
338 		break;
339 	}
340 	if (state->duplex == DUPLEX_FULL) {
341 		mcr_new |= MAC_MCR_FORCE_DPX;
342 		if (state->pause & MLO_PAUSE_TX)
343 			mcr_new |= MAC_MCR_FORCE_TX_FC;
344 		if (state->pause & MLO_PAUSE_RX)
345 			mcr_new |= MAC_MCR_FORCE_RX_FC;
346 	}
347 
348 	/* Only update control register when needed! */
349 	if (mcr_new != mcr_cur)
350 		mtk_w32(mac->hw, mcr_new, MTK_MAC_MCR(mac->id));
351 
352 	return;
353 
354 err_phy:
355 	dev_err(eth->dev, "%s: GMAC%d mode %s not supported!\n", __func__,
356 		mac->id, phy_modes(state->interface));
357 	return;
358 
359 init_err:
360 	dev_err(eth->dev, "%s: GMAC%d mode %s err: %d!\n", __func__,
361 		mac->id, phy_modes(state->interface), err);
362 }
363 
364 static void mtk_mac_pcs_get_state(struct phylink_config *config,
365 				  struct phylink_link_state *state)
366 {
367 	struct mtk_mac *mac = container_of(config, struct mtk_mac,
368 					   phylink_config);
369 	u32 pmsr = mtk_r32(mac->hw, MTK_MAC_MSR(mac->id));
370 
371 	state->link = (pmsr & MAC_MSR_LINK);
372 	state->duplex = (pmsr & MAC_MSR_DPX) >> 1;
373 
374 	switch (pmsr & (MAC_MSR_SPEED_1000 | MAC_MSR_SPEED_100)) {
375 	case 0:
376 		state->speed = SPEED_10;
377 		break;
378 	case MAC_MSR_SPEED_100:
379 		state->speed = SPEED_100;
380 		break;
381 	case MAC_MSR_SPEED_1000:
382 		state->speed = SPEED_1000;
383 		break;
384 	default:
385 		state->speed = SPEED_UNKNOWN;
386 		break;
387 	}
388 
389 	state->pause &= (MLO_PAUSE_RX | MLO_PAUSE_TX);
390 	if (pmsr & MAC_MSR_RX_FC)
391 		state->pause |= MLO_PAUSE_RX;
392 	if (pmsr & MAC_MSR_TX_FC)
393 		state->pause |= MLO_PAUSE_TX;
394 }
395 
396 static void mtk_mac_an_restart(struct phylink_config *config)
397 {
398 	struct mtk_mac *mac = container_of(config, struct mtk_mac,
399 					   phylink_config);
400 
401 	mtk_sgmii_restart_an(mac->hw, mac->id);
402 }
403 
404 static void mtk_mac_link_down(struct phylink_config *config, unsigned int mode,
405 			      phy_interface_t interface)
406 {
407 	struct mtk_mac *mac = container_of(config, struct mtk_mac,
408 					   phylink_config);
409 	u32 mcr = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
410 
411 	mcr &= ~(MAC_MCR_TX_EN | MAC_MCR_RX_EN);
412 	mtk_w32(mac->hw, mcr, MTK_MAC_MCR(mac->id));
413 }
414 
415 static void mtk_mac_link_up(struct phylink_config *config, unsigned int mode,
416 			    phy_interface_t interface,
417 			    struct phy_device *phy)
418 {
419 	struct mtk_mac *mac = container_of(config, struct mtk_mac,
420 					   phylink_config);
421 	u32 mcr = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id));
422 
423 	mcr |= MAC_MCR_TX_EN | MAC_MCR_RX_EN;
424 	mtk_w32(mac->hw, mcr, MTK_MAC_MCR(mac->id));
425 }
426 
427 static void mtk_validate(struct phylink_config *config,
428 			 unsigned long *supported,
429 			 struct phylink_link_state *state)
430 {
431 	struct mtk_mac *mac = container_of(config, struct mtk_mac,
432 					   phylink_config);
433 	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
434 
435 	if (state->interface != PHY_INTERFACE_MODE_NA &&
436 	    state->interface != PHY_INTERFACE_MODE_MII &&
437 	    state->interface != PHY_INTERFACE_MODE_GMII &&
438 	    !(MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII) &&
439 	      phy_interface_mode_is_rgmii(state->interface)) &&
440 	    !(MTK_HAS_CAPS(mac->hw->soc->caps, MTK_TRGMII) &&
441 	      !mac->id && state->interface == PHY_INTERFACE_MODE_TRGMII) &&
442 	    !(MTK_HAS_CAPS(mac->hw->soc->caps, MTK_SGMII) &&
443 	      (state->interface == PHY_INTERFACE_MODE_SGMII ||
444 	       phy_interface_mode_is_8023z(state->interface)))) {
445 		linkmode_zero(supported);
446 		return;
447 	}
448 
449 	phylink_set_port_modes(mask);
450 	phylink_set(mask, Autoneg);
451 
452 	switch (state->interface) {
453 	case PHY_INTERFACE_MODE_TRGMII:
454 		phylink_set(mask, 1000baseT_Full);
455 		break;
456 	case PHY_INTERFACE_MODE_1000BASEX:
457 	case PHY_INTERFACE_MODE_2500BASEX:
458 		phylink_set(mask, 1000baseX_Full);
459 		phylink_set(mask, 2500baseX_Full);
460 		break;
461 	case PHY_INTERFACE_MODE_GMII:
462 	case PHY_INTERFACE_MODE_RGMII:
463 	case PHY_INTERFACE_MODE_RGMII_ID:
464 	case PHY_INTERFACE_MODE_RGMII_RXID:
465 	case PHY_INTERFACE_MODE_RGMII_TXID:
466 		phylink_set(mask, 1000baseT_Half);
467 		/* fall through */
468 	case PHY_INTERFACE_MODE_SGMII:
469 		phylink_set(mask, 1000baseT_Full);
470 		phylink_set(mask, 1000baseX_Full);
471 		/* fall through */
472 	case PHY_INTERFACE_MODE_MII:
473 	case PHY_INTERFACE_MODE_RMII:
474 	case PHY_INTERFACE_MODE_REVMII:
475 	case PHY_INTERFACE_MODE_NA:
476 	default:
477 		phylink_set(mask, 10baseT_Half);
478 		phylink_set(mask, 10baseT_Full);
479 		phylink_set(mask, 100baseT_Half);
480 		phylink_set(mask, 100baseT_Full);
481 		break;
482 	}
483 
484 	if (state->interface == PHY_INTERFACE_MODE_NA) {
485 		if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_SGMII)) {
486 			phylink_set(mask, 1000baseT_Full);
487 			phylink_set(mask, 1000baseX_Full);
488 			phylink_set(mask, 2500baseX_Full);
489 		}
490 		if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII)) {
491 			phylink_set(mask, 1000baseT_Full);
492 			phylink_set(mask, 1000baseT_Half);
493 			phylink_set(mask, 1000baseX_Full);
494 		}
495 		if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_GEPHY)) {
496 			phylink_set(mask, 1000baseT_Full);
497 			phylink_set(mask, 1000baseT_Half);
498 		}
499 	}
500 
501 	phylink_set(mask, Pause);
502 	phylink_set(mask, Asym_Pause);
503 
504 	linkmode_and(supported, supported, mask);
505 	linkmode_and(state->advertising, state->advertising, mask);
506 
507 	/* We can only operate at 2500BaseX or 1000BaseX. If requested
508 	 * to advertise both, only report advertising at 2500BaseX.
509 	 */
510 	phylink_helper_basex_speed(state);
511 }
512 
513 static const struct phylink_mac_ops mtk_phylink_ops = {
514 	.validate = mtk_validate,
515 	.mac_pcs_get_state = mtk_mac_pcs_get_state,
516 	.mac_an_restart = mtk_mac_an_restart,
517 	.mac_config = mtk_mac_config,
518 	.mac_link_down = mtk_mac_link_down,
519 	.mac_link_up = mtk_mac_link_up,
520 };
521 
522 static int mtk_mdio_init(struct mtk_eth *eth)
523 {
524 	struct device_node *mii_np;
525 	int ret;
526 
527 	mii_np = of_get_child_by_name(eth->dev->of_node, "mdio-bus");
528 	if (!mii_np) {
529 		dev_err(eth->dev, "no %s child node found", "mdio-bus");
530 		return -ENODEV;
531 	}
532 
533 	if (!of_device_is_available(mii_np)) {
534 		ret = -ENODEV;
535 		goto err_put_node;
536 	}
537 
538 	eth->mii_bus = devm_mdiobus_alloc(eth->dev);
539 	if (!eth->mii_bus) {
540 		ret = -ENOMEM;
541 		goto err_put_node;
542 	}
543 
544 	eth->mii_bus->name = "mdio";
545 	eth->mii_bus->read = mtk_mdio_read;
546 	eth->mii_bus->write = mtk_mdio_write;
547 	eth->mii_bus->priv = eth;
548 	eth->mii_bus->parent = eth->dev;
549 
550 	snprintf(eth->mii_bus->id, MII_BUS_ID_SIZE, "%pOFn", mii_np);
551 	ret = of_mdiobus_register(eth->mii_bus, mii_np);
552 
553 err_put_node:
554 	of_node_put(mii_np);
555 	return ret;
556 }
557 
558 static void mtk_mdio_cleanup(struct mtk_eth *eth)
559 {
560 	if (!eth->mii_bus)
561 		return;
562 
563 	mdiobus_unregister(eth->mii_bus);
564 }
565 
566 static inline void mtk_tx_irq_disable(struct mtk_eth *eth, u32 mask)
567 {
568 	unsigned long flags;
569 	u32 val;
570 
571 	spin_lock_irqsave(&eth->tx_irq_lock, flags);
572 	val = mtk_r32(eth, eth->tx_int_mask_reg);
573 	mtk_w32(eth, val & ~mask, eth->tx_int_mask_reg);
574 	spin_unlock_irqrestore(&eth->tx_irq_lock, flags);
575 }
576 
577 static inline void mtk_tx_irq_enable(struct mtk_eth *eth, u32 mask)
578 {
579 	unsigned long flags;
580 	u32 val;
581 
582 	spin_lock_irqsave(&eth->tx_irq_lock, flags);
583 	val = mtk_r32(eth, eth->tx_int_mask_reg);
584 	mtk_w32(eth, val | mask, eth->tx_int_mask_reg);
585 	spin_unlock_irqrestore(&eth->tx_irq_lock, flags);
586 }
587 
588 static inline void mtk_rx_irq_disable(struct mtk_eth *eth, u32 mask)
589 {
590 	unsigned long flags;
591 	u32 val;
592 
593 	spin_lock_irqsave(&eth->rx_irq_lock, flags);
594 	val = mtk_r32(eth, MTK_PDMA_INT_MASK);
595 	mtk_w32(eth, val & ~mask, MTK_PDMA_INT_MASK);
596 	spin_unlock_irqrestore(&eth->rx_irq_lock, flags);
597 }
598 
599 static inline void mtk_rx_irq_enable(struct mtk_eth *eth, u32 mask)
600 {
601 	unsigned long flags;
602 	u32 val;
603 
604 	spin_lock_irqsave(&eth->rx_irq_lock, flags);
605 	val = mtk_r32(eth, MTK_PDMA_INT_MASK);
606 	mtk_w32(eth, val | mask, MTK_PDMA_INT_MASK);
607 	spin_unlock_irqrestore(&eth->rx_irq_lock, flags);
608 }
609 
610 static int mtk_set_mac_address(struct net_device *dev, void *p)
611 {
612 	int ret = eth_mac_addr(dev, p);
613 	struct mtk_mac *mac = netdev_priv(dev);
614 	struct mtk_eth *eth = mac->hw;
615 	const char *macaddr = dev->dev_addr;
616 
617 	if (ret)
618 		return ret;
619 
620 	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
621 		return -EBUSY;
622 
623 	spin_lock_bh(&mac->hw->page_lock);
624 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
625 		mtk_w32(mac->hw, (macaddr[0] << 8) | macaddr[1],
626 			MT7628_SDM_MAC_ADRH);
627 		mtk_w32(mac->hw, (macaddr[2] << 24) | (macaddr[3] << 16) |
628 			(macaddr[4] << 8) | macaddr[5],
629 			MT7628_SDM_MAC_ADRL);
630 	} else {
631 		mtk_w32(mac->hw, (macaddr[0] << 8) | macaddr[1],
632 			MTK_GDMA_MAC_ADRH(mac->id));
633 		mtk_w32(mac->hw, (macaddr[2] << 24) | (macaddr[3] << 16) |
634 			(macaddr[4] << 8) | macaddr[5],
635 			MTK_GDMA_MAC_ADRL(mac->id));
636 	}
637 	spin_unlock_bh(&mac->hw->page_lock);
638 
639 	return 0;
640 }
641 
642 void mtk_stats_update_mac(struct mtk_mac *mac)
643 {
644 	struct mtk_hw_stats *hw_stats = mac->hw_stats;
645 	unsigned int base = MTK_GDM1_TX_GBCNT;
646 	u64 stats;
647 
648 	base += hw_stats->reg_offset;
649 
650 	u64_stats_update_begin(&hw_stats->syncp);
651 
652 	hw_stats->rx_bytes += mtk_r32(mac->hw, base);
653 	stats =  mtk_r32(mac->hw, base + 0x04);
654 	if (stats)
655 		hw_stats->rx_bytes += (stats << 32);
656 	hw_stats->rx_packets += mtk_r32(mac->hw, base + 0x08);
657 	hw_stats->rx_overflow += mtk_r32(mac->hw, base + 0x10);
658 	hw_stats->rx_fcs_errors += mtk_r32(mac->hw, base + 0x14);
659 	hw_stats->rx_short_errors += mtk_r32(mac->hw, base + 0x18);
660 	hw_stats->rx_long_errors += mtk_r32(mac->hw, base + 0x1c);
661 	hw_stats->rx_checksum_errors += mtk_r32(mac->hw, base + 0x20);
662 	hw_stats->rx_flow_control_packets +=
663 					mtk_r32(mac->hw, base + 0x24);
664 	hw_stats->tx_skip += mtk_r32(mac->hw, base + 0x28);
665 	hw_stats->tx_collisions += mtk_r32(mac->hw, base + 0x2c);
666 	hw_stats->tx_bytes += mtk_r32(mac->hw, base + 0x30);
667 	stats =  mtk_r32(mac->hw, base + 0x34);
668 	if (stats)
669 		hw_stats->tx_bytes += (stats << 32);
670 	hw_stats->tx_packets += mtk_r32(mac->hw, base + 0x38);
671 	u64_stats_update_end(&hw_stats->syncp);
672 }
673 
674 static void mtk_stats_update(struct mtk_eth *eth)
675 {
676 	int i;
677 
678 	for (i = 0; i < MTK_MAC_COUNT; i++) {
679 		if (!eth->mac[i] || !eth->mac[i]->hw_stats)
680 			continue;
681 		if (spin_trylock(&eth->mac[i]->hw_stats->stats_lock)) {
682 			mtk_stats_update_mac(eth->mac[i]);
683 			spin_unlock(&eth->mac[i]->hw_stats->stats_lock);
684 		}
685 	}
686 }
687 
688 static void mtk_get_stats64(struct net_device *dev,
689 			    struct rtnl_link_stats64 *storage)
690 {
691 	struct mtk_mac *mac = netdev_priv(dev);
692 	struct mtk_hw_stats *hw_stats = mac->hw_stats;
693 	unsigned int start;
694 
695 	if (netif_running(dev) && netif_device_present(dev)) {
696 		if (spin_trylock_bh(&hw_stats->stats_lock)) {
697 			mtk_stats_update_mac(mac);
698 			spin_unlock_bh(&hw_stats->stats_lock);
699 		}
700 	}
701 
702 	do {
703 		start = u64_stats_fetch_begin_irq(&hw_stats->syncp);
704 		storage->rx_packets = hw_stats->rx_packets;
705 		storage->tx_packets = hw_stats->tx_packets;
706 		storage->rx_bytes = hw_stats->rx_bytes;
707 		storage->tx_bytes = hw_stats->tx_bytes;
708 		storage->collisions = hw_stats->tx_collisions;
709 		storage->rx_length_errors = hw_stats->rx_short_errors +
710 			hw_stats->rx_long_errors;
711 		storage->rx_over_errors = hw_stats->rx_overflow;
712 		storage->rx_crc_errors = hw_stats->rx_fcs_errors;
713 		storage->rx_errors = hw_stats->rx_checksum_errors;
714 		storage->tx_aborted_errors = hw_stats->tx_skip;
715 	} while (u64_stats_fetch_retry_irq(&hw_stats->syncp, start));
716 
717 	storage->tx_errors = dev->stats.tx_errors;
718 	storage->rx_dropped = dev->stats.rx_dropped;
719 	storage->tx_dropped = dev->stats.tx_dropped;
720 }
721 
722 static inline int mtk_max_frag_size(int mtu)
723 {
724 	/* make sure buf_size will be at least MTK_MAX_RX_LENGTH */
725 	if (mtu + MTK_RX_ETH_HLEN < MTK_MAX_RX_LENGTH)
726 		mtu = MTK_MAX_RX_LENGTH - MTK_RX_ETH_HLEN;
727 
728 	return SKB_DATA_ALIGN(MTK_RX_HLEN + mtu) +
729 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
730 }
731 
732 static inline int mtk_max_buf_size(int frag_size)
733 {
734 	int buf_size = frag_size - NET_SKB_PAD - NET_IP_ALIGN -
735 		       SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
736 
737 	WARN_ON(buf_size < MTK_MAX_RX_LENGTH);
738 
739 	return buf_size;
740 }
741 
742 static inline void mtk_rx_get_desc(struct mtk_rx_dma *rxd,
743 				   struct mtk_rx_dma *dma_rxd)
744 {
745 	rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
746 	rxd->rxd2 = READ_ONCE(dma_rxd->rxd2);
747 	rxd->rxd3 = READ_ONCE(dma_rxd->rxd3);
748 	rxd->rxd4 = READ_ONCE(dma_rxd->rxd4);
749 }
750 
751 /* the qdma core needs scratch memory to be setup */
752 static int mtk_init_fq_dma(struct mtk_eth *eth)
753 {
754 	dma_addr_t phy_ring_tail;
755 	int cnt = MTK_DMA_SIZE;
756 	dma_addr_t dma_addr;
757 	int i;
758 
759 	eth->scratch_ring = dma_alloc_coherent(eth->dev,
760 					       cnt * sizeof(struct mtk_tx_dma),
761 					       &eth->phy_scratch_ring,
762 					       GFP_ATOMIC);
763 	if (unlikely(!eth->scratch_ring))
764 		return -ENOMEM;
765 
766 	eth->scratch_head = kcalloc(cnt, MTK_QDMA_PAGE_SIZE,
767 				    GFP_KERNEL);
768 	if (unlikely(!eth->scratch_head))
769 		return -ENOMEM;
770 
771 	dma_addr = dma_map_single(eth->dev,
772 				  eth->scratch_head, cnt * MTK_QDMA_PAGE_SIZE,
773 				  DMA_FROM_DEVICE);
774 	if (unlikely(dma_mapping_error(eth->dev, dma_addr)))
775 		return -ENOMEM;
776 
777 	phy_ring_tail = eth->phy_scratch_ring +
778 			(sizeof(struct mtk_tx_dma) * (cnt - 1));
779 
780 	for (i = 0; i < cnt; i++) {
781 		eth->scratch_ring[i].txd1 =
782 					(dma_addr + (i * MTK_QDMA_PAGE_SIZE));
783 		if (i < cnt - 1)
784 			eth->scratch_ring[i].txd2 = (eth->phy_scratch_ring +
785 				((i + 1) * sizeof(struct mtk_tx_dma)));
786 		eth->scratch_ring[i].txd3 = TX_DMA_SDL(MTK_QDMA_PAGE_SIZE);
787 	}
788 
789 	mtk_w32(eth, eth->phy_scratch_ring, MTK_QDMA_FQ_HEAD);
790 	mtk_w32(eth, phy_ring_tail, MTK_QDMA_FQ_TAIL);
791 	mtk_w32(eth, (cnt << 16) | cnt, MTK_QDMA_FQ_CNT);
792 	mtk_w32(eth, MTK_QDMA_PAGE_SIZE << 16, MTK_QDMA_FQ_BLEN);
793 
794 	return 0;
795 }
796 
797 static inline void *mtk_qdma_phys_to_virt(struct mtk_tx_ring *ring, u32 desc)
798 {
799 	void *ret = ring->dma;
800 
801 	return ret + (desc - ring->phys);
802 }
803 
804 static inline struct mtk_tx_buf *mtk_desc_to_tx_buf(struct mtk_tx_ring *ring,
805 						    struct mtk_tx_dma *txd)
806 {
807 	int idx = txd - ring->dma;
808 
809 	return &ring->buf[idx];
810 }
811 
812 static struct mtk_tx_dma *qdma_to_pdma(struct mtk_tx_ring *ring,
813 				       struct mtk_tx_dma *dma)
814 {
815 	return ring->dma_pdma - ring->dma + dma;
816 }
817 
818 static int txd_to_idx(struct mtk_tx_ring *ring, struct mtk_tx_dma *dma)
819 {
820 	return ((void *)dma - (void *)ring->dma) / sizeof(*dma);
821 }
822 
823 static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf)
824 {
825 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
826 		if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) {
827 			dma_unmap_single(eth->dev,
828 					 dma_unmap_addr(tx_buf, dma_addr0),
829 					 dma_unmap_len(tx_buf, dma_len0),
830 					 DMA_TO_DEVICE);
831 		} else if (tx_buf->flags & MTK_TX_FLAGS_PAGE0) {
832 			dma_unmap_page(eth->dev,
833 				       dma_unmap_addr(tx_buf, dma_addr0),
834 				       dma_unmap_len(tx_buf, dma_len0),
835 				       DMA_TO_DEVICE);
836 		}
837 	} else {
838 		if (dma_unmap_len(tx_buf, dma_len0)) {
839 			dma_unmap_page(eth->dev,
840 				       dma_unmap_addr(tx_buf, dma_addr0),
841 				       dma_unmap_len(tx_buf, dma_len0),
842 				       DMA_TO_DEVICE);
843 		}
844 
845 		if (dma_unmap_len(tx_buf, dma_len1)) {
846 			dma_unmap_page(eth->dev,
847 				       dma_unmap_addr(tx_buf, dma_addr1),
848 				       dma_unmap_len(tx_buf, dma_len1),
849 				       DMA_TO_DEVICE);
850 		}
851 	}
852 
853 	tx_buf->flags = 0;
854 	if (tx_buf->skb &&
855 	    (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC))
856 		dev_kfree_skb_any(tx_buf->skb);
857 	tx_buf->skb = NULL;
858 }
859 
860 static void setup_tx_buf(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
861 			 struct mtk_tx_dma *txd, dma_addr_t mapped_addr,
862 			 size_t size, int idx)
863 {
864 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
865 		dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr);
866 		dma_unmap_len_set(tx_buf, dma_len0, size);
867 	} else {
868 		if (idx & 1) {
869 			txd->txd3 = mapped_addr;
870 			txd->txd2 |= TX_DMA_PLEN1(size);
871 			dma_unmap_addr_set(tx_buf, dma_addr1, mapped_addr);
872 			dma_unmap_len_set(tx_buf, dma_len1, size);
873 		} else {
874 			tx_buf->skb = (struct sk_buff *)MTK_DMA_DUMMY_DESC;
875 			txd->txd1 = mapped_addr;
876 			txd->txd2 = TX_DMA_PLEN0(size);
877 			dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr);
878 			dma_unmap_len_set(tx_buf, dma_len0, size);
879 		}
880 	}
881 }
882 
883 static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
884 		      int tx_num, struct mtk_tx_ring *ring, bool gso)
885 {
886 	struct mtk_mac *mac = netdev_priv(dev);
887 	struct mtk_eth *eth = mac->hw;
888 	struct mtk_tx_dma *itxd, *txd;
889 	struct mtk_tx_dma *itxd_pdma, *txd_pdma;
890 	struct mtk_tx_buf *itx_buf, *tx_buf;
891 	dma_addr_t mapped_addr;
892 	unsigned int nr_frags;
893 	int i, n_desc = 1;
894 	u32 txd4 = 0, fport;
895 	int k = 0;
896 
897 	itxd = ring->next_free;
898 	itxd_pdma = qdma_to_pdma(ring, itxd);
899 	if (itxd == ring->last_free)
900 		return -ENOMEM;
901 
902 	/* set the forward port */
903 	fport = (mac->id + 1) << TX_DMA_FPORT_SHIFT;
904 	txd4 |= fport;
905 
906 	itx_buf = mtk_desc_to_tx_buf(ring, itxd);
907 	memset(itx_buf, 0, sizeof(*itx_buf));
908 
909 	if (gso)
910 		txd4 |= TX_DMA_TSO;
911 
912 	/* TX Checksum offload */
913 	if (skb->ip_summed == CHECKSUM_PARTIAL)
914 		txd4 |= TX_DMA_CHKSUM;
915 
916 	/* VLAN header offload */
917 	if (skb_vlan_tag_present(skb))
918 		txd4 |= TX_DMA_INS_VLAN | skb_vlan_tag_get(skb);
919 
920 	mapped_addr = dma_map_single(eth->dev, skb->data,
921 				     skb_headlen(skb), DMA_TO_DEVICE);
922 	if (unlikely(dma_mapping_error(eth->dev, mapped_addr)))
923 		return -ENOMEM;
924 
925 	WRITE_ONCE(itxd->txd1, mapped_addr);
926 	itx_buf->flags |= MTK_TX_FLAGS_SINGLE0;
927 	itx_buf->flags |= (!mac->id) ? MTK_TX_FLAGS_FPORT0 :
928 			  MTK_TX_FLAGS_FPORT1;
929 	setup_tx_buf(eth, itx_buf, itxd_pdma, mapped_addr, skb_headlen(skb),
930 		     k++);
931 
932 	/* TX SG offload */
933 	txd = itxd;
934 	txd_pdma = qdma_to_pdma(ring, txd);
935 	nr_frags = skb_shinfo(skb)->nr_frags;
936 
937 	for (i = 0; i < nr_frags; i++) {
938 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
939 		unsigned int offset = 0;
940 		int frag_size = skb_frag_size(frag);
941 
942 		while (frag_size) {
943 			bool last_frag = false;
944 			unsigned int frag_map_size;
945 			bool new_desc = true;
946 
947 			if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA) ||
948 			    (i & 0x1)) {
949 				txd = mtk_qdma_phys_to_virt(ring, txd->txd2);
950 				txd_pdma = qdma_to_pdma(ring, txd);
951 				if (txd == ring->last_free)
952 					goto err_dma;
953 
954 				n_desc++;
955 			} else {
956 				new_desc = false;
957 			}
958 
959 
960 			frag_map_size = min(frag_size, MTK_TX_DMA_BUF_LEN);
961 			mapped_addr = skb_frag_dma_map(eth->dev, frag, offset,
962 						       frag_map_size,
963 						       DMA_TO_DEVICE);
964 			if (unlikely(dma_mapping_error(eth->dev, mapped_addr)))
965 				goto err_dma;
966 
967 			if (i == nr_frags - 1 &&
968 			    (frag_size - frag_map_size) == 0)
969 				last_frag = true;
970 
971 			WRITE_ONCE(txd->txd1, mapped_addr);
972 			WRITE_ONCE(txd->txd3, (TX_DMA_SWC |
973 					       TX_DMA_PLEN0(frag_map_size) |
974 					       last_frag * TX_DMA_LS0));
975 			WRITE_ONCE(txd->txd4, fport);
976 
977 			tx_buf = mtk_desc_to_tx_buf(ring, txd);
978 			if (new_desc)
979 				memset(tx_buf, 0, sizeof(*tx_buf));
980 			tx_buf->skb = (struct sk_buff *)MTK_DMA_DUMMY_DESC;
981 			tx_buf->flags |= MTK_TX_FLAGS_PAGE0;
982 			tx_buf->flags |= (!mac->id) ? MTK_TX_FLAGS_FPORT0 :
983 					 MTK_TX_FLAGS_FPORT1;
984 
985 			setup_tx_buf(eth, tx_buf, txd_pdma, mapped_addr,
986 				     frag_map_size, k++);
987 
988 			frag_size -= frag_map_size;
989 			offset += frag_map_size;
990 		}
991 	}
992 
993 	/* store skb to cleanup */
994 	itx_buf->skb = skb;
995 
996 	WRITE_ONCE(itxd->txd4, txd4);
997 	WRITE_ONCE(itxd->txd3, (TX_DMA_SWC | TX_DMA_PLEN0(skb_headlen(skb)) |
998 				(!nr_frags * TX_DMA_LS0)));
999 	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1000 		if (k & 0x1)
1001 			txd_pdma->txd2 |= TX_DMA_LS0;
1002 		else
1003 			txd_pdma->txd2 |= TX_DMA_LS1;
1004 	}
1005 
1006 	netdev_sent_queue(dev, skb->len);
1007 	skb_tx_timestamp(skb);
1008 
1009 	ring->next_free = mtk_qdma_phys_to_virt(ring, txd->txd2);
1010 	atomic_sub(n_desc, &ring->free_count);
1011 
1012 	/* make sure that all changes to the dma ring are flushed before we
1013 	 * continue
1014 	 */
1015 	wmb();
1016 
1017 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1018 		if (netif_xmit_stopped(netdev_get_tx_queue(dev, 0)) ||
1019 		    !netdev_xmit_more())
1020 			mtk_w32(eth, txd->txd2, MTK_QTX_CTX_PTR);
1021 	} else {
1022 		int next_idx = NEXT_DESP_IDX(txd_to_idx(ring, txd),
1023 					     ring->dma_size);
1024 		mtk_w32(eth, next_idx, MT7628_TX_CTX_IDX0);
1025 	}
1026 
1027 	return 0;
1028 
1029 err_dma:
1030 	do {
1031 		tx_buf = mtk_desc_to_tx_buf(ring, itxd);
1032 
1033 		/* unmap dma */
1034 		mtk_tx_unmap(eth, tx_buf);
1035 
1036 		itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
1037 		if (!MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
1038 			itxd_pdma->txd2 = TX_DMA_DESP2_DEF;
1039 
1040 		itxd = mtk_qdma_phys_to_virt(ring, itxd->txd2);
1041 		itxd_pdma = qdma_to_pdma(ring, itxd);
1042 	} while (itxd != txd);
1043 
1044 	return -ENOMEM;
1045 }
1046 
1047 static inline int mtk_cal_txd_req(struct sk_buff *skb)
1048 {
1049 	int i, nfrags;
1050 	skb_frag_t *frag;
1051 
1052 	nfrags = 1;
1053 	if (skb_is_gso(skb)) {
1054 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1055 			frag = &skb_shinfo(skb)->frags[i];
1056 			nfrags += DIV_ROUND_UP(skb_frag_size(frag),
1057 						MTK_TX_DMA_BUF_LEN);
1058 		}
1059 	} else {
1060 		nfrags += skb_shinfo(skb)->nr_frags;
1061 	}
1062 
1063 	return nfrags;
1064 }
1065 
1066 static int mtk_queue_stopped(struct mtk_eth *eth)
1067 {
1068 	int i;
1069 
1070 	for (i = 0; i < MTK_MAC_COUNT; i++) {
1071 		if (!eth->netdev[i])
1072 			continue;
1073 		if (netif_queue_stopped(eth->netdev[i]))
1074 			return 1;
1075 	}
1076 
1077 	return 0;
1078 }
1079 
1080 static void mtk_wake_queue(struct mtk_eth *eth)
1081 {
1082 	int i;
1083 
1084 	for (i = 0; i < MTK_MAC_COUNT; i++) {
1085 		if (!eth->netdev[i])
1086 			continue;
1087 		netif_wake_queue(eth->netdev[i]);
1088 	}
1089 }
1090 
1091 static void mtk_stop_queue(struct mtk_eth *eth)
1092 {
1093 	int i;
1094 
1095 	for (i = 0; i < MTK_MAC_COUNT; i++) {
1096 		if (!eth->netdev[i])
1097 			continue;
1098 		netif_stop_queue(eth->netdev[i]);
1099 	}
1100 }
1101 
1102 static int mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
1103 {
1104 	struct mtk_mac *mac = netdev_priv(dev);
1105 	struct mtk_eth *eth = mac->hw;
1106 	struct mtk_tx_ring *ring = &eth->tx_ring;
1107 	struct net_device_stats *stats = &dev->stats;
1108 	bool gso = false;
1109 	int tx_num;
1110 
1111 	/* normally we can rely on the stack not calling this more than once,
1112 	 * however we have 2 queues running on the same ring so we need to lock
1113 	 * the ring access
1114 	 */
1115 	spin_lock(&eth->page_lock);
1116 
1117 	if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
1118 		goto drop;
1119 
1120 	tx_num = mtk_cal_txd_req(skb);
1121 	if (unlikely(atomic_read(&ring->free_count) <= tx_num)) {
1122 		mtk_stop_queue(eth);
1123 		netif_err(eth, tx_queued, dev,
1124 			  "Tx Ring full when queue awake!\n");
1125 		spin_unlock(&eth->page_lock);
1126 		return NETDEV_TX_BUSY;
1127 	}
1128 
1129 	/* TSO: fill MSS info in tcp checksum field */
1130 	if (skb_is_gso(skb)) {
1131 		if (skb_cow_head(skb, 0)) {
1132 			netif_warn(eth, tx_err, dev,
1133 				   "GSO expand head fail.\n");
1134 			goto drop;
1135 		}
1136 
1137 		if (skb_shinfo(skb)->gso_type &
1138 				(SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) {
1139 			gso = true;
1140 			tcp_hdr(skb)->check = htons(skb_shinfo(skb)->gso_size);
1141 		}
1142 	}
1143 
1144 	if (mtk_tx_map(skb, dev, tx_num, ring, gso) < 0)
1145 		goto drop;
1146 
1147 	if (unlikely(atomic_read(&ring->free_count) <= ring->thresh))
1148 		mtk_stop_queue(eth);
1149 
1150 	spin_unlock(&eth->page_lock);
1151 
1152 	return NETDEV_TX_OK;
1153 
1154 drop:
1155 	spin_unlock(&eth->page_lock);
1156 	stats->tx_dropped++;
1157 	dev_kfree_skb_any(skb);
1158 	return NETDEV_TX_OK;
1159 }
1160 
1161 static struct mtk_rx_ring *mtk_get_rx_ring(struct mtk_eth *eth)
1162 {
1163 	int i;
1164 	struct mtk_rx_ring *ring;
1165 	int idx;
1166 
1167 	if (!eth->hwlro)
1168 		return &eth->rx_ring[0];
1169 
1170 	for (i = 0; i < MTK_MAX_RX_RING_NUM; i++) {
1171 		ring = &eth->rx_ring[i];
1172 		idx = NEXT_DESP_IDX(ring->calc_idx, ring->dma_size);
1173 		if (ring->dma[idx].rxd2 & RX_DMA_DONE) {
1174 			ring->calc_idx_update = true;
1175 			return ring;
1176 		}
1177 	}
1178 
1179 	return NULL;
1180 }
1181 
1182 static void mtk_update_rx_cpu_idx(struct mtk_eth *eth)
1183 {
1184 	struct mtk_rx_ring *ring;
1185 	int i;
1186 
1187 	if (!eth->hwlro) {
1188 		ring = &eth->rx_ring[0];
1189 		mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
1190 	} else {
1191 		for (i = 0; i < MTK_MAX_RX_RING_NUM; i++) {
1192 			ring = &eth->rx_ring[i];
1193 			if (ring->calc_idx_update) {
1194 				ring->calc_idx_update = false;
1195 				mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
1196 			}
1197 		}
1198 	}
1199 }
1200 
1201 static int mtk_poll_rx(struct napi_struct *napi, int budget,
1202 		       struct mtk_eth *eth)
1203 {
1204 	struct mtk_rx_ring *ring;
1205 	int idx;
1206 	struct sk_buff *skb;
1207 	u8 *data, *new_data;
1208 	struct mtk_rx_dma *rxd, trxd;
1209 	int done = 0;
1210 
1211 	while (done < budget) {
1212 		struct net_device *netdev;
1213 		unsigned int pktlen;
1214 		dma_addr_t dma_addr;
1215 		int mac;
1216 
1217 		ring = mtk_get_rx_ring(eth);
1218 		if (unlikely(!ring))
1219 			goto rx_done;
1220 
1221 		idx = NEXT_DESP_IDX(ring->calc_idx, ring->dma_size);
1222 		rxd = &ring->dma[idx];
1223 		data = ring->data[idx];
1224 
1225 		mtk_rx_get_desc(&trxd, rxd);
1226 		if (!(trxd.rxd2 & RX_DMA_DONE))
1227 			break;
1228 
1229 		/* find out which mac the packet come from. values start at 1 */
1230 		if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
1231 			mac = 0;
1232 		} else {
1233 			mac = (trxd.rxd4 >> RX_DMA_FPORT_SHIFT) &
1234 				RX_DMA_FPORT_MASK;
1235 			mac--;
1236 		}
1237 
1238 		if (unlikely(mac < 0 || mac >= MTK_MAC_COUNT ||
1239 			     !eth->netdev[mac]))
1240 			goto release_desc;
1241 
1242 		netdev = eth->netdev[mac];
1243 
1244 		if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
1245 			goto release_desc;
1246 
1247 		/* alloc new buffer */
1248 		new_data = napi_alloc_frag(ring->frag_size);
1249 		if (unlikely(!new_data)) {
1250 			netdev->stats.rx_dropped++;
1251 			goto release_desc;
1252 		}
1253 		dma_addr = dma_map_single(eth->dev,
1254 					  new_data + NET_SKB_PAD +
1255 					  eth->ip_align,
1256 					  ring->buf_size,
1257 					  DMA_FROM_DEVICE);
1258 		if (unlikely(dma_mapping_error(eth->dev, dma_addr))) {
1259 			skb_free_frag(new_data);
1260 			netdev->stats.rx_dropped++;
1261 			goto release_desc;
1262 		}
1263 
1264 		/* receive data */
1265 		skb = build_skb(data, ring->frag_size);
1266 		if (unlikely(!skb)) {
1267 			skb_free_frag(new_data);
1268 			netdev->stats.rx_dropped++;
1269 			goto release_desc;
1270 		}
1271 		skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1272 
1273 		dma_unmap_single(eth->dev, trxd.rxd1,
1274 				 ring->buf_size, DMA_FROM_DEVICE);
1275 		pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
1276 		skb->dev = netdev;
1277 		skb_put(skb, pktlen);
1278 		if (trxd.rxd4 & eth->rx_dma_l4_valid)
1279 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1280 		else
1281 			skb_checksum_none_assert(skb);
1282 		skb->protocol = eth_type_trans(skb, netdev);
1283 
1284 		if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX &&
1285 		    RX_DMA_VID(trxd.rxd3))
1286 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1287 					       RX_DMA_VID(trxd.rxd3));
1288 		skb_record_rx_queue(skb, 0);
1289 		napi_gro_receive(napi, skb);
1290 
1291 		ring->data[idx] = new_data;
1292 		rxd->rxd1 = (unsigned int)dma_addr;
1293 
1294 release_desc:
1295 		if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
1296 			rxd->rxd2 = RX_DMA_LSO;
1297 		else
1298 			rxd->rxd2 = RX_DMA_PLEN0(ring->buf_size);
1299 
1300 		ring->calc_idx = idx;
1301 
1302 		done++;
1303 	}
1304 
1305 rx_done:
1306 	if (done) {
1307 		/* make sure that all changes to the dma ring are flushed before
1308 		 * we continue
1309 		 */
1310 		wmb();
1311 		mtk_update_rx_cpu_idx(eth);
1312 	}
1313 
1314 	return done;
1315 }
1316 
1317 static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget,
1318 			    unsigned int *done, unsigned int *bytes)
1319 {
1320 	struct mtk_tx_ring *ring = &eth->tx_ring;
1321 	struct mtk_tx_dma *desc;
1322 	struct sk_buff *skb;
1323 	struct mtk_tx_buf *tx_buf;
1324 	u32 cpu, dma;
1325 
1326 	cpu = mtk_r32(eth, MTK_QTX_CRX_PTR);
1327 	dma = mtk_r32(eth, MTK_QTX_DRX_PTR);
1328 
1329 	desc = mtk_qdma_phys_to_virt(ring, cpu);
1330 
1331 	while ((cpu != dma) && budget) {
1332 		u32 next_cpu = desc->txd2;
1333 		int mac = 0;
1334 
1335 		desc = mtk_qdma_phys_to_virt(ring, desc->txd2);
1336 		if ((desc->txd3 & TX_DMA_OWNER_CPU) == 0)
1337 			break;
1338 
1339 		tx_buf = mtk_desc_to_tx_buf(ring, desc);
1340 		if (tx_buf->flags & MTK_TX_FLAGS_FPORT1)
1341 			mac = 1;
1342 
1343 		skb = tx_buf->skb;
1344 		if (!skb)
1345 			break;
1346 
1347 		if (skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC) {
1348 			bytes[mac] += skb->len;
1349 			done[mac]++;
1350 			budget--;
1351 		}
1352 		mtk_tx_unmap(eth, tx_buf);
1353 
1354 		ring->last_free = desc;
1355 		atomic_inc(&ring->free_count);
1356 
1357 		cpu = next_cpu;
1358 	}
1359 
1360 	mtk_w32(eth, cpu, MTK_QTX_CRX_PTR);
1361 
1362 	return budget;
1363 }
1364 
1365 static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget,
1366 			    unsigned int *done, unsigned int *bytes)
1367 {
1368 	struct mtk_tx_ring *ring = &eth->tx_ring;
1369 	struct mtk_tx_dma *desc;
1370 	struct sk_buff *skb;
1371 	struct mtk_tx_buf *tx_buf;
1372 	u32 cpu, dma;
1373 
1374 	cpu = ring->cpu_idx;
1375 	dma = mtk_r32(eth, MT7628_TX_DTX_IDX0);
1376 
1377 	while ((cpu != dma) && budget) {
1378 		tx_buf = &ring->buf[cpu];
1379 		skb = tx_buf->skb;
1380 		if (!skb)
1381 			break;
1382 
1383 		if (skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC) {
1384 			bytes[0] += skb->len;
1385 			done[0]++;
1386 			budget--;
1387 		}
1388 
1389 		mtk_tx_unmap(eth, tx_buf);
1390 
1391 		desc = &ring->dma[cpu];
1392 		ring->last_free = desc;
1393 		atomic_inc(&ring->free_count);
1394 
1395 		cpu = NEXT_DESP_IDX(cpu, ring->dma_size);
1396 	}
1397 
1398 	ring->cpu_idx = cpu;
1399 
1400 	return budget;
1401 }
1402 
1403 static int mtk_poll_tx(struct mtk_eth *eth, int budget)
1404 {
1405 	struct mtk_tx_ring *ring = &eth->tx_ring;
1406 	unsigned int done[MTK_MAX_DEVS];
1407 	unsigned int bytes[MTK_MAX_DEVS];
1408 	int total = 0, i;
1409 
1410 	memset(done, 0, sizeof(done));
1411 	memset(bytes, 0, sizeof(bytes));
1412 
1413 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
1414 		budget = mtk_poll_tx_qdma(eth, budget, done, bytes);
1415 	else
1416 		budget = mtk_poll_tx_pdma(eth, budget, done, bytes);
1417 
1418 	for (i = 0; i < MTK_MAC_COUNT; i++) {
1419 		if (!eth->netdev[i] || !done[i])
1420 			continue;
1421 		netdev_completed_queue(eth->netdev[i], done[i], bytes[i]);
1422 		total += done[i];
1423 	}
1424 
1425 	if (mtk_queue_stopped(eth) &&
1426 	    (atomic_read(&ring->free_count) > ring->thresh))
1427 		mtk_wake_queue(eth);
1428 
1429 	return total;
1430 }
1431 
1432 static void mtk_handle_status_irq(struct mtk_eth *eth)
1433 {
1434 	u32 status2 = mtk_r32(eth, MTK_INT_STATUS2);
1435 
1436 	if (unlikely(status2 & (MTK_GDM1_AF | MTK_GDM2_AF))) {
1437 		mtk_stats_update(eth);
1438 		mtk_w32(eth, (MTK_GDM1_AF | MTK_GDM2_AF),
1439 			MTK_INT_STATUS2);
1440 	}
1441 }
1442 
1443 static int mtk_napi_tx(struct napi_struct *napi, int budget)
1444 {
1445 	struct mtk_eth *eth = container_of(napi, struct mtk_eth, tx_napi);
1446 	u32 status, mask;
1447 	int tx_done = 0;
1448 
1449 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
1450 		mtk_handle_status_irq(eth);
1451 	mtk_w32(eth, MTK_TX_DONE_INT, eth->tx_int_status_reg);
1452 	tx_done = mtk_poll_tx(eth, budget);
1453 
1454 	if (unlikely(netif_msg_intr(eth))) {
1455 		status = mtk_r32(eth, eth->tx_int_status_reg);
1456 		mask = mtk_r32(eth, eth->tx_int_mask_reg);
1457 		dev_info(eth->dev,
1458 			 "done tx %d, intr 0x%08x/0x%x\n",
1459 			 tx_done, status, mask);
1460 	}
1461 
1462 	if (tx_done == budget)
1463 		return budget;
1464 
1465 	status = mtk_r32(eth, eth->tx_int_status_reg);
1466 	if (status & MTK_TX_DONE_INT)
1467 		return budget;
1468 
1469 	napi_complete(napi);
1470 	mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
1471 
1472 	return tx_done;
1473 }
1474 
1475 static int mtk_napi_rx(struct napi_struct *napi, int budget)
1476 {
1477 	struct mtk_eth *eth = container_of(napi, struct mtk_eth, rx_napi);
1478 	u32 status, mask;
1479 	int rx_done = 0;
1480 	int remain_budget = budget;
1481 
1482 	mtk_handle_status_irq(eth);
1483 
1484 poll_again:
1485 	mtk_w32(eth, MTK_RX_DONE_INT, MTK_PDMA_INT_STATUS);
1486 	rx_done = mtk_poll_rx(napi, remain_budget, eth);
1487 
1488 	if (unlikely(netif_msg_intr(eth))) {
1489 		status = mtk_r32(eth, MTK_PDMA_INT_STATUS);
1490 		mask = mtk_r32(eth, MTK_PDMA_INT_MASK);
1491 		dev_info(eth->dev,
1492 			 "done rx %d, intr 0x%08x/0x%x\n",
1493 			 rx_done, status, mask);
1494 	}
1495 	if (rx_done == remain_budget)
1496 		return budget;
1497 
1498 	status = mtk_r32(eth, MTK_PDMA_INT_STATUS);
1499 	if (status & MTK_RX_DONE_INT) {
1500 		remain_budget -= rx_done;
1501 		goto poll_again;
1502 	}
1503 	napi_complete(napi);
1504 	mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
1505 
1506 	return rx_done + budget - remain_budget;
1507 }
1508 
1509 static int mtk_tx_alloc(struct mtk_eth *eth)
1510 {
1511 	struct mtk_tx_ring *ring = &eth->tx_ring;
1512 	int i, sz = sizeof(*ring->dma);
1513 
1514 	ring->buf = kcalloc(MTK_DMA_SIZE, sizeof(*ring->buf),
1515 			       GFP_KERNEL);
1516 	if (!ring->buf)
1517 		goto no_tx_mem;
1518 
1519 	ring->dma = dma_alloc_coherent(eth->dev, MTK_DMA_SIZE * sz,
1520 				       &ring->phys, GFP_ATOMIC);
1521 	if (!ring->dma)
1522 		goto no_tx_mem;
1523 
1524 	for (i = 0; i < MTK_DMA_SIZE; i++) {
1525 		int next = (i + 1) % MTK_DMA_SIZE;
1526 		u32 next_ptr = ring->phys + next * sz;
1527 
1528 		ring->dma[i].txd2 = next_ptr;
1529 		ring->dma[i].txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
1530 	}
1531 
1532 	/* On MT7688 (PDMA only) this driver uses the ring->dma structs
1533 	 * only as the framework. The real HW descriptors are the PDMA
1534 	 * descriptors in ring->dma_pdma.
1535 	 */
1536 	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1537 		ring->dma_pdma = dma_alloc_coherent(eth->dev, MTK_DMA_SIZE * sz,
1538 						    &ring->phys_pdma,
1539 						    GFP_ATOMIC);
1540 		if (!ring->dma_pdma)
1541 			goto no_tx_mem;
1542 
1543 		for (i = 0; i < MTK_DMA_SIZE; i++) {
1544 			ring->dma_pdma[i].txd2 = TX_DMA_DESP2_DEF;
1545 			ring->dma_pdma[i].txd4 = 0;
1546 		}
1547 	}
1548 
1549 	ring->dma_size = MTK_DMA_SIZE;
1550 	atomic_set(&ring->free_count, MTK_DMA_SIZE - 2);
1551 	ring->next_free = &ring->dma[0];
1552 	ring->last_free = &ring->dma[MTK_DMA_SIZE - 1];
1553 	ring->thresh = MAX_SKB_FRAGS;
1554 
1555 	/* make sure that all changes to the dma ring are flushed before we
1556 	 * continue
1557 	 */
1558 	wmb();
1559 
1560 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1561 		mtk_w32(eth, ring->phys, MTK_QTX_CTX_PTR);
1562 		mtk_w32(eth, ring->phys, MTK_QTX_DTX_PTR);
1563 		mtk_w32(eth,
1564 			ring->phys + ((MTK_DMA_SIZE - 1) * sz),
1565 			MTK_QTX_CRX_PTR);
1566 		mtk_w32(eth,
1567 			ring->phys + ((MTK_DMA_SIZE - 1) * sz),
1568 			MTK_QTX_DRX_PTR);
1569 		mtk_w32(eth, (QDMA_RES_THRES << 8) | QDMA_RES_THRES,
1570 			MTK_QTX_CFG(0));
1571 	} else {
1572 		mtk_w32(eth, ring->phys_pdma, MT7628_TX_BASE_PTR0);
1573 		mtk_w32(eth, MTK_DMA_SIZE, MT7628_TX_MAX_CNT0);
1574 		mtk_w32(eth, 0, MT7628_TX_CTX_IDX0);
1575 		mtk_w32(eth, MT7628_PST_DTX_IDX0, MTK_PDMA_RST_IDX);
1576 	}
1577 
1578 	return 0;
1579 
1580 no_tx_mem:
1581 	return -ENOMEM;
1582 }
1583 
1584 static void mtk_tx_clean(struct mtk_eth *eth)
1585 {
1586 	struct mtk_tx_ring *ring = &eth->tx_ring;
1587 	int i;
1588 
1589 	if (ring->buf) {
1590 		for (i = 0; i < MTK_DMA_SIZE; i++)
1591 			mtk_tx_unmap(eth, &ring->buf[i]);
1592 		kfree(ring->buf);
1593 		ring->buf = NULL;
1594 	}
1595 
1596 	if (ring->dma) {
1597 		dma_free_coherent(eth->dev,
1598 				  MTK_DMA_SIZE * sizeof(*ring->dma),
1599 				  ring->dma,
1600 				  ring->phys);
1601 		ring->dma = NULL;
1602 	}
1603 
1604 	if (ring->dma_pdma) {
1605 		dma_free_coherent(eth->dev,
1606 				  MTK_DMA_SIZE * sizeof(*ring->dma_pdma),
1607 				  ring->dma_pdma,
1608 				  ring->phys_pdma);
1609 		ring->dma_pdma = NULL;
1610 	}
1611 }
1612 
1613 static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
1614 {
1615 	struct mtk_rx_ring *ring;
1616 	int rx_data_len, rx_dma_size;
1617 	int i;
1618 	u32 offset = 0;
1619 
1620 	if (rx_flag == MTK_RX_FLAGS_QDMA) {
1621 		if (ring_no)
1622 			return -EINVAL;
1623 		ring = &eth->rx_ring_qdma;
1624 		offset = 0x1000;
1625 	} else {
1626 		ring = &eth->rx_ring[ring_no];
1627 	}
1628 
1629 	if (rx_flag == MTK_RX_FLAGS_HWLRO) {
1630 		rx_data_len = MTK_MAX_LRO_RX_LENGTH;
1631 		rx_dma_size = MTK_HW_LRO_DMA_SIZE;
1632 	} else {
1633 		rx_data_len = ETH_DATA_LEN;
1634 		rx_dma_size = MTK_DMA_SIZE;
1635 	}
1636 
1637 	ring->frag_size = mtk_max_frag_size(rx_data_len);
1638 	ring->buf_size = mtk_max_buf_size(ring->frag_size);
1639 	ring->data = kcalloc(rx_dma_size, sizeof(*ring->data),
1640 			     GFP_KERNEL);
1641 	if (!ring->data)
1642 		return -ENOMEM;
1643 
1644 	for (i = 0; i < rx_dma_size; i++) {
1645 		ring->data[i] = netdev_alloc_frag(ring->frag_size);
1646 		if (!ring->data[i])
1647 			return -ENOMEM;
1648 	}
1649 
1650 	ring->dma = dma_alloc_coherent(eth->dev,
1651 				       rx_dma_size * sizeof(*ring->dma),
1652 				       &ring->phys, GFP_ATOMIC);
1653 	if (!ring->dma)
1654 		return -ENOMEM;
1655 
1656 	for (i = 0; i < rx_dma_size; i++) {
1657 		dma_addr_t dma_addr = dma_map_single(eth->dev,
1658 				ring->data[i] + NET_SKB_PAD + eth->ip_align,
1659 				ring->buf_size,
1660 				DMA_FROM_DEVICE);
1661 		if (unlikely(dma_mapping_error(eth->dev, dma_addr)))
1662 			return -ENOMEM;
1663 		ring->dma[i].rxd1 = (unsigned int)dma_addr;
1664 
1665 		if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
1666 			ring->dma[i].rxd2 = RX_DMA_LSO;
1667 		else
1668 			ring->dma[i].rxd2 = RX_DMA_PLEN0(ring->buf_size);
1669 	}
1670 	ring->dma_size = rx_dma_size;
1671 	ring->calc_idx_update = false;
1672 	ring->calc_idx = rx_dma_size - 1;
1673 	ring->crx_idx_reg = MTK_PRX_CRX_IDX_CFG(ring_no);
1674 	/* make sure that all changes to the dma ring are flushed before we
1675 	 * continue
1676 	 */
1677 	wmb();
1678 
1679 	mtk_w32(eth, ring->phys, MTK_PRX_BASE_PTR_CFG(ring_no) + offset);
1680 	mtk_w32(eth, rx_dma_size, MTK_PRX_MAX_CNT_CFG(ring_no) + offset);
1681 	mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg + offset);
1682 	mtk_w32(eth, MTK_PST_DRX_IDX_CFG(ring_no), MTK_PDMA_RST_IDX + offset);
1683 
1684 	return 0;
1685 }
1686 
1687 static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring)
1688 {
1689 	int i;
1690 
1691 	if (ring->data && ring->dma) {
1692 		for (i = 0; i < ring->dma_size; i++) {
1693 			if (!ring->data[i])
1694 				continue;
1695 			if (!ring->dma[i].rxd1)
1696 				continue;
1697 			dma_unmap_single(eth->dev,
1698 					 ring->dma[i].rxd1,
1699 					 ring->buf_size,
1700 					 DMA_FROM_DEVICE);
1701 			skb_free_frag(ring->data[i]);
1702 		}
1703 		kfree(ring->data);
1704 		ring->data = NULL;
1705 	}
1706 
1707 	if (ring->dma) {
1708 		dma_free_coherent(eth->dev,
1709 				  ring->dma_size * sizeof(*ring->dma),
1710 				  ring->dma,
1711 				  ring->phys);
1712 		ring->dma = NULL;
1713 	}
1714 }
1715 
1716 static int mtk_hwlro_rx_init(struct mtk_eth *eth)
1717 {
1718 	int i;
1719 	u32 ring_ctrl_dw1 = 0, ring_ctrl_dw2 = 0, ring_ctrl_dw3 = 0;
1720 	u32 lro_ctrl_dw0 = 0, lro_ctrl_dw3 = 0;
1721 
1722 	/* set LRO rings to auto-learn modes */
1723 	ring_ctrl_dw2 |= MTK_RING_AUTO_LERAN_MODE;
1724 
1725 	/* validate LRO ring */
1726 	ring_ctrl_dw2 |= MTK_RING_VLD;
1727 
1728 	/* set AGE timer (unit: 20us) */
1729 	ring_ctrl_dw2 |= MTK_RING_AGE_TIME_H;
1730 	ring_ctrl_dw1 |= MTK_RING_AGE_TIME_L;
1731 
1732 	/* set max AGG timer (unit: 20us) */
1733 	ring_ctrl_dw2 |= MTK_RING_MAX_AGG_TIME;
1734 
1735 	/* set max LRO AGG count */
1736 	ring_ctrl_dw2 |= MTK_RING_MAX_AGG_CNT_L;
1737 	ring_ctrl_dw3 |= MTK_RING_MAX_AGG_CNT_H;
1738 
1739 	for (i = 1; i < MTK_MAX_RX_RING_NUM; i++) {
1740 		mtk_w32(eth, ring_ctrl_dw1, MTK_LRO_CTRL_DW1_CFG(i));
1741 		mtk_w32(eth, ring_ctrl_dw2, MTK_LRO_CTRL_DW2_CFG(i));
1742 		mtk_w32(eth, ring_ctrl_dw3, MTK_LRO_CTRL_DW3_CFG(i));
1743 	}
1744 
1745 	/* IPv4 checksum update enable */
1746 	lro_ctrl_dw0 |= MTK_L3_CKS_UPD_EN;
1747 
1748 	/* switch priority comparison to packet count mode */
1749 	lro_ctrl_dw0 |= MTK_LRO_ALT_PKT_CNT_MODE;
1750 
1751 	/* bandwidth threshold setting */
1752 	mtk_w32(eth, MTK_HW_LRO_BW_THRE, MTK_PDMA_LRO_CTRL_DW2);
1753 
1754 	/* auto-learn score delta setting */
1755 	mtk_w32(eth, MTK_HW_LRO_REPLACE_DELTA, MTK_PDMA_LRO_ALT_SCORE_DELTA);
1756 
1757 	/* set refresh timer for altering flows to 1 sec. (unit: 20us) */
1758 	mtk_w32(eth, (MTK_HW_LRO_TIMER_UNIT << 16) | MTK_HW_LRO_REFRESH_TIME,
1759 		MTK_PDMA_LRO_ALT_REFRESH_TIMER);
1760 
1761 	/* set HW LRO mode & the max aggregation count for rx packets */
1762 	lro_ctrl_dw3 |= MTK_ADMA_MODE | (MTK_HW_LRO_MAX_AGG_CNT & 0xff);
1763 
1764 	/* the minimal remaining room of SDL0 in RXD for lro aggregation */
1765 	lro_ctrl_dw3 |= MTK_LRO_MIN_RXD_SDL;
1766 
1767 	/* enable HW LRO */
1768 	lro_ctrl_dw0 |= MTK_LRO_EN;
1769 
1770 	mtk_w32(eth, lro_ctrl_dw3, MTK_PDMA_LRO_CTRL_DW3);
1771 	mtk_w32(eth, lro_ctrl_dw0, MTK_PDMA_LRO_CTRL_DW0);
1772 
1773 	return 0;
1774 }
1775 
1776 static void mtk_hwlro_rx_uninit(struct mtk_eth *eth)
1777 {
1778 	int i;
1779 	u32 val;
1780 
1781 	/* relinquish lro rings, flush aggregated packets */
1782 	mtk_w32(eth, MTK_LRO_RING_RELINQUISH_REQ, MTK_PDMA_LRO_CTRL_DW0);
1783 
1784 	/* wait for relinquishments done */
1785 	for (i = 0; i < 10; i++) {
1786 		val = mtk_r32(eth, MTK_PDMA_LRO_CTRL_DW0);
1787 		if (val & MTK_LRO_RING_RELINQUISH_DONE) {
1788 			msleep(20);
1789 			continue;
1790 		}
1791 		break;
1792 	}
1793 
1794 	/* invalidate lro rings */
1795 	for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
1796 		mtk_w32(eth, 0, MTK_LRO_CTRL_DW2_CFG(i));
1797 
1798 	/* disable HW LRO */
1799 	mtk_w32(eth, 0, MTK_PDMA_LRO_CTRL_DW0);
1800 }
1801 
1802 static void mtk_hwlro_val_ipaddr(struct mtk_eth *eth, int idx, __be32 ip)
1803 {
1804 	u32 reg_val;
1805 
1806 	reg_val = mtk_r32(eth, MTK_LRO_CTRL_DW2_CFG(idx));
1807 
1808 	/* invalidate the IP setting */
1809 	mtk_w32(eth, (reg_val & ~MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
1810 
1811 	mtk_w32(eth, ip, MTK_LRO_DIP_DW0_CFG(idx));
1812 
1813 	/* validate the IP setting */
1814 	mtk_w32(eth, (reg_val | MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
1815 }
1816 
1817 static void mtk_hwlro_inval_ipaddr(struct mtk_eth *eth, int idx)
1818 {
1819 	u32 reg_val;
1820 
1821 	reg_val = mtk_r32(eth, MTK_LRO_CTRL_DW2_CFG(idx));
1822 
1823 	/* invalidate the IP setting */
1824 	mtk_w32(eth, (reg_val & ~MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
1825 
1826 	mtk_w32(eth, 0, MTK_LRO_DIP_DW0_CFG(idx));
1827 }
1828 
1829 static int mtk_hwlro_get_ip_cnt(struct mtk_mac *mac)
1830 {
1831 	int cnt = 0;
1832 	int i;
1833 
1834 	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
1835 		if (mac->hwlro_ip[i])
1836 			cnt++;
1837 	}
1838 
1839 	return cnt;
1840 }
1841 
1842 static int mtk_hwlro_add_ipaddr(struct net_device *dev,
1843 				struct ethtool_rxnfc *cmd)
1844 {
1845 	struct ethtool_rx_flow_spec *fsp =
1846 		(struct ethtool_rx_flow_spec *)&cmd->fs;
1847 	struct mtk_mac *mac = netdev_priv(dev);
1848 	struct mtk_eth *eth = mac->hw;
1849 	int hwlro_idx;
1850 
1851 	if ((fsp->flow_type != TCP_V4_FLOW) ||
1852 	    (!fsp->h_u.tcp_ip4_spec.ip4dst) ||
1853 	    (fsp->location > 1))
1854 		return -EINVAL;
1855 
1856 	mac->hwlro_ip[fsp->location] = htonl(fsp->h_u.tcp_ip4_spec.ip4dst);
1857 	hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + fsp->location;
1858 
1859 	mac->hwlro_ip_cnt = mtk_hwlro_get_ip_cnt(mac);
1860 
1861 	mtk_hwlro_val_ipaddr(eth, hwlro_idx, mac->hwlro_ip[fsp->location]);
1862 
1863 	return 0;
1864 }
1865 
1866 static int mtk_hwlro_del_ipaddr(struct net_device *dev,
1867 				struct ethtool_rxnfc *cmd)
1868 {
1869 	struct ethtool_rx_flow_spec *fsp =
1870 		(struct ethtool_rx_flow_spec *)&cmd->fs;
1871 	struct mtk_mac *mac = netdev_priv(dev);
1872 	struct mtk_eth *eth = mac->hw;
1873 	int hwlro_idx;
1874 
1875 	if (fsp->location > 1)
1876 		return -EINVAL;
1877 
1878 	mac->hwlro_ip[fsp->location] = 0;
1879 	hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + fsp->location;
1880 
1881 	mac->hwlro_ip_cnt = mtk_hwlro_get_ip_cnt(mac);
1882 
1883 	mtk_hwlro_inval_ipaddr(eth, hwlro_idx);
1884 
1885 	return 0;
1886 }
1887 
1888 static void mtk_hwlro_netdev_disable(struct net_device *dev)
1889 {
1890 	struct mtk_mac *mac = netdev_priv(dev);
1891 	struct mtk_eth *eth = mac->hw;
1892 	int i, hwlro_idx;
1893 
1894 	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
1895 		mac->hwlro_ip[i] = 0;
1896 		hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + i;
1897 
1898 		mtk_hwlro_inval_ipaddr(eth, hwlro_idx);
1899 	}
1900 
1901 	mac->hwlro_ip_cnt = 0;
1902 }
1903 
1904 static int mtk_hwlro_get_fdir_entry(struct net_device *dev,
1905 				    struct ethtool_rxnfc *cmd)
1906 {
1907 	struct mtk_mac *mac = netdev_priv(dev);
1908 	struct ethtool_rx_flow_spec *fsp =
1909 		(struct ethtool_rx_flow_spec *)&cmd->fs;
1910 
1911 	/* only tcp dst ipv4 is meaningful, others are meaningless */
1912 	fsp->flow_type = TCP_V4_FLOW;
1913 	fsp->h_u.tcp_ip4_spec.ip4dst = ntohl(mac->hwlro_ip[fsp->location]);
1914 	fsp->m_u.tcp_ip4_spec.ip4dst = 0;
1915 
1916 	fsp->h_u.tcp_ip4_spec.ip4src = 0;
1917 	fsp->m_u.tcp_ip4_spec.ip4src = 0xffffffff;
1918 	fsp->h_u.tcp_ip4_spec.psrc = 0;
1919 	fsp->m_u.tcp_ip4_spec.psrc = 0xffff;
1920 	fsp->h_u.tcp_ip4_spec.pdst = 0;
1921 	fsp->m_u.tcp_ip4_spec.pdst = 0xffff;
1922 	fsp->h_u.tcp_ip4_spec.tos = 0;
1923 	fsp->m_u.tcp_ip4_spec.tos = 0xff;
1924 
1925 	return 0;
1926 }
1927 
1928 static int mtk_hwlro_get_fdir_all(struct net_device *dev,
1929 				  struct ethtool_rxnfc *cmd,
1930 				  u32 *rule_locs)
1931 {
1932 	struct mtk_mac *mac = netdev_priv(dev);
1933 	int cnt = 0;
1934 	int i;
1935 
1936 	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
1937 		if (mac->hwlro_ip[i]) {
1938 			rule_locs[cnt] = i;
1939 			cnt++;
1940 		}
1941 	}
1942 
1943 	cmd->rule_cnt = cnt;
1944 
1945 	return 0;
1946 }
1947 
1948 static netdev_features_t mtk_fix_features(struct net_device *dev,
1949 					  netdev_features_t features)
1950 {
1951 	if (!(features & NETIF_F_LRO)) {
1952 		struct mtk_mac *mac = netdev_priv(dev);
1953 		int ip_cnt = mtk_hwlro_get_ip_cnt(mac);
1954 
1955 		if (ip_cnt) {
1956 			netdev_info(dev, "RX flow is programmed, LRO should keep on\n");
1957 
1958 			features |= NETIF_F_LRO;
1959 		}
1960 	}
1961 
1962 	return features;
1963 }
1964 
1965 static int mtk_set_features(struct net_device *dev, netdev_features_t features)
1966 {
1967 	int err = 0;
1968 
1969 	if (!((dev->features ^ features) & NETIF_F_LRO))
1970 		return 0;
1971 
1972 	if (!(features & NETIF_F_LRO))
1973 		mtk_hwlro_netdev_disable(dev);
1974 
1975 	return err;
1976 }
1977 
1978 /* wait for DMA to finish whatever it is doing before we start using it again */
1979 static int mtk_dma_busy_wait(struct mtk_eth *eth)
1980 {
1981 	unsigned long t_start = jiffies;
1982 
1983 	while (1) {
1984 		if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1985 			if (!(mtk_r32(eth, MTK_QDMA_GLO_CFG) &
1986 			      (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)))
1987 				return 0;
1988 		} else {
1989 			if (!(mtk_r32(eth, MTK_PDMA_GLO_CFG) &
1990 			      (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)))
1991 				return 0;
1992 		}
1993 
1994 		if (time_after(jiffies, t_start + MTK_DMA_BUSY_TIMEOUT))
1995 			break;
1996 	}
1997 
1998 	dev_err(eth->dev, "DMA init timeout\n");
1999 	return -1;
2000 }
2001 
2002 static int mtk_dma_init(struct mtk_eth *eth)
2003 {
2004 	int err;
2005 	u32 i;
2006 
2007 	if (mtk_dma_busy_wait(eth))
2008 		return -EBUSY;
2009 
2010 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2011 		/* QDMA needs scratch memory for internal reordering of the
2012 		 * descriptors
2013 		 */
2014 		err = mtk_init_fq_dma(eth);
2015 		if (err)
2016 			return err;
2017 	}
2018 
2019 	err = mtk_tx_alloc(eth);
2020 	if (err)
2021 		return err;
2022 
2023 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2024 		err = mtk_rx_alloc(eth, 0, MTK_RX_FLAGS_QDMA);
2025 		if (err)
2026 			return err;
2027 	}
2028 
2029 	err = mtk_rx_alloc(eth, 0, MTK_RX_FLAGS_NORMAL);
2030 	if (err)
2031 		return err;
2032 
2033 	if (eth->hwlro) {
2034 		for (i = 1; i < MTK_MAX_RX_RING_NUM; i++) {
2035 			err = mtk_rx_alloc(eth, i, MTK_RX_FLAGS_HWLRO);
2036 			if (err)
2037 				return err;
2038 		}
2039 		err = mtk_hwlro_rx_init(eth);
2040 		if (err)
2041 			return err;
2042 	}
2043 
2044 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2045 		/* Enable random early drop and set drop threshold
2046 		 * automatically
2047 		 */
2048 		mtk_w32(eth, FC_THRES_DROP_MODE | FC_THRES_DROP_EN |
2049 			FC_THRES_MIN, MTK_QDMA_FC_THRES);
2050 		mtk_w32(eth, 0x0, MTK_QDMA_HRED2);
2051 	}
2052 
2053 	return 0;
2054 }
2055 
2056 static void mtk_dma_free(struct mtk_eth *eth)
2057 {
2058 	int i;
2059 
2060 	for (i = 0; i < MTK_MAC_COUNT; i++)
2061 		if (eth->netdev[i])
2062 			netdev_reset_queue(eth->netdev[i]);
2063 	if (eth->scratch_ring) {
2064 		dma_free_coherent(eth->dev,
2065 				  MTK_DMA_SIZE * sizeof(struct mtk_tx_dma),
2066 				  eth->scratch_ring,
2067 				  eth->phy_scratch_ring);
2068 		eth->scratch_ring = NULL;
2069 		eth->phy_scratch_ring = 0;
2070 	}
2071 	mtk_tx_clean(eth);
2072 	mtk_rx_clean(eth, &eth->rx_ring[0]);
2073 	mtk_rx_clean(eth, &eth->rx_ring_qdma);
2074 
2075 	if (eth->hwlro) {
2076 		mtk_hwlro_rx_uninit(eth);
2077 		for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
2078 			mtk_rx_clean(eth, &eth->rx_ring[i]);
2079 	}
2080 
2081 	kfree(eth->scratch_head);
2082 }
2083 
2084 static void mtk_tx_timeout(struct net_device *dev)
2085 {
2086 	struct mtk_mac *mac = netdev_priv(dev);
2087 	struct mtk_eth *eth = mac->hw;
2088 
2089 	eth->netdev[mac->id]->stats.tx_errors++;
2090 	netif_err(eth, tx_err, dev,
2091 		  "transmit timed out\n");
2092 	schedule_work(&eth->pending_work);
2093 }
2094 
2095 static irqreturn_t mtk_handle_irq_rx(int irq, void *_eth)
2096 {
2097 	struct mtk_eth *eth = _eth;
2098 
2099 	if (likely(napi_schedule_prep(&eth->rx_napi))) {
2100 		__napi_schedule(&eth->rx_napi);
2101 		mtk_rx_irq_disable(eth, MTK_RX_DONE_INT);
2102 	}
2103 
2104 	return IRQ_HANDLED;
2105 }
2106 
2107 static irqreturn_t mtk_handle_irq_tx(int irq, void *_eth)
2108 {
2109 	struct mtk_eth *eth = _eth;
2110 
2111 	if (likely(napi_schedule_prep(&eth->tx_napi))) {
2112 		__napi_schedule(&eth->tx_napi);
2113 		mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
2114 	}
2115 
2116 	return IRQ_HANDLED;
2117 }
2118 
2119 static irqreturn_t mtk_handle_irq(int irq, void *_eth)
2120 {
2121 	struct mtk_eth *eth = _eth;
2122 
2123 	if (mtk_r32(eth, MTK_PDMA_INT_MASK) & MTK_RX_DONE_INT) {
2124 		if (mtk_r32(eth, MTK_PDMA_INT_STATUS) & MTK_RX_DONE_INT)
2125 			mtk_handle_irq_rx(irq, _eth);
2126 	}
2127 	if (mtk_r32(eth, eth->tx_int_mask_reg) & MTK_TX_DONE_INT) {
2128 		if (mtk_r32(eth, eth->tx_int_status_reg) & MTK_TX_DONE_INT)
2129 			mtk_handle_irq_tx(irq, _eth);
2130 	}
2131 
2132 	return IRQ_HANDLED;
2133 }
2134 
2135 #ifdef CONFIG_NET_POLL_CONTROLLER
2136 static void mtk_poll_controller(struct net_device *dev)
2137 {
2138 	struct mtk_mac *mac = netdev_priv(dev);
2139 	struct mtk_eth *eth = mac->hw;
2140 
2141 	mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
2142 	mtk_rx_irq_disable(eth, MTK_RX_DONE_INT);
2143 	mtk_handle_irq_rx(eth->irq[2], dev);
2144 	mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
2145 	mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
2146 }
2147 #endif
2148 
2149 static int mtk_start_dma(struct mtk_eth *eth)
2150 {
2151 	u32 rx_2b_offset = (NET_IP_ALIGN == 2) ? MTK_RX_2B_OFFSET : 0;
2152 	int err;
2153 
2154 	err = mtk_dma_init(eth);
2155 	if (err) {
2156 		mtk_dma_free(eth);
2157 		return err;
2158 	}
2159 
2160 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2161 		mtk_w32(eth,
2162 			MTK_TX_WB_DDONE | MTK_TX_DMA_EN |
2163 			MTK_DMA_SIZE_16DWORDS | MTK_NDP_CO_PRO |
2164 			MTK_RX_DMA_EN | MTK_RX_2B_OFFSET |
2165 			MTK_RX_BT_32DWORDS,
2166 			MTK_QDMA_GLO_CFG);
2167 
2168 		mtk_w32(eth,
2169 			MTK_RX_DMA_EN | rx_2b_offset |
2170 			MTK_RX_BT_32DWORDS | MTK_MULTI_EN,
2171 			MTK_PDMA_GLO_CFG);
2172 	} else {
2173 		mtk_w32(eth, MTK_TX_WB_DDONE | MTK_TX_DMA_EN | MTK_RX_DMA_EN |
2174 			MTK_MULTI_EN | MTK_PDMA_SIZE_8DWORDS,
2175 			MTK_PDMA_GLO_CFG);
2176 	}
2177 
2178 	return 0;
2179 }
2180 
2181 static void mtk_gdm_config(struct mtk_eth *eth, u32 config)
2182 {
2183 	int i;
2184 
2185 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
2186 		return;
2187 
2188 	for (i = 0; i < MTK_MAC_COUNT; i++) {
2189 		u32 val = mtk_r32(eth, MTK_GDMA_FWD_CFG(i));
2190 
2191 		/* default setup the forward port to send frame to PDMA */
2192 		val &= ~0xffff;
2193 
2194 		/* Enable RX checksum */
2195 		val |= MTK_GDMA_ICS_EN | MTK_GDMA_TCS_EN | MTK_GDMA_UCS_EN;
2196 
2197 		val |= config;
2198 
2199 		mtk_w32(eth, val, MTK_GDMA_FWD_CFG(i));
2200 	}
2201 	/* Reset and enable PSE */
2202 	mtk_w32(eth, RST_GL_PSE, MTK_RST_GL);
2203 	mtk_w32(eth, 0, MTK_RST_GL);
2204 }
2205 
2206 static int mtk_open(struct net_device *dev)
2207 {
2208 	struct mtk_mac *mac = netdev_priv(dev);
2209 	struct mtk_eth *eth = mac->hw;
2210 	int err;
2211 
2212 	err = phylink_of_phy_connect(mac->phylink, mac->of_node, 0);
2213 	if (err) {
2214 		netdev_err(dev, "%s: could not attach PHY: %d\n", __func__,
2215 			   err);
2216 		return err;
2217 	}
2218 
2219 	/* we run 2 netdevs on the same dma ring so we only bring it up once */
2220 	if (!refcount_read(&eth->dma_refcnt)) {
2221 		int err = mtk_start_dma(eth);
2222 
2223 		if (err)
2224 			return err;
2225 
2226 		mtk_gdm_config(eth, MTK_GDMA_TO_PDMA);
2227 
2228 		napi_enable(&eth->tx_napi);
2229 		napi_enable(&eth->rx_napi);
2230 		mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
2231 		mtk_rx_irq_enable(eth, MTK_RX_DONE_INT);
2232 		refcount_set(&eth->dma_refcnt, 1);
2233 	}
2234 	else
2235 		refcount_inc(&eth->dma_refcnt);
2236 
2237 	phylink_start(mac->phylink);
2238 	netif_start_queue(dev);
2239 	return 0;
2240 }
2241 
2242 static void mtk_stop_dma(struct mtk_eth *eth, u32 glo_cfg)
2243 {
2244 	u32 val;
2245 	int i;
2246 
2247 	/* stop the dma engine */
2248 	spin_lock_bh(&eth->page_lock);
2249 	val = mtk_r32(eth, glo_cfg);
2250 	mtk_w32(eth, val & ~(MTK_TX_WB_DDONE | MTK_RX_DMA_EN | MTK_TX_DMA_EN),
2251 		glo_cfg);
2252 	spin_unlock_bh(&eth->page_lock);
2253 
2254 	/* wait for dma stop */
2255 	for (i = 0; i < 10; i++) {
2256 		val = mtk_r32(eth, glo_cfg);
2257 		if (val & (MTK_TX_DMA_BUSY | MTK_RX_DMA_BUSY)) {
2258 			msleep(20);
2259 			continue;
2260 		}
2261 		break;
2262 	}
2263 }
2264 
2265 static int mtk_stop(struct net_device *dev)
2266 {
2267 	struct mtk_mac *mac = netdev_priv(dev);
2268 	struct mtk_eth *eth = mac->hw;
2269 
2270 	phylink_stop(mac->phylink);
2271 
2272 	netif_tx_disable(dev);
2273 
2274 	phylink_disconnect_phy(mac->phylink);
2275 
2276 	/* only shutdown DMA if this is the last user */
2277 	if (!refcount_dec_and_test(&eth->dma_refcnt))
2278 		return 0;
2279 
2280 	mtk_gdm_config(eth, MTK_GDMA_DROP_ALL);
2281 
2282 	mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
2283 	mtk_rx_irq_disable(eth, MTK_RX_DONE_INT);
2284 	napi_disable(&eth->tx_napi);
2285 	napi_disable(&eth->rx_napi);
2286 
2287 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
2288 		mtk_stop_dma(eth, MTK_QDMA_GLO_CFG);
2289 	mtk_stop_dma(eth, MTK_PDMA_GLO_CFG);
2290 
2291 	mtk_dma_free(eth);
2292 
2293 	return 0;
2294 }
2295 
2296 static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits)
2297 {
2298 	regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL,
2299 			   reset_bits,
2300 			   reset_bits);
2301 
2302 	usleep_range(1000, 1100);
2303 	regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL,
2304 			   reset_bits,
2305 			   ~reset_bits);
2306 	mdelay(10);
2307 }
2308 
2309 static void mtk_clk_disable(struct mtk_eth *eth)
2310 {
2311 	int clk;
2312 
2313 	for (clk = MTK_CLK_MAX - 1; clk >= 0; clk--)
2314 		clk_disable_unprepare(eth->clks[clk]);
2315 }
2316 
2317 static int mtk_clk_enable(struct mtk_eth *eth)
2318 {
2319 	int clk, ret;
2320 
2321 	for (clk = 0; clk < MTK_CLK_MAX ; clk++) {
2322 		ret = clk_prepare_enable(eth->clks[clk]);
2323 		if (ret)
2324 			goto err_disable_clks;
2325 	}
2326 
2327 	return 0;
2328 
2329 err_disable_clks:
2330 	while (--clk >= 0)
2331 		clk_disable_unprepare(eth->clks[clk]);
2332 
2333 	return ret;
2334 }
2335 
2336 static int mtk_hw_init(struct mtk_eth *eth)
2337 {
2338 	int i, val, ret;
2339 
2340 	if (test_and_set_bit(MTK_HW_INIT, &eth->state))
2341 		return 0;
2342 
2343 	pm_runtime_enable(eth->dev);
2344 	pm_runtime_get_sync(eth->dev);
2345 
2346 	ret = mtk_clk_enable(eth);
2347 	if (ret)
2348 		goto err_disable_pm;
2349 
2350 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
2351 		ret = device_reset(eth->dev);
2352 		if (ret) {
2353 			dev_err(eth->dev, "MAC reset failed!\n");
2354 			goto err_disable_pm;
2355 		}
2356 
2357 		/* enable interrupt delay for RX */
2358 		mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);
2359 
2360 		/* disable delay and normal interrupt */
2361 		mtk_tx_irq_disable(eth, ~0);
2362 		mtk_rx_irq_disable(eth, ~0);
2363 
2364 		return 0;
2365 	}
2366 
2367 	/* Non-MT7628 handling... */
2368 	ethsys_reset(eth, RSTCTRL_FE);
2369 	ethsys_reset(eth, RSTCTRL_PPE);
2370 
2371 	if (eth->pctl) {
2372 		/* Set GE2 driving and slew rate */
2373 		regmap_write(eth->pctl, GPIO_DRV_SEL10, 0xa00);
2374 
2375 		/* set GE2 TDSEL */
2376 		regmap_write(eth->pctl, GPIO_OD33_CTRL8, 0x5);
2377 
2378 		/* set GE2 TUNE */
2379 		regmap_write(eth->pctl, GPIO_BIAS_CTRL, 0x0);
2380 	}
2381 
2382 	/* Set linkdown as the default for each GMAC. Its own MCR would be set
2383 	 * up with the more appropriate value when mtk_mac_config call is being
2384 	 * invoked.
2385 	 */
2386 	for (i = 0; i < MTK_MAC_COUNT; i++)
2387 		mtk_w32(eth, MAC_MCR_FORCE_LINK_DOWN, MTK_MAC_MCR(i));
2388 
2389 	/* Indicates CDM to parse the MTK special tag from CPU
2390 	 * which also is working out for untag packets.
2391 	 */
2392 	val = mtk_r32(eth, MTK_CDMQ_IG_CTRL);
2393 	mtk_w32(eth, val | MTK_CDMQ_STAG_EN, MTK_CDMQ_IG_CTRL);
2394 
2395 	/* Enable RX VLan Offloading */
2396 	mtk_w32(eth, 1, MTK_CDMP_EG_CTRL);
2397 
2398 	/* enable interrupt delay for RX */
2399 	mtk_w32(eth, MTK_PDMA_DELAY_RX_DELAY, MTK_PDMA_DELAY_INT);
2400 
2401 	/* disable delay and normal interrupt */
2402 	mtk_w32(eth, 0, MTK_QDMA_DELAY_INT);
2403 	mtk_tx_irq_disable(eth, ~0);
2404 	mtk_rx_irq_disable(eth, ~0);
2405 
2406 	/* FE int grouping */
2407 	mtk_w32(eth, MTK_TX_DONE_INT, MTK_PDMA_INT_GRP1);
2408 	mtk_w32(eth, MTK_RX_DONE_INT, MTK_PDMA_INT_GRP2);
2409 	mtk_w32(eth, MTK_TX_DONE_INT, MTK_QDMA_INT_GRP1);
2410 	mtk_w32(eth, MTK_RX_DONE_INT, MTK_QDMA_INT_GRP2);
2411 	mtk_w32(eth, 0x21021000, MTK_FE_INT_GRP);
2412 
2413 	return 0;
2414 
2415 err_disable_pm:
2416 	pm_runtime_put_sync(eth->dev);
2417 	pm_runtime_disable(eth->dev);
2418 
2419 	return ret;
2420 }
2421 
2422 static int mtk_hw_deinit(struct mtk_eth *eth)
2423 {
2424 	if (!test_and_clear_bit(MTK_HW_INIT, &eth->state))
2425 		return 0;
2426 
2427 	mtk_clk_disable(eth);
2428 
2429 	pm_runtime_put_sync(eth->dev);
2430 	pm_runtime_disable(eth->dev);
2431 
2432 	return 0;
2433 }
2434 
2435 static int __init mtk_init(struct net_device *dev)
2436 {
2437 	struct mtk_mac *mac = netdev_priv(dev);
2438 	struct mtk_eth *eth = mac->hw;
2439 	const char *mac_addr;
2440 
2441 	mac_addr = of_get_mac_address(mac->of_node);
2442 	if (!IS_ERR(mac_addr))
2443 		ether_addr_copy(dev->dev_addr, mac_addr);
2444 
2445 	/* If the mac address is invalid, use random mac address  */
2446 	if (!is_valid_ether_addr(dev->dev_addr)) {
2447 		eth_hw_addr_random(dev);
2448 		dev_err(eth->dev, "generated random MAC address %pM\n",
2449 			dev->dev_addr);
2450 	}
2451 
2452 	return 0;
2453 }
2454 
2455 static void mtk_uninit(struct net_device *dev)
2456 {
2457 	struct mtk_mac *mac = netdev_priv(dev);
2458 	struct mtk_eth *eth = mac->hw;
2459 
2460 	phylink_disconnect_phy(mac->phylink);
2461 	mtk_tx_irq_disable(eth, ~0);
2462 	mtk_rx_irq_disable(eth, ~0);
2463 }
2464 
2465 static int mtk_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2466 {
2467 	struct mtk_mac *mac = netdev_priv(dev);
2468 
2469 	switch (cmd) {
2470 	case SIOCGMIIPHY:
2471 	case SIOCGMIIREG:
2472 	case SIOCSMIIREG:
2473 		return phylink_mii_ioctl(mac->phylink, ifr, cmd);
2474 	default:
2475 		break;
2476 	}
2477 
2478 	return -EOPNOTSUPP;
2479 }
2480 
2481 static void mtk_pending_work(struct work_struct *work)
2482 {
2483 	struct mtk_eth *eth = container_of(work, struct mtk_eth, pending_work);
2484 	int err, i;
2485 	unsigned long restart = 0;
2486 
2487 	rtnl_lock();
2488 
2489 	dev_dbg(eth->dev, "[%s][%d] reset\n", __func__, __LINE__);
2490 
2491 	while (test_and_set_bit_lock(MTK_RESETTING, &eth->state))
2492 		cpu_relax();
2493 
2494 	dev_dbg(eth->dev, "[%s][%d] mtk_stop starts\n", __func__, __LINE__);
2495 	/* stop all devices to make sure that dma is properly shut down */
2496 	for (i = 0; i < MTK_MAC_COUNT; i++) {
2497 		if (!eth->netdev[i])
2498 			continue;
2499 		mtk_stop(eth->netdev[i]);
2500 		__set_bit(i, &restart);
2501 	}
2502 	dev_dbg(eth->dev, "[%s][%d] mtk_stop ends\n", __func__, __LINE__);
2503 
2504 	/* restart underlying hardware such as power, clock, pin mux
2505 	 * and the connected phy
2506 	 */
2507 	mtk_hw_deinit(eth);
2508 
2509 	if (eth->dev->pins)
2510 		pinctrl_select_state(eth->dev->pins->p,
2511 				     eth->dev->pins->default_state);
2512 	mtk_hw_init(eth);
2513 
2514 	/* restart DMA and enable IRQs */
2515 	for (i = 0; i < MTK_MAC_COUNT; i++) {
2516 		if (!test_bit(i, &restart))
2517 			continue;
2518 		err = mtk_open(eth->netdev[i]);
2519 		if (err) {
2520 			netif_alert(eth, ifup, eth->netdev[i],
2521 			      "Driver up/down cycle failed, closing device.\n");
2522 			dev_close(eth->netdev[i]);
2523 		}
2524 	}
2525 
2526 	dev_dbg(eth->dev, "[%s][%d] reset done\n", __func__, __LINE__);
2527 
2528 	clear_bit_unlock(MTK_RESETTING, &eth->state);
2529 
2530 	rtnl_unlock();
2531 }
2532 
2533 static int mtk_free_dev(struct mtk_eth *eth)
2534 {
2535 	int i;
2536 
2537 	for (i = 0; i < MTK_MAC_COUNT; i++) {
2538 		if (!eth->netdev[i])
2539 			continue;
2540 		free_netdev(eth->netdev[i]);
2541 	}
2542 
2543 	return 0;
2544 }
2545 
2546 static int mtk_unreg_dev(struct mtk_eth *eth)
2547 {
2548 	int i;
2549 
2550 	for (i = 0; i < MTK_MAC_COUNT; i++) {
2551 		if (!eth->netdev[i])
2552 			continue;
2553 		unregister_netdev(eth->netdev[i]);
2554 	}
2555 
2556 	return 0;
2557 }
2558 
2559 static int mtk_cleanup(struct mtk_eth *eth)
2560 {
2561 	mtk_unreg_dev(eth);
2562 	mtk_free_dev(eth);
2563 	cancel_work_sync(&eth->pending_work);
2564 
2565 	return 0;
2566 }
2567 
2568 static int mtk_get_link_ksettings(struct net_device *ndev,
2569 				  struct ethtool_link_ksettings *cmd)
2570 {
2571 	struct mtk_mac *mac = netdev_priv(ndev);
2572 
2573 	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
2574 		return -EBUSY;
2575 
2576 	return phylink_ethtool_ksettings_get(mac->phylink, cmd);
2577 }
2578 
2579 static int mtk_set_link_ksettings(struct net_device *ndev,
2580 				  const struct ethtool_link_ksettings *cmd)
2581 {
2582 	struct mtk_mac *mac = netdev_priv(ndev);
2583 
2584 	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
2585 		return -EBUSY;
2586 
2587 	return phylink_ethtool_ksettings_set(mac->phylink, cmd);
2588 }
2589 
2590 static void mtk_get_drvinfo(struct net_device *dev,
2591 			    struct ethtool_drvinfo *info)
2592 {
2593 	struct mtk_mac *mac = netdev_priv(dev);
2594 
2595 	strlcpy(info->driver, mac->hw->dev->driver->name, sizeof(info->driver));
2596 	strlcpy(info->bus_info, dev_name(mac->hw->dev), sizeof(info->bus_info));
2597 	info->n_stats = ARRAY_SIZE(mtk_ethtool_stats);
2598 }
2599 
2600 static u32 mtk_get_msglevel(struct net_device *dev)
2601 {
2602 	struct mtk_mac *mac = netdev_priv(dev);
2603 
2604 	return mac->hw->msg_enable;
2605 }
2606 
2607 static void mtk_set_msglevel(struct net_device *dev, u32 value)
2608 {
2609 	struct mtk_mac *mac = netdev_priv(dev);
2610 
2611 	mac->hw->msg_enable = value;
2612 }
2613 
2614 static int mtk_nway_reset(struct net_device *dev)
2615 {
2616 	struct mtk_mac *mac = netdev_priv(dev);
2617 
2618 	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
2619 		return -EBUSY;
2620 
2621 	if (!mac->phylink)
2622 		return -ENOTSUPP;
2623 
2624 	return phylink_ethtool_nway_reset(mac->phylink);
2625 }
2626 
2627 static void mtk_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2628 {
2629 	int i;
2630 
2631 	switch (stringset) {
2632 	case ETH_SS_STATS:
2633 		for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++) {
2634 			memcpy(data, mtk_ethtool_stats[i].str, ETH_GSTRING_LEN);
2635 			data += ETH_GSTRING_LEN;
2636 		}
2637 		break;
2638 	}
2639 }
2640 
2641 static int mtk_get_sset_count(struct net_device *dev, int sset)
2642 {
2643 	switch (sset) {
2644 	case ETH_SS_STATS:
2645 		return ARRAY_SIZE(mtk_ethtool_stats);
2646 	default:
2647 		return -EOPNOTSUPP;
2648 	}
2649 }
2650 
2651 static void mtk_get_ethtool_stats(struct net_device *dev,
2652 				  struct ethtool_stats *stats, u64 *data)
2653 {
2654 	struct mtk_mac *mac = netdev_priv(dev);
2655 	struct mtk_hw_stats *hwstats = mac->hw_stats;
2656 	u64 *data_src, *data_dst;
2657 	unsigned int start;
2658 	int i;
2659 
2660 	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
2661 		return;
2662 
2663 	if (netif_running(dev) && netif_device_present(dev)) {
2664 		if (spin_trylock_bh(&hwstats->stats_lock)) {
2665 			mtk_stats_update_mac(mac);
2666 			spin_unlock_bh(&hwstats->stats_lock);
2667 		}
2668 	}
2669 
2670 	data_src = (u64 *)hwstats;
2671 
2672 	do {
2673 		data_dst = data;
2674 		start = u64_stats_fetch_begin_irq(&hwstats->syncp);
2675 
2676 		for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++)
2677 			*data_dst++ = *(data_src + mtk_ethtool_stats[i].offset);
2678 	} while (u64_stats_fetch_retry_irq(&hwstats->syncp, start));
2679 }
2680 
2681 static int mtk_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
2682 			 u32 *rule_locs)
2683 {
2684 	int ret = -EOPNOTSUPP;
2685 
2686 	switch (cmd->cmd) {
2687 	case ETHTOOL_GRXRINGS:
2688 		if (dev->hw_features & NETIF_F_LRO) {
2689 			cmd->data = MTK_MAX_RX_RING_NUM;
2690 			ret = 0;
2691 		}
2692 		break;
2693 	case ETHTOOL_GRXCLSRLCNT:
2694 		if (dev->hw_features & NETIF_F_LRO) {
2695 			struct mtk_mac *mac = netdev_priv(dev);
2696 
2697 			cmd->rule_cnt = mac->hwlro_ip_cnt;
2698 			ret = 0;
2699 		}
2700 		break;
2701 	case ETHTOOL_GRXCLSRULE:
2702 		if (dev->hw_features & NETIF_F_LRO)
2703 			ret = mtk_hwlro_get_fdir_entry(dev, cmd);
2704 		break;
2705 	case ETHTOOL_GRXCLSRLALL:
2706 		if (dev->hw_features & NETIF_F_LRO)
2707 			ret = mtk_hwlro_get_fdir_all(dev, cmd,
2708 						     rule_locs);
2709 		break;
2710 	default:
2711 		break;
2712 	}
2713 
2714 	return ret;
2715 }
2716 
2717 static int mtk_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
2718 {
2719 	int ret = -EOPNOTSUPP;
2720 
2721 	switch (cmd->cmd) {
2722 	case ETHTOOL_SRXCLSRLINS:
2723 		if (dev->hw_features & NETIF_F_LRO)
2724 			ret = mtk_hwlro_add_ipaddr(dev, cmd);
2725 		break;
2726 	case ETHTOOL_SRXCLSRLDEL:
2727 		if (dev->hw_features & NETIF_F_LRO)
2728 			ret = mtk_hwlro_del_ipaddr(dev, cmd);
2729 		break;
2730 	default:
2731 		break;
2732 	}
2733 
2734 	return ret;
2735 }
2736 
2737 static const struct ethtool_ops mtk_ethtool_ops = {
2738 	.get_link_ksettings	= mtk_get_link_ksettings,
2739 	.set_link_ksettings	= mtk_set_link_ksettings,
2740 	.get_drvinfo		= mtk_get_drvinfo,
2741 	.get_msglevel		= mtk_get_msglevel,
2742 	.set_msglevel		= mtk_set_msglevel,
2743 	.nway_reset		= mtk_nway_reset,
2744 	.get_link		= ethtool_op_get_link,
2745 	.get_strings		= mtk_get_strings,
2746 	.get_sset_count		= mtk_get_sset_count,
2747 	.get_ethtool_stats	= mtk_get_ethtool_stats,
2748 	.get_rxnfc		= mtk_get_rxnfc,
2749 	.set_rxnfc              = mtk_set_rxnfc,
2750 };
2751 
2752 static const struct net_device_ops mtk_netdev_ops = {
2753 	.ndo_init		= mtk_init,
2754 	.ndo_uninit		= mtk_uninit,
2755 	.ndo_open		= mtk_open,
2756 	.ndo_stop		= mtk_stop,
2757 	.ndo_start_xmit		= mtk_start_xmit,
2758 	.ndo_set_mac_address	= mtk_set_mac_address,
2759 	.ndo_validate_addr	= eth_validate_addr,
2760 	.ndo_do_ioctl		= mtk_do_ioctl,
2761 	.ndo_tx_timeout		= mtk_tx_timeout,
2762 	.ndo_get_stats64        = mtk_get_stats64,
2763 	.ndo_fix_features	= mtk_fix_features,
2764 	.ndo_set_features	= mtk_set_features,
2765 #ifdef CONFIG_NET_POLL_CONTROLLER
2766 	.ndo_poll_controller	= mtk_poll_controller,
2767 #endif
2768 };
2769 
2770 static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
2771 {
2772 	const __be32 *_id = of_get_property(np, "reg", NULL);
2773 	phy_interface_t phy_mode;
2774 	struct phylink *phylink;
2775 	struct mtk_mac *mac;
2776 	int id, err;
2777 
2778 	if (!_id) {
2779 		dev_err(eth->dev, "missing mac id\n");
2780 		return -EINVAL;
2781 	}
2782 
2783 	id = be32_to_cpup(_id);
2784 	if (id >= MTK_MAC_COUNT) {
2785 		dev_err(eth->dev, "%d is not a valid mac id\n", id);
2786 		return -EINVAL;
2787 	}
2788 
2789 	if (eth->netdev[id]) {
2790 		dev_err(eth->dev, "duplicate mac id found: %d\n", id);
2791 		return -EINVAL;
2792 	}
2793 
2794 	eth->netdev[id] = alloc_etherdev(sizeof(*mac));
2795 	if (!eth->netdev[id]) {
2796 		dev_err(eth->dev, "alloc_etherdev failed\n");
2797 		return -ENOMEM;
2798 	}
2799 	mac = netdev_priv(eth->netdev[id]);
2800 	eth->mac[id] = mac;
2801 	mac->id = id;
2802 	mac->hw = eth;
2803 	mac->of_node = np;
2804 
2805 	memset(mac->hwlro_ip, 0, sizeof(mac->hwlro_ip));
2806 	mac->hwlro_ip_cnt = 0;
2807 
2808 	mac->hw_stats = devm_kzalloc(eth->dev,
2809 				     sizeof(*mac->hw_stats),
2810 				     GFP_KERNEL);
2811 	if (!mac->hw_stats) {
2812 		dev_err(eth->dev, "failed to allocate counter memory\n");
2813 		err = -ENOMEM;
2814 		goto free_netdev;
2815 	}
2816 	spin_lock_init(&mac->hw_stats->stats_lock);
2817 	u64_stats_init(&mac->hw_stats->syncp);
2818 	mac->hw_stats->reg_offset = id * MTK_STAT_OFFSET;
2819 
2820 	/* phylink create */
2821 	err = of_get_phy_mode(np, &phy_mode);
2822 	if (err) {
2823 		dev_err(eth->dev, "incorrect phy-mode\n");
2824 		goto free_netdev;
2825 	}
2826 
2827 	/* mac config is not set */
2828 	mac->interface = PHY_INTERFACE_MODE_NA;
2829 	mac->mode = MLO_AN_PHY;
2830 	mac->speed = SPEED_UNKNOWN;
2831 
2832 	mac->phylink_config.dev = &eth->netdev[id]->dev;
2833 	mac->phylink_config.type = PHYLINK_NETDEV;
2834 
2835 	phylink = phylink_create(&mac->phylink_config,
2836 				 of_fwnode_handle(mac->of_node),
2837 				 phy_mode, &mtk_phylink_ops);
2838 	if (IS_ERR(phylink)) {
2839 		err = PTR_ERR(phylink);
2840 		goto free_netdev;
2841 	}
2842 
2843 	mac->phylink = phylink;
2844 
2845 	SET_NETDEV_DEV(eth->netdev[id], eth->dev);
2846 	eth->netdev[id]->watchdog_timeo = 5 * HZ;
2847 	eth->netdev[id]->netdev_ops = &mtk_netdev_ops;
2848 	eth->netdev[id]->base_addr = (unsigned long)eth->base;
2849 
2850 	eth->netdev[id]->hw_features = eth->soc->hw_features;
2851 	if (eth->hwlro)
2852 		eth->netdev[id]->hw_features |= NETIF_F_LRO;
2853 
2854 	eth->netdev[id]->vlan_features = eth->soc->hw_features &
2855 		~(NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX);
2856 	eth->netdev[id]->features |= eth->soc->hw_features;
2857 	eth->netdev[id]->ethtool_ops = &mtk_ethtool_ops;
2858 
2859 	eth->netdev[id]->irq = eth->irq[0];
2860 	eth->netdev[id]->dev.of_node = np;
2861 
2862 	return 0;
2863 
2864 free_netdev:
2865 	free_netdev(eth->netdev[id]);
2866 	return err;
2867 }
2868 
2869 static int mtk_probe(struct platform_device *pdev)
2870 {
2871 	struct device_node *mac_np;
2872 	struct mtk_eth *eth;
2873 	int err, i;
2874 
2875 	eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL);
2876 	if (!eth)
2877 		return -ENOMEM;
2878 
2879 	eth->soc = of_device_get_match_data(&pdev->dev);
2880 
2881 	eth->dev = &pdev->dev;
2882 	eth->base = devm_platform_ioremap_resource(pdev, 0);
2883 	if (IS_ERR(eth->base))
2884 		return PTR_ERR(eth->base);
2885 
2886 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
2887 		eth->tx_int_mask_reg = MTK_QDMA_INT_MASK;
2888 		eth->tx_int_status_reg = MTK_QDMA_INT_STATUS;
2889 	} else {
2890 		eth->tx_int_mask_reg = MTK_PDMA_INT_MASK;
2891 		eth->tx_int_status_reg = MTK_PDMA_INT_STATUS;
2892 	}
2893 
2894 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
2895 		eth->rx_dma_l4_valid = RX_DMA_L4_VALID_PDMA;
2896 		eth->ip_align = NET_IP_ALIGN;
2897 	} else {
2898 		eth->rx_dma_l4_valid = RX_DMA_L4_VALID;
2899 	}
2900 
2901 	spin_lock_init(&eth->page_lock);
2902 	spin_lock_init(&eth->tx_irq_lock);
2903 	spin_lock_init(&eth->rx_irq_lock);
2904 
2905 	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
2906 		eth->ethsys = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
2907 							      "mediatek,ethsys");
2908 		if (IS_ERR(eth->ethsys)) {
2909 			dev_err(&pdev->dev, "no ethsys regmap found\n");
2910 			return PTR_ERR(eth->ethsys);
2911 		}
2912 	}
2913 
2914 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_INFRA)) {
2915 		eth->infra = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
2916 							     "mediatek,infracfg");
2917 		if (IS_ERR(eth->infra)) {
2918 			dev_err(&pdev->dev, "no infracfg regmap found\n");
2919 			return PTR_ERR(eth->infra);
2920 		}
2921 	}
2922 
2923 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) {
2924 		eth->sgmii = devm_kzalloc(eth->dev, sizeof(*eth->sgmii),
2925 					  GFP_KERNEL);
2926 		if (!eth->sgmii)
2927 			return -ENOMEM;
2928 
2929 		err = mtk_sgmii_init(eth->sgmii, pdev->dev.of_node,
2930 				     eth->soc->ana_rgc3);
2931 
2932 		if (err)
2933 			return err;
2934 	}
2935 
2936 	if (eth->soc->required_pctl) {
2937 		eth->pctl = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
2938 							    "mediatek,pctl");
2939 		if (IS_ERR(eth->pctl)) {
2940 			dev_err(&pdev->dev, "no pctl regmap found\n");
2941 			return PTR_ERR(eth->pctl);
2942 		}
2943 	}
2944 
2945 	for (i = 0; i < 3; i++) {
2946 		if (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_INT) && i > 0)
2947 			eth->irq[i] = eth->irq[0];
2948 		else
2949 			eth->irq[i] = platform_get_irq(pdev, i);
2950 		if (eth->irq[i] < 0) {
2951 			dev_err(&pdev->dev, "no IRQ%d resource found\n", i);
2952 			return -ENXIO;
2953 		}
2954 	}
2955 	for (i = 0; i < ARRAY_SIZE(eth->clks); i++) {
2956 		eth->clks[i] = devm_clk_get(eth->dev,
2957 					    mtk_clks_source_name[i]);
2958 		if (IS_ERR(eth->clks[i])) {
2959 			if (PTR_ERR(eth->clks[i]) == -EPROBE_DEFER)
2960 				return -EPROBE_DEFER;
2961 			if (eth->soc->required_clks & BIT(i)) {
2962 				dev_err(&pdev->dev, "clock %s not found\n",
2963 					mtk_clks_source_name[i]);
2964 				return -EINVAL;
2965 			}
2966 			eth->clks[i] = NULL;
2967 		}
2968 	}
2969 
2970 	eth->msg_enable = netif_msg_init(mtk_msg_level, MTK_DEFAULT_MSG_ENABLE);
2971 	INIT_WORK(&eth->pending_work, mtk_pending_work);
2972 
2973 	err = mtk_hw_init(eth);
2974 	if (err)
2975 		return err;
2976 
2977 	eth->hwlro = MTK_HAS_CAPS(eth->soc->caps, MTK_HWLRO);
2978 
2979 	for_each_child_of_node(pdev->dev.of_node, mac_np) {
2980 		if (!of_device_is_compatible(mac_np,
2981 					     "mediatek,eth-mac"))
2982 			continue;
2983 
2984 		if (!of_device_is_available(mac_np))
2985 			continue;
2986 
2987 		err = mtk_add_mac(eth, mac_np);
2988 		if (err) {
2989 			of_node_put(mac_np);
2990 			goto err_deinit_hw;
2991 		}
2992 	}
2993 
2994 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_INT)) {
2995 		err = devm_request_irq(eth->dev, eth->irq[0],
2996 				       mtk_handle_irq, 0,
2997 				       dev_name(eth->dev), eth);
2998 	} else {
2999 		err = devm_request_irq(eth->dev, eth->irq[1],
3000 				       mtk_handle_irq_tx, 0,
3001 				       dev_name(eth->dev), eth);
3002 		if (err)
3003 			goto err_free_dev;
3004 
3005 		err = devm_request_irq(eth->dev, eth->irq[2],
3006 				       mtk_handle_irq_rx, 0,
3007 				       dev_name(eth->dev), eth);
3008 	}
3009 	if (err)
3010 		goto err_free_dev;
3011 
3012 	/* No MT7628/88 support yet */
3013 	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
3014 		err = mtk_mdio_init(eth);
3015 		if (err)
3016 			goto err_free_dev;
3017 	}
3018 
3019 	for (i = 0; i < MTK_MAX_DEVS; i++) {
3020 		if (!eth->netdev[i])
3021 			continue;
3022 
3023 		err = register_netdev(eth->netdev[i]);
3024 		if (err) {
3025 			dev_err(eth->dev, "error bringing up device\n");
3026 			goto err_deinit_mdio;
3027 		} else
3028 			netif_info(eth, probe, eth->netdev[i],
3029 				   "mediatek frame engine at 0x%08lx, irq %d\n",
3030 				   eth->netdev[i]->base_addr, eth->irq[0]);
3031 	}
3032 
3033 	/* we run 2 devices on the same DMA ring so we need a dummy device
3034 	 * for NAPI to work
3035 	 */
3036 	init_dummy_netdev(&eth->dummy_dev);
3037 	netif_napi_add(&eth->dummy_dev, &eth->tx_napi, mtk_napi_tx,
3038 		       MTK_NAPI_WEIGHT);
3039 	netif_napi_add(&eth->dummy_dev, &eth->rx_napi, mtk_napi_rx,
3040 		       MTK_NAPI_WEIGHT);
3041 
3042 	platform_set_drvdata(pdev, eth);
3043 
3044 	return 0;
3045 
3046 err_deinit_mdio:
3047 	mtk_mdio_cleanup(eth);
3048 err_free_dev:
3049 	mtk_free_dev(eth);
3050 err_deinit_hw:
3051 	mtk_hw_deinit(eth);
3052 
3053 	return err;
3054 }
3055 
3056 static int mtk_remove(struct platform_device *pdev)
3057 {
3058 	struct mtk_eth *eth = platform_get_drvdata(pdev);
3059 	struct mtk_mac *mac;
3060 	int i;
3061 
3062 	/* stop all devices to make sure that dma is properly shut down */
3063 	for (i = 0; i < MTK_MAC_COUNT; i++) {
3064 		if (!eth->netdev[i])
3065 			continue;
3066 		mtk_stop(eth->netdev[i]);
3067 		mac = netdev_priv(eth->netdev[i]);
3068 		phylink_disconnect_phy(mac->phylink);
3069 	}
3070 
3071 	mtk_hw_deinit(eth);
3072 
3073 	netif_napi_del(&eth->tx_napi);
3074 	netif_napi_del(&eth->rx_napi);
3075 	mtk_cleanup(eth);
3076 	mtk_mdio_cleanup(eth);
3077 
3078 	return 0;
3079 }
3080 
3081 static const struct mtk_soc_data mt2701_data = {
3082 	.caps = MT7623_CAPS | MTK_HWLRO,
3083 	.hw_features = MTK_HW_FEATURES,
3084 	.required_clks = MT7623_CLKS_BITMAP,
3085 	.required_pctl = true,
3086 };
3087 
3088 static const struct mtk_soc_data mt7621_data = {
3089 	.caps = MT7621_CAPS,
3090 	.hw_features = MTK_HW_FEATURES,
3091 	.required_clks = MT7621_CLKS_BITMAP,
3092 	.required_pctl = false,
3093 };
3094 
3095 static const struct mtk_soc_data mt7622_data = {
3096 	.ana_rgc3 = 0x2028,
3097 	.caps = MT7622_CAPS | MTK_HWLRO,
3098 	.hw_features = MTK_HW_FEATURES,
3099 	.required_clks = MT7622_CLKS_BITMAP,
3100 	.required_pctl = false,
3101 };
3102 
3103 static const struct mtk_soc_data mt7623_data = {
3104 	.caps = MT7623_CAPS | MTK_HWLRO,
3105 	.hw_features = MTK_HW_FEATURES,
3106 	.required_clks = MT7623_CLKS_BITMAP,
3107 	.required_pctl = true,
3108 };
3109 
3110 static const struct mtk_soc_data mt7629_data = {
3111 	.ana_rgc3 = 0x128,
3112 	.caps = MT7629_CAPS | MTK_HWLRO,
3113 	.hw_features = MTK_HW_FEATURES,
3114 	.required_clks = MT7629_CLKS_BITMAP,
3115 	.required_pctl = false,
3116 };
3117 
3118 static const struct mtk_soc_data rt5350_data = {
3119 	.caps = MT7628_CAPS,
3120 	.hw_features = MTK_HW_FEATURES_MT7628,
3121 	.required_clks = MT7628_CLKS_BITMAP,
3122 	.required_pctl = false,
3123 };
3124 
3125 const struct of_device_id of_mtk_match[] = {
3126 	{ .compatible = "mediatek,mt2701-eth", .data = &mt2701_data},
3127 	{ .compatible = "mediatek,mt7621-eth", .data = &mt7621_data},
3128 	{ .compatible = "mediatek,mt7622-eth", .data = &mt7622_data},
3129 	{ .compatible = "mediatek,mt7623-eth", .data = &mt7623_data},
3130 	{ .compatible = "mediatek,mt7629-eth", .data = &mt7629_data},
3131 	{ .compatible = "ralink,rt5350-eth", .data = &rt5350_data},
3132 	{},
3133 };
3134 MODULE_DEVICE_TABLE(of, of_mtk_match);
3135 
3136 static struct platform_driver mtk_driver = {
3137 	.probe = mtk_probe,
3138 	.remove = mtk_remove,
3139 	.driver = {
3140 		.name = "mtk_soc_eth",
3141 		.of_match_table = of_mtk_match,
3142 	},
3143 };
3144 
3145 module_platform_driver(mtk_driver);
3146 
3147 MODULE_LICENSE("GPL");
3148 MODULE_AUTHOR("John Crispin <blogic@openwrt.org>");
3149 MODULE_DESCRIPTION("Ethernet driver for MediaTek SoC");
3150