1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Cadence MACB/GEM Ethernet Controller driver
4  *
5  * Copyright (C) 2004-2006 Atmel Corporation
6  */
7 
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9 #include <linux/clk.h>
10 #include <linux/clk-provider.h>
11 #include <linux/crc32.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/kernel.h>
15 #include <linux/types.h>
16 #include <linux/circ_buf.h>
17 #include <linux/slab.h>
18 #include <linux/init.h>
19 #include <linux/io.h>
20 #include <linux/gpio.h>
21 #include <linux/gpio/consumer.h>
22 #include <linux/interrupt.h>
23 #include <linux/netdevice.h>
24 #include <linux/etherdevice.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/platform_device.h>
27 #include <linux/phylink.h>
28 #include <linux/of.h>
29 #include <linux/of_device.h>
30 #include <linux/of_gpio.h>
31 #include <linux/of_mdio.h>
32 #include <linux/of_net.h>
33 #include <linux/ip.h>
34 #include <linux/udp.h>
35 #include <linux/tcp.h>
36 #include <linux/iopoll.h>
37 #include <linux/pm_runtime.h>
38 #include "macb.h"
39 
40 /* This structure is only used for MACB on SiFive FU540 devices */
41 struct sifive_fu540_macb_mgmt {
42 	void __iomem *reg;
43 	unsigned long rate;
44 	struct clk_hw hw;
45 };
46 
47 #define MACB_RX_BUFFER_SIZE	128
48 #define RX_BUFFER_MULTIPLE	64  /* bytes */
49 
50 #define DEFAULT_RX_RING_SIZE	512 /* must be power of 2 */
51 #define MIN_RX_RING_SIZE	64
52 #define MAX_RX_RING_SIZE	8192
53 #define RX_RING_BYTES(bp)	(macb_dma_desc_get_size(bp)	\
54 				 * (bp)->rx_ring_size)
55 
56 #define DEFAULT_TX_RING_SIZE	512 /* must be power of 2 */
57 #define MIN_TX_RING_SIZE	64
58 #define MAX_TX_RING_SIZE	4096
59 #define TX_RING_BYTES(bp)	(macb_dma_desc_get_size(bp)	\
60 				 * (bp)->tx_ring_size)
61 
62 /* level of occupied TX descriptors under which we wake up TX process */
63 #define MACB_TX_WAKEUP_THRESH(bp)	(3 * (bp)->tx_ring_size / 4)
64 
65 #define MACB_RX_INT_FLAGS	(MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR))
66 #define MACB_TX_ERR_FLAGS	(MACB_BIT(ISR_TUND)			\
67 					| MACB_BIT(ISR_RLE)		\
68 					| MACB_BIT(TXERR))
69 #define MACB_TX_INT_FLAGS	(MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP)	\
70 					| MACB_BIT(TXUBR))
71 
72 /* Max length of transmit frame must be a multiple of 8 bytes */
73 #define MACB_TX_LEN_ALIGN	8
74 #define MACB_MAX_TX_LEN		((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1)))
75 /* Limit maximum TX length as per Cadence TSO errata. This is to avoid a
76  * false amba_error in TX path from the DMA assuming there is not enough
77  * space in the SRAM (16KB) even when there is.
78  */
79 #define GEM_MAX_TX_LEN		(unsigned int)(0x3FC0)
80 
81 #define GEM_MTU_MIN_SIZE	ETH_MIN_MTU
82 #define MACB_NETIF_LSO		NETIF_F_TSO
83 
84 #define MACB_WOL_HAS_MAGIC_PACKET	(0x1 << 0)
85 #define MACB_WOL_ENABLED		(0x1 << 1)
86 
87 #define HS_SPEED_10000M			4
88 #define MACB_SERDES_RATE_10G		1
89 
90 /* Graceful stop timeouts in us. We should allow up to
91  * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
92  */
93 #define MACB_HALT_TIMEOUT	1230
94 
95 #define MACB_PM_TIMEOUT  100 /* ms */
96 
97 #define MACB_MDIO_TIMEOUT	1000000 /* in usecs */
98 
99 /* DMA buffer descriptor might be different size
100  * depends on hardware configuration:
101  *
102  * 1. dma address width 32 bits:
103  *    word 1: 32 bit address of Data Buffer
104  *    word 2: control
105  *
106  * 2. dma address width 64 bits:
107  *    word 1: 32 bit address of Data Buffer
108  *    word 2: control
109  *    word 3: upper 32 bit address of Data Buffer
110  *    word 4: unused
111  *
112  * 3. dma address width 32 bits with hardware timestamping:
113  *    word 1: 32 bit address of Data Buffer
114  *    word 2: control
115  *    word 3: timestamp word 1
116  *    word 4: timestamp word 2
117  *
118  * 4. dma address width 64 bits with hardware timestamping:
119  *    word 1: 32 bit address of Data Buffer
120  *    word 2: control
121  *    word 3: upper 32 bit address of Data Buffer
122  *    word 4: unused
123  *    word 5: timestamp word 1
124  *    word 6: timestamp word 2
125  */
126 static unsigned int macb_dma_desc_get_size(struct macb *bp)
127 {
128 #ifdef MACB_EXT_DESC
129 	unsigned int desc_size;
130 
131 	switch (bp->hw_dma_cap) {
132 	case HW_DMA_CAP_64B:
133 		desc_size = sizeof(struct macb_dma_desc)
134 			+ sizeof(struct macb_dma_desc_64);
135 		break;
136 	case HW_DMA_CAP_PTP:
137 		desc_size = sizeof(struct macb_dma_desc)
138 			+ sizeof(struct macb_dma_desc_ptp);
139 		break;
140 	case HW_DMA_CAP_64B_PTP:
141 		desc_size = sizeof(struct macb_dma_desc)
142 			+ sizeof(struct macb_dma_desc_64)
143 			+ sizeof(struct macb_dma_desc_ptp);
144 		break;
145 	default:
146 		desc_size = sizeof(struct macb_dma_desc);
147 	}
148 	return desc_size;
149 #endif
150 	return sizeof(struct macb_dma_desc);
151 }
152 
153 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx)
154 {
155 #ifdef MACB_EXT_DESC
156 	switch (bp->hw_dma_cap) {
157 	case HW_DMA_CAP_64B:
158 	case HW_DMA_CAP_PTP:
159 		desc_idx <<= 1;
160 		break;
161 	case HW_DMA_CAP_64B_PTP:
162 		desc_idx *= 3;
163 		break;
164 	default:
165 		break;
166 	}
167 #endif
168 	return desc_idx;
169 }
170 
171 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
172 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc)
173 {
174 	return (struct macb_dma_desc_64 *)((void *)desc
175 		+ sizeof(struct macb_dma_desc));
176 }
177 #endif
178 
179 /* Ring buffer accessors */
180 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index)
181 {
182 	return index & (bp->tx_ring_size - 1);
183 }
184 
185 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue,
186 					  unsigned int index)
187 {
188 	index = macb_tx_ring_wrap(queue->bp, index);
189 	index = macb_adj_dma_desc_idx(queue->bp, index);
190 	return &queue->tx_ring[index];
191 }
192 
193 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue,
194 				       unsigned int index)
195 {
196 	return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)];
197 }
198 
199 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index)
200 {
201 	dma_addr_t offset;
202 
203 	offset = macb_tx_ring_wrap(queue->bp, index) *
204 			macb_dma_desc_get_size(queue->bp);
205 
206 	return queue->tx_ring_dma + offset;
207 }
208 
209 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index)
210 {
211 	return index & (bp->rx_ring_size - 1);
212 }
213 
214 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index)
215 {
216 	index = macb_rx_ring_wrap(queue->bp, index);
217 	index = macb_adj_dma_desc_idx(queue->bp, index);
218 	return &queue->rx_ring[index];
219 }
220 
221 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index)
222 {
223 	return queue->rx_buffers + queue->bp->rx_buffer_size *
224 	       macb_rx_ring_wrap(queue->bp, index);
225 }
226 
227 /* I/O accessors */
228 static u32 hw_readl_native(struct macb *bp, int offset)
229 {
230 	return __raw_readl(bp->regs + offset);
231 }
232 
233 static void hw_writel_native(struct macb *bp, int offset, u32 value)
234 {
235 	__raw_writel(value, bp->regs + offset);
236 }
237 
238 static u32 hw_readl(struct macb *bp, int offset)
239 {
240 	return readl_relaxed(bp->regs + offset);
241 }
242 
243 static void hw_writel(struct macb *bp, int offset, u32 value)
244 {
245 	writel_relaxed(value, bp->regs + offset);
246 }
247 
248 /* Find the CPU endianness by using the loopback bit of NCR register. When the
249  * CPU is in big endian we need to program swapped mode for management
250  * descriptor access.
251  */
252 static bool hw_is_native_io(void __iomem *addr)
253 {
254 	u32 value = MACB_BIT(LLB);
255 
256 	__raw_writel(value, addr + MACB_NCR);
257 	value = __raw_readl(addr + MACB_NCR);
258 
259 	/* Write 0 back to disable everything */
260 	__raw_writel(0, addr + MACB_NCR);
261 
262 	return value == MACB_BIT(LLB);
263 }
264 
265 static bool hw_is_gem(void __iomem *addr, bool native_io)
266 {
267 	u32 id;
268 
269 	if (native_io)
270 		id = __raw_readl(addr + MACB_MID);
271 	else
272 		id = readl_relaxed(addr + MACB_MID);
273 
274 	return MACB_BFEXT(IDNUM, id) >= 0x2;
275 }
276 
277 static void macb_set_hwaddr(struct macb *bp)
278 {
279 	u32 bottom;
280 	u16 top;
281 
282 	bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr));
283 	macb_or_gem_writel(bp, SA1B, bottom);
284 	top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));
285 	macb_or_gem_writel(bp, SA1T, top);
286 
287 	/* Clear unused address register sets */
288 	macb_or_gem_writel(bp, SA2B, 0);
289 	macb_or_gem_writel(bp, SA2T, 0);
290 	macb_or_gem_writel(bp, SA3B, 0);
291 	macb_or_gem_writel(bp, SA3T, 0);
292 	macb_or_gem_writel(bp, SA4B, 0);
293 	macb_or_gem_writel(bp, SA4T, 0);
294 }
295 
296 static void macb_get_hwaddr(struct macb *bp)
297 {
298 	u32 bottom;
299 	u16 top;
300 	u8 addr[6];
301 	int i;
302 
303 	/* Check all 4 address register for valid address */
304 	for (i = 0; i < 4; i++) {
305 		bottom = macb_or_gem_readl(bp, SA1B + i * 8);
306 		top = macb_or_gem_readl(bp, SA1T + i * 8);
307 
308 		addr[0] = bottom & 0xff;
309 		addr[1] = (bottom >> 8) & 0xff;
310 		addr[2] = (bottom >> 16) & 0xff;
311 		addr[3] = (bottom >> 24) & 0xff;
312 		addr[4] = top & 0xff;
313 		addr[5] = (top >> 8) & 0xff;
314 
315 		if (is_valid_ether_addr(addr)) {
316 			eth_hw_addr_set(bp->dev, addr);
317 			return;
318 		}
319 	}
320 
321 	dev_info(&bp->pdev->dev, "invalid hw address, using random\n");
322 	eth_hw_addr_random(bp->dev);
323 }
324 
325 static int macb_mdio_wait_for_idle(struct macb *bp)
326 {
327 	u32 val;
328 
329 	return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE),
330 				  1, MACB_MDIO_TIMEOUT);
331 }
332 
333 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum)
334 {
335 	struct macb *bp = bus->priv;
336 	int status;
337 
338 	status = pm_runtime_get_sync(&bp->pdev->dev);
339 	if (status < 0) {
340 		pm_runtime_put_noidle(&bp->pdev->dev);
341 		goto mdio_pm_exit;
342 	}
343 
344 	status = macb_mdio_wait_for_idle(bp);
345 	if (status < 0)
346 		goto mdio_read_exit;
347 
348 	if (regnum & MII_ADDR_C45) {
349 		macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
350 			    | MACB_BF(RW, MACB_MAN_C45_ADDR)
351 			    | MACB_BF(PHYA, mii_id)
352 			    | MACB_BF(REGA, (regnum >> 16) & 0x1F)
353 			    | MACB_BF(DATA, regnum & 0xFFFF)
354 			    | MACB_BF(CODE, MACB_MAN_C45_CODE)));
355 
356 		status = macb_mdio_wait_for_idle(bp);
357 		if (status < 0)
358 			goto mdio_read_exit;
359 
360 		macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
361 			    | MACB_BF(RW, MACB_MAN_C45_READ)
362 			    | MACB_BF(PHYA, mii_id)
363 			    | MACB_BF(REGA, (regnum >> 16) & 0x1F)
364 			    | MACB_BF(CODE, MACB_MAN_C45_CODE)));
365 	} else {
366 		macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)
367 				| MACB_BF(RW, MACB_MAN_C22_READ)
368 				| MACB_BF(PHYA, mii_id)
369 				| MACB_BF(REGA, regnum)
370 				| MACB_BF(CODE, MACB_MAN_C22_CODE)));
371 	}
372 
373 	status = macb_mdio_wait_for_idle(bp);
374 	if (status < 0)
375 		goto mdio_read_exit;
376 
377 	status = MACB_BFEXT(DATA, macb_readl(bp, MAN));
378 
379 mdio_read_exit:
380 	pm_runtime_mark_last_busy(&bp->pdev->dev);
381 	pm_runtime_put_autosuspend(&bp->pdev->dev);
382 mdio_pm_exit:
383 	return status;
384 }
385 
386 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum,
387 			   u16 value)
388 {
389 	struct macb *bp = bus->priv;
390 	int status;
391 
392 	status = pm_runtime_get_sync(&bp->pdev->dev);
393 	if (status < 0) {
394 		pm_runtime_put_noidle(&bp->pdev->dev);
395 		goto mdio_pm_exit;
396 	}
397 
398 	status = macb_mdio_wait_for_idle(bp);
399 	if (status < 0)
400 		goto mdio_write_exit;
401 
402 	if (regnum & MII_ADDR_C45) {
403 		macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
404 			    | MACB_BF(RW, MACB_MAN_C45_ADDR)
405 			    | MACB_BF(PHYA, mii_id)
406 			    | MACB_BF(REGA, (regnum >> 16) & 0x1F)
407 			    | MACB_BF(DATA, regnum & 0xFFFF)
408 			    | MACB_BF(CODE, MACB_MAN_C45_CODE)));
409 
410 		status = macb_mdio_wait_for_idle(bp);
411 		if (status < 0)
412 			goto mdio_write_exit;
413 
414 		macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
415 			    | MACB_BF(RW, MACB_MAN_C45_WRITE)
416 			    | MACB_BF(PHYA, mii_id)
417 			    | MACB_BF(REGA, (regnum >> 16) & 0x1F)
418 			    | MACB_BF(CODE, MACB_MAN_C45_CODE)
419 			    | MACB_BF(DATA, value)));
420 	} else {
421 		macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)
422 				| MACB_BF(RW, MACB_MAN_C22_WRITE)
423 				| MACB_BF(PHYA, mii_id)
424 				| MACB_BF(REGA, regnum)
425 				| MACB_BF(CODE, MACB_MAN_C22_CODE)
426 				| MACB_BF(DATA, value)));
427 	}
428 
429 	status = macb_mdio_wait_for_idle(bp);
430 	if (status < 0)
431 		goto mdio_write_exit;
432 
433 mdio_write_exit:
434 	pm_runtime_mark_last_busy(&bp->pdev->dev);
435 	pm_runtime_put_autosuspend(&bp->pdev->dev);
436 mdio_pm_exit:
437 	return status;
438 }
439 
440 static void macb_init_buffers(struct macb *bp)
441 {
442 	struct macb_queue *queue;
443 	unsigned int q;
444 
445 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
446 		queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));
447 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
448 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)
449 			queue_writel(queue, RBQPH,
450 				     upper_32_bits(queue->rx_ring_dma));
451 #endif
452 		queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
453 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
454 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)
455 			queue_writel(queue, TBQPH,
456 				     upper_32_bits(queue->tx_ring_dma));
457 #endif
458 	}
459 }
460 
461 /**
462  * macb_set_tx_clk() - Set a clock to a new frequency
463  * @bp:		pointer to struct macb
464  * @speed:	New frequency in Hz
465  */
466 static void macb_set_tx_clk(struct macb *bp, int speed)
467 {
468 	long ferr, rate, rate_rounded;
469 
470 	if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG))
471 		return;
472 
473 	/* In case of MII the PHY is the clock master */
474 	if (bp->phy_interface == PHY_INTERFACE_MODE_MII)
475 		return;
476 
477 	switch (speed) {
478 	case SPEED_10:
479 		rate = 2500000;
480 		break;
481 	case SPEED_100:
482 		rate = 25000000;
483 		break;
484 	case SPEED_1000:
485 		rate = 125000000;
486 		break;
487 	default:
488 		return;
489 	}
490 
491 	rate_rounded = clk_round_rate(bp->tx_clk, rate);
492 	if (rate_rounded < 0)
493 		return;
494 
495 	/* RGMII allows 50 ppm frequency error. Test and warn if this limit
496 	 * is not satisfied.
497 	 */
498 	ferr = abs(rate_rounded - rate);
499 	ferr = DIV_ROUND_UP(ferr, rate / 100000);
500 	if (ferr > 5)
501 		netdev_warn(bp->dev,
502 			    "unable to generate target frequency: %ld Hz\n",
503 			    rate);
504 
505 	if (clk_set_rate(bp->tx_clk, rate_rounded))
506 		netdev_err(bp->dev, "adjusting tx_clk failed.\n");
507 }
508 
509 static void macb_validate(struct phylink_config *config,
510 			  unsigned long *supported,
511 			  struct phylink_link_state *state)
512 {
513 	struct net_device *ndev = to_net_dev(config->dev);
514 	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
515 	struct macb *bp = netdev_priv(ndev);
516 
517 	/* We only support MII, RMII, GMII, RGMII & SGMII. */
518 	if (state->interface != PHY_INTERFACE_MODE_NA &&
519 	    state->interface != PHY_INTERFACE_MODE_MII &&
520 	    state->interface != PHY_INTERFACE_MODE_RMII &&
521 	    state->interface != PHY_INTERFACE_MODE_GMII &&
522 	    state->interface != PHY_INTERFACE_MODE_SGMII &&
523 	    state->interface != PHY_INTERFACE_MODE_10GBASER &&
524 	    !phy_interface_mode_is_rgmii(state->interface)) {
525 		linkmode_zero(supported);
526 		return;
527 	}
528 
529 	if (!macb_is_gem(bp) &&
530 	    (state->interface == PHY_INTERFACE_MODE_GMII ||
531 	     phy_interface_mode_is_rgmii(state->interface))) {
532 		linkmode_zero(supported);
533 		return;
534 	}
535 
536 	if (state->interface == PHY_INTERFACE_MODE_10GBASER &&
537 	    !(bp->caps & MACB_CAPS_HIGH_SPEED &&
538 	      bp->caps & MACB_CAPS_PCS)) {
539 		linkmode_zero(supported);
540 		return;
541 	}
542 
543 	phylink_set_port_modes(mask);
544 	phylink_set(mask, Autoneg);
545 	phylink_set(mask, Asym_Pause);
546 
547 	if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE &&
548 	    (state->interface == PHY_INTERFACE_MODE_NA ||
549 	     state->interface == PHY_INTERFACE_MODE_10GBASER)) {
550 		phylink_set_10g_modes(mask);
551 		phylink_set(mask, 10000baseKR_Full);
552 		if (state->interface != PHY_INTERFACE_MODE_NA)
553 			goto out;
554 	}
555 
556 	phylink_set(mask, 10baseT_Half);
557 	phylink_set(mask, 10baseT_Full);
558 	phylink_set(mask, 100baseT_Half);
559 	phylink_set(mask, 100baseT_Full);
560 
561 	if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE &&
562 	    (state->interface == PHY_INTERFACE_MODE_NA ||
563 	     state->interface == PHY_INTERFACE_MODE_GMII ||
564 	     state->interface == PHY_INTERFACE_MODE_SGMII ||
565 	     phy_interface_mode_is_rgmii(state->interface))) {
566 		phylink_set(mask, 1000baseT_Full);
567 		phylink_set(mask, 1000baseX_Full);
568 
569 		if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF))
570 			phylink_set(mask, 1000baseT_Half);
571 	}
572 out:
573 	linkmode_and(supported, supported, mask);
574 	linkmode_and(state->advertising, state->advertising, mask);
575 }
576 
577 static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode,
578 				 phy_interface_t interface, int speed,
579 				 int duplex)
580 {
581 	struct macb *bp = container_of(pcs, struct macb, phylink_pcs);
582 	u32 config;
583 
584 	config = gem_readl(bp, USX_CONTROL);
585 	config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config);
586 	config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config);
587 	config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS));
588 	config |= GEM_BIT(TX_EN);
589 	gem_writel(bp, USX_CONTROL, config);
590 }
591 
592 static void macb_usx_pcs_get_state(struct phylink_pcs *pcs,
593 				   struct phylink_link_state *state)
594 {
595 	struct macb *bp = container_of(pcs, struct macb, phylink_pcs);
596 	u32 val;
597 
598 	state->speed = SPEED_10000;
599 	state->duplex = 1;
600 	state->an_complete = 1;
601 
602 	val = gem_readl(bp, USX_STATUS);
603 	state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK));
604 	val = gem_readl(bp, NCFGR);
605 	if (val & GEM_BIT(PAE))
606 		state->pause = MLO_PAUSE_RX;
607 }
608 
609 static int macb_usx_pcs_config(struct phylink_pcs *pcs,
610 			       unsigned int mode,
611 			       phy_interface_t interface,
612 			       const unsigned long *advertising,
613 			       bool permit_pause_to_mac)
614 {
615 	struct macb *bp = container_of(pcs, struct macb, phylink_pcs);
616 
617 	gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) |
618 		   GEM_BIT(SIGNAL_OK));
619 
620 	return 0;
621 }
622 
623 static void macb_pcs_get_state(struct phylink_pcs *pcs,
624 			       struct phylink_link_state *state)
625 {
626 	state->link = 0;
627 }
628 
629 static void macb_pcs_an_restart(struct phylink_pcs *pcs)
630 {
631 	/* Not supported */
632 }
633 
634 static int macb_pcs_config(struct phylink_pcs *pcs,
635 			   unsigned int mode,
636 			   phy_interface_t interface,
637 			   const unsigned long *advertising,
638 			   bool permit_pause_to_mac)
639 {
640 	return 0;
641 }
642 
643 static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = {
644 	.pcs_get_state = macb_usx_pcs_get_state,
645 	.pcs_config = macb_usx_pcs_config,
646 	.pcs_link_up = macb_usx_pcs_link_up,
647 };
648 
649 static const struct phylink_pcs_ops macb_phylink_pcs_ops = {
650 	.pcs_get_state = macb_pcs_get_state,
651 	.pcs_an_restart = macb_pcs_an_restart,
652 	.pcs_config = macb_pcs_config,
653 };
654 
655 static void macb_mac_config(struct phylink_config *config, unsigned int mode,
656 			    const struct phylink_link_state *state)
657 {
658 	struct net_device *ndev = to_net_dev(config->dev);
659 	struct macb *bp = netdev_priv(ndev);
660 	unsigned long flags;
661 	u32 old_ctrl, ctrl;
662 	u32 old_ncr, ncr;
663 
664 	spin_lock_irqsave(&bp->lock, flags);
665 
666 	old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR);
667 	old_ncr = ncr = macb_or_gem_readl(bp, NCR);
668 
669 	if (bp->caps & MACB_CAPS_MACB_IS_EMAC) {
670 		if (state->interface == PHY_INTERFACE_MODE_RMII)
671 			ctrl |= MACB_BIT(RM9200_RMII);
672 	} else if (macb_is_gem(bp)) {
673 		ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL));
674 		ncr &= ~GEM_BIT(ENABLE_HS_MAC);
675 
676 		if (state->interface == PHY_INTERFACE_MODE_SGMII) {
677 			ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
678 		} else if (state->interface == PHY_INTERFACE_MODE_10GBASER) {
679 			ctrl |= GEM_BIT(PCSSEL);
680 			ncr |= GEM_BIT(ENABLE_HS_MAC);
681 		} else if (bp->caps & MACB_CAPS_MIIONRGMII &&
682 			   bp->phy_interface == PHY_INTERFACE_MODE_MII) {
683 			ncr |= MACB_BIT(MIIONRGMII);
684 		}
685 	}
686 
687 	/* Apply the new configuration, if any */
688 	if (old_ctrl ^ ctrl)
689 		macb_or_gem_writel(bp, NCFGR, ctrl);
690 
691 	if (old_ncr ^ ncr)
692 		macb_or_gem_writel(bp, NCR, ncr);
693 
694 	/* Disable AN for SGMII fixed link configuration, enable otherwise.
695 	 * Must be written after PCSSEL is set in NCFGR,
696 	 * otherwise writes will not take effect.
697 	 */
698 	if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) {
699 		u32 pcsctrl, old_pcsctrl;
700 
701 		old_pcsctrl = gem_readl(bp, PCSCNTRL);
702 		if (mode == MLO_AN_FIXED)
703 			pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG);
704 		else
705 			pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG);
706 		if (old_pcsctrl != pcsctrl)
707 			gem_writel(bp, PCSCNTRL, pcsctrl);
708 	}
709 
710 	spin_unlock_irqrestore(&bp->lock, flags);
711 }
712 
713 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode,
714 			       phy_interface_t interface)
715 {
716 	struct net_device *ndev = to_net_dev(config->dev);
717 	struct macb *bp = netdev_priv(ndev);
718 	struct macb_queue *queue;
719 	unsigned int q;
720 	u32 ctrl;
721 
722 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))
723 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
724 			queue_writel(queue, IDR,
725 				     bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));
726 
727 	/* Disable Rx and Tx */
728 	ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE));
729 	macb_writel(bp, NCR, ctrl);
730 
731 	netif_tx_stop_all_queues(ndev);
732 }
733 
734 static void macb_mac_link_up(struct phylink_config *config,
735 			     struct phy_device *phy,
736 			     unsigned int mode, phy_interface_t interface,
737 			     int speed, int duplex,
738 			     bool tx_pause, bool rx_pause)
739 {
740 	struct net_device *ndev = to_net_dev(config->dev);
741 	struct macb *bp = netdev_priv(ndev);
742 	struct macb_queue *queue;
743 	unsigned long flags;
744 	unsigned int q;
745 	u32 ctrl;
746 
747 	spin_lock_irqsave(&bp->lock, flags);
748 
749 	ctrl = macb_or_gem_readl(bp, NCFGR);
750 
751 	ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD));
752 
753 	if (speed == SPEED_100)
754 		ctrl |= MACB_BIT(SPD);
755 
756 	if (duplex)
757 		ctrl |= MACB_BIT(FD);
758 
759 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) {
760 		ctrl &= ~MACB_BIT(PAE);
761 		if (macb_is_gem(bp)) {
762 			ctrl &= ~GEM_BIT(GBE);
763 
764 			if (speed == SPEED_1000)
765 				ctrl |= GEM_BIT(GBE);
766 		}
767 
768 		if (rx_pause)
769 			ctrl |= MACB_BIT(PAE);
770 
771 		macb_set_tx_clk(bp, speed);
772 
773 		/* Initialize rings & buffers as clearing MACB_BIT(TE) in link down
774 		 * cleared the pipeline and control registers.
775 		 */
776 		bp->macbgem_ops.mog_init_rings(bp);
777 		macb_init_buffers(bp);
778 
779 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
780 			queue_writel(queue, IER,
781 				     bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));
782 	}
783 
784 	macb_or_gem_writel(bp, NCFGR, ctrl);
785 
786 	if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER)
787 		gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M,
788 							gem_readl(bp, HS_MAC_CONFIG)));
789 
790 	spin_unlock_irqrestore(&bp->lock, flags);
791 
792 	/* Enable Rx and Tx */
793 	macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE));
794 
795 	netif_tx_wake_all_queues(ndev);
796 }
797 
798 static int macb_mac_prepare(struct phylink_config *config, unsigned int mode,
799 			    phy_interface_t interface)
800 {
801 	struct net_device *ndev = to_net_dev(config->dev);
802 	struct macb *bp = netdev_priv(ndev);
803 
804 	if (interface == PHY_INTERFACE_MODE_10GBASER)
805 		bp->phylink_pcs.ops = &macb_phylink_usx_pcs_ops;
806 	else if (interface == PHY_INTERFACE_MODE_SGMII)
807 		bp->phylink_pcs.ops = &macb_phylink_pcs_ops;
808 	else
809 		bp->phylink_pcs.ops = NULL;
810 
811 	if (bp->phylink_pcs.ops)
812 		phylink_set_pcs(bp->phylink, &bp->phylink_pcs);
813 
814 	return 0;
815 }
816 
817 static const struct phylink_mac_ops macb_phylink_ops = {
818 	.validate = macb_validate,
819 	.mac_prepare = macb_mac_prepare,
820 	.mac_config = macb_mac_config,
821 	.mac_link_down = macb_mac_link_down,
822 	.mac_link_up = macb_mac_link_up,
823 };
824 
825 static bool macb_phy_handle_exists(struct device_node *dn)
826 {
827 	dn = of_parse_phandle(dn, "phy-handle", 0);
828 	of_node_put(dn);
829 	return dn != NULL;
830 }
831 
832 static int macb_phylink_connect(struct macb *bp)
833 {
834 	struct device_node *dn = bp->pdev->dev.of_node;
835 	struct net_device *dev = bp->dev;
836 	struct phy_device *phydev;
837 	int ret;
838 
839 	if (dn)
840 		ret = phylink_of_phy_connect(bp->phylink, dn, 0);
841 
842 	if (!dn || (ret && !macb_phy_handle_exists(dn))) {
843 		phydev = phy_find_first(bp->mii_bus);
844 		if (!phydev) {
845 			netdev_err(dev, "no PHY found\n");
846 			return -ENXIO;
847 		}
848 
849 		/* attach the mac to the phy */
850 		ret = phylink_connect_phy(bp->phylink, phydev);
851 	}
852 
853 	if (ret) {
854 		netdev_err(dev, "Could not attach PHY (%d)\n", ret);
855 		return ret;
856 	}
857 
858 	phylink_start(bp->phylink);
859 
860 	return 0;
861 }
862 
863 static void macb_get_pcs_fixed_state(struct phylink_config *config,
864 				     struct phylink_link_state *state)
865 {
866 	struct net_device *ndev = to_net_dev(config->dev);
867 	struct macb *bp = netdev_priv(ndev);
868 
869 	state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0;
870 }
871 
872 /* based on au1000_eth. c*/
873 static int macb_mii_probe(struct net_device *dev)
874 {
875 	struct macb *bp = netdev_priv(dev);
876 
877 	bp->phylink_config.dev = &dev->dev;
878 	bp->phylink_config.type = PHYLINK_NETDEV;
879 
880 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {
881 		bp->phylink_config.poll_fixed_state = true;
882 		bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;
883 	}
884 
885 	bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,
886 				     bp->phy_interface, &macb_phylink_ops);
887 	if (IS_ERR(bp->phylink)) {
888 		netdev_err(dev, "Could not create a phylink instance (%ld)\n",
889 			   PTR_ERR(bp->phylink));
890 		return PTR_ERR(bp->phylink);
891 	}
892 
893 	return 0;
894 }
895 
896 static int macb_mdiobus_register(struct macb *bp)
897 {
898 	struct device_node *child, *np = bp->pdev->dev.of_node;
899 
900 	/* If we have a child named mdio, probe it instead of looking for PHYs
901 	 * directly under the MAC node
902 	 */
903 	child = of_get_child_by_name(np, "mdio");
904 	if (child) {
905 		int ret = of_mdiobus_register(bp->mii_bus, child);
906 
907 		of_node_put(child);
908 		return ret;
909 	}
910 
911 	if (of_phy_is_fixed_link(np))
912 		return mdiobus_register(bp->mii_bus);
913 
914 	/* Only create the PHY from the device tree if at least one PHY is
915 	 * described. Otherwise scan the entire MDIO bus. We do this to support
916 	 * old device tree that did not follow the best practices and did not
917 	 * describe their network PHYs.
918 	 */
919 	for_each_available_child_of_node(np, child)
920 		if (of_mdiobus_child_is_phy(child)) {
921 			/* The loop increments the child refcount,
922 			 * decrement it before returning.
923 			 */
924 			of_node_put(child);
925 
926 			return of_mdiobus_register(bp->mii_bus, np);
927 		}
928 
929 	return mdiobus_register(bp->mii_bus);
930 }
931 
932 static int macb_mii_init(struct macb *bp)
933 {
934 	int err = -ENXIO;
935 
936 	/* Enable management port */
937 	macb_writel(bp, NCR, MACB_BIT(MPE));
938 
939 	bp->mii_bus = mdiobus_alloc();
940 	if (!bp->mii_bus) {
941 		err = -ENOMEM;
942 		goto err_out;
943 	}
944 
945 	bp->mii_bus->name = "MACB_mii_bus";
946 	bp->mii_bus->read = &macb_mdio_read;
947 	bp->mii_bus->write = &macb_mdio_write;
948 	snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
949 		 bp->pdev->name, bp->pdev->id);
950 	bp->mii_bus->priv = bp;
951 	bp->mii_bus->parent = &bp->pdev->dev;
952 
953 	dev_set_drvdata(&bp->dev->dev, bp->mii_bus);
954 
955 	err = macb_mdiobus_register(bp);
956 	if (err)
957 		goto err_out_free_mdiobus;
958 
959 	err = macb_mii_probe(bp->dev);
960 	if (err)
961 		goto err_out_unregister_bus;
962 
963 	return 0;
964 
965 err_out_unregister_bus:
966 	mdiobus_unregister(bp->mii_bus);
967 err_out_free_mdiobus:
968 	mdiobus_free(bp->mii_bus);
969 err_out:
970 	return err;
971 }
972 
973 static void macb_update_stats(struct macb *bp)
974 {
975 	u32 *p = &bp->hw_stats.macb.rx_pause_frames;
976 	u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1;
977 	int offset = MACB_PFR;
978 
979 	WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4);
980 
981 	for (; p < end; p++, offset += 4)
982 		*p += bp->macb_reg_readl(bp, offset);
983 }
984 
985 static int macb_halt_tx(struct macb *bp)
986 {
987 	unsigned long	halt_time, timeout;
988 	u32		status;
989 
990 	macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT));
991 
992 	timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT);
993 	do {
994 		halt_time = jiffies;
995 		status = macb_readl(bp, TSR);
996 		if (!(status & MACB_BIT(TGO)))
997 			return 0;
998 
999 		udelay(250);
1000 	} while (time_before(halt_time, timeout));
1001 
1002 	return -ETIMEDOUT;
1003 }
1004 
1005 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb)
1006 {
1007 	if (tx_skb->mapping) {
1008 		if (tx_skb->mapped_as_page)
1009 			dma_unmap_page(&bp->pdev->dev, tx_skb->mapping,
1010 				       tx_skb->size, DMA_TO_DEVICE);
1011 		else
1012 			dma_unmap_single(&bp->pdev->dev, tx_skb->mapping,
1013 					 tx_skb->size, DMA_TO_DEVICE);
1014 		tx_skb->mapping = 0;
1015 	}
1016 
1017 	if (tx_skb->skb) {
1018 		dev_kfree_skb_any(tx_skb->skb);
1019 		tx_skb->skb = NULL;
1020 	}
1021 }
1022 
1023 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr)
1024 {
1025 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1026 	struct macb_dma_desc_64 *desc_64;
1027 
1028 	if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
1029 		desc_64 = macb_64b_desc(bp, desc);
1030 		desc_64->addrh = upper_32_bits(addr);
1031 		/* The low bits of RX address contain the RX_USED bit, clearing
1032 		 * of which allows packet RX. Make sure the high bits are also
1033 		 * visible to HW at that point.
1034 		 */
1035 		dma_wmb();
1036 	}
1037 #endif
1038 	desc->addr = lower_32_bits(addr);
1039 }
1040 
1041 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc)
1042 {
1043 	dma_addr_t addr = 0;
1044 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1045 	struct macb_dma_desc_64 *desc_64;
1046 
1047 	if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
1048 		desc_64 = macb_64b_desc(bp, desc);
1049 		addr = ((u64)(desc_64->addrh) << 32);
1050 	}
1051 #endif
1052 	addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));
1053 	return addr;
1054 }
1055 
1056 static void macb_tx_error_task(struct work_struct *work)
1057 {
1058 	struct macb_queue	*queue = container_of(work, struct macb_queue,
1059 						      tx_error_task);
1060 	struct macb		*bp = queue->bp;
1061 	struct macb_tx_skb	*tx_skb;
1062 	struct macb_dma_desc	*desc;
1063 	struct sk_buff		*skb;
1064 	unsigned int		tail;
1065 	unsigned long		flags;
1066 
1067 	netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n",
1068 		    (unsigned int)(queue - bp->queues),
1069 		    queue->tx_tail, queue->tx_head);
1070 
1071 	/* Prevent the queue IRQ handlers from running: each of them may call
1072 	 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue().
1073 	 * As explained below, we have to halt the transmission before updating
1074 	 * TBQP registers so we call netif_tx_stop_all_queues() to notify the
1075 	 * network engine about the macb/gem being halted.
1076 	 */
1077 	spin_lock_irqsave(&bp->lock, flags);
1078 
1079 	/* Make sure nobody is trying to queue up new packets */
1080 	netif_tx_stop_all_queues(bp->dev);
1081 
1082 	/* Stop transmission now
1083 	 * (in case we have just queued new packets)
1084 	 * macb/gem must be halted to write TBQP register
1085 	 */
1086 	if (macb_halt_tx(bp))
1087 		/* Just complain for now, reinitializing TX path can be good */
1088 		netdev_err(bp->dev, "BUG: halt tx timed out\n");
1089 
1090 	/* Treat frames in TX queue including the ones that caused the error.
1091 	 * Free transmit buffers in upper layer.
1092 	 */
1093 	for (tail = queue->tx_tail; tail != queue->tx_head; tail++) {
1094 		u32	ctrl;
1095 
1096 		desc = macb_tx_desc(queue, tail);
1097 		ctrl = desc->ctrl;
1098 		tx_skb = macb_tx_skb(queue, tail);
1099 		skb = tx_skb->skb;
1100 
1101 		if (ctrl & MACB_BIT(TX_USED)) {
1102 			/* skb is set for the last buffer of the frame */
1103 			while (!skb) {
1104 				macb_tx_unmap(bp, tx_skb);
1105 				tail++;
1106 				tx_skb = macb_tx_skb(queue, tail);
1107 				skb = tx_skb->skb;
1108 			}
1109 
1110 			/* ctrl still refers to the first buffer descriptor
1111 			 * since it's the only one written back by the hardware
1112 			 */
1113 			if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) {
1114 				netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n",
1115 					    macb_tx_ring_wrap(bp, tail),
1116 					    skb->data);
1117 				bp->dev->stats.tx_packets++;
1118 				queue->stats.tx_packets++;
1119 				bp->dev->stats.tx_bytes += skb->len;
1120 				queue->stats.tx_bytes += skb->len;
1121 			}
1122 		} else {
1123 			/* "Buffers exhausted mid-frame" errors may only happen
1124 			 * if the driver is buggy, so complain loudly about
1125 			 * those. Statistics are updated by hardware.
1126 			 */
1127 			if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED))
1128 				netdev_err(bp->dev,
1129 					   "BUG: TX buffers exhausted mid-frame\n");
1130 
1131 			desc->ctrl = ctrl | MACB_BIT(TX_USED);
1132 		}
1133 
1134 		macb_tx_unmap(bp, tx_skb);
1135 	}
1136 
1137 	/* Set end of TX queue */
1138 	desc = macb_tx_desc(queue, 0);
1139 	macb_set_addr(bp, desc, 0);
1140 	desc->ctrl = MACB_BIT(TX_USED);
1141 
1142 	/* Make descriptor updates visible to hardware */
1143 	wmb();
1144 
1145 	/* Reinitialize the TX desc queue */
1146 	queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
1147 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1148 	if (bp->hw_dma_cap & HW_DMA_CAP_64B)
1149 		queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma));
1150 #endif
1151 	/* Make TX ring reflect state of hardware */
1152 	queue->tx_head = 0;
1153 	queue->tx_tail = 0;
1154 
1155 	/* Housework before enabling TX IRQ */
1156 	macb_writel(bp, TSR, macb_readl(bp, TSR));
1157 	queue_writel(queue, IER, MACB_TX_INT_FLAGS);
1158 
1159 	/* Now we are ready to start transmission again */
1160 	netif_tx_start_all_queues(bp->dev);
1161 	macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
1162 
1163 	spin_unlock_irqrestore(&bp->lock, flags);
1164 }
1165 
1166 static void macb_tx_interrupt(struct macb_queue *queue)
1167 {
1168 	unsigned int tail;
1169 	unsigned int head;
1170 	u32 status;
1171 	struct macb *bp = queue->bp;
1172 	u16 queue_index = queue - bp->queues;
1173 
1174 	status = macb_readl(bp, TSR);
1175 	macb_writel(bp, TSR, status);
1176 
1177 	if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1178 		queue_writel(queue, ISR, MACB_BIT(TCOMP));
1179 
1180 	netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n",
1181 		    (unsigned long)status);
1182 
1183 	head = queue->tx_head;
1184 	for (tail = queue->tx_tail; tail != head; tail++) {
1185 		struct macb_tx_skb	*tx_skb;
1186 		struct sk_buff		*skb;
1187 		struct macb_dma_desc	*desc;
1188 		u32			ctrl;
1189 
1190 		desc = macb_tx_desc(queue, tail);
1191 
1192 		/* Make hw descriptor updates visible to CPU */
1193 		rmb();
1194 
1195 		ctrl = desc->ctrl;
1196 
1197 		/* TX_USED bit is only set by hardware on the very first buffer
1198 		 * descriptor of the transmitted frame.
1199 		 */
1200 		if (!(ctrl & MACB_BIT(TX_USED)))
1201 			break;
1202 
1203 		/* Process all buffers of the current transmitted frame */
1204 		for (;; tail++) {
1205 			tx_skb = macb_tx_skb(queue, tail);
1206 			skb = tx_skb->skb;
1207 
1208 			/* First, update TX stats if needed */
1209 			if (skb) {
1210 				if (unlikely(skb_shinfo(skb)->tx_flags &
1211 					     SKBTX_HW_TSTAMP) &&
1212 				    gem_ptp_do_txstamp(queue, skb, desc) == 0) {
1213 					/* skb now belongs to timestamp buffer
1214 					 * and will be removed later
1215 					 */
1216 					tx_skb->skb = NULL;
1217 				}
1218 				netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n",
1219 					    macb_tx_ring_wrap(bp, tail),
1220 					    skb->data);
1221 				bp->dev->stats.tx_packets++;
1222 				queue->stats.tx_packets++;
1223 				bp->dev->stats.tx_bytes += skb->len;
1224 				queue->stats.tx_bytes += skb->len;
1225 			}
1226 
1227 			/* Now we can safely release resources */
1228 			macb_tx_unmap(bp, tx_skb);
1229 
1230 			/* skb is set only for the last buffer of the frame.
1231 			 * WARNING: at this point skb has been freed by
1232 			 * macb_tx_unmap().
1233 			 */
1234 			if (skb)
1235 				break;
1236 		}
1237 	}
1238 
1239 	queue->tx_tail = tail;
1240 	if (__netif_subqueue_stopped(bp->dev, queue_index) &&
1241 	    CIRC_CNT(queue->tx_head, queue->tx_tail,
1242 		     bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp))
1243 		netif_wake_subqueue(bp->dev, queue_index);
1244 }
1245 
1246 static void gem_rx_refill(struct macb_queue *queue)
1247 {
1248 	unsigned int		entry;
1249 	struct sk_buff		*skb;
1250 	dma_addr_t		paddr;
1251 	struct macb *bp = queue->bp;
1252 	struct macb_dma_desc *desc;
1253 
1254 	while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,
1255 			bp->rx_ring_size) > 0) {
1256 		entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);
1257 
1258 		/* Make hw descriptor updates visible to CPU */
1259 		rmb();
1260 
1261 		queue->rx_prepared_head++;
1262 		desc = macb_rx_desc(queue, entry);
1263 
1264 		if (!queue->rx_skbuff[entry]) {
1265 			/* allocate sk_buff for this free entry in ring */
1266 			skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);
1267 			if (unlikely(!skb)) {
1268 				netdev_err(bp->dev,
1269 					   "Unable to allocate sk_buff\n");
1270 				break;
1271 			}
1272 
1273 			/* now fill corresponding descriptor entry */
1274 			paddr = dma_map_single(&bp->pdev->dev, skb->data,
1275 					       bp->rx_buffer_size,
1276 					       DMA_FROM_DEVICE);
1277 			if (dma_mapping_error(&bp->pdev->dev, paddr)) {
1278 				dev_kfree_skb(skb);
1279 				break;
1280 			}
1281 
1282 			queue->rx_skbuff[entry] = skb;
1283 
1284 			if (entry == bp->rx_ring_size - 1)
1285 				paddr |= MACB_BIT(RX_WRAP);
1286 			desc->ctrl = 0;
1287 			/* Setting addr clears RX_USED and allows reception,
1288 			 * make sure ctrl is cleared first to avoid a race.
1289 			 */
1290 			dma_wmb();
1291 			macb_set_addr(bp, desc, paddr);
1292 
1293 			/* properly align Ethernet header */
1294 			skb_reserve(skb, NET_IP_ALIGN);
1295 		} else {
1296 			desc->ctrl = 0;
1297 			dma_wmb();
1298 			desc->addr &= ~MACB_BIT(RX_USED);
1299 		}
1300 	}
1301 
1302 	/* Make descriptor updates visible to hardware */
1303 	wmb();
1304 
1305 	netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",
1306 			queue, queue->rx_prepared_head, queue->rx_tail);
1307 }
1308 
1309 /* Mark DMA descriptors from begin up to and not including end as unused */
1310 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin,
1311 				  unsigned int end)
1312 {
1313 	unsigned int frag;
1314 
1315 	for (frag = begin; frag != end; frag++) {
1316 		struct macb_dma_desc *desc = macb_rx_desc(queue, frag);
1317 
1318 		desc->addr &= ~MACB_BIT(RX_USED);
1319 	}
1320 
1321 	/* Make descriptor updates visible to hardware */
1322 	wmb();
1323 
1324 	/* When this happens, the hardware stats registers for
1325 	 * whatever caused this is updated, so we don't have to record
1326 	 * anything.
1327 	 */
1328 }
1329 
1330 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi,
1331 		  int budget)
1332 {
1333 	struct macb *bp = queue->bp;
1334 	unsigned int		len;
1335 	unsigned int		entry;
1336 	struct sk_buff		*skb;
1337 	struct macb_dma_desc	*desc;
1338 	int			count = 0;
1339 
1340 	while (count < budget) {
1341 		u32 ctrl;
1342 		dma_addr_t addr;
1343 		bool rxused;
1344 
1345 		entry = macb_rx_ring_wrap(bp, queue->rx_tail);
1346 		desc = macb_rx_desc(queue, entry);
1347 
1348 		/* Make hw descriptor updates visible to CPU */
1349 		rmb();
1350 
1351 		rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;
1352 		addr = macb_get_addr(bp, desc);
1353 
1354 		if (!rxused)
1355 			break;
1356 
1357 		/* Ensure ctrl is at least as up-to-date as rxused */
1358 		dma_rmb();
1359 
1360 		ctrl = desc->ctrl;
1361 
1362 		queue->rx_tail++;
1363 		count++;
1364 
1365 		if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) {
1366 			netdev_err(bp->dev,
1367 				   "not whole frame pointed by descriptor\n");
1368 			bp->dev->stats.rx_dropped++;
1369 			queue->stats.rx_dropped++;
1370 			break;
1371 		}
1372 		skb = queue->rx_skbuff[entry];
1373 		if (unlikely(!skb)) {
1374 			netdev_err(bp->dev,
1375 				   "inconsistent Rx descriptor chain\n");
1376 			bp->dev->stats.rx_dropped++;
1377 			queue->stats.rx_dropped++;
1378 			break;
1379 		}
1380 		/* now everything is ready for receiving packet */
1381 		queue->rx_skbuff[entry] = NULL;
1382 		len = ctrl & bp->rx_frm_len_mask;
1383 
1384 		netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);
1385 
1386 		skb_put(skb, len);
1387 		dma_unmap_single(&bp->pdev->dev, addr,
1388 				 bp->rx_buffer_size, DMA_FROM_DEVICE);
1389 
1390 		skb->protocol = eth_type_trans(skb, bp->dev);
1391 		skb_checksum_none_assert(skb);
1392 		if (bp->dev->features & NETIF_F_RXCSUM &&
1393 		    !(bp->dev->flags & IFF_PROMISC) &&
1394 		    GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK)
1395 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1396 
1397 		bp->dev->stats.rx_packets++;
1398 		queue->stats.rx_packets++;
1399 		bp->dev->stats.rx_bytes += skb->len;
1400 		queue->stats.rx_bytes += skb->len;
1401 
1402 		gem_ptp_do_rxstamp(bp, skb, desc);
1403 
1404 #if defined(DEBUG) && defined(VERBOSE_DEBUG)
1405 		netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",
1406 			    skb->len, skb->csum);
1407 		print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1,
1408 			       skb_mac_header(skb), 16, true);
1409 		print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1,
1410 			       skb->data, 32, true);
1411 #endif
1412 
1413 		napi_gro_receive(napi, skb);
1414 	}
1415 
1416 	gem_rx_refill(queue);
1417 
1418 	return count;
1419 }
1420 
1421 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi,
1422 			 unsigned int first_frag, unsigned int last_frag)
1423 {
1424 	unsigned int len;
1425 	unsigned int frag;
1426 	unsigned int offset;
1427 	struct sk_buff *skb;
1428 	struct macb_dma_desc *desc;
1429 	struct macb *bp = queue->bp;
1430 
1431 	desc = macb_rx_desc(queue, last_frag);
1432 	len = desc->ctrl & bp->rx_frm_len_mask;
1433 
1434 	netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",
1435 		macb_rx_ring_wrap(bp, first_frag),
1436 		macb_rx_ring_wrap(bp, last_frag), len);
1437 
1438 	/* The ethernet header starts NET_IP_ALIGN bytes into the
1439 	 * first buffer. Since the header is 14 bytes, this makes the
1440 	 * payload word-aligned.
1441 	 *
1442 	 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy
1443 	 * the two padding bytes into the skb so that we avoid hitting
1444 	 * the slowpath in memcpy(), and pull them off afterwards.
1445 	 */
1446 	skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN);
1447 	if (!skb) {
1448 		bp->dev->stats.rx_dropped++;
1449 		for (frag = first_frag; ; frag++) {
1450 			desc = macb_rx_desc(queue, frag);
1451 			desc->addr &= ~MACB_BIT(RX_USED);
1452 			if (frag == last_frag)
1453 				break;
1454 		}
1455 
1456 		/* Make descriptor updates visible to hardware */
1457 		wmb();
1458 
1459 		return 1;
1460 	}
1461 
1462 	offset = 0;
1463 	len += NET_IP_ALIGN;
1464 	skb_checksum_none_assert(skb);
1465 	skb_put(skb, len);
1466 
1467 	for (frag = first_frag; ; frag++) {
1468 		unsigned int frag_len = bp->rx_buffer_size;
1469 
1470 		if (offset + frag_len > len) {
1471 			if (unlikely(frag != last_frag)) {
1472 				dev_kfree_skb_any(skb);
1473 				return -1;
1474 			}
1475 			frag_len = len - offset;
1476 		}
1477 		skb_copy_to_linear_data_offset(skb, offset,
1478 					       macb_rx_buffer(queue, frag),
1479 					       frag_len);
1480 		offset += bp->rx_buffer_size;
1481 		desc = macb_rx_desc(queue, frag);
1482 		desc->addr &= ~MACB_BIT(RX_USED);
1483 
1484 		if (frag == last_frag)
1485 			break;
1486 	}
1487 
1488 	/* Make descriptor updates visible to hardware */
1489 	wmb();
1490 
1491 	__skb_pull(skb, NET_IP_ALIGN);
1492 	skb->protocol = eth_type_trans(skb, bp->dev);
1493 
1494 	bp->dev->stats.rx_packets++;
1495 	bp->dev->stats.rx_bytes += skb->len;
1496 	netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",
1497 		    skb->len, skb->csum);
1498 	napi_gro_receive(napi, skb);
1499 
1500 	return 0;
1501 }
1502 
1503 static inline void macb_init_rx_ring(struct macb_queue *queue)
1504 {
1505 	struct macb *bp = queue->bp;
1506 	dma_addr_t addr;
1507 	struct macb_dma_desc *desc = NULL;
1508 	int i;
1509 
1510 	addr = queue->rx_buffers_dma;
1511 	for (i = 0; i < bp->rx_ring_size; i++) {
1512 		desc = macb_rx_desc(queue, i);
1513 		macb_set_addr(bp, desc, addr);
1514 		desc->ctrl = 0;
1515 		addr += bp->rx_buffer_size;
1516 	}
1517 	desc->addr |= MACB_BIT(RX_WRAP);
1518 	queue->rx_tail = 0;
1519 }
1520 
1521 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi,
1522 		   int budget)
1523 {
1524 	struct macb *bp = queue->bp;
1525 	bool reset_rx_queue = false;
1526 	int received = 0;
1527 	unsigned int tail;
1528 	int first_frag = -1;
1529 
1530 	for (tail = queue->rx_tail; budget > 0; tail++) {
1531 		struct macb_dma_desc *desc = macb_rx_desc(queue, tail);
1532 		u32 ctrl;
1533 
1534 		/* Make hw descriptor updates visible to CPU */
1535 		rmb();
1536 
1537 		if (!(desc->addr & MACB_BIT(RX_USED)))
1538 			break;
1539 
1540 		/* Ensure ctrl is at least as up-to-date as addr */
1541 		dma_rmb();
1542 
1543 		ctrl = desc->ctrl;
1544 
1545 		if (ctrl & MACB_BIT(RX_SOF)) {
1546 			if (first_frag != -1)
1547 				discard_partial_frame(queue, first_frag, tail);
1548 			first_frag = tail;
1549 		}
1550 
1551 		if (ctrl & MACB_BIT(RX_EOF)) {
1552 			int dropped;
1553 
1554 			if (unlikely(first_frag == -1)) {
1555 				reset_rx_queue = true;
1556 				continue;
1557 			}
1558 
1559 			dropped = macb_rx_frame(queue, napi, first_frag, tail);
1560 			first_frag = -1;
1561 			if (unlikely(dropped < 0)) {
1562 				reset_rx_queue = true;
1563 				continue;
1564 			}
1565 			if (!dropped) {
1566 				received++;
1567 				budget--;
1568 			}
1569 		}
1570 	}
1571 
1572 	if (unlikely(reset_rx_queue)) {
1573 		unsigned long flags;
1574 		u32 ctrl;
1575 
1576 		netdev_err(bp->dev, "RX queue corruption: reset it\n");
1577 
1578 		spin_lock_irqsave(&bp->lock, flags);
1579 
1580 		ctrl = macb_readl(bp, NCR);
1581 		macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
1582 
1583 		macb_init_rx_ring(queue);
1584 		queue_writel(queue, RBQP, queue->rx_ring_dma);
1585 
1586 		macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
1587 
1588 		spin_unlock_irqrestore(&bp->lock, flags);
1589 		return received;
1590 	}
1591 
1592 	if (first_frag != -1)
1593 		queue->rx_tail = first_frag;
1594 	else
1595 		queue->rx_tail = tail;
1596 
1597 	return received;
1598 }
1599 
1600 static int macb_poll(struct napi_struct *napi, int budget)
1601 {
1602 	struct macb_queue *queue = container_of(napi, struct macb_queue, napi);
1603 	struct macb *bp = queue->bp;
1604 	int work_done;
1605 	u32 status;
1606 
1607 	status = macb_readl(bp, RSR);
1608 	macb_writel(bp, RSR, status);
1609 
1610 	netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n",
1611 		    (unsigned long)status, budget);
1612 
1613 	work_done = bp->macbgem_ops.mog_rx(queue, napi, budget);
1614 	if (work_done < budget) {
1615 		napi_complete_done(napi, work_done);
1616 
1617 		/* Packets received while interrupts were disabled */
1618 		status = macb_readl(bp, RSR);
1619 		if (status) {
1620 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1621 				queue_writel(queue, ISR, MACB_BIT(RCOMP));
1622 			napi_reschedule(napi);
1623 		} else {
1624 			queue_writel(queue, IER, bp->rx_intr_mask);
1625 		}
1626 	}
1627 
1628 	/* TODO: Handle errors */
1629 
1630 	return work_done;
1631 }
1632 
1633 static void macb_hresp_error_task(struct tasklet_struct *t)
1634 {
1635 	struct macb *bp = from_tasklet(bp, t, hresp_err_tasklet);
1636 	struct net_device *dev = bp->dev;
1637 	struct macb_queue *queue;
1638 	unsigned int q;
1639 	u32 ctrl;
1640 
1641 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
1642 		queue_writel(queue, IDR, bp->rx_intr_mask |
1643 					 MACB_TX_INT_FLAGS |
1644 					 MACB_BIT(HRESP));
1645 	}
1646 	ctrl = macb_readl(bp, NCR);
1647 	ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
1648 	macb_writel(bp, NCR, ctrl);
1649 
1650 	netif_tx_stop_all_queues(dev);
1651 	netif_carrier_off(dev);
1652 
1653 	bp->macbgem_ops.mog_init_rings(bp);
1654 
1655 	/* Initialize TX and RX buffers */
1656 	macb_init_buffers(bp);
1657 
1658 	/* Enable interrupts */
1659 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
1660 		queue_writel(queue, IER,
1661 			     bp->rx_intr_mask |
1662 			     MACB_TX_INT_FLAGS |
1663 			     MACB_BIT(HRESP));
1664 
1665 	ctrl |= MACB_BIT(RE) | MACB_BIT(TE);
1666 	macb_writel(bp, NCR, ctrl);
1667 
1668 	netif_carrier_on(dev);
1669 	netif_tx_start_all_queues(dev);
1670 }
1671 
1672 static void macb_tx_restart(struct macb_queue *queue)
1673 {
1674 	unsigned int head = queue->tx_head;
1675 	unsigned int tail = queue->tx_tail;
1676 	struct macb *bp = queue->bp;
1677 
1678 	if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1679 		queue_writel(queue, ISR, MACB_BIT(TXUBR));
1680 
1681 	if (head == tail)
1682 		return;
1683 
1684 	macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
1685 }
1686 
1687 static irqreturn_t macb_wol_interrupt(int irq, void *dev_id)
1688 {
1689 	struct macb_queue *queue = dev_id;
1690 	struct macb *bp = queue->bp;
1691 	u32 status;
1692 
1693 	status = queue_readl(queue, ISR);
1694 
1695 	if (unlikely(!status))
1696 		return IRQ_NONE;
1697 
1698 	spin_lock(&bp->lock);
1699 
1700 	if (status & MACB_BIT(WOL)) {
1701 		queue_writel(queue, IDR, MACB_BIT(WOL));
1702 		macb_writel(bp, WOL, 0);
1703 		netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n",
1704 			    (unsigned int)(queue - bp->queues),
1705 			    (unsigned long)status);
1706 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1707 			queue_writel(queue, ISR, MACB_BIT(WOL));
1708 		pm_wakeup_event(&bp->pdev->dev, 0);
1709 	}
1710 
1711 	spin_unlock(&bp->lock);
1712 
1713 	return IRQ_HANDLED;
1714 }
1715 
1716 static irqreturn_t gem_wol_interrupt(int irq, void *dev_id)
1717 {
1718 	struct macb_queue *queue = dev_id;
1719 	struct macb *bp = queue->bp;
1720 	u32 status;
1721 
1722 	status = queue_readl(queue, ISR);
1723 
1724 	if (unlikely(!status))
1725 		return IRQ_NONE;
1726 
1727 	spin_lock(&bp->lock);
1728 
1729 	if (status & GEM_BIT(WOL)) {
1730 		queue_writel(queue, IDR, GEM_BIT(WOL));
1731 		gem_writel(bp, WOL, 0);
1732 		netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n",
1733 			    (unsigned int)(queue - bp->queues),
1734 			    (unsigned long)status);
1735 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1736 			queue_writel(queue, ISR, GEM_BIT(WOL));
1737 		pm_wakeup_event(&bp->pdev->dev, 0);
1738 	}
1739 
1740 	spin_unlock(&bp->lock);
1741 
1742 	return IRQ_HANDLED;
1743 }
1744 
1745 static irqreturn_t macb_interrupt(int irq, void *dev_id)
1746 {
1747 	struct macb_queue *queue = dev_id;
1748 	struct macb *bp = queue->bp;
1749 	struct net_device *dev = bp->dev;
1750 	u32 status, ctrl;
1751 
1752 	status = queue_readl(queue, ISR);
1753 
1754 	if (unlikely(!status))
1755 		return IRQ_NONE;
1756 
1757 	spin_lock(&bp->lock);
1758 
1759 	while (status) {
1760 		/* close possible race with dev_close */
1761 		if (unlikely(!netif_running(dev))) {
1762 			queue_writel(queue, IDR, -1);
1763 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1764 				queue_writel(queue, ISR, -1);
1765 			break;
1766 		}
1767 
1768 		netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n",
1769 			    (unsigned int)(queue - bp->queues),
1770 			    (unsigned long)status);
1771 
1772 		if (status & bp->rx_intr_mask) {
1773 			/* There's no point taking any more interrupts
1774 			 * until we have processed the buffers. The
1775 			 * scheduling call may fail if the poll routine
1776 			 * is already scheduled, so disable interrupts
1777 			 * now.
1778 			 */
1779 			queue_writel(queue, IDR, bp->rx_intr_mask);
1780 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1781 				queue_writel(queue, ISR, MACB_BIT(RCOMP));
1782 
1783 			if (napi_schedule_prep(&queue->napi)) {
1784 				netdev_vdbg(bp->dev, "scheduling RX softirq\n");
1785 				__napi_schedule(&queue->napi);
1786 			}
1787 		}
1788 
1789 		if (unlikely(status & (MACB_TX_ERR_FLAGS))) {
1790 			queue_writel(queue, IDR, MACB_TX_INT_FLAGS);
1791 			schedule_work(&queue->tx_error_task);
1792 
1793 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1794 				queue_writel(queue, ISR, MACB_TX_ERR_FLAGS);
1795 
1796 			break;
1797 		}
1798 
1799 		if (status & MACB_BIT(TCOMP))
1800 			macb_tx_interrupt(queue);
1801 
1802 		if (status & MACB_BIT(TXUBR))
1803 			macb_tx_restart(queue);
1804 
1805 		/* Link change detection isn't possible with RMII, so we'll
1806 		 * add that if/when we get our hands on a full-blown MII PHY.
1807 		 */
1808 
1809 		/* There is a hardware issue under heavy load where DMA can
1810 		 * stop, this causes endless "used buffer descriptor read"
1811 		 * interrupts but it can be cleared by re-enabling RX. See
1812 		 * the at91rm9200 manual, section 41.3.1 or the Zynq manual
1813 		 * section 16.7.4 for details. RXUBR is only enabled for
1814 		 * these two versions.
1815 		 */
1816 		if (status & MACB_BIT(RXUBR)) {
1817 			ctrl = macb_readl(bp, NCR);
1818 			macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
1819 			wmb();
1820 			macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
1821 
1822 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1823 				queue_writel(queue, ISR, MACB_BIT(RXUBR));
1824 		}
1825 
1826 		if (status & MACB_BIT(ISR_ROVR)) {
1827 			/* We missed at least one packet */
1828 			if (macb_is_gem(bp))
1829 				bp->hw_stats.gem.rx_overruns++;
1830 			else
1831 				bp->hw_stats.macb.rx_overruns++;
1832 
1833 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1834 				queue_writel(queue, ISR, MACB_BIT(ISR_ROVR));
1835 		}
1836 
1837 		if (status & MACB_BIT(HRESP)) {
1838 			tasklet_schedule(&bp->hresp_err_tasklet);
1839 			netdev_err(dev, "DMA bus error: HRESP not OK\n");
1840 
1841 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1842 				queue_writel(queue, ISR, MACB_BIT(HRESP));
1843 		}
1844 		status = queue_readl(queue, ISR);
1845 	}
1846 
1847 	spin_unlock(&bp->lock);
1848 
1849 	return IRQ_HANDLED;
1850 }
1851 
1852 #ifdef CONFIG_NET_POLL_CONTROLLER
1853 /* Polling receive - used by netconsole and other diagnostic tools
1854  * to allow network i/o with interrupts disabled.
1855  */
1856 static void macb_poll_controller(struct net_device *dev)
1857 {
1858 	struct macb *bp = netdev_priv(dev);
1859 	struct macb_queue *queue;
1860 	unsigned long flags;
1861 	unsigned int q;
1862 
1863 	local_irq_save(flags);
1864 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
1865 		macb_interrupt(dev->irq, queue);
1866 	local_irq_restore(flags);
1867 }
1868 #endif
1869 
1870 static unsigned int macb_tx_map(struct macb *bp,
1871 				struct macb_queue *queue,
1872 				struct sk_buff *skb,
1873 				unsigned int hdrlen)
1874 {
1875 	dma_addr_t mapping;
1876 	unsigned int len, entry, i, tx_head = queue->tx_head;
1877 	struct macb_tx_skb *tx_skb = NULL;
1878 	struct macb_dma_desc *desc;
1879 	unsigned int offset, size, count = 0;
1880 	unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags;
1881 	unsigned int eof = 1, mss_mfs = 0;
1882 	u32 ctrl, lso_ctrl = 0, seq_ctrl = 0;
1883 
1884 	/* LSO */
1885 	if (skb_shinfo(skb)->gso_size != 0) {
1886 		if (ip_hdr(skb)->protocol == IPPROTO_UDP)
1887 			/* UDP - UFO */
1888 			lso_ctrl = MACB_LSO_UFO_ENABLE;
1889 		else
1890 			/* TCP - TSO */
1891 			lso_ctrl = MACB_LSO_TSO_ENABLE;
1892 	}
1893 
1894 	/* First, map non-paged data */
1895 	len = skb_headlen(skb);
1896 
1897 	/* first buffer length */
1898 	size = hdrlen;
1899 
1900 	offset = 0;
1901 	while (len) {
1902 		entry = macb_tx_ring_wrap(bp, tx_head);
1903 		tx_skb = &queue->tx_skb[entry];
1904 
1905 		mapping = dma_map_single(&bp->pdev->dev,
1906 					 skb->data + offset,
1907 					 size, DMA_TO_DEVICE);
1908 		if (dma_mapping_error(&bp->pdev->dev, mapping))
1909 			goto dma_error;
1910 
1911 		/* Save info to properly release resources */
1912 		tx_skb->skb = NULL;
1913 		tx_skb->mapping = mapping;
1914 		tx_skb->size = size;
1915 		tx_skb->mapped_as_page = false;
1916 
1917 		len -= size;
1918 		offset += size;
1919 		count++;
1920 		tx_head++;
1921 
1922 		size = min(len, bp->max_tx_length);
1923 	}
1924 
1925 	/* Then, map paged data from fragments */
1926 	for (f = 0; f < nr_frags; f++) {
1927 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
1928 
1929 		len = skb_frag_size(frag);
1930 		offset = 0;
1931 		while (len) {
1932 			size = min(len, bp->max_tx_length);
1933 			entry = macb_tx_ring_wrap(bp, tx_head);
1934 			tx_skb = &queue->tx_skb[entry];
1935 
1936 			mapping = skb_frag_dma_map(&bp->pdev->dev, frag,
1937 						   offset, size, DMA_TO_DEVICE);
1938 			if (dma_mapping_error(&bp->pdev->dev, mapping))
1939 				goto dma_error;
1940 
1941 			/* Save info to properly release resources */
1942 			tx_skb->skb = NULL;
1943 			tx_skb->mapping = mapping;
1944 			tx_skb->size = size;
1945 			tx_skb->mapped_as_page = true;
1946 
1947 			len -= size;
1948 			offset += size;
1949 			count++;
1950 			tx_head++;
1951 		}
1952 	}
1953 
1954 	/* Should never happen */
1955 	if (unlikely(!tx_skb)) {
1956 		netdev_err(bp->dev, "BUG! empty skb!\n");
1957 		return 0;
1958 	}
1959 
1960 	/* This is the last buffer of the frame: save socket buffer */
1961 	tx_skb->skb = skb;
1962 
1963 	/* Update TX ring: update buffer descriptors in reverse order
1964 	 * to avoid race condition
1965 	 */
1966 
1967 	/* Set 'TX_USED' bit in buffer descriptor at tx_head position
1968 	 * to set the end of TX queue
1969 	 */
1970 	i = tx_head;
1971 	entry = macb_tx_ring_wrap(bp, i);
1972 	ctrl = MACB_BIT(TX_USED);
1973 	desc = macb_tx_desc(queue, entry);
1974 	desc->ctrl = ctrl;
1975 
1976 	if (lso_ctrl) {
1977 		if (lso_ctrl == MACB_LSO_UFO_ENABLE)
1978 			/* include header and FCS in value given to h/w */
1979 			mss_mfs = skb_shinfo(skb)->gso_size +
1980 					skb_transport_offset(skb) +
1981 					ETH_FCS_LEN;
1982 		else /* TSO */ {
1983 			mss_mfs = skb_shinfo(skb)->gso_size;
1984 			/* TCP Sequence Number Source Select
1985 			 * can be set only for TSO
1986 			 */
1987 			seq_ctrl = 0;
1988 		}
1989 	}
1990 
1991 	do {
1992 		i--;
1993 		entry = macb_tx_ring_wrap(bp, i);
1994 		tx_skb = &queue->tx_skb[entry];
1995 		desc = macb_tx_desc(queue, entry);
1996 
1997 		ctrl = (u32)tx_skb->size;
1998 		if (eof) {
1999 			ctrl |= MACB_BIT(TX_LAST);
2000 			eof = 0;
2001 		}
2002 		if (unlikely(entry == (bp->tx_ring_size - 1)))
2003 			ctrl |= MACB_BIT(TX_WRAP);
2004 
2005 		/* First descriptor is header descriptor */
2006 		if (i == queue->tx_head) {
2007 			ctrl |= MACB_BF(TX_LSO, lso_ctrl);
2008 			ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl);
2009 			if ((bp->dev->features & NETIF_F_HW_CSUM) &&
2010 			    skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl)
2011 				ctrl |= MACB_BIT(TX_NOCRC);
2012 		} else
2013 			/* Only set MSS/MFS on payload descriptors
2014 			 * (second or later descriptor)
2015 			 */
2016 			ctrl |= MACB_BF(MSS_MFS, mss_mfs);
2017 
2018 		/* Set TX buffer descriptor */
2019 		macb_set_addr(bp, desc, tx_skb->mapping);
2020 		/* desc->addr must be visible to hardware before clearing
2021 		 * 'TX_USED' bit in desc->ctrl.
2022 		 */
2023 		wmb();
2024 		desc->ctrl = ctrl;
2025 	} while (i != queue->tx_head);
2026 
2027 	queue->tx_head = tx_head;
2028 
2029 	return count;
2030 
2031 dma_error:
2032 	netdev_err(bp->dev, "TX DMA map failed\n");
2033 
2034 	for (i = queue->tx_head; i != tx_head; i++) {
2035 		tx_skb = macb_tx_skb(queue, i);
2036 
2037 		macb_tx_unmap(bp, tx_skb);
2038 	}
2039 
2040 	return 0;
2041 }
2042 
2043 static netdev_features_t macb_features_check(struct sk_buff *skb,
2044 					     struct net_device *dev,
2045 					     netdev_features_t features)
2046 {
2047 	unsigned int nr_frags, f;
2048 	unsigned int hdrlen;
2049 
2050 	/* Validate LSO compatibility */
2051 
2052 	/* there is only one buffer or protocol is not UDP */
2053 	if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP))
2054 		return features;
2055 
2056 	/* length of header */
2057 	hdrlen = skb_transport_offset(skb);
2058 
2059 	/* For UFO only:
2060 	 * When software supplies two or more payload buffers all payload buffers
2061 	 * apart from the last must be a multiple of 8 bytes in size.
2062 	 */
2063 	if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN))
2064 		return features & ~MACB_NETIF_LSO;
2065 
2066 	nr_frags = skb_shinfo(skb)->nr_frags;
2067 	/* No need to check last fragment */
2068 	nr_frags--;
2069 	for (f = 0; f < nr_frags; f++) {
2070 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2071 
2072 		if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN))
2073 			return features & ~MACB_NETIF_LSO;
2074 	}
2075 	return features;
2076 }
2077 
2078 static inline int macb_clear_csum(struct sk_buff *skb)
2079 {
2080 	/* no change for packets without checksum offloading */
2081 	if (skb->ip_summed != CHECKSUM_PARTIAL)
2082 		return 0;
2083 
2084 	/* make sure we can modify the header */
2085 	if (unlikely(skb_cow_head(skb, 0)))
2086 		return -1;
2087 
2088 	/* initialize checksum field
2089 	 * This is required - at least for Zynq, which otherwise calculates
2090 	 * wrong UDP header checksums for UDP packets with UDP data len <=2
2091 	 */
2092 	*(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;
2093 	return 0;
2094 }
2095 
2096 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev)
2097 {
2098 	bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) ||
2099 		      skb_is_nonlinear(*skb);
2100 	int padlen = ETH_ZLEN - (*skb)->len;
2101 	int headroom = skb_headroom(*skb);
2102 	int tailroom = skb_tailroom(*skb);
2103 	struct sk_buff *nskb;
2104 	u32 fcs;
2105 
2106 	if (!(ndev->features & NETIF_F_HW_CSUM) ||
2107 	    !((*skb)->ip_summed != CHECKSUM_PARTIAL) ||
2108 	    skb_shinfo(*skb)->gso_size)	/* Not available for GSO */
2109 		return 0;
2110 
2111 	if (padlen <= 0) {
2112 		/* FCS could be appeded to tailroom. */
2113 		if (tailroom >= ETH_FCS_LEN)
2114 			goto add_fcs;
2115 		/* FCS could be appeded by moving data to headroom. */
2116 		else if (!cloned && headroom + tailroom >= ETH_FCS_LEN)
2117 			padlen = 0;
2118 		/* No room for FCS, need to reallocate skb. */
2119 		else
2120 			padlen = ETH_FCS_LEN;
2121 	} else {
2122 		/* Add room for FCS. */
2123 		padlen += ETH_FCS_LEN;
2124 	}
2125 
2126 	if (!cloned && headroom + tailroom >= padlen) {
2127 		(*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len);
2128 		skb_set_tail_pointer(*skb, (*skb)->len);
2129 	} else {
2130 		nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC);
2131 		if (!nskb)
2132 			return -ENOMEM;
2133 
2134 		dev_consume_skb_any(*skb);
2135 		*skb = nskb;
2136 	}
2137 
2138 	if (padlen > ETH_FCS_LEN)
2139 		skb_put_zero(*skb, padlen - ETH_FCS_LEN);
2140 
2141 add_fcs:
2142 	/* set FCS to packet */
2143 	fcs = crc32_le(~0, (*skb)->data, (*skb)->len);
2144 	fcs = ~fcs;
2145 
2146 	skb_put_u8(*skb, fcs		& 0xff);
2147 	skb_put_u8(*skb, (fcs >> 8)	& 0xff);
2148 	skb_put_u8(*skb, (fcs >> 16)	& 0xff);
2149 	skb_put_u8(*skb, (fcs >> 24)	& 0xff);
2150 
2151 	return 0;
2152 }
2153 
2154 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev)
2155 {
2156 	u16 queue_index = skb_get_queue_mapping(skb);
2157 	struct macb *bp = netdev_priv(dev);
2158 	struct macb_queue *queue = &bp->queues[queue_index];
2159 	unsigned long flags;
2160 	unsigned int desc_cnt, nr_frags, frag_size, f;
2161 	unsigned int hdrlen;
2162 	bool is_lso;
2163 	netdev_tx_t ret = NETDEV_TX_OK;
2164 
2165 	if (macb_clear_csum(skb)) {
2166 		dev_kfree_skb_any(skb);
2167 		return ret;
2168 	}
2169 
2170 	if (macb_pad_and_fcs(&skb, dev)) {
2171 		dev_kfree_skb_any(skb);
2172 		return ret;
2173 	}
2174 
2175 	is_lso = (skb_shinfo(skb)->gso_size != 0);
2176 
2177 	if (is_lso) {
2178 		/* length of headers */
2179 		if (ip_hdr(skb)->protocol == IPPROTO_UDP)
2180 			/* only queue eth + ip headers separately for UDP */
2181 			hdrlen = skb_transport_offset(skb);
2182 		else
2183 			hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
2184 		if (skb_headlen(skb) < hdrlen) {
2185 			netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n");
2186 			/* if this is required, would need to copy to single buffer */
2187 			return NETDEV_TX_BUSY;
2188 		}
2189 	} else
2190 		hdrlen = min(skb_headlen(skb), bp->max_tx_length);
2191 
2192 #if defined(DEBUG) && defined(VERBOSE_DEBUG)
2193 	netdev_vdbg(bp->dev,
2194 		    "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n",
2195 		    queue_index, skb->len, skb->head, skb->data,
2196 		    skb_tail_pointer(skb), skb_end_pointer(skb));
2197 	print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1,
2198 		       skb->data, 16, true);
2199 #endif
2200 
2201 	/* Count how many TX buffer descriptors are needed to send this
2202 	 * socket buffer: skb fragments of jumbo frames may need to be
2203 	 * split into many buffer descriptors.
2204 	 */
2205 	if (is_lso && (skb_headlen(skb) > hdrlen))
2206 		/* extra header descriptor if also payload in first buffer */
2207 		desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1;
2208 	else
2209 		desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length);
2210 	nr_frags = skb_shinfo(skb)->nr_frags;
2211 	for (f = 0; f < nr_frags; f++) {
2212 		frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]);
2213 		desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length);
2214 	}
2215 
2216 	spin_lock_irqsave(&bp->lock, flags);
2217 
2218 	/* This is a hard error, log it. */
2219 	if (CIRC_SPACE(queue->tx_head, queue->tx_tail,
2220 		       bp->tx_ring_size) < desc_cnt) {
2221 		netif_stop_subqueue(dev, queue_index);
2222 		spin_unlock_irqrestore(&bp->lock, flags);
2223 		netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n",
2224 			   queue->tx_head, queue->tx_tail);
2225 		return NETDEV_TX_BUSY;
2226 	}
2227 
2228 	/* Map socket buffer for DMA transfer */
2229 	if (!macb_tx_map(bp, queue, skb, hdrlen)) {
2230 		dev_kfree_skb_any(skb);
2231 		goto unlock;
2232 	}
2233 
2234 	/* Make newly initialized descriptor visible to hardware */
2235 	wmb();
2236 	skb_tx_timestamp(skb);
2237 
2238 	macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
2239 
2240 	if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1)
2241 		netif_stop_subqueue(dev, queue_index);
2242 
2243 unlock:
2244 	spin_unlock_irqrestore(&bp->lock, flags);
2245 
2246 	return ret;
2247 }
2248 
2249 static void macb_init_rx_buffer_size(struct macb *bp, size_t size)
2250 {
2251 	if (!macb_is_gem(bp)) {
2252 		bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;
2253 	} else {
2254 		bp->rx_buffer_size = size;
2255 
2256 		if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {
2257 			netdev_dbg(bp->dev,
2258 				   "RX buffer must be multiple of %d bytes, expanding\n",
2259 				   RX_BUFFER_MULTIPLE);
2260 			bp->rx_buffer_size =
2261 				roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE);
2262 		}
2263 	}
2264 
2265 	netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n",
2266 		   bp->dev->mtu, bp->rx_buffer_size);
2267 }
2268 
2269 static void gem_free_rx_buffers(struct macb *bp)
2270 {
2271 	struct sk_buff		*skb;
2272 	struct macb_dma_desc	*desc;
2273 	struct macb_queue *queue;
2274 	dma_addr_t		addr;
2275 	unsigned int q;
2276 	int i;
2277 
2278 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2279 		if (!queue->rx_skbuff)
2280 			continue;
2281 
2282 		for (i = 0; i < bp->rx_ring_size; i++) {
2283 			skb = queue->rx_skbuff[i];
2284 
2285 			if (!skb)
2286 				continue;
2287 
2288 			desc = macb_rx_desc(queue, i);
2289 			addr = macb_get_addr(bp, desc);
2290 
2291 			dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,
2292 					DMA_FROM_DEVICE);
2293 			dev_kfree_skb_any(skb);
2294 			skb = NULL;
2295 		}
2296 
2297 		kfree(queue->rx_skbuff);
2298 		queue->rx_skbuff = NULL;
2299 	}
2300 }
2301 
2302 static void macb_free_rx_buffers(struct macb *bp)
2303 {
2304 	struct macb_queue *queue = &bp->queues[0];
2305 
2306 	if (queue->rx_buffers) {
2307 		dma_free_coherent(&bp->pdev->dev,
2308 				  bp->rx_ring_size * bp->rx_buffer_size,
2309 				  queue->rx_buffers, queue->rx_buffers_dma);
2310 		queue->rx_buffers = NULL;
2311 	}
2312 }
2313 
2314 static void macb_free_consistent(struct macb *bp)
2315 {
2316 	struct macb_queue *queue;
2317 	unsigned int q;
2318 	int size;
2319 
2320 	bp->macbgem_ops.mog_free_rx_buffers(bp);
2321 
2322 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2323 		kfree(queue->tx_skb);
2324 		queue->tx_skb = NULL;
2325 		if (queue->tx_ring) {
2326 			size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
2327 			dma_free_coherent(&bp->pdev->dev, size,
2328 					  queue->tx_ring, queue->tx_ring_dma);
2329 			queue->tx_ring = NULL;
2330 		}
2331 		if (queue->rx_ring) {
2332 			size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;
2333 			dma_free_coherent(&bp->pdev->dev, size,
2334 					  queue->rx_ring, queue->rx_ring_dma);
2335 			queue->rx_ring = NULL;
2336 		}
2337 	}
2338 }
2339 
2340 static int gem_alloc_rx_buffers(struct macb *bp)
2341 {
2342 	struct macb_queue *queue;
2343 	unsigned int q;
2344 	int size;
2345 
2346 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2347 		size = bp->rx_ring_size * sizeof(struct sk_buff *);
2348 		queue->rx_skbuff = kzalloc(size, GFP_KERNEL);
2349 		if (!queue->rx_skbuff)
2350 			return -ENOMEM;
2351 		else
2352 			netdev_dbg(bp->dev,
2353 				   "Allocated %d RX struct sk_buff entries at %p\n",
2354 				   bp->rx_ring_size, queue->rx_skbuff);
2355 	}
2356 	return 0;
2357 }
2358 
2359 static int macb_alloc_rx_buffers(struct macb *bp)
2360 {
2361 	struct macb_queue *queue = &bp->queues[0];
2362 	int size;
2363 
2364 	size = bp->rx_ring_size * bp->rx_buffer_size;
2365 	queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,
2366 					    &queue->rx_buffers_dma, GFP_KERNEL);
2367 	if (!queue->rx_buffers)
2368 		return -ENOMEM;
2369 
2370 	netdev_dbg(bp->dev,
2371 		   "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n",
2372 		   size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);
2373 	return 0;
2374 }
2375 
2376 static int macb_alloc_consistent(struct macb *bp)
2377 {
2378 	struct macb_queue *queue;
2379 	unsigned int q;
2380 	int size;
2381 
2382 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2383 		size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
2384 		queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
2385 						    &queue->tx_ring_dma,
2386 						    GFP_KERNEL);
2387 		if (!queue->tx_ring)
2388 			goto out_err;
2389 		netdev_dbg(bp->dev,
2390 			   "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n",
2391 			   q, size, (unsigned long)queue->tx_ring_dma,
2392 			   queue->tx_ring);
2393 
2394 		size = bp->tx_ring_size * sizeof(struct macb_tx_skb);
2395 		queue->tx_skb = kmalloc(size, GFP_KERNEL);
2396 		if (!queue->tx_skb)
2397 			goto out_err;
2398 
2399 		size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;
2400 		queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
2401 						 &queue->rx_ring_dma, GFP_KERNEL);
2402 		if (!queue->rx_ring)
2403 			goto out_err;
2404 		netdev_dbg(bp->dev,
2405 			   "Allocated RX ring of %d bytes at %08lx (mapped %p)\n",
2406 			   size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);
2407 	}
2408 	if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))
2409 		goto out_err;
2410 
2411 	return 0;
2412 
2413 out_err:
2414 	macb_free_consistent(bp);
2415 	return -ENOMEM;
2416 }
2417 
2418 static void gem_init_rings(struct macb *bp)
2419 {
2420 	struct macb_queue *queue;
2421 	struct macb_dma_desc *desc = NULL;
2422 	unsigned int q;
2423 	int i;
2424 
2425 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2426 		for (i = 0; i < bp->tx_ring_size; i++) {
2427 			desc = macb_tx_desc(queue, i);
2428 			macb_set_addr(bp, desc, 0);
2429 			desc->ctrl = MACB_BIT(TX_USED);
2430 		}
2431 		desc->ctrl |= MACB_BIT(TX_WRAP);
2432 		queue->tx_head = 0;
2433 		queue->tx_tail = 0;
2434 
2435 		queue->rx_tail = 0;
2436 		queue->rx_prepared_head = 0;
2437 
2438 		gem_rx_refill(queue);
2439 	}
2440 
2441 }
2442 
2443 static void macb_init_rings(struct macb *bp)
2444 {
2445 	int i;
2446 	struct macb_dma_desc *desc = NULL;
2447 
2448 	macb_init_rx_ring(&bp->queues[0]);
2449 
2450 	for (i = 0; i < bp->tx_ring_size; i++) {
2451 		desc = macb_tx_desc(&bp->queues[0], i);
2452 		macb_set_addr(bp, desc, 0);
2453 		desc->ctrl = MACB_BIT(TX_USED);
2454 	}
2455 	bp->queues[0].tx_head = 0;
2456 	bp->queues[0].tx_tail = 0;
2457 	desc->ctrl |= MACB_BIT(TX_WRAP);
2458 }
2459 
2460 static void macb_reset_hw(struct macb *bp)
2461 {
2462 	struct macb_queue *queue;
2463 	unsigned int q;
2464 	u32 ctrl = macb_readl(bp, NCR);
2465 
2466 	/* Disable RX and TX (XXX: Should we halt the transmission
2467 	 * more gracefully?)
2468 	 */
2469 	ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
2470 
2471 	/* Clear the stats registers (XXX: Update stats first?) */
2472 	ctrl |= MACB_BIT(CLRSTAT);
2473 
2474 	macb_writel(bp, NCR, ctrl);
2475 
2476 	/* Clear all status flags */
2477 	macb_writel(bp, TSR, -1);
2478 	macb_writel(bp, RSR, -1);
2479 
2480 	/* Disable all interrupts */
2481 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2482 		queue_writel(queue, IDR, -1);
2483 		queue_readl(queue, ISR);
2484 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
2485 			queue_writel(queue, ISR, -1);
2486 	}
2487 }
2488 
2489 static u32 gem_mdc_clk_div(struct macb *bp)
2490 {
2491 	u32 config;
2492 	unsigned long pclk_hz = clk_get_rate(bp->pclk);
2493 
2494 	if (pclk_hz <= 20000000)
2495 		config = GEM_BF(CLK, GEM_CLK_DIV8);
2496 	else if (pclk_hz <= 40000000)
2497 		config = GEM_BF(CLK, GEM_CLK_DIV16);
2498 	else if (pclk_hz <= 80000000)
2499 		config = GEM_BF(CLK, GEM_CLK_DIV32);
2500 	else if (pclk_hz <= 120000000)
2501 		config = GEM_BF(CLK, GEM_CLK_DIV48);
2502 	else if (pclk_hz <= 160000000)
2503 		config = GEM_BF(CLK, GEM_CLK_DIV64);
2504 	else
2505 		config = GEM_BF(CLK, GEM_CLK_DIV96);
2506 
2507 	return config;
2508 }
2509 
2510 static u32 macb_mdc_clk_div(struct macb *bp)
2511 {
2512 	u32 config;
2513 	unsigned long pclk_hz;
2514 
2515 	if (macb_is_gem(bp))
2516 		return gem_mdc_clk_div(bp);
2517 
2518 	pclk_hz = clk_get_rate(bp->pclk);
2519 	if (pclk_hz <= 20000000)
2520 		config = MACB_BF(CLK, MACB_CLK_DIV8);
2521 	else if (pclk_hz <= 40000000)
2522 		config = MACB_BF(CLK, MACB_CLK_DIV16);
2523 	else if (pclk_hz <= 80000000)
2524 		config = MACB_BF(CLK, MACB_CLK_DIV32);
2525 	else
2526 		config = MACB_BF(CLK, MACB_CLK_DIV64);
2527 
2528 	return config;
2529 }
2530 
2531 /* Get the DMA bus width field of the network configuration register that we
2532  * should program.  We find the width from decoding the design configuration
2533  * register to find the maximum supported data bus width.
2534  */
2535 static u32 macb_dbw(struct macb *bp)
2536 {
2537 	if (!macb_is_gem(bp))
2538 		return 0;
2539 
2540 	switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) {
2541 	case 4:
2542 		return GEM_BF(DBW, GEM_DBW128);
2543 	case 2:
2544 		return GEM_BF(DBW, GEM_DBW64);
2545 	case 1:
2546 	default:
2547 		return GEM_BF(DBW, GEM_DBW32);
2548 	}
2549 }
2550 
2551 /* Configure the receive DMA engine
2552  * - use the correct receive buffer size
2553  * - set best burst length for DMA operations
2554  *   (if not supported by FIFO, it will fallback to default)
2555  * - set both rx/tx packet buffers to full memory size
2556  * These are configurable parameters for GEM.
2557  */
2558 static void macb_configure_dma(struct macb *bp)
2559 {
2560 	struct macb_queue *queue;
2561 	u32 buffer_size;
2562 	unsigned int q;
2563 	u32 dmacfg;
2564 
2565 	buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;
2566 	if (macb_is_gem(bp)) {
2567 		dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);
2568 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2569 			if (q)
2570 				queue_writel(queue, RBQS, buffer_size);
2571 			else
2572 				dmacfg |= GEM_BF(RXBS, buffer_size);
2573 		}
2574 		if (bp->dma_burst_length)
2575 			dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);
2576 		dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);
2577 		dmacfg &= ~GEM_BIT(ENDIA_PKT);
2578 
2579 		if (bp->native_io)
2580 			dmacfg &= ~GEM_BIT(ENDIA_DESC);
2581 		else
2582 			dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */
2583 
2584 		if (bp->dev->features & NETIF_F_HW_CSUM)
2585 			dmacfg |= GEM_BIT(TXCOEN);
2586 		else
2587 			dmacfg &= ~GEM_BIT(TXCOEN);
2588 
2589 		dmacfg &= ~GEM_BIT(ADDR64);
2590 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
2591 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)
2592 			dmacfg |= GEM_BIT(ADDR64);
2593 #endif
2594 #ifdef CONFIG_MACB_USE_HWSTAMP
2595 		if (bp->hw_dma_cap & HW_DMA_CAP_PTP)
2596 			dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT);
2597 #endif
2598 		netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",
2599 			   dmacfg);
2600 		gem_writel(bp, DMACFG, dmacfg);
2601 	}
2602 }
2603 
2604 static void macb_init_hw(struct macb *bp)
2605 {
2606 	u32 config;
2607 
2608 	macb_reset_hw(bp);
2609 	macb_set_hwaddr(bp);
2610 
2611 	config = macb_mdc_clk_div(bp);
2612 	config |= MACB_BF(RBOF, NET_IP_ALIGN);	/* Make eth data aligned */
2613 	config |= MACB_BIT(DRFCS);		/* Discard Rx FCS */
2614 	if (bp->caps & MACB_CAPS_JUMBO)
2615 		config |= MACB_BIT(JFRAME);	/* Enable jumbo frames */
2616 	else
2617 		config |= MACB_BIT(BIG);	/* Receive oversized frames */
2618 	if (bp->dev->flags & IFF_PROMISC)
2619 		config |= MACB_BIT(CAF);	/* Copy All Frames */
2620 	else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM)
2621 		config |= GEM_BIT(RXCOEN);
2622 	if (!(bp->dev->flags & IFF_BROADCAST))
2623 		config |= MACB_BIT(NBC);	/* No BroadCast */
2624 	config |= macb_dbw(bp);
2625 	macb_writel(bp, NCFGR, config);
2626 	if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)
2627 		gem_writel(bp, JML, bp->jumbo_max_len);
2628 	bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK;
2629 	if (bp->caps & MACB_CAPS_JUMBO)
2630 		bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK;
2631 
2632 	macb_configure_dma(bp);
2633 }
2634 
2635 /* The hash address register is 64 bits long and takes up two
2636  * locations in the memory map.  The least significant bits are stored
2637  * in EMAC_HSL and the most significant bits in EMAC_HSH.
2638  *
2639  * The unicast hash enable and the multicast hash enable bits in the
2640  * network configuration register enable the reception of hash matched
2641  * frames. The destination address is reduced to a 6 bit index into
2642  * the 64 bit hash register using the following hash function.  The
2643  * hash function is an exclusive or of every sixth bit of the
2644  * destination address.
2645  *
2646  * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47]
2647  * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46]
2648  * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45]
2649  * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44]
2650  * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43]
2651  * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42]
2652  *
2653  * da[0] represents the least significant bit of the first byte
2654  * received, that is, the multicast/unicast indicator, and da[47]
2655  * represents the most significant bit of the last byte received.  If
2656  * the hash index, hi[n], points to a bit that is set in the hash
2657  * register then the frame will be matched according to whether the
2658  * frame is multicast or unicast.  A multicast match will be signalled
2659  * if the multicast hash enable bit is set, da[0] is 1 and the hash
2660  * index points to a bit set in the hash register.  A unicast match
2661  * will be signalled if the unicast hash enable bit is set, da[0] is 0
2662  * and the hash index points to a bit set in the hash register.  To
2663  * receive all multicast frames, the hash register should be set with
2664  * all ones and the multicast hash enable bit should be set in the
2665  * network configuration register.
2666  */
2667 
2668 static inline int hash_bit_value(int bitnr, __u8 *addr)
2669 {
2670 	if (addr[bitnr / 8] & (1 << (bitnr % 8)))
2671 		return 1;
2672 	return 0;
2673 }
2674 
2675 /* Return the hash index value for the specified address. */
2676 static int hash_get_index(__u8 *addr)
2677 {
2678 	int i, j, bitval;
2679 	int hash_index = 0;
2680 
2681 	for (j = 0; j < 6; j++) {
2682 		for (i = 0, bitval = 0; i < 8; i++)
2683 			bitval ^= hash_bit_value(i * 6 + j, addr);
2684 
2685 		hash_index |= (bitval << j);
2686 	}
2687 
2688 	return hash_index;
2689 }
2690 
2691 /* Add multicast addresses to the internal multicast-hash table. */
2692 static void macb_sethashtable(struct net_device *dev)
2693 {
2694 	struct netdev_hw_addr *ha;
2695 	unsigned long mc_filter[2];
2696 	unsigned int bitnr;
2697 	struct macb *bp = netdev_priv(dev);
2698 
2699 	mc_filter[0] = 0;
2700 	mc_filter[1] = 0;
2701 
2702 	netdev_for_each_mc_addr(ha, dev) {
2703 		bitnr = hash_get_index(ha->addr);
2704 		mc_filter[bitnr >> 5] |= 1 << (bitnr & 31);
2705 	}
2706 
2707 	macb_or_gem_writel(bp, HRB, mc_filter[0]);
2708 	macb_or_gem_writel(bp, HRT, mc_filter[1]);
2709 }
2710 
2711 /* Enable/Disable promiscuous and multicast modes. */
2712 static void macb_set_rx_mode(struct net_device *dev)
2713 {
2714 	unsigned long cfg;
2715 	struct macb *bp = netdev_priv(dev);
2716 
2717 	cfg = macb_readl(bp, NCFGR);
2718 
2719 	if (dev->flags & IFF_PROMISC) {
2720 		/* Enable promiscuous mode */
2721 		cfg |= MACB_BIT(CAF);
2722 
2723 		/* Disable RX checksum offload */
2724 		if (macb_is_gem(bp))
2725 			cfg &= ~GEM_BIT(RXCOEN);
2726 	} else {
2727 		/* Disable promiscuous mode */
2728 		cfg &= ~MACB_BIT(CAF);
2729 
2730 		/* Enable RX checksum offload only if requested */
2731 		if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM)
2732 			cfg |= GEM_BIT(RXCOEN);
2733 	}
2734 
2735 	if (dev->flags & IFF_ALLMULTI) {
2736 		/* Enable all multicast mode */
2737 		macb_or_gem_writel(bp, HRB, -1);
2738 		macb_or_gem_writel(bp, HRT, -1);
2739 		cfg |= MACB_BIT(NCFGR_MTI);
2740 	} else if (!netdev_mc_empty(dev)) {
2741 		/* Enable specific multicasts */
2742 		macb_sethashtable(dev);
2743 		cfg |= MACB_BIT(NCFGR_MTI);
2744 	} else if (dev->flags & (~IFF_ALLMULTI)) {
2745 		/* Disable all multicast mode */
2746 		macb_or_gem_writel(bp, HRB, 0);
2747 		macb_or_gem_writel(bp, HRT, 0);
2748 		cfg &= ~MACB_BIT(NCFGR_MTI);
2749 	}
2750 
2751 	macb_writel(bp, NCFGR, cfg);
2752 }
2753 
2754 static int macb_open(struct net_device *dev)
2755 {
2756 	size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;
2757 	struct macb *bp = netdev_priv(dev);
2758 	struct macb_queue *queue;
2759 	unsigned int q;
2760 	int err;
2761 
2762 	netdev_dbg(bp->dev, "open\n");
2763 
2764 	err = pm_runtime_get_sync(&bp->pdev->dev);
2765 	if (err < 0)
2766 		goto pm_exit;
2767 
2768 	/* RX buffers initialization */
2769 	macb_init_rx_buffer_size(bp, bufsz);
2770 
2771 	err = macb_alloc_consistent(bp);
2772 	if (err) {
2773 		netdev_err(dev, "Unable to allocate DMA memory (error %d)\n",
2774 			   err);
2775 		goto pm_exit;
2776 	}
2777 
2778 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2779 		napi_enable(&queue->napi);
2780 
2781 	macb_init_hw(bp);
2782 
2783 	err = macb_phylink_connect(bp);
2784 	if (err)
2785 		goto reset_hw;
2786 
2787 	netif_tx_start_all_queues(dev);
2788 
2789 	if (bp->ptp_info)
2790 		bp->ptp_info->ptp_init(dev);
2791 
2792 	return 0;
2793 
2794 reset_hw:
2795 	macb_reset_hw(bp);
2796 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2797 		napi_disable(&queue->napi);
2798 	macb_free_consistent(bp);
2799 pm_exit:
2800 	pm_runtime_put_sync(&bp->pdev->dev);
2801 	return err;
2802 }
2803 
2804 static int macb_close(struct net_device *dev)
2805 {
2806 	struct macb *bp = netdev_priv(dev);
2807 	struct macb_queue *queue;
2808 	unsigned long flags;
2809 	unsigned int q;
2810 
2811 	netif_tx_stop_all_queues(dev);
2812 
2813 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2814 		napi_disable(&queue->napi);
2815 
2816 	phylink_stop(bp->phylink);
2817 	phylink_disconnect_phy(bp->phylink);
2818 
2819 	spin_lock_irqsave(&bp->lock, flags);
2820 	macb_reset_hw(bp);
2821 	netif_carrier_off(dev);
2822 	spin_unlock_irqrestore(&bp->lock, flags);
2823 
2824 	macb_free_consistent(bp);
2825 
2826 	if (bp->ptp_info)
2827 		bp->ptp_info->ptp_remove(dev);
2828 
2829 	pm_runtime_put(&bp->pdev->dev);
2830 
2831 	return 0;
2832 }
2833 
2834 static int macb_change_mtu(struct net_device *dev, int new_mtu)
2835 {
2836 	if (netif_running(dev))
2837 		return -EBUSY;
2838 
2839 	dev->mtu = new_mtu;
2840 
2841 	return 0;
2842 }
2843 
2844 static void gem_update_stats(struct macb *bp)
2845 {
2846 	struct macb_queue *queue;
2847 	unsigned int i, q, idx;
2848 	unsigned long *stat;
2849 
2850 	u32 *p = &bp->hw_stats.gem.tx_octets_31_0;
2851 
2852 	for (i = 0; i < GEM_STATS_LEN; ++i, ++p) {
2853 		u32 offset = gem_statistics[i].offset;
2854 		u64 val = bp->macb_reg_readl(bp, offset);
2855 
2856 		bp->ethtool_stats[i] += val;
2857 		*p += val;
2858 
2859 		if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) {
2860 			/* Add GEM_OCTTXH, GEM_OCTRXH */
2861 			val = bp->macb_reg_readl(bp, offset + 4);
2862 			bp->ethtool_stats[i] += ((u64)val) << 32;
2863 			*(++p) += val;
2864 		}
2865 	}
2866 
2867 	idx = GEM_STATS_LEN;
2868 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2869 		for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)
2870 			bp->ethtool_stats[idx++] = *stat;
2871 }
2872 
2873 static struct net_device_stats *gem_get_stats(struct macb *bp)
2874 {
2875 	struct gem_stats *hwstat = &bp->hw_stats.gem;
2876 	struct net_device_stats *nstat = &bp->dev->stats;
2877 
2878 	if (!netif_running(bp->dev))
2879 		return nstat;
2880 
2881 	gem_update_stats(bp);
2882 
2883 	nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors +
2884 			    hwstat->rx_alignment_errors +
2885 			    hwstat->rx_resource_errors +
2886 			    hwstat->rx_overruns +
2887 			    hwstat->rx_oversize_frames +
2888 			    hwstat->rx_jabbers +
2889 			    hwstat->rx_undersized_frames +
2890 			    hwstat->rx_length_field_frame_errors);
2891 	nstat->tx_errors = (hwstat->tx_late_collisions +
2892 			    hwstat->tx_excessive_collisions +
2893 			    hwstat->tx_underrun +
2894 			    hwstat->tx_carrier_sense_errors);
2895 	nstat->multicast = hwstat->rx_multicast_frames;
2896 	nstat->collisions = (hwstat->tx_single_collision_frames +
2897 			     hwstat->tx_multiple_collision_frames +
2898 			     hwstat->tx_excessive_collisions);
2899 	nstat->rx_length_errors = (hwstat->rx_oversize_frames +
2900 				   hwstat->rx_jabbers +
2901 				   hwstat->rx_undersized_frames +
2902 				   hwstat->rx_length_field_frame_errors);
2903 	nstat->rx_over_errors = hwstat->rx_resource_errors;
2904 	nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors;
2905 	nstat->rx_frame_errors = hwstat->rx_alignment_errors;
2906 	nstat->rx_fifo_errors = hwstat->rx_overruns;
2907 	nstat->tx_aborted_errors = hwstat->tx_excessive_collisions;
2908 	nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors;
2909 	nstat->tx_fifo_errors = hwstat->tx_underrun;
2910 
2911 	return nstat;
2912 }
2913 
2914 static void gem_get_ethtool_stats(struct net_device *dev,
2915 				  struct ethtool_stats *stats, u64 *data)
2916 {
2917 	struct macb *bp;
2918 
2919 	bp = netdev_priv(dev);
2920 	gem_update_stats(bp);
2921 	memcpy(data, &bp->ethtool_stats, sizeof(u64)
2922 			* (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES));
2923 }
2924 
2925 static int gem_get_sset_count(struct net_device *dev, int sset)
2926 {
2927 	struct macb *bp = netdev_priv(dev);
2928 
2929 	switch (sset) {
2930 	case ETH_SS_STATS:
2931 		return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;
2932 	default:
2933 		return -EOPNOTSUPP;
2934 	}
2935 }
2936 
2937 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p)
2938 {
2939 	char stat_string[ETH_GSTRING_LEN];
2940 	struct macb *bp = netdev_priv(dev);
2941 	struct macb_queue *queue;
2942 	unsigned int i;
2943 	unsigned int q;
2944 
2945 	switch (sset) {
2946 	case ETH_SS_STATS:
2947 		for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN)
2948 			memcpy(p, gem_statistics[i].stat_string,
2949 			       ETH_GSTRING_LEN);
2950 
2951 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2952 			for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) {
2953 				snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s",
2954 						q, queue_statistics[i].stat_string);
2955 				memcpy(p, stat_string, ETH_GSTRING_LEN);
2956 			}
2957 		}
2958 		break;
2959 	}
2960 }
2961 
2962 static struct net_device_stats *macb_get_stats(struct net_device *dev)
2963 {
2964 	struct macb *bp = netdev_priv(dev);
2965 	struct net_device_stats *nstat = &bp->dev->stats;
2966 	struct macb_stats *hwstat = &bp->hw_stats.macb;
2967 
2968 	if (macb_is_gem(bp))
2969 		return gem_get_stats(bp);
2970 
2971 	/* read stats from hardware */
2972 	macb_update_stats(bp);
2973 
2974 	/* Convert HW stats into netdevice stats */
2975 	nstat->rx_errors = (hwstat->rx_fcs_errors +
2976 			    hwstat->rx_align_errors +
2977 			    hwstat->rx_resource_errors +
2978 			    hwstat->rx_overruns +
2979 			    hwstat->rx_oversize_pkts +
2980 			    hwstat->rx_jabbers +
2981 			    hwstat->rx_undersize_pkts +
2982 			    hwstat->rx_length_mismatch);
2983 	nstat->tx_errors = (hwstat->tx_late_cols +
2984 			    hwstat->tx_excessive_cols +
2985 			    hwstat->tx_underruns +
2986 			    hwstat->tx_carrier_errors +
2987 			    hwstat->sqe_test_errors);
2988 	nstat->collisions = (hwstat->tx_single_cols +
2989 			     hwstat->tx_multiple_cols +
2990 			     hwstat->tx_excessive_cols);
2991 	nstat->rx_length_errors = (hwstat->rx_oversize_pkts +
2992 				   hwstat->rx_jabbers +
2993 				   hwstat->rx_undersize_pkts +
2994 				   hwstat->rx_length_mismatch);
2995 	nstat->rx_over_errors = hwstat->rx_resource_errors +
2996 				   hwstat->rx_overruns;
2997 	nstat->rx_crc_errors = hwstat->rx_fcs_errors;
2998 	nstat->rx_frame_errors = hwstat->rx_align_errors;
2999 	nstat->rx_fifo_errors = hwstat->rx_overruns;
3000 	/* XXX: What does "missed" mean? */
3001 	nstat->tx_aborted_errors = hwstat->tx_excessive_cols;
3002 	nstat->tx_carrier_errors = hwstat->tx_carrier_errors;
3003 	nstat->tx_fifo_errors = hwstat->tx_underruns;
3004 	/* Don't know about heartbeat or window errors... */
3005 
3006 	return nstat;
3007 }
3008 
3009 static int macb_get_regs_len(struct net_device *netdev)
3010 {
3011 	return MACB_GREGS_NBR * sizeof(u32);
3012 }
3013 
3014 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs,
3015 			  void *p)
3016 {
3017 	struct macb *bp = netdev_priv(dev);
3018 	unsigned int tail, head;
3019 	u32 *regs_buff = p;
3020 
3021 	regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1))
3022 			| MACB_GREGS_VERSION;
3023 
3024 	tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail);
3025 	head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head);
3026 
3027 	regs_buff[0]  = macb_readl(bp, NCR);
3028 	regs_buff[1]  = macb_or_gem_readl(bp, NCFGR);
3029 	regs_buff[2]  = macb_readl(bp, NSR);
3030 	regs_buff[3]  = macb_readl(bp, TSR);
3031 	regs_buff[4]  = macb_readl(bp, RBQP);
3032 	regs_buff[5]  = macb_readl(bp, TBQP);
3033 	regs_buff[6]  = macb_readl(bp, RSR);
3034 	regs_buff[7]  = macb_readl(bp, IMR);
3035 
3036 	regs_buff[8]  = tail;
3037 	regs_buff[9]  = head;
3038 	regs_buff[10] = macb_tx_dma(&bp->queues[0], tail);
3039 	regs_buff[11] = macb_tx_dma(&bp->queues[0], head);
3040 
3041 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
3042 		regs_buff[12] = macb_or_gem_readl(bp, USRIO);
3043 	if (macb_is_gem(bp))
3044 		regs_buff[13] = gem_readl(bp, DMACFG);
3045 }
3046 
3047 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
3048 {
3049 	struct macb *bp = netdev_priv(netdev);
3050 
3051 	if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) {
3052 		phylink_ethtool_get_wol(bp->phylink, wol);
3053 		wol->supported |= WAKE_MAGIC;
3054 
3055 		if (bp->wol & MACB_WOL_ENABLED)
3056 			wol->wolopts |= WAKE_MAGIC;
3057 	}
3058 }
3059 
3060 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
3061 {
3062 	struct macb *bp = netdev_priv(netdev);
3063 	int ret;
3064 
3065 	/* Pass the order to phylink layer */
3066 	ret = phylink_ethtool_set_wol(bp->phylink, wol);
3067 	/* Don't manage WoL on MAC if handled by the PHY
3068 	 * or if there's a failure in talking to the PHY
3069 	 */
3070 	if (!ret || ret != -EOPNOTSUPP)
3071 		return ret;
3072 
3073 	if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) ||
3074 	    (wol->wolopts & ~WAKE_MAGIC))
3075 		return -EOPNOTSUPP;
3076 
3077 	if (wol->wolopts & WAKE_MAGIC)
3078 		bp->wol |= MACB_WOL_ENABLED;
3079 	else
3080 		bp->wol &= ~MACB_WOL_ENABLED;
3081 
3082 	device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED);
3083 
3084 	return 0;
3085 }
3086 
3087 static int macb_get_link_ksettings(struct net_device *netdev,
3088 				   struct ethtool_link_ksettings *kset)
3089 {
3090 	struct macb *bp = netdev_priv(netdev);
3091 
3092 	return phylink_ethtool_ksettings_get(bp->phylink, kset);
3093 }
3094 
3095 static int macb_set_link_ksettings(struct net_device *netdev,
3096 				   const struct ethtool_link_ksettings *kset)
3097 {
3098 	struct macb *bp = netdev_priv(netdev);
3099 
3100 	return phylink_ethtool_ksettings_set(bp->phylink, kset);
3101 }
3102 
3103 static void macb_get_ringparam(struct net_device *netdev,
3104 			       struct ethtool_ringparam *ring)
3105 {
3106 	struct macb *bp = netdev_priv(netdev);
3107 
3108 	ring->rx_max_pending = MAX_RX_RING_SIZE;
3109 	ring->tx_max_pending = MAX_TX_RING_SIZE;
3110 
3111 	ring->rx_pending = bp->rx_ring_size;
3112 	ring->tx_pending = bp->tx_ring_size;
3113 }
3114 
3115 static int macb_set_ringparam(struct net_device *netdev,
3116 			      struct ethtool_ringparam *ring)
3117 {
3118 	struct macb *bp = netdev_priv(netdev);
3119 	u32 new_rx_size, new_tx_size;
3120 	unsigned int reset = 0;
3121 
3122 	if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
3123 		return -EINVAL;
3124 
3125 	new_rx_size = clamp_t(u32, ring->rx_pending,
3126 			      MIN_RX_RING_SIZE, MAX_RX_RING_SIZE);
3127 	new_rx_size = roundup_pow_of_two(new_rx_size);
3128 
3129 	new_tx_size = clamp_t(u32, ring->tx_pending,
3130 			      MIN_TX_RING_SIZE, MAX_TX_RING_SIZE);
3131 	new_tx_size = roundup_pow_of_two(new_tx_size);
3132 
3133 	if ((new_tx_size == bp->tx_ring_size) &&
3134 	    (new_rx_size == bp->rx_ring_size)) {
3135 		/* nothing to do */
3136 		return 0;
3137 	}
3138 
3139 	if (netif_running(bp->dev)) {
3140 		reset = 1;
3141 		macb_close(bp->dev);
3142 	}
3143 
3144 	bp->rx_ring_size = new_rx_size;
3145 	bp->tx_ring_size = new_tx_size;
3146 
3147 	if (reset)
3148 		macb_open(bp->dev);
3149 
3150 	return 0;
3151 }
3152 
3153 #ifdef CONFIG_MACB_USE_HWSTAMP
3154 static unsigned int gem_get_tsu_rate(struct macb *bp)
3155 {
3156 	struct clk *tsu_clk;
3157 	unsigned int tsu_rate;
3158 
3159 	tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk");
3160 	if (!IS_ERR(tsu_clk))
3161 		tsu_rate = clk_get_rate(tsu_clk);
3162 	/* try pclk instead */
3163 	else if (!IS_ERR(bp->pclk)) {
3164 		tsu_clk = bp->pclk;
3165 		tsu_rate = clk_get_rate(tsu_clk);
3166 	} else
3167 		return -ENOTSUPP;
3168 	return tsu_rate;
3169 }
3170 
3171 static s32 gem_get_ptp_max_adj(void)
3172 {
3173 	return 64000000;
3174 }
3175 
3176 static int gem_get_ts_info(struct net_device *dev,
3177 			   struct ethtool_ts_info *info)
3178 {
3179 	struct macb *bp = netdev_priv(dev);
3180 
3181 	if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) {
3182 		ethtool_op_get_ts_info(dev, info);
3183 		return 0;
3184 	}
3185 
3186 	info->so_timestamping =
3187 		SOF_TIMESTAMPING_TX_SOFTWARE |
3188 		SOF_TIMESTAMPING_RX_SOFTWARE |
3189 		SOF_TIMESTAMPING_SOFTWARE |
3190 		SOF_TIMESTAMPING_TX_HARDWARE |
3191 		SOF_TIMESTAMPING_RX_HARDWARE |
3192 		SOF_TIMESTAMPING_RAW_HARDWARE;
3193 	info->tx_types =
3194 		(1 << HWTSTAMP_TX_ONESTEP_SYNC) |
3195 		(1 << HWTSTAMP_TX_OFF) |
3196 		(1 << HWTSTAMP_TX_ON);
3197 	info->rx_filters =
3198 		(1 << HWTSTAMP_FILTER_NONE) |
3199 		(1 << HWTSTAMP_FILTER_ALL);
3200 
3201 	info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1;
3202 
3203 	return 0;
3204 }
3205 
3206 static struct macb_ptp_info gem_ptp_info = {
3207 	.ptp_init	 = gem_ptp_init,
3208 	.ptp_remove	 = gem_ptp_remove,
3209 	.get_ptp_max_adj = gem_get_ptp_max_adj,
3210 	.get_tsu_rate	 = gem_get_tsu_rate,
3211 	.get_ts_info	 = gem_get_ts_info,
3212 	.get_hwtst	 = gem_get_hwtst,
3213 	.set_hwtst	 = gem_set_hwtst,
3214 };
3215 #endif
3216 
3217 static int macb_get_ts_info(struct net_device *netdev,
3218 			    struct ethtool_ts_info *info)
3219 {
3220 	struct macb *bp = netdev_priv(netdev);
3221 
3222 	if (bp->ptp_info)
3223 		return bp->ptp_info->get_ts_info(netdev, info);
3224 
3225 	return ethtool_op_get_ts_info(netdev, info);
3226 }
3227 
3228 static void gem_enable_flow_filters(struct macb *bp, bool enable)
3229 {
3230 	struct net_device *netdev = bp->dev;
3231 	struct ethtool_rx_fs_item *item;
3232 	u32 t2_scr;
3233 	int num_t2_scr;
3234 
3235 	if (!(netdev->features & NETIF_F_NTUPLE))
3236 		return;
3237 
3238 	num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8));
3239 
3240 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3241 		struct ethtool_rx_flow_spec *fs = &item->fs;
3242 		struct ethtool_tcpip4_spec *tp4sp_m;
3243 
3244 		if (fs->location >= num_t2_scr)
3245 			continue;
3246 
3247 		t2_scr = gem_readl_n(bp, SCRT2, fs->location);
3248 
3249 		/* enable/disable screener regs for the flow entry */
3250 		t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr);
3251 
3252 		/* only enable fields with no masking */
3253 		tp4sp_m = &(fs->m_u.tcp_ip4_spec);
3254 
3255 		if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))
3256 			t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr);
3257 		else
3258 			t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr);
3259 
3260 		if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))
3261 			t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr);
3262 		else
3263 			t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr);
3264 
3265 		if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))
3266 			t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr);
3267 		else
3268 			t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr);
3269 
3270 		gem_writel_n(bp, SCRT2, fs->location, t2_scr);
3271 	}
3272 }
3273 
3274 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs)
3275 {
3276 	struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m;
3277 	uint16_t index = fs->location;
3278 	u32 w0, w1, t2_scr;
3279 	bool cmp_a = false;
3280 	bool cmp_b = false;
3281 	bool cmp_c = false;
3282 
3283 	if (!macb_is_gem(bp))
3284 		return;
3285 
3286 	tp4sp_v = &(fs->h_u.tcp_ip4_spec);
3287 	tp4sp_m = &(fs->m_u.tcp_ip4_spec);
3288 
3289 	/* ignore field if any masking set */
3290 	if (tp4sp_m->ip4src == 0xFFFFFFFF) {
3291 		/* 1st compare reg - IP source address */
3292 		w0 = 0;
3293 		w1 = 0;
3294 		w0 = tp4sp_v->ip4src;
3295 		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3296 		w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
3297 		w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1);
3298 		gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0);
3299 		gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1);
3300 		cmp_a = true;
3301 	}
3302 
3303 	/* ignore field if any masking set */
3304 	if (tp4sp_m->ip4dst == 0xFFFFFFFF) {
3305 		/* 2nd compare reg - IP destination address */
3306 		w0 = 0;
3307 		w1 = 0;
3308 		w0 = tp4sp_v->ip4dst;
3309 		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3310 		w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
3311 		w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1);
3312 		gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0);
3313 		gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1);
3314 		cmp_b = true;
3315 	}
3316 
3317 	/* ignore both port fields if masking set in both */
3318 	if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {
3319 		/* 3rd compare reg - source port, destination port */
3320 		w0 = 0;
3321 		w1 = 0;
3322 		w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1);
3323 		if (tp4sp_m->psrc == tp4sp_m->pdst) {
3324 			w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);
3325 			w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
3326 			w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3327 			w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
3328 		} else {
3329 			/* only one port definition */
3330 			w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */
3331 			w0 = GEM_BFINS(T2MASK, 0xFFFF, w0);
3332 			if (tp4sp_m->psrc == 0xFFFF) { /* src port */
3333 				w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);
3334 				w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
3335 			} else { /* dst port */
3336 				w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
3337 				w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1);
3338 			}
3339 		}
3340 		gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0);
3341 		gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1);
3342 		cmp_c = true;
3343 	}
3344 
3345 	t2_scr = 0;
3346 	t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);
3347 	t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr);
3348 	if (cmp_a)
3349 		t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr);
3350 	if (cmp_b)
3351 		t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr);
3352 	if (cmp_c)
3353 		t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr);
3354 	gem_writel_n(bp, SCRT2, index, t2_scr);
3355 }
3356 
3357 static int gem_add_flow_filter(struct net_device *netdev,
3358 		struct ethtool_rxnfc *cmd)
3359 {
3360 	struct macb *bp = netdev_priv(netdev);
3361 	struct ethtool_rx_flow_spec *fs = &cmd->fs;
3362 	struct ethtool_rx_fs_item *item, *newfs;
3363 	unsigned long flags;
3364 	int ret = -EINVAL;
3365 	bool added = false;
3366 
3367 	newfs = kmalloc(sizeof(*newfs), GFP_KERNEL);
3368 	if (newfs == NULL)
3369 		return -ENOMEM;
3370 	memcpy(&newfs->fs, fs, sizeof(newfs->fs));
3371 
3372 	netdev_dbg(netdev,
3373 			"Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
3374 			fs->flow_type, (int)fs->ring_cookie, fs->location,
3375 			htonl(fs->h_u.tcp_ip4_spec.ip4src),
3376 			htonl(fs->h_u.tcp_ip4_spec.ip4dst),
3377 			htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst));
3378 
3379 	spin_lock_irqsave(&bp->rx_fs_lock, flags);
3380 
3381 	/* find correct place to add in list */
3382 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3383 		if (item->fs.location > newfs->fs.location) {
3384 			list_add_tail(&newfs->list, &item->list);
3385 			added = true;
3386 			break;
3387 		} else if (item->fs.location == fs->location) {
3388 			netdev_err(netdev, "Rule not added: location %d not free!\n",
3389 					fs->location);
3390 			ret = -EBUSY;
3391 			goto err;
3392 		}
3393 	}
3394 	if (!added)
3395 		list_add_tail(&newfs->list, &bp->rx_fs_list.list);
3396 
3397 	gem_prog_cmp_regs(bp, fs);
3398 	bp->rx_fs_list.count++;
3399 	/* enable filtering if NTUPLE on */
3400 	gem_enable_flow_filters(bp, 1);
3401 
3402 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3403 	return 0;
3404 
3405 err:
3406 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3407 	kfree(newfs);
3408 	return ret;
3409 }
3410 
3411 static int gem_del_flow_filter(struct net_device *netdev,
3412 		struct ethtool_rxnfc *cmd)
3413 {
3414 	struct macb *bp = netdev_priv(netdev);
3415 	struct ethtool_rx_fs_item *item;
3416 	struct ethtool_rx_flow_spec *fs;
3417 	unsigned long flags;
3418 
3419 	spin_lock_irqsave(&bp->rx_fs_lock, flags);
3420 
3421 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3422 		if (item->fs.location == cmd->fs.location) {
3423 			/* disable screener regs for the flow entry */
3424 			fs = &(item->fs);
3425 			netdev_dbg(netdev,
3426 					"Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
3427 					fs->flow_type, (int)fs->ring_cookie, fs->location,
3428 					htonl(fs->h_u.tcp_ip4_spec.ip4src),
3429 					htonl(fs->h_u.tcp_ip4_spec.ip4dst),
3430 					htons(fs->h_u.tcp_ip4_spec.psrc),
3431 					htons(fs->h_u.tcp_ip4_spec.pdst));
3432 
3433 			gem_writel_n(bp, SCRT2, fs->location, 0);
3434 
3435 			list_del(&item->list);
3436 			bp->rx_fs_list.count--;
3437 			spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3438 			kfree(item);
3439 			return 0;
3440 		}
3441 	}
3442 
3443 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3444 	return -EINVAL;
3445 }
3446 
3447 static int gem_get_flow_entry(struct net_device *netdev,
3448 		struct ethtool_rxnfc *cmd)
3449 {
3450 	struct macb *bp = netdev_priv(netdev);
3451 	struct ethtool_rx_fs_item *item;
3452 
3453 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3454 		if (item->fs.location == cmd->fs.location) {
3455 			memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));
3456 			return 0;
3457 		}
3458 	}
3459 	return -EINVAL;
3460 }
3461 
3462 static int gem_get_all_flow_entries(struct net_device *netdev,
3463 		struct ethtool_rxnfc *cmd, u32 *rule_locs)
3464 {
3465 	struct macb *bp = netdev_priv(netdev);
3466 	struct ethtool_rx_fs_item *item;
3467 	uint32_t cnt = 0;
3468 
3469 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3470 		if (cnt == cmd->rule_cnt)
3471 			return -EMSGSIZE;
3472 		rule_locs[cnt] = item->fs.location;
3473 		cnt++;
3474 	}
3475 	cmd->data = bp->max_tuples;
3476 	cmd->rule_cnt = cnt;
3477 
3478 	return 0;
3479 }
3480 
3481 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
3482 		u32 *rule_locs)
3483 {
3484 	struct macb *bp = netdev_priv(netdev);
3485 	int ret = 0;
3486 
3487 	switch (cmd->cmd) {
3488 	case ETHTOOL_GRXRINGS:
3489 		cmd->data = bp->num_queues;
3490 		break;
3491 	case ETHTOOL_GRXCLSRLCNT:
3492 		cmd->rule_cnt = bp->rx_fs_list.count;
3493 		break;
3494 	case ETHTOOL_GRXCLSRULE:
3495 		ret = gem_get_flow_entry(netdev, cmd);
3496 		break;
3497 	case ETHTOOL_GRXCLSRLALL:
3498 		ret = gem_get_all_flow_entries(netdev, cmd, rule_locs);
3499 		break;
3500 	default:
3501 		netdev_err(netdev,
3502 			  "Command parameter %d is not supported\n", cmd->cmd);
3503 		ret = -EOPNOTSUPP;
3504 	}
3505 
3506 	return ret;
3507 }
3508 
3509 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
3510 {
3511 	struct macb *bp = netdev_priv(netdev);
3512 	int ret;
3513 
3514 	switch (cmd->cmd) {
3515 	case ETHTOOL_SRXCLSRLINS:
3516 		if ((cmd->fs.location >= bp->max_tuples)
3517 				|| (cmd->fs.ring_cookie >= bp->num_queues)) {
3518 			ret = -EINVAL;
3519 			break;
3520 		}
3521 		ret = gem_add_flow_filter(netdev, cmd);
3522 		break;
3523 	case ETHTOOL_SRXCLSRLDEL:
3524 		ret = gem_del_flow_filter(netdev, cmd);
3525 		break;
3526 	default:
3527 		netdev_err(netdev,
3528 			  "Command parameter %d is not supported\n", cmd->cmd);
3529 		ret = -EOPNOTSUPP;
3530 	}
3531 
3532 	return ret;
3533 }
3534 
3535 static const struct ethtool_ops macb_ethtool_ops = {
3536 	.get_regs_len		= macb_get_regs_len,
3537 	.get_regs		= macb_get_regs,
3538 	.get_link		= ethtool_op_get_link,
3539 	.get_ts_info		= ethtool_op_get_ts_info,
3540 	.get_wol		= macb_get_wol,
3541 	.set_wol		= macb_set_wol,
3542 	.get_link_ksettings     = macb_get_link_ksettings,
3543 	.set_link_ksettings     = macb_set_link_ksettings,
3544 	.get_ringparam		= macb_get_ringparam,
3545 	.set_ringparam		= macb_set_ringparam,
3546 };
3547 
3548 static const struct ethtool_ops gem_ethtool_ops = {
3549 	.get_regs_len		= macb_get_regs_len,
3550 	.get_regs		= macb_get_regs,
3551 	.get_wol		= macb_get_wol,
3552 	.set_wol		= macb_set_wol,
3553 	.get_link		= ethtool_op_get_link,
3554 	.get_ts_info		= macb_get_ts_info,
3555 	.get_ethtool_stats	= gem_get_ethtool_stats,
3556 	.get_strings		= gem_get_ethtool_strings,
3557 	.get_sset_count		= gem_get_sset_count,
3558 	.get_link_ksettings     = macb_get_link_ksettings,
3559 	.set_link_ksettings     = macb_set_link_ksettings,
3560 	.get_ringparam		= macb_get_ringparam,
3561 	.set_ringparam		= macb_set_ringparam,
3562 	.get_rxnfc			= gem_get_rxnfc,
3563 	.set_rxnfc			= gem_set_rxnfc,
3564 };
3565 
3566 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3567 {
3568 	struct macb *bp = netdev_priv(dev);
3569 
3570 	if (!netif_running(dev))
3571 		return -EINVAL;
3572 
3573 	if (bp->ptp_info) {
3574 		switch (cmd) {
3575 		case SIOCSHWTSTAMP:
3576 			return bp->ptp_info->set_hwtst(dev, rq, cmd);
3577 		case SIOCGHWTSTAMP:
3578 			return bp->ptp_info->get_hwtst(dev, rq);
3579 		}
3580 	}
3581 
3582 	return phylink_mii_ioctl(bp->phylink, rq, cmd);
3583 }
3584 
3585 static inline void macb_set_txcsum_feature(struct macb *bp,
3586 					   netdev_features_t features)
3587 {
3588 	u32 val;
3589 
3590 	if (!macb_is_gem(bp))
3591 		return;
3592 
3593 	val = gem_readl(bp, DMACFG);
3594 	if (features & NETIF_F_HW_CSUM)
3595 		val |= GEM_BIT(TXCOEN);
3596 	else
3597 		val &= ~GEM_BIT(TXCOEN);
3598 
3599 	gem_writel(bp, DMACFG, val);
3600 }
3601 
3602 static inline void macb_set_rxcsum_feature(struct macb *bp,
3603 					   netdev_features_t features)
3604 {
3605 	struct net_device *netdev = bp->dev;
3606 	u32 val;
3607 
3608 	if (!macb_is_gem(bp))
3609 		return;
3610 
3611 	val = gem_readl(bp, NCFGR);
3612 	if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC))
3613 		val |= GEM_BIT(RXCOEN);
3614 	else
3615 		val &= ~GEM_BIT(RXCOEN);
3616 
3617 	gem_writel(bp, NCFGR, val);
3618 }
3619 
3620 static inline void macb_set_rxflow_feature(struct macb *bp,
3621 					   netdev_features_t features)
3622 {
3623 	if (!macb_is_gem(bp))
3624 		return;
3625 
3626 	gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE));
3627 }
3628 
3629 static int macb_set_features(struct net_device *netdev,
3630 			     netdev_features_t features)
3631 {
3632 	struct macb *bp = netdev_priv(netdev);
3633 	netdev_features_t changed = features ^ netdev->features;
3634 
3635 	/* TX checksum offload */
3636 	if (changed & NETIF_F_HW_CSUM)
3637 		macb_set_txcsum_feature(bp, features);
3638 
3639 	/* RX checksum offload */
3640 	if (changed & NETIF_F_RXCSUM)
3641 		macb_set_rxcsum_feature(bp, features);
3642 
3643 	/* RX Flow Filters */
3644 	if (changed & NETIF_F_NTUPLE)
3645 		macb_set_rxflow_feature(bp, features);
3646 
3647 	return 0;
3648 }
3649 
3650 static void macb_restore_features(struct macb *bp)
3651 {
3652 	struct net_device *netdev = bp->dev;
3653 	netdev_features_t features = netdev->features;
3654 	struct ethtool_rx_fs_item *item;
3655 
3656 	/* TX checksum offload */
3657 	macb_set_txcsum_feature(bp, features);
3658 
3659 	/* RX checksum offload */
3660 	macb_set_rxcsum_feature(bp, features);
3661 
3662 	/* RX Flow Filters */
3663 	list_for_each_entry(item, &bp->rx_fs_list.list, list)
3664 		gem_prog_cmp_regs(bp, &item->fs);
3665 
3666 	macb_set_rxflow_feature(bp, features);
3667 }
3668 
3669 static const struct net_device_ops macb_netdev_ops = {
3670 	.ndo_open		= macb_open,
3671 	.ndo_stop		= macb_close,
3672 	.ndo_start_xmit		= macb_start_xmit,
3673 	.ndo_set_rx_mode	= macb_set_rx_mode,
3674 	.ndo_get_stats		= macb_get_stats,
3675 	.ndo_eth_ioctl		= macb_ioctl,
3676 	.ndo_validate_addr	= eth_validate_addr,
3677 	.ndo_change_mtu		= macb_change_mtu,
3678 	.ndo_set_mac_address	= eth_mac_addr,
3679 #ifdef CONFIG_NET_POLL_CONTROLLER
3680 	.ndo_poll_controller	= macb_poll_controller,
3681 #endif
3682 	.ndo_set_features	= macb_set_features,
3683 	.ndo_features_check	= macb_features_check,
3684 };
3685 
3686 /* Configure peripheral capabilities according to device tree
3687  * and integration options used
3688  */
3689 static void macb_configure_caps(struct macb *bp,
3690 				const struct macb_config *dt_conf)
3691 {
3692 	u32 dcfg;
3693 
3694 	if (dt_conf)
3695 		bp->caps = dt_conf->caps;
3696 
3697 	if (hw_is_gem(bp->regs, bp->native_io)) {
3698 		bp->caps |= MACB_CAPS_MACB_IS_GEM;
3699 
3700 		dcfg = gem_readl(bp, DCFG1);
3701 		if (GEM_BFEXT(IRQCOR, dcfg) == 0)
3702 			bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE;
3703 		if (GEM_BFEXT(NO_PCS, dcfg) == 0)
3704 			bp->caps |= MACB_CAPS_PCS;
3705 		dcfg = gem_readl(bp, DCFG12);
3706 		if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1)
3707 			bp->caps |= MACB_CAPS_HIGH_SPEED;
3708 		dcfg = gem_readl(bp, DCFG2);
3709 		if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0)
3710 			bp->caps |= MACB_CAPS_FIFO_MODE;
3711 #ifdef CONFIG_MACB_USE_HWSTAMP
3712 		if (gem_has_ptp(bp)) {
3713 			if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5)))
3714 				dev_err(&bp->pdev->dev,
3715 					"GEM doesn't support hardware ptp.\n");
3716 			else {
3717 				bp->hw_dma_cap |= HW_DMA_CAP_PTP;
3718 				bp->ptp_info = &gem_ptp_info;
3719 			}
3720 		}
3721 #endif
3722 	}
3723 
3724 	dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);
3725 }
3726 
3727 static void macb_probe_queues(void __iomem *mem,
3728 			      bool native_io,
3729 			      unsigned int *queue_mask,
3730 			      unsigned int *num_queues)
3731 {
3732 	*queue_mask = 0x1;
3733 	*num_queues = 1;
3734 
3735 	/* is it macb or gem ?
3736 	 *
3737 	 * We need to read directly from the hardware here because
3738 	 * we are early in the probe process and don't have the
3739 	 * MACB_CAPS_MACB_IS_GEM flag positioned
3740 	 */
3741 	if (!hw_is_gem(mem, native_io))
3742 		return;
3743 
3744 	/* bit 0 is never set but queue 0 always exists */
3745 	*queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff;
3746 	*num_queues = hweight32(*queue_mask);
3747 }
3748 
3749 static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk,
3750 			      struct clk *rx_clk, struct clk *tsu_clk)
3751 {
3752 	struct clk_bulk_data clks[] = {
3753 		{ .clk = tsu_clk, },
3754 		{ .clk = rx_clk, },
3755 		{ .clk = pclk, },
3756 		{ .clk = hclk, },
3757 		{ .clk = tx_clk },
3758 	};
3759 
3760 	clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks);
3761 }
3762 
3763 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
3764 			 struct clk **hclk, struct clk **tx_clk,
3765 			 struct clk **rx_clk, struct clk **tsu_clk)
3766 {
3767 	struct macb_platform_data *pdata;
3768 	int err;
3769 
3770 	pdata = dev_get_platdata(&pdev->dev);
3771 	if (pdata) {
3772 		*pclk = pdata->pclk;
3773 		*hclk = pdata->hclk;
3774 	} else {
3775 		*pclk = devm_clk_get(&pdev->dev, "pclk");
3776 		*hclk = devm_clk_get(&pdev->dev, "hclk");
3777 	}
3778 
3779 	if (IS_ERR_OR_NULL(*pclk))
3780 		return dev_err_probe(&pdev->dev,
3781 				     IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV,
3782 				     "failed to get pclk\n");
3783 
3784 	if (IS_ERR_OR_NULL(*hclk))
3785 		return dev_err_probe(&pdev->dev,
3786 				     IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV,
3787 				     "failed to get hclk\n");
3788 
3789 	*tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk");
3790 	if (IS_ERR(*tx_clk))
3791 		return PTR_ERR(*tx_clk);
3792 
3793 	*rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk");
3794 	if (IS_ERR(*rx_clk))
3795 		return PTR_ERR(*rx_clk);
3796 
3797 	*tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk");
3798 	if (IS_ERR(*tsu_clk))
3799 		return PTR_ERR(*tsu_clk);
3800 
3801 	err = clk_prepare_enable(*pclk);
3802 	if (err) {
3803 		dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);
3804 		return err;
3805 	}
3806 
3807 	err = clk_prepare_enable(*hclk);
3808 	if (err) {
3809 		dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err);
3810 		goto err_disable_pclk;
3811 	}
3812 
3813 	err = clk_prepare_enable(*tx_clk);
3814 	if (err) {
3815 		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);
3816 		goto err_disable_hclk;
3817 	}
3818 
3819 	err = clk_prepare_enable(*rx_clk);
3820 	if (err) {
3821 		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);
3822 		goto err_disable_txclk;
3823 	}
3824 
3825 	err = clk_prepare_enable(*tsu_clk);
3826 	if (err) {
3827 		dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err);
3828 		goto err_disable_rxclk;
3829 	}
3830 
3831 	return 0;
3832 
3833 err_disable_rxclk:
3834 	clk_disable_unprepare(*rx_clk);
3835 
3836 err_disable_txclk:
3837 	clk_disable_unprepare(*tx_clk);
3838 
3839 err_disable_hclk:
3840 	clk_disable_unprepare(*hclk);
3841 
3842 err_disable_pclk:
3843 	clk_disable_unprepare(*pclk);
3844 
3845 	return err;
3846 }
3847 
3848 static int macb_init(struct platform_device *pdev)
3849 {
3850 	struct net_device *dev = platform_get_drvdata(pdev);
3851 	unsigned int hw_q, q;
3852 	struct macb *bp = netdev_priv(dev);
3853 	struct macb_queue *queue;
3854 	int err;
3855 	u32 val, reg;
3856 
3857 	bp->tx_ring_size = DEFAULT_TX_RING_SIZE;
3858 	bp->rx_ring_size = DEFAULT_RX_RING_SIZE;
3859 
3860 	/* set the queue register mapping once for all: queue0 has a special
3861 	 * register mapping but we don't want to test the queue index then
3862 	 * compute the corresponding register offset at run time.
3863 	 */
3864 	for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) {
3865 		if (!(bp->queue_mask & (1 << hw_q)))
3866 			continue;
3867 
3868 		queue = &bp->queues[q];
3869 		queue->bp = bp;
3870 		netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT);
3871 		if (hw_q) {
3872 			queue->ISR  = GEM_ISR(hw_q - 1);
3873 			queue->IER  = GEM_IER(hw_q - 1);
3874 			queue->IDR  = GEM_IDR(hw_q - 1);
3875 			queue->IMR  = GEM_IMR(hw_q - 1);
3876 			queue->TBQP = GEM_TBQP(hw_q - 1);
3877 			queue->RBQP = GEM_RBQP(hw_q - 1);
3878 			queue->RBQS = GEM_RBQS(hw_q - 1);
3879 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
3880 			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
3881 				queue->TBQPH = GEM_TBQPH(hw_q - 1);
3882 				queue->RBQPH = GEM_RBQPH(hw_q - 1);
3883 			}
3884 #endif
3885 		} else {
3886 			/* queue0 uses legacy registers */
3887 			queue->ISR  = MACB_ISR;
3888 			queue->IER  = MACB_IER;
3889 			queue->IDR  = MACB_IDR;
3890 			queue->IMR  = MACB_IMR;
3891 			queue->TBQP = MACB_TBQP;
3892 			queue->RBQP = MACB_RBQP;
3893 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
3894 			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
3895 				queue->TBQPH = MACB_TBQPH;
3896 				queue->RBQPH = MACB_RBQPH;
3897 			}
3898 #endif
3899 		}
3900 
3901 		/* get irq: here we use the linux queue index, not the hardware
3902 		 * queue index. the queue irq definitions in the device tree
3903 		 * must remove the optional gaps that could exist in the
3904 		 * hardware queue mask.
3905 		 */
3906 		queue->irq = platform_get_irq(pdev, q);
3907 		err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt,
3908 				       IRQF_SHARED, dev->name, queue);
3909 		if (err) {
3910 			dev_err(&pdev->dev,
3911 				"Unable to request IRQ %d (error %d)\n",
3912 				queue->irq, err);
3913 			return err;
3914 		}
3915 
3916 		INIT_WORK(&queue->tx_error_task, macb_tx_error_task);
3917 		q++;
3918 	}
3919 
3920 	dev->netdev_ops = &macb_netdev_ops;
3921 
3922 	/* setup appropriated routines according to adapter type */
3923 	if (macb_is_gem(bp)) {
3924 		bp->max_tx_length = GEM_MAX_TX_LEN;
3925 		bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers;
3926 		bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers;
3927 		bp->macbgem_ops.mog_init_rings = gem_init_rings;
3928 		bp->macbgem_ops.mog_rx = gem_rx;
3929 		dev->ethtool_ops = &gem_ethtool_ops;
3930 	} else {
3931 		bp->max_tx_length = MACB_MAX_TX_LEN;
3932 		bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers;
3933 		bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers;
3934 		bp->macbgem_ops.mog_init_rings = macb_init_rings;
3935 		bp->macbgem_ops.mog_rx = macb_rx;
3936 		dev->ethtool_ops = &macb_ethtool_ops;
3937 	}
3938 
3939 	/* Set features */
3940 	dev->hw_features = NETIF_F_SG;
3941 
3942 	/* Check LSO capability */
3943 	if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6)))
3944 		dev->hw_features |= MACB_NETIF_LSO;
3945 
3946 	/* Checksum offload is only available on gem with packet buffer */
3947 	if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE))
3948 		dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
3949 	if (bp->caps & MACB_CAPS_SG_DISABLED)
3950 		dev->hw_features &= ~NETIF_F_SG;
3951 	dev->features = dev->hw_features;
3952 
3953 	/* Check RX Flow Filters support.
3954 	 * Max Rx flows set by availability of screeners & compare regs:
3955 	 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs
3956 	 */
3957 	reg = gem_readl(bp, DCFG8);
3958 	bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),
3959 			GEM_BFEXT(T2SCR, reg));
3960 	INIT_LIST_HEAD(&bp->rx_fs_list.list);
3961 	if (bp->max_tuples > 0) {
3962 		/* also needs one ethtype match to check IPv4 */
3963 		if (GEM_BFEXT(SCR2ETH, reg) > 0) {
3964 			/* program this reg now */
3965 			reg = 0;
3966 			reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg);
3967 			gem_writel_n(bp, ETHT, SCRT2_ETHT, reg);
3968 			/* Filtering is supported in hw but don't enable it in kernel now */
3969 			dev->hw_features |= NETIF_F_NTUPLE;
3970 			/* init Rx flow definitions */
3971 			bp->rx_fs_list.count = 0;
3972 			spin_lock_init(&bp->rx_fs_lock);
3973 		} else
3974 			bp->max_tuples = 0;
3975 	}
3976 
3977 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {
3978 		val = 0;
3979 		if (phy_interface_mode_is_rgmii(bp->phy_interface))
3980 			val = bp->usrio->rgmii;
3981 		else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII &&
3982 			 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))
3983 			val = bp->usrio->rmii;
3984 		else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))
3985 			val = bp->usrio->mii;
3986 
3987 		if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN)
3988 			val |= bp->usrio->refclk;
3989 
3990 		macb_or_gem_writel(bp, USRIO, val);
3991 	}
3992 
3993 	/* Set MII management clock divider */
3994 	val = macb_mdc_clk_div(bp);
3995 	val |= macb_dbw(bp);
3996 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)
3997 		val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
3998 	macb_writel(bp, NCFGR, val);
3999 
4000 	return 0;
4001 }
4002 
4003 static const struct macb_usrio_config macb_default_usrio = {
4004 	.mii = MACB_BIT(MII),
4005 	.rmii = MACB_BIT(RMII),
4006 	.rgmii = GEM_BIT(RGMII),
4007 	.refclk = MACB_BIT(CLKEN),
4008 };
4009 
4010 #if defined(CONFIG_OF)
4011 /* 1518 rounded up */
4012 #define AT91ETHER_MAX_RBUFF_SZ	0x600
4013 /* max number of receive buffers */
4014 #define AT91ETHER_MAX_RX_DESCR	9
4015 
4016 static struct sifive_fu540_macb_mgmt *mgmt;
4017 
4018 static int at91ether_alloc_coherent(struct macb *lp)
4019 {
4020 	struct macb_queue *q = &lp->queues[0];
4021 
4022 	q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,
4023 					 (AT91ETHER_MAX_RX_DESCR *
4024 					  macb_dma_desc_get_size(lp)),
4025 					 &q->rx_ring_dma, GFP_KERNEL);
4026 	if (!q->rx_ring)
4027 		return -ENOMEM;
4028 
4029 	q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,
4030 					    AT91ETHER_MAX_RX_DESCR *
4031 					    AT91ETHER_MAX_RBUFF_SZ,
4032 					    &q->rx_buffers_dma, GFP_KERNEL);
4033 	if (!q->rx_buffers) {
4034 		dma_free_coherent(&lp->pdev->dev,
4035 				  AT91ETHER_MAX_RX_DESCR *
4036 				  macb_dma_desc_get_size(lp),
4037 				  q->rx_ring, q->rx_ring_dma);
4038 		q->rx_ring = NULL;
4039 		return -ENOMEM;
4040 	}
4041 
4042 	return 0;
4043 }
4044 
4045 static void at91ether_free_coherent(struct macb *lp)
4046 {
4047 	struct macb_queue *q = &lp->queues[0];
4048 
4049 	if (q->rx_ring) {
4050 		dma_free_coherent(&lp->pdev->dev,
4051 				  AT91ETHER_MAX_RX_DESCR *
4052 				  macb_dma_desc_get_size(lp),
4053 				  q->rx_ring, q->rx_ring_dma);
4054 		q->rx_ring = NULL;
4055 	}
4056 
4057 	if (q->rx_buffers) {
4058 		dma_free_coherent(&lp->pdev->dev,
4059 				  AT91ETHER_MAX_RX_DESCR *
4060 				  AT91ETHER_MAX_RBUFF_SZ,
4061 				  q->rx_buffers, q->rx_buffers_dma);
4062 		q->rx_buffers = NULL;
4063 	}
4064 }
4065 
4066 /* Initialize and start the Receiver and Transmit subsystems */
4067 static int at91ether_start(struct macb *lp)
4068 {
4069 	struct macb_queue *q = &lp->queues[0];
4070 	struct macb_dma_desc *desc;
4071 	dma_addr_t addr;
4072 	u32 ctl;
4073 	int i, ret;
4074 
4075 	ret = at91ether_alloc_coherent(lp);
4076 	if (ret)
4077 		return ret;
4078 
4079 	addr = q->rx_buffers_dma;
4080 	for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) {
4081 		desc = macb_rx_desc(q, i);
4082 		macb_set_addr(lp, desc, addr);
4083 		desc->ctrl = 0;
4084 		addr += AT91ETHER_MAX_RBUFF_SZ;
4085 	}
4086 
4087 	/* Set the Wrap bit on the last descriptor */
4088 	desc->addr |= MACB_BIT(RX_WRAP);
4089 
4090 	/* Reset buffer index */
4091 	q->rx_tail = 0;
4092 
4093 	/* Program address of descriptor list in Rx Buffer Queue register */
4094 	macb_writel(lp, RBQP, q->rx_ring_dma);
4095 
4096 	/* Enable Receive and Transmit */
4097 	ctl = macb_readl(lp, NCR);
4098 	macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE));
4099 
4100 	/* Enable MAC interrupts */
4101 	macb_writel(lp, IER, MACB_BIT(RCOMP)	|
4102 			     MACB_BIT(RXUBR)	|
4103 			     MACB_BIT(ISR_TUND)	|
4104 			     MACB_BIT(ISR_RLE)	|
4105 			     MACB_BIT(TCOMP)	|
4106 			     MACB_BIT(ISR_ROVR)	|
4107 			     MACB_BIT(HRESP));
4108 
4109 	return 0;
4110 }
4111 
4112 static void at91ether_stop(struct macb *lp)
4113 {
4114 	u32 ctl;
4115 
4116 	/* Disable MAC interrupts */
4117 	macb_writel(lp, IDR, MACB_BIT(RCOMP)	|
4118 			     MACB_BIT(RXUBR)	|
4119 			     MACB_BIT(ISR_TUND)	|
4120 			     MACB_BIT(ISR_RLE)	|
4121 			     MACB_BIT(TCOMP)	|
4122 			     MACB_BIT(ISR_ROVR) |
4123 			     MACB_BIT(HRESP));
4124 
4125 	/* Disable Receiver and Transmitter */
4126 	ctl = macb_readl(lp, NCR);
4127 	macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE)));
4128 
4129 	/* Free resources. */
4130 	at91ether_free_coherent(lp);
4131 }
4132 
4133 /* Open the ethernet interface */
4134 static int at91ether_open(struct net_device *dev)
4135 {
4136 	struct macb *lp = netdev_priv(dev);
4137 	u32 ctl;
4138 	int ret;
4139 
4140 	ret = pm_runtime_get_sync(&lp->pdev->dev);
4141 	if (ret < 0) {
4142 		pm_runtime_put_noidle(&lp->pdev->dev);
4143 		return ret;
4144 	}
4145 
4146 	/* Clear internal statistics */
4147 	ctl = macb_readl(lp, NCR);
4148 	macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT));
4149 
4150 	macb_set_hwaddr(lp);
4151 
4152 	ret = at91ether_start(lp);
4153 	if (ret)
4154 		goto pm_exit;
4155 
4156 	ret = macb_phylink_connect(lp);
4157 	if (ret)
4158 		goto stop;
4159 
4160 	netif_start_queue(dev);
4161 
4162 	return 0;
4163 
4164 stop:
4165 	at91ether_stop(lp);
4166 pm_exit:
4167 	pm_runtime_put_sync(&lp->pdev->dev);
4168 	return ret;
4169 }
4170 
4171 /* Close the interface */
4172 static int at91ether_close(struct net_device *dev)
4173 {
4174 	struct macb *lp = netdev_priv(dev);
4175 
4176 	netif_stop_queue(dev);
4177 
4178 	phylink_stop(lp->phylink);
4179 	phylink_disconnect_phy(lp->phylink);
4180 
4181 	at91ether_stop(lp);
4182 
4183 	return pm_runtime_put(&lp->pdev->dev);
4184 }
4185 
4186 /* Transmit packet */
4187 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb,
4188 					struct net_device *dev)
4189 {
4190 	struct macb *lp = netdev_priv(dev);
4191 
4192 	if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) {
4193 		int desc = 0;
4194 
4195 		netif_stop_queue(dev);
4196 
4197 		/* Store packet information (to free when Tx completed) */
4198 		lp->rm9200_txq[desc].skb = skb;
4199 		lp->rm9200_txq[desc].size = skb->len;
4200 		lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data,
4201 							      skb->len, DMA_TO_DEVICE);
4202 		if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) {
4203 			dev_kfree_skb_any(skb);
4204 			dev->stats.tx_dropped++;
4205 			netdev_err(dev, "%s: DMA mapping error\n", __func__);
4206 			return NETDEV_TX_OK;
4207 		}
4208 
4209 		/* Set address of the data in the Transmit Address register */
4210 		macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping);
4211 		/* Set length of the packet in the Transmit Control register */
4212 		macb_writel(lp, TCR, skb->len);
4213 
4214 	} else {
4215 		netdev_err(dev, "%s called, but device is busy!\n", __func__);
4216 		return NETDEV_TX_BUSY;
4217 	}
4218 
4219 	return NETDEV_TX_OK;
4220 }
4221 
4222 /* Extract received frame from buffer descriptors and sent to upper layers.
4223  * (Called from interrupt context)
4224  */
4225 static void at91ether_rx(struct net_device *dev)
4226 {
4227 	struct macb *lp = netdev_priv(dev);
4228 	struct macb_queue *q = &lp->queues[0];
4229 	struct macb_dma_desc *desc;
4230 	unsigned char *p_recv;
4231 	struct sk_buff *skb;
4232 	unsigned int pktlen;
4233 
4234 	desc = macb_rx_desc(q, q->rx_tail);
4235 	while (desc->addr & MACB_BIT(RX_USED)) {
4236 		p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;
4237 		pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);
4238 		skb = netdev_alloc_skb(dev, pktlen + 2);
4239 		if (skb) {
4240 			skb_reserve(skb, 2);
4241 			skb_put_data(skb, p_recv, pktlen);
4242 
4243 			skb->protocol = eth_type_trans(skb, dev);
4244 			dev->stats.rx_packets++;
4245 			dev->stats.rx_bytes += pktlen;
4246 			netif_rx(skb);
4247 		} else {
4248 			dev->stats.rx_dropped++;
4249 		}
4250 
4251 		if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH))
4252 			dev->stats.multicast++;
4253 
4254 		/* reset ownership bit */
4255 		desc->addr &= ~MACB_BIT(RX_USED);
4256 
4257 		/* wrap after last buffer */
4258 		if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)
4259 			q->rx_tail = 0;
4260 		else
4261 			q->rx_tail++;
4262 
4263 		desc = macb_rx_desc(q, q->rx_tail);
4264 	}
4265 }
4266 
4267 /* MAC interrupt handler */
4268 static irqreturn_t at91ether_interrupt(int irq, void *dev_id)
4269 {
4270 	struct net_device *dev = dev_id;
4271 	struct macb *lp = netdev_priv(dev);
4272 	u32 intstatus, ctl;
4273 	unsigned int desc;
4274 
4275 	/* MAC Interrupt Status register indicates what interrupts are pending.
4276 	 * It is automatically cleared once read.
4277 	 */
4278 	intstatus = macb_readl(lp, ISR);
4279 
4280 	/* Receive complete */
4281 	if (intstatus & MACB_BIT(RCOMP))
4282 		at91ether_rx(dev);
4283 
4284 	/* Transmit complete */
4285 	if (intstatus & MACB_BIT(TCOMP)) {
4286 		/* The TCOM bit is set even if the transmission failed */
4287 		if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE)))
4288 			dev->stats.tx_errors++;
4289 
4290 		desc = 0;
4291 		if (lp->rm9200_txq[desc].skb) {
4292 			dev_consume_skb_irq(lp->rm9200_txq[desc].skb);
4293 			lp->rm9200_txq[desc].skb = NULL;
4294 			dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping,
4295 					 lp->rm9200_txq[desc].size, DMA_TO_DEVICE);
4296 			dev->stats.tx_packets++;
4297 			dev->stats.tx_bytes += lp->rm9200_txq[desc].size;
4298 		}
4299 		netif_wake_queue(dev);
4300 	}
4301 
4302 	/* Work-around for EMAC Errata section 41.3.1 */
4303 	if (intstatus & MACB_BIT(RXUBR)) {
4304 		ctl = macb_readl(lp, NCR);
4305 		macb_writel(lp, NCR, ctl & ~MACB_BIT(RE));
4306 		wmb();
4307 		macb_writel(lp, NCR, ctl | MACB_BIT(RE));
4308 	}
4309 
4310 	if (intstatus & MACB_BIT(ISR_ROVR))
4311 		netdev_err(dev, "ROVR error\n");
4312 
4313 	return IRQ_HANDLED;
4314 }
4315 
4316 #ifdef CONFIG_NET_POLL_CONTROLLER
4317 static void at91ether_poll_controller(struct net_device *dev)
4318 {
4319 	unsigned long flags;
4320 
4321 	local_irq_save(flags);
4322 	at91ether_interrupt(dev->irq, dev);
4323 	local_irq_restore(flags);
4324 }
4325 #endif
4326 
4327 static const struct net_device_ops at91ether_netdev_ops = {
4328 	.ndo_open		= at91ether_open,
4329 	.ndo_stop		= at91ether_close,
4330 	.ndo_start_xmit		= at91ether_start_xmit,
4331 	.ndo_get_stats		= macb_get_stats,
4332 	.ndo_set_rx_mode	= macb_set_rx_mode,
4333 	.ndo_set_mac_address	= eth_mac_addr,
4334 	.ndo_eth_ioctl		= macb_ioctl,
4335 	.ndo_validate_addr	= eth_validate_addr,
4336 #ifdef CONFIG_NET_POLL_CONTROLLER
4337 	.ndo_poll_controller	= at91ether_poll_controller,
4338 #endif
4339 };
4340 
4341 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk,
4342 			      struct clk **hclk, struct clk **tx_clk,
4343 			      struct clk **rx_clk, struct clk **tsu_clk)
4344 {
4345 	int err;
4346 
4347 	*hclk = NULL;
4348 	*tx_clk = NULL;
4349 	*rx_clk = NULL;
4350 	*tsu_clk = NULL;
4351 
4352 	*pclk = devm_clk_get(&pdev->dev, "ether_clk");
4353 	if (IS_ERR(*pclk))
4354 		return PTR_ERR(*pclk);
4355 
4356 	err = clk_prepare_enable(*pclk);
4357 	if (err) {
4358 		dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);
4359 		return err;
4360 	}
4361 
4362 	return 0;
4363 }
4364 
4365 static int at91ether_init(struct platform_device *pdev)
4366 {
4367 	struct net_device *dev = platform_get_drvdata(pdev);
4368 	struct macb *bp = netdev_priv(dev);
4369 	int err;
4370 
4371 	bp->queues[0].bp = bp;
4372 
4373 	dev->netdev_ops = &at91ether_netdev_ops;
4374 	dev->ethtool_ops = &macb_ethtool_ops;
4375 
4376 	err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt,
4377 			       0, dev->name, dev);
4378 	if (err)
4379 		return err;
4380 
4381 	macb_writel(bp, NCR, 0);
4382 
4383 	macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG));
4384 
4385 	return 0;
4386 }
4387 
4388 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw,
4389 					       unsigned long parent_rate)
4390 {
4391 	return mgmt->rate;
4392 }
4393 
4394 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate,
4395 				     unsigned long *parent_rate)
4396 {
4397 	if (WARN_ON(rate < 2500000))
4398 		return 2500000;
4399 	else if (rate == 2500000)
4400 		return 2500000;
4401 	else if (WARN_ON(rate < 13750000))
4402 		return 2500000;
4403 	else if (WARN_ON(rate < 25000000))
4404 		return 25000000;
4405 	else if (rate == 25000000)
4406 		return 25000000;
4407 	else if (WARN_ON(rate < 75000000))
4408 		return 25000000;
4409 	else if (WARN_ON(rate < 125000000))
4410 		return 125000000;
4411 	else if (rate == 125000000)
4412 		return 125000000;
4413 
4414 	WARN_ON(rate > 125000000);
4415 
4416 	return 125000000;
4417 }
4418 
4419 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate,
4420 				  unsigned long parent_rate)
4421 {
4422 	rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate);
4423 	if (rate != 125000000)
4424 		iowrite32(1, mgmt->reg);
4425 	else
4426 		iowrite32(0, mgmt->reg);
4427 	mgmt->rate = rate;
4428 
4429 	return 0;
4430 }
4431 
4432 static const struct clk_ops fu540_c000_ops = {
4433 	.recalc_rate = fu540_macb_tx_recalc_rate,
4434 	.round_rate = fu540_macb_tx_round_rate,
4435 	.set_rate = fu540_macb_tx_set_rate,
4436 };
4437 
4438 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk,
4439 			       struct clk **hclk, struct clk **tx_clk,
4440 			       struct clk **rx_clk, struct clk **tsu_clk)
4441 {
4442 	struct clk_init_data init;
4443 	int err = 0;
4444 
4445 	err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk);
4446 	if (err)
4447 		return err;
4448 
4449 	mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL);
4450 	if (!mgmt) {
4451 		err = -ENOMEM;
4452 		goto err_disable_clks;
4453 	}
4454 
4455 	init.name = "sifive-gemgxl-mgmt";
4456 	init.ops = &fu540_c000_ops;
4457 	init.flags = 0;
4458 	init.num_parents = 0;
4459 
4460 	mgmt->rate = 0;
4461 	mgmt->hw.init = &init;
4462 
4463 	*tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw);
4464 	if (IS_ERR(*tx_clk)) {
4465 		err = PTR_ERR(*tx_clk);
4466 		goto err_disable_clks;
4467 	}
4468 
4469 	err = clk_prepare_enable(*tx_clk);
4470 	if (err) {
4471 		dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);
4472 		*tx_clk = NULL;
4473 		goto err_disable_clks;
4474 	} else {
4475 		dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name);
4476 	}
4477 
4478 	return 0;
4479 
4480 err_disable_clks:
4481 	macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk);
4482 
4483 	return err;
4484 }
4485 
4486 static int fu540_c000_init(struct platform_device *pdev)
4487 {
4488 	mgmt->reg = devm_platform_ioremap_resource(pdev, 1);
4489 	if (IS_ERR(mgmt->reg))
4490 		return PTR_ERR(mgmt->reg);
4491 
4492 	return macb_init(pdev);
4493 }
4494 
4495 static const struct macb_usrio_config sama7g5_usrio = {
4496 	.mii = 0,
4497 	.rmii = 1,
4498 	.rgmii = 2,
4499 	.refclk = BIT(2),
4500 	.hdfctlen = BIT(6),
4501 };
4502 
4503 static const struct macb_config fu540_c000_config = {
4504 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |
4505 		MACB_CAPS_GEM_HAS_PTP,
4506 	.dma_burst_length = 16,
4507 	.clk_init = fu540_c000_clk_init,
4508 	.init = fu540_c000_init,
4509 	.jumbo_max_len = 10240,
4510 	.usrio = &macb_default_usrio,
4511 };
4512 
4513 static const struct macb_config at91sam9260_config = {
4514 	.caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4515 	.clk_init = macb_clk_init,
4516 	.init = macb_init,
4517 	.usrio = &macb_default_usrio,
4518 };
4519 
4520 static const struct macb_config sama5d3macb_config = {
4521 	.caps = MACB_CAPS_SG_DISABLED
4522 	      | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4523 	.clk_init = macb_clk_init,
4524 	.init = macb_init,
4525 	.usrio = &macb_default_usrio,
4526 };
4527 
4528 static const struct macb_config pc302gem_config = {
4529 	.caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE,
4530 	.dma_burst_length = 16,
4531 	.clk_init = macb_clk_init,
4532 	.init = macb_init,
4533 	.usrio = &macb_default_usrio,
4534 };
4535 
4536 static const struct macb_config sama5d2_config = {
4537 	.caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4538 	.dma_burst_length = 16,
4539 	.clk_init = macb_clk_init,
4540 	.init = macb_init,
4541 	.usrio = &macb_default_usrio,
4542 };
4543 
4544 static const struct macb_config sama5d29_config = {
4545 	.caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP,
4546 	.dma_burst_length = 16,
4547 	.clk_init = macb_clk_init,
4548 	.init = macb_init,
4549 	.usrio = &macb_default_usrio,
4550 };
4551 
4552 static const struct macb_config sama5d3_config = {
4553 	.caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE
4554 	      | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO,
4555 	.dma_burst_length = 16,
4556 	.clk_init = macb_clk_init,
4557 	.init = macb_init,
4558 	.jumbo_max_len = 10240,
4559 	.usrio = &macb_default_usrio,
4560 };
4561 
4562 static const struct macb_config sama5d4_config = {
4563 	.caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4564 	.dma_burst_length = 4,
4565 	.clk_init = macb_clk_init,
4566 	.init = macb_init,
4567 	.usrio = &macb_default_usrio,
4568 };
4569 
4570 static const struct macb_config emac_config = {
4571 	.caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC,
4572 	.clk_init = at91ether_clk_init,
4573 	.init = at91ether_init,
4574 	.usrio = &macb_default_usrio,
4575 };
4576 
4577 static const struct macb_config np4_config = {
4578 	.caps = MACB_CAPS_USRIO_DISABLED,
4579 	.clk_init = macb_clk_init,
4580 	.init = macb_init,
4581 	.usrio = &macb_default_usrio,
4582 };
4583 
4584 static const struct macb_config zynqmp_config = {
4585 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4586 			MACB_CAPS_JUMBO |
4587 			MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH,
4588 	.dma_burst_length = 16,
4589 	.clk_init = macb_clk_init,
4590 	.init = macb_init,
4591 	.jumbo_max_len = 10240,
4592 	.usrio = &macb_default_usrio,
4593 };
4594 
4595 static const struct macb_config zynq_config = {
4596 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF |
4597 		MACB_CAPS_NEEDS_RSTONUBR,
4598 	.dma_burst_length = 16,
4599 	.clk_init = macb_clk_init,
4600 	.init = macb_init,
4601 	.usrio = &macb_default_usrio,
4602 };
4603 
4604 static const struct macb_config sama7g5_gem_config = {
4605 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG |
4606 		MACB_CAPS_MIIONRGMII,
4607 	.dma_burst_length = 16,
4608 	.clk_init = macb_clk_init,
4609 	.init = macb_init,
4610 	.usrio = &sama7g5_usrio,
4611 };
4612 
4613 static const struct macb_config sama7g5_emac_config = {
4614 	.caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII |
4615 		MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII,
4616 	.dma_burst_length = 16,
4617 	.clk_init = macb_clk_init,
4618 	.init = macb_init,
4619 	.usrio = &sama7g5_usrio,
4620 };
4621 
4622 static const struct of_device_id macb_dt_ids[] = {
4623 	{ .compatible = "cdns,at32ap7000-macb" },
4624 	{ .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
4625 	{ .compatible = "cdns,macb" },
4626 	{ .compatible = "cdns,np4-macb", .data = &np4_config },
4627 	{ .compatible = "cdns,pc302-gem", .data = &pc302gem_config },
4628 	{ .compatible = "cdns,gem", .data = &pc302gem_config },
4629 	{ .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },
4630 	{ .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },
4631 	{ .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config },
4632 	{ .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },
4633 	{ .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },
4634 	{ .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },
4635 	{ .compatible = "cdns,at91rm9200-emac", .data = &emac_config },
4636 	{ .compatible = "cdns,emac", .data = &emac_config },
4637 	{ .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config},
4638 	{ .compatible = "cdns,zynq-gem", .data = &zynq_config },
4639 	{ .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
4640 	{ .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },
4641 	{ .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },
4642 	{ /* sentinel */ }
4643 };
4644 MODULE_DEVICE_TABLE(of, macb_dt_ids);
4645 #endif /* CONFIG_OF */
4646 
4647 static const struct macb_config default_gem_config = {
4648 	.caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4649 			MACB_CAPS_JUMBO |
4650 			MACB_CAPS_GEM_HAS_PTP,
4651 	.dma_burst_length = 16,
4652 	.clk_init = macb_clk_init,
4653 	.init = macb_init,
4654 	.usrio = &macb_default_usrio,
4655 	.jumbo_max_len = 10240,
4656 };
4657 
4658 static int macb_probe(struct platform_device *pdev)
4659 {
4660 	const struct macb_config *macb_config = &default_gem_config;
4661 	int (*clk_init)(struct platform_device *, struct clk **,
4662 			struct clk **, struct clk **,  struct clk **,
4663 			struct clk **) = macb_config->clk_init;
4664 	int (*init)(struct platform_device *) = macb_config->init;
4665 	struct device_node *np = pdev->dev.of_node;
4666 	struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL;
4667 	struct clk *tsu_clk = NULL;
4668 	unsigned int queue_mask, num_queues;
4669 	bool native_io;
4670 	phy_interface_t interface;
4671 	struct net_device *dev;
4672 	struct resource *regs;
4673 	void __iomem *mem;
4674 	struct macb *bp;
4675 	int err, val;
4676 
4677 	mem = devm_platform_get_and_ioremap_resource(pdev, 0, &regs);
4678 	if (IS_ERR(mem))
4679 		return PTR_ERR(mem);
4680 
4681 	if (np) {
4682 		const struct of_device_id *match;
4683 
4684 		match = of_match_node(macb_dt_ids, np);
4685 		if (match && match->data) {
4686 			macb_config = match->data;
4687 			clk_init = macb_config->clk_init;
4688 			init = macb_config->init;
4689 		}
4690 	}
4691 
4692 	err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk);
4693 	if (err)
4694 		return err;
4695 
4696 	pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT);
4697 	pm_runtime_use_autosuspend(&pdev->dev);
4698 	pm_runtime_get_noresume(&pdev->dev);
4699 	pm_runtime_set_active(&pdev->dev);
4700 	pm_runtime_enable(&pdev->dev);
4701 	native_io = hw_is_native_io(mem);
4702 
4703 	macb_probe_queues(mem, native_io, &queue_mask, &num_queues);
4704 	dev = alloc_etherdev_mq(sizeof(*bp), num_queues);
4705 	if (!dev) {
4706 		err = -ENOMEM;
4707 		goto err_disable_clocks;
4708 	}
4709 
4710 	dev->base_addr = regs->start;
4711 
4712 	SET_NETDEV_DEV(dev, &pdev->dev);
4713 
4714 	bp = netdev_priv(dev);
4715 	bp->pdev = pdev;
4716 	bp->dev = dev;
4717 	bp->regs = mem;
4718 	bp->native_io = native_io;
4719 	if (native_io) {
4720 		bp->macb_reg_readl = hw_readl_native;
4721 		bp->macb_reg_writel = hw_writel_native;
4722 	} else {
4723 		bp->macb_reg_readl = hw_readl;
4724 		bp->macb_reg_writel = hw_writel;
4725 	}
4726 	bp->num_queues = num_queues;
4727 	bp->queue_mask = queue_mask;
4728 	if (macb_config)
4729 		bp->dma_burst_length = macb_config->dma_burst_length;
4730 	bp->pclk = pclk;
4731 	bp->hclk = hclk;
4732 	bp->tx_clk = tx_clk;
4733 	bp->rx_clk = rx_clk;
4734 	bp->tsu_clk = tsu_clk;
4735 	if (macb_config)
4736 		bp->jumbo_max_len = macb_config->jumbo_max_len;
4737 
4738 	bp->wol = 0;
4739 	if (of_get_property(np, "magic-packet", NULL))
4740 		bp->wol |= MACB_WOL_HAS_MAGIC_PACKET;
4741 	device_set_wakeup_capable(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET);
4742 
4743 	bp->usrio = macb_config->usrio;
4744 
4745 	spin_lock_init(&bp->lock);
4746 
4747 	/* setup capabilities */
4748 	macb_configure_caps(bp, macb_config);
4749 
4750 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
4751 	if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) {
4752 		dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
4753 		bp->hw_dma_cap |= HW_DMA_CAP_64B;
4754 	}
4755 #endif
4756 	platform_set_drvdata(pdev, dev);
4757 
4758 	dev->irq = platform_get_irq(pdev, 0);
4759 	if (dev->irq < 0) {
4760 		err = dev->irq;
4761 		goto err_out_free_netdev;
4762 	}
4763 
4764 	/* MTU range: 68 - 1500 or 10240 */
4765 	dev->min_mtu = GEM_MTU_MIN_SIZE;
4766 	if (bp->caps & MACB_CAPS_JUMBO)
4767 		dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN;
4768 	else
4769 		dev->max_mtu = ETH_DATA_LEN;
4770 
4771 	if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) {
4772 		val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10));
4773 		if (val)
4774 			bp->rx_bd_rd_prefetch = (2 << (val - 1)) *
4775 						macb_dma_desc_get_size(bp);
4776 
4777 		val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10));
4778 		if (val)
4779 			bp->tx_bd_rd_prefetch = (2 << (val - 1)) *
4780 						macb_dma_desc_get_size(bp);
4781 	}
4782 
4783 	bp->rx_intr_mask = MACB_RX_INT_FLAGS;
4784 	if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR)
4785 		bp->rx_intr_mask |= MACB_BIT(RXUBR);
4786 
4787 	err = of_get_ethdev_address(np, bp->dev);
4788 	if (err == -EPROBE_DEFER)
4789 		goto err_out_free_netdev;
4790 	else if (err)
4791 		macb_get_hwaddr(bp);
4792 
4793 	err = of_get_phy_mode(np, &interface);
4794 	if (err)
4795 		/* not found in DT, MII by default */
4796 		bp->phy_interface = PHY_INTERFACE_MODE_MII;
4797 	else
4798 		bp->phy_interface = interface;
4799 
4800 	/* IP specific init */
4801 	err = init(pdev);
4802 	if (err)
4803 		goto err_out_free_netdev;
4804 
4805 	err = macb_mii_init(bp);
4806 	if (err)
4807 		goto err_out_free_netdev;
4808 
4809 	netif_carrier_off(dev);
4810 
4811 	err = register_netdev(dev);
4812 	if (err) {
4813 		dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
4814 		goto err_out_unregister_mdio;
4815 	}
4816 
4817 	tasklet_setup(&bp->hresp_err_tasklet, macb_hresp_error_task);
4818 
4819 	netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n",
4820 		    macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID),
4821 		    dev->base_addr, dev->irq, dev->dev_addr);
4822 
4823 	pm_runtime_mark_last_busy(&bp->pdev->dev);
4824 	pm_runtime_put_autosuspend(&bp->pdev->dev);
4825 
4826 	return 0;
4827 
4828 err_out_unregister_mdio:
4829 	mdiobus_unregister(bp->mii_bus);
4830 	mdiobus_free(bp->mii_bus);
4831 
4832 err_out_free_netdev:
4833 	free_netdev(dev);
4834 
4835 err_disable_clocks:
4836 	macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk);
4837 	pm_runtime_disable(&pdev->dev);
4838 	pm_runtime_set_suspended(&pdev->dev);
4839 	pm_runtime_dont_use_autosuspend(&pdev->dev);
4840 
4841 	return err;
4842 }
4843 
4844 static int macb_remove(struct platform_device *pdev)
4845 {
4846 	struct net_device *dev;
4847 	struct macb *bp;
4848 
4849 	dev = platform_get_drvdata(pdev);
4850 
4851 	if (dev) {
4852 		bp = netdev_priv(dev);
4853 		mdiobus_unregister(bp->mii_bus);
4854 		mdiobus_free(bp->mii_bus);
4855 
4856 		unregister_netdev(dev);
4857 		tasklet_kill(&bp->hresp_err_tasklet);
4858 		pm_runtime_disable(&pdev->dev);
4859 		pm_runtime_dont_use_autosuspend(&pdev->dev);
4860 		if (!pm_runtime_suspended(&pdev->dev)) {
4861 			macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk,
4862 					  bp->rx_clk, bp->tsu_clk);
4863 			pm_runtime_set_suspended(&pdev->dev);
4864 		}
4865 		phylink_destroy(bp->phylink);
4866 		free_netdev(dev);
4867 	}
4868 
4869 	return 0;
4870 }
4871 
4872 static int __maybe_unused macb_suspend(struct device *dev)
4873 {
4874 	struct net_device *netdev = dev_get_drvdata(dev);
4875 	struct macb *bp = netdev_priv(netdev);
4876 	struct macb_queue *queue;
4877 	unsigned long flags;
4878 	unsigned int q;
4879 	int err;
4880 
4881 	if (!netif_running(netdev))
4882 		return 0;
4883 
4884 	if (bp->wol & MACB_WOL_ENABLED) {
4885 		spin_lock_irqsave(&bp->lock, flags);
4886 		/* Flush all status bits */
4887 		macb_writel(bp, TSR, -1);
4888 		macb_writel(bp, RSR, -1);
4889 		for (q = 0, queue = bp->queues; q < bp->num_queues;
4890 		     ++q, ++queue) {
4891 			/* Disable all interrupts */
4892 			queue_writel(queue, IDR, -1);
4893 			queue_readl(queue, ISR);
4894 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
4895 				queue_writel(queue, ISR, -1);
4896 		}
4897 		/* Change interrupt handler and
4898 		 * Enable WoL IRQ on queue 0
4899 		 */
4900 		devm_free_irq(dev, bp->queues[0].irq, bp->queues);
4901 		if (macb_is_gem(bp)) {
4902 			err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt,
4903 					       IRQF_SHARED, netdev->name, bp->queues);
4904 			if (err) {
4905 				dev_err(dev,
4906 					"Unable to request IRQ %d (error %d)\n",
4907 					bp->queues[0].irq, err);
4908 				spin_unlock_irqrestore(&bp->lock, flags);
4909 				return err;
4910 			}
4911 			queue_writel(bp->queues, IER, GEM_BIT(WOL));
4912 			gem_writel(bp, WOL, MACB_BIT(MAG));
4913 		} else {
4914 			err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt,
4915 					       IRQF_SHARED, netdev->name, bp->queues);
4916 			if (err) {
4917 				dev_err(dev,
4918 					"Unable to request IRQ %d (error %d)\n",
4919 					bp->queues[0].irq, err);
4920 				spin_unlock_irqrestore(&bp->lock, flags);
4921 				return err;
4922 			}
4923 			queue_writel(bp->queues, IER, MACB_BIT(WOL));
4924 			macb_writel(bp, WOL, MACB_BIT(MAG));
4925 		}
4926 		spin_unlock_irqrestore(&bp->lock, flags);
4927 
4928 		enable_irq_wake(bp->queues[0].irq);
4929 	}
4930 
4931 	netif_device_detach(netdev);
4932 	for (q = 0, queue = bp->queues; q < bp->num_queues;
4933 	     ++q, ++queue)
4934 		napi_disable(&queue->napi);
4935 
4936 	if (!(bp->wol & MACB_WOL_ENABLED)) {
4937 		rtnl_lock();
4938 		phylink_stop(bp->phylink);
4939 		rtnl_unlock();
4940 		spin_lock_irqsave(&bp->lock, flags);
4941 		macb_reset_hw(bp);
4942 		spin_unlock_irqrestore(&bp->lock, flags);
4943 	}
4944 
4945 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
4946 		bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO);
4947 
4948 	if (netdev->hw_features & NETIF_F_NTUPLE)
4949 		bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT);
4950 
4951 	if (bp->ptp_info)
4952 		bp->ptp_info->ptp_remove(netdev);
4953 	if (!device_may_wakeup(dev))
4954 		pm_runtime_force_suspend(dev);
4955 
4956 	return 0;
4957 }
4958 
4959 static int __maybe_unused macb_resume(struct device *dev)
4960 {
4961 	struct net_device *netdev = dev_get_drvdata(dev);
4962 	struct macb *bp = netdev_priv(netdev);
4963 	struct macb_queue *queue;
4964 	unsigned long flags;
4965 	unsigned int q;
4966 	int err;
4967 
4968 	if (!netif_running(netdev))
4969 		return 0;
4970 
4971 	if (!device_may_wakeup(dev))
4972 		pm_runtime_force_resume(dev);
4973 
4974 	if (bp->wol & MACB_WOL_ENABLED) {
4975 		spin_lock_irqsave(&bp->lock, flags);
4976 		/* Disable WoL */
4977 		if (macb_is_gem(bp)) {
4978 			queue_writel(bp->queues, IDR, GEM_BIT(WOL));
4979 			gem_writel(bp, WOL, 0);
4980 		} else {
4981 			queue_writel(bp->queues, IDR, MACB_BIT(WOL));
4982 			macb_writel(bp, WOL, 0);
4983 		}
4984 		/* Clear ISR on queue 0 */
4985 		queue_readl(bp->queues, ISR);
4986 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
4987 			queue_writel(bp->queues, ISR, -1);
4988 		/* Replace interrupt handler on queue 0 */
4989 		devm_free_irq(dev, bp->queues[0].irq, bp->queues);
4990 		err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt,
4991 				       IRQF_SHARED, netdev->name, bp->queues);
4992 		if (err) {
4993 			dev_err(dev,
4994 				"Unable to request IRQ %d (error %d)\n",
4995 				bp->queues[0].irq, err);
4996 			spin_unlock_irqrestore(&bp->lock, flags);
4997 			return err;
4998 		}
4999 		spin_unlock_irqrestore(&bp->lock, flags);
5000 
5001 		disable_irq_wake(bp->queues[0].irq);
5002 
5003 		/* Now make sure we disable phy before moving
5004 		 * to common restore path
5005 		 */
5006 		rtnl_lock();
5007 		phylink_stop(bp->phylink);
5008 		rtnl_unlock();
5009 	}
5010 
5011 	for (q = 0, queue = bp->queues; q < bp->num_queues;
5012 	     ++q, ++queue)
5013 		napi_enable(&queue->napi);
5014 
5015 	if (netdev->hw_features & NETIF_F_NTUPLE)
5016 		gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2);
5017 
5018 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
5019 		macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio);
5020 
5021 	macb_writel(bp, NCR, MACB_BIT(MPE));
5022 	macb_init_hw(bp);
5023 	macb_set_rx_mode(netdev);
5024 	macb_restore_features(bp);
5025 	rtnl_lock();
5026 	phylink_start(bp->phylink);
5027 	rtnl_unlock();
5028 
5029 	netif_device_attach(netdev);
5030 	if (bp->ptp_info)
5031 		bp->ptp_info->ptp_init(netdev);
5032 
5033 	return 0;
5034 }
5035 
5036 static int __maybe_unused macb_runtime_suspend(struct device *dev)
5037 {
5038 	struct net_device *netdev = dev_get_drvdata(dev);
5039 	struct macb *bp = netdev_priv(netdev);
5040 
5041 	if (!(device_may_wakeup(dev)))
5042 		macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk);
5043 	else
5044 		macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk);
5045 
5046 	return 0;
5047 }
5048 
5049 static int __maybe_unused macb_runtime_resume(struct device *dev)
5050 {
5051 	struct net_device *netdev = dev_get_drvdata(dev);
5052 	struct macb *bp = netdev_priv(netdev);
5053 
5054 	if (!(device_may_wakeup(dev))) {
5055 		clk_prepare_enable(bp->pclk);
5056 		clk_prepare_enable(bp->hclk);
5057 		clk_prepare_enable(bp->tx_clk);
5058 		clk_prepare_enable(bp->rx_clk);
5059 	}
5060 	clk_prepare_enable(bp->tsu_clk);
5061 
5062 	return 0;
5063 }
5064 
5065 static const struct dev_pm_ops macb_pm_ops = {
5066 	SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume)
5067 	SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL)
5068 };
5069 
5070 static struct platform_driver macb_driver = {
5071 	.probe		= macb_probe,
5072 	.remove		= macb_remove,
5073 	.driver		= {
5074 		.name		= "macb",
5075 		.of_match_table	= of_match_ptr(macb_dt_ids),
5076 		.pm	= &macb_pm_ops,
5077 	},
5078 };
5079 
5080 module_platform_driver(macb_driver);
5081 
5082 MODULE_LICENSE("GPL");
5083 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver");
5084 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
5085 MODULE_ALIAS("platform:macb");
5086