1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2021 Rafał Miłecki <rafal@milecki.pl>
4  */
5 
6 #include <linux/delay.h>
7 #include <linux/etherdevice.h>
8 #include <linux/if_vlan.h>
9 #include <linux/interrupt.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/of_net.h>
13 #include <linux/platform_device.h>
14 #include <linux/slab.h>
15 #include <linux/string.h>
16 
17 #include "bcm4908_enet.h"
18 #include "unimac.h"
19 
20 #define ENET_DMA_CH_RX_CFG			ENET_DMA_CH0_CFG
21 #define ENET_DMA_CH_TX_CFG			ENET_DMA_CH1_CFG
22 #define ENET_DMA_CH_RX_STATE_RAM		ENET_DMA_CH0_STATE_RAM
23 #define ENET_DMA_CH_TX_STATE_RAM		ENET_DMA_CH1_STATE_RAM
24 
25 #define ENET_TX_BDS_NUM				200
26 #define ENET_RX_BDS_NUM				200
27 #define ENET_RX_BDS_NUM_MAX			8192
28 
29 #define ENET_DMA_INT_DEFAULTS			(ENET_DMA_CH_CFG_INT_DONE | \
30 						 ENET_DMA_CH_CFG_INT_NO_DESC | \
31 						 ENET_DMA_CH_CFG_INT_BUFF_DONE)
32 #define ENET_DMA_MAX_BURST_LEN			8 /* in 64 bit words */
33 
34 #define ENET_MTU_MAX				ETH_DATA_LEN /* Is it possible to support 2044? */
35 #define BRCM_MAX_TAG_LEN			6
36 #define ENET_MAX_ETH_OVERHEAD			(ETH_HLEN + BRCM_MAX_TAG_LEN + VLAN_HLEN + \
37 						 ETH_FCS_LEN + 4) /* 32 */
38 
39 struct bcm4908_enet_dma_ring_bd {
40 	__le32 ctl;
41 	__le32 addr;
42 } __packed;
43 
44 struct bcm4908_enet_dma_ring_slot {
45 	struct sk_buff *skb;
46 	unsigned int len;
47 	dma_addr_t dma_addr;
48 };
49 
50 struct bcm4908_enet_dma_ring {
51 	int is_tx;
52 	int read_idx;
53 	int write_idx;
54 	int length;
55 	u16 cfg_block;
56 	u16 st_ram_block;
57 	struct napi_struct napi;
58 
59 	union {
60 		void *cpu_addr;
61 		struct bcm4908_enet_dma_ring_bd *buf_desc;
62 	};
63 	dma_addr_t dma_addr;
64 
65 	struct bcm4908_enet_dma_ring_slot *slots;
66 };
67 
68 struct bcm4908_enet {
69 	struct device *dev;
70 	struct net_device *netdev;
71 	void __iomem *base;
72 	int irq_tx;
73 
74 	struct bcm4908_enet_dma_ring tx_ring;
75 	struct bcm4908_enet_dma_ring rx_ring;
76 };
77 
78 /***
79  * R/W ops
80  */
81 
82 static u32 enet_read(struct bcm4908_enet *enet, u16 offset)
83 {
84 	return readl(enet->base + offset);
85 }
86 
87 static void enet_write(struct bcm4908_enet *enet, u16 offset, u32 value)
88 {
89 	writel(value, enet->base + offset);
90 }
91 
92 static void enet_maskset(struct bcm4908_enet *enet, u16 offset, u32 mask, u32 set)
93 {
94 	u32 val;
95 
96 	WARN_ON(set & ~mask);
97 
98 	val = enet_read(enet, offset);
99 	val = (val & ~mask) | (set & mask);
100 	enet_write(enet, offset, val);
101 }
102 
103 static void enet_set(struct bcm4908_enet *enet, u16 offset, u32 set)
104 {
105 	enet_maskset(enet, offset, set, set);
106 }
107 
108 static u32 enet_umac_read(struct bcm4908_enet *enet, u16 offset)
109 {
110 	return enet_read(enet, ENET_UNIMAC + offset);
111 }
112 
113 static void enet_umac_write(struct bcm4908_enet *enet, u16 offset, u32 value)
114 {
115 	enet_write(enet, ENET_UNIMAC + offset, value);
116 }
117 
118 static void enet_umac_set(struct bcm4908_enet *enet, u16 offset, u32 set)
119 {
120 	enet_set(enet, ENET_UNIMAC + offset, set);
121 }
122 
123 /***
124  * Helpers
125  */
126 
127 static void bcm4908_enet_set_mtu(struct bcm4908_enet *enet, int mtu)
128 {
129 	enet_umac_write(enet, UMAC_MAX_FRAME_LEN, mtu + ENET_MAX_ETH_OVERHEAD);
130 }
131 
132 /***
133  * DMA ring ops
134  */
135 
136 static void bcm4908_enet_dma_ring_intrs_on(struct bcm4908_enet *enet,
137 					   struct bcm4908_enet_dma_ring *ring)
138 {
139 	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_INT_MASK, ENET_DMA_INT_DEFAULTS);
140 }
141 
142 static void bcm4908_enet_dma_ring_intrs_off(struct bcm4908_enet *enet,
143 					    struct bcm4908_enet_dma_ring *ring)
144 {
145 	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_INT_MASK, 0);
146 }
147 
148 static void bcm4908_enet_dma_ring_intrs_ack(struct bcm4908_enet *enet,
149 					    struct bcm4908_enet_dma_ring *ring)
150 {
151 	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_INT_STAT, ENET_DMA_INT_DEFAULTS);
152 }
153 
154 /***
155  * DMA
156  */
157 
158 static int bcm4908_dma_alloc_buf_descs(struct bcm4908_enet *enet,
159 				       struct bcm4908_enet_dma_ring *ring)
160 {
161 	int size = ring->length * sizeof(struct bcm4908_enet_dma_ring_bd);
162 	struct device *dev = enet->dev;
163 
164 	ring->cpu_addr = dma_alloc_coherent(dev, size, &ring->dma_addr, GFP_KERNEL);
165 	if (!ring->cpu_addr)
166 		return -ENOMEM;
167 
168 	if (((uintptr_t)ring->cpu_addr) & (0x40 - 1)) {
169 		dev_err(dev, "Invalid DMA ring alignment\n");
170 		goto err_free_buf_descs;
171 	}
172 
173 	ring->slots = kzalloc(ring->length * sizeof(*ring->slots), GFP_KERNEL);
174 	if (!ring->slots)
175 		goto err_free_buf_descs;
176 
177 	return 0;
178 
179 err_free_buf_descs:
180 	dma_free_coherent(dev, size, ring->cpu_addr, ring->dma_addr);
181 	ring->cpu_addr = NULL;
182 	return -ENOMEM;
183 }
184 
185 static void bcm4908_enet_dma_free(struct bcm4908_enet *enet)
186 {
187 	struct bcm4908_enet_dma_ring *tx_ring = &enet->tx_ring;
188 	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
189 	struct device *dev = enet->dev;
190 	int size;
191 
192 	size = rx_ring->length * sizeof(struct bcm4908_enet_dma_ring_bd);
193 	if (rx_ring->cpu_addr)
194 		dma_free_coherent(dev, size, rx_ring->cpu_addr, rx_ring->dma_addr);
195 	kfree(rx_ring->slots);
196 
197 	size = tx_ring->length * sizeof(struct bcm4908_enet_dma_ring_bd);
198 	if (tx_ring->cpu_addr)
199 		dma_free_coherent(dev, size, tx_ring->cpu_addr, tx_ring->dma_addr);
200 	kfree(tx_ring->slots);
201 }
202 
203 static int bcm4908_enet_dma_alloc(struct bcm4908_enet *enet)
204 {
205 	struct bcm4908_enet_dma_ring *tx_ring = &enet->tx_ring;
206 	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
207 	struct device *dev = enet->dev;
208 	int err;
209 
210 	tx_ring->length = ENET_TX_BDS_NUM;
211 	tx_ring->is_tx = 1;
212 	tx_ring->cfg_block = ENET_DMA_CH_TX_CFG;
213 	tx_ring->st_ram_block = ENET_DMA_CH_TX_STATE_RAM;
214 	err = bcm4908_dma_alloc_buf_descs(enet, tx_ring);
215 	if (err) {
216 		dev_err(dev, "Failed to alloc TX buf descriptors: %d\n", err);
217 		return err;
218 	}
219 
220 	rx_ring->length = ENET_RX_BDS_NUM;
221 	rx_ring->is_tx = 0;
222 	rx_ring->cfg_block = ENET_DMA_CH_RX_CFG;
223 	rx_ring->st_ram_block = ENET_DMA_CH_RX_STATE_RAM;
224 	err = bcm4908_dma_alloc_buf_descs(enet, rx_ring);
225 	if (err) {
226 		dev_err(dev, "Failed to alloc RX buf descriptors: %d\n", err);
227 		bcm4908_enet_dma_free(enet);
228 		return err;
229 	}
230 
231 	return 0;
232 }
233 
234 static void bcm4908_enet_dma_reset(struct bcm4908_enet *enet)
235 {
236 	struct bcm4908_enet_dma_ring *rings[] = { &enet->rx_ring, &enet->tx_ring };
237 	int i;
238 
239 	/* Disable the DMA controller and channel */
240 	for (i = 0; i < ARRAY_SIZE(rings); i++)
241 		enet_write(enet, rings[i]->cfg_block + ENET_DMA_CH_CFG, 0);
242 	enet_maskset(enet, ENET_DMA_CONTROLLER_CFG, ENET_DMA_CTRL_CFG_MASTER_EN, 0);
243 
244 	/* Reset channels state */
245 	for (i = 0; i < ARRAY_SIZE(rings); i++) {
246 		struct bcm4908_enet_dma_ring *ring = rings[i];
247 
248 		enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_BASE_DESC_PTR, 0);
249 		enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_STATE_DATA, 0);
250 		enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_DESC_LEN_STATUS, 0);
251 		enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_DESC_BASE_BUFPTR, 0);
252 	}
253 }
254 
255 static int bcm4908_enet_dma_alloc_rx_buf(struct bcm4908_enet *enet, unsigned int idx)
256 {
257 	struct bcm4908_enet_dma_ring_bd *buf_desc = &enet->rx_ring.buf_desc[idx];
258 	struct bcm4908_enet_dma_ring_slot *slot = &enet->rx_ring.slots[idx];
259 	struct device *dev = enet->dev;
260 	u32 tmp;
261 	int err;
262 
263 	slot->len = ENET_MTU_MAX + ENET_MAX_ETH_OVERHEAD;
264 
265 	slot->skb = netdev_alloc_skb(enet->netdev, slot->len);
266 	if (!slot->skb)
267 		return -ENOMEM;
268 
269 	slot->dma_addr = dma_map_single(dev, slot->skb->data, slot->len, DMA_FROM_DEVICE);
270 	err = dma_mapping_error(dev, slot->dma_addr);
271 	if (err) {
272 		dev_err(dev, "Failed to map DMA buffer: %d\n", err);
273 		kfree_skb(slot->skb);
274 		slot->skb = NULL;
275 		return err;
276 	}
277 
278 	tmp = slot->len << DMA_CTL_LEN_DESC_BUFLENGTH_SHIFT;
279 	tmp |= DMA_CTL_STATUS_OWN;
280 	if (idx == enet->rx_ring.length - 1)
281 		tmp |= DMA_CTL_STATUS_WRAP;
282 	buf_desc->ctl = cpu_to_le32(tmp);
283 	buf_desc->addr = cpu_to_le32(slot->dma_addr);
284 
285 	return 0;
286 }
287 
288 static void bcm4908_enet_dma_ring_init(struct bcm4908_enet *enet,
289 				       struct bcm4908_enet_dma_ring *ring)
290 {
291 	int reset_channel = 0; /* We support only 1 main channel (with TX and RX) */
292 	int reset_subch = ring->is_tx ? 1 : 0;
293 
294 	/* Reset the DMA channel */
295 	enet_write(enet, ENET_DMA_CTRL_CHANNEL_RESET, BIT(reset_channel * 2 + reset_subch));
296 	enet_write(enet, ENET_DMA_CTRL_CHANNEL_RESET, 0);
297 
298 	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG, 0);
299 	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_MAX_BURST, ENET_DMA_MAX_BURST_LEN);
300 	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_INT_MASK, 0);
301 
302 	enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_BASE_DESC_PTR,
303 		   (uint32_t)ring->dma_addr);
304 
305 	ring->read_idx = 0;
306 	ring->write_idx = 0;
307 }
308 
309 static void bcm4908_enet_dma_uninit(struct bcm4908_enet *enet)
310 {
311 	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
312 	struct bcm4908_enet_dma_ring_slot *slot;
313 	struct device *dev = enet->dev;
314 	int i;
315 
316 	for (i = rx_ring->length - 1; i >= 0; i--) {
317 		slot = &rx_ring->slots[i];
318 		if (!slot->skb)
319 			continue;
320 		dma_unmap_single(dev, slot->dma_addr, slot->len, DMA_FROM_DEVICE);
321 		kfree_skb(slot->skb);
322 		slot->skb = NULL;
323 	}
324 }
325 
326 static int bcm4908_enet_dma_init(struct bcm4908_enet *enet)
327 {
328 	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
329 	struct device *dev = enet->dev;
330 	int err;
331 	int i;
332 
333 	for (i = 0; i < rx_ring->length; i++) {
334 		err = bcm4908_enet_dma_alloc_rx_buf(enet, i);
335 		if (err) {
336 			dev_err(dev, "Failed to alloc RX buffer: %d\n", err);
337 			bcm4908_enet_dma_uninit(enet);
338 			return err;
339 		}
340 	}
341 
342 	bcm4908_enet_dma_ring_init(enet, &enet->tx_ring);
343 	bcm4908_enet_dma_ring_init(enet, &enet->rx_ring);
344 
345 	return 0;
346 }
347 
348 static void bcm4908_enet_dma_tx_ring_enable(struct bcm4908_enet *enet,
349 					    struct bcm4908_enet_dma_ring *ring)
350 {
351 	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG, ENET_DMA_CH_CFG_ENABLE);
352 }
353 
354 static void bcm4908_enet_dma_tx_ring_disable(struct bcm4908_enet *enet,
355 					     struct bcm4908_enet_dma_ring *ring)
356 {
357 	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG, 0);
358 }
359 
360 static void bcm4908_enet_dma_rx_ring_enable(struct bcm4908_enet *enet,
361 					    struct bcm4908_enet_dma_ring *ring)
362 {
363 	enet_set(enet, ring->cfg_block + ENET_DMA_CH_CFG, ENET_DMA_CH_CFG_ENABLE);
364 }
365 
366 static void bcm4908_enet_dma_rx_ring_disable(struct bcm4908_enet *enet,
367 					     struct bcm4908_enet_dma_ring *ring)
368 {
369 	unsigned long deadline;
370 	u32 tmp;
371 
372 	enet_maskset(enet, ring->cfg_block + ENET_DMA_CH_CFG, ENET_DMA_CH_CFG_ENABLE, 0);
373 
374 	deadline = jiffies + usecs_to_jiffies(2000);
375 	do {
376 		tmp = enet_read(enet, ring->cfg_block + ENET_DMA_CH_CFG);
377 		if (!(tmp & ENET_DMA_CH_CFG_ENABLE))
378 			return;
379 		enet_maskset(enet, ring->cfg_block + ENET_DMA_CH_CFG, ENET_DMA_CH_CFG_ENABLE, 0);
380 		usleep_range(10, 30);
381 	} while (!time_after_eq(jiffies, deadline));
382 
383 	dev_warn(enet->dev, "Timeout waiting for DMA TX stop\n");
384 }
385 
386 /***
387  * Ethernet driver
388  */
389 
390 static void bcm4908_enet_gmac_init(struct bcm4908_enet *enet)
391 {
392 	u32 cmd;
393 
394 	bcm4908_enet_set_mtu(enet, enet->netdev->mtu);
395 
396 	cmd = enet_umac_read(enet, UMAC_CMD);
397 	enet_umac_write(enet, UMAC_CMD, cmd | CMD_SW_RESET);
398 	enet_umac_write(enet, UMAC_CMD, cmd & ~CMD_SW_RESET);
399 
400 	enet_set(enet, ENET_FLUSH, ENET_FLUSH_RXFIFO_FLUSH | ENET_FLUSH_TXFIFO_FLUSH);
401 	enet_maskset(enet, ENET_FLUSH, ENET_FLUSH_RXFIFO_FLUSH | ENET_FLUSH_TXFIFO_FLUSH, 0);
402 
403 	enet_set(enet, ENET_MIB_CTRL, ENET_MIB_CTRL_CLR_MIB);
404 	enet_maskset(enet, ENET_MIB_CTRL, ENET_MIB_CTRL_CLR_MIB, 0);
405 
406 	cmd = enet_umac_read(enet, UMAC_CMD);
407 	cmd &= ~(CMD_SPEED_MASK << CMD_SPEED_SHIFT);
408 	cmd &= ~CMD_TX_EN;
409 	cmd &= ~CMD_RX_EN;
410 	cmd |= CMD_SPEED_1000 << CMD_SPEED_SHIFT;
411 	enet_umac_write(enet, UMAC_CMD, cmd);
412 
413 	enet_maskset(enet, ENET_GMAC_STATUS,
414 		     ENET_GMAC_STATUS_ETH_SPEED_MASK |
415 		     ENET_GMAC_STATUS_HD |
416 		     ENET_GMAC_STATUS_AUTO_CFG_EN |
417 		     ENET_GMAC_STATUS_LINK_UP,
418 		     ENET_GMAC_STATUS_ETH_SPEED_1000 |
419 		     ENET_GMAC_STATUS_AUTO_CFG_EN |
420 		     ENET_GMAC_STATUS_LINK_UP);
421 }
422 
423 static irqreturn_t bcm4908_enet_irq_handler(int irq, void *dev_id)
424 {
425 	struct bcm4908_enet *enet = dev_id;
426 	struct bcm4908_enet_dma_ring *ring;
427 
428 	ring = (irq == enet->irq_tx) ? &enet->tx_ring : &enet->rx_ring;
429 
430 	bcm4908_enet_dma_ring_intrs_off(enet, ring);
431 	bcm4908_enet_dma_ring_intrs_ack(enet, ring);
432 
433 	napi_schedule(&ring->napi);
434 
435 	return IRQ_HANDLED;
436 }
437 
438 static int bcm4908_enet_open(struct net_device *netdev)
439 {
440 	struct bcm4908_enet *enet = netdev_priv(netdev);
441 	struct bcm4908_enet_dma_ring *tx_ring = &enet->tx_ring;
442 	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
443 	struct device *dev = enet->dev;
444 	int err;
445 
446 	err = request_irq(netdev->irq, bcm4908_enet_irq_handler, 0, "enet", enet);
447 	if (err) {
448 		dev_err(dev, "Failed to request IRQ %d: %d\n", netdev->irq, err);
449 		return err;
450 	}
451 
452 	if (enet->irq_tx > 0) {
453 		err = request_irq(enet->irq_tx, bcm4908_enet_irq_handler, 0,
454 				  "tx", enet);
455 		if (err) {
456 			dev_err(dev, "Failed to request IRQ %d: %d\n",
457 				enet->irq_tx, err);
458 			free_irq(netdev->irq, enet);
459 			return err;
460 		}
461 	}
462 
463 	bcm4908_enet_gmac_init(enet);
464 	bcm4908_enet_dma_reset(enet);
465 	bcm4908_enet_dma_init(enet);
466 
467 	enet_umac_set(enet, UMAC_CMD, CMD_TX_EN | CMD_RX_EN);
468 
469 	enet_set(enet, ENET_DMA_CONTROLLER_CFG, ENET_DMA_CTRL_CFG_MASTER_EN);
470 	enet_maskset(enet, ENET_DMA_CONTROLLER_CFG, ENET_DMA_CTRL_CFG_FLOWC_CH1_EN, 0);
471 
472 	if (enet->irq_tx > 0) {
473 		napi_enable(&tx_ring->napi);
474 		bcm4908_enet_dma_ring_intrs_ack(enet, tx_ring);
475 		bcm4908_enet_dma_ring_intrs_on(enet, tx_ring);
476 	}
477 
478 	bcm4908_enet_dma_rx_ring_enable(enet, rx_ring);
479 	napi_enable(&rx_ring->napi);
480 	netif_carrier_on(netdev);
481 	netif_start_queue(netdev);
482 	bcm4908_enet_dma_ring_intrs_ack(enet, rx_ring);
483 	bcm4908_enet_dma_ring_intrs_on(enet, rx_ring);
484 
485 	return 0;
486 }
487 
488 static int bcm4908_enet_stop(struct net_device *netdev)
489 {
490 	struct bcm4908_enet *enet = netdev_priv(netdev);
491 	struct bcm4908_enet_dma_ring *tx_ring = &enet->tx_ring;
492 	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
493 
494 	netif_stop_queue(netdev);
495 	netif_carrier_off(netdev);
496 	napi_disable(&rx_ring->napi);
497 	napi_disable(&tx_ring->napi);
498 
499 	bcm4908_enet_dma_rx_ring_disable(enet, &enet->rx_ring);
500 	bcm4908_enet_dma_tx_ring_disable(enet, &enet->tx_ring);
501 
502 	bcm4908_enet_dma_uninit(enet);
503 
504 	free_irq(enet->irq_tx, enet);
505 	free_irq(enet->netdev->irq, enet);
506 
507 	return 0;
508 }
509 
510 static int bcm4908_enet_start_xmit(struct sk_buff *skb, struct net_device *netdev)
511 {
512 	struct bcm4908_enet *enet = netdev_priv(netdev);
513 	struct bcm4908_enet_dma_ring *ring = &enet->tx_ring;
514 	struct bcm4908_enet_dma_ring_slot *slot;
515 	struct device *dev = enet->dev;
516 	struct bcm4908_enet_dma_ring_bd *buf_desc;
517 	int free_buf_descs;
518 	u32 tmp;
519 
520 	/* Free transmitted skbs */
521 	if (enet->irq_tx < 0 &&
522 	    !(le32_to_cpu(ring->buf_desc[ring->read_idx].ctl) & DMA_CTL_STATUS_OWN))
523 		napi_schedule(&enet->tx_ring.napi);
524 
525 	/* Don't use the last empty buf descriptor */
526 	if (ring->read_idx <= ring->write_idx)
527 		free_buf_descs = ring->read_idx - ring->write_idx + ring->length;
528 	else
529 		free_buf_descs = ring->read_idx - ring->write_idx;
530 	if (free_buf_descs < 2) {
531 		netif_stop_queue(netdev);
532 		return NETDEV_TX_BUSY;
533 	}
534 
535 	/* Hardware removes OWN bit after sending data */
536 	buf_desc = &ring->buf_desc[ring->write_idx];
537 	if (unlikely(le32_to_cpu(buf_desc->ctl) & DMA_CTL_STATUS_OWN)) {
538 		netif_stop_queue(netdev);
539 		return NETDEV_TX_BUSY;
540 	}
541 
542 	slot = &ring->slots[ring->write_idx];
543 	slot->skb = skb;
544 	slot->len = skb->len;
545 	slot->dma_addr = dma_map_single(dev, skb->data, skb->len, DMA_TO_DEVICE);
546 	if (unlikely(dma_mapping_error(dev, slot->dma_addr)))
547 		return NETDEV_TX_BUSY;
548 
549 	tmp = skb->len << DMA_CTL_LEN_DESC_BUFLENGTH_SHIFT;
550 	tmp |= DMA_CTL_STATUS_OWN;
551 	tmp |= DMA_CTL_STATUS_SOP;
552 	tmp |= DMA_CTL_STATUS_EOP;
553 	tmp |= DMA_CTL_STATUS_APPEND_CRC;
554 	if (ring->write_idx + 1 == ring->length - 1)
555 		tmp |= DMA_CTL_STATUS_WRAP;
556 
557 	buf_desc->addr = cpu_to_le32((uint32_t)slot->dma_addr);
558 	buf_desc->ctl = cpu_to_le32(tmp);
559 
560 	bcm4908_enet_dma_tx_ring_enable(enet, &enet->tx_ring);
561 
562 	if (++ring->write_idx == ring->length - 1)
563 		ring->write_idx = 0;
564 	enet->netdev->stats.tx_bytes += skb->len;
565 	enet->netdev->stats.tx_packets++;
566 
567 	return NETDEV_TX_OK;
568 }
569 
570 static int bcm4908_enet_poll_rx(struct napi_struct *napi, int weight)
571 {
572 	struct bcm4908_enet_dma_ring *rx_ring = container_of(napi, struct bcm4908_enet_dma_ring, napi);
573 	struct bcm4908_enet *enet = container_of(rx_ring, struct bcm4908_enet, rx_ring);
574 	struct device *dev = enet->dev;
575 	int handled = 0;
576 
577 	while (handled < weight) {
578 		struct bcm4908_enet_dma_ring_bd *buf_desc;
579 		struct bcm4908_enet_dma_ring_slot slot;
580 		u32 ctl;
581 		int len;
582 		int err;
583 
584 		buf_desc = &enet->rx_ring.buf_desc[enet->rx_ring.read_idx];
585 		ctl = le32_to_cpu(buf_desc->ctl);
586 		if (ctl & DMA_CTL_STATUS_OWN)
587 			break;
588 
589 		slot = enet->rx_ring.slots[enet->rx_ring.read_idx];
590 
591 		/* Provide new buffer before unpinning the old one */
592 		err = bcm4908_enet_dma_alloc_rx_buf(enet, enet->rx_ring.read_idx);
593 		if (err)
594 			break;
595 
596 		if (++enet->rx_ring.read_idx == enet->rx_ring.length)
597 			enet->rx_ring.read_idx = 0;
598 
599 		len = (ctl & DMA_CTL_LEN_DESC_BUFLENGTH) >> DMA_CTL_LEN_DESC_BUFLENGTH_SHIFT;
600 
601 		if (len < ETH_ZLEN ||
602 		    (ctl & (DMA_CTL_STATUS_SOP | DMA_CTL_STATUS_EOP)) != (DMA_CTL_STATUS_SOP | DMA_CTL_STATUS_EOP)) {
603 			kfree_skb(slot.skb);
604 			enet->netdev->stats.rx_dropped++;
605 			break;
606 		}
607 
608 		dma_unmap_single(dev, slot.dma_addr, slot.len, DMA_FROM_DEVICE);
609 
610 		skb_put(slot.skb, len - ETH_FCS_LEN);
611 		slot.skb->protocol = eth_type_trans(slot.skb, enet->netdev);
612 		netif_receive_skb(slot.skb);
613 
614 		enet->netdev->stats.rx_packets++;
615 		enet->netdev->stats.rx_bytes += len;
616 
617 		handled++;
618 	}
619 
620 	if (handled < weight) {
621 		napi_complete_done(napi, handled);
622 		bcm4908_enet_dma_ring_intrs_on(enet, rx_ring);
623 	}
624 
625 	/* Hardware could disable ring if it run out of descriptors */
626 	bcm4908_enet_dma_rx_ring_enable(enet, &enet->rx_ring);
627 
628 	return handled;
629 }
630 
631 static int bcm4908_enet_poll_tx(struct napi_struct *napi, int weight)
632 {
633 	struct bcm4908_enet_dma_ring *tx_ring = container_of(napi, struct bcm4908_enet_dma_ring, napi);
634 	struct bcm4908_enet *enet = container_of(tx_ring, struct bcm4908_enet, tx_ring);
635 	struct bcm4908_enet_dma_ring_bd *buf_desc;
636 	struct bcm4908_enet_dma_ring_slot *slot;
637 	struct device *dev = enet->dev;
638 	unsigned int bytes = 0;
639 	int handled = 0;
640 
641 	while (handled < weight && tx_ring->read_idx != tx_ring->write_idx) {
642 		buf_desc = &tx_ring->buf_desc[tx_ring->read_idx];
643 		if (le32_to_cpu(buf_desc->ctl) & DMA_CTL_STATUS_OWN)
644 			break;
645 		slot = &tx_ring->slots[tx_ring->read_idx];
646 
647 		dma_unmap_single(dev, slot->dma_addr, slot->len, DMA_TO_DEVICE);
648 		dev_kfree_skb(slot->skb);
649 		bytes += slot->len;
650 		if (++tx_ring->read_idx == tx_ring->length)
651 			tx_ring->read_idx = 0;
652 
653 		handled++;
654 	}
655 
656 	if (handled < weight) {
657 		napi_complete_done(napi, handled);
658 		bcm4908_enet_dma_ring_intrs_on(enet, tx_ring);
659 	}
660 
661 	if (netif_queue_stopped(enet->netdev))
662 		netif_wake_queue(enet->netdev);
663 
664 	return handled;
665 }
666 
667 static int bcm4908_enet_change_mtu(struct net_device *netdev, int new_mtu)
668 {
669 	struct bcm4908_enet *enet = netdev_priv(netdev);
670 
671 	bcm4908_enet_set_mtu(enet, new_mtu);
672 
673 	return 0;
674 }
675 
676 static const struct net_device_ops bcm4908_enet_netdev_ops = {
677 	.ndo_open = bcm4908_enet_open,
678 	.ndo_stop = bcm4908_enet_stop,
679 	.ndo_start_xmit = bcm4908_enet_start_xmit,
680 	.ndo_set_mac_address = eth_mac_addr,
681 	.ndo_change_mtu = bcm4908_enet_change_mtu,
682 };
683 
684 static int bcm4908_enet_probe(struct platform_device *pdev)
685 {
686 	struct device *dev = &pdev->dev;
687 	struct net_device *netdev;
688 	struct bcm4908_enet *enet;
689 	int err;
690 
691 	netdev = devm_alloc_etherdev(dev, sizeof(*enet));
692 	if (!netdev)
693 		return -ENOMEM;
694 
695 	enet = netdev_priv(netdev);
696 	enet->dev = dev;
697 	enet->netdev = netdev;
698 
699 	enet->base = devm_platform_ioremap_resource(pdev, 0);
700 	if (IS_ERR(enet->base)) {
701 		dev_err(dev, "Failed to map registers: %ld\n", PTR_ERR(enet->base));
702 		return PTR_ERR(enet->base);
703 	}
704 
705 	netdev->irq = platform_get_irq_byname(pdev, "rx");
706 	if (netdev->irq < 0)
707 		return netdev->irq;
708 
709 	enet->irq_tx = platform_get_irq_byname(pdev, "tx");
710 
711 	dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
712 
713 	err = bcm4908_enet_dma_alloc(enet);
714 	if (err)
715 		return err;
716 
717 	SET_NETDEV_DEV(netdev, &pdev->dev);
718 	err = of_get_mac_address(dev->of_node, netdev->dev_addr);
719 	if (err)
720 		eth_hw_addr_random(netdev);
721 	netdev->netdev_ops = &bcm4908_enet_netdev_ops;
722 	netdev->min_mtu = ETH_ZLEN;
723 	netdev->mtu = ETH_DATA_LEN;
724 	netdev->max_mtu = ENET_MTU_MAX;
725 	netif_tx_napi_add(netdev, &enet->tx_ring.napi, bcm4908_enet_poll_tx, NAPI_POLL_WEIGHT);
726 	netif_napi_add(netdev, &enet->rx_ring.napi, bcm4908_enet_poll_rx, NAPI_POLL_WEIGHT);
727 
728 	err = register_netdev(netdev);
729 	if (err) {
730 		bcm4908_enet_dma_free(enet);
731 		return err;
732 	}
733 
734 	platform_set_drvdata(pdev, enet);
735 
736 	return 0;
737 }
738 
739 static int bcm4908_enet_remove(struct platform_device *pdev)
740 {
741 	struct bcm4908_enet *enet = platform_get_drvdata(pdev);
742 
743 	unregister_netdev(enet->netdev);
744 	netif_napi_del(&enet->rx_ring.napi);
745 	netif_napi_del(&enet->tx_ring.napi);
746 	bcm4908_enet_dma_free(enet);
747 
748 	return 0;
749 }
750 
751 static const struct of_device_id bcm4908_enet_of_match[] = {
752 	{ .compatible = "brcm,bcm4908-enet"},
753 	{},
754 };
755 
756 static struct platform_driver bcm4908_enet_driver = {
757 	.driver = {
758 		.name = "bcm4908_enet",
759 		.of_match_table = bcm4908_enet_of_match,
760 	},
761 	.probe	= bcm4908_enet_probe,
762 	.remove = bcm4908_enet_remove,
763 };
764 module_platform_driver(bcm4908_enet_driver);
765 
766 MODULE_LICENSE("GPL v2");
767 MODULE_DEVICE_TABLE(of, bcm4908_enet_of_match);
768