1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */
3
4 /* TSN endpoint Ethernet MAC driver
5 *
6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time
7 * communication. It is designed for endpoints within TSN (Time Sensitive
8 * Networking) networks; e.g., for PLCs in the industrial automation case.
9 *
10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used
11 * by the driver.
12 *
13 * More information can be found here:
14 * - www.embedded-experts.at/tsn
15 * - www.engleder-embedded.com
16 */
17
18 #include "tsnep.h"
19 #include "tsnep_hw.h"
20
21 #include <linux/module.h>
22 #include <linux/of.h>
23 #include <linux/of_net.h>
24 #include <linux/of_mdio.h>
25 #include <linux/interrupt.h>
26 #include <linux/etherdevice.h>
27 #include <linux/phy.h>
28 #include <linux/iopoll.h>
29 #include <linux/bpf.h>
30 #include <linux/bpf_trace.h>
31 #include <net/page_pool/helpers.h>
32 #include <net/xdp_sock_drv.h>
33
34 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
35 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)
36 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \
37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
38 /* XSK buffer shall store at least Q-in-Q frame */
39 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \
40 ETH_FRAME_LEN + ETH_FCS_LEN + \
41 VLAN_HLEN * 2, 4))
42
43 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
44 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))
45 #else
46 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0))
47 #endif
48 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF))
49
50 #define TSNEP_COALESCE_USECS_DEFAULT 64
51 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \
52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1)
53
54 /* mapping type */
55 #define TSNEP_TX_TYPE_MAP BIT(0)
56 #define TSNEP_TX_TYPE_MAP_PAGE BIT(1)
57 #define TSNEP_TX_TYPE_INLINE BIT(2)
58 /* buffer type */
59 #define TSNEP_TX_TYPE_SKB BIT(8)
60 #define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP)
61 #define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE)
62 #define TSNEP_TX_TYPE_SKB_FRAG BIT(9)
63 #define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE)
64 #define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE)
65 #define TSNEP_TX_TYPE_XDP_TX BIT(10)
66 #define TSNEP_TX_TYPE_XDP_NDO BIT(11)
67 #define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE)
68 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)
69 #define TSNEP_TX_TYPE_XSK BIT(12)
70 #define TSNEP_TX_TYPE_TSTAMP BIT(13)
71 #define TSNEP_TX_TYPE_SKB_TSTAMP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_TSTAMP)
72
73 #define TSNEP_XDP_TX BIT(0)
74 #define TSNEP_XDP_REDIRECT BIT(1)
75
tsnep_enable_irq(struct tsnep_adapter * adapter,u32 mask)76 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask)
77 {
78 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
79 }
80
tsnep_disable_irq(struct tsnep_adapter * adapter,u32 mask)81 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask)
82 {
83 mask |= ECM_INT_DISABLE;
84 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
85 }
86
tsnep_irq(int irq,void * arg)87 static irqreturn_t tsnep_irq(int irq, void *arg)
88 {
89 struct tsnep_adapter *adapter = arg;
90 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE);
91
92 /* acknowledge interrupt */
93 if (active != 0)
94 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE);
95
96 /* handle link interrupt */
97 if ((active & ECM_INT_LINK) != 0)
98 phy_mac_interrupt(adapter->netdev->phydev);
99
100 /* handle TX/RX queue 0 interrupt */
101 if ((active & adapter->queue[0].irq_mask) != 0) {
102 if (napi_schedule_prep(&adapter->queue[0].napi)) {
103 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask);
104 /* schedule after masking to avoid races */
105 __napi_schedule(&adapter->queue[0].napi);
106 }
107 }
108
109 return IRQ_HANDLED;
110 }
111
tsnep_irq_txrx(int irq,void * arg)112 static irqreturn_t tsnep_irq_txrx(int irq, void *arg)
113 {
114 struct tsnep_queue *queue = arg;
115
116 /* handle TX/RX queue interrupt */
117 if (napi_schedule_prep(&queue->napi)) {
118 tsnep_disable_irq(queue->adapter, queue->irq_mask);
119 /* schedule after masking to avoid races */
120 __napi_schedule(&queue->napi);
121 }
122
123 return IRQ_HANDLED;
124 }
125
tsnep_set_irq_coalesce(struct tsnep_queue * queue,u32 usecs)126 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs)
127 {
128 if (usecs > TSNEP_COALESCE_USECS_MAX)
129 return -ERANGE;
130
131 usecs /= ECM_INT_DELAY_BASE_US;
132 usecs <<= ECM_INT_DELAY_SHIFT;
133 usecs &= ECM_INT_DELAY_MASK;
134
135 queue->irq_delay &= ~ECM_INT_DELAY_MASK;
136 queue->irq_delay |= usecs;
137 iowrite8(queue->irq_delay, queue->irq_delay_addr);
138
139 return 0;
140 }
141
tsnep_get_irq_coalesce(struct tsnep_queue * queue)142 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue)
143 {
144 u32 usecs;
145
146 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK);
147 usecs >>= ECM_INT_DELAY_SHIFT;
148 usecs *= ECM_INT_DELAY_BASE_US;
149
150 return usecs;
151 }
152
tsnep_mdiobus_read(struct mii_bus * bus,int addr,int regnum)153 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum)
154 {
155 struct tsnep_adapter *adapter = bus->priv;
156 u32 md;
157 int retval;
158
159 md = ECM_MD_READ;
160 if (!adapter->suppress_preamble)
161 md |= ECM_MD_PREAMBLE;
162 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
163 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
164 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
165 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
166 !(md & ECM_MD_BUSY), 16, 1000);
167 if (retval != 0)
168 return retval;
169
170 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT;
171 }
172
tsnep_mdiobus_write(struct mii_bus * bus,int addr,int regnum,u16 val)173 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum,
174 u16 val)
175 {
176 struct tsnep_adapter *adapter = bus->priv;
177 u32 md;
178 int retval;
179
180 md = ECM_MD_WRITE;
181 if (!adapter->suppress_preamble)
182 md |= ECM_MD_PREAMBLE;
183 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
184 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
185 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK;
186 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
187 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
188 !(md & ECM_MD_BUSY), 16, 1000);
189 if (retval != 0)
190 return retval;
191
192 return 0;
193 }
194
tsnep_set_link_mode(struct tsnep_adapter * adapter)195 static void tsnep_set_link_mode(struct tsnep_adapter *adapter)
196 {
197 u32 mode;
198
199 switch (adapter->phydev->speed) {
200 case SPEED_100:
201 mode = ECM_LINK_MODE_100;
202 break;
203 case SPEED_1000:
204 mode = ECM_LINK_MODE_1000;
205 break;
206 default:
207 mode = ECM_LINK_MODE_OFF;
208 break;
209 }
210 iowrite32(mode, adapter->addr + ECM_STATUS);
211 }
212
tsnep_phy_link_status_change(struct net_device * netdev)213 static void tsnep_phy_link_status_change(struct net_device *netdev)
214 {
215 struct tsnep_adapter *adapter = netdev_priv(netdev);
216 struct phy_device *phydev = netdev->phydev;
217
218 if (phydev->link)
219 tsnep_set_link_mode(adapter);
220
221 phy_print_status(netdev->phydev);
222 }
223
tsnep_phy_loopback(struct tsnep_adapter * adapter,bool enable)224 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable)
225 {
226 int retval;
227
228 retval = phy_loopback(adapter->phydev, enable);
229
230 /* PHY link state change is not signaled if loopback is enabled, it
231 * would delay a working loopback anyway, let's ensure that loopback
232 * is working immediately by setting link mode directly
233 */
234 if (!retval && enable)
235 tsnep_set_link_mode(adapter);
236
237 return retval;
238 }
239
tsnep_phy_open(struct tsnep_adapter * adapter)240 static int tsnep_phy_open(struct tsnep_adapter *adapter)
241 {
242 struct phy_device *phydev;
243 struct ethtool_eee ethtool_eee;
244 int retval;
245
246 retval = phy_connect_direct(adapter->netdev, adapter->phydev,
247 tsnep_phy_link_status_change,
248 adapter->phy_mode);
249 if (retval)
250 return retval;
251 phydev = adapter->netdev->phydev;
252
253 /* MAC supports only 100Mbps|1000Mbps full duplex
254 * SPE (Single Pair Ethernet) is also an option but not implemented yet
255 */
256 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);
257 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT);
258 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT);
259 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
260
261 /* disable EEE autoneg, EEE not supported by TSNEP */
262 memset(ðtool_eee, 0, sizeof(ethtool_eee));
263 phy_ethtool_set_eee(adapter->phydev, ðtool_eee);
264
265 adapter->phydev->irq = PHY_MAC_INTERRUPT;
266 phy_start(adapter->phydev);
267
268 return 0;
269 }
270
tsnep_phy_close(struct tsnep_adapter * adapter)271 static void tsnep_phy_close(struct tsnep_adapter *adapter)
272 {
273 phy_stop(adapter->netdev->phydev);
274 phy_disconnect(adapter->netdev->phydev);
275 }
276
tsnep_tx_ring_cleanup(struct tsnep_tx * tx)277 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)
278 {
279 struct device *dmadev = tx->adapter->dmadev;
280 int i;
281
282 memset(tx->entry, 0, sizeof(tx->entry));
283
284 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
285 if (tx->page[i]) {
286 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i],
287 tx->page_dma[i]);
288 tx->page[i] = NULL;
289 tx->page_dma[i] = 0;
290 }
291 }
292 }
293
tsnep_tx_ring_create(struct tsnep_tx * tx)294 static int tsnep_tx_ring_create(struct tsnep_tx *tx)
295 {
296 struct device *dmadev = tx->adapter->dmadev;
297 struct tsnep_tx_entry *entry;
298 struct tsnep_tx_entry *next_entry;
299 int i, j;
300 int retval;
301
302 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
303 tx->page[i] =
304 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i],
305 GFP_KERNEL);
306 if (!tx->page[i]) {
307 retval = -ENOMEM;
308 goto alloc_failed;
309 }
310 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
311 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
312 entry->desc_wb = (struct tsnep_tx_desc_wb *)
313 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j);
314 entry->desc = (struct tsnep_tx_desc *)
315 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
316 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j;
317 entry->owner_user_flag = false;
318 }
319 }
320 for (i = 0; i < TSNEP_RING_SIZE; i++) {
321 entry = &tx->entry[i];
322 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK];
323 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
324 }
325
326 return 0;
327
328 alloc_failed:
329 tsnep_tx_ring_cleanup(tx);
330 return retval;
331 }
332
tsnep_tx_init(struct tsnep_tx * tx)333 static void tsnep_tx_init(struct tsnep_tx *tx)
334 {
335 dma_addr_t dma;
336
337 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
338 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);
339 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);
340 tx->write = 0;
341 tx->read = 0;
342 tx->owner_counter = 1;
343 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
344 }
345
tsnep_tx_enable(struct tsnep_tx * tx)346 static void tsnep_tx_enable(struct tsnep_tx *tx)
347 {
348 struct netdev_queue *nq;
349
350 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
351
352 __netif_tx_lock_bh(nq);
353 netif_tx_wake_queue(nq);
354 __netif_tx_unlock_bh(nq);
355 }
356
tsnep_tx_disable(struct tsnep_tx * tx,struct napi_struct * napi)357 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)
358 {
359 struct netdev_queue *nq;
360 u32 val;
361
362 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
363
364 __netif_tx_lock_bh(nq);
365 netif_tx_stop_queue(nq);
366 __netif_tx_unlock_bh(nq);
367
368 /* wait until TX is done in hardware */
369 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
370 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
371 1000000);
372
373 /* wait until TX is also done in software */
374 while (READ_ONCE(tx->read) != tx->write) {
375 napi_schedule(napi);
376 napi_synchronize(napi);
377 }
378 }
379
tsnep_tx_activate(struct tsnep_tx * tx,int index,int length,bool last)380 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
381 bool last)
382 {
383 struct tsnep_tx_entry *entry = &tx->entry[index];
384
385 entry->properties = 0;
386 /* xdpf and zc are union with skb */
387 if (entry->skb) {
388 entry->properties = length & TSNEP_DESC_LENGTH_MASK;
389 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
390 if ((entry->type & TSNEP_TX_TYPE_SKB_TSTAMP) == TSNEP_TX_TYPE_SKB_TSTAMP)
391 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG;
392
393 /* toggle user flag to prevent false acknowledge
394 *
395 * Only the first fragment is acknowledged. For all other
396 * fragments no acknowledge is done and the last written owner
397 * counter stays in the writeback descriptor. Therefore, it is
398 * possible that the last written owner counter is identical to
399 * the new incremented owner counter and a false acknowledge is
400 * detected before the real acknowledge has been done by
401 * hardware.
402 *
403 * The user flag is used to prevent this situation. The user
404 * flag is copied to the writeback descriptor by the hardware
405 * and is used as additional acknowledge data. By toggeling the
406 * user flag only for the first fragment (which is
407 * acknowledged), it is guaranteed that the last acknowledge
408 * done for this descriptor has used a different user flag and
409 * cannot be detected as false acknowledge.
410 */
411 entry->owner_user_flag = !entry->owner_user_flag;
412 }
413 if (last)
414 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG;
415 if (index == tx->increment_owner_counter) {
416 tx->owner_counter++;
417 if (tx->owner_counter == 4)
418 tx->owner_counter = 1;
419 tx->increment_owner_counter--;
420 if (tx->increment_owner_counter < 0)
421 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
422 }
423 entry->properties |=
424 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
425 TSNEP_DESC_OWNER_COUNTER_MASK;
426 if (entry->owner_user_flag)
427 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG;
428 entry->desc->more_properties =
429 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK);
430 if (entry->type & TSNEP_TX_TYPE_INLINE)
431 entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG;
432
433 /* descriptor properties shall be written last, because valid data is
434 * signaled there
435 */
436 dma_wmb();
437
438 entry->desc->properties = __cpu_to_le32(entry->properties);
439 }
440
tsnep_tx_desc_available(struct tsnep_tx * tx)441 static int tsnep_tx_desc_available(struct tsnep_tx *tx)
442 {
443 if (tx->read <= tx->write)
444 return TSNEP_RING_SIZE - tx->write + tx->read - 1;
445 else
446 return tx->read - tx->write - 1;
447 }
448
tsnep_tx_map_frag(skb_frag_t * frag,struct tsnep_tx_entry * entry,struct device * dmadev,dma_addr_t * dma)449 static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry,
450 struct device *dmadev, dma_addr_t *dma)
451 {
452 unsigned int len;
453 int mapped;
454
455 len = skb_frag_size(frag);
456 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
457 *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE);
458 if (dma_mapping_error(dmadev, *dma))
459 return -ENOMEM;
460 entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE;
461 mapped = 1;
462 } else {
463 void *fragdata = skb_frag_address_safe(frag);
464
465 if (likely(fragdata)) {
466 memcpy(&entry->desc->tx, fragdata, len);
467 } else {
468 struct page *page = skb_frag_page(frag);
469
470 fragdata = kmap_local_page(page);
471 memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag),
472 len);
473 kunmap_local(fragdata);
474 }
475 entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE;
476 mapped = 0;
477 }
478
479 return mapped;
480 }
481
tsnep_tx_map(struct sk_buff * skb,struct tsnep_tx * tx,int count,bool do_tstamp)482 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count,
483 bool do_tstamp)
484 {
485 struct device *dmadev = tx->adapter->dmadev;
486 struct tsnep_tx_entry *entry;
487 unsigned int len;
488 int map_len = 0;
489 dma_addr_t dma;
490 int i, mapped;
491
492 for (i = 0; i < count; i++) {
493 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
494
495 if (!i) {
496 len = skb_headlen(skb);
497 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
498 dma = dma_map_single(dmadev, skb->data, len,
499 DMA_TO_DEVICE);
500 if (dma_mapping_error(dmadev, dma))
501 return -ENOMEM;
502 entry->type = TSNEP_TX_TYPE_SKB_MAP;
503 mapped = 1;
504 } else {
505 memcpy(&entry->desc->tx, skb->data, len);
506 entry->type = TSNEP_TX_TYPE_SKB_INLINE;
507 mapped = 0;
508 }
509
510 if (do_tstamp)
511 entry->type |= TSNEP_TX_TYPE_TSTAMP;
512 } else {
513 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
514
515 len = skb_frag_size(frag);
516 mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma);
517 if (mapped < 0)
518 return mapped;
519 }
520
521 entry->len = len;
522 if (likely(mapped)) {
523 dma_unmap_addr_set(entry, dma, dma);
524 entry->desc->tx = __cpu_to_le64(dma);
525 }
526
527 map_len += len;
528 }
529
530 return map_len;
531 }
532
tsnep_tx_unmap(struct tsnep_tx * tx,int index,int count)533 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
534 {
535 struct device *dmadev = tx->adapter->dmadev;
536 struct tsnep_tx_entry *entry;
537 int map_len = 0;
538 int i;
539
540 for (i = 0; i < count; i++) {
541 entry = &tx->entry[(index + i) & TSNEP_RING_MASK];
542
543 if (entry->len) {
544 if (entry->type & TSNEP_TX_TYPE_MAP)
545 dma_unmap_single(dmadev,
546 dma_unmap_addr(entry, dma),
547 dma_unmap_len(entry, len),
548 DMA_TO_DEVICE);
549 else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE)
550 dma_unmap_page(dmadev,
551 dma_unmap_addr(entry, dma),
552 dma_unmap_len(entry, len),
553 DMA_TO_DEVICE);
554 map_len += entry->len;
555 entry->len = 0;
556 }
557 }
558
559 return map_len;
560 }
561
tsnep_xmit_frame_ring(struct sk_buff * skb,struct tsnep_tx * tx)562 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
563 struct tsnep_tx *tx)
564 {
565 struct tsnep_tx_entry *entry;
566 bool do_tstamp = false;
567 int count = 1;
568 int length;
569 int retval;
570 int i;
571
572 if (skb_shinfo(skb)->nr_frags > 0)
573 count += skb_shinfo(skb)->nr_frags;
574
575 if (tsnep_tx_desc_available(tx) < count) {
576 /* ring full, shall not happen because queue is stopped if full
577 * below
578 */
579 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
580
581 return NETDEV_TX_BUSY;
582 }
583
584 entry = &tx->entry[tx->write];
585 entry->skb = skb;
586
587 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
588 tx->adapter->hwtstamp_config.tx_type == HWTSTAMP_TX_ON) {
589 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
590 do_tstamp = true;
591 }
592
593 retval = tsnep_tx_map(skb, tx, count, do_tstamp);
594 if (retval < 0) {
595 tsnep_tx_unmap(tx, tx->write, count);
596 dev_kfree_skb_any(entry->skb);
597 entry->skb = NULL;
598
599 tx->dropped++;
600
601 return NETDEV_TX_OK;
602 }
603 length = retval;
604
605 for (i = 0; i < count; i++)
606 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
607 i == count - 1);
608 tx->write = (tx->write + count) & TSNEP_RING_MASK;
609
610 skb_tx_timestamp(skb);
611
612 /* descriptor properties shall be valid before hardware is notified */
613 dma_wmb();
614
615 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
616
617 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) {
618 /* ring can get full with next frame */
619 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
620 }
621
622 return NETDEV_TX_OK;
623 }
624
tsnep_xdp_tx_map(struct xdp_frame * xdpf,struct tsnep_tx * tx,struct skb_shared_info * shinfo,int count,u32 type)625 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx,
626 struct skb_shared_info *shinfo, int count, u32 type)
627 {
628 struct device *dmadev = tx->adapter->dmadev;
629 struct tsnep_tx_entry *entry;
630 struct page *page;
631 skb_frag_t *frag;
632 unsigned int len;
633 int map_len = 0;
634 dma_addr_t dma;
635 void *data;
636 int i;
637
638 frag = NULL;
639 len = xdpf->len;
640 for (i = 0; i < count; i++) {
641 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
642 if (type & TSNEP_TX_TYPE_XDP_NDO) {
643 data = unlikely(frag) ? skb_frag_address(frag) :
644 xdpf->data;
645 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE);
646 if (dma_mapping_error(dmadev, dma))
647 return -ENOMEM;
648
649 entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE;
650 } else {
651 page = unlikely(frag) ? skb_frag_page(frag) :
652 virt_to_page(xdpf->data);
653 dma = page_pool_get_dma_addr(page);
654 if (unlikely(frag))
655 dma += skb_frag_off(frag);
656 else
657 dma += sizeof(*xdpf) + xdpf->headroom;
658 dma_sync_single_for_device(dmadev, dma, len,
659 DMA_BIDIRECTIONAL);
660
661 entry->type = TSNEP_TX_TYPE_XDP_TX;
662 }
663
664 entry->len = len;
665 dma_unmap_addr_set(entry, dma, dma);
666
667 entry->desc->tx = __cpu_to_le64(dma);
668
669 map_len += len;
670
671 if (i + 1 < count) {
672 frag = &shinfo->frags[i];
673 len = skb_frag_size(frag);
674 }
675 }
676
677 return map_len;
678 }
679
680 /* This function requires __netif_tx_lock is held by the caller. */
tsnep_xdp_xmit_frame_ring(struct xdp_frame * xdpf,struct tsnep_tx * tx,u32 type)681 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf,
682 struct tsnep_tx *tx, u32 type)
683 {
684 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf);
685 struct tsnep_tx_entry *entry;
686 int count, length, retval, i;
687
688 count = 1;
689 if (unlikely(xdp_frame_has_frags(xdpf)))
690 count += shinfo->nr_frags;
691
692 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
693 * will be available for normal TX path and queue is stopped there if
694 * necessary
695 */
696 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count))
697 return false;
698
699 entry = &tx->entry[tx->write];
700 entry->xdpf = xdpf;
701
702 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type);
703 if (retval < 0) {
704 tsnep_tx_unmap(tx, tx->write, count);
705 entry->xdpf = NULL;
706
707 tx->dropped++;
708
709 return false;
710 }
711 length = retval;
712
713 for (i = 0; i < count; i++)
714 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
715 i == count - 1);
716 tx->write = (tx->write + count) & TSNEP_RING_MASK;
717
718 /* descriptor properties shall be valid before hardware is notified */
719 dma_wmb();
720
721 return true;
722 }
723
tsnep_xdp_xmit_flush(struct tsnep_tx * tx)724 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx)
725 {
726 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
727 }
728
tsnep_xdp_xmit_back(struct tsnep_adapter * adapter,struct xdp_buff * xdp,struct netdev_queue * tx_nq,struct tsnep_tx * tx,bool zc)729 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,
730 struct xdp_buff *xdp,
731 struct netdev_queue *tx_nq, struct tsnep_tx *tx,
732 bool zc)
733 {
734 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
735 bool xmit;
736 u32 type;
737
738 if (unlikely(!xdpf))
739 return false;
740
741 /* no page pool for zero copy */
742 if (zc)
743 type = TSNEP_TX_TYPE_XDP_NDO;
744 else
745 type = TSNEP_TX_TYPE_XDP_TX;
746
747 __netif_tx_lock(tx_nq, smp_processor_id());
748
749 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, type);
750
751 /* Avoid transmit queue timeout since we share it with the slow path */
752 if (xmit)
753 txq_trans_cond_update(tx_nq);
754
755 __netif_tx_unlock(tx_nq);
756
757 return xmit;
758 }
759
tsnep_xdp_tx_map_zc(struct xdp_desc * xdpd,struct tsnep_tx * tx)760 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)
761 {
762 struct tsnep_tx_entry *entry;
763 dma_addr_t dma;
764
765 entry = &tx->entry[tx->write];
766 entry->zc = true;
767
768 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr);
769 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len);
770
771 entry->type = TSNEP_TX_TYPE_XSK;
772 entry->len = xdpd->len;
773
774 entry->desc->tx = __cpu_to_le64(dma);
775
776 return xdpd->len;
777 }
778
tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc * xdpd,struct tsnep_tx * tx)779 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,
780 struct tsnep_tx *tx)
781 {
782 int length;
783
784 length = tsnep_xdp_tx_map_zc(xdpd, tx);
785
786 tsnep_tx_activate(tx, tx->write, length, true);
787 tx->write = (tx->write + 1) & TSNEP_RING_MASK;
788 }
789
tsnep_xdp_xmit_zc(struct tsnep_tx * tx)790 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)
791 {
792 int desc_available = tsnep_tx_desc_available(tx);
793 struct xdp_desc *descs = tx->xsk_pool->tx_descs;
794 int batch, i;
795
796 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
797 * will be available for normal TX path and queue is stopped there if
798 * necessary
799 */
800 if (desc_available <= (MAX_SKB_FRAGS + 1))
801 return;
802 desc_available -= MAX_SKB_FRAGS + 1;
803
804 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available);
805 for (i = 0; i < batch; i++)
806 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx);
807
808 if (batch) {
809 /* descriptor properties shall be valid before hardware is
810 * notified
811 */
812 dma_wmb();
813
814 tsnep_xdp_xmit_flush(tx);
815 }
816 }
817
tsnep_tx_poll(struct tsnep_tx * tx,int napi_budget)818 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
819 {
820 struct tsnep_tx_entry *entry;
821 struct netdev_queue *nq;
822 int xsk_frames = 0;
823 int budget = 128;
824 int length;
825 int count;
826
827 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
828 __netif_tx_lock(nq, smp_processor_id());
829
830 do {
831 if (tx->read == tx->write)
832 break;
833
834 entry = &tx->entry[tx->read];
835 if ((__le32_to_cpu(entry->desc_wb->properties) &
836 TSNEP_TX_DESC_OWNER_MASK) !=
837 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
838 break;
839
840 /* descriptor properties shall be read first, because valid data
841 * is signaled there
842 */
843 dma_rmb();
844
845 count = 1;
846 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
847 skb_shinfo(entry->skb)->nr_frags > 0)
848 count += skb_shinfo(entry->skb)->nr_frags;
849 else if ((entry->type & TSNEP_TX_TYPE_XDP) &&
850 xdp_frame_has_frags(entry->xdpf))
851 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags;
852
853 length = tsnep_tx_unmap(tx, tx->read, count);
854
855 if (((entry->type & TSNEP_TX_TYPE_SKB_TSTAMP) == TSNEP_TX_TYPE_SKB_TSTAMP) &&
856 (__le32_to_cpu(entry->desc_wb->properties) &
857 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) {
858 struct skb_shared_hwtstamps hwtstamps;
859 u64 timestamp;
860
861 if (skb_shinfo(entry->skb)->tx_flags &
862 SKBTX_HW_TSTAMP_USE_CYCLES)
863 timestamp =
864 __le64_to_cpu(entry->desc_wb->counter);
865 else
866 timestamp =
867 __le64_to_cpu(entry->desc_wb->timestamp);
868
869 memset(&hwtstamps, 0, sizeof(hwtstamps));
870 hwtstamps.hwtstamp = ns_to_ktime(timestamp);
871
872 skb_tstamp_tx(entry->skb, &hwtstamps);
873 }
874
875 if (entry->type & TSNEP_TX_TYPE_SKB)
876 napi_consume_skb(entry->skb, napi_budget);
877 else if (entry->type & TSNEP_TX_TYPE_XDP)
878 xdp_return_frame_rx_napi(entry->xdpf);
879 else
880 xsk_frames++;
881 /* xdpf and zc are union with skb */
882 entry->skb = NULL;
883
884 tx->read = (tx->read + count) & TSNEP_RING_MASK;
885
886 tx->packets++;
887 tx->bytes += length + ETH_FCS_LEN;
888
889 budget--;
890 } while (likely(budget));
891
892 if (tx->xsk_pool) {
893 if (xsk_frames)
894 xsk_tx_completed(tx->xsk_pool, xsk_frames);
895 if (xsk_uses_need_wakeup(tx->xsk_pool))
896 xsk_set_tx_need_wakeup(tx->xsk_pool);
897 tsnep_xdp_xmit_zc(tx);
898 }
899
900 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&
901 netif_tx_queue_stopped(nq)) {
902 netif_tx_wake_queue(nq);
903 }
904
905 __netif_tx_unlock(nq);
906
907 return budget != 0;
908 }
909
tsnep_tx_pending(struct tsnep_tx * tx)910 static bool tsnep_tx_pending(struct tsnep_tx *tx)
911 {
912 struct tsnep_tx_entry *entry;
913 struct netdev_queue *nq;
914 bool pending = false;
915
916 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
917 __netif_tx_lock(nq, smp_processor_id());
918
919 if (tx->read != tx->write) {
920 entry = &tx->entry[tx->read];
921 if ((__le32_to_cpu(entry->desc_wb->properties) &
922 TSNEP_TX_DESC_OWNER_MASK) ==
923 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
924 pending = true;
925 }
926
927 __netif_tx_unlock(nq);
928
929 return pending;
930 }
931
tsnep_tx_open(struct tsnep_tx * tx)932 static int tsnep_tx_open(struct tsnep_tx *tx)
933 {
934 int retval;
935
936 retval = tsnep_tx_ring_create(tx);
937 if (retval)
938 return retval;
939
940 tsnep_tx_init(tx);
941
942 return 0;
943 }
944
tsnep_tx_close(struct tsnep_tx * tx)945 static void tsnep_tx_close(struct tsnep_tx *tx)
946 {
947 tsnep_tx_ring_cleanup(tx);
948 }
949
tsnep_rx_ring_cleanup(struct tsnep_rx * rx)950 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
951 {
952 struct device *dmadev = rx->adapter->dmadev;
953 struct tsnep_rx_entry *entry;
954 int i;
955
956 for (i = 0; i < TSNEP_RING_SIZE; i++) {
957 entry = &rx->entry[i];
958 if (!rx->xsk_pool && entry->page)
959 page_pool_put_full_page(rx->page_pool, entry->page,
960 false);
961 if (rx->xsk_pool && entry->xdp)
962 xsk_buff_free(entry->xdp);
963 /* xdp is union with page */
964 entry->page = NULL;
965 }
966
967 if (rx->page_pool)
968 page_pool_destroy(rx->page_pool);
969
970 memset(rx->entry, 0, sizeof(rx->entry));
971
972 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
973 if (rx->page[i]) {
974 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i],
975 rx->page_dma[i]);
976 rx->page[i] = NULL;
977 rx->page_dma[i] = 0;
978 }
979 }
980 }
981
tsnep_rx_ring_create(struct tsnep_rx * rx)982 static int tsnep_rx_ring_create(struct tsnep_rx *rx)
983 {
984 struct device *dmadev = rx->adapter->dmadev;
985 struct tsnep_rx_entry *entry;
986 struct page_pool_params pp_params = { 0 };
987 struct tsnep_rx_entry *next_entry;
988 int i, j;
989 int retval;
990
991 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
992 rx->page[i] =
993 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i],
994 GFP_KERNEL);
995 if (!rx->page[i]) {
996 retval = -ENOMEM;
997 goto failed;
998 }
999 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
1000 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
1001 entry->desc_wb = (struct tsnep_rx_desc_wb *)
1002 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);
1003 entry->desc = (struct tsnep_rx_desc *)
1004 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
1005 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;
1006 }
1007 }
1008
1009 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
1010 pp_params.order = 0;
1011 pp_params.pool_size = TSNEP_RING_SIZE;
1012 pp_params.nid = dev_to_node(dmadev);
1013 pp_params.dev = dmadev;
1014 pp_params.dma_dir = DMA_BIDIRECTIONAL;
1015 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE;
1016 pp_params.offset = TSNEP_RX_OFFSET;
1017 rx->page_pool = page_pool_create(&pp_params);
1018 if (IS_ERR(rx->page_pool)) {
1019 retval = PTR_ERR(rx->page_pool);
1020 rx->page_pool = NULL;
1021 goto failed;
1022 }
1023
1024 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1025 entry = &rx->entry[i];
1026 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK];
1027 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
1028 }
1029
1030 return 0;
1031
1032 failed:
1033 tsnep_rx_ring_cleanup(rx);
1034 return retval;
1035 }
1036
tsnep_rx_init(struct tsnep_rx * rx)1037 static void tsnep_rx_init(struct tsnep_rx *rx)
1038 {
1039 dma_addr_t dma;
1040
1041 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
1042 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);
1043 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);
1044 rx->write = 0;
1045 rx->read = 0;
1046 rx->owner_counter = 1;
1047 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1048 }
1049
tsnep_rx_enable(struct tsnep_rx * rx)1050 static void tsnep_rx_enable(struct tsnep_rx *rx)
1051 {
1052 /* descriptor properties shall be valid before hardware is notified */
1053 dma_wmb();
1054
1055 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
1056 }
1057
tsnep_rx_disable(struct tsnep_rx * rx)1058 static void tsnep_rx_disable(struct tsnep_rx *rx)
1059 {
1060 u32 val;
1061
1062 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);
1063 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,
1064 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,
1065 1000000);
1066 }
1067
tsnep_rx_desc_available(struct tsnep_rx * rx)1068 static int tsnep_rx_desc_available(struct tsnep_rx *rx)
1069 {
1070 if (rx->read <= rx->write)
1071 return TSNEP_RING_SIZE - rx->write + rx->read - 1;
1072 else
1073 return rx->read - rx->write - 1;
1074 }
1075
tsnep_rx_free_page_buffer(struct tsnep_rx * rx)1076 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx)
1077 {
1078 struct page **page;
1079
1080 /* last entry of page_buffer is always zero, because ring cannot be
1081 * filled completely
1082 */
1083 page = rx->page_buffer;
1084 while (*page) {
1085 page_pool_put_full_page(rx->page_pool, *page, false);
1086 *page = NULL;
1087 page++;
1088 }
1089 }
1090
tsnep_rx_alloc_page_buffer(struct tsnep_rx * rx)1091 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx)
1092 {
1093 int i;
1094
1095 /* alloc for all ring entries except the last one, because ring cannot
1096 * be filled completely
1097 */
1098 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) {
1099 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool);
1100 if (!rx->page_buffer[i]) {
1101 tsnep_rx_free_page_buffer(rx);
1102
1103 return -ENOMEM;
1104 }
1105 }
1106
1107 return 0;
1108 }
1109
tsnep_rx_set_page(struct tsnep_rx * rx,struct tsnep_rx_entry * entry,struct page * page)1110 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1111 struct page *page)
1112 {
1113 entry->page = page;
1114 entry->len = TSNEP_MAX_RX_BUF_SIZE;
1115 entry->dma = page_pool_get_dma_addr(entry->page);
1116 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET);
1117 }
1118
tsnep_rx_alloc_buffer(struct tsnep_rx * rx,int index)1119 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index)
1120 {
1121 struct tsnep_rx_entry *entry = &rx->entry[index];
1122 struct page *page;
1123
1124 page = page_pool_dev_alloc_pages(rx->page_pool);
1125 if (unlikely(!page))
1126 return -ENOMEM;
1127 tsnep_rx_set_page(rx, entry, page);
1128
1129 return 0;
1130 }
1131
tsnep_rx_reuse_buffer(struct tsnep_rx * rx,int index)1132 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index)
1133 {
1134 struct tsnep_rx_entry *entry = &rx->entry[index];
1135 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1136
1137 tsnep_rx_set_page(rx, entry, read->page);
1138 read->page = NULL;
1139 }
1140
tsnep_rx_activate(struct tsnep_rx * rx,int index)1141 static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
1142 {
1143 struct tsnep_rx_entry *entry = &rx->entry[index];
1144
1145 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */
1146 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;
1147 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
1148 if (index == rx->increment_owner_counter) {
1149 rx->owner_counter++;
1150 if (rx->owner_counter == 4)
1151 rx->owner_counter = 1;
1152 rx->increment_owner_counter--;
1153 if (rx->increment_owner_counter < 0)
1154 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1155 }
1156 entry->properties |=
1157 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
1158 TSNEP_DESC_OWNER_COUNTER_MASK;
1159
1160 /* descriptor properties shall be written last, because valid data is
1161 * signaled there
1162 */
1163 dma_wmb();
1164
1165 entry->desc->properties = __cpu_to_le32(entry->properties);
1166 }
1167
tsnep_rx_alloc(struct tsnep_rx * rx,int count,bool reuse)1168 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse)
1169 {
1170 bool alloc_failed = false;
1171 int i, index;
1172
1173 for (i = 0; i < count && !alloc_failed; i++) {
1174 index = (rx->write + i) & TSNEP_RING_MASK;
1175
1176 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) {
1177 rx->alloc_failed++;
1178 alloc_failed = true;
1179
1180 /* reuse only if no other allocation was successful */
1181 if (i == 0 && reuse)
1182 tsnep_rx_reuse_buffer(rx, index);
1183 else
1184 break;
1185 }
1186
1187 tsnep_rx_activate(rx, index);
1188 }
1189
1190 if (i)
1191 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1192
1193 return i;
1194 }
1195
tsnep_rx_refill(struct tsnep_rx * rx,int count,bool reuse)1196 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
1197 {
1198 int desc_refilled;
1199
1200 desc_refilled = tsnep_rx_alloc(rx, count, reuse);
1201 if (desc_refilled)
1202 tsnep_rx_enable(rx);
1203
1204 return desc_refilled;
1205 }
1206
tsnep_rx_set_xdp(struct tsnep_rx * rx,struct tsnep_rx_entry * entry,struct xdp_buff * xdp)1207 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1208 struct xdp_buff *xdp)
1209 {
1210 entry->xdp = xdp;
1211 entry->len = TSNEP_XSK_RX_BUF_SIZE;
1212 entry->dma = xsk_buff_xdp_get_dma(entry->xdp);
1213 entry->desc->rx = __cpu_to_le64(entry->dma);
1214 }
1215
tsnep_rx_reuse_buffer_zc(struct tsnep_rx * rx,int index)1216 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index)
1217 {
1218 struct tsnep_rx_entry *entry = &rx->entry[index];
1219 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1220
1221 tsnep_rx_set_xdp(rx, entry, read->xdp);
1222 read->xdp = NULL;
1223 }
1224
tsnep_rx_alloc_zc(struct tsnep_rx * rx,int count,bool reuse)1225 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse)
1226 {
1227 u32 allocated;
1228 int i;
1229
1230 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count);
1231 for (i = 0; i < allocated; i++) {
1232 int index = (rx->write + i) & TSNEP_RING_MASK;
1233 struct tsnep_rx_entry *entry = &rx->entry[index];
1234
1235 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]);
1236 tsnep_rx_activate(rx, index);
1237 }
1238 if (i == 0) {
1239 rx->alloc_failed++;
1240
1241 if (reuse) {
1242 tsnep_rx_reuse_buffer_zc(rx, rx->write);
1243 tsnep_rx_activate(rx, rx->write);
1244 }
1245 }
1246
1247 if (i)
1248 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1249
1250 return i;
1251 }
1252
tsnep_rx_free_zc(struct tsnep_rx * rx)1253 static void tsnep_rx_free_zc(struct tsnep_rx *rx)
1254 {
1255 int i;
1256
1257 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1258 struct tsnep_rx_entry *entry = &rx->entry[i];
1259
1260 if (entry->xdp)
1261 xsk_buff_free(entry->xdp);
1262 entry->xdp = NULL;
1263 }
1264 }
1265
tsnep_rx_refill_zc(struct tsnep_rx * rx,int count,bool reuse)1266 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse)
1267 {
1268 int desc_refilled;
1269
1270 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse);
1271 if (desc_refilled)
1272 tsnep_rx_enable(rx);
1273
1274 return desc_refilled;
1275 }
1276
tsnep_xdp_run_prog(struct tsnep_rx * rx,struct bpf_prog * prog,struct xdp_buff * xdp,int * status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1277 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,
1278 struct xdp_buff *xdp, int *status,
1279 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1280 {
1281 unsigned int length;
1282 unsigned int sync;
1283 u32 act;
1284
1285 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;
1286
1287 act = bpf_prog_run_xdp(prog, xdp);
1288 switch (act) {
1289 case XDP_PASS:
1290 return false;
1291 case XDP_TX:
1292 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, false))
1293 goto out_failure;
1294 *status |= TSNEP_XDP_TX;
1295 return true;
1296 case XDP_REDIRECT:
1297 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1298 goto out_failure;
1299 *status |= TSNEP_XDP_REDIRECT;
1300 return true;
1301 default:
1302 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1303 fallthrough;
1304 case XDP_ABORTED:
1305 out_failure:
1306 trace_xdp_exception(rx->adapter->netdev, prog, act);
1307 fallthrough;
1308 case XDP_DROP:
1309 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU
1310 * touch
1311 */
1312 sync = xdp->data_end - xdp->data_hard_start -
1313 XDP_PACKET_HEADROOM;
1314 sync = max(sync, length);
1315 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data),
1316 sync, true);
1317 return true;
1318 }
1319 }
1320
tsnep_xdp_run_prog_zc(struct tsnep_rx * rx,struct bpf_prog * prog,struct xdp_buff * xdp,int * status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1321 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog,
1322 struct xdp_buff *xdp, int *status,
1323 struct netdev_queue *tx_nq,
1324 struct tsnep_tx *tx)
1325 {
1326 u32 act;
1327
1328 act = bpf_prog_run_xdp(prog, xdp);
1329
1330 /* XDP_REDIRECT is the main action for zero-copy */
1331 if (likely(act == XDP_REDIRECT)) {
1332 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1333 goto out_failure;
1334 *status |= TSNEP_XDP_REDIRECT;
1335 return true;
1336 }
1337
1338 switch (act) {
1339 case XDP_PASS:
1340 return false;
1341 case XDP_TX:
1342 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, true))
1343 goto out_failure;
1344 *status |= TSNEP_XDP_TX;
1345 return true;
1346 default:
1347 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1348 fallthrough;
1349 case XDP_ABORTED:
1350 out_failure:
1351 trace_xdp_exception(rx->adapter->netdev, prog, act);
1352 fallthrough;
1353 case XDP_DROP:
1354 xsk_buff_free(xdp);
1355 return true;
1356 }
1357 }
1358
tsnep_finalize_xdp(struct tsnep_adapter * adapter,int status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1359 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,
1360 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1361 {
1362 if (status & TSNEP_XDP_TX) {
1363 __netif_tx_lock(tx_nq, smp_processor_id());
1364 tsnep_xdp_xmit_flush(tx);
1365 __netif_tx_unlock(tx_nq);
1366 }
1367
1368 if (status & TSNEP_XDP_REDIRECT)
1369 xdp_do_flush();
1370 }
1371
tsnep_build_skb(struct tsnep_rx * rx,struct page * page,int length)1372 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
1373 int length)
1374 {
1375 struct sk_buff *skb;
1376
1377 skb = napi_build_skb(page_address(page), PAGE_SIZE);
1378 if (unlikely(!skb))
1379 return NULL;
1380
1381 /* update pointers within the skb to store the data */
1382 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE);
1383 __skb_put(skb, length - ETH_FCS_LEN);
1384
1385 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {
1386 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
1387 struct tsnep_rx_inline *rx_inline =
1388 (struct tsnep_rx_inline *)(page_address(page) +
1389 TSNEP_RX_OFFSET);
1390
1391 skb_shinfo(skb)->tx_flags |=
1392 SKBTX_HW_TSTAMP_NETDEV;
1393 memset(hwtstamps, 0, sizeof(*hwtstamps));
1394 hwtstamps->netdev_data = rx_inline;
1395 }
1396
1397 skb_record_rx_queue(skb, rx->queue_index);
1398 skb->protocol = eth_type_trans(skb, rx->adapter->netdev);
1399
1400 return skb;
1401 }
1402
tsnep_rx_page(struct tsnep_rx * rx,struct napi_struct * napi,struct page * page,int length)1403 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi,
1404 struct page *page, int length)
1405 {
1406 struct sk_buff *skb;
1407
1408 skb = tsnep_build_skb(rx, page, length);
1409 if (skb) {
1410 skb_mark_for_recycle(skb);
1411
1412 rx->packets++;
1413 rx->bytes += length;
1414 if (skb->pkt_type == PACKET_MULTICAST)
1415 rx->multicast++;
1416
1417 napi_gro_receive(napi, skb);
1418 } else {
1419 page_pool_recycle_direct(rx->page_pool, page);
1420
1421 rx->dropped++;
1422 }
1423 }
1424
tsnep_rx_poll(struct tsnep_rx * rx,struct napi_struct * napi,int budget)1425 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
1426 int budget)
1427 {
1428 struct device *dmadev = rx->adapter->dmadev;
1429 enum dma_data_direction dma_dir;
1430 struct tsnep_rx_entry *entry;
1431 struct netdev_queue *tx_nq;
1432 struct bpf_prog *prog;
1433 struct xdp_buff xdp;
1434 struct tsnep_tx *tx;
1435 int desc_available;
1436 int xdp_status = 0;
1437 int done = 0;
1438 int length;
1439
1440 desc_available = tsnep_rx_desc_available(rx);
1441 dma_dir = page_pool_get_dma_dir(rx->page_pool);
1442 prog = READ_ONCE(rx->adapter->xdp_prog);
1443 if (prog) {
1444 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1445 rx->tx_queue_index);
1446 tx = &rx->adapter->tx[rx->tx_queue_index];
1447
1448 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq);
1449 }
1450
1451 while (likely(done < budget) && (rx->read != rx->write)) {
1452 entry = &rx->entry[rx->read];
1453 if ((__le32_to_cpu(entry->desc_wb->properties) &
1454 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1455 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1456 break;
1457 done++;
1458
1459 if (desc_available >= TSNEP_RING_RX_REFILL) {
1460 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1461
1462 desc_available -= tsnep_rx_refill(rx, desc_available,
1463 reuse);
1464 if (!entry->page) {
1465 /* buffer has been reused for refill to prevent
1466 * empty RX ring, thus buffer cannot be used for
1467 * RX processing
1468 */
1469 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1470 desc_available++;
1471
1472 rx->dropped++;
1473
1474 continue;
1475 }
1476 }
1477
1478 /* descriptor properties shall be read first, because valid data
1479 * is signaled there
1480 */
1481 dma_rmb();
1482
1483 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET);
1484 length = __le32_to_cpu(entry->desc_wb->properties) &
1485 TSNEP_DESC_LENGTH_MASK;
1486 dma_sync_single_range_for_cpu(dmadev, entry->dma,
1487 TSNEP_RX_OFFSET, length, dma_dir);
1488
1489 /* RX metadata with timestamps is in front of actual data,
1490 * subtract metadata size to get length of actual data and
1491 * consider metadata size as offset of actual data during RX
1492 * processing
1493 */
1494 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1495
1496 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1497 desc_available++;
1498
1499 if (prog) {
1500 bool consume;
1501
1502 xdp_prepare_buff(&xdp, page_address(entry->page),
1503 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE,
1504 length - ETH_FCS_LEN, false);
1505
1506 consume = tsnep_xdp_run_prog(rx, prog, &xdp,
1507 &xdp_status, tx_nq, tx);
1508 if (consume) {
1509 rx->packets++;
1510 rx->bytes += length;
1511
1512 entry->page = NULL;
1513
1514 continue;
1515 }
1516 }
1517
1518 tsnep_rx_page(rx, napi, entry->page, length);
1519 entry->page = NULL;
1520 }
1521
1522 if (xdp_status)
1523 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1524
1525 if (desc_available)
1526 tsnep_rx_refill(rx, desc_available, false);
1527
1528 return done;
1529 }
1530
tsnep_rx_poll_zc(struct tsnep_rx * rx,struct napi_struct * napi,int budget)1531 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,
1532 int budget)
1533 {
1534 struct tsnep_rx_entry *entry;
1535 struct netdev_queue *tx_nq;
1536 struct bpf_prog *prog;
1537 struct tsnep_tx *tx;
1538 int desc_available;
1539 int xdp_status = 0;
1540 struct page *page;
1541 int done = 0;
1542 int length;
1543
1544 desc_available = tsnep_rx_desc_available(rx);
1545 prog = READ_ONCE(rx->adapter->xdp_prog);
1546 if (prog) {
1547 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1548 rx->tx_queue_index);
1549 tx = &rx->adapter->tx[rx->tx_queue_index];
1550 }
1551
1552 while (likely(done < budget) && (rx->read != rx->write)) {
1553 entry = &rx->entry[rx->read];
1554 if ((__le32_to_cpu(entry->desc_wb->properties) &
1555 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1556 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1557 break;
1558 done++;
1559
1560 if (desc_available >= TSNEP_RING_RX_REFILL) {
1561 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1562
1563 desc_available -= tsnep_rx_refill_zc(rx, desc_available,
1564 reuse);
1565 if (!entry->xdp) {
1566 /* buffer has been reused for refill to prevent
1567 * empty RX ring, thus buffer cannot be used for
1568 * RX processing
1569 */
1570 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1571 desc_available++;
1572
1573 rx->dropped++;
1574
1575 continue;
1576 }
1577 }
1578
1579 /* descriptor properties shall be read first, because valid data
1580 * is signaled there
1581 */
1582 dma_rmb();
1583
1584 prefetch(entry->xdp->data);
1585 length = __le32_to_cpu(entry->desc_wb->properties) &
1586 TSNEP_DESC_LENGTH_MASK;
1587 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN);
1588 xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool);
1589
1590 /* RX metadata with timestamps is in front of actual data,
1591 * subtract metadata size to get length of actual data and
1592 * consider metadata size as offset of actual data during RX
1593 * processing
1594 */
1595 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1596
1597 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1598 desc_available++;
1599
1600 if (prog) {
1601 bool consume;
1602
1603 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE;
1604 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE;
1605
1606 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp,
1607 &xdp_status, tx_nq, tx);
1608 if (consume) {
1609 rx->packets++;
1610 rx->bytes += length;
1611
1612 entry->xdp = NULL;
1613
1614 continue;
1615 }
1616 }
1617
1618 page = page_pool_dev_alloc_pages(rx->page_pool);
1619 if (page) {
1620 memcpy(page_address(page) + TSNEP_RX_OFFSET,
1621 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE,
1622 length + TSNEP_RX_INLINE_METADATA_SIZE);
1623 tsnep_rx_page(rx, napi, page, length);
1624 } else {
1625 rx->dropped++;
1626 }
1627 xsk_buff_free(entry->xdp);
1628 entry->xdp = NULL;
1629 }
1630
1631 if (xdp_status)
1632 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1633
1634 if (desc_available)
1635 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false);
1636
1637 if (xsk_uses_need_wakeup(rx->xsk_pool)) {
1638 if (desc_available)
1639 xsk_set_rx_need_wakeup(rx->xsk_pool);
1640 else
1641 xsk_clear_rx_need_wakeup(rx->xsk_pool);
1642
1643 return done;
1644 }
1645
1646 return desc_available ? budget : done;
1647 }
1648
tsnep_rx_pending(struct tsnep_rx * rx)1649 static bool tsnep_rx_pending(struct tsnep_rx *rx)
1650 {
1651 struct tsnep_rx_entry *entry;
1652
1653 if (rx->read != rx->write) {
1654 entry = &rx->entry[rx->read];
1655 if ((__le32_to_cpu(entry->desc_wb->properties) &
1656 TSNEP_DESC_OWNER_COUNTER_MASK) ==
1657 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1658 return true;
1659 }
1660
1661 return false;
1662 }
1663
tsnep_rx_open(struct tsnep_rx * rx)1664 static int tsnep_rx_open(struct tsnep_rx *rx)
1665 {
1666 int desc_available;
1667 int retval;
1668
1669 retval = tsnep_rx_ring_create(rx);
1670 if (retval)
1671 return retval;
1672
1673 tsnep_rx_init(rx);
1674
1675 desc_available = tsnep_rx_desc_available(rx);
1676 if (rx->xsk_pool)
1677 retval = tsnep_rx_alloc_zc(rx, desc_available, false);
1678 else
1679 retval = tsnep_rx_alloc(rx, desc_available, false);
1680 if (retval != desc_available) {
1681 retval = -ENOMEM;
1682
1683 goto alloc_failed;
1684 }
1685
1686 /* prealloc pages to prevent allocation failures when XSK pool is
1687 * disabled at runtime
1688 */
1689 if (rx->xsk_pool) {
1690 retval = tsnep_rx_alloc_page_buffer(rx);
1691 if (retval)
1692 goto alloc_failed;
1693 }
1694
1695 return 0;
1696
1697 alloc_failed:
1698 tsnep_rx_ring_cleanup(rx);
1699 return retval;
1700 }
1701
tsnep_rx_close(struct tsnep_rx * rx)1702 static void tsnep_rx_close(struct tsnep_rx *rx)
1703 {
1704 if (rx->xsk_pool)
1705 tsnep_rx_free_page_buffer(rx);
1706
1707 tsnep_rx_ring_cleanup(rx);
1708 }
1709
tsnep_rx_reopen(struct tsnep_rx * rx)1710 static void tsnep_rx_reopen(struct tsnep_rx *rx)
1711 {
1712 struct page **page = rx->page_buffer;
1713 int i;
1714
1715 tsnep_rx_init(rx);
1716
1717 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1718 struct tsnep_rx_entry *entry = &rx->entry[i];
1719
1720 /* defined initial values for properties are required for
1721 * correct owner counter checking
1722 */
1723 entry->desc->properties = 0;
1724 entry->desc_wb->properties = 0;
1725
1726 /* prevent allocation failures by reusing kept pages */
1727 if (*page) {
1728 tsnep_rx_set_page(rx, entry, *page);
1729 tsnep_rx_activate(rx, rx->write);
1730 rx->write++;
1731
1732 *page = NULL;
1733 page++;
1734 }
1735 }
1736 }
1737
tsnep_rx_reopen_xsk(struct tsnep_rx * rx)1738 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx)
1739 {
1740 struct page **page = rx->page_buffer;
1741 u32 allocated;
1742 int i;
1743
1744 tsnep_rx_init(rx);
1745
1746 /* alloc all ring entries except the last one, because ring cannot be
1747 * filled completely, as many buffers as possible is enough as wakeup is
1748 * done if new buffers are available
1749 */
1750 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch,
1751 TSNEP_RING_SIZE - 1);
1752
1753 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1754 struct tsnep_rx_entry *entry = &rx->entry[i];
1755
1756 /* keep pages to prevent allocation failures when xsk is
1757 * disabled
1758 */
1759 if (entry->page) {
1760 *page = entry->page;
1761 entry->page = NULL;
1762
1763 page++;
1764 }
1765
1766 /* defined initial values for properties are required for
1767 * correct owner counter checking
1768 */
1769 entry->desc->properties = 0;
1770 entry->desc_wb->properties = 0;
1771
1772 if (allocated) {
1773 tsnep_rx_set_xdp(rx, entry,
1774 rx->xdp_batch[allocated - 1]);
1775 tsnep_rx_activate(rx, rx->write);
1776 rx->write++;
1777
1778 allocated--;
1779 }
1780 }
1781
1782 /* set need wakeup flag immediately if ring is not filled completely,
1783 * first polling would be too late as need wakeup signalisation would
1784 * be delayed for an indefinite time
1785 */
1786 if (xsk_uses_need_wakeup(rx->xsk_pool)) {
1787 int desc_available = tsnep_rx_desc_available(rx);
1788
1789 if (desc_available)
1790 xsk_set_rx_need_wakeup(rx->xsk_pool);
1791 else
1792 xsk_clear_rx_need_wakeup(rx->xsk_pool);
1793 }
1794 }
1795
tsnep_pending(struct tsnep_queue * queue)1796 static bool tsnep_pending(struct tsnep_queue *queue)
1797 {
1798 if (queue->tx && tsnep_tx_pending(queue->tx))
1799 return true;
1800
1801 if (queue->rx && tsnep_rx_pending(queue->rx))
1802 return true;
1803
1804 return false;
1805 }
1806
tsnep_poll(struct napi_struct * napi,int budget)1807 static int tsnep_poll(struct napi_struct *napi, int budget)
1808 {
1809 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue,
1810 napi);
1811 bool complete = true;
1812 int done = 0;
1813
1814 if (queue->tx)
1815 complete = tsnep_tx_poll(queue->tx, budget);
1816
1817 /* handle case where we are called by netpoll with a budget of 0 */
1818 if (unlikely(budget <= 0))
1819 return budget;
1820
1821 if (queue->rx) {
1822 done = queue->rx->xsk_pool ?
1823 tsnep_rx_poll_zc(queue->rx, napi, budget) :
1824 tsnep_rx_poll(queue->rx, napi, budget);
1825 if (done >= budget)
1826 complete = false;
1827 }
1828
1829 /* if all work not completed, return budget and keep polling */
1830 if (!complete)
1831 return budget;
1832
1833 if (likely(napi_complete_done(napi, done))) {
1834 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1835
1836 /* reschedule if work is already pending, prevent rotten packets
1837 * which are transmitted or received after polling but before
1838 * interrupt enable
1839 */
1840 if (tsnep_pending(queue)) {
1841 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1842 napi_schedule(napi);
1843 }
1844 }
1845
1846 return min(done, budget - 1);
1847 }
1848
tsnep_request_irq(struct tsnep_queue * queue,bool first)1849 static int tsnep_request_irq(struct tsnep_queue *queue, bool first)
1850 {
1851 const char *name = netdev_name(queue->adapter->netdev);
1852 irq_handler_t handler;
1853 void *dev;
1854 int retval;
1855
1856 if (first) {
1857 sprintf(queue->name, "%s-mac", name);
1858 handler = tsnep_irq;
1859 dev = queue->adapter;
1860 } else {
1861 if (queue->tx && queue->rx)
1862 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d",
1863 name, queue->rx->queue_index);
1864 else if (queue->tx)
1865 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d",
1866 name, queue->tx->queue_index);
1867 else
1868 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d",
1869 name, queue->rx->queue_index);
1870 handler = tsnep_irq_txrx;
1871 dev = queue;
1872 }
1873
1874 retval = request_irq(queue->irq, handler, 0, queue->name, dev);
1875 if (retval) {
1876 /* if name is empty, then interrupt won't be freed */
1877 memset(queue->name, 0, sizeof(queue->name));
1878 }
1879
1880 return retval;
1881 }
1882
tsnep_free_irq(struct tsnep_queue * queue,bool first)1883 static void tsnep_free_irq(struct tsnep_queue *queue, bool first)
1884 {
1885 void *dev;
1886
1887 if (!strlen(queue->name))
1888 return;
1889
1890 if (first)
1891 dev = queue->adapter;
1892 else
1893 dev = queue;
1894
1895 free_irq(queue->irq, dev);
1896 memset(queue->name, 0, sizeof(queue->name));
1897 }
1898
tsnep_queue_close(struct tsnep_queue * queue,bool first)1899 static void tsnep_queue_close(struct tsnep_queue *queue, bool first)
1900 {
1901 struct tsnep_rx *rx = queue->rx;
1902
1903 tsnep_free_irq(queue, first);
1904
1905 if (rx) {
1906 if (xdp_rxq_info_is_reg(&rx->xdp_rxq))
1907 xdp_rxq_info_unreg(&rx->xdp_rxq);
1908 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc))
1909 xdp_rxq_info_unreg(&rx->xdp_rxq_zc);
1910 }
1911
1912 netif_napi_del(&queue->napi);
1913 }
1914
tsnep_queue_open(struct tsnep_adapter * adapter,struct tsnep_queue * queue,bool first)1915 static int tsnep_queue_open(struct tsnep_adapter *adapter,
1916 struct tsnep_queue *queue, bool first)
1917 {
1918 struct tsnep_rx *rx = queue->rx;
1919 struct tsnep_tx *tx = queue->tx;
1920 int retval;
1921
1922 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll);
1923
1924 if (rx) {
1925 /* choose TX queue for XDP_TX */
1926 if (tx)
1927 rx->tx_queue_index = tx->queue_index;
1928 else if (rx->queue_index < adapter->num_tx_queues)
1929 rx->tx_queue_index = rx->queue_index;
1930 else
1931 rx->tx_queue_index = 0;
1932
1933 /* prepare both memory models to eliminate possible registration
1934 * errors when memory model is switched between page pool and
1935 * XSK pool during runtime
1936 */
1937 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev,
1938 rx->queue_index, queue->napi.napi_id);
1939 if (retval)
1940 goto failed;
1941 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq,
1942 MEM_TYPE_PAGE_POOL,
1943 rx->page_pool);
1944 if (retval)
1945 goto failed;
1946 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev,
1947 rx->queue_index, queue->napi.napi_id);
1948 if (retval)
1949 goto failed;
1950 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc,
1951 MEM_TYPE_XSK_BUFF_POOL,
1952 NULL);
1953 if (retval)
1954 goto failed;
1955 if (rx->xsk_pool)
1956 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc);
1957 }
1958
1959 retval = tsnep_request_irq(queue, first);
1960 if (retval) {
1961 netif_err(adapter, drv, adapter->netdev,
1962 "can't get assigned irq %d.\n", queue->irq);
1963 goto failed;
1964 }
1965
1966 return 0;
1967
1968 failed:
1969 tsnep_queue_close(queue, first);
1970
1971 return retval;
1972 }
1973
tsnep_queue_enable(struct tsnep_queue * queue)1974 static void tsnep_queue_enable(struct tsnep_queue *queue)
1975 {
1976 napi_enable(&queue->napi);
1977 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1978
1979 if (queue->tx)
1980 tsnep_tx_enable(queue->tx);
1981
1982 if (queue->rx)
1983 tsnep_rx_enable(queue->rx);
1984 }
1985
tsnep_queue_disable(struct tsnep_queue * queue)1986 static void tsnep_queue_disable(struct tsnep_queue *queue)
1987 {
1988 if (queue->tx)
1989 tsnep_tx_disable(queue->tx, &queue->napi);
1990
1991 napi_disable(&queue->napi);
1992 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1993
1994 /* disable RX after NAPI polling has been disabled, because RX can be
1995 * enabled during NAPI polling
1996 */
1997 if (queue->rx)
1998 tsnep_rx_disable(queue->rx);
1999 }
2000
tsnep_netdev_open(struct net_device * netdev)2001 static int tsnep_netdev_open(struct net_device *netdev)
2002 {
2003 struct tsnep_adapter *adapter = netdev_priv(netdev);
2004 int i, retval;
2005
2006 for (i = 0; i < adapter->num_queues; i++) {
2007 if (adapter->queue[i].tx) {
2008 retval = tsnep_tx_open(adapter->queue[i].tx);
2009 if (retval)
2010 goto failed;
2011 }
2012 if (adapter->queue[i].rx) {
2013 retval = tsnep_rx_open(adapter->queue[i].rx);
2014 if (retval)
2015 goto failed;
2016 }
2017
2018 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0);
2019 if (retval)
2020 goto failed;
2021 }
2022
2023 retval = netif_set_real_num_tx_queues(adapter->netdev,
2024 adapter->num_tx_queues);
2025 if (retval)
2026 goto failed;
2027 retval = netif_set_real_num_rx_queues(adapter->netdev,
2028 adapter->num_rx_queues);
2029 if (retval)
2030 goto failed;
2031
2032 tsnep_enable_irq(adapter, ECM_INT_LINK);
2033 retval = tsnep_phy_open(adapter);
2034 if (retval)
2035 goto phy_failed;
2036
2037 for (i = 0; i < adapter->num_queues; i++)
2038 tsnep_queue_enable(&adapter->queue[i]);
2039
2040 return 0;
2041
2042 phy_failed:
2043 tsnep_disable_irq(adapter, ECM_INT_LINK);
2044 failed:
2045 for (i = 0; i < adapter->num_queues; i++) {
2046 tsnep_queue_close(&adapter->queue[i], i == 0);
2047
2048 if (adapter->queue[i].rx)
2049 tsnep_rx_close(adapter->queue[i].rx);
2050 if (adapter->queue[i].tx)
2051 tsnep_tx_close(adapter->queue[i].tx);
2052 }
2053 return retval;
2054 }
2055
tsnep_netdev_close(struct net_device * netdev)2056 static int tsnep_netdev_close(struct net_device *netdev)
2057 {
2058 struct tsnep_adapter *adapter = netdev_priv(netdev);
2059 int i;
2060
2061 tsnep_disable_irq(adapter, ECM_INT_LINK);
2062 tsnep_phy_close(adapter);
2063
2064 for (i = 0; i < adapter->num_queues; i++) {
2065 tsnep_queue_disable(&adapter->queue[i]);
2066
2067 tsnep_queue_close(&adapter->queue[i], i == 0);
2068
2069 if (adapter->queue[i].rx)
2070 tsnep_rx_close(adapter->queue[i].rx);
2071 if (adapter->queue[i].tx)
2072 tsnep_tx_close(adapter->queue[i].tx);
2073 }
2074
2075 return 0;
2076 }
2077
tsnep_enable_xsk(struct tsnep_queue * queue,struct xsk_buff_pool * pool)2078 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)
2079 {
2080 bool running = netif_running(queue->adapter->netdev);
2081 u32 frame_size;
2082
2083 frame_size = xsk_pool_get_rx_frame_size(pool);
2084 if (frame_size < TSNEP_XSK_RX_BUF_SIZE)
2085 return -EOPNOTSUPP;
2086
2087 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE,
2088 sizeof(*queue->rx->page_buffer),
2089 GFP_KERNEL);
2090 if (!queue->rx->page_buffer)
2091 return -ENOMEM;
2092 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE,
2093 sizeof(*queue->rx->xdp_batch),
2094 GFP_KERNEL);
2095 if (!queue->rx->xdp_batch) {
2096 kfree(queue->rx->page_buffer);
2097 queue->rx->page_buffer = NULL;
2098
2099 return -ENOMEM;
2100 }
2101
2102 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc);
2103
2104 if (running)
2105 tsnep_queue_disable(queue);
2106
2107 queue->tx->xsk_pool = pool;
2108 queue->rx->xsk_pool = pool;
2109
2110 if (running) {
2111 tsnep_rx_reopen_xsk(queue->rx);
2112 tsnep_queue_enable(queue);
2113 }
2114
2115 return 0;
2116 }
2117
tsnep_disable_xsk(struct tsnep_queue * queue)2118 void tsnep_disable_xsk(struct tsnep_queue *queue)
2119 {
2120 bool running = netif_running(queue->adapter->netdev);
2121
2122 if (running)
2123 tsnep_queue_disable(queue);
2124
2125 tsnep_rx_free_zc(queue->rx);
2126
2127 queue->rx->xsk_pool = NULL;
2128 queue->tx->xsk_pool = NULL;
2129
2130 if (running) {
2131 tsnep_rx_reopen(queue->rx);
2132 tsnep_queue_enable(queue);
2133 }
2134
2135 kfree(queue->rx->xdp_batch);
2136 queue->rx->xdp_batch = NULL;
2137 kfree(queue->rx->page_buffer);
2138 queue->rx->page_buffer = NULL;
2139 }
2140
tsnep_netdev_xmit_frame(struct sk_buff * skb,struct net_device * netdev)2141 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb,
2142 struct net_device *netdev)
2143 {
2144 struct tsnep_adapter *adapter = netdev_priv(netdev);
2145 u16 queue_mapping = skb_get_queue_mapping(skb);
2146
2147 if (queue_mapping >= adapter->num_tx_queues)
2148 queue_mapping = 0;
2149
2150 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]);
2151 }
2152
tsnep_netdev_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2153 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr,
2154 int cmd)
2155 {
2156 if (!netif_running(netdev))
2157 return -EINVAL;
2158 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP)
2159 return tsnep_ptp_ioctl(netdev, ifr, cmd);
2160 return phy_mii_ioctl(netdev->phydev, ifr, cmd);
2161 }
2162
tsnep_netdev_set_multicast(struct net_device * netdev)2163 static void tsnep_netdev_set_multicast(struct net_device *netdev)
2164 {
2165 struct tsnep_adapter *adapter = netdev_priv(netdev);
2166
2167 u16 rx_filter = 0;
2168
2169 /* configured MAC address and broadcasts are never filtered */
2170 if (netdev->flags & IFF_PROMISC) {
2171 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2172 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS;
2173 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) {
2174 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2175 }
2176 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER);
2177 }
2178
tsnep_netdev_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)2179 static void tsnep_netdev_get_stats64(struct net_device *netdev,
2180 struct rtnl_link_stats64 *stats)
2181 {
2182 struct tsnep_adapter *adapter = netdev_priv(netdev);
2183 u32 reg;
2184 u32 val;
2185 int i;
2186
2187 for (i = 0; i < adapter->num_tx_queues; i++) {
2188 stats->tx_packets += adapter->tx[i].packets;
2189 stats->tx_bytes += adapter->tx[i].bytes;
2190 stats->tx_dropped += adapter->tx[i].dropped;
2191 }
2192 for (i = 0; i < adapter->num_rx_queues; i++) {
2193 stats->rx_packets += adapter->rx[i].packets;
2194 stats->rx_bytes += adapter->rx[i].bytes;
2195 stats->rx_dropped += adapter->rx[i].dropped;
2196 stats->multicast += adapter->rx[i].multicast;
2197
2198 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) +
2199 TSNEP_RX_STATISTIC);
2200 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >>
2201 TSNEP_RX_STATISTIC_NO_DESC_SHIFT;
2202 stats->rx_dropped += val;
2203 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >>
2204 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT;
2205 stats->rx_dropped += val;
2206 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >>
2207 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT;
2208 stats->rx_errors += val;
2209 stats->rx_fifo_errors += val;
2210 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >>
2211 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT;
2212 stats->rx_errors += val;
2213 stats->rx_frame_errors += val;
2214 }
2215
2216 reg = ioread32(adapter->addr + ECM_STAT);
2217 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT;
2218 stats->rx_errors += val;
2219 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT;
2220 stats->rx_errors += val;
2221 stats->rx_crc_errors += val;
2222 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT;
2223 stats->rx_errors += val;
2224 }
2225
tsnep_mac_set_address(struct tsnep_adapter * adapter,u8 * addr)2226 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr)
2227 {
2228 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2229 iowrite16(*(u16 *)(addr + sizeof(u32)),
2230 adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2231
2232 ether_addr_copy(adapter->mac_address, addr);
2233 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n",
2234 addr);
2235 }
2236
tsnep_netdev_set_mac_address(struct net_device * netdev,void * addr)2237 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr)
2238 {
2239 struct tsnep_adapter *adapter = netdev_priv(netdev);
2240 struct sockaddr *sock_addr = addr;
2241 int retval;
2242
2243 retval = eth_prepare_mac_addr_change(netdev, sock_addr);
2244 if (retval)
2245 return retval;
2246 eth_hw_addr_set(netdev, sock_addr->sa_data);
2247 tsnep_mac_set_address(adapter, sock_addr->sa_data);
2248
2249 return 0;
2250 }
2251
tsnep_netdev_set_features(struct net_device * netdev,netdev_features_t features)2252 static int tsnep_netdev_set_features(struct net_device *netdev,
2253 netdev_features_t features)
2254 {
2255 struct tsnep_adapter *adapter = netdev_priv(netdev);
2256 netdev_features_t changed = netdev->features ^ features;
2257 bool enable;
2258 int retval = 0;
2259
2260 if (changed & NETIF_F_LOOPBACK) {
2261 enable = !!(features & NETIF_F_LOOPBACK);
2262 retval = tsnep_phy_loopback(adapter, enable);
2263 }
2264
2265 return retval;
2266 }
2267
tsnep_netdev_get_tstamp(struct net_device * netdev,const struct skb_shared_hwtstamps * hwtstamps,bool cycles)2268 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev,
2269 const struct skb_shared_hwtstamps *hwtstamps,
2270 bool cycles)
2271 {
2272 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data;
2273 u64 timestamp;
2274
2275 if (cycles)
2276 timestamp = __le64_to_cpu(rx_inline->counter);
2277 else
2278 timestamp = __le64_to_cpu(rx_inline->timestamp);
2279
2280 return ns_to_ktime(timestamp);
2281 }
2282
tsnep_netdev_bpf(struct net_device * dev,struct netdev_bpf * bpf)2283 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf)
2284 {
2285 struct tsnep_adapter *adapter = netdev_priv(dev);
2286
2287 switch (bpf->command) {
2288 case XDP_SETUP_PROG:
2289 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack);
2290 case XDP_SETUP_XSK_POOL:
2291 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool,
2292 bpf->xsk.queue_id);
2293 default:
2294 return -EOPNOTSUPP;
2295 }
2296 }
2297
tsnep_xdp_get_tx(struct tsnep_adapter * adapter,u32 cpu)2298 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu)
2299 {
2300 if (cpu >= TSNEP_MAX_QUEUES)
2301 cpu &= TSNEP_MAX_QUEUES - 1;
2302
2303 while (cpu >= adapter->num_tx_queues)
2304 cpu -= adapter->num_tx_queues;
2305
2306 return &adapter->tx[cpu];
2307 }
2308
tsnep_netdev_xdp_xmit(struct net_device * dev,int n,struct xdp_frame ** xdp,u32 flags)2309 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n,
2310 struct xdp_frame **xdp, u32 flags)
2311 {
2312 struct tsnep_adapter *adapter = netdev_priv(dev);
2313 u32 cpu = smp_processor_id();
2314 struct netdev_queue *nq;
2315 struct tsnep_tx *tx;
2316 int nxmit;
2317 bool xmit;
2318
2319 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2320 return -EINVAL;
2321
2322 tx = tsnep_xdp_get_tx(adapter, cpu);
2323 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index);
2324
2325 __netif_tx_lock(nq, cpu);
2326
2327 for (nxmit = 0; nxmit < n; nxmit++) {
2328 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx,
2329 TSNEP_TX_TYPE_XDP_NDO);
2330 if (!xmit)
2331 break;
2332
2333 /* avoid transmit queue timeout since we share it with the slow
2334 * path
2335 */
2336 txq_trans_cond_update(nq);
2337 }
2338
2339 if (flags & XDP_XMIT_FLUSH)
2340 tsnep_xdp_xmit_flush(tx);
2341
2342 __netif_tx_unlock(nq);
2343
2344 return nxmit;
2345 }
2346
tsnep_netdev_xsk_wakeup(struct net_device * dev,u32 queue_id,u32 flags)2347 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id,
2348 u32 flags)
2349 {
2350 struct tsnep_adapter *adapter = netdev_priv(dev);
2351 struct tsnep_queue *queue;
2352
2353 if (queue_id >= adapter->num_rx_queues ||
2354 queue_id >= adapter->num_tx_queues)
2355 return -EINVAL;
2356
2357 queue = &adapter->queue[queue_id];
2358
2359 if (!napi_if_scheduled_mark_missed(&queue->napi))
2360 napi_schedule(&queue->napi);
2361
2362 return 0;
2363 }
2364
2365 static const struct net_device_ops tsnep_netdev_ops = {
2366 .ndo_open = tsnep_netdev_open,
2367 .ndo_stop = tsnep_netdev_close,
2368 .ndo_start_xmit = tsnep_netdev_xmit_frame,
2369 .ndo_eth_ioctl = tsnep_netdev_ioctl,
2370 .ndo_set_rx_mode = tsnep_netdev_set_multicast,
2371 .ndo_get_stats64 = tsnep_netdev_get_stats64,
2372 .ndo_set_mac_address = tsnep_netdev_set_mac_address,
2373 .ndo_set_features = tsnep_netdev_set_features,
2374 .ndo_get_tstamp = tsnep_netdev_get_tstamp,
2375 .ndo_setup_tc = tsnep_tc_setup,
2376 .ndo_bpf = tsnep_netdev_bpf,
2377 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit,
2378 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup,
2379 };
2380
tsnep_mac_init(struct tsnep_adapter * adapter)2381 static int tsnep_mac_init(struct tsnep_adapter *adapter)
2382 {
2383 int retval;
2384
2385 /* initialize RX filtering, at least configured MAC address and
2386 * broadcast are not filtered
2387 */
2388 iowrite16(0, adapter->addr + TSNEP_RX_FILTER);
2389
2390 /* try to get MAC address in the following order:
2391 * - device tree
2392 * - valid MAC address already set
2393 * - MAC address register if valid
2394 * - random MAC address
2395 */
2396 retval = of_get_mac_address(adapter->pdev->dev.of_node,
2397 adapter->mac_address);
2398 if (retval == -EPROBE_DEFER)
2399 return retval;
2400 if (retval && !is_valid_ether_addr(adapter->mac_address)) {
2401 *(u32 *)adapter->mac_address =
2402 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2403 *(u16 *)(adapter->mac_address + sizeof(u32)) =
2404 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2405 if (!is_valid_ether_addr(adapter->mac_address))
2406 eth_random_addr(adapter->mac_address);
2407 }
2408
2409 tsnep_mac_set_address(adapter, adapter->mac_address);
2410 eth_hw_addr_set(adapter->netdev, adapter->mac_address);
2411
2412 return 0;
2413 }
2414
tsnep_mdio_init(struct tsnep_adapter * adapter)2415 static int tsnep_mdio_init(struct tsnep_adapter *adapter)
2416 {
2417 struct device_node *np = adapter->pdev->dev.of_node;
2418 int retval;
2419
2420 if (np) {
2421 np = of_get_child_by_name(np, "mdio");
2422 if (!np)
2423 return 0;
2424
2425 adapter->suppress_preamble =
2426 of_property_read_bool(np, "suppress-preamble");
2427 }
2428
2429 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev);
2430 if (!adapter->mdiobus) {
2431 retval = -ENOMEM;
2432
2433 goto out;
2434 }
2435
2436 adapter->mdiobus->priv = (void *)adapter;
2437 adapter->mdiobus->parent = &adapter->pdev->dev;
2438 adapter->mdiobus->read = tsnep_mdiobus_read;
2439 adapter->mdiobus->write = tsnep_mdiobus_write;
2440 adapter->mdiobus->name = TSNEP "-mdiobus";
2441 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s",
2442 adapter->pdev->name);
2443
2444 /* do not scan broadcast address */
2445 adapter->mdiobus->phy_mask = 0x0000001;
2446
2447 retval = of_mdiobus_register(adapter->mdiobus, np);
2448
2449 out:
2450 of_node_put(np);
2451
2452 return retval;
2453 }
2454
tsnep_phy_init(struct tsnep_adapter * adapter)2455 static int tsnep_phy_init(struct tsnep_adapter *adapter)
2456 {
2457 struct device_node *phy_node;
2458 int retval;
2459
2460 retval = of_get_phy_mode(adapter->pdev->dev.of_node,
2461 &adapter->phy_mode);
2462 if (retval)
2463 adapter->phy_mode = PHY_INTERFACE_MODE_GMII;
2464
2465 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle",
2466 0);
2467 adapter->phydev = of_phy_find_device(phy_node);
2468 of_node_put(phy_node);
2469 if (!adapter->phydev && adapter->mdiobus)
2470 adapter->phydev = phy_find_first(adapter->mdiobus);
2471 if (!adapter->phydev)
2472 return -EIO;
2473
2474 return 0;
2475 }
2476
tsnep_queue_init(struct tsnep_adapter * adapter,int queue_count)2477 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
2478 {
2479 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;
2480 char name[8];
2481 int i;
2482 int retval;
2483
2484 /* one TX/RX queue pair for netdev is mandatory */
2485 if (platform_irq_count(adapter->pdev) == 1)
2486 retval = platform_get_irq(adapter->pdev, 0);
2487 else
2488 retval = platform_get_irq_byname(adapter->pdev, "mac");
2489 if (retval < 0)
2490 return retval;
2491 adapter->num_tx_queues = 1;
2492 adapter->num_rx_queues = 1;
2493 adapter->num_queues = 1;
2494 adapter->queue[0].adapter = adapter;
2495 adapter->queue[0].irq = retval;
2496 adapter->queue[0].tx = &adapter->tx[0];
2497 adapter->queue[0].tx->adapter = adapter;
2498 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0);
2499 adapter->queue[0].tx->queue_index = 0;
2500 adapter->queue[0].rx = &adapter->rx[0];
2501 adapter->queue[0].rx->adapter = adapter;
2502 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0);
2503 adapter->queue[0].rx->queue_index = 0;
2504 adapter->queue[0].irq_mask = irq_mask;
2505 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY;
2506 retval = tsnep_set_irq_coalesce(&adapter->queue[0],
2507 TSNEP_COALESCE_USECS_DEFAULT);
2508 if (retval < 0)
2509 return retval;
2510
2511 adapter->netdev->irq = adapter->queue[0].irq;
2512
2513 /* add additional TX/RX queue pairs only if dedicated interrupt is
2514 * available
2515 */
2516 for (i = 1; i < queue_count; i++) {
2517 sprintf(name, "txrx-%d", i);
2518 retval = platform_get_irq_byname_optional(adapter->pdev, name);
2519 if (retval < 0)
2520 break;
2521
2522 adapter->num_tx_queues++;
2523 adapter->num_rx_queues++;
2524 adapter->num_queues++;
2525 adapter->queue[i].adapter = adapter;
2526 adapter->queue[i].irq = retval;
2527 adapter->queue[i].tx = &adapter->tx[i];
2528 adapter->queue[i].tx->adapter = adapter;
2529 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i);
2530 adapter->queue[i].tx->queue_index = i;
2531 adapter->queue[i].rx = &adapter->rx[i];
2532 adapter->queue[i].rx->adapter = adapter;
2533 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i);
2534 adapter->queue[i].rx->queue_index = i;
2535 adapter->queue[i].irq_mask =
2536 irq_mask << (ECM_INT_TXRX_SHIFT * i);
2537 adapter->queue[i].irq_delay_addr =
2538 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i;
2539 retval = tsnep_set_irq_coalesce(&adapter->queue[i],
2540 TSNEP_COALESCE_USECS_DEFAULT);
2541 if (retval < 0)
2542 return retval;
2543 }
2544
2545 return 0;
2546 }
2547
tsnep_probe(struct platform_device * pdev)2548 static int tsnep_probe(struct platform_device *pdev)
2549 {
2550 struct tsnep_adapter *adapter;
2551 struct net_device *netdev;
2552 struct resource *io;
2553 u32 type;
2554 int revision;
2555 int version;
2556 int queue_count;
2557 int retval;
2558
2559 netdev = devm_alloc_etherdev_mqs(&pdev->dev,
2560 sizeof(struct tsnep_adapter),
2561 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES);
2562 if (!netdev)
2563 return -ENODEV;
2564 SET_NETDEV_DEV(netdev, &pdev->dev);
2565 adapter = netdev_priv(netdev);
2566 platform_set_drvdata(pdev, adapter);
2567 adapter->pdev = pdev;
2568 adapter->dmadev = &pdev->dev;
2569 adapter->netdev = netdev;
2570 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
2571 NETIF_MSG_LINK | NETIF_MSG_IFUP |
2572 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
2573
2574 netdev->min_mtu = ETH_MIN_MTU;
2575 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE;
2576
2577 mutex_init(&adapter->gate_control_lock);
2578 mutex_init(&adapter->rxnfc_lock);
2579 INIT_LIST_HEAD(&adapter->rxnfc_rules);
2580
2581 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2582 adapter->addr = devm_ioremap_resource(&pdev->dev, io);
2583 if (IS_ERR(adapter->addr))
2584 return PTR_ERR(adapter->addr);
2585 netdev->mem_start = io->start;
2586 netdev->mem_end = io->end;
2587
2588 type = ioread32(adapter->addr + ECM_TYPE);
2589 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT;
2590 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT;
2591 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT;
2592 adapter->gate_control = type & ECM_GATE_CONTROL;
2593 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT;
2594
2595 tsnep_disable_irq(adapter, ECM_INT_ALL);
2596
2597 retval = tsnep_queue_init(adapter, queue_count);
2598 if (retval)
2599 return retval;
2600
2601 retval = dma_set_mask_and_coherent(&adapter->pdev->dev,
2602 DMA_BIT_MASK(64));
2603 if (retval) {
2604 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n");
2605 return retval;
2606 }
2607
2608 retval = tsnep_mac_init(adapter);
2609 if (retval)
2610 return retval;
2611
2612 retval = tsnep_mdio_init(adapter);
2613 if (retval)
2614 goto mdio_init_failed;
2615
2616 retval = tsnep_phy_init(adapter);
2617 if (retval)
2618 goto phy_init_failed;
2619
2620 retval = tsnep_ptp_init(adapter);
2621 if (retval)
2622 goto ptp_init_failed;
2623
2624 retval = tsnep_tc_init(adapter);
2625 if (retval)
2626 goto tc_init_failed;
2627
2628 retval = tsnep_rxnfc_init(adapter);
2629 if (retval)
2630 goto rxnfc_init_failed;
2631
2632 netdev->netdev_ops = &tsnep_netdev_ops;
2633 netdev->ethtool_ops = &tsnep_ethtool_ops;
2634 netdev->features = NETIF_F_SG;
2635 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK;
2636
2637 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
2638 NETDEV_XDP_ACT_NDO_XMIT |
2639 NETDEV_XDP_ACT_NDO_XMIT_SG |
2640 NETDEV_XDP_ACT_XSK_ZEROCOPY;
2641
2642 /* carrier off reporting is important to ethtool even BEFORE open */
2643 netif_carrier_off(netdev);
2644
2645 retval = register_netdev(netdev);
2646 if (retval)
2647 goto register_failed;
2648
2649 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version,
2650 revision);
2651 if (adapter->gate_control)
2652 dev_info(&adapter->pdev->dev, "gate control detected\n");
2653
2654 return 0;
2655
2656 register_failed:
2657 tsnep_rxnfc_cleanup(adapter);
2658 rxnfc_init_failed:
2659 tsnep_tc_cleanup(adapter);
2660 tc_init_failed:
2661 tsnep_ptp_cleanup(adapter);
2662 ptp_init_failed:
2663 phy_init_failed:
2664 if (adapter->mdiobus)
2665 mdiobus_unregister(adapter->mdiobus);
2666 mdio_init_failed:
2667 return retval;
2668 }
2669
tsnep_remove(struct platform_device * pdev)2670 static int tsnep_remove(struct platform_device *pdev)
2671 {
2672 struct tsnep_adapter *adapter = platform_get_drvdata(pdev);
2673
2674 unregister_netdev(adapter->netdev);
2675
2676 tsnep_rxnfc_cleanup(adapter);
2677
2678 tsnep_tc_cleanup(adapter);
2679
2680 tsnep_ptp_cleanup(adapter);
2681
2682 if (adapter->mdiobus)
2683 mdiobus_unregister(adapter->mdiobus);
2684
2685 tsnep_disable_irq(adapter, ECM_INT_ALL);
2686
2687 return 0;
2688 }
2689
2690 static const struct of_device_id tsnep_of_match[] = {
2691 { .compatible = "engleder,tsnep", },
2692 { },
2693 };
2694 MODULE_DEVICE_TABLE(of, tsnep_of_match);
2695
2696 static struct platform_driver tsnep_driver = {
2697 .driver = {
2698 .name = TSNEP,
2699 .of_match_table = tsnep_of_match,
2700 },
2701 .probe = tsnep_probe,
2702 .remove = tsnep_remove,
2703 };
2704 module_platform_driver(tsnep_driver);
2705
2706 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>");
2707 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver");
2708 MODULE_LICENSE("GPL");
2709