1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ 3 4 /* TSN endpoint Ethernet MAC driver 5 * 6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time 7 * communication. It is designed for endpoints within TSN (Time Sensitive 8 * Networking) networks; e.g., for PLCs in the industrial automation case. 9 * 10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used 11 * by the driver. 12 * 13 * More information can be found here: 14 * - www.embedded-experts.at/tsn 15 * - www.engleder-embedded.com 16 */ 17 18 #include "tsnep.h" 19 #include "tsnep_hw.h" 20 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_net.h> 24 #include <linux/of_mdio.h> 25 #include <linux/interrupt.h> 26 #include <linux/etherdevice.h> 27 #include <linux/phy.h> 28 #include <linux/iopoll.h> 29 #include <linux/bpf.h> 30 #include <linux/bpf_trace.h> 31 #include <net/page_pool/helpers.h> 32 #include <net/xdp_sock_drv.h> 33 34 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN) 35 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4) 36 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \ 37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 38 /* XSK buffer shall store at least Q-in-Q frame */ 39 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \ 40 ETH_FRAME_LEN + ETH_FCS_LEN + \ 41 VLAN_HLEN * 2, 4)) 42 43 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 44 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF)) 45 #else 46 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0)) 47 #endif 48 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF)) 49 50 #define TSNEP_COALESCE_USECS_DEFAULT 64 51 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \ 52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1) 53 54 #define TSNEP_TX_TYPE_SKB BIT(0) 55 #define TSNEP_TX_TYPE_SKB_FRAG BIT(1) 56 #define TSNEP_TX_TYPE_XDP_TX BIT(2) 57 #define TSNEP_TX_TYPE_XDP_NDO BIT(3) 58 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO) 59 #define TSNEP_TX_TYPE_XSK BIT(4) 60 61 #define TSNEP_XDP_TX BIT(0) 62 #define TSNEP_XDP_REDIRECT BIT(1) 63 64 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask) 65 { 66 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 67 } 68 69 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask) 70 { 71 mask |= ECM_INT_DISABLE; 72 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 73 } 74 75 static irqreturn_t tsnep_irq(int irq, void *arg) 76 { 77 struct tsnep_adapter *adapter = arg; 78 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE); 79 80 /* acknowledge interrupt */ 81 if (active != 0) 82 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE); 83 84 /* handle link interrupt */ 85 if ((active & ECM_INT_LINK) != 0) 86 phy_mac_interrupt(adapter->netdev->phydev); 87 88 /* handle TX/RX queue 0 interrupt */ 89 if ((active & adapter->queue[0].irq_mask) != 0) { 90 if (napi_schedule_prep(&adapter->queue[0].napi)) { 91 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); 92 /* schedule after masking to avoid races */ 93 __napi_schedule(&adapter->queue[0].napi); 94 } 95 } 96 97 return IRQ_HANDLED; 98 } 99 100 static irqreturn_t tsnep_irq_txrx(int irq, void *arg) 101 { 102 struct tsnep_queue *queue = arg; 103 104 /* handle TX/RX queue interrupt */ 105 if (napi_schedule_prep(&queue->napi)) { 106 tsnep_disable_irq(queue->adapter, queue->irq_mask); 107 /* schedule after masking to avoid races */ 108 __napi_schedule(&queue->napi); 109 } 110 111 return IRQ_HANDLED; 112 } 113 114 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs) 115 { 116 if (usecs > TSNEP_COALESCE_USECS_MAX) 117 return -ERANGE; 118 119 usecs /= ECM_INT_DELAY_BASE_US; 120 usecs <<= ECM_INT_DELAY_SHIFT; 121 usecs &= ECM_INT_DELAY_MASK; 122 123 queue->irq_delay &= ~ECM_INT_DELAY_MASK; 124 queue->irq_delay |= usecs; 125 iowrite8(queue->irq_delay, queue->irq_delay_addr); 126 127 return 0; 128 } 129 130 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue) 131 { 132 u32 usecs; 133 134 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK); 135 usecs >>= ECM_INT_DELAY_SHIFT; 136 usecs *= ECM_INT_DELAY_BASE_US; 137 138 return usecs; 139 } 140 141 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) 142 { 143 struct tsnep_adapter *adapter = bus->priv; 144 u32 md; 145 int retval; 146 147 md = ECM_MD_READ; 148 if (!adapter->suppress_preamble) 149 md |= ECM_MD_PREAMBLE; 150 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 151 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 152 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 153 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 154 !(md & ECM_MD_BUSY), 16, 1000); 155 if (retval != 0) 156 return retval; 157 158 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT; 159 } 160 161 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum, 162 u16 val) 163 { 164 struct tsnep_adapter *adapter = bus->priv; 165 u32 md; 166 int retval; 167 168 md = ECM_MD_WRITE; 169 if (!adapter->suppress_preamble) 170 md |= ECM_MD_PREAMBLE; 171 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 172 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 173 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK; 174 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 175 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 176 !(md & ECM_MD_BUSY), 16, 1000); 177 if (retval != 0) 178 return retval; 179 180 return 0; 181 } 182 183 static void tsnep_set_link_mode(struct tsnep_adapter *adapter) 184 { 185 u32 mode; 186 187 switch (adapter->phydev->speed) { 188 case SPEED_100: 189 mode = ECM_LINK_MODE_100; 190 break; 191 case SPEED_1000: 192 mode = ECM_LINK_MODE_1000; 193 break; 194 default: 195 mode = ECM_LINK_MODE_OFF; 196 break; 197 } 198 iowrite32(mode, adapter->addr + ECM_STATUS); 199 } 200 201 static void tsnep_phy_link_status_change(struct net_device *netdev) 202 { 203 struct tsnep_adapter *adapter = netdev_priv(netdev); 204 struct phy_device *phydev = netdev->phydev; 205 206 if (phydev->link) 207 tsnep_set_link_mode(adapter); 208 209 phy_print_status(netdev->phydev); 210 } 211 212 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable) 213 { 214 int retval; 215 216 retval = phy_loopback(adapter->phydev, enable); 217 218 /* PHY link state change is not signaled if loopback is enabled, it 219 * would delay a working loopback anyway, let's ensure that loopback 220 * is working immediately by setting link mode directly 221 */ 222 if (!retval && enable) 223 tsnep_set_link_mode(adapter); 224 225 return retval; 226 } 227 228 static int tsnep_phy_open(struct tsnep_adapter *adapter) 229 { 230 struct phy_device *phydev; 231 struct ethtool_eee ethtool_eee; 232 int retval; 233 234 retval = phy_connect_direct(adapter->netdev, adapter->phydev, 235 tsnep_phy_link_status_change, 236 adapter->phy_mode); 237 if (retval) 238 return retval; 239 phydev = adapter->netdev->phydev; 240 241 /* MAC supports only 100Mbps|1000Mbps full duplex 242 * SPE (Single Pair Ethernet) is also an option but not implemented yet 243 */ 244 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); 245 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); 246 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); 247 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 248 249 /* disable EEE autoneg, EEE not supported by TSNEP */ 250 memset(ðtool_eee, 0, sizeof(ethtool_eee)); 251 phy_ethtool_set_eee(adapter->phydev, ðtool_eee); 252 253 adapter->phydev->irq = PHY_MAC_INTERRUPT; 254 phy_start(adapter->phydev); 255 256 return 0; 257 } 258 259 static void tsnep_phy_close(struct tsnep_adapter *adapter) 260 { 261 phy_stop(adapter->netdev->phydev); 262 phy_disconnect(adapter->netdev->phydev); 263 } 264 265 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx) 266 { 267 struct device *dmadev = tx->adapter->dmadev; 268 int i; 269 270 memset(tx->entry, 0, sizeof(tx->entry)); 271 272 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 273 if (tx->page[i]) { 274 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i], 275 tx->page_dma[i]); 276 tx->page[i] = NULL; 277 tx->page_dma[i] = 0; 278 } 279 } 280 } 281 282 static int tsnep_tx_ring_create(struct tsnep_tx *tx) 283 { 284 struct device *dmadev = tx->adapter->dmadev; 285 struct tsnep_tx_entry *entry; 286 struct tsnep_tx_entry *next_entry; 287 int i, j; 288 int retval; 289 290 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 291 tx->page[i] = 292 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i], 293 GFP_KERNEL); 294 if (!tx->page[i]) { 295 retval = -ENOMEM; 296 goto alloc_failed; 297 } 298 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 299 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 300 entry->desc_wb = (struct tsnep_tx_desc_wb *) 301 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j); 302 entry->desc = (struct tsnep_tx_desc *) 303 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 304 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j; 305 entry->owner_user_flag = false; 306 } 307 } 308 for (i = 0; i < TSNEP_RING_SIZE; i++) { 309 entry = &tx->entry[i]; 310 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK]; 311 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 312 } 313 314 return 0; 315 316 alloc_failed: 317 tsnep_tx_ring_cleanup(tx); 318 return retval; 319 } 320 321 static void tsnep_tx_init(struct tsnep_tx *tx) 322 { 323 dma_addr_t dma; 324 325 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 326 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW); 327 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH); 328 tx->write = 0; 329 tx->read = 0; 330 tx->owner_counter = 1; 331 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 332 } 333 334 static void tsnep_tx_enable(struct tsnep_tx *tx) 335 { 336 struct netdev_queue *nq; 337 338 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 339 340 __netif_tx_lock_bh(nq); 341 netif_tx_wake_queue(nq); 342 __netif_tx_unlock_bh(nq); 343 } 344 345 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi) 346 { 347 struct netdev_queue *nq; 348 u32 val; 349 350 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 351 352 __netif_tx_lock_bh(nq); 353 netif_tx_stop_queue(nq); 354 __netif_tx_unlock_bh(nq); 355 356 /* wait until TX is done in hardware */ 357 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val, 358 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000, 359 1000000); 360 361 /* wait until TX is also done in software */ 362 while (READ_ONCE(tx->read) != tx->write) { 363 napi_schedule(napi); 364 napi_synchronize(napi); 365 } 366 } 367 368 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, 369 bool last) 370 { 371 struct tsnep_tx_entry *entry = &tx->entry[index]; 372 373 entry->properties = 0; 374 /* xdpf and zc are union with skb */ 375 if (entry->skb) { 376 entry->properties = length & TSNEP_DESC_LENGTH_MASK; 377 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 378 if ((entry->type & TSNEP_TX_TYPE_SKB) && 379 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS)) 380 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; 381 382 /* toggle user flag to prevent false acknowledge 383 * 384 * Only the first fragment is acknowledged. For all other 385 * fragments no acknowledge is done and the last written owner 386 * counter stays in the writeback descriptor. Therefore, it is 387 * possible that the last written owner counter is identical to 388 * the new incremented owner counter and a false acknowledge is 389 * detected before the real acknowledge has been done by 390 * hardware. 391 * 392 * The user flag is used to prevent this situation. The user 393 * flag is copied to the writeback descriptor by the hardware 394 * and is used as additional acknowledge data. By toggeling the 395 * user flag only for the first fragment (which is 396 * acknowledged), it is guaranteed that the last acknowledge 397 * done for this descriptor has used a different user flag and 398 * cannot be detected as false acknowledge. 399 */ 400 entry->owner_user_flag = !entry->owner_user_flag; 401 } 402 if (last) 403 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG; 404 if (index == tx->increment_owner_counter) { 405 tx->owner_counter++; 406 if (tx->owner_counter == 4) 407 tx->owner_counter = 1; 408 tx->increment_owner_counter--; 409 if (tx->increment_owner_counter < 0) 410 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 411 } 412 entry->properties |= 413 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 414 TSNEP_DESC_OWNER_COUNTER_MASK; 415 if (entry->owner_user_flag) 416 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG; 417 entry->desc->more_properties = 418 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK); 419 420 /* descriptor properties shall be written last, because valid data is 421 * signaled there 422 */ 423 dma_wmb(); 424 425 entry->desc->properties = __cpu_to_le32(entry->properties); 426 } 427 428 static int tsnep_tx_desc_available(struct tsnep_tx *tx) 429 { 430 if (tx->read <= tx->write) 431 return TSNEP_RING_SIZE - tx->write + tx->read - 1; 432 else 433 return tx->read - tx->write - 1; 434 } 435 436 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) 437 { 438 struct device *dmadev = tx->adapter->dmadev; 439 struct tsnep_tx_entry *entry; 440 unsigned int len; 441 dma_addr_t dma; 442 int map_len = 0; 443 int i; 444 445 for (i = 0; i < count; i++) { 446 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 447 448 if (!i) { 449 len = skb_headlen(skb); 450 dma = dma_map_single(dmadev, skb->data, len, 451 DMA_TO_DEVICE); 452 453 entry->type = TSNEP_TX_TYPE_SKB; 454 } else { 455 len = skb_frag_size(&skb_shinfo(skb)->frags[i - 1]); 456 dma = skb_frag_dma_map(dmadev, 457 &skb_shinfo(skb)->frags[i - 1], 458 0, len, DMA_TO_DEVICE); 459 460 entry->type = TSNEP_TX_TYPE_SKB_FRAG; 461 } 462 if (dma_mapping_error(dmadev, dma)) 463 return -ENOMEM; 464 465 entry->len = len; 466 dma_unmap_addr_set(entry, dma, dma); 467 468 entry->desc->tx = __cpu_to_le64(dma); 469 470 map_len += len; 471 } 472 473 return map_len; 474 } 475 476 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) 477 { 478 struct device *dmadev = tx->adapter->dmadev; 479 struct tsnep_tx_entry *entry; 480 int map_len = 0; 481 int i; 482 483 for (i = 0; i < count; i++) { 484 entry = &tx->entry[(index + i) & TSNEP_RING_MASK]; 485 486 if (entry->len) { 487 if (entry->type & TSNEP_TX_TYPE_SKB) 488 dma_unmap_single(dmadev, 489 dma_unmap_addr(entry, dma), 490 dma_unmap_len(entry, len), 491 DMA_TO_DEVICE); 492 else if (entry->type & 493 (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_XDP_NDO)) 494 dma_unmap_page(dmadev, 495 dma_unmap_addr(entry, dma), 496 dma_unmap_len(entry, len), 497 DMA_TO_DEVICE); 498 map_len += entry->len; 499 entry->len = 0; 500 } 501 } 502 503 return map_len; 504 } 505 506 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, 507 struct tsnep_tx *tx) 508 { 509 int count = 1; 510 struct tsnep_tx_entry *entry; 511 int length; 512 int i; 513 int retval; 514 515 if (skb_shinfo(skb)->nr_frags > 0) 516 count += skb_shinfo(skb)->nr_frags; 517 518 if (tsnep_tx_desc_available(tx) < count) { 519 /* ring full, shall not happen because queue is stopped if full 520 * below 521 */ 522 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 523 524 return NETDEV_TX_BUSY; 525 } 526 527 entry = &tx->entry[tx->write]; 528 entry->skb = skb; 529 530 retval = tsnep_tx_map(skb, tx, count); 531 if (retval < 0) { 532 tsnep_tx_unmap(tx, tx->write, count); 533 dev_kfree_skb_any(entry->skb); 534 entry->skb = NULL; 535 536 tx->dropped++; 537 538 return NETDEV_TX_OK; 539 } 540 length = retval; 541 542 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 543 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 544 545 for (i = 0; i < count; i++) 546 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 547 i == count - 1); 548 tx->write = (tx->write + count) & TSNEP_RING_MASK; 549 550 skb_tx_timestamp(skb); 551 552 /* descriptor properties shall be valid before hardware is notified */ 553 dma_wmb(); 554 555 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 556 557 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) { 558 /* ring can get full with next frame */ 559 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 560 } 561 562 return NETDEV_TX_OK; 563 } 564 565 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx, 566 struct skb_shared_info *shinfo, int count, u32 type) 567 { 568 struct device *dmadev = tx->adapter->dmadev; 569 struct tsnep_tx_entry *entry; 570 struct page *page; 571 skb_frag_t *frag; 572 unsigned int len; 573 int map_len = 0; 574 dma_addr_t dma; 575 void *data; 576 int i; 577 578 frag = NULL; 579 len = xdpf->len; 580 for (i = 0; i < count; i++) { 581 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 582 if (type & TSNEP_TX_TYPE_XDP_NDO) { 583 data = unlikely(frag) ? skb_frag_address(frag) : 584 xdpf->data; 585 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE); 586 if (dma_mapping_error(dmadev, dma)) 587 return -ENOMEM; 588 589 entry->type = TSNEP_TX_TYPE_XDP_NDO; 590 } else { 591 page = unlikely(frag) ? skb_frag_page(frag) : 592 virt_to_page(xdpf->data); 593 dma = page_pool_get_dma_addr(page); 594 if (unlikely(frag)) 595 dma += skb_frag_off(frag); 596 else 597 dma += sizeof(*xdpf) + xdpf->headroom; 598 dma_sync_single_for_device(dmadev, dma, len, 599 DMA_BIDIRECTIONAL); 600 601 entry->type = TSNEP_TX_TYPE_XDP_TX; 602 } 603 604 entry->len = len; 605 dma_unmap_addr_set(entry, dma, dma); 606 607 entry->desc->tx = __cpu_to_le64(dma); 608 609 map_len += len; 610 611 if (i + 1 < count) { 612 frag = &shinfo->frags[i]; 613 len = skb_frag_size(frag); 614 } 615 } 616 617 return map_len; 618 } 619 620 /* This function requires __netif_tx_lock is held by the caller. */ 621 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf, 622 struct tsnep_tx *tx, u32 type) 623 { 624 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf); 625 struct tsnep_tx_entry *entry; 626 int count, length, retval, i; 627 628 count = 1; 629 if (unlikely(xdp_frame_has_frags(xdpf))) 630 count += shinfo->nr_frags; 631 632 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 633 * will be available for normal TX path and queue is stopped there if 634 * necessary 635 */ 636 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count)) 637 return false; 638 639 entry = &tx->entry[tx->write]; 640 entry->xdpf = xdpf; 641 642 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type); 643 if (retval < 0) { 644 tsnep_tx_unmap(tx, tx->write, count); 645 entry->xdpf = NULL; 646 647 tx->dropped++; 648 649 return false; 650 } 651 length = retval; 652 653 for (i = 0; i < count; i++) 654 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 655 i == count - 1); 656 tx->write = (tx->write + count) & TSNEP_RING_MASK; 657 658 /* descriptor properties shall be valid before hardware is notified */ 659 dma_wmb(); 660 661 return true; 662 } 663 664 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx) 665 { 666 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 667 } 668 669 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter, 670 struct xdp_buff *xdp, 671 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 672 { 673 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 674 bool xmit; 675 676 if (unlikely(!xdpf)) 677 return false; 678 679 __netif_tx_lock(tx_nq, smp_processor_id()); 680 681 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, TSNEP_TX_TYPE_XDP_TX); 682 683 /* Avoid transmit queue timeout since we share it with the slow path */ 684 if (xmit) 685 txq_trans_cond_update(tx_nq); 686 687 __netif_tx_unlock(tx_nq); 688 689 return xmit; 690 } 691 692 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx) 693 { 694 struct tsnep_tx_entry *entry; 695 dma_addr_t dma; 696 697 entry = &tx->entry[tx->write]; 698 entry->zc = true; 699 700 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr); 701 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len); 702 703 entry->type = TSNEP_TX_TYPE_XSK; 704 entry->len = xdpd->len; 705 706 entry->desc->tx = __cpu_to_le64(dma); 707 708 return xdpd->len; 709 } 710 711 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd, 712 struct tsnep_tx *tx) 713 { 714 int length; 715 716 length = tsnep_xdp_tx_map_zc(xdpd, tx); 717 718 tsnep_tx_activate(tx, tx->write, length, true); 719 tx->write = (tx->write + 1) & TSNEP_RING_MASK; 720 } 721 722 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx) 723 { 724 int desc_available = tsnep_tx_desc_available(tx); 725 struct xdp_desc *descs = tx->xsk_pool->tx_descs; 726 int batch, i; 727 728 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 729 * will be available for normal TX path and queue is stopped there if 730 * necessary 731 */ 732 if (desc_available <= (MAX_SKB_FRAGS + 1)) 733 return; 734 desc_available -= MAX_SKB_FRAGS + 1; 735 736 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available); 737 for (i = 0; i < batch; i++) 738 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx); 739 740 if (batch) { 741 /* descriptor properties shall be valid before hardware is 742 * notified 743 */ 744 dma_wmb(); 745 746 tsnep_xdp_xmit_flush(tx); 747 } 748 } 749 750 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) 751 { 752 struct tsnep_tx_entry *entry; 753 struct netdev_queue *nq; 754 int xsk_frames = 0; 755 int budget = 128; 756 int length; 757 int count; 758 759 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 760 __netif_tx_lock(nq, smp_processor_id()); 761 762 do { 763 if (tx->read == tx->write) 764 break; 765 766 entry = &tx->entry[tx->read]; 767 if ((__le32_to_cpu(entry->desc_wb->properties) & 768 TSNEP_TX_DESC_OWNER_MASK) != 769 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 770 break; 771 772 /* descriptor properties shall be read first, because valid data 773 * is signaled there 774 */ 775 dma_rmb(); 776 777 count = 1; 778 if ((entry->type & TSNEP_TX_TYPE_SKB) && 779 skb_shinfo(entry->skb)->nr_frags > 0) 780 count += skb_shinfo(entry->skb)->nr_frags; 781 else if ((entry->type & TSNEP_TX_TYPE_XDP) && 782 xdp_frame_has_frags(entry->xdpf)) 783 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags; 784 785 length = tsnep_tx_unmap(tx, tx->read, count); 786 787 if ((entry->type & TSNEP_TX_TYPE_SKB) && 788 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && 789 (__le32_to_cpu(entry->desc_wb->properties) & 790 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) { 791 struct skb_shared_hwtstamps hwtstamps; 792 u64 timestamp; 793 794 if (skb_shinfo(entry->skb)->tx_flags & 795 SKBTX_HW_TSTAMP_USE_CYCLES) 796 timestamp = 797 __le64_to_cpu(entry->desc_wb->counter); 798 else 799 timestamp = 800 __le64_to_cpu(entry->desc_wb->timestamp); 801 802 memset(&hwtstamps, 0, sizeof(hwtstamps)); 803 hwtstamps.hwtstamp = ns_to_ktime(timestamp); 804 805 skb_tstamp_tx(entry->skb, &hwtstamps); 806 } 807 808 if (entry->type & TSNEP_TX_TYPE_SKB) 809 napi_consume_skb(entry->skb, napi_budget); 810 else if (entry->type & TSNEP_TX_TYPE_XDP) 811 xdp_return_frame_rx_napi(entry->xdpf); 812 else 813 xsk_frames++; 814 /* xdpf and zc are union with skb */ 815 entry->skb = NULL; 816 817 tx->read = (tx->read + count) & TSNEP_RING_MASK; 818 819 tx->packets++; 820 tx->bytes += length + ETH_FCS_LEN; 821 822 budget--; 823 } while (likely(budget)); 824 825 if (tx->xsk_pool) { 826 if (xsk_frames) 827 xsk_tx_completed(tx->xsk_pool, xsk_frames); 828 if (xsk_uses_need_wakeup(tx->xsk_pool)) 829 xsk_set_tx_need_wakeup(tx->xsk_pool); 830 tsnep_xdp_xmit_zc(tx); 831 } 832 833 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) && 834 netif_tx_queue_stopped(nq)) { 835 netif_tx_wake_queue(nq); 836 } 837 838 __netif_tx_unlock(nq); 839 840 return budget != 0; 841 } 842 843 static bool tsnep_tx_pending(struct tsnep_tx *tx) 844 { 845 struct tsnep_tx_entry *entry; 846 struct netdev_queue *nq; 847 bool pending = false; 848 849 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 850 __netif_tx_lock(nq, smp_processor_id()); 851 852 if (tx->read != tx->write) { 853 entry = &tx->entry[tx->read]; 854 if ((__le32_to_cpu(entry->desc_wb->properties) & 855 TSNEP_TX_DESC_OWNER_MASK) == 856 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 857 pending = true; 858 } 859 860 __netif_tx_unlock(nq); 861 862 return pending; 863 } 864 865 static int tsnep_tx_open(struct tsnep_tx *tx) 866 { 867 int retval; 868 869 retval = tsnep_tx_ring_create(tx); 870 if (retval) 871 return retval; 872 873 tsnep_tx_init(tx); 874 875 return 0; 876 } 877 878 static void tsnep_tx_close(struct tsnep_tx *tx) 879 { 880 tsnep_tx_ring_cleanup(tx); 881 } 882 883 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) 884 { 885 struct device *dmadev = rx->adapter->dmadev; 886 struct tsnep_rx_entry *entry; 887 int i; 888 889 for (i = 0; i < TSNEP_RING_SIZE; i++) { 890 entry = &rx->entry[i]; 891 if (!rx->xsk_pool && entry->page) 892 page_pool_put_full_page(rx->page_pool, entry->page, 893 false); 894 if (rx->xsk_pool && entry->xdp) 895 xsk_buff_free(entry->xdp); 896 /* xdp is union with page */ 897 entry->page = NULL; 898 } 899 900 if (rx->page_pool) 901 page_pool_destroy(rx->page_pool); 902 903 memset(rx->entry, 0, sizeof(rx->entry)); 904 905 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 906 if (rx->page[i]) { 907 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i], 908 rx->page_dma[i]); 909 rx->page[i] = NULL; 910 rx->page_dma[i] = 0; 911 } 912 } 913 } 914 915 static int tsnep_rx_ring_create(struct tsnep_rx *rx) 916 { 917 struct device *dmadev = rx->adapter->dmadev; 918 struct tsnep_rx_entry *entry; 919 struct page_pool_params pp_params = { 0 }; 920 struct tsnep_rx_entry *next_entry; 921 int i, j; 922 int retval; 923 924 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 925 rx->page[i] = 926 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i], 927 GFP_KERNEL); 928 if (!rx->page[i]) { 929 retval = -ENOMEM; 930 goto failed; 931 } 932 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 933 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 934 entry->desc_wb = (struct tsnep_rx_desc_wb *) 935 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j); 936 entry->desc = (struct tsnep_rx_desc *) 937 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 938 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j; 939 } 940 } 941 942 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 943 pp_params.order = 0; 944 pp_params.pool_size = TSNEP_RING_SIZE; 945 pp_params.nid = dev_to_node(dmadev); 946 pp_params.dev = dmadev; 947 pp_params.dma_dir = DMA_BIDIRECTIONAL; 948 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE; 949 pp_params.offset = TSNEP_RX_OFFSET; 950 rx->page_pool = page_pool_create(&pp_params); 951 if (IS_ERR(rx->page_pool)) { 952 retval = PTR_ERR(rx->page_pool); 953 rx->page_pool = NULL; 954 goto failed; 955 } 956 957 for (i = 0; i < TSNEP_RING_SIZE; i++) { 958 entry = &rx->entry[i]; 959 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK]; 960 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 961 } 962 963 return 0; 964 965 failed: 966 tsnep_rx_ring_cleanup(rx); 967 return retval; 968 } 969 970 static void tsnep_rx_init(struct tsnep_rx *rx) 971 { 972 dma_addr_t dma; 973 974 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 975 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW); 976 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH); 977 rx->write = 0; 978 rx->read = 0; 979 rx->owner_counter = 1; 980 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 981 } 982 983 static void tsnep_rx_enable(struct tsnep_rx *rx) 984 { 985 /* descriptor properties shall be valid before hardware is notified */ 986 dma_wmb(); 987 988 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); 989 } 990 991 static void tsnep_rx_disable(struct tsnep_rx *rx) 992 { 993 u32 val; 994 995 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL); 996 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val, 997 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000, 998 1000000); 999 } 1000 1001 static int tsnep_rx_desc_available(struct tsnep_rx *rx) 1002 { 1003 if (rx->read <= rx->write) 1004 return TSNEP_RING_SIZE - rx->write + rx->read - 1; 1005 else 1006 return rx->read - rx->write - 1; 1007 } 1008 1009 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx) 1010 { 1011 struct page **page; 1012 1013 /* last entry of page_buffer is always zero, because ring cannot be 1014 * filled completely 1015 */ 1016 page = rx->page_buffer; 1017 while (*page) { 1018 page_pool_put_full_page(rx->page_pool, *page, false); 1019 *page = NULL; 1020 page++; 1021 } 1022 } 1023 1024 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx) 1025 { 1026 int i; 1027 1028 /* alloc for all ring entries except the last one, because ring cannot 1029 * be filled completely 1030 */ 1031 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) { 1032 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool); 1033 if (!rx->page_buffer[i]) { 1034 tsnep_rx_free_page_buffer(rx); 1035 1036 return -ENOMEM; 1037 } 1038 } 1039 1040 return 0; 1041 } 1042 1043 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1044 struct page *page) 1045 { 1046 entry->page = page; 1047 entry->len = TSNEP_MAX_RX_BUF_SIZE; 1048 entry->dma = page_pool_get_dma_addr(entry->page); 1049 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET); 1050 } 1051 1052 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index) 1053 { 1054 struct tsnep_rx_entry *entry = &rx->entry[index]; 1055 struct page *page; 1056 1057 page = page_pool_dev_alloc_pages(rx->page_pool); 1058 if (unlikely(!page)) 1059 return -ENOMEM; 1060 tsnep_rx_set_page(rx, entry, page); 1061 1062 return 0; 1063 } 1064 1065 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index) 1066 { 1067 struct tsnep_rx_entry *entry = &rx->entry[index]; 1068 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1069 1070 tsnep_rx_set_page(rx, entry, read->page); 1071 read->page = NULL; 1072 } 1073 1074 static void tsnep_rx_activate(struct tsnep_rx *rx, int index) 1075 { 1076 struct tsnep_rx_entry *entry = &rx->entry[index]; 1077 1078 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */ 1079 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK; 1080 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 1081 if (index == rx->increment_owner_counter) { 1082 rx->owner_counter++; 1083 if (rx->owner_counter == 4) 1084 rx->owner_counter = 1; 1085 rx->increment_owner_counter--; 1086 if (rx->increment_owner_counter < 0) 1087 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1088 } 1089 entry->properties |= 1090 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 1091 TSNEP_DESC_OWNER_COUNTER_MASK; 1092 1093 /* descriptor properties shall be written last, because valid data is 1094 * signaled there 1095 */ 1096 dma_wmb(); 1097 1098 entry->desc->properties = __cpu_to_le32(entry->properties); 1099 } 1100 1101 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse) 1102 { 1103 bool alloc_failed = false; 1104 int i, index; 1105 1106 for (i = 0; i < count && !alloc_failed; i++) { 1107 index = (rx->write + i) & TSNEP_RING_MASK; 1108 1109 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) { 1110 rx->alloc_failed++; 1111 alloc_failed = true; 1112 1113 /* reuse only if no other allocation was successful */ 1114 if (i == 0 && reuse) 1115 tsnep_rx_reuse_buffer(rx, index); 1116 else 1117 break; 1118 } 1119 1120 tsnep_rx_activate(rx, index); 1121 } 1122 1123 if (i) 1124 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1125 1126 return i; 1127 } 1128 1129 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse) 1130 { 1131 int desc_refilled; 1132 1133 desc_refilled = tsnep_rx_alloc(rx, count, reuse); 1134 if (desc_refilled) 1135 tsnep_rx_enable(rx); 1136 1137 return desc_refilled; 1138 } 1139 1140 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1141 struct xdp_buff *xdp) 1142 { 1143 entry->xdp = xdp; 1144 entry->len = TSNEP_XSK_RX_BUF_SIZE; 1145 entry->dma = xsk_buff_xdp_get_dma(entry->xdp); 1146 entry->desc->rx = __cpu_to_le64(entry->dma); 1147 } 1148 1149 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index) 1150 { 1151 struct tsnep_rx_entry *entry = &rx->entry[index]; 1152 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1153 1154 tsnep_rx_set_xdp(rx, entry, read->xdp); 1155 read->xdp = NULL; 1156 } 1157 1158 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse) 1159 { 1160 u32 allocated; 1161 int i; 1162 1163 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count); 1164 for (i = 0; i < allocated; i++) { 1165 int index = (rx->write + i) & TSNEP_RING_MASK; 1166 struct tsnep_rx_entry *entry = &rx->entry[index]; 1167 1168 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]); 1169 tsnep_rx_activate(rx, index); 1170 } 1171 if (i == 0) { 1172 rx->alloc_failed++; 1173 1174 if (reuse) { 1175 tsnep_rx_reuse_buffer_zc(rx, rx->write); 1176 tsnep_rx_activate(rx, rx->write); 1177 } 1178 } 1179 1180 if (i) 1181 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1182 1183 return i; 1184 } 1185 1186 static void tsnep_rx_free_zc(struct tsnep_rx *rx) 1187 { 1188 int i; 1189 1190 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1191 struct tsnep_rx_entry *entry = &rx->entry[i]; 1192 1193 if (entry->xdp) 1194 xsk_buff_free(entry->xdp); 1195 entry->xdp = NULL; 1196 } 1197 } 1198 1199 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse) 1200 { 1201 int desc_refilled; 1202 1203 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse); 1204 if (desc_refilled) 1205 tsnep_rx_enable(rx); 1206 1207 return desc_refilled; 1208 } 1209 1210 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog, 1211 struct xdp_buff *xdp, int *status, 1212 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1213 { 1214 unsigned int length; 1215 unsigned int sync; 1216 u32 act; 1217 1218 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM; 1219 1220 act = bpf_prog_run_xdp(prog, xdp); 1221 switch (act) { 1222 case XDP_PASS: 1223 return false; 1224 case XDP_TX: 1225 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx)) 1226 goto out_failure; 1227 *status |= TSNEP_XDP_TX; 1228 return true; 1229 case XDP_REDIRECT: 1230 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1231 goto out_failure; 1232 *status |= TSNEP_XDP_REDIRECT; 1233 return true; 1234 default: 1235 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1236 fallthrough; 1237 case XDP_ABORTED: 1238 out_failure: 1239 trace_xdp_exception(rx->adapter->netdev, prog, act); 1240 fallthrough; 1241 case XDP_DROP: 1242 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU 1243 * touch 1244 */ 1245 sync = xdp->data_end - xdp->data_hard_start - 1246 XDP_PACKET_HEADROOM; 1247 sync = max(sync, length); 1248 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data), 1249 sync, true); 1250 return true; 1251 } 1252 } 1253 1254 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog, 1255 struct xdp_buff *xdp, int *status, 1256 struct netdev_queue *tx_nq, 1257 struct tsnep_tx *tx) 1258 { 1259 u32 act; 1260 1261 act = bpf_prog_run_xdp(prog, xdp); 1262 1263 /* XDP_REDIRECT is the main action for zero-copy */ 1264 if (likely(act == XDP_REDIRECT)) { 1265 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1266 goto out_failure; 1267 *status |= TSNEP_XDP_REDIRECT; 1268 return true; 1269 } 1270 1271 switch (act) { 1272 case XDP_PASS: 1273 return false; 1274 case XDP_TX: 1275 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx)) 1276 goto out_failure; 1277 *status |= TSNEP_XDP_TX; 1278 return true; 1279 default: 1280 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1281 fallthrough; 1282 case XDP_ABORTED: 1283 out_failure: 1284 trace_xdp_exception(rx->adapter->netdev, prog, act); 1285 fallthrough; 1286 case XDP_DROP: 1287 xsk_buff_free(xdp); 1288 return true; 1289 } 1290 } 1291 1292 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status, 1293 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1294 { 1295 if (status & TSNEP_XDP_TX) { 1296 __netif_tx_lock(tx_nq, smp_processor_id()); 1297 tsnep_xdp_xmit_flush(tx); 1298 __netif_tx_unlock(tx_nq); 1299 } 1300 1301 if (status & TSNEP_XDP_REDIRECT) 1302 xdp_do_flush(); 1303 } 1304 1305 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page, 1306 int length) 1307 { 1308 struct sk_buff *skb; 1309 1310 skb = napi_build_skb(page_address(page), PAGE_SIZE); 1311 if (unlikely(!skb)) 1312 return NULL; 1313 1314 /* update pointers within the skb to store the data */ 1315 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE); 1316 __skb_put(skb, length - ETH_FCS_LEN); 1317 1318 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) { 1319 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb); 1320 struct tsnep_rx_inline *rx_inline = 1321 (struct tsnep_rx_inline *)(page_address(page) + 1322 TSNEP_RX_OFFSET); 1323 1324 skb_shinfo(skb)->tx_flags |= 1325 SKBTX_HW_TSTAMP_NETDEV; 1326 memset(hwtstamps, 0, sizeof(*hwtstamps)); 1327 hwtstamps->netdev_data = rx_inline; 1328 } 1329 1330 skb_record_rx_queue(skb, rx->queue_index); 1331 skb->protocol = eth_type_trans(skb, rx->adapter->netdev); 1332 1333 return skb; 1334 } 1335 1336 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi, 1337 struct page *page, int length) 1338 { 1339 struct sk_buff *skb; 1340 1341 skb = tsnep_build_skb(rx, page, length); 1342 if (skb) { 1343 skb_mark_for_recycle(skb); 1344 1345 rx->packets++; 1346 rx->bytes += length; 1347 if (skb->pkt_type == PACKET_MULTICAST) 1348 rx->multicast++; 1349 1350 napi_gro_receive(napi, skb); 1351 } else { 1352 page_pool_recycle_direct(rx->page_pool, page); 1353 1354 rx->dropped++; 1355 } 1356 } 1357 1358 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 1359 int budget) 1360 { 1361 struct device *dmadev = rx->adapter->dmadev; 1362 enum dma_data_direction dma_dir; 1363 struct tsnep_rx_entry *entry; 1364 struct netdev_queue *tx_nq; 1365 struct bpf_prog *prog; 1366 struct xdp_buff xdp; 1367 struct tsnep_tx *tx; 1368 int desc_available; 1369 int xdp_status = 0; 1370 int done = 0; 1371 int length; 1372 1373 desc_available = tsnep_rx_desc_available(rx); 1374 dma_dir = page_pool_get_dma_dir(rx->page_pool); 1375 prog = READ_ONCE(rx->adapter->xdp_prog); 1376 if (prog) { 1377 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1378 rx->tx_queue_index); 1379 tx = &rx->adapter->tx[rx->tx_queue_index]; 1380 1381 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq); 1382 } 1383 1384 while (likely(done < budget) && (rx->read != rx->write)) { 1385 entry = &rx->entry[rx->read]; 1386 if ((__le32_to_cpu(entry->desc_wb->properties) & 1387 TSNEP_DESC_OWNER_COUNTER_MASK) != 1388 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1389 break; 1390 done++; 1391 1392 if (desc_available >= TSNEP_RING_RX_REFILL) { 1393 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1394 1395 desc_available -= tsnep_rx_refill(rx, desc_available, 1396 reuse); 1397 if (!entry->page) { 1398 /* buffer has been reused for refill to prevent 1399 * empty RX ring, thus buffer cannot be used for 1400 * RX processing 1401 */ 1402 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1403 desc_available++; 1404 1405 rx->dropped++; 1406 1407 continue; 1408 } 1409 } 1410 1411 /* descriptor properties shall be read first, because valid data 1412 * is signaled there 1413 */ 1414 dma_rmb(); 1415 1416 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET); 1417 length = __le32_to_cpu(entry->desc_wb->properties) & 1418 TSNEP_DESC_LENGTH_MASK; 1419 dma_sync_single_range_for_cpu(dmadev, entry->dma, 1420 TSNEP_RX_OFFSET, length, dma_dir); 1421 1422 /* RX metadata with timestamps is in front of actual data, 1423 * subtract metadata size to get length of actual data and 1424 * consider metadata size as offset of actual data during RX 1425 * processing 1426 */ 1427 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1428 1429 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1430 desc_available++; 1431 1432 if (prog) { 1433 bool consume; 1434 1435 xdp_prepare_buff(&xdp, page_address(entry->page), 1436 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE, 1437 length - ETH_FCS_LEN, false); 1438 1439 consume = tsnep_xdp_run_prog(rx, prog, &xdp, 1440 &xdp_status, tx_nq, tx); 1441 if (consume) { 1442 rx->packets++; 1443 rx->bytes += length; 1444 1445 entry->page = NULL; 1446 1447 continue; 1448 } 1449 } 1450 1451 tsnep_rx_page(rx, napi, entry->page, length); 1452 entry->page = NULL; 1453 } 1454 1455 if (xdp_status) 1456 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1457 1458 if (desc_available) 1459 tsnep_rx_refill(rx, desc_available, false); 1460 1461 return done; 1462 } 1463 1464 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi, 1465 int budget) 1466 { 1467 struct tsnep_rx_entry *entry; 1468 struct netdev_queue *tx_nq; 1469 struct bpf_prog *prog; 1470 struct tsnep_tx *tx; 1471 int desc_available; 1472 int xdp_status = 0; 1473 struct page *page; 1474 int done = 0; 1475 int length; 1476 1477 desc_available = tsnep_rx_desc_available(rx); 1478 prog = READ_ONCE(rx->adapter->xdp_prog); 1479 if (prog) { 1480 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1481 rx->tx_queue_index); 1482 tx = &rx->adapter->tx[rx->tx_queue_index]; 1483 } 1484 1485 while (likely(done < budget) && (rx->read != rx->write)) { 1486 entry = &rx->entry[rx->read]; 1487 if ((__le32_to_cpu(entry->desc_wb->properties) & 1488 TSNEP_DESC_OWNER_COUNTER_MASK) != 1489 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1490 break; 1491 done++; 1492 1493 if (desc_available >= TSNEP_RING_RX_REFILL) { 1494 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1495 1496 desc_available -= tsnep_rx_refill_zc(rx, desc_available, 1497 reuse); 1498 if (!entry->xdp) { 1499 /* buffer has been reused for refill to prevent 1500 * empty RX ring, thus buffer cannot be used for 1501 * RX processing 1502 */ 1503 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1504 desc_available++; 1505 1506 rx->dropped++; 1507 1508 continue; 1509 } 1510 } 1511 1512 /* descriptor properties shall be read first, because valid data 1513 * is signaled there 1514 */ 1515 dma_rmb(); 1516 1517 prefetch(entry->xdp->data); 1518 length = __le32_to_cpu(entry->desc_wb->properties) & 1519 TSNEP_DESC_LENGTH_MASK; 1520 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN); 1521 xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool); 1522 1523 /* RX metadata with timestamps is in front of actual data, 1524 * subtract metadata size to get length of actual data and 1525 * consider metadata size as offset of actual data during RX 1526 * processing 1527 */ 1528 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1529 1530 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1531 desc_available++; 1532 1533 if (prog) { 1534 bool consume; 1535 1536 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE; 1537 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE; 1538 1539 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp, 1540 &xdp_status, tx_nq, tx); 1541 if (consume) { 1542 rx->packets++; 1543 rx->bytes += length; 1544 1545 entry->xdp = NULL; 1546 1547 continue; 1548 } 1549 } 1550 1551 page = page_pool_dev_alloc_pages(rx->page_pool); 1552 if (page) { 1553 memcpy(page_address(page) + TSNEP_RX_OFFSET, 1554 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE, 1555 length + TSNEP_RX_INLINE_METADATA_SIZE); 1556 tsnep_rx_page(rx, napi, page, length); 1557 } else { 1558 rx->dropped++; 1559 } 1560 xsk_buff_free(entry->xdp); 1561 entry->xdp = NULL; 1562 } 1563 1564 if (xdp_status) 1565 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1566 1567 if (desc_available) 1568 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false); 1569 1570 if (xsk_uses_need_wakeup(rx->xsk_pool)) { 1571 if (desc_available) 1572 xsk_set_rx_need_wakeup(rx->xsk_pool); 1573 else 1574 xsk_clear_rx_need_wakeup(rx->xsk_pool); 1575 1576 return done; 1577 } 1578 1579 return desc_available ? budget : done; 1580 } 1581 1582 static bool tsnep_rx_pending(struct tsnep_rx *rx) 1583 { 1584 struct tsnep_rx_entry *entry; 1585 1586 if (rx->read != rx->write) { 1587 entry = &rx->entry[rx->read]; 1588 if ((__le32_to_cpu(entry->desc_wb->properties) & 1589 TSNEP_DESC_OWNER_COUNTER_MASK) == 1590 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1591 return true; 1592 } 1593 1594 return false; 1595 } 1596 1597 static int tsnep_rx_open(struct tsnep_rx *rx) 1598 { 1599 int desc_available; 1600 int retval; 1601 1602 retval = tsnep_rx_ring_create(rx); 1603 if (retval) 1604 return retval; 1605 1606 tsnep_rx_init(rx); 1607 1608 desc_available = tsnep_rx_desc_available(rx); 1609 if (rx->xsk_pool) 1610 retval = tsnep_rx_alloc_zc(rx, desc_available, false); 1611 else 1612 retval = tsnep_rx_alloc(rx, desc_available, false); 1613 if (retval != desc_available) { 1614 retval = -ENOMEM; 1615 1616 goto alloc_failed; 1617 } 1618 1619 /* prealloc pages to prevent allocation failures when XSK pool is 1620 * disabled at runtime 1621 */ 1622 if (rx->xsk_pool) { 1623 retval = tsnep_rx_alloc_page_buffer(rx); 1624 if (retval) 1625 goto alloc_failed; 1626 } 1627 1628 return 0; 1629 1630 alloc_failed: 1631 tsnep_rx_ring_cleanup(rx); 1632 return retval; 1633 } 1634 1635 static void tsnep_rx_close(struct tsnep_rx *rx) 1636 { 1637 if (rx->xsk_pool) 1638 tsnep_rx_free_page_buffer(rx); 1639 1640 tsnep_rx_ring_cleanup(rx); 1641 } 1642 1643 static void tsnep_rx_reopen(struct tsnep_rx *rx) 1644 { 1645 struct page **page = rx->page_buffer; 1646 int i; 1647 1648 tsnep_rx_init(rx); 1649 1650 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1651 struct tsnep_rx_entry *entry = &rx->entry[i]; 1652 1653 /* defined initial values for properties are required for 1654 * correct owner counter checking 1655 */ 1656 entry->desc->properties = 0; 1657 entry->desc_wb->properties = 0; 1658 1659 /* prevent allocation failures by reusing kept pages */ 1660 if (*page) { 1661 tsnep_rx_set_page(rx, entry, *page); 1662 tsnep_rx_activate(rx, rx->write); 1663 rx->write++; 1664 1665 *page = NULL; 1666 page++; 1667 } 1668 } 1669 } 1670 1671 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx) 1672 { 1673 struct page **page = rx->page_buffer; 1674 u32 allocated; 1675 int i; 1676 1677 tsnep_rx_init(rx); 1678 1679 /* alloc all ring entries except the last one, because ring cannot be 1680 * filled completely, as many buffers as possible is enough as wakeup is 1681 * done if new buffers are available 1682 */ 1683 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, 1684 TSNEP_RING_SIZE - 1); 1685 1686 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1687 struct tsnep_rx_entry *entry = &rx->entry[i]; 1688 1689 /* keep pages to prevent allocation failures when xsk is 1690 * disabled 1691 */ 1692 if (entry->page) { 1693 *page = entry->page; 1694 entry->page = NULL; 1695 1696 page++; 1697 } 1698 1699 /* defined initial values for properties are required for 1700 * correct owner counter checking 1701 */ 1702 entry->desc->properties = 0; 1703 entry->desc_wb->properties = 0; 1704 1705 if (allocated) { 1706 tsnep_rx_set_xdp(rx, entry, 1707 rx->xdp_batch[allocated - 1]); 1708 tsnep_rx_activate(rx, rx->write); 1709 rx->write++; 1710 1711 allocated--; 1712 } 1713 } 1714 1715 /* set need wakeup flag immediately if ring is not filled completely, 1716 * first polling would be too late as need wakeup signalisation would 1717 * be delayed for an indefinite time 1718 */ 1719 if (xsk_uses_need_wakeup(rx->xsk_pool)) { 1720 int desc_available = tsnep_rx_desc_available(rx); 1721 1722 if (desc_available) 1723 xsk_set_rx_need_wakeup(rx->xsk_pool); 1724 else 1725 xsk_clear_rx_need_wakeup(rx->xsk_pool); 1726 } 1727 } 1728 1729 static bool tsnep_pending(struct tsnep_queue *queue) 1730 { 1731 if (queue->tx && tsnep_tx_pending(queue->tx)) 1732 return true; 1733 1734 if (queue->rx && tsnep_rx_pending(queue->rx)) 1735 return true; 1736 1737 return false; 1738 } 1739 1740 static int tsnep_poll(struct napi_struct *napi, int budget) 1741 { 1742 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue, 1743 napi); 1744 bool complete = true; 1745 int done = 0; 1746 1747 if (queue->tx) 1748 complete = tsnep_tx_poll(queue->tx, budget); 1749 1750 /* handle case where we are called by netpoll with a budget of 0 */ 1751 if (unlikely(budget <= 0)) 1752 return budget; 1753 1754 if (queue->rx) { 1755 done = queue->rx->xsk_pool ? 1756 tsnep_rx_poll_zc(queue->rx, napi, budget) : 1757 tsnep_rx_poll(queue->rx, napi, budget); 1758 if (done >= budget) 1759 complete = false; 1760 } 1761 1762 /* if all work not completed, return budget and keep polling */ 1763 if (!complete) 1764 return budget; 1765 1766 if (likely(napi_complete_done(napi, done))) { 1767 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1768 1769 /* reschedule if work is already pending, prevent rotten packets 1770 * which are transmitted or received after polling but before 1771 * interrupt enable 1772 */ 1773 if (tsnep_pending(queue)) { 1774 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1775 napi_schedule(napi); 1776 } 1777 } 1778 1779 return min(done, budget - 1); 1780 } 1781 1782 static int tsnep_request_irq(struct tsnep_queue *queue, bool first) 1783 { 1784 const char *name = netdev_name(queue->adapter->netdev); 1785 irq_handler_t handler; 1786 void *dev; 1787 int retval; 1788 1789 if (first) { 1790 sprintf(queue->name, "%s-mac", name); 1791 handler = tsnep_irq; 1792 dev = queue->adapter; 1793 } else { 1794 if (queue->tx && queue->rx) 1795 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d", 1796 name, queue->rx->queue_index); 1797 else if (queue->tx) 1798 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d", 1799 name, queue->tx->queue_index); 1800 else 1801 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d", 1802 name, queue->rx->queue_index); 1803 handler = tsnep_irq_txrx; 1804 dev = queue; 1805 } 1806 1807 retval = request_irq(queue->irq, handler, 0, queue->name, dev); 1808 if (retval) { 1809 /* if name is empty, then interrupt won't be freed */ 1810 memset(queue->name, 0, sizeof(queue->name)); 1811 } 1812 1813 return retval; 1814 } 1815 1816 static void tsnep_free_irq(struct tsnep_queue *queue, bool first) 1817 { 1818 void *dev; 1819 1820 if (!strlen(queue->name)) 1821 return; 1822 1823 if (first) 1824 dev = queue->adapter; 1825 else 1826 dev = queue; 1827 1828 free_irq(queue->irq, dev); 1829 memset(queue->name, 0, sizeof(queue->name)); 1830 } 1831 1832 static void tsnep_queue_close(struct tsnep_queue *queue, bool first) 1833 { 1834 struct tsnep_rx *rx = queue->rx; 1835 1836 tsnep_free_irq(queue, first); 1837 1838 if (rx) { 1839 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1840 xdp_rxq_info_unreg(&rx->xdp_rxq); 1841 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc)) 1842 xdp_rxq_info_unreg(&rx->xdp_rxq_zc); 1843 } 1844 1845 netif_napi_del(&queue->napi); 1846 } 1847 1848 static int tsnep_queue_open(struct tsnep_adapter *adapter, 1849 struct tsnep_queue *queue, bool first) 1850 { 1851 struct tsnep_rx *rx = queue->rx; 1852 struct tsnep_tx *tx = queue->tx; 1853 int retval; 1854 1855 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll); 1856 1857 if (rx) { 1858 /* choose TX queue for XDP_TX */ 1859 if (tx) 1860 rx->tx_queue_index = tx->queue_index; 1861 else if (rx->queue_index < adapter->num_tx_queues) 1862 rx->tx_queue_index = rx->queue_index; 1863 else 1864 rx->tx_queue_index = 0; 1865 1866 /* prepare both memory models to eliminate possible registration 1867 * errors when memory model is switched between page pool and 1868 * XSK pool during runtime 1869 */ 1870 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev, 1871 rx->queue_index, queue->napi.napi_id); 1872 if (retval) 1873 goto failed; 1874 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1875 MEM_TYPE_PAGE_POOL, 1876 rx->page_pool); 1877 if (retval) 1878 goto failed; 1879 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev, 1880 rx->queue_index, queue->napi.napi_id); 1881 if (retval) 1882 goto failed; 1883 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc, 1884 MEM_TYPE_XSK_BUFF_POOL, 1885 NULL); 1886 if (retval) 1887 goto failed; 1888 if (rx->xsk_pool) 1889 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc); 1890 } 1891 1892 retval = tsnep_request_irq(queue, first); 1893 if (retval) { 1894 netif_err(adapter, drv, adapter->netdev, 1895 "can't get assigned irq %d.\n", queue->irq); 1896 goto failed; 1897 } 1898 1899 return 0; 1900 1901 failed: 1902 tsnep_queue_close(queue, first); 1903 1904 return retval; 1905 } 1906 1907 static void tsnep_queue_enable(struct tsnep_queue *queue) 1908 { 1909 napi_enable(&queue->napi); 1910 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1911 1912 if (queue->tx) 1913 tsnep_tx_enable(queue->tx); 1914 1915 if (queue->rx) 1916 tsnep_rx_enable(queue->rx); 1917 } 1918 1919 static void tsnep_queue_disable(struct tsnep_queue *queue) 1920 { 1921 if (queue->tx) 1922 tsnep_tx_disable(queue->tx, &queue->napi); 1923 1924 napi_disable(&queue->napi); 1925 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1926 1927 /* disable RX after NAPI polling has been disabled, because RX can be 1928 * enabled during NAPI polling 1929 */ 1930 if (queue->rx) 1931 tsnep_rx_disable(queue->rx); 1932 } 1933 1934 static int tsnep_netdev_open(struct net_device *netdev) 1935 { 1936 struct tsnep_adapter *adapter = netdev_priv(netdev); 1937 int i, retval; 1938 1939 for (i = 0; i < adapter->num_queues; i++) { 1940 if (adapter->queue[i].tx) { 1941 retval = tsnep_tx_open(adapter->queue[i].tx); 1942 if (retval) 1943 goto failed; 1944 } 1945 if (adapter->queue[i].rx) { 1946 retval = tsnep_rx_open(adapter->queue[i].rx); 1947 if (retval) 1948 goto failed; 1949 } 1950 1951 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0); 1952 if (retval) 1953 goto failed; 1954 } 1955 1956 retval = netif_set_real_num_tx_queues(adapter->netdev, 1957 adapter->num_tx_queues); 1958 if (retval) 1959 goto failed; 1960 retval = netif_set_real_num_rx_queues(adapter->netdev, 1961 adapter->num_rx_queues); 1962 if (retval) 1963 goto failed; 1964 1965 tsnep_enable_irq(adapter, ECM_INT_LINK); 1966 retval = tsnep_phy_open(adapter); 1967 if (retval) 1968 goto phy_failed; 1969 1970 for (i = 0; i < adapter->num_queues; i++) 1971 tsnep_queue_enable(&adapter->queue[i]); 1972 1973 return 0; 1974 1975 phy_failed: 1976 tsnep_disable_irq(adapter, ECM_INT_LINK); 1977 failed: 1978 for (i = 0; i < adapter->num_queues; i++) { 1979 tsnep_queue_close(&adapter->queue[i], i == 0); 1980 1981 if (adapter->queue[i].rx) 1982 tsnep_rx_close(adapter->queue[i].rx); 1983 if (adapter->queue[i].tx) 1984 tsnep_tx_close(adapter->queue[i].tx); 1985 } 1986 return retval; 1987 } 1988 1989 static int tsnep_netdev_close(struct net_device *netdev) 1990 { 1991 struct tsnep_adapter *adapter = netdev_priv(netdev); 1992 int i; 1993 1994 tsnep_disable_irq(adapter, ECM_INT_LINK); 1995 tsnep_phy_close(adapter); 1996 1997 for (i = 0; i < adapter->num_queues; i++) { 1998 tsnep_queue_disable(&adapter->queue[i]); 1999 2000 tsnep_queue_close(&adapter->queue[i], i == 0); 2001 2002 if (adapter->queue[i].rx) 2003 tsnep_rx_close(adapter->queue[i].rx); 2004 if (adapter->queue[i].tx) 2005 tsnep_tx_close(adapter->queue[i].tx); 2006 } 2007 2008 return 0; 2009 } 2010 2011 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool) 2012 { 2013 bool running = netif_running(queue->adapter->netdev); 2014 u32 frame_size; 2015 2016 frame_size = xsk_pool_get_rx_frame_size(pool); 2017 if (frame_size < TSNEP_XSK_RX_BUF_SIZE) 2018 return -EOPNOTSUPP; 2019 2020 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE, 2021 sizeof(*queue->rx->page_buffer), 2022 GFP_KERNEL); 2023 if (!queue->rx->page_buffer) 2024 return -ENOMEM; 2025 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE, 2026 sizeof(*queue->rx->xdp_batch), 2027 GFP_KERNEL); 2028 if (!queue->rx->xdp_batch) { 2029 kfree(queue->rx->page_buffer); 2030 queue->rx->page_buffer = NULL; 2031 2032 return -ENOMEM; 2033 } 2034 2035 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc); 2036 2037 if (running) 2038 tsnep_queue_disable(queue); 2039 2040 queue->tx->xsk_pool = pool; 2041 queue->rx->xsk_pool = pool; 2042 2043 if (running) { 2044 tsnep_rx_reopen_xsk(queue->rx); 2045 tsnep_queue_enable(queue); 2046 } 2047 2048 return 0; 2049 } 2050 2051 void tsnep_disable_xsk(struct tsnep_queue *queue) 2052 { 2053 bool running = netif_running(queue->adapter->netdev); 2054 2055 if (running) 2056 tsnep_queue_disable(queue); 2057 2058 tsnep_rx_free_zc(queue->rx); 2059 2060 queue->rx->xsk_pool = NULL; 2061 queue->tx->xsk_pool = NULL; 2062 2063 if (running) { 2064 tsnep_rx_reopen(queue->rx); 2065 tsnep_queue_enable(queue); 2066 } 2067 2068 kfree(queue->rx->xdp_batch); 2069 queue->rx->xdp_batch = NULL; 2070 kfree(queue->rx->page_buffer); 2071 queue->rx->page_buffer = NULL; 2072 } 2073 2074 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb, 2075 struct net_device *netdev) 2076 { 2077 struct tsnep_adapter *adapter = netdev_priv(netdev); 2078 u16 queue_mapping = skb_get_queue_mapping(skb); 2079 2080 if (queue_mapping >= adapter->num_tx_queues) 2081 queue_mapping = 0; 2082 2083 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]); 2084 } 2085 2086 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr, 2087 int cmd) 2088 { 2089 if (!netif_running(netdev)) 2090 return -EINVAL; 2091 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP) 2092 return tsnep_ptp_ioctl(netdev, ifr, cmd); 2093 return phy_mii_ioctl(netdev->phydev, ifr, cmd); 2094 } 2095 2096 static void tsnep_netdev_set_multicast(struct net_device *netdev) 2097 { 2098 struct tsnep_adapter *adapter = netdev_priv(netdev); 2099 2100 u16 rx_filter = 0; 2101 2102 /* configured MAC address and broadcasts are never filtered */ 2103 if (netdev->flags & IFF_PROMISC) { 2104 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2105 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS; 2106 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) { 2107 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2108 } 2109 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER); 2110 } 2111 2112 static void tsnep_netdev_get_stats64(struct net_device *netdev, 2113 struct rtnl_link_stats64 *stats) 2114 { 2115 struct tsnep_adapter *adapter = netdev_priv(netdev); 2116 u32 reg; 2117 u32 val; 2118 int i; 2119 2120 for (i = 0; i < adapter->num_tx_queues; i++) { 2121 stats->tx_packets += adapter->tx[i].packets; 2122 stats->tx_bytes += adapter->tx[i].bytes; 2123 stats->tx_dropped += adapter->tx[i].dropped; 2124 } 2125 for (i = 0; i < adapter->num_rx_queues; i++) { 2126 stats->rx_packets += adapter->rx[i].packets; 2127 stats->rx_bytes += adapter->rx[i].bytes; 2128 stats->rx_dropped += adapter->rx[i].dropped; 2129 stats->multicast += adapter->rx[i].multicast; 2130 2131 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + 2132 TSNEP_RX_STATISTIC); 2133 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >> 2134 TSNEP_RX_STATISTIC_NO_DESC_SHIFT; 2135 stats->rx_dropped += val; 2136 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >> 2137 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT; 2138 stats->rx_dropped += val; 2139 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >> 2140 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT; 2141 stats->rx_errors += val; 2142 stats->rx_fifo_errors += val; 2143 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >> 2144 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT; 2145 stats->rx_errors += val; 2146 stats->rx_frame_errors += val; 2147 } 2148 2149 reg = ioread32(adapter->addr + ECM_STAT); 2150 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT; 2151 stats->rx_errors += val; 2152 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT; 2153 stats->rx_errors += val; 2154 stats->rx_crc_errors += val; 2155 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT; 2156 stats->rx_errors += val; 2157 } 2158 2159 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr) 2160 { 2161 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2162 iowrite16(*(u16 *)(addr + sizeof(u32)), 2163 adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2164 2165 ether_addr_copy(adapter->mac_address, addr); 2166 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n", 2167 addr); 2168 } 2169 2170 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr) 2171 { 2172 struct tsnep_adapter *adapter = netdev_priv(netdev); 2173 struct sockaddr *sock_addr = addr; 2174 int retval; 2175 2176 retval = eth_prepare_mac_addr_change(netdev, sock_addr); 2177 if (retval) 2178 return retval; 2179 eth_hw_addr_set(netdev, sock_addr->sa_data); 2180 tsnep_mac_set_address(adapter, sock_addr->sa_data); 2181 2182 return 0; 2183 } 2184 2185 static int tsnep_netdev_set_features(struct net_device *netdev, 2186 netdev_features_t features) 2187 { 2188 struct tsnep_adapter *adapter = netdev_priv(netdev); 2189 netdev_features_t changed = netdev->features ^ features; 2190 bool enable; 2191 int retval = 0; 2192 2193 if (changed & NETIF_F_LOOPBACK) { 2194 enable = !!(features & NETIF_F_LOOPBACK); 2195 retval = tsnep_phy_loopback(adapter, enable); 2196 } 2197 2198 return retval; 2199 } 2200 2201 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev, 2202 const struct skb_shared_hwtstamps *hwtstamps, 2203 bool cycles) 2204 { 2205 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data; 2206 u64 timestamp; 2207 2208 if (cycles) 2209 timestamp = __le64_to_cpu(rx_inline->counter); 2210 else 2211 timestamp = __le64_to_cpu(rx_inline->timestamp); 2212 2213 return ns_to_ktime(timestamp); 2214 } 2215 2216 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf) 2217 { 2218 struct tsnep_adapter *adapter = netdev_priv(dev); 2219 2220 switch (bpf->command) { 2221 case XDP_SETUP_PROG: 2222 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack); 2223 case XDP_SETUP_XSK_POOL: 2224 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool, 2225 bpf->xsk.queue_id); 2226 default: 2227 return -EOPNOTSUPP; 2228 } 2229 } 2230 2231 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu) 2232 { 2233 if (cpu >= TSNEP_MAX_QUEUES) 2234 cpu &= TSNEP_MAX_QUEUES - 1; 2235 2236 while (cpu >= adapter->num_tx_queues) 2237 cpu -= adapter->num_tx_queues; 2238 2239 return &adapter->tx[cpu]; 2240 } 2241 2242 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n, 2243 struct xdp_frame **xdp, u32 flags) 2244 { 2245 struct tsnep_adapter *adapter = netdev_priv(dev); 2246 u32 cpu = smp_processor_id(); 2247 struct netdev_queue *nq; 2248 struct tsnep_tx *tx; 2249 int nxmit; 2250 bool xmit; 2251 2252 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2253 return -EINVAL; 2254 2255 tx = tsnep_xdp_get_tx(adapter, cpu); 2256 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index); 2257 2258 __netif_tx_lock(nq, cpu); 2259 2260 for (nxmit = 0; nxmit < n; nxmit++) { 2261 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx, 2262 TSNEP_TX_TYPE_XDP_NDO); 2263 if (!xmit) 2264 break; 2265 2266 /* avoid transmit queue timeout since we share it with the slow 2267 * path 2268 */ 2269 txq_trans_cond_update(nq); 2270 } 2271 2272 if (flags & XDP_XMIT_FLUSH) 2273 tsnep_xdp_xmit_flush(tx); 2274 2275 __netif_tx_unlock(nq); 2276 2277 return nxmit; 2278 } 2279 2280 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id, 2281 u32 flags) 2282 { 2283 struct tsnep_adapter *adapter = netdev_priv(dev); 2284 struct tsnep_queue *queue; 2285 2286 if (queue_id >= adapter->num_rx_queues || 2287 queue_id >= adapter->num_tx_queues) 2288 return -EINVAL; 2289 2290 queue = &adapter->queue[queue_id]; 2291 2292 if (!napi_if_scheduled_mark_missed(&queue->napi)) 2293 napi_schedule(&queue->napi); 2294 2295 return 0; 2296 } 2297 2298 static const struct net_device_ops tsnep_netdev_ops = { 2299 .ndo_open = tsnep_netdev_open, 2300 .ndo_stop = tsnep_netdev_close, 2301 .ndo_start_xmit = tsnep_netdev_xmit_frame, 2302 .ndo_eth_ioctl = tsnep_netdev_ioctl, 2303 .ndo_set_rx_mode = tsnep_netdev_set_multicast, 2304 .ndo_get_stats64 = tsnep_netdev_get_stats64, 2305 .ndo_set_mac_address = tsnep_netdev_set_mac_address, 2306 .ndo_set_features = tsnep_netdev_set_features, 2307 .ndo_get_tstamp = tsnep_netdev_get_tstamp, 2308 .ndo_setup_tc = tsnep_tc_setup, 2309 .ndo_bpf = tsnep_netdev_bpf, 2310 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit, 2311 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup, 2312 }; 2313 2314 static int tsnep_mac_init(struct tsnep_adapter *adapter) 2315 { 2316 int retval; 2317 2318 /* initialize RX filtering, at least configured MAC address and 2319 * broadcast are not filtered 2320 */ 2321 iowrite16(0, adapter->addr + TSNEP_RX_FILTER); 2322 2323 /* try to get MAC address in the following order: 2324 * - device tree 2325 * - valid MAC address already set 2326 * - MAC address register if valid 2327 * - random MAC address 2328 */ 2329 retval = of_get_mac_address(adapter->pdev->dev.of_node, 2330 adapter->mac_address); 2331 if (retval == -EPROBE_DEFER) 2332 return retval; 2333 if (retval && !is_valid_ether_addr(adapter->mac_address)) { 2334 *(u32 *)adapter->mac_address = 2335 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2336 *(u16 *)(adapter->mac_address + sizeof(u32)) = 2337 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2338 if (!is_valid_ether_addr(adapter->mac_address)) 2339 eth_random_addr(adapter->mac_address); 2340 } 2341 2342 tsnep_mac_set_address(adapter, adapter->mac_address); 2343 eth_hw_addr_set(adapter->netdev, adapter->mac_address); 2344 2345 return 0; 2346 } 2347 2348 static int tsnep_mdio_init(struct tsnep_adapter *adapter) 2349 { 2350 struct device_node *np = adapter->pdev->dev.of_node; 2351 int retval; 2352 2353 if (np) { 2354 np = of_get_child_by_name(np, "mdio"); 2355 if (!np) 2356 return 0; 2357 2358 adapter->suppress_preamble = 2359 of_property_read_bool(np, "suppress-preamble"); 2360 } 2361 2362 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev); 2363 if (!adapter->mdiobus) { 2364 retval = -ENOMEM; 2365 2366 goto out; 2367 } 2368 2369 adapter->mdiobus->priv = (void *)adapter; 2370 adapter->mdiobus->parent = &adapter->pdev->dev; 2371 adapter->mdiobus->read = tsnep_mdiobus_read; 2372 adapter->mdiobus->write = tsnep_mdiobus_write; 2373 adapter->mdiobus->name = TSNEP "-mdiobus"; 2374 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s", 2375 adapter->pdev->name); 2376 2377 /* do not scan broadcast address */ 2378 adapter->mdiobus->phy_mask = 0x0000001; 2379 2380 retval = of_mdiobus_register(adapter->mdiobus, np); 2381 2382 out: 2383 of_node_put(np); 2384 2385 return retval; 2386 } 2387 2388 static int tsnep_phy_init(struct tsnep_adapter *adapter) 2389 { 2390 struct device_node *phy_node; 2391 int retval; 2392 2393 retval = of_get_phy_mode(adapter->pdev->dev.of_node, 2394 &adapter->phy_mode); 2395 if (retval) 2396 adapter->phy_mode = PHY_INTERFACE_MODE_GMII; 2397 2398 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle", 2399 0); 2400 adapter->phydev = of_phy_find_device(phy_node); 2401 of_node_put(phy_node); 2402 if (!adapter->phydev && adapter->mdiobus) 2403 adapter->phydev = phy_find_first(adapter->mdiobus); 2404 if (!adapter->phydev) 2405 return -EIO; 2406 2407 return 0; 2408 } 2409 2410 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) 2411 { 2412 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; 2413 char name[8]; 2414 int i; 2415 int retval; 2416 2417 /* one TX/RX queue pair for netdev is mandatory */ 2418 if (platform_irq_count(adapter->pdev) == 1) 2419 retval = platform_get_irq(adapter->pdev, 0); 2420 else 2421 retval = platform_get_irq_byname(adapter->pdev, "mac"); 2422 if (retval < 0) 2423 return retval; 2424 adapter->num_tx_queues = 1; 2425 adapter->num_rx_queues = 1; 2426 adapter->num_queues = 1; 2427 adapter->queue[0].adapter = adapter; 2428 adapter->queue[0].irq = retval; 2429 adapter->queue[0].tx = &adapter->tx[0]; 2430 adapter->queue[0].tx->adapter = adapter; 2431 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0); 2432 adapter->queue[0].tx->queue_index = 0; 2433 adapter->queue[0].rx = &adapter->rx[0]; 2434 adapter->queue[0].rx->adapter = adapter; 2435 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0); 2436 adapter->queue[0].rx->queue_index = 0; 2437 adapter->queue[0].irq_mask = irq_mask; 2438 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY; 2439 retval = tsnep_set_irq_coalesce(&adapter->queue[0], 2440 TSNEP_COALESCE_USECS_DEFAULT); 2441 if (retval < 0) 2442 return retval; 2443 2444 adapter->netdev->irq = adapter->queue[0].irq; 2445 2446 /* add additional TX/RX queue pairs only if dedicated interrupt is 2447 * available 2448 */ 2449 for (i = 1; i < queue_count; i++) { 2450 sprintf(name, "txrx-%d", i); 2451 retval = platform_get_irq_byname_optional(adapter->pdev, name); 2452 if (retval < 0) 2453 break; 2454 2455 adapter->num_tx_queues++; 2456 adapter->num_rx_queues++; 2457 adapter->num_queues++; 2458 adapter->queue[i].adapter = adapter; 2459 adapter->queue[i].irq = retval; 2460 adapter->queue[i].tx = &adapter->tx[i]; 2461 adapter->queue[i].tx->adapter = adapter; 2462 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i); 2463 adapter->queue[i].tx->queue_index = i; 2464 adapter->queue[i].rx = &adapter->rx[i]; 2465 adapter->queue[i].rx->adapter = adapter; 2466 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i); 2467 adapter->queue[i].rx->queue_index = i; 2468 adapter->queue[i].irq_mask = 2469 irq_mask << (ECM_INT_TXRX_SHIFT * i); 2470 adapter->queue[i].irq_delay_addr = 2471 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i; 2472 retval = tsnep_set_irq_coalesce(&adapter->queue[i], 2473 TSNEP_COALESCE_USECS_DEFAULT); 2474 if (retval < 0) 2475 return retval; 2476 } 2477 2478 return 0; 2479 } 2480 2481 static int tsnep_probe(struct platform_device *pdev) 2482 { 2483 struct tsnep_adapter *adapter; 2484 struct net_device *netdev; 2485 struct resource *io; 2486 u32 type; 2487 int revision; 2488 int version; 2489 int queue_count; 2490 int retval; 2491 2492 netdev = devm_alloc_etherdev_mqs(&pdev->dev, 2493 sizeof(struct tsnep_adapter), 2494 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES); 2495 if (!netdev) 2496 return -ENODEV; 2497 SET_NETDEV_DEV(netdev, &pdev->dev); 2498 adapter = netdev_priv(netdev); 2499 platform_set_drvdata(pdev, adapter); 2500 adapter->pdev = pdev; 2501 adapter->dmadev = &pdev->dev; 2502 adapter->netdev = netdev; 2503 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | 2504 NETIF_MSG_LINK | NETIF_MSG_IFUP | 2505 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED; 2506 2507 netdev->min_mtu = ETH_MIN_MTU; 2508 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE; 2509 2510 mutex_init(&adapter->gate_control_lock); 2511 mutex_init(&adapter->rxnfc_lock); 2512 INIT_LIST_HEAD(&adapter->rxnfc_rules); 2513 2514 io = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2515 adapter->addr = devm_ioremap_resource(&pdev->dev, io); 2516 if (IS_ERR(adapter->addr)) 2517 return PTR_ERR(adapter->addr); 2518 netdev->mem_start = io->start; 2519 netdev->mem_end = io->end; 2520 2521 type = ioread32(adapter->addr + ECM_TYPE); 2522 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT; 2523 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT; 2524 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT; 2525 adapter->gate_control = type & ECM_GATE_CONTROL; 2526 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT; 2527 2528 tsnep_disable_irq(adapter, ECM_INT_ALL); 2529 2530 retval = tsnep_queue_init(adapter, queue_count); 2531 if (retval) 2532 return retval; 2533 2534 retval = dma_set_mask_and_coherent(&adapter->pdev->dev, 2535 DMA_BIT_MASK(64)); 2536 if (retval) { 2537 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n"); 2538 return retval; 2539 } 2540 2541 retval = tsnep_mac_init(adapter); 2542 if (retval) 2543 return retval; 2544 2545 retval = tsnep_mdio_init(adapter); 2546 if (retval) 2547 goto mdio_init_failed; 2548 2549 retval = tsnep_phy_init(adapter); 2550 if (retval) 2551 goto phy_init_failed; 2552 2553 retval = tsnep_ptp_init(adapter); 2554 if (retval) 2555 goto ptp_init_failed; 2556 2557 retval = tsnep_tc_init(adapter); 2558 if (retval) 2559 goto tc_init_failed; 2560 2561 retval = tsnep_rxnfc_init(adapter); 2562 if (retval) 2563 goto rxnfc_init_failed; 2564 2565 netdev->netdev_ops = &tsnep_netdev_ops; 2566 netdev->ethtool_ops = &tsnep_ethtool_ops; 2567 netdev->features = NETIF_F_SG; 2568 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK; 2569 2570 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 2571 NETDEV_XDP_ACT_NDO_XMIT | 2572 NETDEV_XDP_ACT_NDO_XMIT_SG | 2573 NETDEV_XDP_ACT_XSK_ZEROCOPY; 2574 2575 /* carrier off reporting is important to ethtool even BEFORE open */ 2576 netif_carrier_off(netdev); 2577 2578 retval = register_netdev(netdev); 2579 if (retval) 2580 goto register_failed; 2581 2582 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version, 2583 revision); 2584 if (adapter->gate_control) 2585 dev_info(&adapter->pdev->dev, "gate control detected\n"); 2586 2587 return 0; 2588 2589 register_failed: 2590 tsnep_rxnfc_cleanup(adapter); 2591 rxnfc_init_failed: 2592 tsnep_tc_cleanup(adapter); 2593 tc_init_failed: 2594 tsnep_ptp_cleanup(adapter); 2595 ptp_init_failed: 2596 phy_init_failed: 2597 if (adapter->mdiobus) 2598 mdiobus_unregister(adapter->mdiobus); 2599 mdio_init_failed: 2600 return retval; 2601 } 2602 2603 static int tsnep_remove(struct platform_device *pdev) 2604 { 2605 struct tsnep_adapter *adapter = platform_get_drvdata(pdev); 2606 2607 unregister_netdev(adapter->netdev); 2608 2609 tsnep_rxnfc_cleanup(adapter); 2610 2611 tsnep_tc_cleanup(adapter); 2612 2613 tsnep_ptp_cleanup(adapter); 2614 2615 if (adapter->mdiobus) 2616 mdiobus_unregister(adapter->mdiobus); 2617 2618 tsnep_disable_irq(adapter, ECM_INT_ALL); 2619 2620 return 0; 2621 } 2622 2623 static const struct of_device_id tsnep_of_match[] = { 2624 { .compatible = "engleder,tsnep", }, 2625 { }, 2626 }; 2627 MODULE_DEVICE_TABLE(of, tsnep_of_match); 2628 2629 static struct platform_driver tsnep_driver = { 2630 .driver = { 2631 .name = TSNEP, 2632 .of_match_table = tsnep_of_match, 2633 }, 2634 .probe = tsnep_probe, 2635 .remove = tsnep_remove, 2636 }; 2637 module_platform_driver(tsnep_driver); 2638 2639 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>"); 2640 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver"); 2641 MODULE_LICENSE("GPL"); 2642