1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ 3 4 /* TSN endpoint Ethernet MAC driver 5 * 6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time 7 * communication. It is designed for endpoints within TSN (Time Sensitive 8 * Networking) networks; e.g., for PLCs in the industrial automation case. 9 * 10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used 11 * by the driver. 12 * 13 * More information can be found here: 14 * - www.embedded-experts.at/tsn 15 * - www.engleder-embedded.com 16 */ 17 18 #include "tsnep.h" 19 #include "tsnep_hw.h" 20 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_net.h> 24 #include <linux/of_mdio.h> 25 #include <linux/interrupt.h> 26 #include <linux/etherdevice.h> 27 #include <linux/phy.h> 28 #include <linux/iopoll.h> 29 #include <linux/bpf.h> 30 #include <linux/bpf_trace.h> 31 #include <net/xdp_sock_drv.h> 32 33 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN) 34 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4) 35 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \ 36 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 37 /* XSK buffer shall store at least Q-in-Q frame */ 38 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \ 39 ETH_FRAME_LEN + ETH_FCS_LEN + \ 40 VLAN_HLEN * 2, 4)) 41 42 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 43 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF)) 44 #else 45 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0)) 46 #endif 47 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF)) 48 49 #define TSNEP_COALESCE_USECS_DEFAULT 64 50 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \ 51 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1) 52 53 #define TSNEP_TX_TYPE_SKB BIT(0) 54 #define TSNEP_TX_TYPE_SKB_FRAG BIT(1) 55 #define TSNEP_TX_TYPE_XDP_TX BIT(2) 56 #define TSNEP_TX_TYPE_XDP_NDO BIT(3) 57 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO) 58 #define TSNEP_TX_TYPE_XSK BIT(4) 59 60 #define TSNEP_XDP_TX BIT(0) 61 #define TSNEP_XDP_REDIRECT BIT(1) 62 63 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask) 64 { 65 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 66 } 67 68 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask) 69 { 70 mask |= ECM_INT_DISABLE; 71 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 72 } 73 74 static irqreturn_t tsnep_irq(int irq, void *arg) 75 { 76 struct tsnep_adapter *adapter = arg; 77 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE); 78 79 /* acknowledge interrupt */ 80 if (active != 0) 81 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE); 82 83 /* handle link interrupt */ 84 if ((active & ECM_INT_LINK) != 0) 85 phy_mac_interrupt(adapter->netdev->phydev); 86 87 /* handle TX/RX queue 0 interrupt */ 88 if ((active & adapter->queue[0].irq_mask) != 0) { 89 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); 90 napi_schedule(&adapter->queue[0].napi); 91 } 92 93 return IRQ_HANDLED; 94 } 95 96 static irqreturn_t tsnep_irq_txrx(int irq, void *arg) 97 { 98 struct tsnep_queue *queue = arg; 99 100 /* handle TX/RX queue interrupt */ 101 tsnep_disable_irq(queue->adapter, queue->irq_mask); 102 napi_schedule(&queue->napi); 103 104 return IRQ_HANDLED; 105 } 106 107 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs) 108 { 109 if (usecs > TSNEP_COALESCE_USECS_MAX) 110 return -ERANGE; 111 112 usecs /= ECM_INT_DELAY_BASE_US; 113 usecs <<= ECM_INT_DELAY_SHIFT; 114 usecs &= ECM_INT_DELAY_MASK; 115 116 queue->irq_delay &= ~ECM_INT_DELAY_MASK; 117 queue->irq_delay |= usecs; 118 iowrite8(queue->irq_delay, queue->irq_delay_addr); 119 120 return 0; 121 } 122 123 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue) 124 { 125 u32 usecs; 126 127 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK); 128 usecs >>= ECM_INT_DELAY_SHIFT; 129 usecs *= ECM_INT_DELAY_BASE_US; 130 131 return usecs; 132 } 133 134 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) 135 { 136 struct tsnep_adapter *adapter = bus->priv; 137 u32 md; 138 int retval; 139 140 md = ECM_MD_READ; 141 if (!adapter->suppress_preamble) 142 md |= ECM_MD_PREAMBLE; 143 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 144 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 145 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 146 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 147 !(md & ECM_MD_BUSY), 16, 1000); 148 if (retval != 0) 149 return retval; 150 151 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT; 152 } 153 154 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum, 155 u16 val) 156 { 157 struct tsnep_adapter *adapter = bus->priv; 158 u32 md; 159 int retval; 160 161 md = ECM_MD_WRITE; 162 if (!adapter->suppress_preamble) 163 md |= ECM_MD_PREAMBLE; 164 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 165 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 166 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK; 167 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 168 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 169 !(md & ECM_MD_BUSY), 16, 1000); 170 if (retval != 0) 171 return retval; 172 173 return 0; 174 } 175 176 static void tsnep_set_link_mode(struct tsnep_adapter *adapter) 177 { 178 u32 mode; 179 180 switch (adapter->phydev->speed) { 181 case SPEED_100: 182 mode = ECM_LINK_MODE_100; 183 break; 184 case SPEED_1000: 185 mode = ECM_LINK_MODE_1000; 186 break; 187 default: 188 mode = ECM_LINK_MODE_OFF; 189 break; 190 } 191 iowrite32(mode, adapter->addr + ECM_STATUS); 192 } 193 194 static void tsnep_phy_link_status_change(struct net_device *netdev) 195 { 196 struct tsnep_adapter *adapter = netdev_priv(netdev); 197 struct phy_device *phydev = netdev->phydev; 198 199 if (phydev->link) 200 tsnep_set_link_mode(adapter); 201 202 phy_print_status(netdev->phydev); 203 } 204 205 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable) 206 { 207 int retval; 208 209 retval = phy_loopback(adapter->phydev, enable); 210 211 /* PHY link state change is not signaled if loopback is enabled, it 212 * would delay a working loopback anyway, let's ensure that loopback 213 * is working immediately by setting link mode directly 214 */ 215 if (!retval && enable) 216 tsnep_set_link_mode(adapter); 217 218 return retval; 219 } 220 221 static int tsnep_phy_open(struct tsnep_adapter *adapter) 222 { 223 struct phy_device *phydev; 224 struct ethtool_eee ethtool_eee; 225 int retval; 226 227 retval = phy_connect_direct(adapter->netdev, adapter->phydev, 228 tsnep_phy_link_status_change, 229 adapter->phy_mode); 230 if (retval) 231 return retval; 232 phydev = adapter->netdev->phydev; 233 234 /* MAC supports only 100Mbps|1000Mbps full duplex 235 * SPE (Single Pair Ethernet) is also an option but not implemented yet 236 */ 237 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); 238 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); 239 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); 240 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 241 242 /* disable EEE autoneg, EEE not supported by TSNEP */ 243 memset(ðtool_eee, 0, sizeof(ethtool_eee)); 244 phy_ethtool_set_eee(adapter->phydev, ðtool_eee); 245 246 adapter->phydev->irq = PHY_MAC_INTERRUPT; 247 phy_start(adapter->phydev); 248 249 return 0; 250 } 251 252 static void tsnep_phy_close(struct tsnep_adapter *adapter) 253 { 254 phy_stop(adapter->netdev->phydev); 255 phy_disconnect(adapter->netdev->phydev); 256 } 257 258 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx) 259 { 260 struct device *dmadev = tx->adapter->dmadev; 261 int i; 262 263 memset(tx->entry, 0, sizeof(tx->entry)); 264 265 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 266 if (tx->page[i]) { 267 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i], 268 tx->page_dma[i]); 269 tx->page[i] = NULL; 270 tx->page_dma[i] = 0; 271 } 272 } 273 } 274 275 static int tsnep_tx_ring_create(struct tsnep_tx *tx) 276 { 277 struct device *dmadev = tx->adapter->dmadev; 278 struct tsnep_tx_entry *entry; 279 struct tsnep_tx_entry *next_entry; 280 int i, j; 281 int retval; 282 283 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 284 tx->page[i] = 285 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i], 286 GFP_KERNEL); 287 if (!tx->page[i]) { 288 retval = -ENOMEM; 289 goto alloc_failed; 290 } 291 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 292 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 293 entry->desc_wb = (struct tsnep_tx_desc_wb *) 294 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j); 295 entry->desc = (struct tsnep_tx_desc *) 296 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 297 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j; 298 entry->owner_user_flag = false; 299 } 300 } 301 for (i = 0; i < TSNEP_RING_SIZE; i++) { 302 entry = &tx->entry[i]; 303 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK]; 304 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 305 } 306 307 return 0; 308 309 alloc_failed: 310 tsnep_tx_ring_cleanup(tx); 311 return retval; 312 } 313 314 static void tsnep_tx_init(struct tsnep_tx *tx) 315 { 316 dma_addr_t dma; 317 318 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 319 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW); 320 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH); 321 tx->write = 0; 322 tx->read = 0; 323 tx->owner_counter = 1; 324 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 325 } 326 327 static void tsnep_tx_enable(struct tsnep_tx *tx) 328 { 329 struct netdev_queue *nq; 330 331 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 332 333 __netif_tx_lock_bh(nq); 334 netif_tx_wake_queue(nq); 335 __netif_tx_unlock_bh(nq); 336 } 337 338 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi) 339 { 340 struct netdev_queue *nq; 341 u32 val; 342 343 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 344 345 __netif_tx_lock_bh(nq); 346 netif_tx_stop_queue(nq); 347 __netif_tx_unlock_bh(nq); 348 349 /* wait until TX is done in hardware */ 350 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val, 351 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000, 352 1000000); 353 354 /* wait until TX is also done in software */ 355 while (READ_ONCE(tx->read) != tx->write) { 356 napi_schedule(napi); 357 napi_synchronize(napi); 358 } 359 } 360 361 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, 362 bool last) 363 { 364 struct tsnep_tx_entry *entry = &tx->entry[index]; 365 366 entry->properties = 0; 367 /* xdpf and zc are union with skb */ 368 if (entry->skb) { 369 entry->properties = length & TSNEP_DESC_LENGTH_MASK; 370 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 371 if ((entry->type & TSNEP_TX_TYPE_SKB) && 372 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS)) 373 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; 374 375 /* toggle user flag to prevent false acknowledge 376 * 377 * Only the first fragment is acknowledged. For all other 378 * fragments no acknowledge is done and the last written owner 379 * counter stays in the writeback descriptor. Therefore, it is 380 * possible that the last written owner counter is identical to 381 * the new incremented owner counter and a false acknowledge is 382 * detected before the real acknowledge has been done by 383 * hardware. 384 * 385 * The user flag is used to prevent this situation. The user 386 * flag is copied to the writeback descriptor by the hardware 387 * and is used as additional acknowledge data. By toggeling the 388 * user flag only for the first fragment (which is 389 * acknowledged), it is guaranteed that the last acknowledge 390 * done for this descriptor has used a different user flag and 391 * cannot be detected as false acknowledge. 392 */ 393 entry->owner_user_flag = !entry->owner_user_flag; 394 } 395 if (last) 396 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG; 397 if (index == tx->increment_owner_counter) { 398 tx->owner_counter++; 399 if (tx->owner_counter == 4) 400 tx->owner_counter = 1; 401 tx->increment_owner_counter--; 402 if (tx->increment_owner_counter < 0) 403 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 404 } 405 entry->properties |= 406 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 407 TSNEP_DESC_OWNER_COUNTER_MASK; 408 if (entry->owner_user_flag) 409 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG; 410 entry->desc->more_properties = 411 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK); 412 413 /* descriptor properties shall be written last, because valid data is 414 * signaled there 415 */ 416 dma_wmb(); 417 418 entry->desc->properties = __cpu_to_le32(entry->properties); 419 } 420 421 static int tsnep_tx_desc_available(struct tsnep_tx *tx) 422 { 423 if (tx->read <= tx->write) 424 return TSNEP_RING_SIZE - tx->write + tx->read - 1; 425 else 426 return tx->read - tx->write - 1; 427 } 428 429 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) 430 { 431 struct device *dmadev = tx->adapter->dmadev; 432 struct tsnep_tx_entry *entry; 433 unsigned int len; 434 dma_addr_t dma; 435 int map_len = 0; 436 int i; 437 438 for (i = 0; i < count; i++) { 439 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 440 441 if (!i) { 442 len = skb_headlen(skb); 443 dma = dma_map_single(dmadev, skb->data, len, 444 DMA_TO_DEVICE); 445 446 entry->type = TSNEP_TX_TYPE_SKB; 447 } else { 448 len = skb_frag_size(&skb_shinfo(skb)->frags[i - 1]); 449 dma = skb_frag_dma_map(dmadev, 450 &skb_shinfo(skb)->frags[i - 1], 451 0, len, DMA_TO_DEVICE); 452 453 entry->type = TSNEP_TX_TYPE_SKB_FRAG; 454 } 455 if (dma_mapping_error(dmadev, dma)) 456 return -ENOMEM; 457 458 entry->len = len; 459 dma_unmap_addr_set(entry, dma, dma); 460 461 entry->desc->tx = __cpu_to_le64(dma); 462 463 map_len += len; 464 } 465 466 return map_len; 467 } 468 469 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) 470 { 471 struct device *dmadev = tx->adapter->dmadev; 472 struct tsnep_tx_entry *entry; 473 int map_len = 0; 474 int i; 475 476 for (i = 0; i < count; i++) { 477 entry = &tx->entry[(index + i) & TSNEP_RING_MASK]; 478 479 if (entry->len) { 480 if (entry->type & TSNEP_TX_TYPE_SKB) 481 dma_unmap_single(dmadev, 482 dma_unmap_addr(entry, dma), 483 dma_unmap_len(entry, len), 484 DMA_TO_DEVICE); 485 else if (entry->type & 486 (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_XDP_NDO)) 487 dma_unmap_page(dmadev, 488 dma_unmap_addr(entry, dma), 489 dma_unmap_len(entry, len), 490 DMA_TO_DEVICE); 491 map_len += entry->len; 492 entry->len = 0; 493 } 494 } 495 496 return map_len; 497 } 498 499 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, 500 struct tsnep_tx *tx) 501 { 502 int count = 1; 503 struct tsnep_tx_entry *entry; 504 int length; 505 int i; 506 int retval; 507 508 if (skb_shinfo(skb)->nr_frags > 0) 509 count += skb_shinfo(skb)->nr_frags; 510 511 if (tsnep_tx_desc_available(tx) < count) { 512 /* ring full, shall not happen because queue is stopped if full 513 * below 514 */ 515 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 516 517 return NETDEV_TX_BUSY; 518 } 519 520 entry = &tx->entry[tx->write]; 521 entry->skb = skb; 522 523 retval = tsnep_tx_map(skb, tx, count); 524 if (retval < 0) { 525 tsnep_tx_unmap(tx, tx->write, count); 526 dev_kfree_skb_any(entry->skb); 527 entry->skb = NULL; 528 529 tx->dropped++; 530 531 return NETDEV_TX_OK; 532 } 533 length = retval; 534 535 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 536 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 537 538 for (i = 0; i < count; i++) 539 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 540 i == count - 1); 541 tx->write = (tx->write + count) & TSNEP_RING_MASK; 542 543 skb_tx_timestamp(skb); 544 545 /* descriptor properties shall be valid before hardware is notified */ 546 dma_wmb(); 547 548 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 549 550 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) { 551 /* ring can get full with next frame */ 552 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 553 } 554 555 return NETDEV_TX_OK; 556 } 557 558 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx, 559 struct skb_shared_info *shinfo, int count, u32 type) 560 { 561 struct device *dmadev = tx->adapter->dmadev; 562 struct tsnep_tx_entry *entry; 563 struct page *page; 564 skb_frag_t *frag; 565 unsigned int len; 566 int map_len = 0; 567 dma_addr_t dma; 568 void *data; 569 int i; 570 571 frag = NULL; 572 len = xdpf->len; 573 for (i = 0; i < count; i++) { 574 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 575 if (type & TSNEP_TX_TYPE_XDP_NDO) { 576 data = unlikely(frag) ? skb_frag_address(frag) : 577 xdpf->data; 578 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE); 579 if (dma_mapping_error(dmadev, dma)) 580 return -ENOMEM; 581 582 entry->type = TSNEP_TX_TYPE_XDP_NDO; 583 } else { 584 page = unlikely(frag) ? skb_frag_page(frag) : 585 virt_to_page(xdpf->data); 586 dma = page_pool_get_dma_addr(page); 587 if (unlikely(frag)) 588 dma += skb_frag_off(frag); 589 else 590 dma += sizeof(*xdpf) + xdpf->headroom; 591 dma_sync_single_for_device(dmadev, dma, len, 592 DMA_BIDIRECTIONAL); 593 594 entry->type = TSNEP_TX_TYPE_XDP_TX; 595 } 596 597 entry->len = len; 598 dma_unmap_addr_set(entry, dma, dma); 599 600 entry->desc->tx = __cpu_to_le64(dma); 601 602 map_len += len; 603 604 if (i + 1 < count) { 605 frag = &shinfo->frags[i]; 606 len = skb_frag_size(frag); 607 } 608 } 609 610 return map_len; 611 } 612 613 /* This function requires __netif_tx_lock is held by the caller. */ 614 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf, 615 struct tsnep_tx *tx, u32 type) 616 { 617 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf); 618 struct tsnep_tx_entry *entry; 619 int count, length, retval, i; 620 621 count = 1; 622 if (unlikely(xdp_frame_has_frags(xdpf))) 623 count += shinfo->nr_frags; 624 625 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 626 * will be available for normal TX path and queue is stopped there if 627 * necessary 628 */ 629 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count)) 630 return false; 631 632 entry = &tx->entry[tx->write]; 633 entry->xdpf = xdpf; 634 635 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type); 636 if (retval < 0) { 637 tsnep_tx_unmap(tx, tx->write, count); 638 entry->xdpf = NULL; 639 640 tx->dropped++; 641 642 return false; 643 } 644 length = retval; 645 646 for (i = 0; i < count; i++) 647 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 648 i == count - 1); 649 tx->write = (tx->write + count) & TSNEP_RING_MASK; 650 651 /* descriptor properties shall be valid before hardware is notified */ 652 dma_wmb(); 653 654 return true; 655 } 656 657 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx) 658 { 659 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 660 } 661 662 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter, 663 struct xdp_buff *xdp, 664 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 665 { 666 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 667 bool xmit; 668 669 if (unlikely(!xdpf)) 670 return false; 671 672 __netif_tx_lock(tx_nq, smp_processor_id()); 673 674 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, TSNEP_TX_TYPE_XDP_TX); 675 676 /* Avoid transmit queue timeout since we share it with the slow path */ 677 if (xmit) 678 txq_trans_cond_update(tx_nq); 679 680 __netif_tx_unlock(tx_nq); 681 682 return xmit; 683 } 684 685 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx) 686 { 687 struct tsnep_tx_entry *entry; 688 dma_addr_t dma; 689 690 entry = &tx->entry[tx->write]; 691 entry->zc = true; 692 693 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr); 694 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len); 695 696 entry->type = TSNEP_TX_TYPE_XSK; 697 entry->len = xdpd->len; 698 699 entry->desc->tx = __cpu_to_le64(dma); 700 701 return xdpd->len; 702 } 703 704 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd, 705 struct tsnep_tx *tx) 706 { 707 int length; 708 709 length = tsnep_xdp_tx_map_zc(xdpd, tx); 710 711 tsnep_tx_activate(tx, tx->write, length, true); 712 tx->write = (tx->write + 1) & TSNEP_RING_MASK; 713 } 714 715 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx) 716 { 717 int desc_available = tsnep_tx_desc_available(tx); 718 struct xdp_desc *descs = tx->xsk_pool->tx_descs; 719 int batch, i; 720 721 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 722 * will be available for normal TX path and queue is stopped there if 723 * necessary 724 */ 725 if (desc_available <= (MAX_SKB_FRAGS + 1)) 726 return; 727 desc_available -= MAX_SKB_FRAGS + 1; 728 729 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available); 730 for (i = 0; i < batch; i++) 731 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx); 732 733 if (batch) { 734 /* descriptor properties shall be valid before hardware is 735 * notified 736 */ 737 dma_wmb(); 738 739 tsnep_xdp_xmit_flush(tx); 740 } 741 } 742 743 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) 744 { 745 struct tsnep_tx_entry *entry; 746 struct netdev_queue *nq; 747 int xsk_frames = 0; 748 int budget = 128; 749 int length; 750 int count; 751 752 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 753 __netif_tx_lock(nq, smp_processor_id()); 754 755 do { 756 if (tx->read == tx->write) 757 break; 758 759 entry = &tx->entry[tx->read]; 760 if ((__le32_to_cpu(entry->desc_wb->properties) & 761 TSNEP_TX_DESC_OWNER_MASK) != 762 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 763 break; 764 765 /* descriptor properties shall be read first, because valid data 766 * is signaled there 767 */ 768 dma_rmb(); 769 770 count = 1; 771 if ((entry->type & TSNEP_TX_TYPE_SKB) && 772 skb_shinfo(entry->skb)->nr_frags > 0) 773 count += skb_shinfo(entry->skb)->nr_frags; 774 else if ((entry->type & TSNEP_TX_TYPE_XDP) && 775 xdp_frame_has_frags(entry->xdpf)) 776 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags; 777 778 length = tsnep_tx_unmap(tx, tx->read, count); 779 780 if ((entry->type & TSNEP_TX_TYPE_SKB) && 781 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && 782 (__le32_to_cpu(entry->desc_wb->properties) & 783 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) { 784 struct skb_shared_hwtstamps hwtstamps; 785 u64 timestamp; 786 787 if (skb_shinfo(entry->skb)->tx_flags & 788 SKBTX_HW_TSTAMP_USE_CYCLES) 789 timestamp = 790 __le64_to_cpu(entry->desc_wb->counter); 791 else 792 timestamp = 793 __le64_to_cpu(entry->desc_wb->timestamp); 794 795 memset(&hwtstamps, 0, sizeof(hwtstamps)); 796 hwtstamps.hwtstamp = ns_to_ktime(timestamp); 797 798 skb_tstamp_tx(entry->skb, &hwtstamps); 799 } 800 801 if (entry->type & TSNEP_TX_TYPE_SKB) 802 napi_consume_skb(entry->skb, napi_budget); 803 else if (entry->type & TSNEP_TX_TYPE_XDP) 804 xdp_return_frame_rx_napi(entry->xdpf); 805 else 806 xsk_frames++; 807 /* xdpf and zc are union with skb */ 808 entry->skb = NULL; 809 810 tx->read = (tx->read + count) & TSNEP_RING_MASK; 811 812 tx->packets++; 813 tx->bytes += length + ETH_FCS_LEN; 814 815 budget--; 816 } while (likely(budget)); 817 818 if (tx->xsk_pool) { 819 if (xsk_frames) 820 xsk_tx_completed(tx->xsk_pool, xsk_frames); 821 if (xsk_uses_need_wakeup(tx->xsk_pool)) 822 xsk_set_tx_need_wakeup(tx->xsk_pool); 823 tsnep_xdp_xmit_zc(tx); 824 } 825 826 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) && 827 netif_tx_queue_stopped(nq)) { 828 netif_tx_wake_queue(nq); 829 } 830 831 __netif_tx_unlock(nq); 832 833 return budget != 0; 834 } 835 836 static bool tsnep_tx_pending(struct tsnep_tx *tx) 837 { 838 struct tsnep_tx_entry *entry; 839 struct netdev_queue *nq; 840 bool pending = false; 841 842 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 843 __netif_tx_lock(nq, smp_processor_id()); 844 845 if (tx->read != tx->write) { 846 entry = &tx->entry[tx->read]; 847 if ((__le32_to_cpu(entry->desc_wb->properties) & 848 TSNEP_TX_DESC_OWNER_MASK) == 849 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 850 pending = true; 851 } 852 853 __netif_tx_unlock(nq); 854 855 return pending; 856 } 857 858 static int tsnep_tx_open(struct tsnep_tx *tx) 859 { 860 int retval; 861 862 retval = tsnep_tx_ring_create(tx); 863 if (retval) 864 return retval; 865 866 tsnep_tx_init(tx); 867 868 return 0; 869 } 870 871 static void tsnep_tx_close(struct tsnep_tx *tx) 872 { 873 tsnep_tx_ring_cleanup(tx); 874 } 875 876 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) 877 { 878 struct device *dmadev = rx->adapter->dmadev; 879 struct tsnep_rx_entry *entry; 880 int i; 881 882 for (i = 0; i < TSNEP_RING_SIZE; i++) { 883 entry = &rx->entry[i]; 884 if (!rx->xsk_pool && entry->page) 885 page_pool_put_full_page(rx->page_pool, entry->page, 886 false); 887 if (rx->xsk_pool && entry->xdp) 888 xsk_buff_free(entry->xdp); 889 /* xdp is union with page */ 890 entry->page = NULL; 891 } 892 893 if (rx->page_pool) 894 page_pool_destroy(rx->page_pool); 895 896 memset(rx->entry, 0, sizeof(rx->entry)); 897 898 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 899 if (rx->page[i]) { 900 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i], 901 rx->page_dma[i]); 902 rx->page[i] = NULL; 903 rx->page_dma[i] = 0; 904 } 905 } 906 } 907 908 static int tsnep_rx_ring_create(struct tsnep_rx *rx) 909 { 910 struct device *dmadev = rx->adapter->dmadev; 911 struct tsnep_rx_entry *entry; 912 struct page_pool_params pp_params = { 0 }; 913 struct tsnep_rx_entry *next_entry; 914 int i, j; 915 int retval; 916 917 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 918 rx->page[i] = 919 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i], 920 GFP_KERNEL); 921 if (!rx->page[i]) { 922 retval = -ENOMEM; 923 goto failed; 924 } 925 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 926 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 927 entry->desc_wb = (struct tsnep_rx_desc_wb *) 928 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j); 929 entry->desc = (struct tsnep_rx_desc *) 930 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 931 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j; 932 } 933 } 934 935 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 936 pp_params.order = 0; 937 pp_params.pool_size = TSNEP_RING_SIZE; 938 pp_params.nid = dev_to_node(dmadev); 939 pp_params.dev = dmadev; 940 pp_params.dma_dir = DMA_BIDIRECTIONAL; 941 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE; 942 pp_params.offset = TSNEP_RX_OFFSET; 943 rx->page_pool = page_pool_create(&pp_params); 944 if (IS_ERR(rx->page_pool)) { 945 retval = PTR_ERR(rx->page_pool); 946 rx->page_pool = NULL; 947 goto failed; 948 } 949 950 for (i = 0; i < TSNEP_RING_SIZE; i++) { 951 entry = &rx->entry[i]; 952 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK]; 953 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 954 } 955 956 return 0; 957 958 failed: 959 tsnep_rx_ring_cleanup(rx); 960 return retval; 961 } 962 963 static void tsnep_rx_init(struct tsnep_rx *rx) 964 { 965 dma_addr_t dma; 966 967 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 968 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW); 969 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH); 970 rx->write = 0; 971 rx->read = 0; 972 rx->owner_counter = 1; 973 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 974 } 975 976 static void tsnep_rx_enable(struct tsnep_rx *rx) 977 { 978 /* descriptor properties shall be valid before hardware is notified */ 979 dma_wmb(); 980 981 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); 982 } 983 984 static void tsnep_rx_disable(struct tsnep_rx *rx) 985 { 986 u32 val; 987 988 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL); 989 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val, 990 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000, 991 1000000); 992 } 993 994 static int tsnep_rx_desc_available(struct tsnep_rx *rx) 995 { 996 if (rx->read <= rx->write) 997 return TSNEP_RING_SIZE - rx->write + rx->read - 1; 998 else 999 return rx->read - rx->write - 1; 1000 } 1001 1002 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx) 1003 { 1004 struct page **page; 1005 1006 /* last entry of page_buffer is always zero, because ring cannot be 1007 * filled completely 1008 */ 1009 page = rx->page_buffer; 1010 while (*page) { 1011 page_pool_put_full_page(rx->page_pool, *page, false); 1012 *page = NULL; 1013 page++; 1014 } 1015 } 1016 1017 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx) 1018 { 1019 int i; 1020 1021 /* alloc for all ring entries except the last one, because ring cannot 1022 * be filled completely 1023 */ 1024 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) { 1025 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool); 1026 if (!rx->page_buffer[i]) { 1027 tsnep_rx_free_page_buffer(rx); 1028 1029 return -ENOMEM; 1030 } 1031 } 1032 1033 return 0; 1034 } 1035 1036 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1037 struct page *page) 1038 { 1039 entry->page = page; 1040 entry->len = TSNEP_MAX_RX_BUF_SIZE; 1041 entry->dma = page_pool_get_dma_addr(entry->page); 1042 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET); 1043 } 1044 1045 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index) 1046 { 1047 struct tsnep_rx_entry *entry = &rx->entry[index]; 1048 struct page *page; 1049 1050 page = page_pool_dev_alloc_pages(rx->page_pool); 1051 if (unlikely(!page)) 1052 return -ENOMEM; 1053 tsnep_rx_set_page(rx, entry, page); 1054 1055 return 0; 1056 } 1057 1058 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index) 1059 { 1060 struct tsnep_rx_entry *entry = &rx->entry[index]; 1061 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1062 1063 tsnep_rx_set_page(rx, entry, read->page); 1064 read->page = NULL; 1065 } 1066 1067 static void tsnep_rx_activate(struct tsnep_rx *rx, int index) 1068 { 1069 struct tsnep_rx_entry *entry = &rx->entry[index]; 1070 1071 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */ 1072 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK; 1073 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 1074 if (index == rx->increment_owner_counter) { 1075 rx->owner_counter++; 1076 if (rx->owner_counter == 4) 1077 rx->owner_counter = 1; 1078 rx->increment_owner_counter--; 1079 if (rx->increment_owner_counter < 0) 1080 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1081 } 1082 entry->properties |= 1083 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 1084 TSNEP_DESC_OWNER_COUNTER_MASK; 1085 1086 /* descriptor properties shall be written last, because valid data is 1087 * signaled there 1088 */ 1089 dma_wmb(); 1090 1091 entry->desc->properties = __cpu_to_le32(entry->properties); 1092 } 1093 1094 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse) 1095 { 1096 bool alloc_failed = false; 1097 int i, index; 1098 1099 for (i = 0; i < count && !alloc_failed; i++) { 1100 index = (rx->write + i) & TSNEP_RING_MASK; 1101 1102 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) { 1103 rx->alloc_failed++; 1104 alloc_failed = true; 1105 1106 /* reuse only if no other allocation was successful */ 1107 if (i == 0 && reuse) 1108 tsnep_rx_reuse_buffer(rx, index); 1109 else 1110 break; 1111 } 1112 1113 tsnep_rx_activate(rx, index); 1114 } 1115 1116 if (i) 1117 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1118 1119 return i; 1120 } 1121 1122 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse) 1123 { 1124 int desc_refilled; 1125 1126 desc_refilled = tsnep_rx_alloc(rx, count, reuse); 1127 if (desc_refilled) 1128 tsnep_rx_enable(rx); 1129 1130 return desc_refilled; 1131 } 1132 1133 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1134 struct xdp_buff *xdp) 1135 { 1136 entry->xdp = xdp; 1137 entry->len = TSNEP_XSK_RX_BUF_SIZE; 1138 entry->dma = xsk_buff_xdp_get_dma(entry->xdp); 1139 entry->desc->rx = __cpu_to_le64(entry->dma); 1140 } 1141 1142 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index) 1143 { 1144 struct tsnep_rx_entry *entry = &rx->entry[index]; 1145 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1146 1147 tsnep_rx_set_xdp(rx, entry, read->xdp); 1148 read->xdp = NULL; 1149 } 1150 1151 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse) 1152 { 1153 u32 allocated; 1154 int i; 1155 1156 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count); 1157 for (i = 0; i < allocated; i++) { 1158 int index = (rx->write + i) & TSNEP_RING_MASK; 1159 struct tsnep_rx_entry *entry = &rx->entry[index]; 1160 1161 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]); 1162 tsnep_rx_activate(rx, index); 1163 } 1164 if (i == 0) { 1165 rx->alloc_failed++; 1166 1167 if (reuse) { 1168 tsnep_rx_reuse_buffer_zc(rx, rx->write); 1169 tsnep_rx_activate(rx, rx->write); 1170 } 1171 } 1172 1173 if (i) 1174 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1175 1176 return i; 1177 } 1178 1179 static void tsnep_rx_free_zc(struct tsnep_rx *rx) 1180 { 1181 int i; 1182 1183 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1184 struct tsnep_rx_entry *entry = &rx->entry[i]; 1185 1186 if (entry->xdp) 1187 xsk_buff_free(entry->xdp); 1188 entry->xdp = NULL; 1189 } 1190 } 1191 1192 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse) 1193 { 1194 int desc_refilled; 1195 1196 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse); 1197 if (desc_refilled) 1198 tsnep_rx_enable(rx); 1199 1200 return desc_refilled; 1201 } 1202 1203 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog, 1204 struct xdp_buff *xdp, int *status, 1205 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1206 { 1207 unsigned int length; 1208 unsigned int sync; 1209 u32 act; 1210 1211 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM; 1212 1213 act = bpf_prog_run_xdp(prog, xdp); 1214 switch (act) { 1215 case XDP_PASS: 1216 return false; 1217 case XDP_TX: 1218 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx)) 1219 goto out_failure; 1220 *status |= TSNEP_XDP_TX; 1221 return true; 1222 case XDP_REDIRECT: 1223 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1224 goto out_failure; 1225 *status |= TSNEP_XDP_REDIRECT; 1226 return true; 1227 default: 1228 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1229 fallthrough; 1230 case XDP_ABORTED: 1231 out_failure: 1232 trace_xdp_exception(rx->adapter->netdev, prog, act); 1233 fallthrough; 1234 case XDP_DROP: 1235 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU 1236 * touch 1237 */ 1238 sync = xdp->data_end - xdp->data_hard_start - 1239 XDP_PACKET_HEADROOM; 1240 sync = max(sync, length); 1241 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data), 1242 sync, true); 1243 return true; 1244 } 1245 } 1246 1247 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog, 1248 struct xdp_buff *xdp, int *status, 1249 struct netdev_queue *tx_nq, 1250 struct tsnep_tx *tx) 1251 { 1252 u32 act; 1253 1254 act = bpf_prog_run_xdp(prog, xdp); 1255 1256 /* XDP_REDIRECT is the main action for zero-copy */ 1257 if (likely(act == XDP_REDIRECT)) { 1258 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1259 goto out_failure; 1260 *status |= TSNEP_XDP_REDIRECT; 1261 return true; 1262 } 1263 1264 switch (act) { 1265 case XDP_PASS: 1266 return false; 1267 case XDP_TX: 1268 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx)) 1269 goto out_failure; 1270 *status |= TSNEP_XDP_TX; 1271 return true; 1272 default: 1273 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1274 fallthrough; 1275 case XDP_ABORTED: 1276 out_failure: 1277 trace_xdp_exception(rx->adapter->netdev, prog, act); 1278 fallthrough; 1279 case XDP_DROP: 1280 xsk_buff_free(xdp); 1281 return true; 1282 } 1283 } 1284 1285 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status, 1286 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1287 { 1288 if (status & TSNEP_XDP_TX) { 1289 __netif_tx_lock(tx_nq, smp_processor_id()); 1290 tsnep_xdp_xmit_flush(tx); 1291 __netif_tx_unlock(tx_nq); 1292 } 1293 1294 if (status & TSNEP_XDP_REDIRECT) 1295 xdp_do_flush(); 1296 } 1297 1298 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page, 1299 int length) 1300 { 1301 struct sk_buff *skb; 1302 1303 skb = napi_build_skb(page_address(page), PAGE_SIZE); 1304 if (unlikely(!skb)) 1305 return NULL; 1306 1307 /* update pointers within the skb to store the data */ 1308 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE); 1309 __skb_put(skb, length - ETH_FCS_LEN); 1310 1311 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) { 1312 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb); 1313 struct tsnep_rx_inline *rx_inline = 1314 (struct tsnep_rx_inline *)(page_address(page) + 1315 TSNEP_RX_OFFSET); 1316 1317 skb_shinfo(skb)->tx_flags |= 1318 SKBTX_HW_TSTAMP_NETDEV; 1319 memset(hwtstamps, 0, sizeof(*hwtstamps)); 1320 hwtstamps->netdev_data = rx_inline; 1321 } 1322 1323 skb_record_rx_queue(skb, rx->queue_index); 1324 skb->protocol = eth_type_trans(skb, rx->adapter->netdev); 1325 1326 return skb; 1327 } 1328 1329 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi, 1330 struct page *page, int length) 1331 { 1332 struct sk_buff *skb; 1333 1334 skb = tsnep_build_skb(rx, page, length); 1335 if (skb) { 1336 page_pool_release_page(rx->page_pool, page); 1337 1338 rx->packets++; 1339 rx->bytes += length; 1340 if (skb->pkt_type == PACKET_MULTICAST) 1341 rx->multicast++; 1342 1343 napi_gro_receive(napi, skb); 1344 } else { 1345 page_pool_recycle_direct(rx->page_pool, page); 1346 1347 rx->dropped++; 1348 } 1349 } 1350 1351 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 1352 int budget) 1353 { 1354 struct device *dmadev = rx->adapter->dmadev; 1355 enum dma_data_direction dma_dir; 1356 struct tsnep_rx_entry *entry; 1357 struct netdev_queue *tx_nq; 1358 struct bpf_prog *prog; 1359 struct xdp_buff xdp; 1360 struct tsnep_tx *tx; 1361 int desc_available; 1362 int xdp_status = 0; 1363 int done = 0; 1364 int length; 1365 1366 desc_available = tsnep_rx_desc_available(rx); 1367 dma_dir = page_pool_get_dma_dir(rx->page_pool); 1368 prog = READ_ONCE(rx->adapter->xdp_prog); 1369 if (prog) { 1370 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1371 rx->tx_queue_index); 1372 tx = &rx->adapter->tx[rx->tx_queue_index]; 1373 1374 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq); 1375 } 1376 1377 while (likely(done < budget) && (rx->read != rx->write)) { 1378 entry = &rx->entry[rx->read]; 1379 if ((__le32_to_cpu(entry->desc_wb->properties) & 1380 TSNEP_DESC_OWNER_COUNTER_MASK) != 1381 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1382 break; 1383 done++; 1384 1385 if (desc_available >= TSNEP_RING_RX_REFILL) { 1386 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1387 1388 desc_available -= tsnep_rx_refill(rx, desc_available, 1389 reuse); 1390 if (!entry->page) { 1391 /* buffer has been reused for refill to prevent 1392 * empty RX ring, thus buffer cannot be used for 1393 * RX processing 1394 */ 1395 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1396 desc_available++; 1397 1398 rx->dropped++; 1399 1400 continue; 1401 } 1402 } 1403 1404 /* descriptor properties shall be read first, because valid data 1405 * is signaled there 1406 */ 1407 dma_rmb(); 1408 1409 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET); 1410 length = __le32_to_cpu(entry->desc_wb->properties) & 1411 TSNEP_DESC_LENGTH_MASK; 1412 dma_sync_single_range_for_cpu(dmadev, entry->dma, 1413 TSNEP_RX_OFFSET, length, dma_dir); 1414 1415 /* RX metadata with timestamps is in front of actual data, 1416 * subtract metadata size to get length of actual data and 1417 * consider metadata size as offset of actual data during RX 1418 * processing 1419 */ 1420 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1421 1422 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1423 desc_available++; 1424 1425 if (prog) { 1426 bool consume; 1427 1428 xdp_prepare_buff(&xdp, page_address(entry->page), 1429 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE, 1430 length, false); 1431 1432 consume = tsnep_xdp_run_prog(rx, prog, &xdp, 1433 &xdp_status, tx_nq, tx); 1434 if (consume) { 1435 rx->packets++; 1436 rx->bytes += length; 1437 1438 entry->page = NULL; 1439 1440 continue; 1441 } 1442 } 1443 1444 tsnep_rx_page(rx, napi, entry->page, length); 1445 entry->page = NULL; 1446 } 1447 1448 if (xdp_status) 1449 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1450 1451 if (desc_available) 1452 tsnep_rx_refill(rx, desc_available, false); 1453 1454 return done; 1455 } 1456 1457 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi, 1458 int budget) 1459 { 1460 struct tsnep_rx_entry *entry; 1461 struct netdev_queue *tx_nq; 1462 struct bpf_prog *prog; 1463 struct tsnep_tx *tx; 1464 int desc_available; 1465 int xdp_status = 0; 1466 struct page *page; 1467 int done = 0; 1468 int length; 1469 1470 desc_available = tsnep_rx_desc_available(rx); 1471 prog = READ_ONCE(rx->adapter->xdp_prog); 1472 if (prog) { 1473 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1474 rx->tx_queue_index); 1475 tx = &rx->adapter->tx[rx->tx_queue_index]; 1476 } 1477 1478 while (likely(done < budget) && (rx->read != rx->write)) { 1479 entry = &rx->entry[rx->read]; 1480 if ((__le32_to_cpu(entry->desc_wb->properties) & 1481 TSNEP_DESC_OWNER_COUNTER_MASK) != 1482 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1483 break; 1484 done++; 1485 1486 if (desc_available >= TSNEP_RING_RX_REFILL) { 1487 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1488 1489 desc_available -= tsnep_rx_refill_zc(rx, desc_available, 1490 reuse); 1491 if (!entry->xdp) { 1492 /* buffer has been reused for refill to prevent 1493 * empty RX ring, thus buffer cannot be used for 1494 * RX processing 1495 */ 1496 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1497 desc_available++; 1498 1499 rx->dropped++; 1500 1501 continue; 1502 } 1503 } 1504 1505 /* descriptor properties shall be read first, because valid data 1506 * is signaled there 1507 */ 1508 dma_rmb(); 1509 1510 prefetch(entry->xdp->data); 1511 length = __le32_to_cpu(entry->desc_wb->properties) & 1512 TSNEP_DESC_LENGTH_MASK; 1513 xsk_buff_set_size(entry->xdp, length); 1514 xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool); 1515 1516 /* RX metadata with timestamps is in front of actual data, 1517 * subtract metadata size to get length of actual data and 1518 * consider metadata size as offset of actual data during RX 1519 * processing 1520 */ 1521 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1522 1523 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1524 desc_available++; 1525 1526 if (prog) { 1527 bool consume; 1528 1529 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE; 1530 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE; 1531 1532 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp, 1533 &xdp_status, tx_nq, tx); 1534 if (consume) { 1535 rx->packets++; 1536 rx->bytes += length; 1537 1538 entry->xdp = NULL; 1539 1540 continue; 1541 } 1542 } 1543 1544 page = page_pool_dev_alloc_pages(rx->page_pool); 1545 if (page) { 1546 memcpy(page_address(page) + TSNEP_RX_OFFSET, 1547 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE, 1548 length + TSNEP_RX_INLINE_METADATA_SIZE); 1549 tsnep_rx_page(rx, napi, page, length); 1550 } else { 1551 rx->dropped++; 1552 } 1553 xsk_buff_free(entry->xdp); 1554 entry->xdp = NULL; 1555 } 1556 1557 if (xdp_status) 1558 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1559 1560 if (desc_available) 1561 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false); 1562 1563 if (xsk_uses_need_wakeup(rx->xsk_pool)) { 1564 if (desc_available) 1565 xsk_set_rx_need_wakeup(rx->xsk_pool); 1566 else 1567 xsk_clear_rx_need_wakeup(rx->xsk_pool); 1568 1569 return done; 1570 } 1571 1572 return desc_available ? budget : done; 1573 } 1574 1575 static bool tsnep_rx_pending(struct tsnep_rx *rx) 1576 { 1577 struct tsnep_rx_entry *entry; 1578 1579 if (rx->read != rx->write) { 1580 entry = &rx->entry[rx->read]; 1581 if ((__le32_to_cpu(entry->desc_wb->properties) & 1582 TSNEP_DESC_OWNER_COUNTER_MASK) == 1583 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1584 return true; 1585 } 1586 1587 return false; 1588 } 1589 1590 static int tsnep_rx_open(struct tsnep_rx *rx) 1591 { 1592 int desc_available; 1593 int retval; 1594 1595 retval = tsnep_rx_ring_create(rx); 1596 if (retval) 1597 return retval; 1598 1599 tsnep_rx_init(rx); 1600 1601 desc_available = tsnep_rx_desc_available(rx); 1602 if (rx->xsk_pool) 1603 retval = tsnep_rx_alloc_zc(rx, desc_available, false); 1604 else 1605 retval = tsnep_rx_alloc(rx, desc_available, false); 1606 if (retval != desc_available) { 1607 retval = -ENOMEM; 1608 1609 goto alloc_failed; 1610 } 1611 1612 /* prealloc pages to prevent allocation failures when XSK pool is 1613 * disabled at runtime 1614 */ 1615 if (rx->xsk_pool) { 1616 retval = tsnep_rx_alloc_page_buffer(rx); 1617 if (retval) 1618 goto alloc_failed; 1619 } 1620 1621 return 0; 1622 1623 alloc_failed: 1624 tsnep_rx_ring_cleanup(rx); 1625 return retval; 1626 } 1627 1628 static void tsnep_rx_close(struct tsnep_rx *rx) 1629 { 1630 if (rx->xsk_pool) 1631 tsnep_rx_free_page_buffer(rx); 1632 1633 tsnep_rx_ring_cleanup(rx); 1634 } 1635 1636 static void tsnep_rx_reopen(struct tsnep_rx *rx) 1637 { 1638 struct page **page = rx->page_buffer; 1639 int i; 1640 1641 tsnep_rx_init(rx); 1642 1643 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1644 struct tsnep_rx_entry *entry = &rx->entry[i]; 1645 1646 /* defined initial values for properties are required for 1647 * correct owner counter checking 1648 */ 1649 entry->desc->properties = 0; 1650 entry->desc_wb->properties = 0; 1651 1652 /* prevent allocation failures by reusing kept pages */ 1653 if (*page) { 1654 tsnep_rx_set_page(rx, entry, *page); 1655 tsnep_rx_activate(rx, rx->write); 1656 rx->write++; 1657 1658 *page = NULL; 1659 page++; 1660 } 1661 } 1662 } 1663 1664 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx) 1665 { 1666 struct page **page = rx->page_buffer; 1667 u32 allocated; 1668 int i; 1669 1670 tsnep_rx_init(rx); 1671 1672 /* alloc all ring entries except the last one, because ring cannot be 1673 * filled completely, as many buffers as possible is enough as wakeup is 1674 * done if new buffers are available 1675 */ 1676 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, 1677 TSNEP_RING_SIZE - 1); 1678 1679 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1680 struct tsnep_rx_entry *entry = &rx->entry[i]; 1681 1682 /* keep pages to prevent allocation failures when xsk is 1683 * disabled 1684 */ 1685 if (entry->page) { 1686 *page = entry->page; 1687 entry->page = NULL; 1688 1689 page++; 1690 } 1691 1692 /* defined initial values for properties are required for 1693 * correct owner counter checking 1694 */ 1695 entry->desc->properties = 0; 1696 entry->desc_wb->properties = 0; 1697 1698 if (allocated) { 1699 tsnep_rx_set_xdp(rx, entry, 1700 rx->xdp_batch[allocated - 1]); 1701 tsnep_rx_activate(rx, rx->write); 1702 rx->write++; 1703 1704 allocated--; 1705 } 1706 } 1707 } 1708 1709 static bool tsnep_pending(struct tsnep_queue *queue) 1710 { 1711 if (queue->tx && tsnep_tx_pending(queue->tx)) 1712 return true; 1713 1714 if (queue->rx && tsnep_rx_pending(queue->rx)) 1715 return true; 1716 1717 return false; 1718 } 1719 1720 static int tsnep_poll(struct napi_struct *napi, int budget) 1721 { 1722 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue, 1723 napi); 1724 bool complete = true; 1725 int done = 0; 1726 1727 if (queue->tx) 1728 complete = tsnep_tx_poll(queue->tx, budget); 1729 1730 if (queue->rx) { 1731 done = queue->rx->xsk_pool ? 1732 tsnep_rx_poll_zc(queue->rx, napi, budget) : 1733 tsnep_rx_poll(queue->rx, napi, budget); 1734 if (done >= budget) 1735 complete = false; 1736 } 1737 1738 /* if all work not completed, return budget and keep polling */ 1739 if (!complete) 1740 return budget; 1741 1742 if (likely(napi_complete_done(napi, done))) { 1743 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1744 1745 /* reschedule if work is already pending, prevent rotten packets 1746 * which are transmitted or received after polling but before 1747 * interrupt enable 1748 */ 1749 if (tsnep_pending(queue)) { 1750 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1751 napi_schedule(napi); 1752 } 1753 } 1754 1755 return min(done, budget - 1); 1756 } 1757 1758 static int tsnep_request_irq(struct tsnep_queue *queue, bool first) 1759 { 1760 const char *name = netdev_name(queue->adapter->netdev); 1761 irq_handler_t handler; 1762 void *dev; 1763 int retval; 1764 1765 if (first) { 1766 sprintf(queue->name, "%s-mac", name); 1767 handler = tsnep_irq; 1768 dev = queue->adapter; 1769 } else { 1770 if (queue->tx && queue->rx) 1771 sprintf(queue->name, "%s-txrx-%d", name, 1772 queue->rx->queue_index); 1773 else if (queue->tx) 1774 sprintf(queue->name, "%s-tx-%d", name, 1775 queue->tx->queue_index); 1776 else 1777 sprintf(queue->name, "%s-rx-%d", name, 1778 queue->rx->queue_index); 1779 handler = tsnep_irq_txrx; 1780 dev = queue; 1781 } 1782 1783 retval = request_irq(queue->irq, handler, 0, queue->name, dev); 1784 if (retval) { 1785 /* if name is empty, then interrupt won't be freed */ 1786 memset(queue->name, 0, sizeof(queue->name)); 1787 } 1788 1789 return retval; 1790 } 1791 1792 static void tsnep_free_irq(struct tsnep_queue *queue, bool first) 1793 { 1794 void *dev; 1795 1796 if (!strlen(queue->name)) 1797 return; 1798 1799 if (first) 1800 dev = queue->adapter; 1801 else 1802 dev = queue; 1803 1804 free_irq(queue->irq, dev); 1805 memset(queue->name, 0, sizeof(queue->name)); 1806 } 1807 1808 static void tsnep_queue_close(struct tsnep_queue *queue, bool first) 1809 { 1810 struct tsnep_rx *rx = queue->rx; 1811 1812 tsnep_free_irq(queue, first); 1813 1814 if (rx) { 1815 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1816 xdp_rxq_info_unreg(&rx->xdp_rxq); 1817 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc)) 1818 xdp_rxq_info_unreg(&rx->xdp_rxq_zc); 1819 } 1820 1821 netif_napi_del(&queue->napi); 1822 } 1823 1824 static int tsnep_queue_open(struct tsnep_adapter *adapter, 1825 struct tsnep_queue *queue, bool first) 1826 { 1827 struct tsnep_rx *rx = queue->rx; 1828 struct tsnep_tx *tx = queue->tx; 1829 int retval; 1830 1831 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll); 1832 1833 if (rx) { 1834 /* choose TX queue for XDP_TX */ 1835 if (tx) 1836 rx->tx_queue_index = tx->queue_index; 1837 else if (rx->queue_index < adapter->num_tx_queues) 1838 rx->tx_queue_index = rx->queue_index; 1839 else 1840 rx->tx_queue_index = 0; 1841 1842 /* prepare both memory models to eliminate possible registration 1843 * errors when memory model is switched between page pool and 1844 * XSK pool during runtime 1845 */ 1846 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev, 1847 rx->queue_index, queue->napi.napi_id); 1848 if (retval) 1849 goto failed; 1850 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1851 MEM_TYPE_PAGE_POOL, 1852 rx->page_pool); 1853 if (retval) 1854 goto failed; 1855 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev, 1856 rx->queue_index, queue->napi.napi_id); 1857 if (retval) 1858 goto failed; 1859 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc, 1860 MEM_TYPE_XSK_BUFF_POOL, 1861 NULL); 1862 if (retval) 1863 goto failed; 1864 if (rx->xsk_pool) 1865 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc); 1866 } 1867 1868 retval = tsnep_request_irq(queue, first); 1869 if (retval) { 1870 netif_err(adapter, drv, adapter->netdev, 1871 "can't get assigned irq %d.\n", queue->irq); 1872 goto failed; 1873 } 1874 1875 return 0; 1876 1877 failed: 1878 tsnep_queue_close(queue, first); 1879 1880 return retval; 1881 } 1882 1883 static void tsnep_queue_enable(struct tsnep_queue *queue) 1884 { 1885 napi_enable(&queue->napi); 1886 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1887 1888 if (queue->tx) 1889 tsnep_tx_enable(queue->tx); 1890 1891 if (queue->rx) 1892 tsnep_rx_enable(queue->rx); 1893 } 1894 1895 static void tsnep_queue_disable(struct tsnep_queue *queue) 1896 { 1897 if (queue->tx) 1898 tsnep_tx_disable(queue->tx, &queue->napi); 1899 1900 napi_disable(&queue->napi); 1901 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1902 1903 /* disable RX after NAPI polling has been disabled, because RX can be 1904 * enabled during NAPI polling 1905 */ 1906 if (queue->rx) 1907 tsnep_rx_disable(queue->rx); 1908 } 1909 1910 static int tsnep_netdev_open(struct net_device *netdev) 1911 { 1912 struct tsnep_adapter *adapter = netdev_priv(netdev); 1913 int i, retval; 1914 1915 for (i = 0; i < adapter->num_queues; i++) { 1916 if (adapter->queue[i].tx) { 1917 retval = tsnep_tx_open(adapter->queue[i].tx); 1918 if (retval) 1919 goto failed; 1920 } 1921 if (adapter->queue[i].rx) { 1922 retval = tsnep_rx_open(adapter->queue[i].rx); 1923 if (retval) 1924 goto failed; 1925 } 1926 1927 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0); 1928 if (retval) 1929 goto failed; 1930 } 1931 1932 retval = netif_set_real_num_tx_queues(adapter->netdev, 1933 adapter->num_tx_queues); 1934 if (retval) 1935 goto failed; 1936 retval = netif_set_real_num_rx_queues(adapter->netdev, 1937 adapter->num_rx_queues); 1938 if (retval) 1939 goto failed; 1940 1941 tsnep_enable_irq(adapter, ECM_INT_LINK); 1942 retval = tsnep_phy_open(adapter); 1943 if (retval) 1944 goto phy_failed; 1945 1946 for (i = 0; i < adapter->num_queues; i++) 1947 tsnep_queue_enable(&adapter->queue[i]); 1948 1949 return 0; 1950 1951 phy_failed: 1952 tsnep_disable_irq(adapter, ECM_INT_LINK); 1953 failed: 1954 for (i = 0; i < adapter->num_queues; i++) { 1955 tsnep_queue_close(&adapter->queue[i], i == 0); 1956 1957 if (adapter->queue[i].rx) 1958 tsnep_rx_close(adapter->queue[i].rx); 1959 if (adapter->queue[i].tx) 1960 tsnep_tx_close(adapter->queue[i].tx); 1961 } 1962 return retval; 1963 } 1964 1965 static int tsnep_netdev_close(struct net_device *netdev) 1966 { 1967 struct tsnep_adapter *adapter = netdev_priv(netdev); 1968 int i; 1969 1970 tsnep_disable_irq(adapter, ECM_INT_LINK); 1971 tsnep_phy_close(adapter); 1972 1973 for (i = 0; i < adapter->num_queues; i++) { 1974 tsnep_queue_disable(&adapter->queue[i]); 1975 1976 tsnep_queue_close(&adapter->queue[i], i == 0); 1977 1978 if (adapter->queue[i].rx) 1979 tsnep_rx_close(adapter->queue[i].rx); 1980 if (adapter->queue[i].tx) 1981 tsnep_tx_close(adapter->queue[i].tx); 1982 } 1983 1984 return 0; 1985 } 1986 1987 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool) 1988 { 1989 bool running = netif_running(queue->adapter->netdev); 1990 u32 frame_size; 1991 1992 frame_size = xsk_pool_get_rx_frame_size(pool); 1993 if (frame_size < TSNEP_XSK_RX_BUF_SIZE) 1994 return -EOPNOTSUPP; 1995 1996 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE, 1997 sizeof(*queue->rx->page_buffer), 1998 GFP_KERNEL); 1999 if (!queue->rx->page_buffer) 2000 return -ENOMEM; 2001 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE, 2002 sizeof(*queue->rx->xdp_batch), 2003 GFP_KERNEL); 2004 if (!queue->rx->xdp_batch) { 2005 kfree(queue->rx->page_buffer); 2006 queue->rx->page_buffer = NULL; 2007 2008 return -ENOMEM; 2009 } 2010 2011 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc); 2012 2013 if (running) 2014 tsnep_queue_disable(queue); 2015 2016 queue->tx->xsk_pool = pool; 2017 queue->rx->xsk_pool = pool; 2018 2019 if (running) { 2020 tsnep_rx_reopen_xsk(queue->rx); 2021 tsnep_queue_enable(queue); 2022 } 2023 2024 return 0; 2025 } 2026 2027 void tsnep_disable_xsk(struct tsnep_queue *queue) 2028 { 2029 bool running = netif_running(queue->adapter->netdev); 2030 2031 if (running) 2032 tsnep_queue_disable(queue); 2033 2034 tsnep_rx_free_zc(queue->rx); 2035 2036 queue->rx->xsk_pool = NULL; 2037 queue->tx->xsk_pool = NULL; 2038 2039 if (running) { 2040 tsnep_rx_reopen(queue->rx); 2041 tsnep_queue_enable(queue); 2042 } 2043 2044 kfree(queue->rx->xdp_batch); 2045 queue->rx->xdp_batch = NULL; 2046 kfree(queue->rx->page_buffer); 2047 queue->rx->page_buffer = NULL; 2048 } 2049 2050 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb, 2051 struct net_device *netdev) 2052 { 2053 struct tsnep_adapter *adapter = netdev_priv(netdev); 2054 u16 queue_mapping = skb_get_queue_mapping(skb); 2055 2056 if (queue_mapping >= adapter->num_tx_queues) 2057 queue_mapping = 0; 2058 2059 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]); 2060 } 2061 2062 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr, 2063 int cmd) 2064 { 2065 if (!netif_running(netdev)) 2066 return -EINVAL; 2067 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP) 2068 return tsnep_ptp_ioctl(netdev, ifr, cmd); 2069 return phy_mii_ioctl(netdev->phydev, ifr, cmd); 2070 } 2071 2072 static void tsnep_netdev_set_multicast(struct net_device *netdev) 2073 { 2074 struct tsnep_adapter *adapter = netdev_priv(netdev); 2075 2076 u16 rx_filter = 0; 2077 2078 /* configured MAC address and broadcasts are never filtered */ 2079 if (netdev->flags & IFF_PROMISC) { 2080 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2081 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS; 2082 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) { 2083 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2084 } 2085 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER); 2086 } 2087 2088 static void tsnep_netdev_get_stats64(struct net_device *netdev, 2089 struct rtnl_link_stats64 *stats) 2090 { 2091 struct tsnep_adapter *adapter = netdev_priv(netdev); 2092 u32 reg; 2093 u32 val; 2094 int i; 2095 2096 for (i = 0; i < adapter->num_tx_queues; i++) { 2097 stats->tx_packets += adapter->tx[i].packets; 2098 stats->tx_bytes += adapter->tx[i].bytes; 2099 stats->tx_dropped += adapter->tx[i].dropped; 2100 } 2101 for (i = 0; i < adapter->num_rx_queues; i++) { 2102 stats->rx_packets += adapter->rx[i].packets; 2103 stats->rx_bytes += adapter->rx[i].bytes; 2104 stats->rx_dropped += adapter->rx[i].dropped; 2105 stats->multicast += adapter->rx[i].multicast; 2106 2107 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + 2108 TSNEP_RX_STATISTIC); 2109 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >> 2110 TSNEP_RX_STATISTIC_NO_DESC_SHIFT; 2111 stats->rx_dropped += val; 2112 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >> 2113 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT; 2114 stats->rx_dropped += val; 2115 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >> 2116 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT; 2117 stats->rx_errors += val; 2118 stats->rx_fifo_errors += val; 2119 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >> 2120 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT; 2121 stats->rx_errors += val; 2122 stats->rx_frame_errors += val; 2123 } 2124 2125 reg = ioread32(adapter->addr + ECM_STAT); 2126 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT; 2127 stats->rx_errors += val; 2128 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT; 2129 stats->rx_errors += val; 2130 stats->rx_crc_errors += val; 2131 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT; 2132 stats->rx_errors += val; 2133 } 2134 2135 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr) 2136 { 2137 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2138 iowrite16(*(u16 *)(addr + sizeof(u32)), 2139 adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2140 2141 ether_addr_copy(adapter->mac_address, addr); 2142 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n", 2143 addr); 2144 } 2145 2146 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr) 2147 { 2148 struct tsnep_adapter *adapter = netdev_priv(netdev); 2149 struct sockaddr *sock_addr = addr; 2150 int retval; 2151 2152 retval = eth_prepare_mac_addr_change(netdev, sock_addr); 2153 if (retval) 2154 return retval; 2155 eth_hw_addr_set(netdev, sock_addr->sa_data); 2156 tsnep_mac_set_address(adapter, sock_addr->sa_data); 2157 2158 return 0; 2159 } 2160 2161 static int tsnep_netdev_set_features(struct net_device *netdev, 2162 netdev_features_t features) 2163 { 2164 struct tsnep_adapter *adapter = netdev_priv(netdev); 2165 netdev_features_t changed = netdev->features ^ features; 2166 bool enable; 2167 int retval = 0; 2168 2169 if (changed & NETIF_F_LOOPBACK) { 2170 enable = !!(features & NETIF_F_LOOPBACK); 2171 retval = tsnep_phy_loopback(adapter, enable); 2172 } 2173 2174 return retval; 2175 } 2176 2177 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev, 2178 const struct skb_shared_hwtstamps *hwtstamps, 2179 bool cycles) 2180 { 2181 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data; 2182 u64 timestamp; 2183 2184 if (cycles) 2185 timestamp = __le64_to_cpu(rx_inline->counter); 2186 else 2187 timestamp = __le64_to_cpu(rx_inline->timestamp); 2188 2189 return ns_to_ktime(timestamp); 2190 } 2191 2192 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf) 2193 { 2194 struct tsnep_adapter *adapter = netdev_priv(dev); 2195 2196 switch (bpf->command) { 2197 case XDP_SETUP_PROG: 2198 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack); 2199 case XDP_SETUP_XSK_POOL: 2200 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool, 2201 bpf->xsk.queue_id); 2202 default: 2203 return -EOPNOTSUPP; 2204 } 2205 } 2206 2207 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu) 2208 { 2209 if (cpu >= TSNEP_MAX_QUEUES) 2210 cpu &= TSNEP_MAX_QUEUES - 1; 2211 2212 while (cpu >= adapter->num_tx_queues) 2213 cpu -= adapter->num_tx_queues; 2214 2215 return &adapter->tx[cpu]; 2216 } 2217 2218 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n, 2219 struct xdp_frame **xdp, u32 flags) 2220 { 2221 struct tsnep_adapter *adapter = netdev_priv(dev); 2222 u32 cpu = smp_processor_id(); 2223 struct netdev_queue *nq; 2224 struct tsnep_tx *tx; 2225 int nxmit; 2226 bool xmit; 2227 2228 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2229 return -EINVAL; 2230 2231 tx = tsnep_xdp_get_tx(adapter, cpu); 2232 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index); 2233 2234 __netif_tx_lock(nq, cpu); 2235 2236 for (nxmit = 0; nxmit < n; nxmit++) { 2237 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx, 2238 TSNEP_TX_TYPE_XDP_NDO); 2239 if (!xmit) 2240 break; 2241 2242 /* avoid transmit queue timeout since we share it with the slow 2243 * path 2244 */ 2245 txq_trans_cond_update(nq); 2246 } 2247 2248 if (flags & XDP_XMIT_FLUSH) 2249 tsnep_xdp_xmit_flush(tx); 2250 2251 __netif_tx_unlock(nq); 2252 2253 return nxmit; 2254 } 2255 2256 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id, 2257 u32 flags) 2258 { 2259 struct tsnep_adapter *adapter = netdev_priv(dev); 2260 struct tsnep_queue *queue; 2261 2262 if (queue_id >= adapter->num_rx_queues || 2263 queue_id >= adapter->num_tx_queues) 2264 return -EINVAL; 2265 2266 queue = &adapter->queue[queue_id]; 2267 2268 if (!napi_if_scheduled_mark_missed(&queue->napi)) 2269 napi_schedule(&queue->napi); 2270 2271 return 0; 2272 } 2273 2274 static const struct net_device_ops tsnep_netdev_ops = { 2275 .ndo_open = tsnep_netdev_open, 2276 .ndo_stop = tsnep_netdev_close, 2277 .ndo_start_xmit = tsnep_netdev_xmit_frame, 2278 .ndo_eth_ioctl = tsnep_netdev_ioctl, 2279 .ndo_set_rx_mode = tsnep_netdev_set_multicast, 2280 .ndo_get_stats64 = tsnep_netdev_get_stats64, 2281 .ndo_set_mac_address = tsnep_netdev_set_mac_address, 2282 .ndo_set_features = tsnep_netdev_set_features, 2283 .ndo_get_tstamp = tsnep_netdev_get_tstamp, 2284 .ndo_setup_tc = tsnep_tc_setup, 2285 .ndo_bpf = tsnep_netdev_bpf, 2286 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit, 2287 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup, 2288 }; 2289 2290 static int tsnep_mac_init(struct tsnep_adapter *adapter) 2291 { 2292 int retval; 2293 2294 /* initialize RX filtering, at least configured MAC address and 2295 * broadcast are not filtered 2296 */ 2297 iowrite16(0, adapter->addr + TSNEP_RX_FILTER); 2298 2299 /* try to get MAC address in the following order: 2300 * - device tree 2301 * - valid MAC address already set 2302 * - MAC address register if valid 2303 * - random MAC address 2304 */ 2305 retval = of_get_mac_address(adapter->pdev->dev.of_node, 2306 adapter->mac_address); 2307 if (retval == -EPROBE_DEFER) 2308 return retval; 2309 if (retval && !is_valid_ether_addr(adapter->mac_address)) { 2310 *(u32 *)adapter->mac_address = 2311 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2312 *(u16 *)(adapter->mac_address + sizeof(u32)) = 2313 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2314 if (!is_valid_ether_addr(adapter->mac_address)) 2315 eth_random_addr(adapter->mac_address); 2316 } 2317 2318 tsnep_mac_set_address(adapter, adapter->mac_address); 2319 eth_hw_addr_set(adapter->netdev, adapter->mac_address); 2320 2321 return 0; 2322 } 2323 2324 static int tsnep_mdio_init(struct tsnep_adapter *adapter) 2325 { 2326 struct device_node *np = adapter->pdev->dev.of_node; 2327 int retval; 2328 2329 if (np) { 2330 np = of_get_child_by_name(np, "mdio"); 2331 if (!np) 2332 return 0; 2333 2334 adapter->suppress_preamble = 2335 of_property_read_bool(np, "suppress-preamble"); 2336 } 2337 2338 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev); 2339 if (!adapter->mdiobus) { 2340 retval = -ENOMEM; 2341 2342 goto out; 2343 } 2344 2345 adapter->mdiobus->priv = (void *)adapter; 2346 adapter->mdiobus->parent = &adapter->pdev->dev; 2347 adapter->mdiobus->read = tsnep_mdiobus_read; 2348 adapter->mdiobus->write = tsnep_mdiobus_write; 2349 adapter->mdiobus->name = TSNEP "-mdiobus"; 2350 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s", 2351 adapter->pdev->name); 2352 2353 /* do not scan broadcast address */ 2354 adapter->mdiobus->phy_mask = 0x0000001; 2355 2356 retval = of_mdiobus_register(adapter->mdiobus, np); 2357 2358 out: 2359 of_node_put(np); 2360 2361 return retval; 2362 } 2363 2364 static int tsnep_phy_init(struct tsnep_adapter *adapter) 2365 { 2366 struct device_node *phy_node; 2367 int retval; 2368 2369 retval = of_get_phy_mode(adapter->pdev->dev.of_node, 2370 &adapter->phy_mode); 2371 if (retval) 2372 adapter->phy_mode = PHY_INTERFACE_MODE_GMII; 2373 2374 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle", 2375 0); 2376 adapter->phydev = of_phy_find_device(phy_node); 2377 of_node_put(phy_node); 2378 if (!adapter->phydev && adapter->mdiobus) 2379 adapter->phydev = phy_find_first(adapter->mdiobus); 2380 if (!adapter->phydev) 2381 return -EIO; 2382 2383 return 0; 2384 } 2385 2386 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) 2387 { 2388 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; 2389 char name[8]; 2390 int i; 2391 int retval; 2392 2393 /* one TX/RX queue pair for netdev is mandatory */ 2394 if (platform_irq_count(adapter->pdev) == 1) 2395 retval = platform_get_irq(adapter->pdev, 0); 2396 else 2397 retval = platform_get_irq_byname(adapter->pdev, "mac"); 2398 if (retval < 0) 2399 return retval; 2400 adapter->num_tx_queues = 1; 2401 adapter->num_rx_queues = 1; 2402 adapter->num_queues = 1; 2403 adapter->queue[0].adapter = adapter; 2404 adapter->queue[0].irq = retval; 2405 adapter->queue[0].tx = &adapter->tx[0]; 2406 adapter->queue[0].tx->adapter = adapter; 2407 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0); 2408 adapter->queue[0].tx->queue_index = 0; 2409 adapter->queue[0].rx = &adapter->rx[0]; 2410 adapter->queue[0].rx->adapter = adapter; 2411 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0); 2412 adapter->queue[0].rx->queue_index = 0; 2413 adapter->queue[0].irq_mask = irq_mask; 2414 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY; 2415 retval = tsnep_set_irq_coalesce(&adapter->queue[0], 2416 TSNEP_COALESCE_USECS_DEFAULT); 2417 if (retval < 0) 2418 return retval; 2419 2420 adapter->netdev->irq = adapter->queue[0].irq; 2421 2422 /* add additional TX/RX queue pairs only if dedicated interrupt is 2423 * available 2424 */ 2425 for (i = 1; i < queue_count; i++) { 2426 sprintf(name, "txrx-%d", i); 2427 retval = platform_get_irq_byname_optional(adapter->pdev, name); 2428 if (retval < 0) 2429 break; 2430 2431 adapter->num_tx_queues++; 2432 adapter->num_rx_queues++; 2433 adapter->num_queues++; 2434 adapter->queue[i].adapter = adapter; 2435 adapter->queue[i].irq = retval; 2436 adapter->queue[i].tx = &adapter->tx[i]; 2437 adapter->queue[i].tx->adapter = adapter; 2438 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i); 2439 adapter->queue[i].tx->queue_index = i; 2440 adapter->queue[i].rx = &adapter->rx[i]; 2441 adapter->queue[i].rx->adapter = adapter; 2442 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i); 2443 adapter->queue[i].rx->queue_index = i; 2444 adapter->queue[i].irq_mask = 2445 irq_mask << (ECM_INT_TXRX_SHIFT * i); 2446 adapter->queue[i].irq_delay_addr = 2447 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i; 2448 retval = tsnep_set_irq_coalesce(&adapter->queue[i], 2449 TSNEP_COALESCE_USECS_DEFAULT); 2450 if (retval < 0) 2451 return retval; 2452 } 2453 2454 return 0; 2455 } 2456 2457 static int tsnep_probe(struct platform_device *pdev) 2458 { 2459 struct tsnep_adapter *adapter; 2460 struct net_device *netdev; 2461 struct resource *io; 2462 u32 type; 2463 int revision; 2464 int version; 2465 int queue_count; 2466 int retval; 2467 2468 netdev = devm_alloc_etherdev_mqs(&pdev->dev, 2469 sizeof(struct tsnep_adapter), 2470 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES); 2471 if (!netdev) 2472 return -ENODEV; 2473 SET_NETDEV_DEV(netdev, &pdev->dev); 2474 adapter = netdev_priv(netdev); 2475 platform_set_drvdata(pdev, adapter); 2476 adapter->pdev = pdev; 2477 adapter->dmadev = &pdev->dev; 2478 adapter->netdev = netdev; 2479 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | 2480 NETIF_MSG_LINK | NETIF_MSG_IFUP | 2481 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED; 2482 2483 netdev->min_mtu = ETH_MIN_MTU; 2484 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE; 2485 2486 mutex_init(&adapter->gate_control_lock); 2487 mutex_init(&adapter->rxnfc_lock); 2488 INIT_LIST_HEAD(&adapter->rxnfc_rules); 2489 2490 io = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2491 adapter->addr = devm_ioremap_resource(&pdev->dev, io); 2492 if (IS_ERR(adapter->addr)) 2493 return PTR_ERR(adapter->addr); 2494 netdev->mem_start = io->start; 2495 netdev->mem_end = io->end; 2496 2497 type = ioread32(adapter->addr + ECM_TYPE); 2498 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT; 2499 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT; 2500 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT; 2501 adapter->gate_control = type & ECM_GATE_CONTROL; 2502 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT; 2503 2504 tsnep_disable_irq(adapter, ECM_INT_ALL); 2505 2506 retval = tsnep_queue_init(adapter, queue_count); 2507 if (retval) 2508 return retval; 2509 2510 retval = dma_set_mask_and_coherent(&adapter->pdev->dev, 2511 DMA_BIT_MASK(64)); 2512 if (retval) { 2513 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n"); 2514 return retval; 2515 } 2516 2517 retval = tsnep_mac_init(adapter); 2518 if (retval) 2519 return retval; 2520 2521 retval = tsnep_mdio_init(adapter); 2522 if (retval) 2523 goto mdio_init_failed; 2524 2525 retval = tsnep_phy_init(adapter); 2526 if (retval) 2527 goto phy_init_failed; 2528 2529 retval = tsnep_ptp_init(adapter); 2530 if (retval) 2531 goto ptp_init_failed; 2532 2533 retval = tsnep_tc_init(adapter); 2534 if (retval) 2535 goto tc_init_failed; 2536 2537 retval = tsnep_rxnfc_init(adapter); 2538 if (retval) 2539 goto rxnfc_init_failed; 2540 2541 netdev->netdev_ops = &tsnep_netdev_ops; 2542 netdev->ethtool_ops = &tsnep_ethtool_ops; 2543 netdev->features = NETIF_F_SG; 2544 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK; 2545 2546 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 2547 NETDEV_XDP_ACT_NDO_XMIT | 2548 NETDEV_XDP_ACT_NDO_XMIT_SG | 2549 NETDEV_XDP_ACT_XSK_ZEROCOPY; 2550 2551 /* carrier off reporting is important to ethtool even BEFORE open */ 2552 netif_carrier_off(netdev); 2553 2554 retval = register_netdev(netdev); 2555 if (retval) 2556 goto register_failed; 2557 2558 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version, 2559 revision); 2560 if (adapter->gate_control) 2561 dev_info(&adapter->pdev->dev, "gate control detected\n"); 2562 2563 return 0; 2564 2565 register_failed: 2566 tsnep_rxnfc_cleanup(adapter); 2567 rxnfc_init_failed: 2568 tsnep_tc_cleanup(adapter); 2569 tc_init_failed: 2570 tsnep_ptp_cleanup(adapter); 2571 ptp_init_failed: 2572 phy_init_failed: 2573 if (adapter->mdiobus) 2574 mdiobus_unregister(adapter->mdiobus); 2575 mdio_init_failed: 2576 return retval; 2577 } 2578 2579 static int tsnep_remove(struct platform_device *pdev) 2580 { 2581 struct tsnep_adapter *adapter = platform_get_drvdata(pdev); 2582 2583 unregister_netdev(adapter->netdev); 2584 2585 tsnep_rxnfc_cleanup(adapter); 2586 2587 tsnep_tc_cleanup(adapter); 2588 2589 tsnep_ptp_cleanup(adapter); 2590 2591 if (adapter->mdiobus) 2592 mdiobus_unregister(adapter->mdiobus); 2593 2594 tsnep_disable_irq(adapter, ECM_INT_ALL); 2595 2596 return 0; 2597 } 2598 2599 static const struct of_device_id tsnep_of_match[] = { 2600 { .compatible = "engleder,tsnep", }, 2601 { }, 2602 }; 2603 MODULE_DEVICE_TABLE(of, tsnep_of_match); 2604 2605 static struct platform_driver tsnep_driver = { 2606 .driver = { 2607 .name = TSNEP, 2608 .of_match_table = tsnep_of_match, 2609 }, 2610 .probe = tsnep_probe, 2611 .remove = tsnep_remove, 2612 }; 2613 module_platform_driver(tsnep_driver); 2614 2615 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>"); 2616 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver"); 2617 MODULE_LICENSE("GPL"); 2618