1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ 3 4 /* TSN endpoint Ethernet MAC driver 5 * 6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time 7 * communication. It is designed for endpoints within TSN (Time Sensitive 8 * Networking) networks; e.g., for PLCs in the industrial automation case. 9 * 10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used 11 * by the driver. 12 * 13 * More information can be found here: 14 * - www.embedded-experts.at/tsn 15 * - www.engleder-embedded.com 16 */ 17 18 #include "tsnep.h" 19 #include "tsnep_hw.h" 20 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_net.h> 24 #include <linux/of_mdio.h> 25 #include <linux/interrupt.h> 26 #include <linux/etherdevice.h> 27 #include <linux/phy.h> 28 #include <linux/iopoll.h> 29 #include <linux/bpf.h> 30 #include <linux/bpf_trace.h> 31 32 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN) 33 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4) 34 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \ 35 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 36 37 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 38 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF)) 39 #else 40 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0)) 41 #endif 42 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF)) 43 44 #define TSNEP_COALESCE_USECS_DEFAULT 64 45 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \ 46 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1) 47 48 #define TSNEP_TX_TYPE_SKB BIT(0) 49 #define TSNEP_TX_TYPE_SKB_FRAG BIT(1) 50 #define TSNEP_TX_TYPE_XDP_TX BIT(2) 51 #define TSNEP_TX_TYPE_XDP_NDO BIT(3) 52 53 #define TSNEP_XDP_TX BIT(0) 54 #define TSNEP_XDP_REDIRECT BIT(1) 55 56 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask) 57 { 58 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 59 } 60 61 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask) 62 { 63 mask |= ECM_INT_DISABLE; 64 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 65 } 66 67 static irqreturn_t tsnep_irq(int irq, void *arg) 68 { 69 struct tsnep_adapter *adapter = arg; 70 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE); 71 72 /* acknowledge interrupt */ 73 if (active != 0) 74 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE); 75 76 /* handle link interrupt */ 77 if ((active & ECM_INT_LINK) != 0) 78 phy_mac_interrupt(adapter->netdev->phydev); 79 80 /* handle TX/RX queue 0 interrupt */ 81 if ((active & adapter->queue[0].irq_mask) != 0) { 82 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); 83 napi_schedule(&adapter->queue[0].napi); 84 } 85 86 return IRQ_HANDLED; 87 } 88 89 static irqreturn_t tsnep_irq_txrx(int irq, void *arg) 90 { 91 struct tsnep_queue *queue = arg; 92 93 /* handle TX/RX queue interrupt */ 94 tsnep_disable_irq(queue->adapter, queue->irq_mask); 95 napi_schedule(&queue->napi); 96 97 return IRQ_HANDLED; 98 } 99 100 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs) 101 { 102 if (usecs > TSNEP_COALESCE_USECS_MAX) 103 return -ERANGE; 104 105 usecs /= ECM_INT_DELAY_BASE_US; 106 usecs <<= ECM_INT_DELAY_SHIFT; 107 usecs &= ECM_INT_DELAY_MASK; 108 109 queue->irq_delay &= ~ECM_INT_DELAY_MASK; 110 queue->irq_delay |= usecs; 111 iowrite8(queue->irq_delay, queue->irq_delay_addr); 112 113 return 0; 114 } 115 116 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue) 117 { 118 u32 usecs; 119 120 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK); 121 usecs >>= ECM_INT_DELAY_SHIFT; 122 usecs *= ECM_INT_DELAY_BASE_US; 123 124 return usecs; 125 } 126 127 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) 128 { 129 struct tsnep_adapter *adapter = bus->priv; 130 u32 md; 131 int retval; 132 133 md = ECM_MD_READ; 134 if (!adapter->suppress_preamble) 135 md |= ECM_MD_PREAMBLE; 136 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 137 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 138 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 139 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 140 !(md & ECM_MD_BUSY), 16, 1000); 141 if (retval != 0) 142 return retval; 143 144 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT; 145 } 146 147 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum, 148 u16 val) 149 { 150 struct tsnep_adapter *adapter = bus->priv; 151 u32 md; 152 int retval; 153 154 md = ECM_MD_WRITE; 155 if (!adapter->suppress_preamble) 156 md |= ECM_MD_PREAMBLE; 157 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 158 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 159 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK; 160 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 161 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 162 !(md & ECM_MD_BUSY), 16, 1000); 163 if (retval != 0) 164 return retval; 165 166 return 0; 167 } 168 169 static void tsnep_set_link_mode(struct tsnep_adapter *adapter) 170 { 171 u32 mode; 172 173 switch (adapter->phydev->speed) { 174 case SPEED_100: 175 mode = ECM_LINK_MODE_100; 176 break; 177 case SPEED_1000: 178 mode = ECM_LINK_MODE_1000; 179 break; 180 default: 181 mode = ECM_LINK_MODE_OFF; 182 break; 183 } 184 iowrite32(mode, adapter->addr + ECM_STATUS); 185 } 186 187 static void tsnep_phy_link_status_change(struct net_device *netdev) 188 { 189 struct tsnep_adapter *adapter = netdev_priv(netdev); 190 struct phy_device *phydev = netdev->phydev; 191 192 if (phydev->link) 193 tsnep_set_link_mode(adapter); 194 195 phy_print_status(netdev->phydev); 196 } 197 198 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable) 199 { 200 int retval; 201 202 retval = phy_loopback(adapter->phydev, enable); 203 204 /* PHY link state change is not signaled if loopback is enabled, it 205 * would delay a working loopback anyway, let's ensure that loopback 206 * is working immediately by setting link mode directly 207 */ 208 if (!retval && enable) 209 tsnep_set_link_mode(adapter); 210 211 return retval; 212 } 213 214 static int tsnep_phy_open(struct tsnep_adapter *adapter) 215 { 216 struct phy_device *phydev; 217 struct ethtool_eee ethtool_eee; 218 int retval; 219 220 retval = phy_connect_direct(adapter->netdev, adapter->phydev, 221 tsnep_phy_link_status_change, 222 adapter->phy_mode); 223 if (retval) 224 return retval; 225 phydev = adapter->netdev->phydev; 226 227 /* MAC supports only 100Mbps|1000Mbps full duplex 228 * SPE (Single Pair Ethernet) is also an option but not implemented yet 229 */ 230 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); 231 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); 232 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); 233 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 234 235 /* disable EEE autoneg, EEE not supported by TSNEP */ 236 memset(ðtool_eee, 0, sizeof(ethtool_eee)); 237 phy_ethtool_set_eee(adapter->phydev, ðtool_eee); 238 239 adapter->phydev->irq = PHY_MAC_INTERRUPT; 240 phy_start(adapter->phydev); 241 242 return 0; 243 } 244 245 static void tsnep_phy_close(struct tsnep_adapter *adapter) 246 { 247 phy_stop(adapter->netdev->phydev); 248 phy_disconnect(adapter->netdev->phydev); 249 } 250 251 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx) 252 { 253 struct device *dmadev = tx->adapter->dmadev; 254 int i; 255 256 memset(tx->entry, 0, sizeof(tx->entry)); 257 258 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 259 if (tx->page[i]) { 260 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i], 261 tx->page_dma[i]); 262 tx->page[i] = NULL; 263 tx->page_dma[i] = 0; 264 } 265 } 266 } 267 268 static int tsnep_tx_ring_init(struct tsnep_tx *tx) 269 { 270 struct device *dmadev = tx->adapter->dmadev; 271 struct tsnep_tx_entry *entry; 272 struct tsnep_tx_entry *next_entry; 273 int i, j; 274 int retval; 275 276 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 277 tx->page[i] = 278 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i], 279 GFP_KERNEL); 280 if (!tx->page[i]) { 281 retval = -ENOMEM; 282 goto alloc_failed; 283 } 284 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 285 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 286 entry->desc_wb = (struct tsnep_tx_desc_wb *) 287 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j); 288 entry->desc = (struct tsnep_tx_desc *) 289 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 290 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j; 291 } 292 } 293 for (i = 0; i < TSNEP_RING_SIZE; i++) { 294 entry = &tx->entry[i]; 295 next_entry = &tx->entry[(i + 1) % TSNEP_RING_SIZE]; 296 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 297 } 298 299 return 0; 300 301 alloc_failed: 302 tsnep_tx_ring_cleanup(tx); 303 return retval; 304 } 305 306 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, 307 bool last) 308 { 309 struct tsnep_tx_entry *entry = &tx->entry[index]; 310 311 entry->properties = 0; 312 /* xdpf is union with skb */ 313 if (entry->skb) { 314 entry->properties = length & TSNEP_DESC_LENGTH_MASK; 315 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 316 if ((entry->type & TSNEP_TX_TYPE_SKB) && 317 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS)) 318 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; 319 320 /* toggle user flag to prevent false acknowledge 321 * 322 * Only the first fragment is acknowledged. For all other 323 * fragments no acknowledge is done and the last written owner 324 * counter stays in the writeback descriptor. Therefore, it is 325 * possible that the last written owner counter is identical to 326 * the new incremented owner counter and a false acknowledge is 327 * detected before the real acknowledge has been done by 328 * hardware. 329 * 330 * The user flag is used to prevent this situation. The user 331 * flag is copied to the writeback descriptor by the hardware 332 * and is used as additional acknowledge data. By toggeling the 333 * user flag only for the first fragment (which is 334 * acknowledged), it is guaranteed that the last acknowledge 335 * done for this descriptor has used a different user flag and 336 * cannot be detected as false acknowledge. 337 */ 338 entry->owner_user_flag = !entry->owner_user_flag; 339 } 340 if (last) 341 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG; 342 if (index == tx->increment_owner_counter) { 343 tx->owner_counter++; 344 if (tx->owner_counter == 4) 345 tx->owner_counter = 1; 346 tx->increment_owner_counter--; 347 if (tx->increment_owner_counter < 0) 348 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 349 } 350 entry->properties |= 351 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 352 TSNEP_DESC_OWNER_COUNTER_MASK; 353 if (entry->owner_user_flag) 354 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG; 355 entry->desc->more_properties = 356 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK); 357 358 /* descriptor properties shall be written last, because valid data is 359 * signaled there 360 */ 361 dma_wmb(); 362 363 entry->desc->properties = __cpu_to_le32(entry->properties); 364 } 365 366 static int tsnep_tx_desc_available(struct tsnep_tx *tx) 367 { 368 if (tx->read <= tx->write) 369 return TSNEP_RING_SIZE - tx->write + tx->read - 1; 370 else 371 return tx->read - tx->write - 1; 372 } 373 374 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) 375 { 376 struct device *dmadev = tx->adapter->dmadev; 377 struct tsnep_tx_entry *entry; 378 unsigned int len; 379 dma_addr_t dma; 380 int map_len = 0; 381 int i; 382 383 for (i = 0; i < count; i++) { 384 entry = &tx->entry[(tx->write + i) % TSNEP_RING_SIZE]; 385 386 if (!i) { 387 len = skb_headlen(skb); 388 dma = dma_map_single(dmadev, skb->data, len, 389 DMA_TO_DEVICE); 390 391 entry->type = TSNEP_TX_TYPE_SKB; 392 } else { 393 len = skb_frag_size(&skb_shinfo(skb)->frags[i - 1]); 394 dma = skb_frag_dma_map(dmadev, 395 &skb_shinfo(skb)->frags[i - 1], 396 0, len, DMA_TO_DEVICE); 397 398 entry->type = TSNEP_TX_TYPE_SKB_FRAG; 399 } 400 if (dma_mapping_error(dmadev, dma)) 401 return -ENOMEM; 402 403 entry->len = len; 404 dma_unmap_addr_set(entry, dma, dma); 405 406 entry->desc->tx = __cpu_to_le64(dma); 407 408 map_len += len; 409 } 410 411 return map_len; 412 } 413 414 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) 415 { 416 struct device *dmadev = tx->adapter->dmadev; 417 struct tsnep_tx_entry *entry; 418 int map_len = 0; 419 int i; 420 421 for (i = 0; i < count; i++) { 422 entry = &tx->entry[(index + i) % TSNEP_RING_SIZE]; 423 424 if (entry->len) { 425 if (entry->type & TSNEP_TX_TYPE_SKB) 426 dma_unmap_single(dmadev, 427 dma_unmap_addr(entry, dma), 428 dma_unmap_len(entry, len), 429 DMA_TO_DEVICE); 430 else if (entry->type & 431 (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_XDP_NDO)) 432 dma_unmap_page(dmadev, 433 dma_unmap_addr(entry, dma), 434 dma_unmap_len(entry, len), 435 DMA_TO_DEVICE); 436 map_len += entry->len; 437 entry->len = 0; 438 } 439 } 440 441 return map_len; 442 } 443 444 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, 445 struct tsnep_tx *tx) 446 { 447 int count = 1; 448 struct tsnep_tx_entry *entry; 449 int length; 450 int i; 451 int retval; 452 453 if (skb_shinfo(skb)->nr_frags > 0) 454 count += skb_shinfo(skb)->nr_frags; 455 456 if (tsnep_tx_desc_available(tx) < count) { 457 /* ring full, shall not happen because queue is stopped if full 458 * below 459 */ 460 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 461 462 return NETDEV_TX_BUSY; 463 } 464 465 entry = &tx->entry[tx->write]; 466 entry->skb = skb; 467 468 retval = tsnep_tx_map(skb, tx, count); 469 if (retval < 0) { 470 tsnep_tx_unmap(tx, tx->write, count); 471 dev_kfree_skb_any(entry->skb); 472 entry->skb = NULL; 473 474 tx->dropped++; 475 476 return NETDEV_TX_OK; 477 } 478 length = retval; 479 480 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 481 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 482 483 for (i = 0; i < count; i++) 484 tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE, length, 485 i == count - 1); 486 tx->write = (tx->write + count) % TSNEP_RING_SIZE; 487 488 skb_tx_timestamp(skb); 489 490 /* descriptor properties shall be valid before hardware is notified */ 491 dma_wmb(); 492 493 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 494 495 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) { 496 /* ring can get full with next frame */ 497 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 498 } 499 500 return NETDEV_TX_OK; 501 } 502 503 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx, 504 struct skb_shared_info *shinfo, int count, u32 type) 505 { 506 struct device *dmadev = tx->adapter->dmadev; 507 struct tsnep_tx_entry *entry; 508 struct page *page; 509 skb_frag_t *frag; 510 unsigned int len; 511 int map_len = 0; 512 dma_addr_t dma; 513 void *data; 514 int i; 515 516 frag = NULL; 517 len = xdpf->len; 518 for (i = 0; i < count; i++) { 519 entry = &tx->entry[(tx->write + i) % TSNEP_RING_SIZE]; 520 if (type & TSNEP_TX_TYPE_XDP_NDO) { 521 data = unlikely(frag) ? skb_frag_address(frag) : 522 xdpf->data; 523 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE); 524 if (dma_mapping_error(dmadev, dma)) 525 return -ENOMEM; 526 527 entry->type = TSNEP_TX_TYPE_XDP_NDO; 528 } else { 529 page = unlikely(frag) ? skb_frag_page(frag) : 530 virt_to_page(xdpf->data); 531 dma = page_pool_get_dma_addr(page); 532 if (unlikely(frag)) 533 dma += skb_frag_off(frag); 534 else 535 dma += sizeof(*xdpf) + xdpf->headroom; 536 dma_sync_single_for_device(dmadev, dma, len, 537 DMA_BIDIRECTIONAL); 538 539 entry->type = TSNEP_TX_TYPE_XDP_TX; 540 } 541 542 entry->len = len; 543 dma_unmap_addr_set(entry, dma, dma); 544 545 entry->desc->tx = __cpu_to_le64(dma); 546 547 map_len += len; 548 549 if (i + 1 < count) { 550 frag = &shinfo->frags[i]; 551 len = skb_frag_size(frag); 552 } 553 } 554 555 return map_len; 556 } 557 558 /* This function requires __netif_tx_lock is held by the caller. */ 559 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf, 560 struct tsnep_tx *tx, u32 type) 561 { 562 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf); 563 struct tsnep_tx_entry *entry; 564 int count, length, retval, i; 565 566 count = 1; 567 if (unlikely(xdp_frame_has_frags(xdpf))) 568 count += shinfo->nr_frags; 569 570 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 571 * will be available for normal TX path and queue is stopped there if 572 * necessary 573 */ 574 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count)) 575 return false; 576 577 entry = &tx->entry[tx->write]; 578 entry->xdpf = xdpf; 579 580 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type); 581 if (retval < 0) { 582 tsnep_tx_unmap(tx, tx->write, count); 583 entry->xdpf = NULL; 584 585 tx->dropped++; 586 587 return false; 588 } 589 length = retval; 590 591 for (i = 0; i < count; i++) 592 tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE, length, 593 i == count - 1); 594 tx->write = (tx->write + count) % TSNEP_RING_SIZE; 595 596 /* descriptor properties shall be valid before hardware is notified */ 597 dma_wmb(); 598 599 return true; 600 } 601 602 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx) 603 { 604 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 605 } 606 607 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter, 608 struct xdp_buff *xdp, 609 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 610 { 611 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 612 bool xmit; 613 614 if (unlikely(!xdpf)) 615 return false; 616 617 __netif_tx_lock(tx_nq, smp_processor_id()); 618 619 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, TSNEP_TX_TYPE_XDP_TX); 620 621 /* Avoid transmit queue timeout since we share it with the slow path */ 622 if (xmit) 623 txq_trans_cond_update(tx_nq); 624 625 __netif_tx_unlock(tx_nq); 626 627 return xmit; 628 } 629 630 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) 631 { 632 struct tsnep_tx_entry *entry; 633 struct netdev_queue *nq; 634 int budget = 128; 635 int length; 636 int count; 637 638 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 639 __netif_tx_lock(nq, smp_processor_id()); 640 641 do { 642 if (tx->read == tx->write) 643 break; 644 645 entry = &tx->entry[tx->read]; 646 if ((__le32_to_cpu(entry->desc_wb->properties) & 647 TSNEP_TX_DESC_OWNER_MASK) != 648 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 649 break; 650 651 /* descriptor properties shall be read first, because valid data 652 * is signaled there 653 */ 654 dma_rmb(); 655 656 count = 1; 657 if ((entry->type & TSNEP_TX_TYPE_SKB) && 658 skb_shinfo(entry->skb)->nr_frags > 0) 659 count += skb_shinfo(entry->skb)->nr_frags; 660 else if (!(entry->type & TSNEP_TX_TYPE_SKB) && 661 xdp_frame_has_frags(entry->xdpf)) 662 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags; 663 664 length = tsnep_tx_unmap(tx, tx->read, count); 665 666 if ((entry->type & TSNEP_TX_TYPE_SKB) && 667 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && 668 (__le32_to_cpu(entry->desc_wb->properties) & 669 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) { 670 struct skb_shared_hwtstamps hwtstamps; 671 u64 timestamp; 672 673 if (skb_shinfo(entry->skb)->tx_flags & 674 SKBTX_HW_TSTAMP_USE_CYCLES) 675 timestamp = 676 __le64_to_cpu(entry->desc_wb->counter); 677 else 678 timestamp = 679 __le64_to_cpu(entry->desc_wb->timestamp); 680 681 memset(&hwtstamps, 0, sizeof(hwtstamps)); 682 hwtstamps.hwtstamp = ns_to_ktime(timestamp); 683 684 skb_tstamp_tx(entry->skb, &hwtstamps); 685 } 686 687 if (entry->type & TSNEP_TX_TYPE_SKB) 688 napi_consume_skb(entry->skb, napi_budget); 689 else 690 xdp_return_frame_rx_napi(entry->xdpf); 691 /* xdpf is union with skb */ 692 entry->skb = NULL; 693 694 tx->read = (tx->read + count) % TSNEP_RING_SIZE; 695 696 tx->packets++; 697 tx->bytes += length + ETH_FCS_LEN; 698 699 budget--; 700 } while (likely(budget)); 701 702 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) && 703 netif_tx_queue_stopped(nq)) { 704 netif_tx_wake_queue(nq); 705 } 706 707 __netif_tx_unlock(nq); 708 709 return budget != 0; 710 } 711 712 static bool tsnep_tx_pending(struct tsnep_tx *tx) 713 { 714 struct tsnep_tx_entry *entry; 715 struct netdev_queue *nq; 716 bool pending = false; 717 718 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 719 __netif_tx_lock(nq, smp_processor_id()); 720 721 if (tx->read != tx->write) { 722 entry = &tx->entry[tx->read]; 723 if ((__le32_to_cpu(entry->desc_wb->properties) & 724 TSNEP_TX_DESC_OWNER_MASK) == 725 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 726 pending = true; 727 } 728 729 __netif_tx_unlock(nq); 730 731 return pending; 732 } 733 734 static int tsnep_tx_open(struct tsnep_adapter *adapter, void __iomem *addr, 735 int queue_index, struct tsnep_tx *tx) 736 { 737 dma_addr_t dma; 738 int retval; 739 740 memset(tx, 0, sizeof(*tx)); 741 tx->adapter = adapter; 742 tx->addr = addr; 743 tx->queue_index = queue_index; 744 745 retval = tsnep_tx_ring_init(tx); 746 if (retval) 747 return retval; 748 749 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 750 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW); 751 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH); 752 tx->owner_counter = 1; 753 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 754 755 return 0; 756 } 757 758 static void tsnep_tx_close(struct tsnep_tx *tx) 759 { 760 u32 val; 761 762 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val, 763 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000, 764 1000000); 765 766 tsnep_tx_ring_cleanup(tx); 767 } 768 769 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) 770 { 771 struct device *dmadev = rx->adapter->dmadev; 772 struct tsnep_rx_entry *entry; 773 int i; 774 775 for (i = 0; i < TSNEP_RING_SIZE; i++) { 776 entry = &rx->entry[i]; 777 if (entry->page) 778 page_pool_put_full_page(rx->page_pool, entry->page, 779 false); 780 entry->page = NULL; 781 } 782 783 if (rx->page_pool) 784 page_pool_destroy(rx->page_pool); 785 786 memset(rx->entry, 0, sizeof(rx->entry)); 787 788 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 789 if (rx->page[i]) { 790 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i], 791 rx->page_dma[i]); 792 rx->page[i] = NULL; 793 rx->page_dma[i] = 0; 794 } 795 } 796 } 797 798 static int tsnep_rx_ring_init(struct tsnep_rx *rx) 799 { 800 struct device *dmadev = rx->adapter->dmadev; 801 struct tsnep_rx_entry *entry; 802 struct page_pool_params pp_params = { 0 }; 803 struct tsnep_rx_entry *next_entry; 804 int i, j; 805 int retval; 806 807 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 808 rx->page[i] = 809 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i], 810 GFP_KERNEL); 811 if (!rx->page[i]) { 812 retval = -ENOMEM; 813 goto failed; 814 } 815 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 816 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 817 entry->desc_wb = (struct tsnep_rx_desc_wb *) 818 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j); 819 entry->desc = (struct tsnep_rx_desc *) 820 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 821 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j; 822 } 823 } 824 825 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 826 pp_params.order = 0; 827 pp_params.pool_size = TSNEP_RING_SIZE; 828 pp_params.nid = dev_to_node(dmadev); 829 pp_params.dev = dmadev; 830 pp_params.dma_dir = DMA_BIDIRECTIONAL; 831 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE; 832 pp_params.offset = TSNEP_RX_OFFSET; 833 rx->page_pool = page_pool_create(&pp_params); 834 if (IS_ERR(rx->page_pool)) { 835 retval = PTR_ERR(rx->page_pool); 836 rx->page_pool = NULL; 837 goto failed; 838 } 839 840 for (i = 0; i < TSNEP_RING_SIZE; i++) { 841 entry = &rx->entry[i]; 842 next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE]; 843 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 844 } 845 846 return 0; 847 848 failed: 849 tsnep_rx_ring_cleanup(rx); 850 return retval; 851 } 852 853 static int tsnep_rx_desc_available(struct tsnep_rx *rx) 854 { 855 if (rx->read <= rx->write) 856 return TSNEP_RING_SIZE - rx->write + rx->read - 1; 857 else 858 return rx->read - rx->write - 1; 859 } 860 861 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 862 struct page *page) 863 { 864 entry->page = page; 865 entry->len = TSNEP_MAX_RX_BUF_SIZE; 866 entry->dma = page_pool_get_dma_addr(entry->page); 867 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET); 868 } 869 870 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index) 871 { 872 struct tsnep_rx_entry *entry = &rx->entry[index]; 873 struct page *page; 874 875 page = page_pool_dev_alloc_pages(rx->page_pool); 876 if (unlikely(!page)) 877 return -ENOMEM; 878 tsnep_rx_set_page(rx, entry, page); 879 880 return 0; 881 } 882 883 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index) 884 { 885 struct tsnep_rx_entry *entry = &rx->entry[index]; 886 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 887 888 tsnep_rx_set_page(rx, entry, read->page); 889 read->page = NULL; 890 } 891 892 static void tsnep_rx_activate(struct tsnep_rx *rx, int index) 893 { 894 struct tsnep_rx_entry *entry = &rx->entry[index]; 895 896 /* TSNEP_MAX_RX_BUF_SIZE is a multiple of 4 */ 897 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK; 898 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 899 if (index == rx->increment_owner_counter) { 900 rx->owner_counter++; 901 if (rx->owner_counter == 4) 902 rx->owner_counter = 1; 903 rx->increment_owner_counter--; 904 if (rx->increment_owner_counter < 0) 905 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 906 } 907 entry->properties |= 908 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 909 TSNEP_DESC_OWNER_COUNTER_MASK; 910 911 /* descriptor properties shall be written last, because valid data is 912 * signaled there 913 */ 914 dma_wmb(); 915 916 entry->desc->properties = __cpu_to_le32(entry->properties); 917 } 918 919 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse) 920 { 921 int index; 922 bool alloc_failed = false; 923 bool enable = false; 924 int i; 925 int retval; 926 927 for (i = 0; i < count && !alloc_failed; i++) { 928 index = (rx->write + i) % TSNEP_RING_SIZE; 929 930 retval = tsnep_rx_alloc_buffer(rx, index); 931 if (unlikely(retval)) { 932 rx->alloc_failed++; 933 alloc_failed = true; 934 935 /* reuse only if no other allocation was successful */ 936 if (i == 0 && reuse) 937 tsnep_rx_reuse_buffer(rx, index); 938 else 939 break; 940 } 941 942 tsnep_rx_activate(rx, index); 943 944 enable = true; 945 } 946 947 if (enable) { 948 rx->write = (rx->write + i) % TSNEP_RING_SIZE; 949 950 /* descriptor properties shall be valid before hardware is 951 * notified 952 */ 953 dma_wmb(); 954 955 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); 956 } 957 958 return i; 959 } 960 961 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog, 962 struct xdp_buff *xdp, int *status, 963 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 964 { 965 unsigned int length; 966 unsigned int sync; 967 u32 act; 968 969 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM; 970 971 act = bpf_prog_run_xdp(prog, xdp); 972 973 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */ 974 sync = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM; 975 sync = max(sync, length); 976 977 switch (act) { 978 case XDP_PASS: 979 return false; 980 case XDP_TX: 981 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx)) 982 goto out_failure; 983 *status |= TSNEP_XDP_TX; 984 return true; 985 case XDP_REDIRECT: 986 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 987 goto out_failure; 988 *status |= TSNEP_XDP_REDIRECT; 989 return true; 990 default: 991 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 992 fallthrough; 993 case XDP_ABORTED: 994 out_failure: 995 trace_xdp_exception(rx->adapter->netdev, prog, act); 996 fallthrough; 997 case XDP_DROP: 998 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data), 999 sync, true); 1000 return true; 1001 } 1002 } 1003 1004 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status, 1005 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1006 { 1007 if (status & TSNEP_XDP_TX) { 1008 __netif_tx_lock(tx_nq, smp_processor_id()); 1009 tsnep_xdp_xmit_flush(tx); 1010 __netif_tx_unlock(tx_nq); 1011 } 1012 1013 if (status & TSNEP_XDP_REDIRECT) 1014 xdp_do_flush(); 1015 } 1016 1017 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page, 1018 int length) 1019 { 1020 struct sk_buff *skb; 1021 1022 skb = napi_build_skb(page_address(page), PAGE_SIZE); 1023 if (unlikely(!skb)) 1024 return NULL; 1025 1026 /* update pointers within the skb to store the data */ 1027 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE); 1028 __skb_put(skb, length - ETH_FCS_LEN); 1029 1030 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) { 1031 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb); 1032 struct tsnep_rx_inline *rx_inline = 1033 (struct tsnep_rx_inline *)(page_address(page) + 1034 TSNEP_RX_OFFSET); 1035 1036 skb_shinfo(skb)->tx_flags |= 1037 SKBTX_HW_TSTAMP_NETDEV; 1038 memset(hwtstamps, 0, sizeof(*hwtstamps)); 1039 hwtstamps->netdev_data = rx_inline; 1040 } 1041 1042 skb_record_rx_queue(skb, rx->queue_index); 1043 skb->protocol = eth_type_trans(skb, rx->adapter->netdev); 1044 1045 return skb; 1046 } 1047 1048 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 1049 int budget) 1050 { 1051 struct device *dmadev = rx->adapter->dmadev; 1052 enum dma_data_direction dma_dir; 1053 struct tsnep_rx_entry *entry; 1054 struct netdev_queue *tx_nq; 1055 struct bpf_prog *prog; 1056 struct xdp_buff xdp; 1057 struct sk_buff *skb; 1058 struct tsnep_tx *tx; 1059 int desc_available; 1060 int xdp_status = 0; 1061 int done = 0; 1062 int length; 1063 1064 desc_available = tsnep_rx_desc_available(rx); 1065 dma_dir = page_pool_get_dma_dir(rx->page_pool); 1066 prog = READ_ONCE(rx->adapter->xdp_prog); 1067 if (prog) { 1068 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1069 rx->tx_queue_index); 1070 tx = &rx->adapter->tx[rx->tx_queue_index]; 1071 1072 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq); 1073 } 1074 1075 while (likely(done < budget) && (rx->read != rx->write)) { 1076 entry = &rx->entry[rx->read]; 1077 if ((__le32_to_cpu(entry->desc_wb->properties) & 1078 TSNEP_DESC_OWNER_COUNTER_MASK) != 1079 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1080 break; 1081 done++; 1082 1083 if (desc_available >= TSNEP_RING_RX_REFILL) { 1084 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1085 1086 desc_available -= tsnep_rx_refill(rx, desc_available, 1087 reuse); 1088 if (!entry->page) { 1089 /* buffer has been reused for refill to prevent 1090 * empty RX ring, thus buffer cannot be used for 1091 * RX processing 1092 */ 1093 rx->read = (rx->read + 1) % TSNEP_RING_SIZE; 1094 desc_available++; 1095 1096 rx->dropped++; 1097 1098 continue; 1099 } 1100 } 1101 1102 /* descriptor properties shall be read first, because valid data 1103 * is signaled there 1104 */ 1105 dma_rmb(); 1106 1107 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET); 1108 length = __le32_to_cpu(entry->desc_wb->properties) & 1109 TSNEP_DESC_LENGTH_MASK; 1110 dma_sync_single_range_for_cpu(dmadev, entry->dma, 1111 TSNEP_RX_OFFSET, length, dma_dir); 1112 1113 /* RX metadata with timestamps is in front of actual data, 1114 * subtract metadata size to get length of actual data and 1115 * consider metadata size as offset of actual data during RX 1116 * processing 1117 */ 1118 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1119 1120 rx->read = (rx->read + 1) % TSNEP_RING_SIZE; 1121 desc_available++; 1122 1123 if (prog) { 1124 bool consume; 1125 1126 xdp_prepare_buff(&xdp, page_address(entry->page), 1127 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE, 1128 length, false); 1129 1130 consume = tsnep_xdp_run_prog(rx, prog, &xdp, 1131 &xdp_status, tx_nq, tx); 1132 if (consume) { 1133 rx->packets++; 1134 rx->bytes += length; 1135 1136 entry->page = NULL; 1137 1138 continue; 1139 } 1140 } 1141 1142 skb = tsnep_build_skb(rx, entry->page, length); 1143 if (skb) { 1144 page_pool_release_page(rx->page_pool, entry->page); 1145 1146 rx->packets++; 1147 rx->bytes += length; 1148 if (skb->pkt_type == PACKET_MULTICAST) 1149 rx->multicast++; 1150 1151 napi_gro_receive(napi, skb); 1152 } else { 1153 page_pool_recycle_direct(rx->page_pool, entry->page); 1154 1155 rx->dropped++; 1156 } 1157 entry->page = NULL; 1158 } 1159 1160 if (xdp_status) 1161 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1162 1163 if (desc_available) 1164 tsnep_rx_refill(rx, desc_available, false); 1165 1166 return done; 1167 } 1168 1169 static bool tsnep_rx_pending(struct tsnep_rx *rx) 1170 { 1171 struct tsnep_rx_entry *entry; 1172 1173 if (rx->read != rx->write) { 1174 entry = &rx->entry[rx->read]; 1175 if ((__le32_to_cpu(entry->desc_wb->properties) & 1176 TSNEP_DESC_OWNER_COUNTER_MASK) == 1177 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1178 return true; 1179 } 1180 1181 return false; 1182 } 1183 1184 static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr, 1185 int queue_index, struct tsnep_rx *rx) 1186 { 1187 dma_addr_t dma; 1188 int retval; 1189 1190 memset(rx, 0, sizeof(*rx)); 1191 rx->adapter = adapter; 1192 rx->addr = addr; 1193 rx->queue_index = queue_index; 1194 1195 retval = tsnep_rx_ring_init(rx); 1196 if (retval) 1197 return retval; 1198 1199 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 1200 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW); 1201 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH); 1202 rx->owner_counter = 1; 1203 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1204 1205 tsnep_rx_refill(rx, tsnep_rx_desc_available(rx), false); 1206 1207 return 0; 1208 } 1209 1210 static void tsnep_rx_close(struct tsnep_rx *rx) 1211 { 1212 u32 val; 1213 1214 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL); 1215 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val, 1216 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000, 1217 1000000); 1218 1219 tsnep_rx_ring_cleanup(rx); 1220 } 1221 1222 static bool tsnep_pending(struct tsnep_queue *queue) 1223 { 1224 if (queue->tx && tsnep_tx_pending(queue->tx)) 1225 return true; 1226 1227 if (queue->rx && tsnep_rx_pending(queue->rx)) 1228 return true; 1229 1230 return false; 1231 } 1232 1233 static int tsnep_poll(struct napi_struct *napi, int budget) 1234 { 1235 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue, 1236 napi); 1237 bool complete = true; 1238 int done = 0; 1239 1240 if (queue->tx) 1241 complete = tsnep_tx_poll(queue->tx, budget); 1242 1243 if (queue->rx) { 1244 done = tsnep_rx_poll(queue->rx, napi, budget); 1245 if (done >= budget) 1246 complete = false; 1247 } 1248 1249 /* if all work not completed, return budget and keep polling */ 1250 if (!complete) 1251 return budget; 1252 1253 if (likely(napi_complete_done(napi, done))) { 1254 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1255 1256 /* reschedule if work is already pending, prevent rotten packets 1257 * which are transmitted or received after polling but before 1258 * interrupt enable 1259 */ 1260 if (tsnep_pending(queue)) { 1261 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1262 napi_schedule(napi); 1263 } 1264 } 1265 1266 return min(done, budget - 1); 1267 } 1268 1269 static int tsnep_request_irq(struct tsnep_queue *queue, bool first) 1270 { 1271 const char *name = netdev_name(queue->adapter->netdev); 1272 irq_handler_t handler; 1273 void *dev; 1274 int retval; 1275 1276 if (first) { 1277 sprintf(queue->name, "%s-mac", name); 1278 handler = tsnep_irq; 1279 dev = queue->adapter; 1280 } else { 1281 if (queue->tx && queue->rx) 1282 sprintf(queue->name, "%s-txrx-%d", name, 1283 queue->rx->queue_index); 1284 else if (queue->tx) 1285 sprintf(queue->name, "%s-tx-%d", name, 1286 queue->tx->queue_index); 1287 else 1288 sprintf(queue->name, "%s-rx-%d", name, 1289 queue->rx->queue_index); 1290 handler = tsnep_irq_txrx; 1291 dev = queue; 1292 } 1293 1294 retval = request_irq(queue->irq, handler, 0, queue->name, dev); 1295 if (retval) { 1296 /* if name is empty, then interrupt won't be freed */ 1297 memset(queue->name, 0, sizeof(queue->name)); 1298 } 1299 1300 return retval; 1301 } 1302 1303 static void tsnep_free_irq(struct tsnep_queue *queue, bool first) 1304 { 1305 void *dev; 1306 1307 if (!strlen(queue->name)) 1308 return; 1309 1310 if (first) 1311 dev = queue->adapter; 1312 else 1313 dev = queue; 1314 1315 free_irq(queue->irq, dev); 1316 memset(queue->name, 0, sizeof(queue->name)); 1317 } 1318 1319 static void tsnep_queue_close(struct tsnep_queue *queue, bool first) 1320 { 1321 struct tsnep_rx *rx = queue->rx; 1322 1323 tsnep_free_irq(queue, first); 1324 1325 if (rx && xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1326 xdp_rxq_info_unreg(&rx->xdp_rxq); 1327 1328 netif_napi_del(&queue->napi); 1329 } 1330 1331 static int tsnep_queue_open(struct tsnep_adapter *adapter, 1332 struct tsnep_queue *queue, bool first) 1333 { 1334 struct tsnep_rx *rx = queue->rx; 1335 struct tsnep_tx *tx = queue->tx; 1336 int retval; 1337 1338 queue->adapter = adapter; 1339 1340 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll); 1341 1342 if (rx) { 1343 /* choose TX queue for XDP_TX */ 1344 if (tx) 1345 rx->tx_queue_index = tx->queue_index; 1346 else if (rx->queue_index < adapter->num_tx_queues) 1347 rx->tx_queue_index = rx->queue_index; 1348 else 1349 rx->tx_queue_index = 0; 1350 1351 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev, 1352 rx->queue_index, queue->napi.napi_id); 1353 if (retval) 1354 goto failed; 1355 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1356 MEM_TYPE_PAGE_POOL, 1357 rx->page_pool); 1358 if (retval) 1359 goto failed; 1360 } 1361 1362 retval = tsnep_request_irq(queue, first); 1363 if (retval) { 1364 netif_err(adapter, drv, adapter->netdev, 1365 "can't get assigned irq %d.\n", queue->irq); 1366 goto failed; 1367 } 1368 1369 return 0; 1370 1371 failed: 1372 tsnep_queue_close(queue, first); 1373 1374 return retval; 1375 } 1376 1377 static int tsnep_netdev_open(struct net_device *netdev) 1378 { 1379 struct tsnep_adapter *adapter = netdev_priv(netdev); 1380 int tx_queue_index = 0; 1381 int rx_queue_index = 0; 1382 void __iomem *addr; 1383 int i, retval; 1384 1385 for (i = 0; i < adapter->num_queues; i++) { 1386 if (adapter->queue[i].tx) { 1387 addr = adapter->addr + TSNEP_QUEUE(tx_queue_index); 1388 retval = tsnep_tx_open(adapter, addr, tx_queue_index, 1389 adapter->queue[i].tx); 1390 if (retval) 1391 goto failed; 1392 tx_queue_index++; 1393 } 1394 if (adapter->queue[i].rx) { 1395 addr = adapter->addr + TSNEP_QUEUE(rx_queue_index); 1396 retval = tsnep_rx_open(adapter, addr, rx_queue_index, 1397 adapter->queue[i].rx); 1398 if (retval) 1399 goto failed; 1400 rx_queue_index++; 1401 } 1402 1403 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0); 1404 if (retval) 1405 goto failed; 1406 } 1407 1408 retval = netif_set_real_num_tx_queues(adapter->netdev, 1409 adapter->num_tx_queues); 1410 if (retval) 1411 goto failed; 1412 retval = netif_set_real_num_rx_queues(adapter->netdev, 1413 adapter->num_rx_queues); 1414 if (retval) 1415 goto failed; 1416 1417 tsnep_enable_irq(adapter, ECM_INT_LINK); 1418 retval = tsnep_phy_open(adapter); 1419 if (retval) 1420 goto phy_failed; 1421 1422 for (i = 0; i < adapter->num_queues; i++) { 1423 napi_enable(&adapter->queue[i].napi); 1424 1425 tsnep_enable_irq(adapter, adapter->queue[i].irq_mask); 1426 } 1427 1428 return 0; 1429 1430 phy_failed: 1431 tsnep_disable_irq(adapter, ECM_INT_LINK); 1432 failed: 1433 for (i = 0; i < adapter->num_queues; i++) { 1434 tsnep_queue_close(&adapter->queue[i], i == 0); 1435 1436 if (adapter->queue[i].rx) 1437 tsnep_rx_close(adapter->queue[i].rx); 1438 if (adapter->queue[i].tx) 1439 tsnep_tx_close(adapter->queue[i].tx); 1440 } 1441 return retval; 1442 } 1443 1444 static int tsnep_netdev_close(struct net_device *netdev) 1445 { 1446 struct tsnep_adapter *adapter = netdev_priv(netdev); 1447 int i; 1448 1449 tsnep_disable_irq(adapter, ECM_INT_LINK); 1450 tsnep_phy_close(adapter); 1451 1452 for (i = 0; i < adapter->num_queues; i++) { 1453 tsnep_disable_irq(adapter, adapter->queue[i].irq_mask); 1454 1455 napi_disable(&adapter->queue[i].napi); 1456 1457 tsnep_queue_close(&adapter->queue[i], i == 0); 1458 1459 if (adapter->queue[i].rx) 1460 tsnep_rx_close(adapter->queue[i].rx); 1461 if (adapter->queue[i].tx) 1462 tsnep_tx_close(adapter->queue[i].tx); 1463 } 1464 1465 return 0; 1466 } 1467 1468 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb, 1469 struct net_device *netdev) 1470 { 1471 struct tsnep_adapter *adapter = netdev_priv(netdev); 1472 u16 queue_mapping = skb_get_queue_mapping(skb); 1473 1474 if (queue_mapping >= adapter->num_tx_queues) 1475 queue_mapping = 0; 1476 1477 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]); 1478 } 1479 1480 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr, 1481 int cmd) 1482 { 1483 if (!netif_running(netdev)) 1484 return -EINVAL; 1485 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP) 1486 return tsnep_ptp_ioctl(netdev, ifr, cmd); 1487 return phy_mii_ioctl(netdev->phydev, ifr, cmd); 1488 } 1489 1490 static void tsnep_netdev_set_multicast(struct net_device *netdev) 1491 { 1492 struct tsnep_adapter *adapter = netdev_priv(netdev); 1493 1494 u16 rx_filter = 0; 1495 1496 /* configured MAC address and broadcasts are never filtered */ 1497 if (netdev->flags & IFF_PROMISC) { 1498 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 1499 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS; 1500 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) { 1501 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 1502 } 1503 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER); 1504 } 1505 1506 static void tsnep_netdev_get_stats64(struct net_device *netdev, 1507 struct rtnl_link_stats64 *stats) 1508 { 1509 struct tsnep_adapter *adapter = netdev_priv(netdev); 1510 u32 reg; 1511 u32 val; 1512 int i; 1513 1514 for (i = 0; i < adapter->num_tx_queues; i++) { 1515 stats->tx_packets += adapter->tx[i].packets; 1516 stats->tx_bytes += adapter->tx[i].bytes; 1517 stats->tx_dropped += adapter->tx[i].dropped; 1518 } 1519 for (i = 0; i < adapter->num_rx_queues; i++) { 1520 stats->rx_packets += adapter->rx[i].packets; 1521 stats->rx_bytes += adapter->rx[i].bytes; 1522 stats->rx_dropped += adapter->rx[i].dropped; 1523 stats->multicast += adapter->rx[i].multicast; 1524 1525 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + 1526 TSNEP_RX_STATISTIC); 1527 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >> 1528 TSNEP_RX_STATISTIC_NO_DESC_SHIFT; 1529 stats->rx_dropped += val; 1530 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >> 1531 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT; 1532 stats->rx_dropped += val; 1533 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >> 1534 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT; 1535 stats->rx_errors += val; 1536 stats->rx_fifo_errors += val; 1537 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >> 1538 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT; 1539 stats->rx_errors += val; 1540 stats->rx_frame_errors += val; 1541 } 1542 1543 reg = ioread32(adapter->addr + ECM_STAT); 1544 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT; 1545 stats->rx_errors += val; 1546 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT; 1547 stats->rx_errors += val; 1548 stats->rx_crc_errors += val; 1549 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT; 1550 stats->rx_errors += val; 1551 } 1552 1553 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr) 1554 { 1555 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW); 1556 iowrite16(*(u16 *)(addr + sizeof(u32)), 1557 adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 1558 1559 ether_addr_copy(adapter->mac_address, addr); 1560 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n", 1561 addr); 1562 } 1563 1564 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr) 1565 { 1566 struct tsnep_adapter *adapter = netdev_priv(netdev); 1567 struct sockaddr *sock_addr = addr; 1568 int retval; 1569 1570 retval = eth_prepare_mac_addr_change(netdev, sock_addr); 1571 if (retval) 1572 return retval; 1573 eth_hw_addr_set(netdev, sock_addr->sa_data); 1574 tsnep_mac_set_address(adapter, sock_addr->sa_data); 1575 1576 return 0; 1577 } 1578 1579 static int tsnep_netdev_set_features(struct net_device *netdev, 1580 netdev_features_t features) 1581 { 1582 struct tsnep_adapter *adapter = netdev_priv(netdev); 1583 netdev_features_t changed = netdev->features ^ features; 1584 bool enable; 1585 int retval = 0; 1586 1587 if (changed & NETIF_F_LOOPBACK) { 1588 enable = !!(features & NETIF_F_LOOPBACK); 1589 retval = tsnep_phy_loopback(adapter, enable); 1590 } 1591 1592 return retval; 1593 } 1594 1595 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev, 1596 const struct skb_shared_hwtstamps *hwtstamps, 1597 bool cycles) 1598 { 1599 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data; 1600 u64 timestamp; 1601 1602 if (cycles) 1603 timestamp = __le64_to_cpu(rx_inline->counter); 1604 else 1605 timestamp = __le64_to_cpu(rx_inline->timestamp); 1606 1607 return ns_to_ktime(timestamp); 1608 } 1609 1610 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf) 1611 { 1612 struct tsnep_adapter *adapter = netdev_priv(dev); 1613 1614 switch (bpf->command) { 1615 case XDP_SETUP_PROG: 1616 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack); 1617 default: 1618 return -EOPNOTSUPP; 1619 } 1620 } 1621 1622 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu) 1623 { 1624 if (cpu >= TSNEP_MAX_QUEUES) 1625 cpu &= TSNEP_MAX_QUEUES - 1; 1626 1627 while (cpu >= adapter->num_tx_queues) 1628 cpu -= adapter->num_tx_queues; 1629 1630 return &adapter->tx[cpu]; 1631 } 1632 1633 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n, 1634 struct xdp_frame **xdp, u32 flags) 1635 { 1636 struct tsnep_adapter *adapter = netdev_priv(dev); 1637 u32 cpu = smp_processor_id(); 1638 struct netdev_queue *nq; 1639 struct tsnep_tx *tx; 1640 int nxmit; 1641 bool xmit; 1642 1643 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1644 return -EINVAL; 1645 1646 tx = tsnep_xdp_get_tx(adapter, cpu); 1647 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index); 1648 1649 __netif_tx_lock(nq, cpu); 1650 1651 for (nxmit = 0; nxmit < n; nxmit++) { 1652 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx, 1653 TSNEP_TX_TYPE_XDP_NDO); 1654 if (!xmit) 1655 break; 1656 1657 /* avoid transmit queue timeout since we share it with the slow 1658 * path 1659 */ 1660 txq_trans_cond_update(nq); 1661 } 1662 1663 if (flags & XDP_XMIT_FLUSH) 1664 tsnep_xdp_xmit_flush(tx); 1665 1666 __netif_tx_unlock(nq); 1667 1668 return nxmit; 1669 } 1670 1671 static const struct net_device_ops tsnep_netdev_ops = { 1672 .ndo_open = tsnep_netdev_open, 1673 .ndo_stop = tsnep_netdev_close, 1674 .ndo_start_xmit = tsnep_netdev_xmit_frame, 1675 .ndo_eth_ioctl = tsnep_netdev_ioctl, 1676 .ndo_set_rx_mode = tsnep_netdev_set_multicast, 1677 .ndo_get_stats64 = tsnep_netdev_get_stats64, 1678 .ndo_set_mac_address = tsnep_netdev_set_mac_address, 1679 .ndo_set_features = tsnep_netdev_set_features, 1680 .ndo_get_tstamp = tsnep_netdev_get_tstamp, 1681 .ndo_setup_tc = tsnep_tc_setup, 1682 .ndo_bpf = tsnep_netdev_bpf, 1683 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit, 1684 }; 1685 1686 static int tsnep_mac_init(struct tsnep_adapter *adapter) 1687 { 1688 int retval; 1689 1690 /* initialize RX filtering, at least configured MAC address and 1691 * broadcast are not filtered 1692 */ 1693 iowrite16(0, adapter->addr + TSNEP_RX_FILTER); 1694 1695 /* try to get MAC address in the following order: 1696 * - device tree 1697 * - valid MAC address already set 1698 * - MAC address register if valid 1699 * - random MAC address 1700 */ 1701 retval = of_get_mac_address(adapter->pdev->dev.of_node, 1702 adapter->mac_address); 1703 if (retval == -EPROBE_DEFER) 1704 return retval; 1705 if (retval && !is_valid_ether_addr(adapter->mac_address)) { 1706 *(u32 *)adapter->mac_address = 1707 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW); 1708 *(u16 *)(adapter->mac_address + sizeof(u32)) = 1709 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 1710 if (!is_valid_ether_addr(adapter->mac_address)) 1711 eth_random_addr(adapter->mac_address); 1712 } 1713 1714 tsnep_mac_set_address(adapter, adapter->mac_address); 1715 eth_hw_addr_set(adapter->netdev, adapter->mac_address); 1716 1717 return 0; 1718 } 1719 1720 static int tsnep_mdio_init(struct tsnep_adapter *adapter) 1721 { 1722 struct device_node *np = adapter->pdev->dev.of_node; 1723 int retval; 1724 1725 if (np) { 1726 np = of_get_child_by_name(np, "mdio"); 1727 if (!np) 1728 return 0; 1729 1730 adapter->suppress_preamble = 1731 of_property_read_bool(np, "suppress-preamble"); 1732 } 1733 1734 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev); 1735 if (!adapter->mdiobus) { 1736 retval = -ENOMEM; 1737 1738 goto out; 1739 } 1740 1741 adapter->mdiobus->priv = (void *)adapter; 1742 adapter->mdiobus->parent = &adapter->pdev->dev; 1743 adapter->mdiobus->read = tsnep_mdiobus_read; 1744 adapter->mdiobus->write = tsnep_mdiobus_write; 1745 adapter->mdiobus->name = TSNEP "-mdiobus"; 1746 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s", 1747 adapter->pdev->name); 1748 1749 /* do not scan broadcast address */ 1750 adapter->mdiobus->phy_mask = 0x0000001; 1751 1752 retval = of_mdiobus_register(adapter->mdiobus, np); 1753 1754 out: 1755 of_node_put(np); 1756 1757 return retval; 1758 } 1759 1760 static int tsnep_phy_init(struct tsnep_adapter *adapter) 1761 { 1762 struct device_node *phy_node; 1763 int retval; 1764 1765 retval = of_get_phy_mode(adapter->pdev->dev.of_node, 1766 &adapter->phy_mode); 1767 if (retval) 1768 adapter->phy_mode = PHY_INTERFACE_MODE_GMII; 1769 1770 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle", 1771 0); 1772 adapter->phydev = of_phy_find_device(phy_node); 1773 of_node_put(phy_node); 1774 if (!adapter->phydev && adapter->mdiobus) 1775 adapter->phydev = phy_find_first(adapter->mdiobus); 1776 if (!adapter->phydev) 1777 return -EIO; 1778 1779 return 0; 1780 } 1781 1782 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) 1783 { 1784 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; 1785 char name[8]; 1786 int i; 1787 int retval; 1788 1789 /* one TX/RX queue pair for netdev is mandatory */ 1790 if (platform_irq_count(adapter->pdev) == 1) 1791 retval = platform_get_irq(adapter->pdev, 0); 1792 else 1793 retval = platform_get_irq_byname(adapter->pdev, "mac"); 1794 if (retval < 0) 1795 return retval; 1796 adapter->num_tx_queues = 1; 1797 adapter->num_rx_queues = 1; 1798 adapter->num_queues = 1; 1799 adapter->queue[0].irq = retval; 1800 adapter->queue[0].tx = &adapter->tx[0]; 1801 adapter->queue[0].rx = &adapter->rx[0]; 1802 adapter->queue[0].irq_mask = irq_mask; 1803 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY; 1804 retval = tsnep_set_irq_coalesce(&adapter->queue[0], 1805 TSNEP_COALESCE_USECS_DEFAULT); 1806 if (retval < 0) 1807 return retval; 1808 1809 adapter->netdev->irq = adapter->queue[0].irq; 1810 1811 /* add additional TX/RX queue pairs only if dedicated interrupt is 1812 * available 1813 */ 1814 for (i = 1; i < queue_count; i++) { 1815 sprintf(name, "txrx-%d", i); 1816 retval = platform_get_irq_byname_optional(adapter->pdev, name); 1817 if (retval < 0) 1818 break; 1819 1820 adapter->num_tx_queues++; 1821 adapter->num_rx_queues++; 1822 adapter->num_queues++; 1823 adapter->queue[i].irq = retval; 1824 adapter->queue[i].tx = &adapter->tx[i]; 1825 adapter->queue[i].rx = &adapter->rx[i]; 1826 adapter->queue[i].irq_mask = 1827 irq_mask << (ECM_INT_TXRX_SHIFT * i); 1828 adapter->queue[i].irq_delay_addr = 1829 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i; 1830 retval = tsnep_set_irq_coalesce(&adapter->queue[i], 1831 TSNEP_COALESCE_USECS_DEFAULT); 1832 if (retval < 0) 1833 return retval; 1834 } 1835 1836 return 0; 1837 } 1838 1839 static int tsnep_probe(struct platform_device *pdev) 1840 { 1841 struct tsnep_adapter *adapter; 1842 struct net_device *netdev; 1843 struct resource *io; 1844 u32 type; 1845 int revision; 1846 int version; 1847 int queue_count; 1848 int retval; 1849 1850 netdev = devm_alloc_etherdev_mqs(&pdev->dev, 1851 sizeof(struct tsnep_adapter), 1852 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES); 1853 if (!netdev) 1854 return -ENODEV; 1855 SET_NETDEV_DEV(netdev, &pdev->dev); 1856 adapter = netdev_priv(netdev); 1857 platform_set_drvdata(pdev, adapter); 1858 adapter->pdev = pdev; 1859 adapter->dmadev = &pdev->dev; 1860 adapter->netdev = netdev; 1861 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | 1862 NETIF_MSG_LINK | NETIF_MSG_IFUP | 1863 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED; 1864 1865 netdev->min_mtu = ETH_MIN_MTU; 1866 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE; 1867 1868 mutex_init(&adapter->gate_control_lock); 1869 mutex_init(&adapter->rxnfc_lock); 1870 INIT_LIST_HEAD(&adapter->rxnfc_rules); 1871 1872 io = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1873 adapter->addr = devm_ioremap_resource(&pdev->dev, io); 1874 if (IS_ERR(adapter->addr)) 1875 return PTR_ERR(adapter->addr); 1876 netdev->mem_start = io->start; 1877 netdev->mem_end = io->end; 1878 1879 type = ioread32(adapter->addr + ECM_TYPE); 1880 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT; 1881 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT; 1882 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT; 1883 adapter->gate_control = type & ECM_GATE_CONTROL; 1884 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT; 1885 1886 tsnep_disable_irq(adapter, ECM_INT_ALL); 1887 1888 retval = tsnep_queue_init(adapter, queue_count); 1889 if (retval) 1890 return retval; 1891 1892 retval = dma_set_mask_and_coherent(&adapter->pdev->dev, 1893 DMA_BIT_MASK(64)); 1894 if (retval) { 1895 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n"); 1896 return retval; 1897 } 1898 1899 retval = tsnep_mac_init(adapter); 1900 if (retval) 1901 return retval; 1902 1903 retval = tsnep_mdio_init(adapter); 1904 if (retval) 1905 goto mdio_init_failed; 1906 1907 retval = tsnep_phy_init(adapter); 1908 if (retval) 1909 goto phy_init_failed; 1910 1911 retval = tsnep_ptp_init(adapter); 1912 if (retval) 1913 goto ptp_init_failed; 1914 1915 retval = tsnep_tc_init(adapter); 1916 if (retval) 1917 goto tc_init_failed; 1918 1919 retval = tsnep_rxnfc_init(adapter); 1920 if (retval) 1921 goto rxnfc_init_failed; 1922 1923 netdev->netdev_ops = &tsnep_netdev_ops; 1924 netdev->ethtool_ops = &tsnep_ethtool_ops; 1925 netdev->features = NETIF_F_SG; 1926 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK; 1927 1928 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 1929 NETDEV_XDP_ACT_NDO_XMIT | 1930 NETDEV_XDP_ACT_NDO_XMIT_SG; 1931 1932 /* carrier off reporting is important to ethtool even BEFORE open */ 1933 netif_carrier_off(netdev); 1934 1935 retval = register_netdev(netdev); 1936 if (retval) 1937 goto register_failed; 1938 1939 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version, 1940 revision); 1941 if (adapter->gate_control) 1942 dev_info(&adapter->pdev->dev, "gate control detected\n"); 1943 1944 return 0; 1945 1946 register_failed: 1947 tsnep_rxnfc_cleanup(adapter); 1948 rxnfc_init_failed: 1949 tsnep_tc_cleanup(adapter); 1950 tc_init_failed: 1951 tsnep_ptp_cleanup(adapter); 1952 ptp_init_failed: 1953 phy_init_failed: 1954 if (adapter->mdiobus) 1955 mdiobus_unregister(adapter->mdiobus); 1956 mdio_init_failed: 1957 return retval; 1958 } 1959 1960 static int tsnep_remove(struct platform_device *pdev) 1961 { 1962 struct tsnep_adapter *adapter = platform_get_drvdata(pdev); 1963 1964 unregister_netdev(adapter->netdev); 1965 1966 tsnep_rxnfc_cleanup(adapter); 1967 1968 tsnep_tc_cleanup(adapter); 1969 1970 tsnep_ptp_cleanup(adapter); 1971 1972 if (adapter->mdiobus) 1973 mdiobus_unregister(adapter->mdiobus); 1974 1975 tsnep_disable_irq(adapter, ECM_INT_ALL); 1976 1977 return 0; 1978 } 1979 1980 static const struct of_device_id tsnep_of_match[] = { 1981 { .compatible = "engleder,tsnep", }, 1982 { }, 1983 }; 1984 MODULE_DEVICE_TABLE(of, tsnep_of_match); 1985 1986 static struct platform_driver tsnep_driver = { 1987 .driver = { 1988 .name = TSNEP, 1989 .of_match_table = tsnep_of_match, 1990 }, 1991 .probe = tsnep_probe, 1992 .remove = tsnep_remove, 1993 }; 1994 module_platform_driver(tsnep_driver); 1995 1996 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>"); 1997 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver"); 1998 MODULE_LICENSE("GPL"); 1999