1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/clk.h> 10 #include <linux/crc32.h> 11 #include <linux/module.h> 12 #include <linux/moduleparam.h> 13 #include <linux/kernel.h> 14 #include <linux/types.h> 15 #include <linux/circ_buf.h> 16 #include <linux/slab.h> 17 #include <linux/init.h> 18 #include <linux/io.h> 19 #include <linux/gpio.h> 20 #include <linux/gpio/consumer.h> 21 #include <linux/interrupt.h> 22 #include <linux/netdevice.h> 23 #include <linux/etherdevice.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/platform_data/macb.h> 26 #include <linux/platform_device.h> 27 #include <linux/phy.h> 28 #include <linux/of.h> 29 #include <linux/of_device.h> 30 #include <linux/of_gpio.h> 31 #include <linux/of_mdio.h> 32 #include <linux/of_net.h> 33 #include <linux/ip.h> 34 #include <linux/udp.h> 35 #include <linux/tcp.h> 36 #include <linux/iopoll.h> 37 #include <linux/pm_runtime.h> 38 #include "macb.h" 39 40 #define MACB_RX_BUFFER_SIZE 128 41 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 42 43 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 44 #define MIN_RX_RING_SIZE 64 45 #define MAX_RX_RING_SIZE 8192 46 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 47 * (bp)->rx_ring_size) 48 49 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 50 #define MIN_TX_RING_SIZE 64 51 #define MAX_TX_RING_SIZE 4096 52 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 53 * (bp)->tx_ring_size) 54 55 /* level of occupied TX descriptors under which we wake up TX process */ 56 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 57 58 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 59 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 60 | MACB_BIT(ISR_RLE) \ 61 | MACB_BIT(TXERR)) 62 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 63 | MACB_BIT(TXUBR)) 64 65 /* Max length of transmit frame must be a multiple of 8 bytes */ 66 #define MACB_TX_LEN_ALIGN 8 67 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 68 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 69 70 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 71 #define MACB_NETIF_LSO NETIF_F_TSO 72 73 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 74 #define MACB_WOL_ENABLED (0x1 << 1) 75 76 /* Graceful stop timeouts in us. We should allow up to 77 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 78 */ 79 #define MACB_HALT_TIMEOUT 1230 80 81 #define MACB_PM_TIMEOUT 100 /* ms */ 82 83 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 84 85 /* DMA buffer descriptor might be different size 86 * depends on hardware configuration: 87 * 88 * 1. dma address width 32 bits: 89 * word 1: 32 bit address of Data Buffer 90 * word 2: control 91 * 92 * 2. dma address width 64 bits: 93 * word 1: 32 bit address of Data Buffer 94 * word 2: control 95 * word 3: upper 32 bit address of Data Buffer 96 * word 4: unused 97 * 98 * 3. dma address width 32 bits with hardware timestamping: 99 * word 1: 32 bit address of Data Buffer 100 * word 2: control 101 * word 3: timestamp word 1 102 * word 4: timestamp word 2 103 * 104 * 4. dma address width 64 bits with hardware timestamping: 105 * word 1: 32 bit address of Data Buffer 106 * word 2: control 107 * word 3: upper 32 bit address of Data Buffer 108 * word 4: unused 109 * word 5: timestamp word 1 110 * word 6: timestamp word 2 111 */ 112 static unsigned int macb_dma_desc_get_size(struct macb *bp) 113 { 114 #ifdef MACB_EXT_DESC 115 unsigned int desc_size; 116 117 switch (bp->hw_dma_cap) { 118 case HW_DMA_CAP_64B: 119 desc_size = sizeof(struct macb_dma_desc) 120 + sizeof(struct macb_dma_desc_64); 121 break; 122 case HW_DMA_CAP_PTP: 123 desc_size = sizeof(struct macb_dma_desc) 124 + sizeof(struct macb_dma_desc_ptp); 125 break; 126 case HW_DMA_CAP_64B_PTP: 127 desc_size = sizeof(struct macb_dma_desc) 128 + sizeof(struct macb_dma_desc_64) 129 + sizeof(struct macb_dma_desc_ptp); 130 break; 131 default: 132 desc_size = sizeof(struct macb_dma_desc); 133 } 134 return desc_size; 135 #endif 136 return sizeof(struct macb_dma_desc); 137 } 138 139 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 140 { 141 #ifdef MACB_EXT_DESC 142 switch (bp->hw_dma_cap) { 143 case HW_DMA_CAP_64B: 144 case HW_DMA_CAP_PTP: 145 desc_idx <<= 1; 146 break; 147 case HW_DMA_CAP_64B_PTP: 148 desc_idx *= 3; 149 break; 150 default: 151 break; 152 } 153 #endif 154 return desc_idx; 155 } 156 157 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 158 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 159 { 160 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 161 return (struct macb_dma_desc_64 *)((void *)desc + sizeof(struct macb_dma_desc)); 162 return NULL; 163 } 164 #endif 165 166 /* Ring buffer accessors */ 167 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 168 { 169 return index & (bp->tx_ring_size - 1); 170 } 171 172 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 173 unsigned int index) 174 { 175 index = macb_tx_ring_wrap(queue->bp, index); 176 index = macb_adj_dma_desc_idx(queue->bp, index); 177 return &queue->tx_ring[index]; 178 } 179 180 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 181 unsigned int index) 182 { 183 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 184 } 185 186 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 187 { 188 dma_addr_t offset; 189 190 offset = macb_tx_ring_wrap(queue->bp, index) * 191 macb_dma_desc_get_size(queue->bp); 192 193 return queue->tx_ring_dma + offset; 194 } 195 196 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 197 { 198 return index & (bp->rx_ring_size - 1); 199 } 200 201 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 202 { 203 index = macb_rx_ring_wrap(queue->bp, index); 204 index = macb_adj_dma_desc_idx(queue->bp, index); 205 return &queue->rx_ring[index]; 206 } 207 208 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 209 { 210 return queue->rx_buffers + queue->bp->rx_buffer_size * 211 macb_rx_ring_wrap(queue->bp, index); 212 } 213 214 /* I/O accessors */ 215 static u32 hw_readl_native(struct macb *bp, int offset) 216 { 217 return __raw_readl(bp->regs + offset); 218 } 219 220 static void hw_writel_native(struct macb *bp, int offset, u32 value) 221 { 222 __raw_writel(value, bp->regs + offset); 223 } 224 225 static u32 hw_readl(struct macb *bp, int offset) 226 { 227 return readl_relaxed(bp->regs + offset); 228 } 229 230 static void hw_writel(struct macb *bp, int offset, u32 value) 231 { 232 writel_relaxed(value, bp->regs + offset); 233 } 234 235 /* Find the CPU endianness by using the loopback bit of NCR register. When the 236 * CPU is in big endian we need to program swapped mode for management 237 * descriptor access. 238 */ 239 static bool hw_is_native_io(void __iomem *addr) 240 { 241 u32 value = MACB_BIT(LLB); 242 243 __raw_writel(value, addr + MACB_NCR); 244 value = __raw_readl(addr + MACB_NCR); 245 246 /* Write 0 back to disable everything */ 247 __raw_writel(0, addr + MACB_NCR); 248 249 return value == MACB_BIT(LLB); 250 } 251 252 static bool hw_is_gem(void __iomem *addr, bool native_io) 253 { 254 u32 id; 255 256 if (native_io) 257 id = __raw_readl(addr + MACB_MID); 258 else 259 id = readl_relaxed(addr + MACB_MID); 260 261 return MACB_BFEXT(IDNUM, id) >= 0x2; 262 } 263 264 static void macb_set_hwaddr(struct macb *bp) 265 { 266 u32 bottom; 267 u16 top; 268 269 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 270 macb_or_gem_writel(bp, SA1B, bottom); 271 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 272 macb_or_gem_writel(bp, SA1T, top); 273 274 /* Clear unused address register sets */ 275 macb_or_gem_writel(bp, SA2B, 0); 276 macb_or_gem_writel(bp, SA2T, 0); 277 macb_or_gem_writel(bp, SA3B, 0); 278 macb_or_gem_writel(bp, SA3T, 0); 279 macb_or_gem_writel(bp, SA4B, 0); 280 macb_or_gem_writel(bp, SA4T, 0); 281 } 282 283 static void macb_get_hwaddr(struct macb *bp) 284 { 285 u32 bottom; 286 u16 top; 287 u8 addr[6]; 288 int i; 289 290 /* Check all 4 address register for valid address */ 291 for (i = 0; i < 4; i++) { 292 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 293 top = macb_or_gem_readl(bp, SA1T + i * 8); 294 295 addr[0] = bottom & 0xff; 296 addr[1] = (bottom >> 8) & 0xff; 297 addr[2] = (bottom >> 16) & 0xff; 298 addr[3] = (bottom >> 24) & 0xff; 299 addr[4] = top & 0xff; 300 addr[5] = (top >> 8) & 0xff; 301 302 if (is_valid_ether_addr(addr)) { 303 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 304 return; 305 } 306 } 307 308 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 309 eth_hw_addr_random(bp->dev); 310 } 311 312 static int macb_mdio_wait_for_idle(struct macb *bp) 313 { 314 u32 val; 315 316 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 317 1, MACB_MDIO_TIMEOUT); 318 } 319 320 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 321 { 322 struct macb *bp = bus->priv; 323 int status; 324 325 status = pm_runtime_get_sync(&bp->pdev->dev); 326 if (status < 0) 327 goto mdio_pm_exit; 328 329 status = macb_mdio_wait_for_idle(bp); 330 if (status < 0) 331 goto mdio_read_exit; 332 333 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 334 | MACB_BF(RW, MACB_MAN_READ) 335 | MACB_BF(PHYA, mii_id) 336 | MACB_BF(REGA, regnum) 337 | MACB_BF(CODE, MACB_MAN_CODE))); 338 339 status = macb_mdio_wait_for_idle(bp); 340 if (status < 0) 341 goto mdio_read_exit; 342 343 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 344 345 mdio_read_exit: 346 pm_runtime_mark_last_busy(&bp->pdev->dev); 347 pm_runtime_put_autosuspend(&bp->pdev->dev); 348 mdio_pm_exit: 349 return status; 350 } 351 352 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 353 u16 value) 354 { 355 struct macb *bp = bus->priv; 356 int status; 357 358 status = pm_runtime_get_sync(&bp->pdev->dev); 359 if (status < 0) 360 goto mdio_pm_exit; 361 362 status = macb_mdio_wait_for_idle(bp); 363 if (status < 0) 364 goto mdio_write_exit; 365 366 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 367 | MACB_BF(RW, MACB_MAN_WRITE) 368 | MACB_BF(PHYA, mii_id) 369 | MACB_BF(REGA, regnum) 370 | MACB_BF(CODE, MACB_MAN_CODE) 371 | MACB_BF(DATA, value))); 372 373 status = macb_mdio_wait_for_idle(bp); 374 if (status < 0) 375 goto mdio_write_exit; 376 377 mdio_write_exit: 378 pm_runtime_mark_last_busy(&bp->pdev->dev); 379 pm_runtime_put_autosuspend(&bp->pdev->dev); 380 mdio_pm_exit: 381 return status; 382 } 383 384 /** 385 * macb_set_tx_clk() - Set a clock to a new frequency 386 * @clk Pointer to the clock to change 387 * @rate New frequency in Hz 388 * @dev Pointer to the struct net_device 389 */ 390 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 391 { 392 long ferr, rate, rate_rounded; 393 394 if (!clk) 395 return; 396 397 switch (speed) { 398 case SPEED_10: 399 rate = 2500000; 400 break; 401 case SPEED_100: 402 rate = 25000000; 403 break; 404 case SPEED_1000: 405 rate = 125000000; 406 break; 407 default: 408 return; 409 } 410 411 rate_rounded = clk_round_rate(clk, rate); 412 if (rate_rounded < 0) 413 return; 414 415 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 416 * is not satisfied. 417 */ 418 ferr = abs(rate_rounded - rate); 419 ferr = DIV_ROUND_UP(ferr, rate / 100000); 420 if (ferr > 5) 421 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 422 rate); 423 424 if (clk_set_rate(clk, rate_rounded)) 425 netdev_err(dev, "adjusting tx_clk failed.\n"); 426 } 427 428 static void macb_handle_link_change(struct net_device *dev) 429 { 430 struct macb *bp = netdev_priv(dev); 431 struct phy_device *phydev = dev->phydev; 432 unsigned long flags; 433 int status_change = 0; 434 435 spin_lock_irqsave(&bp->lock, flags); 436 437 if (phydev->link) { 438 if ((bp->speed != phydev->speed) || 439 (bp->duplex != phydev->duplex)) { 440 u32 reg; 441 442 reg = macb_readl(bp, NCFGR); 443 reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 444 if (macb_is_gem(bp)) 445 reg &= ~GEM_BIT(GBE); 446 447 if (phydev->duplex) 448 reg |= MACB_BIT(FD); 449 if (phydev->speed == SPEED_100) 450 reg |= MACB_BIT(SPD); 451 if (phydev->speed == SPEED_1000 && 452 bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 453 reg |= GEM_BIT(GBE); 454 455 macb_or_gem_writel(bp, NCFGR, reg); 456 457 bp->speed = phydev->speed; 458 bp->duplex = phydev->duplex; 459 status_change = 1; 460 } 461 } 462 463 if (phydev->link != bp->link) { 464 if (!phydev->link) { 465 bp->speed = 0; 466 bp->duplex = -1; 467 } 468 bp->link = phydev->link; 469 470 status_change = 1; 471 } 472 473 spin_unlock_irqrestore(&bp->lock, flags); 474 475 if (status_change) { 476 if (phydev->link) { 477 /* Update the TX clock rate if and only if the link is 478 * up and there has been a link change. 479 */ 480 macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); 481 482 netif_carrier_on(dev); 483 netdev_info(dev, "link up (%d/%s)\n", 484 phydev->speed, 485 phydev->duplex == DUPLEX_FULL ? 486 "Full" : "Half"); 487 } else { 488 netif_carrier_off(dev); 489 netdev_info(dev, "link down\n"); 490 } 491 } 492 } 493 494 /* based on au1000_eth. c*/ 495 static int macb_mii_probe(struct net_device *dev) 496 { 497 struct macb *bp = netdev_priv(dev); 498 struct phy_device *phydev; 499 struct device_node *np; 500 int ret, i; 501 502 np = bp->pdev->dev.of_node; 503 ret = 0; 504 505 if (np) { 506 if (of_phy_is_fixed_link(np)) { 507 bp->phy_node = of_node_get(np); 508 } else { 509 bp->phy_node = of_parse_phandle(np, "phy-handle", 0); 510 /* fallback to standard phy registration if no 511 * phy-handle was found nor any phy found during 512 * dt phy registration 513 */ 514 if (!bp->phy_node && !phy_find_first(bp->mii_bus)) { 515 for (i = 0; i < PHY_MAX_ADDR; i++) { 516 phydev = mdiobus_scan(bp->mii_bus, i); 517 if (IS_ERR(phydev) && 518 PTR_ERR(phydev) != -ENODEV) { 519 ret = PTR_ERR(phydev); 520 break; 521 } 522 } 523 524 if (ret) 525 return -ENODEV; 526 } 527 } 528 } 529 530 if (bp->phy_node) { 531 phydev = of_phy_connect(dev, bp->phy_node, 532 &macb_handle_link_change, 0, 533 bp->phy_interface); 534 if (!phydev) 535 return -ENODEV; 536 } else { 537 phydev = phy_find_first(bp->mii_bus); 538 if (!phydev) { 539 netdev_err(dev, "no PHY found\n"); 540 return -ENXIO; 541 } 542 543 /* attach the mac to the phy */ 544 ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, 545 bp->phy_interface); 546 if (ret) { 547 netdev_err(dev, "Could not attach to PHY\n"); 548 return ret; 549 } 550 } 551 552 /* mask with MAC supported features */ 553 if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 554 phy_set_max_speed(phydev, SPEED_1000); 555 else 556 phy_set_max_speed(phydev, SPEED_100); 557 558 if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) 559 phy_remove_link_mode(phydev, 560 ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 561 562 bp->link = 0; 563 bp->speed = 0; 564 bp->duplex = -1; 565 566 return 0; 567 } 568 569 static int macb_mii_init(struct macb *bp) 570 { 571 struct device_node *np; 572 int err = -ENXIO; 573 574 /* Enable management port */ 575 macb_writel(bp, NCR, MACB_BIT(MPE)); 576 577 bp->mii_bus = mdiobus_alloc(); 578 if (!bp->mii_bus) { 579 err = -ENOMEM; 580 goto err_out; 581 } 582 583 bp->mii_bus->name = "MACB_mii_bus"; 584 bp->mii_bus->read = &macb_mdio_read; 585 bp->mii_bus->write = &macb_mdio_write; 586 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 587 bp->pdev->name, bp->pdev->id); 588 bp->mii_bus->priv = bp; 589 bp->mii_bus->parent = &bp->pdev->dev; 590 591 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 592 593 np = bp->pdev->dev.of_node; 594 if (np && of_phy_is_fixed_link(np)) { 595 if (of_phy_register_fixed_link(np) < 0) { 596 dev_err(&bp->pdev->dev, 597 "broken fixed-link specification %pOF\n", np); 598 goto err_out_free_mdiobus; 599 } 600 601 err = mdiobus_register(bp->mii_bus); 602 } else { 603 err = of_mdiobus_register(bp->mii_bus, np); 604 } 605 606 if (err) 607 goto err_out_free_fixed_link; 608 609 err = macb_mii_probe(bp->dev); 610 if (err) 611 goto err_out_unregister_bus; 612 613 return 0; 614 615 err_out_unregister_bus: 616 mdiobus_unregister(bp->mii_bus); 617 err_out_free_fixed_link: 618 if (np && of_phy_is_fixed_link(np)) 619 of_phy_deregister_fixed_link(np); 620 err_out_free_mdiobus: 621 of_node_put(bp->phy_node); 622 mdiobus_free(bp->mii_bus); 623 err_out: 624 return err; 625 } 626 627 static void macb_update_stats(struct macb *bp) 628 { 629 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 630 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 631 int offset = MACB_PFR; 632 633 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 634 635 for (; p < end; p++, offset += 4) 636 *p += bp->macb_reg_readl(bp, offset); 637 } 638 639 static int macb_halt_tx(struct macb *bp) 640 { 641 unsigned long halt_time, timeout; 642 u32 status; 643 644 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 645 646 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 647 do { 648 halt_time = jiffies; 649 status = macb_readl(bp, TSR); 650 if (!(status & MACB_BIT(TGO))) 651 return 0; 652 653 udelay(250); 654 } while (time_before(halt_time, timeout)); 655 656 return -ETIMEDOUT; 657 } 658 659 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 660 { 661 if (tx_skb->mapping) { 662 if (tx_skb->mapped_as_page) 663 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 664 tx_skb->size, DMA_TO_DEVICE); 665 else 666 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 667 tx_skb->size, DMA_TO_DEVICE); 668 tx_skb->mapping = 0; 669 } 670 671 if (tx_skb->skb) { 672 dev_kfree_skb_any(tx_skb->skb); 673 tx_skb->skb = NULL; 674 } 675 } 676 677 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 678 { 679 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 680 struct macb_dma_desc_64 *desc_64; 681 682 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 683 desc_64 = macb_64b_desc(bp, desc); 684 desc_64->addrh = upper_32_bits(addr); 685 /* The low bits of RX address contain the RX_USED bit, clearing 686 * of which allows packet RX. Make sure the high bits are also 687 * visible to HW at that point. 688 */ 689 dma_wmb(); 690 } 691 #endif 692 desc->addr = lower_32_bits(addr); 693 } 694 695 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 696 { 697 dma_addr_t addr = 0; 698 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 699 struct macb_dma_desc_64 *desc_64; 700 701 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 702 desc_64 = macb_64b_desc(bp, desc); 703 addr = ((u64)(desc_64->addrh) << 32); 704 } 705 #endif 706 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 707 return addr; 708 } 709 710 static void macb_tx_error_task(struct work_struct *work) 711 { 712 struct macb_queue *queue = container_of(work, struct macb_queue, 713 tx_error_task); 714 struct macb *bp = queue->bp; 715 struct macb_tx_skb *tx_skb; 716 struct macb_dma_desc *desc; 717 struct sk_buff *skb; 718 unsigned int tail; 719 unsigned long flags; 720 721 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 722 (unsigned int)(queue - bp->queues), 723 queue->tx_tail, queue->tx_head); 724 725 /* Prevent the queue IRQ handlers from running: each of them may call 726 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 727 * As explained below, we have to halt the transmission before updating 728 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 729 * network engine about the macb/gem being halted. 730 */ 731 spin_lock_irqsave(&bp->lock, flags); 732 733 /* Make sure nobody is trying to queue up new packets */ 734 netif_tx_stop_all_queues(bp->dev); 735 736 /* Stop transmission now 737 * (in case we have just queued new packets) 738 * macb/gem must be halted to write TBQP register 739 */ 740 if (macb_halt_tx(bp)) 741 /* Just complain for now, reinitializing TX path can be good */ 742 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 743 744 /* Treat frames in TX queue including the ones that caused the error. 745 * Free transmit buffers in upper layer. 746 */ 747 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 748 u32 ctrl; 749 750 desc = macb_tx_desc(queue, tail); 751 ctrl = desc->ctrl; 752 tx_skb = macb_tx_skb(queue, tail); 753 skb = tx_skb->skb; 754 755 if (ctrl & MACB_BIT(TX_USED)) { 756 /* skb is set for the last buffer of the frame */ 757 while (!skb) { 758 macb_tx_unmap(bp, tx_skb); 759 tail++; 760 tx_skb = macb_tx_skb(queue, tail); 761 skb = tx_skb->skb; 762 } 763 764 /* ctrl still refers to the first buffer descriptor 765 * since it's the only one written back by the hardware 766 */ 767 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 768 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 769 macb_tx_ring_wrap(bp, tail), 770 skb->data); 771 bp->dev->stats.tx_packets++; 772 queue->stats.tx_packets++; 773 bp->dev->stats.tx_bytes += skb->len; 774 queue->stats.tx_bytes += skb->len; 775 } 776 } else { 777 /* "Buffers exhausted mid-frame" errors may only happen 778 * if the driver is buggy, so complain loudly about 779 * those. Statistics are updated by hardware. 780 */ 781 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 782 netdev_err(bp->dev, 783 "BUG: TX buffers exhausted mid-frame\n"); 784 785 desc->ctrl = ctrl | MACB_BIT(TX_USED); 786 } 787 788 macb_tx_unmap(bp, tx_skb); 789 } 790 791 /* Set end of TX queue */ 792 desc = macb_tx_desc(queue, 0); 793 macb_set_addr(bp, desc, 0); 794 desc->ctrl = MACB_BIT(TX_USED); 795 796 /* Make descriptor updates visible to hardware */ 797 wmb(); 798 799 /* Reinitialize the TX desc queue */ 800 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 801 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 802 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 803 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 804 #endif 805 /* Make TX ring reflect state of hardware */ 806 queue->tx_head = 0; 807 queue->tx_tail = 0; 808 809 /* Housework before enabling TX IRQ */ 810 macb_writel(bp, TSR, macb_readl(bp, TSR)); 811 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 812 813 /* Now we are ready to start transmission again */ 814 netif_tx_start_all_queues(bp->dev); 815 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 816 817 spin_unlock_irqrestore(&bp->lock, flags); 818 } 819 820 static void macb_tx_interrupt(struct macb_queue *queue) 821 { 822 unsigned int tail; 823 unsigned int head; 824 u32 status; 825 struct macb *bp = queue->bp; 826 u16 queue_index = queue - bp->queues; 827 828 status = macb_readl(bp, TSR); 829 macb_writel(bp, TSR, status); 830 831 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 832 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 833 834 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 835 (unsigned long)status); 836 837 head = queue->tx_head; 838 for (tail = queue->tx_tail; tail != head; tail++) { 839 struct macb_tx_skb *tx_skb; 840 struct sk_buff *skb; 841 struct macb_dma_desc *desc; 842 u32 ctrl; 843 844 desc = macb_tx_desc(queue, tail); 845 846 /* Make hw descriptor updates visible to CPU */ 847 rmb(); 848 849 ctrl = desc->ctrl; 850 851 /* TX_USED bit is only set by hardware on the very first buffer 852 * descriptor of the transmitted frame. 853 */ 854 if (!(ctrl & MACB_BIT(TX_USED))) 855 break; 856 857 /* Process all buffers of the current transmitted frame */ 858 for (;; tail++) { 859 tx_skb = macb_tx_skb(queue, tail); 860 skb = tx_skb->skb; 861 862 /* First, update TX stats if needed */ 863 if (skb) { 864 if (unlikely(skb_shinfo(skb)->tx_flags & 865 SKBTX_HW_TSTAMP) && 866 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 867 /* skb now belongs to timestamp buffer 868 * and will be removed later 869 */ 870 tx_skb->skb = NULL; 871 } 872 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 873 macb_tx_ring_wrap(bp, tail), 874 skb->data); 875 bp->dev->stats.tx_packets++; 876 queue->stats.tx_packets++; 877 bp->dev->stats.tx_bytes += skb->len; 878 queue->stats.tx_bytes += skb->len; 879 } 880 881 /* Now we can safely release resources */ 882 macb_tx_unmap(bp, tx_skb); 883 884 /* skb is set only for the last buffer of the frame. 885 * WARNING: at this point skb has been freed by 886 * macb_tx_unmap(). 887 */ 888 if (skb) 889 break; 890 } 891 } 892 893 queue->tx_tail = tail; 894 if (__netif_subqueue_stopped(bp->dev, queue_index) && 895 CIRC_CNT(queue->tx_head, queue->tx_tail, 896 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 897 netif_wake_subqueue(bp->dev, queue_index); 898 } 899 900 static void gem_rx_refill(struct macb_queue *queue) 901 { 902 unsigned int entry; 903 struct sk_buff *skb; 904 dma_addr_t paddr; 905 struct macb *bp = queue->bp; 906 struct macb_dma_desc *desc; 907 908 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 909 bp->rx_ring_size) > 0) { 910 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 911 912 /* Make hw descriptor updates visible to CPU */ 913 rmb(); 914 915 queue->rx_prepared_head++; 916 desc = macb_rx_desc(queue, entry); 917 918 if (!queue->rx_skbuff[entry]) { 919 /* allocate sk_buff for this free entry in ring */ 920 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 921 if (unlikely(!skb)) { 922 netdev_err(bp->dev, 923 "Unable to allocate sk_buff\n"); 924 break; 925 } 926 927 /* now fill corresponding descriptor entry */ 928 paddr = dma_map_single(&bp->pdev->dev, skb->data, 929 bp->rx_buffer_size, 930 DMA_FROM_DEVICE); 931 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 932 dev_kfree_skb(skb); 933 break; 934 } 935 936 queue->rx_skbuff[entry] = skb; 937 938 if (entry == bp->rx_ring_size - 1) 939 paddr |= MACB_BIT(RX_WRAP); 940 desc->ctrl = 0; 941 /* Setting addr clears RX_USED and allows reception, 942 * make sure ctrl is cleared first to avoid a race. 943 */ 944 dma_wmb(); 945 macb_set_addr(bp, desc, paddr); 946 947 /* properly align Ethernet header */ 948 skb_reserve(skb, NET_IP_ALIGN); 949 } else { 950 desc->ctrl = 0; 951 dma_wmb(); 952 desc->addr &= ~MACB_BIT(RX_USED); 953 } 954 } 955 956 /* Make descriptor updates visible to hardware */ 957 wmb(); 958 959 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 960 queue, queue->rx_prepared_head, queue->rx_tail); 961 } 962 963 /* Mark DMA descriptors from begin up to and not including end as unused */ 964 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 965 unsigned int end) 966 { 967 unsigned int frag; 968 969 for (frag = begin; frag != end; frag++) { 970 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 971 972 desc->addr &= ~MACB_BIT(RX_USED); 973 } 974 975 /* Make descriptor updates visible to hardware */ 976 wmb(); 977 978 /* When this happens, the hardware stats registers for 979 * whatever caused this is updated, so we don't have to record 980 * anything. 981 */ 982 } 983 984 static int gem_rx(struct macb_queue *queue, int budget) 985 { 986 struct macb *bp = queue->bp; 987 unsigned int len; 988 unsigned int entry; 989 struct sk_buff *skb; 990 struct macb_dma_desc *desc; 991 int count = 0; 992 993 while (count < budget) { 994 u32 ctrl; 995 dma_addr_t addr; 996 bool rxused; 997 998 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 999 desc = macb_rx_desc(queue, entry); 1000 1001 /* Make hw descriptor updates visible to CPU */ 1002 rmb(); 1003 1004 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1005 addr = macb_get_addr(bp, desc); 1006 1007 if (!rxused) 1008 break; 1009 1010 /* Ensure ctrl is at least as up-to-date as rxused */ 1011 dma_rmb(); 1012 1013 ctrl = desc->ctrl; 1014 1015 queue->rx_tail++; 1016 count++; 1017 1018 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1019 netdev_err(bp->dev, 1020 "not whole frame pointed by descriptor\n"); 1021 bp->dev->stats.rx_dropped++; 1022 queue->stats.rx_dropped++; 1023 break; 1024 } 1025 skb = queue->rx_skbuff[entry]; 1026 if (unlikely(!skb)) { 1027 netdev_err(bp->dev, 1028 "inconsistent Rx descriptor chain\n"); 1029 bp->dev->stats.rx_dropped++; 1030 queue->stats.rx_dropped++; 1031 break; 1032 } 1033 /* now everything is ready for receiving packet */ 1034 queue->rx_skbuff[entry] = NULL; 1035 len = ctrl & bp->rx_frm_len_mask; 1036 1037 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1038 1039 skb_put(skb, len); 1040 dma_unmap_single(&bp->pdev->dev, addr, 1041 bp->rx_buffer_size, DMA_FROM_DEVICE); 1042 1043 skb->protocol = eth_type_trans(skb, bp->dev); 1044 skb_checksum_none_assert(skb); 1045 if (bp->dev->features & NETIF_F_RXCSUM && 1046 !(bp->dev->flags & IFF_PROMISC) && 1047 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1048 skb->ip_summed = CHECKSUM_UNNECESSARY; 1049 1050 bp->dev->stats.rx_packets++; 1051 queue->stats.rx_packets++; 1052 bp->dev->stats.rx_bytes += skb->len; 1053 queue->stats.rx_bytes += skb->len; 1054 1055 gem_ptp_do_rxstamp(bp, skb, desc); 1056 1057 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1058 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1059 skb->len, skb->csum); 1060 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1061 skb_mac_header(skb), 16, true); 1062 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1063 skb->data, 32, true); 1064 #endif 1065 1066 netif_receive_skb(skb); 1067 } 1068 1069 gem_rx_refill(queue); 1070 1071 return count; 1072 } 1073 1074 static int macb_rx_frame(struct macb_queue *queue, unsigned int first_frag, 1075 unsigned int last_frag) 1076 { 1077 unsigned int len; 1078 unsigned int frag; 1079 unsigned int offset; 1080 struct sk_buff *skb; 1081 struct macb_dma_desc *desc; 1082 struct macb *bp = queue->bp; 1083 1084 desc = macb_rx_desc(queue, last_frag); 1085 len = desc->ctrl & bp->rx_frm_len_mask; 1086 1087 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1088 macb_rx_ring_wrap(bp, first_frag), 1089 macb_rx_ring_wrap(bp, last_frag), len); 1090 1091 /* The ethernet header starts NET_IP_ALIGN bytes into the 1092 * first buffer. Since the header is 14 bytes, this makes the 1093 * payload word-aligned. 1094 * 1095 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1096 * the two padding bytes into the skb so that we avoid hitting 1097 * the slowpath in memcpy(), and pull them off afterwards. 1098 */ 1099 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1100 if (!skb) { 1101 bp->dev->stats.rx_dropped++; 1102 for (frag = first_frag; ; frag++) { 1103 desc = macb_rx_desc(queue, frag); 1104 desc->addr &= ~MACB_BIT(RX_USED); 1105 if (frag == last_frag) 1106 break; 1107 } 1108 1109 /* Make descriptor updates visible to hardware */ 1110 wmb(); 1111 1112 return 1; 1113 } 1114 1115 offset = 0; 1116 len += NET_IP_ALIGN; 1117 skb_checksum_none_assert(skb); 1118 skb_put(skb, len); 1119 1120 for (frag = first_frag; ; frag++) { 1121 unsigned int frag_len = bp->rx_buffer_size; 1122 1123 if (offset + frag_len > len) { 1124 if (unlikely(frag != last_frag)) { 1125 dev_kfree_skb_any(skb); 1126 return -1; 1127 } 1128 frag_len = len - offset; 1129 } 1130 skb_copy_to_linear_data_offset(skb, offset, 1131 macb_rx_buffer(queue, frag), 1132 frag_len); 1133 offset += bp->rx_buffer_size; 1134 desc = macb_rx_desc(queue, frag); 1135 desc->addr &= ~MACB_BIT(RX_USED); 1136 1137 if (frag == last_frag) 1138 break; 1139 } 1140 1141 /* Make descriptor updates visible to hardware */ 1142 wmb(); 1143 1144 __skb_pull(skb, NET_IP_ALIGN); 1145 skb->protocol = eth_type_trans(skb, bp->dev); 1146 1147 bp->dev->stats.rx_packets++; 1148 bp->dev->stats.rx_bytes += skb->len; 1149 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1150 skb->len, skb->csum); 1151 netif_receive_skb(skb); 1152 1153 return 0; 1154 } 1155 1156 static inline void macb_init_rx_ring(struct macb_queue *queue) 1157 { 1158 struct macb *bp = queue->bp; 1159 dma_addr_t addr; 1160 struct macb_dma_desc *desc = NULL; 1161 int i; 1162 1163 addr = queue->rx_buffers_dma; 1164 for (i = 0; i < bp->rx_ring_size; i++) { 1165 desc = macb_rx_desc(queue, i); 1166 macb_set_addr(bp, desc, addr); 1167 desc->ctrl = 0; 1168 addr += bp->rx_buffer_size; 1169 } 1170 desc->addr |= MACB_BIT(RX_WRAP); 1171 queue->rx_tail = 0; 1172 } 1173 1174 static int macb_rx(struct macb_queue *queue, int budget) 1175 { 1176 struct macb *bp = queue->bp; 1177 bool reset_rx_queue = false; 1178 int received = 0; 1179 unsigned int tail; 1180 int first_frag = -1; 1181 1182 for (tail = queue->rx_tail; budget > 0; tail++) { 1183 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1184 u32 ctrl; 1185 1186 /* Make hw descriptor updates visible to CPU */ 1187 rmb(); 1188 1189 if (!(desc->addr & MACB_BIT(RX_USED))) 1190 break; 1191 1192 /* Ensure ctrl is at least as up-to-date as addr */ 1193 dma_rmb(); 1194 1195 ctrl = desc->ctrl; 1196 1197 if (ctrl & MACB_BIT(RX_SOF)) { 1198 if (first_frag != -1) 1199 discard_partial_frame(queue, first_frag, tail); 1200 first_frag = tail; 1201 } 1202 1203 if (ctrl & MACB_BIT(RX_EOF)) { 1204 int dropped; 1205 1206 if (unlikely(first_frag == -1)) { 1207 reset_rx_queue = true; 1208 continue; 1209 } 1210 1211 dropped = macb_rx_frame(queue, first_frag, tail); 1212 first_frag = -1; 1213 if (unlikely(dropped < 0)) { 1214 reset_rx_queue = true; 1215 continue; 1216 } 1217 if (!dropped) { 1218 received++; 1219 budget--; 1220 } 1221 } 1222 } 1223 1224 if (unlikely(reset_rx_queue)) { 1225 unsigned long flags; 1226 u32 ctrl; 1227 1228 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1229 1230 spin_lock_irqsave(&bp->lock, flags); 1231 1232 ctrl = macb_readl(bp, NCR); 1233 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1234 1235 macb_init_rx_ring(queue); 1236 queue_writel(queue, RBQP, queue->rx_ring_dma); 1237 1238 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1239 1240 spin_unlock_irqrestore(&bp->lock, flags); 1241 return received; 1242 } 1243 1244 if (first_frag != -1) 1245 queue->rx_tail = first_frag; 1246 else 1247 queue->rx_tail = tail; 1248 1249 return received; 1250 } 1251 1252 static int macb_poll(struct napi_struct *napi, int budget) 1253 { 1254 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1255 struct macb *bp = queue->bp; 1256 int work_done; 1257 u32 status; 1258 1259 status = macb_readl(bp, RSR); 1260 macb_writel(bp, RSR, status); 1261 1262 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1263 (unsigned long)status, budget); 1264 1265 work_done = bp->macbgem_ops.mog_rx(queue, budget); 1266 if (work_done < budget) { 1267 napi_complete_done(napi, work_done); 1268 1269 /* Packets received while interrupts were disabled */ 1270 status = macb_readl(bp, RSR); 1271 if (status) { 1272 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1273 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1274 napi_reschedule(napi); 1275 } else { 1276 queue_writel(queue, IER, bp->rx_intr_mask); 1277 } 1278 } 1279 1280 /* TODO: Handle errors */ 1281 1282 return work_done; 1283 } 1284 1285 static void macb_hresp_error_task(unsigned long data) 1286 { 1287 struct macb *bp = (struct macb *)data; 1288 struct net_device *dev = bp->dev; 1289 struct macb_queue *queue = bp->queues; 1290 unsigned int q; 1291 u32 ctrl; 1292 1293 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1294 queue_writel(queue, IDR, bp->rx_intr_mask | 1295 MACB_TX_INT_FLAGS | 1296 MACB_BIT(HRESP)); 1297 } 1298 ctrl = macb_readl(bp, NCR); 1299 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1300 macb_writel(bp, NCR, ctrl); 1301 1302 netif_tx_stop_all_queues(dev); 1303 netif_carrier_off(dev); 1304 1305 bp->macbgem_ops.mog_init_rings(bp); 1306 1307 /* Initialize TX and RX buffers */ 1308 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1309 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 1310 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1311 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1312 queue_writel(queue, RBQPH, 1313 upper_32_bits(queue->rx_ring_dma)); 1314 #endif 1315 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1316 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1317 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1318 queue_writel(queue, TBQPH, 1319 upper_32_bits(queue->tx_ring_dma)); 1320 #endif 1321 1322 /* Enable interrupts */ 1323 queue_writel(queue, IER, 1324 bp->rx_intr_mask | 1325 MACB_TX_INT_FLAGS | 1326 MACB_BIT(HRESP)); 1327 } 1328 1329 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1330 macb_writel(bp, NCR, ctrl); 1331 1332 netif_carrier_on(dev); 1333 netif_tx_start_all_queues(dev); 1334 } 1335 1336 static void macb_tx_restart(struct macb_queue *queue) 1337 { 1338 unsigned int head = queue->tx_head; 1339 unsigned int tail = queue->tx_tail; 1340 struct macb *bp = queue->bp; 1341 1342 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1343 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1344 1345 if (head == tail) 1346 return; 1347 1348 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1349 } 1350 1351 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1352 { 1353 struct macb_queue *queue = dev_id; 1354 struct macb *bp = queue->bp; 1355 struct net_device *dev = bp->dev; 1356 u32 status, ctrl; 1357 1358 status = queue_readl(queue, ISR); 1359 1360 if (unlikely(!status)) 1361 return IRQ_NONE; 1362 1363 spin_lock(&bp->lock); 1364 1365 while (status) { 1366 /* close possible race with dev_close */ 1367 if (unlikely(!netif_running(dev))) { 1368 queue_writel(queue, IDR, -1); 1369 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1370 queue_writel(queue, ISR, -1); 1371 break; 1372 } 1373 1374 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1375 (unsigned int)(queue - bp->queues), 1376 (unsigned long)status); 1377 1378 if (status & bp->rx_intr_mask) { 1379 /* There's no point taking any more interrupts 1380 * until we have processed the buffers. The 1381 * scheduling call may fail if the poll routine 1382 * is already scheduled, so disable interrupts 1383 * now. 1384 */ 1385 queue_writel(queue, IDR, bp->rx_intr_mask); 1386 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1387 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1388 1389 if (napi_schedule_prep(&queue->napi)) { 1390 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1391 __napi_schedule(&queue->napi); 1392 } 1393 } 1394 1395 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1396 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1397 schedule_work(&queue->tx_error_task); 1398 1399 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1400 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1401 1402 break; 1403 } 1404 1405 if (status & MACB_BIT(TCOMP)) 1406 macb_tx_interrupt(queue); 1407 1408 if (status & MACB_BIT(TXUBR)) 1409 macb_tx_restart(queue); 1410 1411 /* Link change detection isn't possible with RMII, so we'll 1412 * add that if/when we get our hands on a full-blown MII PHY. 1413 */ 1414 1415 /* There is a hardware issue under heavy load where DMA can 1416 * stop, this causes endless "used buffer descriptor read" 1417 * interrupts but it can be cleared by re-enabling RX. See 1418 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1419 * section 16.7.4 for details. RXUBR is only enabled for 1420 * these two versions. 1421 */ 1422 if (status & MACB_BIT(RXUBR)) { 1423 ctrl = macb_readl(bp, NCR); 1424 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1425 wmb(); 1426 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1427 1428 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1429 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1430 } 1431 1432 if (status & MACB_BIT(ISR_ROVR)) { 1433 /* We missed at least one packet */ 1434 if (macb_is_gem(bp)) 1435 bp->hw_stats.gem.rx_overruns++; 1436 else 1437 bp->hw_stats.macb.rx_overruns++; 1438 1439 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1440 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1441 } 1442 1443 if (status & MACB_BIT(HRESP)) { 1444 tasklet_schedule(&bp->hresp_err_tasklet); 1445 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1446 1447 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1448 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1449 } 1450 status = queue_readl(queue, ISR); 1451 } 1452 1453 spin_unlock(&bp->lock); 1454 1455 return IRQ_HANDLED; 1456 } 1457 1458 #ifdef CONFIG_NET_POLL_CONTROLLER 1459 /* Polling receive - used by netconsole and other diagnostic tools 1460 * to allow network i/o with interrupts disabled. 1461 */ 1462 static void macb_poll_controller(struct net_device *dev) 1463 { 1464 struct macb *bp = netdev_priv(dev); 1465 struct macb_queue *queue; 1466 unsigned long flags; 1467 unsigned int q; 1468 1469 local_irq_save(flags); 1470 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1471 macb_interrupt(dev->irq, queue); 1472 local_irq_restore(flags); 1473 } 1474 #endif 1475 1476 static unsigned int macb_tx_map(struct macb *bp, 1477 struct macb_queue *queue, 1478 struct sk_buff *skb, 1479 unsigned int hdrlen) 1480 { 1481 dma_addr_t mapping; 1482 unsigned int len, entry, i, tx_head = queue->tx_head; 1483 struct macb_tx_skb *tx_skb = NULL; 1484 struct macb_dma_desc *desc; 1485 unsigned int offset, size, count = 0; 1486 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1487 unsigned int eof = 1, mss_mfs = 0; 1488 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1489 1490 /* LSO */ 1491 if (skb_shinfo(skb)->gso_size != 0) { 1492 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1493 /* UDP - UFO */ 1494 lso_ctrl = MACB_LSO_UFO_ENABLE; 1495 else 1496 /* TCP - TSO */ 1497 lso_ctrl = MACB_LSO_TSO_ENABLE; 1498 } 1499 1500 /* First, map non-paged data */ 1501 len = skb_headlen(skb); 1502 1503 /* first buffer length */ 1504 size = hdrlen; 1505 1506 offset = 0; 1507 while (len) { 1508 entry = macb_tx_ring_wrap(bp, tx_head); 1509 tx_skb = &queue->tx_skb[entry]; 1510 1511 mapping = dma_map_single(&bp->pdev->dev, 1512 skb->data + offset, 1513 size, DMA_TO_DEVICE); 1514 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1515 goto dma_error; 1516 1517 /* Save info to properly release resources */ 1518 tx_skb->skb = NULL; 1519 tx_skb->mapping = mapping; 1520 tx_skb->size = size; 1521 tx_skb->mapped_as_page = false; 1522 1523 len -= size; 1524 offset += size; 1525 count++; 1526 tx_head++; 1527 1528 size = min(len, bp->max_tx_length); 1529 } 1530 1531 /* Then, map paged data from fragments */ 1532 for (f = 0; f < nr_frags; f++) { 1533 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1534 1535 len = skb_frag_size(frag); 1536 offset = 0; 1537 while (len) { 1538 size = min(len, bp->max_tx_length); 1539 entry = macb_tx_ring_wrap(bp, tx_head); 1540 tx_skb = &queue->tx_skb[entry]; 1541 1542 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1543 offset, size, DMA_TO_DEVICE); 1544 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1545 goto dma_error; 1546 1547 /* Save info to properly release resources */ 1548 tx_skb->skb = NULL; 1549 tx_skb->mapping = mapping; 1550 tx_skb->size = size; 1551 tx_skb->mapped_as_page = true; 1552 1553 len -= size; 1554 offset += size; 1555 count++; 1556 tx_head++; 1557 } 1558 } 1559 1560 /* Should never happen */ 1561 if (unlikely(!tx_skb)) { 1562 netdev_err(bp->dev, "BUG! empty skb!\n"); 1563 return 0; 1564 } 1565 1566 /* This is the last buffer of the frame: save socket buffer */ 1567 tx_skb->skb = skb; 1568 1569 /* Update TX ring: update buffer descriptors in reverse order 1570 * to avoid race condition 1571 */ 1572 1573 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1574 * to set the end of TX queue 1575 */ 1576 i = tx_head; 1577 entry = macb_tx_ring_wrap(bp, i); 1578 ctrl = MACB_BIT(TX_USED); 1579 desc = macb_tx_desc(queue, entry); 1580 desc->ctrl = ctrl; 1581 1582 if (lso_ctrl) { 1583 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1584 /* include header and FCS in value given to h/w */ 1585 mss_mfs = skb_shinfo(skb)->gso_size + 1586 skb_transport_offset(skb) + 1587 ETH_FCS_LEN; 1588 else /* TSO */ { 1589 mss_mfs = skb_shinfo(skb)->gso_size; 1590 /* TCP Sequence Number Source Select 1591 * can be set only for TSO 1592 */ 1593 seq_ctrl = 0; 1594 } 1595 } 1596 1597 do { 1598 i--; 1599 entry = macb_tx_ring_wrap(bp, i); 1600 tx_skb = &queue->tx_skb[entry]; 1601 desc = macb_tx_desc(queue, entry); 1602 1603 ctrl = (u32)tx_skb->size; 1604 if (eof) { 1605 ctrl |= MACB_BIT(TX_LAST); 1606 eof = 0; 1607 } 1608 if (unlikely(entry == (bp->tx_ring_size - 1))) 1609 ctrl |= MACB_BIT(TX_WRAP); 1610 1611 /* First descriptor is header descriptor */ 1612 if (i == queue->tx_head) { 1613 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1614 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1615 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1616 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1617 ctrl |= MACB_BIT(TX_NOCRC); 1618 } else 1619 /* Only set MSS/MFS on payload descriptors 1620 * (second or later descriptor) 1621 */ 1622 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1623 1624 /* Set TX buffer descriptor */ 1625 macb_set_addr(bp, desc, tx_skb->mapping); 1626 /* desc->addr must be visible to hardware before clearing 1627 * 'TX_USED' bit in desc->ctrl. 1628 */ 1629 wmb(); 1630 desc->ctrl = ctrl; 1631 } while (i != queue->tx_head); 1632 1633 queue->tx_head = tx_head; 1634 1635 return count; 1636 1637 dma_error: 1638 netdev_err(bp->dev, "TX DMA map failed\n"); 1639 1640 for (i = queue->tx_head; i != tx_head; i++) { 1641 tx_skb = macb_tx_skb(queue, i); 1642 1643 macb_tx_unmap(bp, tx_skb); 1644 } 1645 1646 return 0; 1647 } 1648 1649 static netdev_features_t macb_features_check(struct sk_buff *skb, 1650 struct net_device *dev, 1651 netdev_features_t features) 1652 { 1653 unsigned int nr_frags, f; 1654 unsigned int hdrlen; 1655 1656 /* Validate LSO compatibility */ 1657 1658 /* there is only one buffer */ 1659 if (!skb_is_nonlinear(skb)) 1660 return features; 1661 1662 /* length of header */ 1663 hdrlen = skb_transport_offset(skb); 1664 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1665 hdrlen += tcp_hdrlen(skb); 1666 1667 /* For LSO: 1668 * When software supplies two or more payload buffers all payload buffers 1669 * apart from the last must be a multiple of 8 bytes in size. 1670 */ 1671 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1672 return features & ~MACB_NETIF_LSO; 1673 1674 nr_frags = skb_shinfo(skb)->nr_frags; 1675 /* No need to check last fragment */ 1676 nr_frags--; 1677 for (f = 0; f < nr_frags; f++) { 1678 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1679 1680 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1681 return features & ~MACB_NETIF_LSO; 1682 } 1683 return features; 1684 } 1685 1686 static inline int macb_clear_csum(struct sk_buff *skb) 1687 { 1688 /* no change for packets without checksum offloading */ 1689 if (skb->ip_summed != CHECKSUM_PARTIAL) 1690 return 0; 1691 1692 /* make sure we can modify the header */ 1693 if (unlikely(skb_cow_head(skb, 0))) 1694 return -1; 1695 1696 /* initialize checksum field 1697 * This is required - at least for Zynq, which otherwise calculates 1698 * wrong UDP header checksums for UDP packets with UDP data len <=2 1699 */ 1700 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1701 return 0; 1702 } 1703 1704 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1705 { 1706 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1707 int padlen = ETH_ZLEN - (*skb)->len; 1708 int headroom = skb_headroom(*skb); 1709 int tailroom = skb_tailroom(*skb); 1710 struct sk_buff *nskb; 1711 u32 fcs; 1712 1713 if (!(ndev->features & NETIF_F_HW_CSUM) || 1714 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1715 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1716 return 0; 1717 1718 if (padlen <= 0) { 1719 /* FCS could be appeded to tailroom. */ 1720 if (tailroom >= ETH_FCS_LEN) 1721 goto add_fcs; 1722 /* FCS could be appeded by moving data to headroom. */ 1723 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1724 padlen = 0; 1725 /* No room for FCS, need to reallocate skb. */ 1726 else 1727 padlen = ETH_FCS_LEN; 1728 } else { 1729 /* Add room for FCS. */ 1730 padlen += ETH_FCS_LEN; 1731 } 1732 1733 if (!cloned && headroom + tailroom >= padlen) { 1734 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1735 skb_set_tail_pointer(*skb, (*skb)->len); 1736 } else { 1737 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1738 if (!nskb) 1739 return -ENOMEM; 1740 1741 dev_consume_skb_any(*skb); 1742 *skb = nskb; 1743 } 1744 1745 if (padlen > ETH_FCS_LEN) 1746 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1747 1748 add_fcs: 1749 /* set FCS to packet */ 1750 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1751 fcs = ~fcs; 1752 1753 skb_put_u8(*skb, fcs & 0xff); 1754 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1755 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1756 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1757 1758 return 0; 1759 } 1760 1761 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1762 { 1763 u16 queue_index = skb_get_queue_mapping(skb); 1764 struct macb *bp = netdev_priv(dev); 1765 struct macb_queue *queue = &bp->queues[queue_index]; 1766 unsigned long flags; 1767 unsigned int desc_cnt, nr_frags, frag_size, f; 1768 unsigned int hdrlen; 1769 bool is_lso, is_udp = 0; 1770 netdev_tx_t ret = NETDEV_TX_OK; 1771 1772 if (macb_clear_csum(skb)) { 1773 dev_kfree_skb_any(skb); 1774 return ret; 1775 } 1776 1777 if (macb_pad_and_fcs(&skb, dev)) { 1778 dev_kfree_skb_any(skb); 1779 return ret; 1780 } 1781 1782 is_lso = (skb_shinfo(skb)->gso_size != 0); 1783 1784 if (is_lso) { 1785 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1786 1787 /* length of headers */ 1788 if (is_udp) 1789 /* only queue eth + ip headers separately for UDP */ 1790 hdrlen = skb_transport_offset(skb); 1791 else 1792 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1793 if (skb_headlen(skb) < hdrlen) { 1794 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1795 /* if this is required, would need to copy to single buffer */ 1796 return NETDEV_TX_BUSY; 1797 } 1798 } else 1799 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1800 1801 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1802 netdev_vdbg(bp->dev, 1803 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1804 queue_index, skb->len, skb->head, skb->data, 1805 skb_tail_pointer(skb), skb_end_pointer(skb)); 1806 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1807 skb->data, 16, true); 1808 #endif 1809 1810 /* Count how many TX buffer descriptors are needed to send this 1811 * socket buffer: skb fragments of jumbo frames may need to be 1812 * split into many buffer descriptors. 1813 */ 1814 if (is_lso && (skb_headlen(skb) > hdrlen)) 1815 /* extra header descriptor if also payload in first buffer */ 1816 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1817 else 1818 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1819 nr_frags = skb_shinfo(skb)->nr_frags; 1820 for (f = 0; f < nr_frags; f++) { 1821 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1822 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1823 } 1824 1825 spin_lock_irqsave(&bp->lock, flags); 1826 1827 /* This is a hard error, log it. */ 1828 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1829 bp->tx_ring_size) < desc_cnt) { 1830 netif_stop_subqueue(dev, queue_index); 1831 spin_unlock_irqrestore(&bp->lock, flags); 1832 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1833 queue->tx_head, queue->tx_tail); 1834 return NETDEV_TX_BUSY; 1835 } 1836 1837 /* Map socket buffer for DMA transfer */ 1838 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1839 dev_kfree_skb_any(skb); 1840 goto unlock; 1841 } 1842 1843 /* Make newly initialized descriptor visible to hardware */ 1844 wmb(); 1845 skb_tx_timestamp(skb); 1846 1847 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1848 1849 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1850 netif_stop_subqueue(dev, queue_index); 1851 1852 unlock: 1853 spin_unlock_irqrestore(&bp->lock, flags); 1854 1855 return ret; 1856 } 1857 1858 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1859 { 1860 if (!macb_is_gem(bp)) { 1861 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1862 } else { 1863 bp->rx_buffer_size = size; 1864 1865 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1866 netdev_dbg(bp->dev, 1867 "RX buffer must be multiple of %d bytes, expanding\n", 1868 RX_BUFFER_MULTIPLE); 1869 bp->rx_buffer_size = 1870 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1871 } 1872 } 1873 1874 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1875 bp->dev->mtu, bp->rx_buffer_size); 1876 } 1877 1878 static void gem_free_rx_buffers(struct macb *bp) 1879 { 1880 struct sk_buff *skb; 1881 struct macb_dma_desc *desc; 1882 struct macb_queue *queue; 1883 dma_addr_t addr; 1884 unsigned int q; 1885 int i; 1886 1887 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1888 if (!queue->rx_skbuff) 1889 continue; 1890 1891 for (i = 0; i < bp->rx_ring_size; i++) { 1892 skb = queue->rx_skbuff[i]; 1893 1894 if (!skb) 1895 continue; 1896 1897 desc = macb_rx_desc(queue, i); 1898 addr = macb_get_addr(bp, desc); 1899 1900 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1901 DMA_FROM_DEVICE); 1902 dev_kfree_skb_any(skb); 1903 skb = NULL; 1904 } 1905 1906 kfree(queue->rx_skbuff); 1907 queue->rx_skbuff = NULL; 1908 } 1909 } 1910 1911 static void macb_free_rx_buffers(struct macb *bp) 1912 { 1913 struct macb_queue *queue = &bp->queues[0]; 1914 1915 if (queue->rx_buffers) { 1916 dma_free_coherent(&bp->pdev->dev, 1917 bp->rx_ring_size * bp->rx_buffer_size, 1918 queue->rx_buffers, queue->rx_buffers_dma); 1919 queue->rx_buffers = NULL; 1920 } 1921 } 1922 1923 static void macb_free_consistent(struct macb *bp) 1924 { 1925 struct macb_queue *queue; 1926 unsigned int q; 1927 int size; 1928 1929 bp->macbgem_ops.mog_free_rx_buffers(bp); 1930 1931 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1932 kfree(queue->tx_skb); 1933 queue->tx_skb = NULL; 1934 if (queue->tx_ring) { 1935 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1936 dma_free_coherent(&bp->pdev->dev, size, 1937 queue->tx_ring, queue->tx_ring_dma); 1938 queue->tx_ring = NULL; 1939 } 1940 if (queue->rx_ring) { 1941 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 1942 dma_free_coherent(&bp->pdev->dev, size, 1943 queue->rx_ring, queue->rx_ring_dma); 1944 queue->rx_ring = NULL; 1945 } 1946 } 1947 } 1948 1949 static int gem_alloc_rx_buffers(struct macb *bp) 1950 { 1951 struct macb_queue *queue; 1952 unsigned int q; 1953 int size; 1954 1955 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1956 size = bp->rx_ring_size * sizeof(struct sk_buff *); 1957 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 1958 if (!queue->rx_skbuff) 1959 return -ENOMEM; 1960 else 1961 netdev_dbg(bp->dev, 1962 "Allocated %d RX struct sk_buff entries at %p\n", 1963 bp->rx_ring_size, queue->rx_skbuff); 1964 } 1965 return 0; 1966 } 1967 1968 static int macb_alloc_rx_buffers(struct macb *bp) 1969 { 1970 struct macb_queue *queue = &bp->queues[0]; 1971 int size; 1972 1973 size = bp->rx_ring_size * bp->rx_buffer_size; 1974 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 1975 &queue->rx_buffers_dma, GFP_KERNEL); 1976 if (!queue->rx_buffers) 1977 return -ENOMEM; 1978 1979 netdev_dbg(bp->dev, 1980 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 1981 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 1982 return 0; 1983 } 1984 1985 static int macb_alloc_consistent(struct macb *bp) 1986 { 1987 struct macb_queue *queue; 1988 unsigned int q; 1989 int size; 1990 1991 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1992 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1993 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1994 &queue->tx_ring_dma, 1995 GFP_KERNEL); 1996 if (!queue->tx_ring) 1997 goto out_err; 1998 netdev_dbg(bp->dev, 1999 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2000 q, size, (unsigned long)queue->tx_ring_dma, 2001 queue->tx_ring); 2002 2003 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2004 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2005 if (!queue->tx_skb) 2006 goto out_err; 2007 2008 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2009 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2010 &queue->rx_ring_dma, GFP_KERNEL); 2011 if (!queue->rx_ring) 2012 goto out_err; 2013 netdev_dbg(bp->dev, 2014 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2015 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2016 } 2017 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2018 goto out_err; 2019 2020 return 0; 2021 2022 out_err: 2023 macb_free_consistent(bp); 2024 return -ENOMEM; 2025 } 2026 2027 static void gem_init_rings(struct macb *bp) 2028 { 2029 struct macb_queue *queue; 2030 struct macb_dma_desc *desc = NULL; 2031 unsigned int q; 2032 int i; 2033 2034 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2035 for (i = 0; i < bp->tx_ring_size; i++) { 2036 desc = macb_tx_desc(queue, i); 2037 macb_set_addr(bp, desc, 0); 2038 desc->ctrl = MACB_BIT(TX_USED); 2039 } 2040 desc->ctrl |= MACB_BIT(TX_WRAP); 2041 queue->tx_head = 0; 2042 queue->tx_tail = 0; 2043 2044 queue->rx_tail = 0; 2045 queue->rx_prepared_head = 0; 2046 2047 gem_rx_refill(queue); 2048 } 2049 2050 } 2051 2052 static void macb_init_rings(struct macb *bp) 2053 { 2054 int i; 2055 struct macb_dma_desc *desc = NULL; 2056 2057 macb_init_rx_ring(&bp->queues[0]); 2058 2059 for (i = 0; i < bp->tx_ring_size; i++) { 2060 desc = macb_tx_desc(&bp->queues[0], i); 2061 macb_set_addr(bp, desc, 0); 2062 desc->ctrl = MACB_BIT(TX_USED); 2063 } 2064 bp->queues[0].tx_head = 0; 2065 bp->queues[0].tx_tail = 0; 2066 desc->ctrl |= MACB_BIT(TX_WRAP); 2067 } 2068 2069 static void macb_reset_hw(struct macb *bp) 2070 { 2071 struct macb_queue *queue; 2072 unsigned int q; 2073 u32 ctrl = macb_readl(bp, NCR); 2074 2075 /* Disable RX and TX (XXX: Should we halt the transmission 2076 * more gracefully?) 2077 */ 2078 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2079 2080 /* Clear the stats registers (XXX: Update stats first?) */ 2081 ctrl |= MACB_BIT(CLRSTAT); 2082 2083 macb_writel(bp, NCR, ctrl); 2084 2085 /* Clear all status flags */ 2086 macb_writel(bp, TSR, -1); 2087 macb_writel(bp, RSR, -1); 2088 2089 /* Disable all interrupts */ 2090 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2091 queue_writel(queue, IDR, -1); 2092 queue_readl(queue, ISR); 2093 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2094 queue_writel(queue, ISR, -1); 2095 } 2096 } 2097 2098 static u32 gem_mdc_clk_div(struct macb *bp) 2099 { 2100 u32 config; 2101 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2102 2103 if (pclk_hz <= 20000000) 2104 config = GEM_BF(CLK, GEM_CLK_DIV8); 2105 else if (pclk_hz <= 40000000) 2106 config = GEM_BF(CLK, GEM_CLK_DIV16); 2107 else if (pclk_hz <= 80000000) 2108 config = GEM_BF(CLK, GEM_CLK_DIV32); 2109 else if (pclk_hz <= 120000000) 2110 config = GEM_BF(CLK, GEM_CLK_DIV48); 2111 else if (pclk_hz <= 160000000) 2112 config = GEM_BF(CLK, GEM_CLK_DIV64); 2113 else 2114 config = GEM_BF(CLK, GEM_CLK_DIV96); 2115 2116 return config; 2117 } 2118 2119 static u32 macb_mdc_clk_div(struct macb *bp) 2120 { 2121 u32 config; 2122 unsigned long pclk_hz; 2123 2124 if (macb_is_gem(bp)) 2125 return gem_mdc_clk_div(bp); 2126 2127 pclk_hz = clk_get_rate(bp->pclk); 2128 if (pclk_hz <= 20000000) 2129 config = MACB_BF(CLK, MACB_CLK_DIV8); 2130 else if (pclk_hz <= 40000000) 2131 config = MACB_BF(CLK, MACB_CLK_DIV16); 2132 else if (pclk_hz <= 80000000) 2133 config = MACB_BF(CLK, MACB_CLK_DIV32); 2134 else 2135 config = MACB_BF(CLK, MACB_CLK_DIV64); 2136 2137 return config; 2138 } 2139 2140 /* Get the DMA bus width field of the network configuration register that we 2141 * should program. We find the width from decoding the design configuration 2142 * register to find the maximum supported data bus width. 2143 */ 2144 static u32 macb_dbw(struct macb *bp) 2145 { 2146 if (!macb_is_gem(bp)) 2147 return 0; 2148 2149 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2150 case 4: 2151 return GEM_BF(DBW, GEM_DBW128); 2152 case 2: 2153 return GEM_BF(DBW, GEM_DBW64); 2154 case 1: 2155 default: 2156 return GEM_BF(DBW, GEM_DBW32); 2157 } 2158 } 2159 2160 /* Configure the receive DMA engine 2161 * - use the correct receive buffer size 2162 * - set best burst length for DMA operations 2163 * (if not supported by FIFO, it will fallback to default) 2164 * - set both rx/tx packet buffers to full memory size 2165 * These are configurable parameters for GEM. 2166 */ 2167 static void macb_configure_dma(struct macb *bp) 2168 { 2169 struct macb_queue *queue; 2170 u32 buffer_size; 2171 unsigned int q; 2172 u32 dmacfg; 2173 2174 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2175 if (macb_is_gem(bp)) { 2176 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2177 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2178 if (q) 2179 queue_writel(queue, RBQS, buffer_size); 2180 else 2181 dmacfg |= GEM_BF(RXBS, buffer_size); 2182 } 2183 if (bp->dma_burst_length) 2184 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2185 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2186 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2187 2188 if (bp->native_io) 2189 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2190 else 2191 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2192 2193 if (bp->dev->features & NETIF_F_HW_CSUM) 2194 dmacfg |= GEM_BIT(TXCOEN); 2195 else 2196 dmacfg &= ~GEM_BIT(TXCOEN); 2197 2198 dmacfg &= ~GEM_BIT(ADDR64); 2199 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2200 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2201 dmacfg |= GEM_BIT(ADDR64); 2202 #endif 2203 #ifdef CONFIG_MACB_USE_HWSTAMP 2204 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2205 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2206 #endif 2207 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2208 dmacfg); 2209 gem_writel(bp, DMACFG, dmacfg); 2210 } 2211 } 2212 2213 static void macb_init_hw(struct macb *bp) 2214 { 2215 struct macb_queue *queue; 2216 unsigned int q; 2217 2218 u32 config; 2219 2220 macb_reset_hw(bp); 2221 macb_set_hwaddr(bp); 2222 2223 config = macb_mdc_clk_div(bp); 2224 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2225 config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2226 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2227 config |= MACB_BIT(PAE); /* PAuse Enable */ 2228 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2229 if (bp->caps & MACB_CAPS_JUMBO) 2230 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2231 else 2232 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2233 if (bp->dev->flags & IFF_PROMISC) 2234 config |= MACB_BIT(CAF); /* Copy All Frames */ 2235 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2236 config |= GEM_BIT(RXCOEN); 2237 if (!(bp->dev->flags & IFF_BROADCAST)) 2238 config |= MACB_BIT(NBC); /* No BroadCast */ 2239 config |= macb_dbw(bp); 2240 macb_writel(bp, NCFGR, config); 2241 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2242 gem_writel(bp, JML, bp->jumbo_max_len); 2243 bp->speed = SPEED_10; 2244 bp->duplex = DUPLEX_HALF; 2245 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2246 if (bp->caps & MACB_CAPS_JUMBO) 2247 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2248 2249 macb_configure_dma(bp); 2250 2251 /* Initialize TX and RX buffers */ 2252 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2253 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 2254 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2255 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2256 queue_writel(queue, RBQPH, upper_32_bits(queue->rx_ring_dma)); 2257 #endif 2258 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 2259 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2260 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2261 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 2262 #endif 2263 2264 /* Enable interrupts */ 2265 queue_writel(queue, IER, 2266 bp->rx_intr_mask | 2267 MACB_TX_INT_FLAGS | 2268 MACB_BIT(HRESP)); 2269 } 2270 2271 /* Enable TX and RX */ 2272 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 2273 } 2274 2275 /* The hash address register is 64 bits long and takes up two 2276 * locations in the memory map. The least significant bits are stored 2277 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2278 * 2279 * The unicast hash enable and the multicast hash enable bits in the 2280 * network configuration register enable the reception of hash matched 2281 * frames. The destination address is reduced to a 6 bit index into 2282 * the 64 bit hash register using the following hash function. The 2283 * hash function is an exclusive or of every sixth bit of the 2284 * destination address. 2285 * 2286 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2287 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2288 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2289 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2290 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2291 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2292 * 2293 * da[0] represents the least significant bit of the first byte 2294 * received, that is, the multicast/unicast indicator, and da[47] 2295 * represents the most significant bit of the last byte received. If 2296 * the hash index, hi[n], points to a bit that is set in the hash 2297 * register then the frame will be matched according to whether the 2298 * frame is multicast or unicast. A multicast match will be signalled 2299 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2300 * index points to a bit set in the hash register. A unicast match 2301 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2302 * and the hash index points to a bit set in the hash register. To 2303 * receive all multicast frames, the hash register should be set with 2304 * all ones and the multicast hash enable bit should be set in the 2305 * network configuration register. 2306 */ 2307 2308 static inline int hash_bit_value(int bitnr, __u8 *addr) 2309 { 2310 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2311 return 1; 2312 return 0; 2313 } 2314 2315 /* Return the hash index value for the specified address. */ 2316 static int hash_get_index(__u8 *addr) 2317 { 2318 int i, j, bitval; 2319 int hash_index = 0; 2320 2321 for (j = 0; j < 6; j++) { 2322 for (i = 0, bitval = 0; i < 8; i++) 2323 bitval ^= hash_bit_value(i * 6 + j, addr); 2324 2325 hash_index |= (bitval << j); 2326 } 2327 2328 return hash_index; 2329 } 2330 2331 /* Add multicast addresses to the internal multicast-hash table. */ 2332 static void macb_sethashtable(struct net_device *dev) 2333 { 2334 struct netdev_hw_addr *ha; 2335 unsigned long mc_filter[2]; 2336 unsigned int bitnr; 2337 struct macb *bp = netdev_priv(dev); 2338 2339 mc_filter[0] = 0; 2340 mc_filter[1] = 0; 2341 2342 netdev_for_each_mc_addr(ha, dev) { 2343 bitnr = hash_get_index(ha->addr); 2344 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2345 } 2346 2347 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2348 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2349 } 2350 2351 /* Enable/Disable promiscuous and multicast modes. */ 2352 static void macb_set_rx_mode(struct net_device *dev) 2353 { 2354 unsigned long cfg; 2355 struct macb *bp = netdev_priv(dev); 2356 2357 cfg = macb_readl(bp, NCFGR); 2358 2359 if (dev->flags & IFF_PROMISC) { 2360 /* Enable promiscuous mode */ 2361 cfg |= MACB_BIT(CAF); 2362 2363 /* Disable RX checksum offload */ 2364 if (macb_is_gem(bp)) 2365 cfg &= ~GEM_BIT(RXCOEN); 2366 } else { 2367 /* Disable promiscuous mode */ 2368 cfg &= ~MACB_BIT(CAF); 2369 2370 /* Enable RX checksum offload only if requested */ 2371 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2372 cfg |= GEM_BIT(RXCOEN); 2373 } 2374 2375 if (dev->flags & IFF_ALLMULTI) { 2376 /* Enable all multicast mode */ 2377 macb_or_gem_writel(bp, HRB, -1); 2378 macb_or_gem_writel(bp, HRT, -1); 2379 cfg |= MACB_BIT(NCFGR_MTI); 2380 } else if (!netdev_mc_empty(dev)) { 2381 /* Enable specific multicasts */ 2382 macb_sethashtable(dev); 2383 cfg |= MACB_BIT(NCFGR_MTI); 2384 } else if (dev->flags & (~IFF_ALLMULTI)) { 2385 /* Disable all multicast mode */ 2386 macb_or_gem_writel(bp, HRB, 0); 2387 macb_or_gem_writel(bp, HRT, 0); 2388 cfg &= ~MACB_BIT(NCFGR_MTI); 2389 } 2390 2391 macb_writel(bp, NCFGR, cfg); 2392 } 2393 2394 static int macb_open(struct net_device *dev) 2395 { 2396 struct macb *bp = netdev_priv(dev); 2397 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2398 struct macb_queue *queue; 2399 unsigned int q; 2400 int err; 2401 2402 netdev_dbg(bp->dev, "open\n"); 2403 2404 err = pm_runtime_get_sync(&bp->pdev->dev); 2405 if (err < 0) 2406 goto pm_exit; 2407 2408 /* carrier starts down */ 2409 netif_carrier_off(dev); 2410 2411 /* if the phy is not yet register, retry later*/ 2412 if (!dev->phydev) { 2413 err = -EAGAIN; 2414 goto pm_exit; 2415 } 2416 2417 /* RX buffers initialization */ 2418 macb_init_rx_buffer_size(bp, bufsz); 2419 2420 err = macb_alloc_consistent(bp); 2421 if (err) { 2422 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2423 err); 2424 goto pm_exit; 2425 } 2426 2427 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2428 napi_enable(&queue->napi); 2429 2430 bp->macbgem_ops.mog_init_rings(bp); 2431 macb_init_hw(bp); 2432 2433 /* schedule a link state check */ 2434 phy_start(dev->phydev); 2435 2436 netif_tx_start_all_queues(dev); 2437 2438 if (bp->ptp_info) 2439 bp->ptp_info->ptp_init(dev); 2440 2441 pm_exit: 2442 if (err) { 2443 pm_runtime_put_sync(&bp->pdev->dev); 2444 return err; 2445 } 2446 return 0; 2447 } 2448 2449 static int macb_close(struct net_device *dev) 2450 { 2451 struct macb *bp = netdev_priv(dev); 2452 struct macb_queue *queue; 2453 unsigned long flags; 2454 unsigned int q; 2455 2456 netif_tx_stop_all_queues(dev); 2457 2458 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2459 napi_disable(&queue->napi); 2460 2461 if (dev->phydev) 2462 phy_stop(dev->phydev); 2463 2464 spin_lock_irqsave(&bp->lock, flags); 2465 macb_reset_hw(bp); 2466 netif_carrier_off(dev); 2467 spin_unlock_irqrestore(&bp->lock, flags); 2468 2469 macb_free_consistent(bp); 2470 2471 if (bp->ptp_info) 2472 bp->ptp_info->ptp_remove(dev); 2473 2474 pm_runtime_put(&bp->pdev->dev); 2475 2476 return 0; 2477 } 2478 2479 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2480 { 2481 if (netif_running(dev)) 2482 return -EBUSY; 2483 2484 dev->mtu = new_mtu; 2485 2486 return 0; 2487 } 2488 2489 static void gem_update_stats(struct macb *bp) 2490 { 2491 struct macb_queue *queue; 2492 unsigned int i, q, idx; 2493 unsigned long *stat; 2494 2495 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2496 2497 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2498 u32 offset = gem_statistics[i].offset; 2499 u64 val = bp->macb_reg_readl(bp, offset); 2500 2501 bp->ethtool_stats[i] += val; 2502 *p += val; 2503 2504 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2505 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2506 val = bp->macb_reg_readl(bp, offset + 4); 2507 bp->ethtool_stats[i] += ((u64)val) << 32; 2508 *(++p) += val; 2509 } 2510 } 2511 2512 idx = GEM_STATS_LEN; 2513 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2514 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2515 bp->ethtool_stats[idx++] = *stat; 2516 } 2517 2518 static struct net_device_stats *gem_get_stats(struct macb *bp) 2519 { 2520 struct gem_stats *hwstat = &bp->hw_stats.gem; 2521 struct net_device_stats *nstat = &bp->dev->stats; 2522 2523 gem_update_stats(bp); 2524 2525 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2526 hwstat->rx_alignment_errors + 2527 hwstat->rx_resource_errors + 2528 hwstat->rx_overruns + 2529 hwstat->rx_oversize_frames + 2530 hwstat->rx_jabbers + 2531 hwstat->rx_undersized_frames + 2532 hwstat->rx_length_field_frame_errors); 2533 nstat->tx_errors = (hwstat->tx_late_collisions + 2534 hwstat->tx_excessive_collisions + 2535 hwstat->tx_underrun + 2536 hwstat->tx_carrier_sense_errors); 2537 nstat->multicast = hwstat->rx_multicast_frames; 2538 nstat->collisions = (hwstat->tx_single_collision_frames + 2539 hwstat->tx_multiple_collision_frames + 2540 hwstat->tx_excessive_collisions); 2541 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2542 hwstat->rx_jabbers + 2543 hwstat->rx_undersized_frames + 2544 hwstat->rx_length_field_frame_errors); 2545 nstat->rx_over_errors = hwstat->rx_resource_errors; 2546 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2547 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2548 nstat->rx_fifo_errors = hwstat->rx_overruns; 2549 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2550 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2551 nstat->tx_fifo_errors = hwstat->tx_underrun; 2552 2553 return nstat; 2554 } 2555 2556 static void gem_get_ethtool_stats(struct net_device *dev, 2557 struct ethtool_stats *stats, u64 *data) 2558 { 2559 struct macb *bp; 2560 2561 bp = netdev_priv(dev); 2562 gem_update_stats(bp); 2563 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2564 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2565 } 2566 2567 static int gem_get_sset_count(struct net_device *dev, int sset) 2568 { 2569 struct macb *bp = netdev_priv(dev); 2570 2571 switch (sset) { 2572 case ETH_SS_STATS: 2573 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2574 default: 2575 return -EOPNOTSUPP; 2576 } 2577 } 2578 2579 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2580 { 2581 char stat_string[ETH_GSTRING_LEN]; 2582 struct macb *bp = netdev_priv(dev); 2583 struct macb_queue *queue; 2584 unsigned int i; 2585 unsigned int q; 2586 2587 switch (sset) { 2588 case ETH_SS_STATS: 2589 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2590 memcpy(p, gem_statistics[i].stat_string, 2591 ETH_GSTRING_LEN); 2592 2593 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2594 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2595 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2596 q, queue_statistics[i].stat_string); 2597 memcpy(p, stat_string, ETH_GSTRING_LEN); 2598 } 2599 } 2600 break; 2601 } 2602 } 2603 2604 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2605 { 2606 struct macb *bp = netdev_priv(dev); 2607 struct net_device_stats *nstat = &bp->dev->stats; 2608 struct macb_stats *hwstat = &bp->hw_stats.macb; 2609 2610 if (macb_is_gem(bp)) 2611 return gem_get_stats(bp); 2612 2613 /* read stats from hardware */ 2614 macb_update_stats(bp); 2615 2616 /* Convert HW stats into netdevice stats */ 2617 nstat->rx_errors = (hwstat->rx_fcs_errors + 2618 hwstat->rx_align_errors + 2619 hwstat->rx_resource_errors + 2620 hwstat->rx_overruns + 2621 hwstat->rx_oversize_pkts + 2622 hwstat->rx_jabbers + 2623 hwstat->rx_undersize_pkts + 2624 hwstat->rx_length_mismatch); 2625 nstat->tx_errors = (hwstat->tx_late_cols + 2626 hwstat->tx_excessive_cols + 2627 hwstat->tx_underruns + 2628 hwstat->tx_carrier_errors + 2629 hwstat->sqe_test_errors); 2630 nstat->collisions = (hwstat->tx_single_cols + 2631 hwstat->tx_multiple_cols + 2632 hwstat->tx_excessive_cols); 2633 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2634 hwstat->rx_jabbers + 2635 hwstat->rx_undersize_pkts + 2636 hwstat->rx_length_mismatch); 2637 nstat->rx_over_errors = hwstat->rx_resource_errors + 2638 hwstat->rx_overruns; 2639 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2640 nstat->rx_frame_errors = hwstat->rx_align_errors; 2641 nstat->rx_fifo_errors = hwstat->rx_overruns; 2642 /* XXX: What does "missed" mean? */ 2643 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2644 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2645 nstat->tx_fifo_errors = hwstat->tx_underruns; 2646 /* Don't know about heartbeat or window errors... */ 2647 2648 return nstat; 2649 } 2650 2651 static int macb_get_regs_len(struct net_device *netdev) 2652 { 2653 return MACB_GREGS_NBR * sizeof(u32); 2654 } 2655 2656 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2657 void *p) 2658 { 2659 struct macb *bp = netdev_priv(dev); 2660 unsigned int tail, head; 2661 u32 *regs_buff = p; 2662 2663 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2664 | MACB_GREGS_VERSION; 2665 2666 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2667 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2668 2669 regs_buff[0] = macb_readl(bp, NCR); 2670 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2671 regs_buff[2] = macb_readl(bp, NSR); 2672 regs_buff[3] = macb_readl(bp, TSR); 2673 regs_buff[4] = macb_readl(bp, RBQP); 2674 regs_buff[5] = macb_readl(bp, TBQP); 2675 regs_buff[6] = macb_readl(bp, RSR); 2676 regs_buff[7] = macb_readl(bp, IMR); 2677 2678 regs_buff[8] = tail; 2679 regs_buff[9] = head; 2680 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2681 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2682 2683 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2684 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2685 if (macb_is_gem(bp)) 2686 regs_buff[13] = gem_readl(bp, DMACFG); 2687 } 2688 2689 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2690 { 2691 struct macb *bp = netdev_priv(netdev); 2692 2693 wol->supported = 0; 2694 wol->wolopts = 0; 2695 2696 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2697 wol->supported = WAKE_MAGIC; 2698 2699 if (bp->wol & MACB_WOL_ENABLED) 2700 wol->wolopts |= WAKE_MAGIC; 2701 } 2702 } 2703 2704 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2705 { 2706 struct macb *bp = netdev_priv(netdev); 2707 2708 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2709 (wol->wolopts & ~WAKE_MAGIC)) 2710 return -EOPNOTSUPP; 2711 2712 if (wol->wolopts & WAKE_MAGIC) 2713 bp->wol |= MACB_WOL_ENABLED; 2714 else 2715 bp->wol &= ~MACB_WOL_ENABLED; 2716 2717 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2718 2719 return 0; 2720 } 2721 2722 static void macb_get_ringparam(struct net_device *netdev, 2723 struct ethtool_ringparam *ring) 2724 { 2725 struct macb *bp = netdev_priv(netdev); 2726 2727 ring->rx_max_pending = MAX_RX_RING_SIZE; 2728 ring->tx_max_pending = MAX_TX_RING_SIZE; 2729 2730 ring->rx_pending = bp->rx_ring_size; 2731 ring->tx_pending = bp->tx_ring_size; 2732 } 2733 2734 static int macb_set_ringparam(struct net_device *netdev, 2735 struct ethtool_ringparam *ring) 2736 { 2737 struct macb *bp = netdev_priv(netdev); 2738 u32 new_rx_size, new_tx_size; 2739 unsigned int reset = 0; 2740 2741 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2742 return -EINVAL; 2743 2744 new_rx_size = clamp_t(u32, ring->rx_pending, 2745 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2746 new_rx_size = roundup_pow_of_two(new_rx_size); 2747 2748 new_tx_size = clamp_t(u32, ring->tx_pending, 2749 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2750 new_tx_size = roundup_pow_of_two(new_tx_size); 2751 2752 if ((new_tx_size == bp->tx_ring_size) && 2753 (new_rx_size == bp->rx_ring_size)) { 2754 /* nothing to do */ 2755 return 0; 2756 } 2757 2758 if (netif_running(bp->dev)) { 2759 reset = 1; 2760 macb_close(bp->dev); 2761 } 2762 2763 bp->rx_ring_size = new_rx_size; 2764 bp->tx_ring_size = new_tx_size; 2765 2766 if (reset) 2767 macb_open(bp->dev); 2768 2769 return 0; 2770 } 2771 2772 #ifdef CONFIG_MACB_USE_HWSTAMP 2773 static unsigned int gem_get_tsu_rate(struct macb *bp) 2774 { 2775 struct clk *tsu_clk; 2776 unsigned int tsu_rate; 2777 2778 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2779 if (!IS_ERR(tsu_clk)) 2780 tsu_rate = clk_get_rate(tsu_clk); 2781 /* try pclk instead */ 2782 else if (!IS_ERR(bp->pclk)) { 2783 tsu_clk = bp->pclk; 2784 tsu_rate = clk_get_rate(tsu_clk); 2785 } else 2786 return -ENOTSUPP; 2787 return tsu_rate; 2788 } 2789 2790 static s32 gem_get_ptp_max_adj(void) 2791 { 2792 return 64000000; 2793 } 2794 2795 static int gem_get_ts_info(struct net_device *dev, 2796 struct ethtool_ts_info *info) 2797 { 2798 struct macb *bp = netdev_priv(dev); 2799 2800 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2801 ethtool_op_get_ts_info(dev, info); 2802 return 0; 2803 } 2804 2805 info->so_timestamping = 2806 SOF_TIMESTAMPING_TX_SOFTWARE | 2807 SOF_TIMESTAMPING_RX_SOFTWARE | 2808 SOF_TIMESTAMPING_SOFTWARE | 2809 SOF_TIMESTAMPING_TX_HARDWARE | 2810 SOF_TIMESTAMPING_RX_HARDWARE | 2811 SOF_TIMESTAMPING_RAW_HARDWARE; 2812 info->tx_types = 2813 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2814 (1 << HWTSTAMP_TX_OFF) | 2815 (1 << HWTSTAMP_TX_ON); 2816 info->rx_filters = 2817 (1 << HWTSTAMP_FILTER_NONE) | 2818 (1 << HWTSTAMP_FILTER_ALL); 2819 2820 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2821 2822 return 0; 2823 } 2824 2825 static struct macb_ptp_info gem_ptp_info = { 2826 .ptp_init = gem_ptp_init, 2827 .ptp_remove = gem_ptp_remove, 2828 .get_ptp_max_adj = gem_get_ptp_max_adj, 2829 .get_tsu_rate = gem_get_tsu_rate, 2830 .get_ts_info = gem_get_ts_info, 2831 .get_hwtst = gem_get_hwtst, 2832 .set_hwtst = gem_set_hwtst, 2833 }; 2834 #endif 2835 2836 static int macb_get_ts_info(struct net_device *netdev, 2837 struct ethtool_ts_info *info) 2838 { 2839 struct macb *bp = netdev_priv(netdev); 2840 2841 if (bp->ptp_info) 2842 return bp->ptp_info->get_ts_info(netdev, info); 2843 2844 return ethtool_op_get_ts_info(netdev, info); 2845 } 2846 2847 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2848 { 2849 struct net_device *netdev = bp->dev; 2850 struct ethtool_rx_fs_item *item; 2851 u32 t2_scr; 2852 int num_t2_scr; 2853 2854 if (!(netdev->features & NETIF_F_NTUPLE)) 2855 return; 2856 2857 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2858 2859 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2860 struct ethtool_rx_flow_spec *fs = &item->fs; 2861 struct ethtool_tcpip4_spec *tp4sp_m; 2862 2863 if (fs->location >= num_t2_scr) 2864 continue; 2865 2866 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2867 2868 /* enable/disable screener regs for the flow entry */ 2869 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2870 2871 /* only enable fields with no masking */ 2872 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2873 2874 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2875 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2876 else 2877 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2878 2879 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2880 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2881 else 2882 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2883 2884 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2885 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2886 else 2887 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 2888 2889 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 2890 } 2891 } 2892 2893 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 2894 { 2895 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 2896 uint16_t index = fs->location; 2897 u32 w0, w1, t2_scr; 2898 bool cmp_a = false; 2899 bool cmp_b = false; 2900 bool cmp_c = false; 2901 2902 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 2903 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2904 2905 /* ignore field if any masking set */ 2906 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 2907 /* 1st compare reg - IP source address */ 2908 w0 = 0; 2909 w1 = 0; 2910 w0 = tp4sp_v->ip4src; 2911 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2912 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2913 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 2914 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 2915 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 2916 cmp_a = true; 2917 } 2918 2919 /* ignore field if any masking set */ 2920 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 2921 /* 2nd compare reg - IP destination address */ 2922 w0 = 0; 2923 w1 = 0; 2924 w0 = tp4sp_v->ip4dst; 2925 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2926 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2927 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 2928 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 2929 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 2930 cmp_b = true; 2931 } 2932 2933 /* ignore both port fields if masking set in both */ 2934 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 2935 /* 3rd compare reg - source port, destination port */ 2936 w0 = 0; 2937 w1 = 0; 2938 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 2939 if (tp4sp_m->psrc == tp4sp_m->pdst) { 2940 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 2941 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2942 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2943 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2944 } else { 2945 /* only one port definition */ 2946 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 2947 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 2948 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 2949 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 2950 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2951 } else { /* dst port */ 2952 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2953 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 2954 } 2955 } 2956 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 2957 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 2958 cmp_c = true; 2959 } 2960 2961 t2_scr = 0; 2962 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 2963 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 2964 if (cmp_a) 2965 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 2966 if (cmp_b) 2967 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 2968 if (cmp_c) 2969 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 2970 gem_writel_n(bp, SCRT2, index, t2_scr); 2971 } 2972 2973 static int gem_add_flow_filter(struct net_device *netdev, 2974 struct ethtool_rxnfc *cmd) 2975 { 2976 struct macb *bp = netdev_priv(netdev); 2977 struct ethtool_rx_flow_spec *fs = &cmd->fs; 2978 struct ethtool_rx_fs_item *item, *newfs; 2979 unsigned long flags; 2980 int ret = -EINVAL; 2981 bool added = false; 2982 2983 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 2984 if (newfs == NULL) 2985 return -ENOMEM; 2986 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 2987 2988 netdev_dbg(netdev, 2989 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2990 fs->flow_type, (int)fs->ring_cookie, fs->location, 2991 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2992 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2993 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 2994 2995 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2996 2997 /* find correct place to add in list */ 2998 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2999 if (item->fs.location > newfs->fs.location) { 3000 list_add_tail(&newfs->list, &item->list); 3001 added = true; 3002 break; 3003 } else if (item->fs.location == fs->location) { 3004 netdev_err(netdev, "Rule not added: location %d not free!\n", 3005 fs->location); 3006 ret = -EBUSY; 3007 goto err; 3008 } 3009 } 3010 if (!added) 3011 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3012 3013 gem_prog_cmp_regs(bp, fs); 3014 bp->rx_fs_list.count++; 3015 /* enable filtering if NTUPLE on */ 3016 gem_enable_flow_filters(bp, 1); 3017 3018 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3019 return 0; 3020 3021 err: 3022 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3023 kfree(newfs); 3024 return ret; 3025 } 3026 3027 static int gem_del_flow_filter(struct net_device *netdev, 3028 struct ethtool_rxnfc *cmd) 3029 { 3030 struct macb *bp = netdev_priv(netdev); 3031 struct ethtool_rx_fs_item *item; 3032 struct ethtool_rx_flow_spec *fs; 3033 unsigned long flags; 3034 3035 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3036 3037 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3038 if (item->fs.location == cmd->fs.location) { 3039 /* disable screener regs for the flow entry */ 3040 fs = &(item->fs); 3041 netdev_dbg(netdev, 3042 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3043 fs->flow_type, (int)fs->ring_cookie, fs->location, 3044 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3045 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3046 htons(fs->h_u.tcp_ip4_spec.psrc), 3047 htons(fs->h_u.tcp_ip4_spec.pdst)); 3048 3049 gem_writel_n(bp, SCRT2, fs->location, 0); 3050 3051 list_del(&item->list); 3052 bp->rx_fs_list.count--; 3053 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3054 kfree(item); 3055 return 0; 3056 } 3057 } 3058 3059 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3060 return -EINVAL; 3061 } 3062 3063 static int gem_get_flow_entry(struct net_device *netdev, 3064 struct ethtool_rxnfc *cmd) 3065 { 3066 struct macb *bp = netdev_priv(netdev); 3067 struct ethtool_rx_fs_item *item; 3068 3069 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3070 if (item->fs.location == cmd->fs.location) { 3071 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3072 return 0; 3073 } 3074 } 3075 return -EINVAL; 3076 } 3077 3078 static int gem_get_all_flow_entries(struct net_device *netdev, 3079 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3080 { 3081 struct macb *bp = netdev_priv(netdev); 3082 struct ethtool_rx_fs_item *item; 3083 uint32_t cnt = 0; 3084 3085 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3086 if (cnt == cmd->rule_cnt) 3087 return -EMSGSIZE; 3088 rule_locs[cnt] = item->fs.location; 3089 cnt++; 3090 } 3091 cmd->data = bp->max_tuples; 3092 cmd->rule_cnt = cnt; 3093 3094 return 0; 3095 } 3096 3097 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3098 u32 *rule_locs) 3099 { 3100 struct macb *bp = netdev_priv(netdev); 3101 int ret = 0; 3102 3103 switch (cmd->cmd) { 3104 case ETHTOOL_GRXRINGS: 3105 cmd->data = bp->num_queues; 3106 break; 3107 case ETHTOOL_GRXCLSRLCNT: 3108 cmd->rule_cnt = bp->rx_fs_list.count; 3109 break; 3110 case ETHTOOL_GRXCLSRULE: 3111 ret = gem_get_flow_entry(netdev, cmd); 3112 break; 3113 case ETHTOOL_GRXCLSRLALL: 3114 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3115 break; 3116 default: 3117 netdev_err(netdev, 3118 "Command parameter %d is not supported\n", cmd->cmd); 3119 ret = -EOPNOTSUPP; 3120 } 3121 3122 return ret; 3123 } 3124 3125 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3126 { 3127 struct macb *bp = netdev_priv(netdev); 3128 int ret; 3129 3130 switch (cmd->cmd) { 3131 case ETHTOOL_SRXCLSRLINS: 3132 if ((cmd->fs.location >= bp->max_tuples) 3133 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3134 ret = -EINVAL; 3135 break; 3136 } 3137 ret = gem_add_flow_filter(netdev, cmd); 3138 break; 3139 case ETHTOOL_SRXCLSRLDEL: 3140 ret = gem_del_flow_filter(netdev, cmd); 3141 break; 3142 default: 3143 netdev_err(netdev, 3144 "Command parameter %d is not supported\n", cmd->cmd); 3145 ret = -EOPNOTSUPP; 3146 } 3147 3148 return ret; 3149 } 3150 3151 static const struct ethtool_ops macb_ethtool_ops = { 3152 .get_regs_len = macb_get_regs_len, 3153 .get_regs = macb_get_regs, 3154 .get_link = ethtool_op_get_link, 3155 .get_ts_info = ethtool_op_get_ts_info, 3156 .get_wol = macb_get_wol, 3157 .set_wol = macb_set_wol, 3158 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3159 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3160 .get_ringparam = macb_get_ringparam, 3161 .set_ringparam = macb_set_ringparam, 3162 }; 3163 3164 static const struct ethtool_ops gem_ethtool_ops = { 3165 .get_regs_len = macb_get_regs_len, 3166 .get_regs = macb_get_regs, 3167 .get_link = ethtool_op_get_link, 3168 .get_ts_info = macb_get_ts_info, 3169 .get_ethtool_stats = gem_get_ethtool_stats, 3170 .get_strings = gem_get_ethtool_strings, 3171 .get_sset_count = gem_get_sset_count, 3172 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3173 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3174 .get_ringparam = macb_get_ringparam, 3175 .set_ringparam = macb_set_ringparam, 3176 .get_rxnfc = gem_get_rxnfc, 3177 .set_rxnfc = gem_set_rxnfc, 3178 }; 3179 3180 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3181 { 3182 struct phy_device *phydev = dev->phydev; 3183 struct macb *bp = netdev_priv(dev); 3184 3185 if (!netif_running(dev)) 3186 return -EINVAL; 3187 3188 if (!phydev) 3189 return -ENODEV; 3190 3191 if (!bp->ptp_info) 3192 return phy_mii_ioctl(phydev, rq, cmd); 3193 3194 switch (cmd) { 3195 case SIOCSHWTSTAMP: 3196 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3197 case SIOCGHWTSTAMP: 3198 return bp->ptp_info->get_hwtst(dev, rq); 3199 default: 3200 return phy_mii_ioctl(phydev, rq, cmd); 3201 } 3202 } 3203 3204 static inline void macb_set_txcsum_feature(struct macb *bp, 3205 netdev_features_t features) 3206 { 3207 u32 val; 3208 3209 if (!macb_is_gem(bp)) 3210 return; 3211 3212 val = gem_readl(bp, DMACFG); 3213 if (features & NETIF_F_HW_CSUM) 3214 val |= GEM_BIT(TXCOEN); 3215 else 3216 val &= ~GEM_BIT(TXCOEN); 3217 3218 gem_writel(bp, DMACFG, val); 3219 } 3220 3221 static inline void macb_set_rxcsum_feature(struct macb *bp, 3222 netdev_features_t features) 3223 { 3224 struct net_device *netdev = bp->dev; 3225 u32 val; 3226 3227 if (!macb_is_gem(bp)) 3228 return; 3229 3230 val = gem_readl(bp, NCFGR); 3231 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 3232 val |= GEM_BIT(RXCOEN); 3233 else 3234 val &= ~GEM_BIT(RXCOEN); 3235 3236 gem_writel(bp, NCFGR, val); 3237 } 3238 3239 static inline void macb_set_rxflow_feature(struct macb *bp, 3240 netdev_features_t features) 3241 { 3242 if (!macb_is_gem(bp)) 3243 return; 3244 3245 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 3246 } 3247 3248 static int macb_set_features(struct net_device *netdev, 3249 netdev_features_t features) 3250 { 3251 struct macb *bp = netdev_priv(netdev); 3252 netdev_features_t changed = features ^ netdev->features; 3253 3254 /* TX checksum offload */ 3255 if (changed & NETIF_F_HW_CSUM) 3256 macb_set_txcsum_feature(bp, features); 3257 3258 /* RX checksum offload */ 3259 if (changed & NETIF_F_RXCSUM) 3260 macb_set_rxcsum_feature(bp, features); 3261 3262 /* RX Flow Filters */ 3263 if (changed & NETIF_F_NTUPLE) 3264 macb_set_rxflow_feature(bp, features); 3265 3266 return 0; 3267 } 3268 3269 static void macb_restore_features(struct macb *bp) 3270 { 3271 struct net_device *netdev = bp->dev; 3272 netdev_features_t features = netdev->features; 3273 3274 /* TX checksum offload */ 3275 macb_set_txcsum_feature(bp, features); 3276 3277 /* RX checksum offload */ 3278 macb_set_rxcsum_feature(bp, features); 3279 3280 /* RX Flow Filters */ 3281 macb_set_rxflow_feature(bp, features); 3282 } 3283 3284 static const struct net_device_ops macb_netdev_ops = { 3285 .ndo_open = macb_open, 3286 .ndo_stop = macb_close, 3287 .ndo_start_xmit = macb_start_xmit, 3288 .ndo_set_rx_mode = macb_set_rx_mode, 3289 .ndo_get_stats = macb_get_stats, 3290 .ndo_do_ioctl = macb_ioctl, 3291 .ndo_validate_addr = eth_validate_addr, 3292 .ndo_change_mtu = macb_change_mtu, 3293 .ndo_set_mac_address = eth_mac_addr, 3294 #ifdef CONFIG_NET_POLL_CONTROLLER 3295 .ndo_poll_controller = macb_poll_controller, 3296 #endif 3297 .ndo_set_features = macb_set_features, 3298 .ndo_features_check = macb_features_check, 3299 }; 3300 3301 /* Configure peripheral capabilities according to device tree 3302 * and integration options used 3303 */ 3304 static void macb_configure_caps(struct macb *bp, 3305 const struct macb_config *dt_conf) 3306 { 3307 u32 dcfg; 3308 3309 if (dt_conf) 3310 bp->caps = dt_conf->caps; 3311 3312 if (hw_is_gem(bp->regs, bp->native_io)) { 3313 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3314 3315 dcfg = gem_readl(bp, DCFG1); 3316 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3317 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3318 dcfg = gem_readl(bp, DCFG2); 3319 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3320 bp->caps |= MACB_CAPS_FIFO_MODE; 3321 #ifdef CONFIG_MACB_USE_HWSTAMP 3322 if (gem_has_ptp(bp)) { 3323 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3324 pr_err("GEM doesn't support hardware ptp.\n"); 3325 else { 3326 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3327 bp->ptp_info = &gem_ptp_info; 3328 } 3329 } 3330 #endif 3331 } 3332 3333 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3334 } 3335 3336 static void macb_probe_queues(void __iomem *mem, 3337 bool native_io, 3338 unsigned int *queue_mask, 3339 unsigned int *num_queues) 3340 { 3341 unsigned int hw_q; 3342 3343 *queue_mask = 0x1; 3344 *num_queues = 1; 3345 3346 /* is it macb or gem ? 3347 * 3348 * We need to read directly from the hardware here because 3349 * we are early in the probe process and don't have the 3350 * MACB_CAPS_MACB_IS_GEM flag positioned 3351 */ 3352 if (!hw_is_gem(mem, native_io)) 3353 return; 3354 3355 /* bit 0 is never set but queue 0 always exists */ 3356 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3357 3358 *queue_mask |= 0x1; 3359 3360 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3361 if (*queue_mask & (1 << hw_q)) 3362 (*num_queues)++; 3363 } 3364 3365 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3366 struct clk **hclk, struct clk **tx_clk, 3367 struct clk **rx_clk, struct clk **tsu_clk) 3368 { 3369 struct macb_platform_data *pdata; 3370 int err; 3371 3372 pdata = dev_get_platdata(&pdev->dev); 3373 if (pdata) { 3374 *pclk = pdata->pclk; 3375 *hclk = pdata->hclk; 3376 } else { 3377 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3378 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3379 } 3380 3381 if (IS_ERR_OR_NULL(*pclk)) { 3382 err = PTR_ERR(*pclk); 3383 if (!err) 3384 err = -ENODEV; 3385 3386 dev_err(&pdev->dev, "failed to get macb_clk (%d)\n", err); 3387 return err; 3388 } 3389 3390 if (IS_ERR_OR_NULL(*hclk)) { 3391 err = PTR_ERR(*hclk); 3392 if (!err) 3393 err = -ENODEV; 3394 3395 dev_err(&pdev->dev, "failed to get hclk (%d)\n", err); 3396 return err; 3397 } 3398 3399 *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); 3400 if (IS_ERR(*tx_clk)) 3401 *tx_clk = NULL; 3402 3403 *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); 3404 if (IS_ERR(*rx_clk)) 3405 *rx_clk = NULL; 3406 3407 *tsu_clk = devm_clk_get(&pdev->dev, "tsu_clk"); 3408 if (IS_ERR(*tsu_clk)) 3409 *tsu_clk = NULL; 3410 3411 err = clk_prepare_enable(*pclk); 3412 if (err) { 3413 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3414 return err; 3415 } 3416 3417 err = clk_prepare_enable(*hclk); 3418 if (err) { 3419 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 3420 goto err_disable_pclk; 3421 } 3422 3423 err = clk_prepare_enable(*tx_clk); 3424 if (err) { 3425 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 3426 goto err_disable_hclk; 3427 } 3428 3429 err = clk_prepare_enable(*rx_clk); 3430 if (err) { 3431 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 3432 goto err_disable_txclk; 3433 } 3434 3435 err = clk_prepare_enable(*tsu_clk); 3436 if (err) { 3437 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 3438 goto err_disable_rxclk; 3439 } 3440 3441 return 0; 3442 3443 err_disable_rxclk: 3444 clk_disable_unprepare(*rx_clk); 3445 3446 err_disable_txclk: 3447 clk_disable_unprepare(*tx_clk); 3448 3449 err_disable_hclk: 3450 clk_disable_unprepare(*hclk); 3451 3452 err_disable_pclk: 3453 clk_disable_unprepare(*pclk); 3454 3455 return err; 3456 } 3457 3458 static int macb_init(struct platform_device *pdev) 3459 { 3460 struct net_device *dev = platform_get_drvdata(pdev); 3461 unsigned int hw_q, q; 3462 struct macb *bp = netdev_priv(dev); 3463 struct macb_queue *queue; 3464 int err; 3465 u32 val, reg; 3466 3467 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3468 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3469 3470 /* set the queue register mapping once for all: queue0 has a special 3471 * register mapping but we don't want to test the queue index then 3472 * compute the corresponding register offset at run time. 3473 */ 3474 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3475 if (!(bp->queue_mask & (1 << hw_q))) 3476 continue; 3477 3478 queue = &bp->queues[q]; 3479 queue->bp = bp; 3480 netif_napi_add(dev, &queue->napi, macb_poll, 64); 3481 if (hw_q) { 3482 queue->ISR = GEM_ISR(hw_q - 1); 3483 queue->IER = GEM_IER(hw_q - 1); 3484 queue->IDR = GEM_IDR(hw_q - 1); 3485 queue->IMR = GEM_IMR(hw_q - 1); 3486 queue->TBQP = GEM_TBQP(hw_q - 1); 3487 queue->RBQP = GEM_RBQP(hw_q - 1); 3488 queue->RBQS = GEM_RBQS(hw_q - 1); 3489 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3490 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3491 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3492 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3493 } 3494 #endif 3495 } else { 3496 /* queue0 uses legacy registers */ 3497 queue->ISR = MACB_ISR; 3498 queue->IER = MACB_IER; 3499 queue->IDR = MACB_IDR; 3500 queue->IMR = MACB_IMR; 3501 queue->TBQP = MACB_TBQP; 3502 queue->RBQP = MACB_RBQP; 3503 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3504 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3505 queue->TBQPH = MACB_TBQPH; 3506 queue->RBQPH = MACB_RBQPH; 3507 } 3508 #endif 3509 } 3510 3511 /* get irq: here we use the linux queue index, not the hardware 3512 * queue index. the queue irq definitions in the device tree 3513 * must remove the optional gaps that could exist in the 3514 * hardware queue mask. 3515 */ 3516 queue->irq = platform_get_irq(pdev, q); 3517 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3518 IRQF_SHARED, dev->name, queue); 3519 if (err) { 3520 dev_err(&pdev->dev, 3521 "Unable to request IRQ %d (error %d)\n", 3522 queue->irq, err); 3523 return err; 3524 } 3525 3526 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3527 q++; 3528 } 3529 3530 dev->netdev_ops = &macb_netdev_ops; 3531 3532 /* setup appropriated routines according to adapter type */ 3533 if (macb_is_gem(bp)) { 3534 bp->max_tx_length = GEM_MAX_TX_LEN; 3535 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3536 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3537 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3538 bp->macbgem_ops.mog_rx = gem_rx; 3539 dev->ethtool_ops = &gem_ethtool_ops; 3540 } else { 3541 bp->max_tx_length = MACB_MAX_TX_LEN; 3542 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3543 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3544 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3545 bp->macbgem_ops.mog_rx = macb_rx; 3546 dev->ethtool_ops = &macb_ethtool_ops; 3547 } 3548 3549 /* Set features */ 3550 dev->hw_features = NETIF_F_SG; 3551 3552 /* Check LSO capability */ 3553 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3554 dev->hw_features |= MACB_NETIF_LSO; 3555 3556 /* Checksum offload is only available on gem with packet buffer */ 3557 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3558 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3559 if (bp->caps & MACB_CAPS_SG_DISABLED) 3560 dev->hw_features &= ~NETIF_F_SG; 3561 dev->features = dev->hw_features; 3562 3563 /* Check RX Flow Filters support. 3564 * Max Rx flows set by availability of screeners & compare regs: 3565 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3566 */ 3567 reg = gem_readl(bp, DCFG8); 3568 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3569 GEM_BFEXT(T2SCR, reg)); 3570 if (bp->max_tuples > 0) { 3571 /* also needs one ethtype match to check IPv4 */ 3572 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3573 /* program this reg now */ 3574 reg = 0; 3575 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3576 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3577 /* Filtering is supported in hw but don't enable it in kernel now */ 3578 dev->hw_features |= NETIF_F_NTUPLE; 3579 /* init Rx flow definitions */ 3580 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3581 bp->rx_fs_list.count = 0; 3582 spin_lock_init(&bp->rx_fs_lock); 3583 } else 3584 bp->max_tuples = 0; 3585 } 3586 3587 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3588 val = 0; 3589 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3590 val = GEM_BIT(RGMII); 3591 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3592 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3593 val = MACB_BIT(RMII); 3594 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3595 val = MACB_BIT(MII); 3596 3597 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3598 val |= MACB_BIT(CLKEN); 3599 3600 macb_or_gem_writel(bp, USRIO, val); 3601 } 3602 3603 /* Set MII management clock divider */ 3604 val = macb_mdc_clk_div(bp); 3605 val |= macb_dbw(bp); 3606 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3607 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3608 macb_writel(bp, NCFGR, val); 3609 3610 return 0; 3611 } 3612 3613 #if defined(CONFIG_OF) 3614 /* 1518 rounded up */ 3615 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3616 /* max number of receive buffers */ 3617 #define AT91ETHER_MAX_RX_DESCR 9 3618 3619 /* Initialize and start the Receiver and Transmit subsystems */ 3620 static int at91ether_start(struct net_device *dev) 3621 { 3622 struct macb *lp = netdev_priv(dev); 3623 struct macb_queue *q = &lp->queues[0]; 3624 struct macb_dma_desc *desc; 3625 dma_addr_t addr; 3626 u32 ctl; 3627 int i; 3628 3629 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3630 (AT91ETHER_MAX_RX_DESCR * 3631 macb_dma_desc_get_size(lp)), 3632 &q->rx_ring_dma, GFP_KERNEL); 3633 if (!q->rx_ring) 3634 return -ENOMEM; 3635 3636 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3637 AT91ETHER_MAX_RX_DESCR * 3638 AT91ETHER_MAX_RBUFF_SZ, 3639 &q->rx_buffers_dma, GFP_KERNEL); 3640 if (!q->rx_buffers) { 3641 dma_free_coherent(&lp->pdev->dev, 3642 AT91ETHER_MAX_RX_DESCR * 3643 macb_dma_desc_get_size(lp), 3644 q->rx_ring, q->rx_ring_dma); 3645 q->rx_ring = NULL; 3646 return -ENOMEM; 3647 } 3648 3649 addr = q->rx_buffers_dma; 3650 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3651 desc = macb_rx_desc(q, i); 3652 macb_set_addr(lp, desc, addr); 3653 desc->ctrl = 0; 3654 addr += AT91ETHER_MAX_RBUFF_SZ; 3655 } 3656 3657 /* Set the Wrap bit on the last descriptor */ 3658 desc->addr |= MACB_BIT(RX_WRAP); 3659 3660 /* Reset buffer index */ 3661 q->rx_tail = 0; 3662 3663 /* Program address of descriptor list in Rx Buffer Queue register */ 3664 macb_writel(lp, RBQP, q->rx_ring_dma); 3665 3666 /* Enable Receive and Transmit */ 3667 ctl = macb_readl(lp, NCR); 3668 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3669 3670 return 0; 3671 } 3672 3673 /* Open the ethernet interface */ 3674 static int at91ether_open(struct net_device *dev) 3675 { 3676 struct macb *lp = netdev_priv(dev); 3677 u32 ctl; 3678 int ret; 3679 3680 /* Clear internal statistics */ 3681 ctl = macb_readl(lp, NCR); 3682 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3683 3684 macb_set_hwaddr(lp); 3685 3686 ret = at91ether_start(dev); 3687 if (ret) 3688 return ret; 3689 3690 /* Enable MAC interrupts */ 3691 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3692 MACB_BIT(RXUBR) | 3693 MACB_BIT(ISR_TUND) | 3694 MACB_BIT(ISR_RLE) | 3695 MACB_BIT(TCOMP) | 3696 MACB_BIT(ISR_ROVR) | 3697 MACB_BIT(HRESP)); 3698 3699 /* schedule a link state check */ 3700 phy_start(dev->phydev); 3701 3702 netif_start_queue(dev); 3703 3704 return 0; 3705 } 3706 3707 /* Close the interface */ 3708 static int at91ether_close(struct net_device *dev) 3709 { 3710 struct macb *lp = netdev_priv(dev); 3711 struct macb_queue *q = &lp->queues[0]; 3712 u32 ctl; 3713 3714 /* Disable Receiver and Transmitter */ 3715 ctl = macb_readl(lp, NCR); 3716 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3717 3718 /* Disable MAC interrupts */ 3719 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3720 MACB_BIT(RXUBR) | 3721 MACB_BIT(ISR_TUND) | 3722 MACB_BIT(ISR_RLE) | 3723 MACB_BIT(TCOMP) | 3724 MACB_BIT(ISR_ROVR) | 3725 MACB_BIT(HRESP)); 3726 3727 netif_stop_queue(dev); 3728 3729 dma_free_coherent(&lp->pdev->dev, 3730 AT91ETHER_MAX_RX_DESCR * 3731 macb_dma_desc_get_size(lp), 3732 q->rx_ring, q->rx_ring_dma); 3733 q->rx_ring = NULL; 3734 3735 dma_free_coherent(&lp->pdev->dev, 3736 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3737 q->rx_buffers, q->rx_buffers_dma); 3738 q->rx_buffers = NULL; 3739 3740 return 0; 3741 } 3742 3743 /* Transmit packet */ 3744 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3745 struct net_device *dev) 3746 { 3747 struct macb *lp = netdev_priv(dev); 3748 3749 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3750 netif_stop_queue(dev); 3751 3752 /* Store packet information (to free when Tx completed) */ 3753 lp->skb = skb; 3754 lp->skb_length = skb->len; 3755 lp->skb_physaddr = dma_map_single(&lp->pdev->dev, skb->data, 3756 skb->len, DMA_TO_DEVICE); 3757 if (dma_mapping_error(&lp->pdev->dev, lp->skb_physaddr)) { 3758 dev_kfree_skb_any(skb); 3759 dev->stats.tx_dropped++; 3760 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3761 return NETDEV_TX_OK; 3762 } 3763 3764 /* Set address of the data in the Transmit Address register */ 3765 macb_writel(lp, TAR, lp->skb_physaddr); 3766 /* Set length of the packet in the Transmit Control register */ 3767 macb_writel(lp, TCR, skb->len); 3768 3769 } else { 3770 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3771 return NETDEV_TX_BUSY; 3772 } 3773 3774 return NETDEV_TX_OK; 3775 } 3776 3777 /* Extract received frame from buffer descriptors and sent to upper layers. 3778 * (Called from interrupt context) 3779 */ 3780 static void at91ether_rx(struct net_device *dev) 3781 { 3782 struct macb *lp = netdev_priv(dev); 3783 struct macb_queue *q = &lp->queues[0]; 3784 struct macb_dma_desc *desc; 3785 unsigned char *p_recv; 3786 struct sk_buff *skb; 3787 unsigned int pktlen; 3788 3789 desc = macb_rx_desc(q, q->rx_tail); 3790 while (desc->addr & MACB_BIT(RX_USED)) { 3791 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3792 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3793 skb = netdev_alloc_skb(dev, pktlen + 2); 3794 if (skb) { 3795 skb_reserve(skb, 2); 3796 skb_put_data(skb, p_recv, pktlen); 3797 3798 skb->protocol = eth_type_trans(skb, dev); 3799 dev->stats.rx_packets++; 3800 dev->stats.rx_bytes += pktlen; 3801 netif_rx(skb); 3802 } else { 3803 dev->stats.rx_dropped++; 3804 } 3805 3806 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3807 dev->stats.multicast++; 3808 3809 /* reset ownership bit */ 3810 desc->addr &= ~MACB_BIT(RX_USED); 3811 3812 /* wrap after last buffer */ 3813 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3814 q->rx_tail = 0; 3815 else 3816 q->rx_tail++; 3817 3818 desc = macb_rx_desc(q, q->rx_tail); 3819 } 3820 } 3821 3822 /* MAC interrupt handler */ 3823 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3824 { 3825 struct net_device *dev = dev_id; 3826 struct macb *lp = netdev_priv(dev); 3827 u32 intstatus, ctl; 3828 3829 /* MAC Interrupt Status register indicates what interrupts are pending. 3830 * It is automatically cleared once read. 3831 */ 3832 intstatus = macb_readl(lp, ISR); 3833 3834 /* Receive complete */ 3835 if (intstatus & MACB_BIT(RCOMP)) 3836 at91ether_rx(dev); 3837 3838 /* Transmit complete */ 3839 if (intstatus & MACB_BIT(TCOMP)) { 3840 /* The TCOM bit is set even if the transmission failed */ 3841 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3842 dev->stats.tx_errors++; 3843 3844 if (lp->skb) { 3845 dev_consume_skb_irq(lp->skb); 3846 lp->skb = NULL; 3847 dma_unmap_single(&lp->pdev->dev, lp->skb_physaddr, 3848 lp->skb_length, DMA_TO_DEVICE); 3849 dev->stats.tx_packets++; 3850 dev->stats.tx_bytes += lp->skb_length; 3851 } 3852 netif_wake_queue(dev); 3853 } 3854 3855 /* Work-around for EMAC Errata section 41.3.1 */ 3856 if (intstatus & MACB_BIT(RXUBR)) { 3857 ctl = macb_readl(lp, NCR); 3858 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3859 wmb(); 3860 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3861 } 3862 3863 if (intstatus & MACB_BIT(ISR_ROVR)) 3864 netdev_err(dev, "ROVR error\n"); 3865 3866 return IRQ_HANDLED; 3867 } 3868 3869 #ifdef CONFIG_NET_POLL_CONTROLLER 3870 static void at91ether_poll_controller(struct net_device *dev) 3871 { 3872 unsigned long flags; 3873 3874 local_irq_save(flags); 3875 at91ether_interrupt(dev->irq, dev); 3876 local_irq_restore(flags); 3877 } 3878 #endif 3879 3880 static const struct net_device_ops at91ether_netdev_ops = { 3881 .ndo_open = at91ether_open, 3882 .ndo_stop = at91ether_close, 3883 .ndo_start_xmit = at91ether_start_xmit, 3884 .ndo_get_stats = macb_get_stats, 3885 .ndo_set_rx_mode = macb_set_rx_mode, 3886 .ndo_set_mac_address = eth_mac_addr, 3887 .ndo_do_ioctl = macb_ioctl, 3888 .ndo_validate_addr = eth_validate_addr, 3889 #ifdef CONFIG_NET_POLL_CONTROLLER 3890 .ndo_poll_controller = at91ether_poll_controller, 3891 #endif 3892 }; 3893 3894 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3895 struct clk **hclk, struct clk **tx_clk, 3896 struct clk **rx_clk, struct clk **tsu_clk) 3897 { 3898 int err; 3899 3900 *hclk = NULL; 3901 *tx_clk = NULL; 3902 *rx_clk = NULL; 3903 *tsu_clk = NULL; 3904 3905 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3906 if (IS_ERR(*pclk)) 3907 return PTR_ERR(*pclk); 3908 3909 err = clk_prepare_enable(*pclk); 3910 if (err) { 3911 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3912 return err; 3913 } 3914 3915 return 0; 3916 } 3917 3918 static int at91ether_init(struct platform_device *pdev) 3919 { 3920 struct net_device *dev = platform_get_drvdata(pdev); 3921 struct macb *bp = netdev_priv(dev); 3922 int err; 3923 u32 reg; 3924 3925 bp->queues[0].bp = bp; 3926 3927 dev->netdev_ops = &at91ether_netdev_ops; 3928 dev->ethtool_ops = &macb_ethtool_ops; 3929 3930 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3931 0, dev->name, dev); 3932 if (err) 3933 return err; 3934 3935 macb_writel(bp, NCR, 0); 3936 3937 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3938 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3939 reg |= MACB_BIT(RM9200_RMII); 3940 3941 macb_writel(bp, NCFGR, reg); 3942 3943 return 0; 3944 } 3945 3946 static const struct macb_config at91sam9260_config = { 3947 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3948 .clk_init = macb_clk_init, 3949 .init = macb_init, 3950 }; 3951 3952 static const struct macb_config sama5d3macb_config = { 3953 .caps = MACB_CAPS_SG_DISABLED 3954 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3955 .clk_init = macb_clk_init, 3956 .init = macb_init, 3957 }; 3958 3959 static const struct macb_config pc302gem_config = { 3960 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 3961 .dma_burst_length = 16, 3962 .clk_init = macb_clk_init, 3963 .init = macb_init, 3964 }; 3965 3966 static const struct macb_config sama5d2_config = { 3967 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3968 .dma_burst_length = 16, 3969 .clk_init = macb_clk_init, 3970 .init = macb_init, 3971 }; 3972 3973 static const struct macb_config sama5d3_config = { 3974 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 3975 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 3976 .dma_burst_length = 16, 3977 .clk_init = macb_clk_init, 3978 .init = macb_init, 3979 .jumbo_max_len = 10240, 3980 }; 3981 3982 static const struct macb_config sama5d4_config = { 3983 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3984 .dma_burst_length = 4, 3985 .clk_init = macb_clk_init, 3986 .init = macb_init, 3987 }; 3988 3989 static const struct macb_config emac_config = { 3990 .caps = MACB_CAPS_NEEDS_RSTONUBR, 3991 .clk_init = at91ether_clk_init, 3992 .init = at91ether_init, 3993 }; 3994 3995 static const struct macb_config np4_config = { 3996 .caps = MACB_CAPS_USRIO_DISABLED, 3997 .clk_init = macb_clk_init, 3998 .init = macb_init, 3999 }; 4000 4001 static const struct macb_config zynqmp_config = { 4002 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4003 MACB_CAPS_JUMBO | 4004 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 4005 .dma_burst_length = 16, 4006 .clk_init = macb_clk_init, 4007 .init = macb_init, 4008 .jumbo_max_len = 10240, 4009 }; 4010 4011 static const struct macb_config zynq_config = { 4012 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 4013 MACB_CAPS_NEEDS_RSTONUBR, 4014 .dma_burst_length = 16, 4015 .clk_init = macb_clk_init, 4016 .init = macb_init, 4017 }; 4018 4019 static const struct of_device_id macb_dt_ids[] = { 4020 { .compatible = "cdns,at32ap7000-macb" }, 4021 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 4022 { .compatible = "cdns,macb" }, 4023 { .compatible = "cdns,np4-macb", .data = &np4_config }, 4024 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 4025 { .compatible = "cdns,gem", .data = &pc302gem_config }, 4026 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 4027 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 4028 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 4029 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 4030 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 4031 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 4032 { .compatible = "cdns,emac", .data = &emac_config }, 4033 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 4034 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 4035 { /* sentinel */ } 4036 }; 4037 MODULE_DEVICE_TABLE(of, macb_dt_ids); 4038 #endif /* CONFIG_OF */ 4039 4040 static const struct macb_config default_gem_config = { 4041 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4042 MACB_CAPS_JUMBO | 4043 MACB_CAPS_GEM_HAS_PTP, 4044 .dma_burst_length = 16, 4045 .clk_init = macb_clk_init, 4046 .init = macb_init, 4047 .jumbo_max_len = 10240, 4048 }; 4049 4050 static int macb_probe(struct platform_device *pdev) 4051 { 4052 const struct macb_config *macb_config = &default_gem_config; 4053 int (*clk_init)(struct platform_device *, struct clk **, 4054 struct clk **, struct clk **, struct clk **, 4055 struct clk **) = macb_config->clk_init; 4056 int (*init)(struct platform_device *) = macb_config->init; 4057 struct device_node *np = pdev->dev.of_node; 4058 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4059 struct clk *tsu_clk = NULL; 4060 unsigned int queue_mask, num_queues; 4061 bool native_io; 4062 struct phy_device *phydev; 4063 struct net_device *dev; 4064 struct resource *regs; 4065 void __iomem *mem; 4066 const char *mac; 4067 struct macb *bp; 4068 int err, val; 4069 4070 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4071 mem = devm_ioremap_resource(&pdev->dev, regs); 4072 if (IS_ERR(mem)) 4073 return PTR_ERR(mem); 4074 4075 if (np) { 4076 const struct of_device_id *match; 4077 4078 match = of_match_node(macb_dt_ids, np); 4079 if (match && match->data) { 4080 macb_config = match->data; 4081 clk_init = macb_config->clk_init; 4082 init = macb_config->init; 4083 } 4084 } 4085 4086 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4087 if (err) 4088 return err; 4089 4090 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4091 pm_runtime_use_autosuspend(&pdev->dev); 4092 pm_runtime_get_noresume(&pdev->dev); 4093 pm_runtime_set_active(&pdev->dev); 4094 pm_runtime_enable(&pdev->dev); 4095 native_io = hw_is_native_io(mem); 4096 4097 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4098 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4099 if (!dev) { 4100 err = -ENOMEM; 4101 goto err_disable_clocks; 4102 } 4103 4104 dev->base_addr = regs->start; 4105 4106 SET_NETDEV_DEV(dev, &pdev->dev); 4107 4108 bp = netdev_priv(dev); 4109 bp->pdev = pdev; 4110 bp->dev = dev; 4111 bp->regs = mem; 4112 bp->native_io = native_io; 4113 if (native_io) { 4114 bp->macb_reg_readl = hw_readl_native; 4115 bp->macb_reg_writel = hw_writel_native; 4116 } else { 4117 bp->macb_reg_readl = hw_readl; 4118 bp->macb_reg_writel = hw_writel; 4119 } 4120 bp->num_queues = num_queues; 4121 bp->queue_mask = queue_mask; 4122 if (macb_config) 4123 bp->dma_burst_length = macb_config->dma_burst_length; 4124 bp->pclk = pclk; 4125 bp->hclk = hclk; 4126 bp->tx_clk = tx_clk; 4127 bp->rx_clk = rx_clk; 4128 bp->tsu_clk = tsu_clk; 4129 if (macb_config) 4130 bp->jumbo_max_len = macb_config->jumbo_max_len; 4131 4132 bp->wol = 0; 4133 if (of_get_property(np, "magic-packet", NULL)) 4134 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4135 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4136 4137 spin_lock_init(&bp->lock); 4138 4139 /* setup capabilities */ 4140 macb_configure_caps(bp, macb_config); 4141 4142 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4143 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4144 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4145 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4146 } 4147 #endif 4148 platform_set_drvdata(pdev, dev); 4149 4150 dev->irq = platform_get_irq(pdev, 0); 4151 if (dev->irq < 0) { 4152 err = dev->irq; 4153 goto err_out_free_netdev; 4154 } 4155 4156 /* MTU range: 68 - 1500 or 10240 */ 4157 dev->min_mtu = GEM_MTU_MIN_SIZE; 4158 if (bp->caps & MACB_CAPS_JUMBO) 4159 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4160 else 4161 dev->max_mtu = ETH_DATA_LEN; 4162 4163 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4164 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4165 if (val) 4166 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4167 macb_dma_desc_get_size(bp); 4168 4169 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4170 if (val) 4171 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4172 macb_dma_desc_get_size(bp); 4173 } 4174 4175 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4176 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4177 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4178 4179 mac = of_get_mac_address(np); 4180 if (PTR_ERR(mac) == -EPROBE_DEFER) { 4181 err = -EPROBE_DEFER; 4182 goto err_out_free_netdev; 4183 } else if (!IS_ERR_OR_NULL(mac)) { 4184 ether_addr_copy(bp->dev->dev_addr, mac); 4185 } else { 4186 macb_get_hwaddr(bp); 4187 } 4188 4189 err = of_get_phy_mode(np); 4190 if (err < 0) 4191 /* not found in DT, MII by default */ 4192 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4193 else 4194 bp->phy_interface = err; 4195 4196 /* IP specific init */ 4197 err = init(pdev); 4198 if (err) 4199 goto err_out_free_netdev; 4200 4201 err = macb_mii_init(bp); 4202 if (err) 4203 goto err_out_free_netdev; 4204 4205 phydev = dev->phydev; 4206 4207 netif_carrier_off(dev); 4208 4209 err = register_netdev(dev); 4210 if (err) { 4211 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4212 goto err_out_unregister_mdio; 4213 } 4214 4215 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4216 (unsigned long)bp); 4217 4218 phy_attached_info(phydev); 4219 4220 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4221 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4222 dev->base_addr, dev->irq, dev->dev_addr); 4223 4224 pm_runtime_mark_last_busy(&bp->pdev->dev); 4225 pm_runtime_put_autosuspend(&bp->pdev->dev); 4226 4227 return 0; 4228 4229 err_out_unregister_mdio: 4230 phy_disconnect(dev->phydev); 4231 mdiobus_unregister(bp->mii_bus); 4232 of_node_put(bp->phy_node); 4233 if (np && of_phy_is_fixed_link(np)) 4234 of_phy_deregister_fixed_link(np); 4235 mdiobus_free(bp->mii_bus); 4236 4237 err_out_free_netdev: 4238 free_netdev(dev); 4239 4240 err_disable_clocks: 4241 clk_disable_unprepare(tx_clk); 4242 clk_disable_unprepare(hclk); 4243 clk_disable_unprepare(pclk); 4244 clk_disable_unprepare(rx_clk); 4245 clk_disable_unprepare(tsu_clk); 4246 pm_runtime_disable(&pdev->dev); 4247 pm_runtime_set_suspended(&pdev->dev); 4248 pm_runtime_dont_use_autosuspend(&pdev->dev); 4249 4250 return err; 4251 } 4252 4253 static int macb_remove(struct platform_device *pdev) 4254 { 4255 struct net_device *dev; 4256 struct macb *bp; 4257 struct device_node *np = pdev->dev.of_node; 4258 4259 dev = platform_get_drvdata(pdev); 4260 4261 if (dev) { 4262 bp = netdev_priv(dev); 4263 if (dev->phydev) 4264 phy_disconnect(dev->phydev); 4265 mdiobus_unregister(bp->mii_bus); 4266 if (np && of_phy_is_fixed_link(np)) 4267 of_phy_deregister_fixed_link(np); 4268 dev->phydev = NULL; 4269 mdiobus_free(bp->mii_bus); 4270 4271 unregister_netdev(dev); 4272 pm_runtime_disable(&pdev->dev); 4273 pm_runtime_dont_use_autosuspend(&pdev->dev); 4274 if (!pm_runtime_suspended(&pdev->dev)) { 4275 clk_disable_unprepare(bp->tx_clk); 4276 clk_disable_unprepare(bp->hclk); 4277 clk_disable_unprepare(bp->pclk); 4278 clk_disable_unprepare(bp->rx_clk); 4279 clk_disable_unprepare(bp->tsu_clk); 4280 pm_runtime_set_suspended(&pdev->dev); 4281 } 4282 of_node_put(bp->phy_node); 4283 free_netdev(dev); 4284 } 4285 4286 return 0; 4287 } 4288 4289 static int __maybe_unused macb_suspend(struct device *dev) 4290 { 4291 struct net_device *netdev = dev_get_drvdata(dev); 4292 struct macb *bp = netdev_priv(netdev); 4293 struct macb_queue *queue = bp->queues; 4294 unsigned long flags; 4295 unsigned int q; 4296 4297 if (!netif_running(netdev)) 4298 return 0; 4299 4300 4301 if (bp->wol & MACB_WOL_ENABLED) { 4302 macb_writel(bp, IER, MACB_BIT(WOL)); 4303 macb_writel(bp, WOL, MACB_BIT(MAG)); 4304 enable_irq_wake(bp->queues[0].irq); 4305 netif_device_detach(netdev); 4306 } else { 4307 netif_device_detach(netdev); 4308 for (q = 0, queue = bp->queues; q < bp->num_queues; 4309 ++q, ++queue) 4310 napi_disable(&queue->napi); 4311 phy_stop(netdev->phydev); 4312 phy_suspend(netdev->phydev); 4313 spin_lock_irqsave(&bp->lock, flags); 4314 macb_reset_hw(bp); 4315 spin_unlock_irqrestore(&bp->lock, flags); 4316 4317 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4318 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 4319 4320 if (netdev->hw_features & NETIF_F_NTUPLE) 4321 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 4322 } 4323 4324 netif_carrier_off(netdev); 4325 if (bp->ptp_info) 4326 bp->ptp_info->ptp_remove(netdev); 4327 pm_runtime_force_suspend(dev); 4328 4329 return 0; 4330 } 4331 4332 static int __maybe_unused macb_resume(struct device *dev) 4333 { 4334 struct net_device *netdev = dev_get_drvdata(dev); 4335 struct macb *bp = netdev_priv(netdev); 4336 struct macb_queue *queue = bp->queues; 4337 unsigned int q; 4338 4339 if (!netif_running(netdev)) 4340 return 0; 4341 4342 pm_runtime_force_resume(dev); 4343 4344 if (bp->wol & MACB_WOL_ENABLED) { 4345 macb_writel(bp, IDR, MACB_BIT(WOL)); 4346 macb_writel(bp, WOL, 0); 4347 disable_irq_wake(bp->queues[0].irq); 4348 } else { 4349 macb_writel(bp, NCR, MACB_BIT(MPE)); 4350 4351 if (netdev->hw_features & NETIF_F_NTUPLE) 4352 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 4353 4354 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4355 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 4356 4357 for (q = 0, queue = bp->queues; q < bp->num_queues; 4358 ++q, ++queue) 4359 napi_enable(&queue->napi); 4360 phy_resume(netdev->phydev); 4361 phy_init_hw(netdev->phydev); 4362 phy_start(netdev->phydev); 4363 } 4364 4365 bp->macbgem_ops.mog_init_rings(bp); 4366 macb_init_hw(bp); 4367 macb_set_rx_mode(netdev); 4368 macb_restore_features(bp); 4369 netif_device_attach(netdev); 4370 if (bp->ptp_info) 4371 bp->ptp_info->ptp_init(netdev); 4372 4373 return 0; 4374 } 4375 4376 static int __maybe_unused macb_runtime_suspend(struct device *dev) 4377 { 4378 struct net_device *netdev = dev_get_drvdata(dev); 4379 struct macb *bp = netdev_priv(netdev); 4380 4381 if (!(device_may_wakeup(&bp->dev->dev))) { 4382 clk_disable_unprepare(bp->tx_clk); 4383 clk_disable_unprepare(bp->hclk); 4384 clk_disable_unprepare(bp->pclk); 4385 clk_disable_unprepare(bp->rx_clk); 4386 } 4387 clk_disable_unprepare(bp->tsu_clk); 4388 4389 return 0; 4390 } 4391 4392 static int __maybe_unused macb_runtime_resume(struct device *dev) 4393 { 4394 struct net_device *netdev = dev_get_drvdata(dev); 4395 struct macb *bp = netdev_priv(netdev); 4396 4397 if (!(device_may_wakeup(&bp->dev->dev))) { 4398 clk_prepare_enable(bp->pclk); 4399 clk_prepare_enable(bp->hclk); 4400 clk_prepare_enable(bp->tx_clk); 4401 clk_prepare_enable(bp->rx_clk); 4402 } 4403 clk_prepare_enable(bp->tsu_clk); 4404 4405 return 0; 4406 } 4407 4408 static const struct dev_pm_ops macb_pm_ops = { 4409 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 4410 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 4411 }; 4412 4413 static struct platform_driver macb_driver = { 4414 .probe = macb_probe, 4415 .remove = macb_remove, 4416 .driver = { 4417 .name = "macb", 4418 .of_match_table = of_match_ptr(macb_dt_ids), 4419 .pm = &macb_pm_ops, 4420 }, 4421 }; 4422 4423 module_platform_driver(macb_driver); 4424 4425 MODULE_LICENSE("GPL"); 4426 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4427 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4428 MODULE_ALIAS("platform:macb"); 4429