1 /* 2 * Cadence MACB/GEM Ethernet Controller driver 3 * 4 * Copyright (C) 2004-2006 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 #include <linux/clk.h> 13 #include <linux/crc32.h> 14 #include <linux/module.h> 15 #include <linux/moduleparam.h> 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/circ_buf.h> 19 #include <linux/slab.h> 20 #include <linux/init.h> 21 #include <linux/io.h> 22 #include <linux/gpio.h> 23 #include <linux/gpio/consumer.h> 24 #include <linux/interrupt.h> 25 #include <linux/netdevice.h> 26 #include <linux/etherdevice.h> 27 #include <linux/dma-mapping.h> 28 #include <linux/platform_data/macb.h> 29 #include <linux/platform_device.h> 30 #include <linux/phy.h> 31 #include <linux/of.h> 32 #include <linux/of_device.h> 33 #include <linux/of_gpio.h> 34 #include <linux/of_mdio.h> 35 #include <linux/of_net.h> 36 #include <linux/ip.h> 37 #include <linux/udp.h> 38 #include <linux/tcp.h> 39 #include <linux/iopoll.h> 40 #include <linux/pm_runtime.h> 41 #include "macb.h" 42 43 #define MACB_RX_BUFFER_SIZE 128 44 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 45 46 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 47 #define MIN_RX_RING_SIZE 64 48 #define MAX_RX_RING_SIZE 8192 49 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 50 * (bp)->rx_ring_size) 51 52 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 53 #define MIN_TX_RING_SIZE 64 54 #define MAX_TX_RING_SIZE 4096 55 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 56 * (bp)->tx_ring_size) 57 58 /* level of occupied TX descriptors under which we wake up TX process */ 59 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 60 61 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 62 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 63 | MACB_BIT(ISR_RLE) \ 64 | MACB_BIT(TXERR)) 65 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 66 | MACB_BIT(TXUBR)) 67 68 /* Max length of transmit frame must be a multiple of 8 bytes */ 69 #define MACB_TX_LEN_ALIGN 8 70 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 71 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 72 73 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 74 #define MACB_NETIF_LSO NETIF_F_TSO 75 76 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 77 #define MACB_WOL_ENABLED (0x1 << 1) 78 79 /* Graceful stop timeouts in us. We should allow up to 80 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 81 */ 82 #define MACB_HALT_TIMEOUT 1230 83 84 #define MACB_PM_TIMEOUT 100 /* ms */ 85 86 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 87 88 /* DMA buffer descriptor might be different size 89 * depends on hardware configuration: 90 * 91 * 1. dma address width 32 bits: 92 * word 1: 32 bit address of Data Buffer 93 * word 2: control 94 * 95 * 2. dma address width 64 bits: 96 * word 1: 32 bit address of Data Buffer 97 * word 2: control 98 * word 3: upper 32 bit address of Data Buffer 99 * word 4: unused 100 * 101 * 3. dma address width 32 bits with hardware timestamping: 102 * word 1: 32 bit address of Data Buffer 103 * word 2: control 104 * word 3: timestamp word 1 105 * word 4: timestamp word 2 106 * 107 * 4. dma address width 64 bits with hardware timestamping: 108 * word 1: 32 bit address of Data Buffer 109 * word 2: control 110 * word 3: upper 32 bit address of Data Buffer 111 * word 4: unused 112 * word 5: timestamp word 1 113 * word 6: timestamp word 2 114 */ 115 static unsigned int macb_dma_desc_get_size(struct macb *bp) 116 { 117 #ifdef MACB_EXT_DESC 118 unsigned int desc_size; 119 120 switch (bp->hw_dma_cap) { 121 case HW_DMA_CAP_64B: 122 desc_size = sizeof(struct macb_dma_desc) 123 + sizeof(struct macb_dma_desc_64); 124 break; 125 case HW_DMA_CAP_PTP: 126 desc_size = sizeof(struct macb_dma_desc) 127 + sizeof(struct macb_dma_desc_ptp); 128 break; 129 case HW_DMA_CAP_64B_PTP: 130 desc_size = sizeof(struct macb_dma_desc) 131 + sizeof(struct macb_dma_desc_64) 132 + sizeof(struct macb_dma_desc_ptp); 133 break; 134 default: 135 desc_size = sizeof(struct macb_dma_desc); 136 } 137 return desc_size; 138 #endif 139 return sizeof(struct macb_dma_desc); 140 } 141 142 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 143 { 144 #ifdef MACB_EXT_DESC 145 switch (bp->hw_dma_cap) { 146 case HW_DMA_CAP_64B: 147 case HW_DMA_CAP_PTP: 148 desc_idx <<= 1; 149 break; 150 case HW_DMA_CAP_64B_PTP: 151 desc_idx *= 3; 152 break; 153 default: 154 break; 155 } 156 #endif 157 return desc_idx; 158 } 159 160 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 161 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 162 { 163 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 164 return (struct macb_dma_desc_64 *)((void *)desc + sizeof(struct macb_dma_desc)); 165 return NULL; 166 } 167 #endif 168 169 /* Ring buffer accessors */ 170 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 171 { 172 return index & (bp->tx_ring_size - 1); 173 } 174 175 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 176 unsigned int index) 177 { 178 index = macb_tx_ring_wrap(queue->bp, index); 179 index = macb_adj_dma_desc_idx(queue->bp, index); 180 return &queue->tx_ring[index]; 181 } 182 183 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 184 unsigned int index) 185 { 186 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 187 } 188 189 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 190 { 191 dma_addr_t offset; 192 193 offset = macb_tx_ring_wrap(queue->bp, index) * 194 macb_dma_desc_get_size(queue->bp); 195 196 return queue->tx_ring_dma + offset; 197 } 198 199 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 200 { 201 return index & (bp->rx_ring_size - 1); 202 } 203 204 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 205 { 206 index = macb_rx_ring_wrap(queue->bp, index); 207 index = macb_adj_dma_desc_idx(queue->bp, index); 208 return &queue->rx_ring[index]; 209 } 210 211 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 212 { 213 return queue->rx_buffers + queue->bp->rx_buffer_size * 214 macb_rx_ring_wrap(queue->bp, index); 215 } 216 217 /* I/O accessors */ 218 static u32 hw_readl_native(struct macb *bp, int offset) 219 { 220 return __raw_readl(bp->regs + offset); 221 } 222 223 static void hw_writel_native(struct macb *bp, int offset, u32 value) 224 { 225 __raw_writel(value, bp->regs + offset); 226 } 227 228 static u32 hw_readl(struct macb *bp, int offset) 229 { 230 return readl_relaxed(bp->regs + offset); 231 } 232 233 static void hw_writel(struct macb *bp, int offset, u32 value) 234 { 235 writel_relaxed(value, bp->regs + offset); 236 } 237 238 /* Find the CPU endianness by using the loopback bit of NCR register. When the 239 * CPU is in big endian we need to program swapped mode for management 240 * descriptor access. 241 */ 242 static bool hw_is_native_io(void __iomem *addr) 243 { 244 u32 value = MACB_BIT(LLB); 245 246 __raw_writel(value, addr + MACB_NCR); 247 value = __raw_readl(addr + MACB_NCR); 248 249 /* Write 0 back to disable everything */ 250 __raw_writel(0, addr + MACB_NCR); 251 252 return value == MACB_BIT(LLB); 253 } 254 255 static bool hw_is_gem(void __iomem *addr, bool native_io) 256 { 257 u32 id; 258 259 if (native_io) 260 id = __raw_readl(addr + MACB_MID); 261 else 262 id = readl_relaxed(addr + MACB_MID); 263 264 return MACB_BFEXT(IDNUM, id) >= 0x2; 265 } 266 267 static void macb_set_hwaddr(struct macb *bp) 268 { 269 u32 bottom; 270 u16 top; 271 272 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 273 macb_or_gem_writel(bp, SA1B, bottom); 274 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 275 macb_or_gem_writel(bp, SA1T, top); 276 277 /* Clear unused address register sets */ 278 macb_or_gem_writel(bp, SA2B, 0); 279 macb_or_gem_writel(bp, SA2T, 0); 280 macb_or_gem_writel(bp, SA3B, 0); 281 macb_or_gem_writel(bp, SA3T, 0); 282 macb_or_gem_writel(bp, SA4B, 0); 283 macb_or_gem_writel(bp, SA4T, 0); 284 } 285 286 static void macb_get_hwaddr(struct macb *bp) 287 { 288 u32 bottom; 289 u16 top; 290 u8 addr[6]; 291 int i; 292 293 /* Check all 4 address register for valid address */ 294 for (i = 0; i < 4; i++) { 295 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 296 top = macb_or_gem_readl(bp, SA1T + i * 8); 297 298 addr[0] = bottom & 0xff; 299 addr[1] = (bottom >> 8) & 0xff; 300 addr[2] = (bottom >> 16) & 0xff; 301 addr[3] = (bottom >> 24) & 0xff; 302 addr[4] = top & 0xff; 303 addr[5] = (top >> 8) & 0xff; 304 305 if (is_valid_ether_addr(addr)) { 306 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 307 return; 308 } 309 } 310 311 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 312 eth_hw_addr_random(bp->dev); 313 } 314 315 static int macb_mdio_wait_for_idle(struct macb *bp) 316 { 317 u32 val; 318 319 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 320 1, MACB_MDIO_TIMEOUT); 321 } 322 323 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 324 { 325 struct macb *bp = bus->priv; 326 int status; 327 328 status = pm_runtime_get_sync(&bp->pdev->dev); 329 if (status < 0) 330 goto mdio_pm_exit; 331 332 status = macb_mdio_wait_for_idle(bp); 333 if (status < 0) 334 goto mdio_read_exit; 335 336 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 337 | MACB_BF(RW, MACB_MAN_READ) 338 | MACB_BF(PHYA, mii_id) 339 | MACB_BF(REGA, regnum) 340 | MACB_BF(CODE, MACB_MAN_CODE))); 341 342 status = macb_mdio_wait_for_idle(bp); 343 if (status < 0) 344 goto mdio_read_exit; 345 346 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 347 348 mdio_read_exit: 349 pm_runtime_mark_last_busy(&bp->pdev->dev); 350 pm_runtime_put_autosuspend(&bp->pdev->dev); 351 mdio_pm_exit: 352 return status; 353 } 354 355 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 356 u16 value) 357 { 358 struct macb *bp = bus->priv; 359 int status; 360 361 status = pm_runtime_get_sync(&bp->pdev->dev); 362 if (status < 0) 363 goto mdio_pm_exit; 364 365 status = macb_mdio_wait_for_idle(bp); 366 if (status < 0) 367 goto mdio_write_exit; 368 369 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 370 | MACB_BF(RW, MACB_MAN_WRITE) 371 | MACB_BF(PHYA, mii_id) 372 | MACB_BF(REGA, regnum) 373 | MACB_BF(CODE, MACB_MAN_CODE) 374 | MACB_BF(DATA, value))); 375 376 status = macb_mdio_wait_for_idle(bp); 377 if (status < 0) 378 goto mdio_write_exit; 379 380 mdio_write_exit: 381 pm_runtime_mark_last_busy(&bp->pdev->dev); 382 pm_runtime_put_autosuspend(&bp->pdev->dev); 383 mdio_pm_exit: 384 return status; 385 } 386 387 /** 388 * macb_set_tx_clk() - Set a clock to a new frequency 389 * @clk Pointer to the clock to change 390 * @rate New frequency in Hz 391 * @dev Pointer to the struct net_device 392 */ 393 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 394 { 395 long ferr, rate, rate_rounded; 396 397 if (!clk) 398 return; 399 400 switch (speed) { 401 case SPEED_10: 402 rate = 2500000; 403 break; 404 case SPEED_100: 405 rate = 25000000; 406 break; 407 case SPEED_1000: 408 rate = 125000000; 409 break; 410 default: 411 return; 412 } 413 414 rate_rounded = clk_round_rate(clk, rate); 415 if (rate_rounded < 0) 416 return; 417 418 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 419 * is not satisfied. 420 */ 421 ferr = abs(rate_rounded - rate); 422 ferr = DIV_ROUND_UP(ferr, rate / 100000); 423 if (ferr > 5) 424 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 425 rate); 426 427 if (clk_set_rate(clk, rate_rounded)) 428 netdev_err(dev, "adjusting tx_clk failed.\n"); 429 } 430 431 static void macb_handle_link_change(struct net_device *dev) 432 { 433 struct macb *bp = netdev_priv(dev); 434 struct phy_device *phydev = dev->phydev; 435 unsigned long flags; 436 int status_change = 0; 437 438 spin_lock_irqsave(&bp->lock, flags); 439 440 if (phydev->link) { 441 if ((bp->speed != phydev->speed) || 442 (bp->duplex != phydev->duplex)) { 443 u32 reg; 444 445 reg = macb_readl(bp, NCFGR); 446 reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 447 if (macb_is_gem(bp)) 448 reg &= ~GEM_BIT(GBE); 449 450 if (phydev->duplex) 451 reg |= MACB_BIT(FD); 452 if (phydev->speed == SPEED_100) 453 reg |= MACB_BIT(SPD); 454 if (phydev->speed == SPEED_1000 && 455 bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 456 reg |= GEM_BIT(GBE); 457 458 macb_or_gem_writel(bp, NCFGR, reg); 459 460 bp->speed = phydev->speed; 461 bp->duplex = phydev->duplex; 462 status_change = 1; 463 } 464 } 465 466 if (phydev->link != bp->link) { 467 if (!phydev->link) { 468 bp->speed = 0; 469 bp->duplex = -1; 470 } 471 bp->link = phydev->link; 472 473 status_change = 1; 474 } 475 476 spin_unlock_irqrestore(&bp->lock, flags); 477 478 if (status_change) { 479 if (phydev->link) { 480 /* Update the TX clock rate if and only if the link is 481 * up and there has been a link change. 482 */ 483 macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); 484 485 netif_carrier_on(dev); 486 netdev_info(dev, "link up (%d/%s)\n", 487 phydev->speed, 488 phydev->duplex == DUPLEX_FULL ? 489 "Full" : "Half"); 490 } else { 491 netif_carrier_off(dev); 492 netdev_info(dev, "link down\n"); 493 } 494 } 495 } 496 497 /* based on au1000_eth. c*/ 498 static int macb_mii_probe(struct net_device *dev) 499 { 500 struct macb *bp = netdev_priv(dev); 501 struct phy_device *phydev; 502 struct device_node *np; 503 int ret, i; 504 505 np = bp->pdev->dev.of_node; 506 ret = 0; 507 508 if (np) { 509 if (of_phy_is_fixed_link(np)) { 510 bp->phy_node = of_node_get(np); 511 } else { 512 bp->phy_node = of_parse_phandle(np, "phy-handle", 0); 513 /* fallback to standard phy registration if no 514 * phy-handle was found nor any phy found during 515 * dt phy registration 516 */ 517 if (!bp->phy_node && !phy_find_first(bp->mii_bus)) { 518 for (i = 0; i < PHY_MAX_ADDR; i++) { 519 phydev = mdiobus_scan(bp->mii_bus, i); 520 if (IS_ERR(phydev) && 521 PTR_ERR(phydev) != -ENODEV) { 522 ret = PTR_ERR(phydev); 523 break; 524 } 525 } 526 527 if (ret) 528 return -ENODEV; 529 } 530 } 531 } 532 533 if (bp->phy_node) { 534 phydev = of_phy_connect(dev, bp->phy_node, 535 &macb_handle_link_change, 0, 536 bp->phy_interface); 537 if (!phydev) 538 return -ENODEV; 539 } else { 540 phydev = phy_find_first(bp->mii_bus); 541 if (!phydev) { 542 netdev_err(dev, "no PHY found\n"); 543 return -ENXIO; 544 } 545 546 /* attach the mac to the phy */ 547 ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, 548 bp->phy_interface); 549 if (ret) { 550 netdev_err(dev, "Could not attach to PHY\n"); 551 return ret; 552 } 553 } 554 555 /* mask with MAC supported features */ 556 if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 557 phy_set_max_speed(phydev, SPEED_1000); 558 else 559 phy_set_max_speed(phydev, SPEED_100); 560 561 if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) 562 phy_remove_link_mode(phydev, 563 ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 564 565 bp->link = 0; 566 bp->speed = 0; 567 bp->duplex = -1; 568 569 return 0; 570 } 571 572 static int macb_mii_init(struct macb *bp) 573 { 574 struct device_node *np; 575 int err = -ENXIO; 576 577 /* Enable management port */ 578 macb_writel(bp, NCR, MACB_BIT(MPE)); 579 580 bp->mii_bus = mdiobus_alloc(); 581 if (!bp->mii_bus) { 582 err = -ENOMEM; 583 goto err_out; 584 } 585 586 bp->mii_bus->name = "MACB_mii_bus"; 587 bp->mii_bus->read = &macb_mdio_read; 588 bp->mii_bus->write = &macb_mdio_write; 589 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 590 bp->pdev->name, bp->pdev->id); 591 bp->mii_bus->priv = bp; 592 bp->mii_bus->parent = &bp->pdev->dev; 593 594 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 595 596 np = bp->pdev->dev.of_node; 597 if (np && of_phy_is_fixed_link(np)) { 598 if (of_phy_register_fixed_link(np) < 0) { 599 dev_err(&bp->pdev->dev, 600 "broken fixed-link specification %pOF\n", np); 601 goto err_out_free_mdiobus; 602 } 603 604 err = mdiobus_register(bp->mii_bus); 605 } else { 606 err = of_mdiobus_register(bp->mii_bus, np); 607 } 608 609 if (err) 610 goto err_out_free_fixed_link; 611 612 err = macb_mii_probe(bp->dev); 613 if (err) 614 goto err_out_unregister_bus; 615 616 return 0; 617 618 err_out_unregister_bus: 619 mdiobus_unregister(bp->mii_bus); 620 err_out_free_fixed_link: 621 if (np && of_phy_is_fixed_link(np)) 622 of_phy_deregister_fixed_link(np); 623 err_out_free_mdiobus: 624 of_node_put(bp->phy_node); 625 mdiobus_free(bp->mii_bus); 626 err_out: 627 return err; 628 } 629 630 static void macb_update_stats(struct macb *bp) 631 { 632 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 633 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 634 int offset = MACB_PFR; 635 636 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 637 638 for (; p < end; p++, offset += 4) 639 *p += bp->macb_reg_readl(bp, offset); 640 } 641 642 static int macb_halt_tx(struct macb *bp) 643 { 644 unsigned long halt_time, timeout; 645 u32 status; 646 647 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 648 649 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 650 do { 651 halt_time = jiffies; 652 status = macb_readl(bp, TSR); 653 if (!(status & MACB_BIT(TGO))) 654 return 0; 655 656 udelay(250); 657 } while (time_before(halt_time, timeout)); 658 659 return -ETIMEDOUT; 660 } 661 662 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 663 { 664 if (tx_skb->mapping) { 665 if (tx_skb->mapped_as_page) 666 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 667 tx_skb->size, DMA_TO_DEVICE); 668 else 669 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 670 tx_skb->size, DMA_TO_DEVICE); 671 tx_skb->mapping = 0; 672 } 673 674 if (tx_skb->skb) { 675 dev_kfree_skb_any(tx_skb->skb); 676 tx_skb->skb = NULL; 677 } 678 } 679 680 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 681 { 682 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 683 struct macb_dma_desc_64 *desc_64; 684 685 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 686 desc_64 = macb_64b_desc(bp, desc); 687 desc_64->addrh = upper_32_bits(addr); 688 /* The low bits of RX address contain the RX_USED bit, clearing 689 * of which allows packet RX. Make sure the high bits are also 690 * visible to HW at that point. 691 */ 692 dma_wmb(); 693 } 694 #endif 695 desc->addr = lower_32_bits(addr); 696 } 697 698 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 699 { 700 dma_addr_t addr = 0; 701 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 702 struct macb_dma_desc_64 *desc_64; 703 704 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 705 desc_64 = macb_64b_desc(bp, desc); 706 addr = ((u64)(desc_64->addrh) << 32); 707 } 708 #endif 709 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 710 return addr; 711 } 712 713 static void macb_tx_error_task(struct work_struct *work) 714 { 715 struct macb_queue *queue = container_of(work, struct macb_queue, 716 tx_error_task); 717 struct macb *bp = queue->bp; 718 struct macb_tx_skb *tx_skb; 719 struct macb_dma_desc *desc; 720 struct sk_buff *skb; 721 unsigned int tail; 722 unsigned long flags; 723 724 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 725 (unsigned int)(queue - bp->queues), 726 queue->tx_tail, queue->tx_head); 727 728 /* Prevent the queue IRQ handlers from running: each of them may call 729 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 730 * As explained below, we have to halt the transmission before updating 731 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 732 * network engine about the macb/gem being halted. 733 */ 734 spin_lock_irqsave(&bp->lock, flags); 735 736 /* Make sure nobody is trying to queue up new packets */ 737 netif_tx_stop_all_queues(bp->dev); 738 739 /* Stop transmission now 740 * (in case we have just queued new packets) 741 * macb/gem must be halted to write TBQP register 742 */ 743 if (macb_halt_tx(bp)) 744 /* Just complain for now, reinitializing TX path can be good */ 745 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 746 747 /* Treat frames in TX queue including the ones that caused the error. 748 * Free transmit buffers in upper layer. 749 */ 750 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 751 u32 ctrl; 752 753 desc = macb_tx_desc(queue, tail); 754 ctrl = desc->ctrl; 755 tx_skb = macb_tx_skb(queue, tail); 756 skb = tx_skb->skb; 757 758 if (ctrl & MACB_BIT(TX_USED)) { 759 /* skb is set for the last buffer of the frame */ 760 while (!skb) { 761 macb_tx_unmap(bp, tx_skb); 762 tail++; 763 tx_skb = macb_tx_skb(queue, tail); 764 skb = tx_skb->skb; 765 } 766 767 /* ctrl still refers to the first buffer descriptor 768 * since it's the only one written back by the hardware 769 */ 770 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 771 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 772 macb_tx_ring_wrap(bp, tail), 773 skb->data); 774 bp->dev->stats.tx_packets++; 775 queue->stats.tx_packets++; 776 bp->dev->stats.tx_bytes += skb->len; 777 queue->stats.tx_bytes += skb->len; 778 } 779 } else { 780 /* "Buffers exhausted mid-frame" errors may only happen 781 * if the driver is buggy, so complain loudly about 782 * those. Statistics are updated by hardware. 783 */ 784 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 785 netdev_err(bp->dev, 786 "BUG: TX buffers exhausted mid-frame\n"); 787 788 desc->ctrl = ctrl | MACB_BIT(TX_USED); 789 } 790 791 macb_tx_unmap(bp, tx_skb); 792 } 793 794 /* Set end of TX queue */ 795 desc = macb_tx_desc(queue, 0); 796 macb_set_addr(bp, desc, 0); 797 desc->ctrl = MACB_BIT(TX_USED); 798 799 /* Make descriptor updates visible to hardware */ 800 wmb(); 801 802 /* Reinitialize the TX desc queue */ 803 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 804 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 805 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 806 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 807 #endif 808 /* Make TX ring reflect state of hardware */ 809 queue->tx_head = 0; 810 queue->tx_tail = 0; 811 812 /* Housework before enabling TX IRQ */ 813 macb_writel(bp, TSR, macb_readl(bp, TSR)); 814 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 815 816 /* Now we are ready to start transmission again */ 817 netif_tx_start_all_queues(bp->dev); 818 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 819 820 spin_unlock_irqrestore(&bp->lock, flags); 821 } 822 823 static void macb_tx_interrupt(struct macb_queue *queue) 824 { 825 unsigned int tail; 826 unsigned int head; 827 u32 status; 828 struct macb *bp = queue->bp; 829 u16 queue_index = queue - bp->queues; 830 831 status = macb_readl(bp, TSR); 832 macb_writel(bp, TSR, status); 833 834 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 835 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 836 837 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 838 (unsigned long)status); 839 840 head = queue->tx_head; 841 for (tail = queue->tx_tail; tail != head; tail++) { 842 struct macb_tx_skb *tx_skb; 843 struct sk_buff *skb; 844 struct macb_dma_desc *desc; 845 u32 ctrl; 846 847 desc = macb_tx_desc(queue, tail); 848 849 /* Make hw descriptor updates visible to CPU */ 850 rmb(); 851 852 ctrl = desc->ctrl; 853 854 /* TX_USED bit is only set by hardware on the very first buffer 855 * descriptor of the transmitted frame. 856 */ 857 if (!(ctrl & MACB_BIT(TX_USED))) 858 break; 859 860 /* Process all buffers of the current transmitted frame */ 861 for (;; tail++) { 862 tx_skb = macb_tx_skb(queue, tail); 863 skb = tx_skb->skb; 864 865 /* First, update TX stats if needed */ 866 if (skb) { 867 if (unlikely(skb_shinfo(skb)->tx_flags & 868 SKBTX_HW_TSTAMP) && 869 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 870 /* skb now belongs to timestamp buffer 871 * and will be removed later 872 */ 873 tx_skb->skb = NULL; 874 } 875 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 876 macb_tx_ring_wrap(bp, tail), 877 skb->data); 878 bp->dev->stats.tx_packets++; 879 queue->stats.tx_packets++; 880 bp->dev->stats.tx_bytes += skb->len; 881 queue->stats.tx_bytes += skb->len; 882 } 883 884 /* Now we can safely release resources */ 885 macb_tx_unmap(bp, tx_skb); 886 887 /* skb is set only for the last buffer of the frame. 888 * WARNING: at this point skb has been freed by 889 * macb_tx_unmap(). 890 */ 891 if (skb) 892 break; 893 } 894 } 895 896 queue->tx_tail = tail; 897 if (__netif_subqueue_stopped(bp->dev, queue_index) && 898 CIRC_CNT(queue->tx_head, queue->tx_tail, 899 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 900 netif_wake_subqueue(bp->dev, queue_index); 901 } 902 903 static void gem_rx_refill(struct macb_queue *queue) 904 { 905 unsigned int entry; 906 struct sk_buff *skb; 907 dma_addr_t paddr; 908 struct macb *bp = queue->bp; 909 struct macb_dma_desc *desc; 910 911 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 912 bp->rx_ring_size) > 0) { 913 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 914 915 /* Make hw descriptor updates visible to CPU */ 916 rmb(); 917 918 queue->rx_prepared_head++; 919 desc = macb_rx_desc(queue, entry); 920 921 if (!queue->rx_skbuff[entry]) { 922 /* allocate sk_buff for this free entry in ring */ 923 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 924 if (unlikely(!skb)) { 925 netdev_err(bp->dev, 926 "Unable to allocate sk_buff\n"); 927 break; 928 } 929 930 /* now fill corresponding descriptor entry */ 931 paddr = dma_map_single(&bp->pdev->dev, skb->data, 932 bp->rx_buffer_size, 933 DMA_FROM_DEVICE); 934 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 935 dev_kfree_skb(skb); 936 break; 937 } 938 939 queue->rx_skbuff[entry] = skb; 940 941 if (entry == bp->rx_ring_size - 1) 942 paddr |= MACB_BIT(RX_WRAP); 943 desc->ctrl = 0; 944 /* Setting addr clears RX_USED and allows reception, 945 * make sure ctrl is cleared first to avoid a race. 946 */ 947 dma_wmb(); 948 macb_set_addr(bp, desc, paddr); 949 950 /* properly align Ethernet header */ 951 skb_reserve(skb, NET_IP_ALIGN); 952 } else { 953 desc->ctrl = 0; 954 dma_wmb(); 955 desc->addr &= ~MACB_BIT(RX_USED); 956 } 957 } 958 959 /* Make descriptor updates visible to hardware */ 960 wmb(); 961 962 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 963 queue, queue->rx_prepared_head, queue->rx_tail); 964 } 965 966 /* Mark DMA descriptors from begin up to and not including end as unused */ 967 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 968 unsigned int end) 969 { 970 unsigned int frag; 971 972 for (frag = begin; frag != end; frag++) { 973 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 974 975 desc->addr &= ~MACB_BIT(RX_USED); 976 } 977 978 /* Make descriptor updates visible to hardware */ 979 wmb(); 980 981 /* When this happens, the hardware stats registers for 982 * whatever caused this is updated, so we don't have to record 983 * anything. 984 */ 985 } 986 987 static int gem_rx(struct macb_queue *queue, int budget) 988 { 989 struct macb *bp = queue->bp; 990 unsigned int len; 991 unsigned int entry; 992 struct sk_buff *skb; 993 struct macb_dma_desc *desc; 994 int count = 0; 995 996 while (count < budget) { 997 u32 ctrl; 998 dma_addr_t addr; 999 bool rxused; 1000 1001 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1002 desc = macb_rx_desc(queue, entry); 1003 1004 /* Make hw descriptor updates visible to CPU */ 1005 rmb(); 1006 1007 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1008 addr = macb_get_addr(bp, desc); 1009 1010 if (!rxused) 1011 break; 1012 1013 /* Ensure ctrl is at least as up-to-date as rxused */ 1014 dma_rmb(); 1015 1016 ctrl = desc->ctrl; 1017 1018 queue->rx_tail++; 1019 count++; 1020 1021 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1022 netdev_err(bp->dev, 1023 "not whole frame pointed by descriptor\n"); 1024 bp->dev->stats.rx_dropped++; 1025 queue->stats.rx_dropped++; 1026 break; 1027 } 1028 skb = queue->rx_skbuff[entry]; 1029 if (unlikely(!skb)) { 1030 netdev_err(bp->dev, 1031 "inconsistent Rx descriptor chain\n"); 1032 bp->dev->stats.rx_dropped++; 1033 queue->stats.rx_dropped++; 1034 break; 1035 } 1036 /* now everything is ready for receiving packet */ 1037 queue->rx_skbuff[entry] = NULL; 1038 len = ctrl & bp->rx_frm_len_mask; 1039 1040 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1041 1042 skb_put(skb, len); 1043 dma_unmap_single(&bp->pdev->dev, addr, 1044 bp->rx_buffer_size, DMA_FROM_DEVICE); 1045 1046 skb->protocol = eth_type_trans(skb, bp->dev); 1047 skb_checksum_none_assert(skb); 1048 if (bp->dev->features & NETIF_F_RXCSUM && 1049 !(bp->dev->flags & IFF_PROMISC) && 1050 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1051 skb->ip_summed = CHECKSUM_UNNECESSARY; 1052 1053 bp->dev->stats.rx_packets++; 1054 queue->stats.rx_packets++; 1055 bp->dev->stats.rx_bytes += skb->len; 1056 queue->stats.rx_bytes += skb->len; 1057 1058 gem_ptp_do_rxstamp(bp, skb, desc); 1059 1060 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1061 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1062 skb->len, skb->csum); 1063 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1064 skb_mac_header(skb), 16, true); 1065 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1066 skb->data, 32, true); 1067 #endif 1068 1069 netif_receive_skb(skb); 1070 } 1071 1072 gem_rx_refill(queue); 1073 1074 return count; 1075 } 1076 1077 static int macb_rx_frame(struct macb_queue *queue, unsigned int first_frag, 1078 unsigned int last_frag) 1079 { 1080 unsigned int len; 1081 unsigned int frag; 1082 unsigned int offset; 1083 struct sk_buff *skb; 1084 struct macb_dma_desc *desc; 1085 struct macb *bp = queue->bp; 1086 1087 desc = macb_rx_desc(queue, last_frag); 1088 len = desc->ctrl & bp->rx_frm_len_mask; 1089 1090 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1091 macb_rx_ring_wrap(bp, first_frag), 1092 macb_rx_ring_wrap(bp, last_frag), len); 1093 1094 /* The ethernet header starts NET_IP_ALIGN bytes into the 1095 * first buffer. Since the header is 14 bytes, this makes the 1096 * payload word-aligned. 1097 * 1098 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1099 * the two padding bytes into the skb so that we avoid hitting 1100 * the slowpath in memcpy(), and pull them off afterwards. 1101 */ 1102 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1103 if (!skb) { 1104 bp->dev->stats.rx_dropped++; 1105 for (frag = first_frag; ; frag++) { 1106 desc = macb_rx_desc(queue, frag); 1107 desc->addr &= ~MACB_BIT(RX_USED); 1108 if (frag == last_frag) 1109 break; 1110 } 1111 1112 /* Make descriptor updates visible to hardware */ 1113 wmb(); 1114 1115 return 1; 1116 } 1117 1118 offset = 0; 1119 len += NET_IP_ALIGN; 1120 skb_checksum_none_assert(skb); 1121 skb_put(skb, len); 1122 1123 for (frag = first_frag; ; frag++) { 1124 unsigned int frag_len = bp->rx_buffer_size; 1125 1126 if (offset + frag_len > len) { 1127 if (unlikely(frag != last_frag)) { 1128 dev_kfree_skb_any(skb); 1129 return -1; 1130 } 1131 frag_len = len - offset; 1132 } 1133 skb_copy_to_linear_data_offset(skb, offset, 1134 macb_rx_buffer(queue, frag), 1135 frag_len); 1136 offset += bp->rx_buffer_size; 1137 desc = macb_rx_desc(queue, frag); 1138 desc->addr &= ~MACB_BIT(RX_USED); 1139 1140 if (frag == last_frag) 1141 break; 1142 } 1143 1144 /* Make descriptor updates visible to hardware */ 1145 wmb(); 1146 1147 __skb_pull(skb, NET_IP_ALIGN); 1148 skb->protocol = eth_type_trans(skb, bp->dev); 1149 1150 bp->dev->stats.rx_packets++; 1151 bp->dev->stats.rx_bytes += skb->len; 1152 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1153 skb->len, skb->csum); 1154 netif_receive_skb(skb); 1155 1156 return 0; 1157 } 1158 1159 static inline void macb_init_rx_ring(struct macb_queue *queue) 1160 { 1161 struct macb *bp = queue->bp; 1162 dma_addr_t addr; 1163 struct macb_dma_desc *desc = NULL; 1164 int i; 1165 1166 addr = queue->rx_buffers_dma; 1167 for (i = 0; i < bp->rx_ring_size; i++) { 1168 desc = macb_rx_desc(queue, i); 1169 macb_set_addr(bp, desc, addr); 1170 desc->ctrl = 0; 1171 addr += bp->rx_buffer_size; 1172 } 1173 desc->addr |= MACB_BIT(RX_WRAP); 1174 queue->rx_tail = 0; 1175 } 1176 1177 static int macb_rx(struct macb_queue *queue, int budget) 1178 { 1179 struct macb *bp = queue->bp; 1180 bool reset_rx_queue = false; 1181 int received = 0; 1182 unsigned int tail; 1183 int first_frag = -1; 1184 1185 for (tail = queue->rx_tail; budget > 0; tail++) { 1186 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1187 u32 ctrl; 1188 1189 /* Make hw descriptor updates visible to CPU */ 1190 rmb(); 1191 1192 if (!(desc->addr & MACB_BIT(RX_USED))) 1193 break; 1194 1195 /* Ensure ctrl is at least as up-to-date as addr */ 1196 dma_rmb(); 1197 1198 ctrl = desc->ctrl; 1199 1200 if (ctrl & MACB_BIT(RX_SOF)) { 1201 if (first_frag != -1) 1202 discard_partial_frame(queue, first_frag, tail); 1203 first_frag = tail; 1204 } 1205 1206 if (ctrl & MACB_BIT(RX_EOF)) { 1207 int dropped; 1208 1209 if (unlikely(first_frag == -1)) { 1210 reset_rx_queue = true; 1211 continue; 1212 } 1213 1214 dropped = macb_rx_frame(queue, first_frag, tail); 1215 first_frag = -1; 1216 if (unlikely(dropped < 0)) { 1217 reset_rx_queue = true; 1218 continue; 1219 } 1220 if (!dropped) { 1221 received++; 1222 budget--; 1223 } 1224 } 1225 } 1226 1227 if (unlikely(reset_rx_queue)) { 1228 unsigned long flags; 1229 u32 ctrl; 1230 1231 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1232 1233 spin_lock_irqsave(&bp->lock, flags); 1234 1235 ctrl = macb_readl(bp, NCR); 1236 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1237 1238 macb_init_rx_ring(queue); 1239 queue_writel(queue, RBQP, queue->rx_ring_dma); 1240 1241 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1242 1243 spin_unlock_irqrestore(&bp->lock, flags); 1244 return received; 1245 } 1246 1247 if (first_frag != -1) 1248 queue->rx_tail = first_frag; 1249 else 1250 queue->rx_tail = tail; 1251 1252 return received; 1253 } 1254 1255 static int macb_poll(struct napi_struct *napi, int budget) 1256 { 1257 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1258 struct macb *bp = queue->bp; 1259 int work_done; 1260 u32 status; 1261 1262 status = macb_readl(bp, RSR); 1263 macb_writel(bp, RSR, status); 1264 1265 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1266 (unsigned long)status, budget); 1267 1268 work_done = bp->macbgem_ops.mog_rx(queue, budget); 1269 if (work_done < budget) { 1270 napi_complete_done(napi, work_done); 1271 1272 /* Packets received while interrupts were disabled */ 1273 status = macb_readl(bp, RSR); 1274 if (status) { 1275 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1276 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1277 napi_reschedule(napi); 1278 } else { 1279 queue_writel(queue, IER, bp->rx_intr_mask); 1280 } 1281 } 1282 1283 /* TODO: Handle errors */ 1284 1285 return work_done; 1286 } 1287 1288 static void macb_hresp_error_task(unsigned long data) 1289 { 1290 struct macb *bp = (struct macb *)data; 1291 struct net_device *dev = bp->dev; 1292 struct macb_queue *queue = bp->queues; 1293 unsigned int q; 1294 u32 ctrl; 1295 1296 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1297 queue_writel(queue, IDR, bp->rx_intr_mask | 1298 MACB_TX_INT_FLAGS | 1299 MACB_BIT(HRESP)); 1300 } 1301 ctrl = macb_readl(bp, NCR); 1302 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1303 macb_writel(bp, NCR, ctrl); 1304 1305 netif_tx_stop_all_queues(dev); 1306 netif_carrier_off(dev); 1307 1308 bp->macbgem_ops.mog_init_rings(bp); 1309 1310 /* Initialize TX and RX buffers */ 1311 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1312 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 1313 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1314 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1315 queue_writel(queue, RBQPH, 1316 upper_32_bits(queue->rx_ring_dma)); 1317 #endif 1318 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1319 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1320 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1321 queue_writel(queue, TBQPH, 1322 upper_32_bits(queue->tx_ring_dma)); 1323 #endif 1324 1325 /* Enable interrupts */ 1326 queue_writel(queue, IER, 1327 bp->rx_intr_mask | 1328 MACB_TX_INT_FLAGS | 1329 MACB_BIT(HRESP)); 1330 } 1331 1332 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1333 macb_writel(bp, NCR, ctrl); 1334 1335 netif_carrier_on(dev); 1336 netif_tx_start_all_queues(dev); 1337 } 1338 1339 static void macb_tx_restart(struct macb_queue *queue) 1340 { 1341 unsigned int head = queue->tx_head; 1342 unsigned int tail = queue->tx_tail; 1343 struct macb *bp = queue->bp; 1344 1345 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1346 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1347 1348 if (head == tail) 1349 return; 1350 1351 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1352 } 1353 1354 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1355 { 1356 struct macb_queue *queue = dev_id; 1357 struct macb *bp = queue->bp; 1358 struct net_device *dev = bp->dev; 1359 u32 status, ctrl; 1360 1361 status = queue_readl(queue, ISR); 1362 1363 if (unlikely(!status)) 1364 return IRQ_NONE; 1365 1366 spin_lock(&bp->lock); 1367 1368 while (status) { 1369 /* close possible race with dev_close */ 1370 if (unlikely(!netif_running(dev))) { 1371 queue_writel(queue, IDR, -1); 1372 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1373 queue_writel(queue, ISR, -1); 1374 break; 1375 } 1376 1377 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1378 (unsigned int)(queue - bp->queues), 1379 (unsigned long)status); 1380 1381 if (status & bp->rx_intr_mask) { 1382 /* There's no point taking any more interrupts 1383 * until we have processed the buffers. The 1384 * scheduling call may fail if the poll routine 1385 * is already scheduled, so disable interrupts 1386 * now. 1387 */ 1388 queue_writel(queue, IDR, bp->rx_intr_mask); 1389 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1390 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1391 1392 if (napi_schedule_prep(&queue->napi)) { 1393 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1394 __napi_schedule(&queue->napi); 1395 } 1396 } 1397 1398 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1399 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1400 schedule_work(&queue->tx_error_task); 1401 1402 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1403 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1404 1405 break; 1406 } 1407 1408 if (status & MACB_BIT(TCOMP)) 1409 macb_tx_interrupt(queue); 1410 1411 if (status & MACB_BIT(TXUBR)) 1412 macb_tx_restart(queue); 1413 1414 /* Link change detection isn't possible with RMII, so we'll 1415 * add that if/when we get our hands on a full-blown MII PHY. 1416 */ 1417 1418 /* There is a hardware issue under heavy load where DMA can 1419 * stop, this causes endless "used buffer descriptor read" 1420 * interrupts but it can be cleared by re-enabling RX. See 1421 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1422 * section 16.7.4 for details. RXUBR is only enabled for 1423 * these two versions. 1424 */ 1425 if (status & MACB_BIT(RXUBR)) { 1426 ctrl = macb_readl(bp, NCR); 1427 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1428 wmb(); 1429 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1430 1431 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1432 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1433 } 1434 1435 if (status & MACB_BIT(ISR_ROVR)) { 1436 /* We missed at least one packet */ 1437 if (macb_is_gem(bp)) 1438 bp->hw_stats.gem.rx_overruns++; 1439 else 1440 bp->hw_stats.macb.rx_overruns++; 1441 1442 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1443 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1444 } 1445 1446 if (status & MACB_BIT(HRESP)) { 1447 tasklet_schedule(&bp->hresp_err_tasklet); 1448 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1449 1450 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1451 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1452 } 1453 status = queue_readl(queue, ISR); 1454 } 1455 1456 spin_unlock(&bp->lock); 1457 1458 return IRQ_HANDLED; 1459 } 1460 1461 #ifdef CONFIG_NET_POLL_CONTROLLER 1462 /* Polling receive - used by netconsole and other diagnostic tools 1463 * to allow network i/o with interrupts disabled. 1464 */ 1465 static void macb_poll_controller(struct net_device *dev) 1466 { 1467 struct macb *bp = netdev_priv(dev); 1468 struct macb_queue *queue; 1469 unsigned long flags; 1470 unsigned int q; 1471 1472 local_irq_save(flags); 1473 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1474 macb_interrupt(dev->irq, queue); 1475 local_irq_restore(flags); 1476 } 1477 #endif 1478 1479 static unsigned int macb_tx_map(struct macb *bp, 1480 struct macb_queue *queue, 1481 struct sk_buff *skb, 1482 unsigned int hdrlen) 1483 { 1484 dma_addr_t mapping; 1485 unsigned int len, entry, i, tx_head = queue->tx_head; 1486 struct macb_tx_skb *tx_skb = NULL; 1487 struct macb_dma_desc *desc; 1488 unsigned int offset, size, count = 0; 1489 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1490 unsigned int eof = 1, mss_mfs = 0; 1491 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1492 1493 /* LSO */ 1494 if (skb_shinfo(skb)->gso_size != 0) { 1495 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1496 /* UDP - UFO */ 1497 lso_ctrl = MACB_LSO_UFO_ENABLE; 1498 else 1499 /* TCP - TSO */ 1500 lso_ctrl = MACB_LSO_TSO_ENABLE; 1501 } 1502 1503 /* First, map non-paged data */ 1504 len = skb_headlen(skb); 1505 1506 /* first buffer length */ 1507 size = hdrlen; 1508 1509 offset = 0; 1510 while (len) { 1511 entry = macb_tx_ring_wrap(bp, tx_head); 1512 tx_skb = &queue->tx_skb[entry]; 1513 1514 mapping = dma_map_single(&bp->pdev->dev, 1515 skb->data + offset, 1516 size, DMA_TO_DEVICE); 1517 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1518 goto dma_error; 1519 1520 /* Save info to properly release resources */ 1521 tx_skb->skb = NULL; 1522 tx_skb->mapping = mapping; 1523 tx_skb->size = size; 1524 tx_skb->mapped_as_page = false; 1525 1526 len -= size; 1527 offset += size; 1528 count++; 1529 tx_head++; 1530 1531 size = min(len, bp->max_tx_length); 1532 } 1533 1534 /* Then, map paged data from fragments */ 1535 for (f = 0; f < nr_frags; f++) { 1536 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1537 1538 len = skb_frag_size(frag); 1539 offset = 0; 1540 while (len) { 1541 size = min(len, bp->max_tx_length); 1542 entry = macb_tx_ring_wrap(bp, tx_head); 1543 tx_skb = &queue->tx_skb[entry]; 1544 1545 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1546 offset, size, DMA_TO_DEVICE); 1547 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1548 goto dma_error; 1549 1550 /* Save info to properly release resources */ 1551 tx_skb->skb = NULL; 1552 tx_skb->mapping = mapping; 1553 tx_skb->size = size; 1554 tx_skb->mapped_as_page = true; 1555 1556 len -= size; 1557 offset += size; 1558 count++; 1559 tx_head++; 1560 } 1561 } 1562 1563 /* Should never happen */ 1564 if (unlikely(!tx_skb)) { 1565 netdev_err(bp->dev, "BUG! empty skb!\n"); 1566 return 0; 1567 } 1568 1569 /* This is the last buffer of the frame: save socket buffer */ 1570 tx_skb->skb = skb; 1571 1572 /* Update TX ring: update buffer descriptors in reverse order 1573 * to avoid race condition 1574 */ 1575 1576 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1577 * to set the end of TX queue 1578 */ 1579 i = tx_head; 1580 entry = macb_tx_ring_wrap(bp, i); 1581 ctrl = MACB_BIT(TX_USED); 1582 desc = macb_tx_desc(queue, entry); 1583 desc->ctrl = ctrl; 1584 1585 if (lso_ctrl) { 1586 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1587 /* include header and FCS in value given to h/w */ 1588 mss_mfs = skb_shinfo(skb)->gso_size + 1589 skb_transport_offset(skb) + 1590 ETH_FCS_LEN; 1591 else /* TSO */ { 1592 mss_mfs = skb_shinfo(skb)->gso_size; 1593 /* TCP Sequence Number Source Select 1594 * can be set only for TSO 1595 */ 1596 seq_ctrl = 0; 1597 } 1598 } 1599 1600 do { 1601 i--; 1602 entry = macb_tx_ring_wrap(bp, i); 1603 tx_skb = &queue->tx_skb[entry]; 1604 desc = macb_tx_desc(queue, entry); 1605 1606 ctrl = (u32)tx_skb->size; 1607 if (eof) { 1608 ctrl |= MACB_BIT(TX_LAST); 1609 eof = 0; 1610 } 1611 if (unlikely(entry == (bp->tx_ring_size - 1))) 1612 ctrl |= MACB_BIT(TX_WRAP); 1613 1614 /* First descriptor is header descriptor */ 1615 if (i == queue->tx_head) { 1616 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1617 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1618 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1619 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1620 ctrl |= MACB_BIT(TX_NOCRC); 1621 } else 1622 /* Only set MSS/MFS on payload descriptors 1623 * (second or later descriptor) 1624 */ 1625 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1626 1627 /* Set TX buffer descriptor */ 1628 macb_set_addr(bp, desc, tx_skb->mapping); 1629 /* desc->addr must be visible to hardware before clearing 1630 * 'TX_USED' bit in desc->ctrl. 1631 */ 1632 wmb(); 1633 desc->ctrl = ctrl; 1634 } while (i != queue->tx_head); 1635 1636 queue->tx_head = tx_head; 1637 1638 return count; 1639 1640 dma_error: 1641 netdev_err(bp->dev, "TX DMA map failed\n"); 1642 1643 for (i = queue->tx_head; i != tx_head; i++) { 1644 tx_skb = macb_tx_skb(queue, i); 1645 1646 macb_tx_unmap(bp, tx_skb); 1647 } 1648 1649 return 0; 1650 } 1651 1652 static netdev_features_t macb_features_check(struct sk_buff *skb, 1653 struct net_device *dev, 1654 netdev_features_t features) 1655 { 1656 unsigned int nr_frags, f; 1657 unsigned int hdrlen; 1658 1659 /* Validate LSO compatibility */ 1660 1661 /* there is only one buffer */ 1662 if (!skb_is_nonlinear(skb)) 1663 return features; 1664 1665 /* length of header */ 1666 hdrlen = skb_transport_offset(skb); 1667 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1668 hdrlen += tcp_hdrlen(skb); 1669 1670 /* For LSO: 1671 * When software supplies two or more payload buffers all payload buffers 1672 * apart from the last must be a multiple of 8 bytes in size. 1673 */ 1674 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1675 return features & ~MACB_NETIF_LSO; 1676 1677 nr_frags = skb_shinfo(skb)->nr_frags; 1678 /* No need to check last fragment */ 1679 nr_frags--; 1680 for (f = 0; f < nr_frags; f++) { 1681 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1682 1683 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1684 return features & ~MACB_NETIF_LSO; 1685 } 1686 return features; 1687 } 1688 1689 static inline int macb_clear_csum(struct sk_buff *skb) 1690 { 1691 /* no change for packets without checksum offloading */ 1692 if (skb->ip_summed != CHECKSUM_PARTIAL) 1693 return 0; 1694 1695 /* make sure we can modify the header */ 1696 if (unlikely(skb_cow_head(skb, 0))) 1697 return -1; 1698 1699 /* initialize checksum field 1700 * This is required - at least for Zynq, which otherwise calculates 1701 * wrong UDP header checksums for UDP packets with UDP data len <=2 1702 */ 1703 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1704 return 0; 1705 } 1706 1707 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1708 { 1709 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1710 int padlen = ETH_ZLEN - (*skb)->len; 1711 int headroom = skb_headroom(*skb); 1712 int tailroom = skb_tailroom(*skb); 1713 struct sk_buff *nskb; 1714 u32 fcs; 1715 1716 if (!(ndev->features & NETIF_F_HW_CSUM) || 1717 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1718 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1719 return 0; 1720 1721 if (padlen <= 0) { 1722 /* FCS could be appeded to tailroom. */ 1723 if (tailroom >= ETH_FCS_LEN) 1724 goto add_fcs; 1725 /* FCS could be appeded by moving data to headroom. */ 1726 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1727 padlen = 0; 1728 /* No room for FCS, need to reallocate skb. */ 1729 else 1730 padlen = ETH_FCS_LEN; 1731 } else { 1732 /* Add room for FCS. */ 1733 padlen += ETH_FCS_LEN; 1734 } 1735 1736 if (!cloned && headroom + tailroom >= padlen) { 1737 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1738 skb_set_tail_pointer(*skb, (*skb)->len); 1739 } else { 1740 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1741 if (!nskb) 1742 return -ENOMEM; 1743 1744 dev_consume_skb_any(*skb); 1745 *skb = nskb; 1746 } 1747 1748 if (padlen > ETH_FCS_LEN) 1749 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1750 1751 add_fcs: 1752 /* set FCS to packet */ 1753 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1754 fcs = ~fcs; 1755 1756 skb_put_u8(*skb, fcs & 0xff); 1757 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1758 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1759 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1760 1761 return 0; 1762 } 1763 1764 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1765 { 1766 u16 queue_index = skb_get_queue_mapping(skb); 1767 struct macb *bp = netdev_priv(dev); 1768 struct macb_queue *queue = &bp->queues[queue_index]; 1769 unsigned long flags; 1770 unsigned int desc_cnt, nr_frags, frag_size, f; 1771 unsigned int hdrlen; 1772 bool is_lso, is_udp = 0; 1773 netdev_tx_t ret = NETDEV_TX_OK; 1774 1775 if (macb_clear_csum(skb)) { 1776 dev_kfree_skb_any(skb); 1777 return ret; 1778 } 1779 1780 if (macb_pad_and_fcs(&skb, dev)) { 1781 dev_kfree_skb_any(skb); 1782 return ret; 1783 } 1784 1785 is_lso = (skb_shinfo(skb)->gso_size != 0); 1786 1787 if (is_lso) { 1788 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1789 1790 /* length of headers */ 1791 if (is_udp) 1792 /* only queue eth + ip headers separately for UDP */ 1793 hdrlen = skb_transport_offset(skb); 1794 else 1795 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1796 if (skb_headlen(skb) < hdrlen) { 1797 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1798 /* if this is required, would need to copy to single buffer */ 1799 return NETDEV_TX_BUSY; 1800 } 1801 } else 1802 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1803 1804 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1805 netdev_vdbg(bp->dev, 1806 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1807 queue_index, skb->len, skb->head, skb->data, 1808 skb_tail_pointer(skb), skb_end_pointer(skb)); 1809 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1810 skb->data, 16, true); 1811 #endif 1812 1813 /* Count how many TX buffer descriptors are needed to send this 1814 * socket buffer: skb fragments of jumbo frames may need to be 1815 * split into many buffer descriptors. 1816 */ 1817 if (is_lso && (skb_headlen(skb) > hdrlen)) 1818 /* extra header descriptor if also payload in first buffer */ 1819 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1820 else 1821 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1822 nr_frags = skb_shinfo(skb)->nr_frags; 1823 for (f = 0; f < nr_frags; f++) { 1824 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1825 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1826 } 1827 1828 spin_lock_irqsave(&bp->lock, flags); 1829 1830 /* This is a hard error, log it. */ 1831 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1832 bp->tx_ring_size) < desc_cnt) { 1833 netif_stop_subqueue(dev, queue_index); 1834 spin_unlock_irqrestore(&bp->lock, flags); 1835 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1836 queue->tx_head, queue->tx_tail); 1837 return NETDEV_TX_BUSY; 1838 } 1839 1840 /* Map socket buffer for DMA transfer */ 1841 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1842 dev_kfree_skb_any(skb); 1843 goto unlock; 1844 } 1845 1846 /* Make newly initialized descriptor visible to hardware */ 1847 wmb(); 1848 skb_tx_timestamp(skb); 1849 1850 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1851 1852 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1853 netif_stop_subqueue(dev, queue_index); 1854 1855 unlock: 1856 spin_unlock_irqrestore(&bp->lock, flags); 1857 1858 return ret; 1859 } 1860 1861 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1862 { 1863 if (!macb_is_gem(bp)) { 1864 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1865 } else { 1866 bp->rx_buffer_size = size; 1867 1868 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1869 netdev_dbg(bp->dev, 1870 "RX buffer must be multiple of %d bytes, expanding\n", 1871 RX_BUFFER_MULTIPLE); 1872 bp->rx_buffer_size = 1873 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1874 } 1875 } 1876 1877 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1878 bp->dev->mtu, bp->rx_buffer_size); 1879 } 1880 1881 static void gem_free_rx_buffers(struct macb *bp) 1882 { 1883 struct sk_buff *skb; 1884 struct macb_dma_desc *desc; 1885 struct macb_queue *queue; 1886 dma_addr_t addr; 1887 unsigned int q; 1888 int i; 1889 1890 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1891 if (!queue->rx_skbuff) 1892 continue; 1893 1894 for (i = 0; i < bp->rx_ring_size; i++) { 1895 skb = queue->rx_skbuff[i]; 1896 1897 if (!skb) 1898 continue; 1899 1900 desc = macb_rx_desc(queue, i); 1901 addr = macb_get_addr(bp, desc); 1902 1903 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1904 DMA_FROM_DEVICE); 1905 dev_kfree_skb_any(skb); 1906 skb = NULL; 1907 } 1908 1909 kfree(queue->rx_skbuff); 1910 queue->rx_skbuff = NULL; 1911 } 1912 } 1913 1914 static void macb_free_rx_buffers(struct macb *bp) 1915 { 1916 struct macb_queue *queue = &bp->queues[0]; 1917 1918 if (queue->rx_buffers) { 1919 dma_free_coherent(&bp->pdev->dev, 1920 bp->rx_ring_size * bp->rx_buffer_size, 1921 queue->rx_buffers, queue->rx_buffers_dma); 1922 queue->rx_buffers = NULL; 1923 } 1924 } 1925 1926 static void macb_free_consistent(struct macb *bp) 1927 { 1928 struct macb_queue *queue; 1929 unsigned int q; 1930 int size; 1931 1932 bp->macbgem_ops.mog_free_rx_buffers(bp); 1933 1934 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1935 kfree(queue->tx_skb); 1936 queue->tx_skb = NULL; 1937 if (queue->tx_ring) { 1938 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1939 dma_free_coherent(&bp->pdev->dev, size, 1940 queue->tx_ring, queue->tx_ring_dma); 1941 queue->tx_ring = NULL; 1942 } 1943 if (queue->rx_ring) { 1944 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 1945 dma_free_coherent(&bp->pdev->dev, size, 1946 queue->rx_ring, queue->rx_ring_dma); 1947 queue->rx_ring = NULL; 1948 } 1949 } 1950 } 1951 1952 static int gem_alloc_rx_buffers(struct macb *bp) 1953 { 1954 struct macb_queue *queue; 1955 unsigned int q; 1956 int size; 1957 1958 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1959 size = bp->rx_ring_size * sizeof(struct sk_buff *); 1960 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 1961 if (!queue->rx_skbuff) 1962 return -ENOMEM; 1963 else 1964 netdev_dbg(bp->dev, 1965 "Allocated %d RX struct sk_buff entries at %p\n", 1966 bp->rx_ring_size, queue->rx_skbuff); 1967 } 1968 return 0; 1969 } 1970 1971 static int macb_alloc_rx_buffers(struct macb *bp) 1972 { 1973 struct macb_queue *queue = &bp->queues[0]; 1974 int size; 1975 1976 size = bp->rx_ring_size * bp->rx_buffer_size; 1977 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 1978 &queue->rx_buffers_dma, GFP_KERNEL); 1979 if (!queue->rx_buffers) 1980 return -ENOMEM; 1981 1982 netdev_dbg(bp->dev, 1983 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 1984 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 1985 return 0; 1986 } 1987 1988 static int macb_alloc_consistent(struct macb *bp) 1989 { 1990 struct macb_queue *queue; 1991 unsigned int q; 1992 int size; 1993 1994 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1995 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1996 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1997 &queue->tx_ring_dma, 1998 GFP_KERNEL); 1999 if (!queue->tx_ring) 2000 goto out_err; 2001 netdev_dbg(bp->dev, 2002 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2003 q, size, (unsigned long)queue->tx_ring_dma, 2004 queue->tx_ring); 2005 2006 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2007 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2008 if (!queue->tx_skb) 2009 goto out_err; 2010 2011 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2012 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2013 &queue->rx_ring_dma, GFP_KERNEL); 2014 if (!queue->rx_ring) 2015 goto out_err; 2016 netdev_dbg(bp->dev, 2017 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2018 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2019 } 2020 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2021 goto out_err; 2022 2023 return 0; 2024 2025 out_err: 2026 macb_free_consistent(bp); 2027 return -ENOMEM; 2028 } 2029 2030 static void gem_init_rings(struct macb *bp) 2031 { 2032 struct macb_queue *queue; 2033 struct macb_dma_desc *desc = NULL; 2034 unsigned int q; 2035 int i; 2036 2037 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2038 for (i = 0; i < bp->tx_ring_size; i++) { 2039 desc = macb_tx_desc(queue, i); 2040 macb_set_addr(bp, desc, 0); 2041 desc->ctrl = MACB_BIT(TX_USED); 2042 } 2043 desc->ctrl |= MACB_BIT(TX_WRAP); 2044 queue->tx_head = 0; 2045 queue->tx_tail = 0; 2046 2047 queue->rx_tail = 0; 2048 queue->rx_prepared_head = 0; 2049 2050 gem_rx_refill(queue); 2051 } 2052 2053 } 2054 2055 static void macb_init_rings(struct macb *bp) 2056 { 2057 int i; 2058 struct macb_dma_desc *desc = NULL; 2059 2060 macb_init_rx_ring(&bp->queues[0]); 2061 2062 for (i = 0; i < bp->tx_ring_size; i++) { 2063 desc = macb_tx_desc(&bp->queues[0], i); 2064 macb_set_addr(bp, desc, 0); 2065 desc->ctrl = MACB_BIT(TX_USED); 2066 } 2067 bp->queues[0].tx_head = 0; 2068 bp->queues[0].tx_tail = 0; 2069 desc->ctrl |= MACB_BIT(TX_WRAP); 2070 } 2071 2072 static void macb_reset_hw(struct macb *bp) 2073 { 2074 struct macb_queue *queue; 2075 unsigned int q; 2076 u32 ctrl = macb_readl(bp, NCR); 2077 2078 /* Disable RX and TX (XXX: Should we halt the transmission 2079 * more gracefully?) 2080 */ 2081 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2082 2083 /* Clear the stats registers (XXX: Update stats first?) */ 2084 ctrl |= MACB_BIT(CLRSTAT); 2085 2086 macb_writel(bp, NCR, ctrl); 2087 2088 /* Clear all status flags */ 2089 macb_writel(bp, TSR, -1); 2090 macb_writel(bp, RSR, -1); 2091 2092 /* Disable all interrupts */ 2093 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2094 queue_writel(queue, IDR, -1); 2095 queue_readl(queue, ISR); 2096 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2097 queue_writel(queue, ISR, -1); 2098 } 2099 } 2100 2101 static u32 gem_mdc_clk_div(struct macb *bp) 2102 { 2103 u32 config; 2104 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2105 2106 if (pclk_hz <= 20000000) 2107 config = GEM_BF(CLK, GEM_CLK_DIV8); 2108 else if (pclk_hz <= 40000000) 2109 config = GEM_BF(CLK, GEM_CLK_DIV16); 2110 else if (pclk_hz <= 80000000) 2111 config = GEM_BF(CLK, GEM_CLK_DIV32); 2112 else if (pclk_hz <= 120000000) 2113 config = GEM_BF(CLK, GEM_CLK_DIV48); 2114 else if (pclk_hz <= 160000000) 2115 config = GEM_BF(CLK, GEM_CLK_DIV64); 2116 else 2117 config = GEM_BF(CLK, GEM_CLK_DIV96); 2118 2119 return config; 2120 } 2121 2122 static u32 macb_mdc_clk_div(struct macb *bp) 2123 { 2124 u32 config; 2125 unsigned long pclk_hz; 2126 2127 if (macb_is_gem(bp)) 2128 return gem_mdc_clk_div(bp); 2129 2130 pclk_hz = clk_get_rate(bp->pclk); 2131 if (pclk_hz <= 20000000) 2132 config = MACB_BF(CLK, MACB_CLK_DIV8); 2133 else if (pclk_hz <= 40000000) 2134 config = MACB_BF(CLK, MACB_CLK_DIV16); 2135 else if (pclk_hz <= 80000000) 2136 config = MACB_BF(CLK, MACB_CLK_DIV32); 2137 else 2138 config = MACB_BF(CLK, MACB_CLK_DIV64); 2139 2140 return config; 2141 } 2142 2143 /* Get the DMA bus width field of the network configuration register that we 2144 * should program. We find the width from decoding the design configuration 2145 * register to find the maximum supported data bus width. 2146 */ 2147 static u32 macb_dbw(struct macb *bp) 2148 { 2149 if (!macb_is_gem(bp)) 2150 return 0; 2151 2152 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2153 case 4: 2154 return GEM_BF(DBW, GEM_DBW128); 2155 case 2: 2156 return GEM_BF(DBW, GEM_DBW64); 2157 case 1: 2158 default: 2159 return GEM_BF(DBW, GEM_DBW32); 2160 } 2161 } 2162 2163 /* Configure the receive DMA engine 2164 * - use the correct receive buffer size 2165 * - set best burst length for DMA operations 2166 * (if not supported by FIFO, it will fallback to default) 2167 * - set both rx/tx packet buffers to full memory size 2168 * These are configurable parameters for GEM. 2169 */ 2170 static void macb_configure_dma(struct macb *bp) 2171 { 2172 struct macb_queue *queue; 2173 u32 buffer_size; 2174 unsigned int q; 2175 u32 dmacfg; 2176 2177 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2178 if (macb_is_gem(bp)) { 2179 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2180 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2181 if (q) 2182 queue_writel(queue, RBQS, buffer_size); 2183 else 2184 dmacfg |= GEM_BF(RXBS, buffer_size); 2185 } 2186 if (bp->dma_burst_length) 2187 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2188 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2189 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2190 2191 if (bp->native_io) 2192 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2193 else 2194 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2195 2196 if (bp->dev->features & NETIF_F_HW_CSUM) 2197 dmacfg |= GEM_BIT(TXCOEN); 2198 else 2199 dmacfg &= ~GEM_BIT(TXCOEN); 2200 2201 dmacfg &= ~GEM_BIT(ADDR64); 2202 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2203 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2204 dmacfg |= GEM_BIT(ADDR64); 2205 #endif 2206 #ifdef CONFIG_MACB_USE_HWSTAMP 2207 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2208 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2209 #endif 2210 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2211 dmacfg); 2212 gem_writel(bp, DMACFG, dmacfg); 2213 } 2214 } 2215 2216 static void macb_init_hw(struct macb *bp) 2217 { 2218 struct macb_queue *queue; 2219 unsigned int q; 2220 2221 u32 config; 2222 2223 macb_reset_hw(bp); 2224 macb_set_hwaddr(bp); 2225 2226 config = macb_mdc_clk_div(bp); 2227 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2228 config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2229 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2230 config |= MACB_BIT(PAE); /* PAuse Enable */ 2231 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2232 if (bp->caps & MACB_CAPS_JUMBO) 2233 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2234 else 2235 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2236 if (bp->dev->flags & IFF_PROMISC) 2237 config |= MACB_BIT(CAF); /* Copy All Frames */ 2238 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2239 config |= GEM_BIT(RXCOEN); 2240 if (!(bp->dev->flags & IFF_BROADCAST)) 2241 config |= MACB_BIT(NBC); /* No BroadCast */ 2242 config |= macb_dbw(bp); 2243 macb_writel(bp, NCFGR, config); 2244 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2245 gem_writel(bp, JML, bp->jumbo_max_len); 2246 bp->speed = SPEED_10; 2247 bp->duplex = DUPLEX_HALF; 2248 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2249 if (bp->caps & MACB_CAPS_JUMBO) 2250 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2251 2252 macb_configure_dma(bp); 2253 2254 /* Initialize TX and RX buffers */ 2255 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2256 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 2257 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2258 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2259 queue_writel(queue, RBQPH, upper_32_bits(queue->rx_ring_dma)); 2260 #endif 2261 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 2262 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2263 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2264 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 2265 #endif 2266 2267 /* Enable interrupts */ 2268 queue_writel(queue, IER, 2269 bp->rx_intr_mask | 2270 MACB_TX_INT_FLAGS | 2271 MACB_BIT(HRESP)); 2272 } 2273 2274 /* Enable TX and RX */ 2275 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 2276 } 2277 2278 /* The hash address register is 64 bits long and takes up two 2279 * locations in the memory map. The least significant bits are stored 2280 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2281 * 2282 * The unicast hash enable and the multicast hash enable bits in the 2283 * network configuration register enable the reception of hash matched 2284 * frames. The destination address is reduced to a 6 bit index into 2285 * the 64 bit hash register using the following hash function. The 2286 * hash function is an exclusive or of every sixth bit of the 2287 * destination address. 2288 * 2289 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2290 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2291 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2292 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2293 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2294 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2295 * 2296 * da[0] represents the least significant bit of the first byte 2297 * received, that is, the multicast/unicast indicator, and da[47] 2298 * represents the most significant bit of the last byte received. If 2299 * the hash index, hi[n], points to a bit that is set in the hash 2300 * register then the frame will be matched according to whether the 2301 * frame is multicast or unicast. A multicast match will be signalled 2302 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2303 * index points to a bit set in the hash register. A unicast match 2304 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2305 * and the hash index points to a bit set in the hash register. To 2306 * receive all multicast frames, the hash register should be set with 2307 * all ones and the multicast hash enable bit should be set in the 2308 * network configuration register. 2309 */ 2310 2311 static inline int hash_bit_value(int bitnr, __u8 *addr) 2312 { 2313 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2314 return 1; 2315 return 0; 2316 } 2317 2318 /* Return the hash index value for the specified address. */ 2319 static int hash_get_index(__u8 *addr) 2320 { 2321 int i, j, bitval; 2322 int hash_index = 0; 2323 2324 for (j = 0; j < 6; j++) { 2325 for (i = 0, bitval = 0; i < 8; i++) 2326 bitval ^= hash_bit_value(i * 6 + j, addr); 2327 2328 hash_index |= (bitval << j); 2329 } 2330 2331 return hash_index; 2332 } 2333 2334 /* Add multicast addresses to the internal multicast-hash table. */ 2335 static void macb_sethashtable(struct net_device *dev) 2336 { 2337 struct netdev_hw_addr *ha; 2338 unsigned long mc_filter[2]; 2339 unsigned int bitnr; 2340 struct macb *bp = netdev_priv(dev); 2341 2342 mc_filter[0] = 0; 2343 mc_filter[1] = 0; 2344 2345 netdev_for_each_mc_addr(ha, dev) { 2346 bitnr = hash_get_index(ha->addr); 2347 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2348 } 2349 2350 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2351 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2352 } 2353 2354 /* Enable/Disable promiscuous and multicast modes. */ 2355 static void macb_set_rx_mode(struct net_device *dev) 2356 { 2357 unsigned long cfg; 2358 struct macb *bp = netdev_priv(dev); 2359 2360 cfg = macb_readl(bp, NCFGR); 2361 2362 if (dev->flags & IFF_PROMISC) { 2363 /* Enable promiscuous mode */ 2364 cfg |= MACB_BIT(CAF); 2365 2366 /* Disable RX checksum offload */ 2367 if (macb_is_gem(bp)) 2368 cfg &= ~GEM_BIT(RXCOEN); 2369 } else { 2370 /* Disable promiscuous mode */ 2371 cfg &= ~MACB_BIT(CAF); 2372 2373 /* Enable RX checksum offload only if requested */ 2374 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2375 cfg |= GEM_BIT(RXCOEN); 2376 } 2377 2378 if (dev->flags & IFF_ALLMULTI) { 2379 /* Enable all multicast mode */ 2380 macb_or_gem_writel(bp, HRB, -1); 2381 macb_or_gem_writel(bp, HRT, -1); 2382 cfg |= MACB_BIT(NCFGR_MTI); 2383 } else if (!netdev_mc_empty(dev)) { 2384 /* Enable specific multicasts */ 2385 macb_sethashtable(dev); 2386 cfg |= MACB_BIT(NCFGR_MTI); 2387 } else if (dev->flags & (~IFF_ALLMULTI)) { 2388 /* Disable all multicast mode */ 2389 macb_or_gem_writel(bp, HRB, 0); 2390 macb_or_gem_writel(bp, HRT, 0); 2391 cfg &= ~MACB_BIT(NCFGR_MTI); 2392 } 2393 2394 macb_writel(bp, NCFGR, cfg); 2395 } 2396 2397 static int macb_open(struct net_device *dev) 2398 { 2399 struct macb *bp = netdev_priv(dev); 2400 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2401 struct macb_queue *queue; 2402 unsigned int q; 2403 int err; 2404 2405 netdev_dbg(bp->dev, "open\n"); 2406 2407 err = pm_runtime_get_sync(&bp->pdev->dev); 2408 if (err < 0) 2409 goto pm_exit; 2410 2411 /* carrier starts down */ 2412 netif_carrier_off(dev); 2413 2414 /* if the phy is not yet register, retry later*/ 2415 if (!dev->phydev) { 2416 err = -EAGAIN; 2417 goto pm_exit; 2418 } 2419 2420 /* RX buffers initialization */ 2421 macb_init_rx_buffer_size(bp, bufsz); 2422 2423 err = macb_alloc_consistent(bp); 2424 if (err) { 2425 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2426 err); 2427 goto pm_exit; 2428 } 2429 2430 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2431 napi_enable(&queue->napi); 2432 2433 bp->macbgem_ops.mog_init_rings(bp); 2434 macb_init_hw(bp); 2435 2436 /* schedule a link state check */ 2437 phy_start(dev->phydev); 2438 2439 netif_tx_start_all_queues(dev); 2440 2441 if (bp->ptp_info) 2442 bp->ptp_info->ptp_init(dev); 2443 2444 pm_exit: 2445 if (err) { 2446 pm_runtime_put_sync(&bp->pdev->dev); 2447 return err; 2448 } 2449 return 0; 2450 } 2451 2452 static int macb_close(struct net_device *dev) 2453 { 2454 struct macb *bp = netdev_priv(dev); 2455 struct macb_queue *queue; 2456 unsigned long flags; 2457 unsigned int q; 2458 2459 netif_tx_stop_all_queues(dev); 2460 2461 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2462 napi_disable(&queue->napi); 2463 2464 if (dev->phydev) 2465 phy_stop(dev->phydev); 2466 2467 spin_lock_irqsave(&bp->lock, flags); 2468 macb_reset_hw(bp); 2469 netif_carrier_off(dev); 2470 spin_unlock_irqrestore(&bp->lock, flags); 2471 2472 macb_free_consistent(bp); 2473 2474 if (bp->ptp_info) 2475 bp->ptp_info->ptp_remove(dev); 2476 2477 pm_runtime_put(&bp->pdev->dev); 2478 2479 return 0; 2480 } 2481 2482 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2483 { 2484 if (netif_running(dev)) 2485 return -EBUSY; 2486 2487 dev->mtu = new_mtu; 2488 2489 return 0; 2490 } 2491 2492 static void gem_update_stats(struct macb *bp) 2493 { 2494 struct macb_queue *queue; 2495 unsigned int i, q, idx; 2496 unsigned long *stat; 2497 2498 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2499 2500 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2501 u32 offset = gem_statistics[i].offset; 2502 u64 val = bp->macb_reg_readl(bp, offset); 2503 2504 bp->ethtool_stats[i] += val; 2505 *p += val; 2506 2507 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2508 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2509 val = bp->macb_reg_readl(bp, offset + 4); 2510 bp->ethtool_stats[i] += ((u64)val) << 32; 2511 *(++p) += val; 2512 } 2513 } 2514 2515 idx = GEM_STATS_LEN; 2516 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2517 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2518 bp->ethtool_stats[idx++] = *stat; 2519 } 2520 2521 static struct net_device_stats *gem_get_stats(struct macb *bp) 2522 { 2523 struct gem_stats *hwstat = &bp->hw_stats.gem; 2524 struct net_device_stats *nstat = &bp->dev->stats; 2525 2526 gem_update_stats(bp); 2527 2528 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2529 hwstat->rx_alignment_errors + 2530 hwstat->rx_resource_errors + 2531 hwstat->rx_overruns + 2532 hwstat->rx_oversize_frames + 2533 hwstat->rx_jabbers + 2534 hwstat->rx_undersized_frames + 2535 hwstat->rx_length_field_frame_errors); 2536 nstat->tx_errors = (hwstat->tx_late_collisions + 2537 hwstat->tx_excessive_collisions + 2538 hwstat->tx_underrun + 2539 hwstat->tx_carrier_sense_errors); 2540 nstat->multicast = hwstat->rx_multicast_frames; 2541 nstat->collisions = (hwstat->tx_single_collision_frames + 2542 hwstat->tx_multiple_collision_frames + 2543 hwstat->tx_excessive_collisions); 2544 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2545 hwstat->rx_jabbers + 2546 hwstat->rx_undersized_frames + 2547 hwstat->rx_length_field_frame_errors); 2548 nstat->rx_over_errors = hwstat->rx_resource_errors; 2549 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2550 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2551 nstat->rx_fifo_errors = hwstat->rx_overruns; 2552 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2553 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2554 nstat->tx_fifo_errors = hwstat->tx_underrun; 2555 2556 return nstat; 2557 } 2558 2559 static void gem_get_ethtool_stats(struct net_device *dev, 2560 struct ethtool_stats *stats, u64 *data) 2561 { 2562 struct macb *bp; 2563 2564 bp = netdev_priv(dev); 2565 gem_update_stats(bp); 2566 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2567 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2568 } 2569 2570 static int gem_get_sset_count(struct net_device *dev, int sset) 2571 { 2572 struct macb *bp = netdev_priv(dev); 2573 2574 switch (sset) { 2575 case ETH_SS_STATS: 2576 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2577 default: 2578 return -EOPNOTSUPP; 2579 } 2580 } 2581 2582 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2583 { 2584 char stat_string[ETH_GSTRING_LEN]; 2585 struct macb *bp = netdev_priv(dev); 2586 struct macb_queue *queue; 2587 unsigned int i; 2588 unsigned int q; 2589 2590 switch (sset) { 2591 case ETH_SS_STATS: 2592 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2593 memcpy(p, gem_statistics[i].stat_string, 2594 ETH_GSTRING_LEN); 2595 2596 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2597 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2598 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2599 q, queue_statistics[i].stat_string); 2600 memcpy(p, stat_string, ETH_GSTRING_LEN); 2601 } 2602 } 2603 break; 2604 } 2605 } 2606 2607 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2608 { 2609 struct macb *bp = netdev_priv(dev); 2610 struct net_device_stats *nstat = &bp->dev->stats; 2611 struct macb_stats *hwstat = &bp->hw_stats.macb; 2612 2613 if (macb_is_gem(bp)) 2614 return gem_get_stats(bp); 2615 2616 /* read stats from hardware */ 2617 macb_update_stats(bp); 2618 2619 /* Convert HW stats into netdevice stats */ 2620 nstat->rx_errors = (hwstat->rx_fcs_errors + 2621 hwstat->rx_align_errors + 2622 hwstat->rx_resource_errors + 2623 hwstat->rx_overruns + 2624 hwstat->rx_oversize_pkts + 2625 hwstat->rx_jabbers + 2626 hwstat->rx_undersize_pkts + 2627 hwstat->rx_length_mismatch); 2628 nstat->tx_errors = (hwstat->tx_late_cols + 2629 hwstat->tx_excessive_cols + 2630 hwstat->tx_underruns + 2631 hwstat->tx_carrier_errors + 2632 hwstat->sqe_test_errors); 2633 nstat->collisions = (hwstat->tx_single_cols + 2634 hwstat->tx_multiple_cols + 2635 hwstat->tx_excessive_cols); 2636 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2637 hwstat->rx_jabbers + 2638 hwstat->rx_undersize_pkts + 2639 hwstat->rx_length_mismatch); 2640 nstat->rx_over_errors = hwstat->rx_resource_errors + 2641 hwstat->rx_overruns; 2642 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2643 nstat->rx_frame_errors = hwstat->rx_align_errors; 2644 nstat->rx_fifo_errors = hwstat->rx_overruns; 2645 /* XXX: What does "missed" mean? */ 2646 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2647 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2648 nstat->tx_fifo_errors = hwstat->tx_underruns; 2649 /* Don't know about heartbeat or window errors... */ 2650 2651 return nstat; 2652 } 2653 2654 static int macb_get_regs_len(struct net_device *netdev) 2655 { 2656 return MACB_GREGS_NBR * sizeof(u32); 2657 } 2658 2659 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2660 void *p) 2661 { 2662 struct macb *bp = netdev_priv(dev); 2663 unsigned int tail, head; 2664 u32 *regs_buff = p; 2665 2666 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2667 | MACB_GREGS_VERSION; 2668 2669 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2670 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2671 2672 regs_buff[0] = macb_readl(bp, NCR); 2673 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2674 regs_buff[2] = macb_readl(bp, NSR); 2675 regs_buff[3] = macb_readl(bp, TSR); 2676 regs_buff[4] = macb_readl(bp, RBQP); 2677 regs_buff[5] = macb_readl(bp, TBQP); 2678 regs_buff[6] = macb_readl(bp, RSR); 2679 regs_buff[7] = macb_readl(bp, IMR); 2680 2681 regs_buff[8] = tail; 2682 regs_buff[9] = head; 2683 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2684 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2685 2686 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2687 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2688 if (macb_is_gem(bp)) 2689 regs_buff[13] = gem_readl(bp, DMACFG); 2690 } 2691 2692 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2693 { 2694 struct macb *bp = netdev_priv(netdev); 2695 2696 wol->supported = 0; 2697 wol->wolopts = 0; 2698 2699 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2700 wol->supported = WAKE_MAGIC; 2701 2702 if (bp->wol & MACB_WOL_ENABLED) 2703 wol->wolopts |= WAKE_MAGIC; 2704 } 2705 } 2706 2707 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2708 { 2709 struct macb *bp = netdev_priv(netdev); 2710 2711 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2712 (wol->wolopts & ~WAKE_MAGIC)) 2713 return -EOPNOTSUPP; 2714 2715 if (wol->wolopts & WAKE_MAGIC) 2716 bp->wol |= MACB_WOL_ENABLED; 2717 else 2718 bp->wol &= ~MACB_WOL_ENABLED; 2719 2720 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2721 2722 return 0; 2723 } 2724 2725 static void macb_get_ringparam(struct net_device *netdev, 2726 struct ethtool_ringparam *ring) 2727 { 2728 struct macb *bp = netdev_priv(netdev); 2729 2730 ring->rx_max_pending = MAX_RX_RING_SIZE; 2731 ring->tx_max_pending = MAX_TX_RING_SIZE; 2732 2733 ring->rx_pending = bp->rx_ring_size; 2734 ring->tx_pending = bp->tx_ring_size; 2735 } 2736 2737 static int macb_set_ringparam(struct net_device *netdev, 2738 struct ethtool_ringparam *ring) 2739 { 2740 struct macb *bp = netdev_priv(netdev); 2741 u32 new_rx_size, new_tx_size; 2742 unsigned int reset = 0; 2743 2744 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2745 return -EINVAL; 2746 2747 new_rx_size = clamp_t(u32, ring->rx_pending, 2748 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2749 new_rx_size = roundup_pow_of_two(new_rx_size); 2750 2751 new_tx_size = clamp_t(u32, ring->tx_pending, 2752 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2753 new_tx_size = roundup_pow_of_two(new_tx_size); 2754 2755 if ((new_tx_size == bp->tx_ring_size) && 2756 (new_rx_size == bp->rx_ring_size)) { 2757 /* nothing to do */ 2758 return 0; 2759 } 2760 2761 if (netif_running(bp->dev)) { 2762 reset = 1; 2763 macb_close(bp->dev); 2764 } 2765 2766 bp->rx_ring_size = new_rx_size; 2767 bp->tx_ring_size = new_tx_size; 2768 2769 if (reset) 2770 macb_open(bp->dev); 2771 2772 return 0; 2773 } 2774 2775 #ifdef CONFIG_MACB_USE_HWSTAMP 2776 static unsigned int gem_get_tsu_rate(struct macb *bp) 2777 { 2778 struct clk *tsu_clk; 2779 unsigned int tsu_rate; 2780 2781 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2782 if (!IS_ERR(tsu_clk)) 2783 tsu_rate = clk_get_rate(tsu_clk); 2784 /* try pclk instead */ 2785 else if (!IS_ERR(bp->pclk)) { 2786 tsu_clk = bp->pclk; 2787 tsu_rate = clk_get_rate(tsu_clk); 2788 } else 2789 return -ENOTSUPP; 2790 return tsu_rate; 2791 } 2792 2793 static s32 gem_get_ptp_max_adj(void) 2794 { 2795 return 64000000; 2796 } 2797 2798 static int gem_get_ts_info(struct net_device *dev, 2799 struct ethtool_ts_info *info) 2800 { 2801 struct macb *bp = netdev_priv(dev); 2802 2803 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2804 ethtool_op_get_ts_info(dev, info); 2805 return 0; 2806 } 2807 2808 info->so_timestamping = 2809 SOF_TIMESTAMPING_TX_SOFTWARE | 2810 SOF_TIMESTAMPING_RX_SOFTWARE | 2811 SOF_TIMESTAMPING_SOFTWARE | 2812 SOF_TIMESTAMPING_TX_HARDWARE | 2813 SOF_TIMESTAMPING_RX_HARDWARE | 2814 SOF_TIMESTAMPING_RAW_HARDWARE; 2815 info->tx_types = 2816 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2817 (1 << HWTSTAMP_TX_OFF) | 2818 (1 << HWTSTAMP_TX_ON); 2819 info->rx_filters = 2820 (1 << HWTSTAMP_FILTER_NONE) | 2821 (1 << HWTSTAMP_FILTER_ALL); 2822 2823 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2824 2825 return 0; 2826 } 2827 2828 static struct macb_ptp_info gem_ptp_info = { 2829 .ptp_init = gem_ptp_init, 2830 .ptp_remove = gem_ptp_remove, 2831 .get_ptp_max_adj = gem_get_ptp_max_adj, 2832 .get_tsu_rate = gem_get_tsu_rate, 2833 .get_ts_info = gem_get_ts_info, 2834 .get_hwtst = gem_get_hwtst, 2835 .set_hwtst = gem_set_hwtst, 2836 }; 2837 #endif 2838 2839 static int macb_get_ts_info(struct net_device *netdev, 2840 struct ethtool_ts_info *info) 2841 { 2842 struct macb *bp = netdev_priv(netdev); 2843 2844 if (bp->ptp_info) 2845 return bp->ptp_info->get_ts_info(netdev, info); 2846 2847 return ethtool_op_get_ts_info(netdev, info); 2848 } 2849 2850 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2851 { 2852 struct ethtool_rx_fs_item *item; 2853 u32 t2_scr; 2854 int num_t2_scr; 2855 2856 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2857 2858 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2859 struct ethtool_rx_flow_spec *fs = &item->fs; 2860 struct ethtool_tcpip4_spec *tp4sp_m; 2861 2862 if (fs->location >= num_t2_scr) 2863 continue; 2864 2865 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2866 2867 /* enable/disable screener regs for the flow entry */ 2868 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2869 2870 /* only enable fields with no masking */ 2871 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2872 2873 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2874 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2875 else 2876 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2877 2878 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2879 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2880 else 2881 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2882 2883 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2884 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2885 else 2886 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 2887 2888 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 2889 } 2890 } 2891 2892 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 2893 { 2894 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 2895 uint16_t index = fs->location; 2896 u32 w0, w1, t2_scr; 2897 bool cmp_a = false; 2898 bool cmp_b = false; 2899 bool cmp_c = false; 2900 2901 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 2902 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2903 2904 /* ignore field if any masking set */ 2905 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 2906 /* 1st compare reg - IP source address */ 2907 w0 = 0; 2908 w1 = 0; 2909 w0 = tp4sp_v->ip4src; 2910 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2911 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2912 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 2913 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 2914 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 2915 cmp_a = true; 2916 } 2917 2918 /* ignore field if any masking set */ 2919 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 2920 /* 2nd compare reg - IP destination address */ 2921 w0 = 0; 2922 w1 = 0; 2923 w0 = tp4sp_v->ip4dst; 2924 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2925 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2926 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 2927 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 2928 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 2929 cmp_b = true; 2930 } 2931 2932 /* ignore both port fields if masking set in both */ 2933 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 2934 /* 3rd compare reg - source port, destination port */ 2935 w0 = 0; 2936 w1 = 0; 2937 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 2938 if (tp4sp_m->psrc == tp4sp_m->pdst) { 2939 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 2940 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2941 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2942 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2943 } else { 2944 /* only one port definition */ 2945 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 2946 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 2947 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 2948 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 2949 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2950 } else { /* dst port */ 2951 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2952 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 2953 } 2954 } 2955 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 2956 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 2957 cmp_c = true; 2958 } 2959 2960 t2_scr = 0; 2961 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 2962 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 2963 if (cmp_a) 2964 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 2965 if (cmp_b) 2966 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 2967 if (cmp_c) 2968 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 2969 gem_writel_n(bp, SCRT2, index, t2_scr); 2970 } 2971 2972 static int gem_add_flow_filter(struct net_device *netdev, 2973 struct ethtool_rxnfc *cmd) 2974 { 2975 struct macb *bp = netdev_priv(netdev); 2976 struct ethtool_rx_flow_spec *fs = &cmd->fs; 2977 struct ethtool_rx_fs_item *item, *newfs; 2978 unsigned long flags; 2979 int ret = -EINVAL; 2980 bool added = false; 2981 2982 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 2983 if (newfs == NULL) 2984 return -ENOMEM; 2985 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 2986 2987 netdev_dbg(netdev, 2988 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2989 fs->flow_type, (int)fs->ring_cookie, fs->location, 2990 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2991 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2992 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 2993 2994 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2995 2996 /* find correct place to add in list */ 2997 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2998 if (item->fs.location > newfs->fs.location) { 2999 list_add_tail(&newfs->list, &item->list); 3000 added = true; 3001 break; 3002 } else if (item->fs.location == fs->location) { 3003 netdev_err(netdev, "Rule not added: location %d not free!\n", 3004 fs->location); 3005 ret = -EBUSY; 3006 goto err; 3007 } 3008 } 3009 if (!added) 3010 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3011 3012 gem_prog_cmp_regs(bp, fs); 3013 bp->rx_fs_list.count++; 3014 /* enable filtering if NTUPLE on */ 3015 if (netdev->features & NETIF_F_NTUPLE) 3016 gem_enable_flow_filters(bp, 1); 3017 3018 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3019 return 0; 3020 3021 err: 3022 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3023 kfree(newfs); 3024 return ret; 3025 } 3026 3027 static int gem_del_flow_filter(struct net_device *netdev, 3028 struct ethtool_rxnfc *cmd) 3029 { 3030 struct macb *bp = netdev_priv(netdev); 3031 struct ethtool_rx_fs_item *item; 3032 struct ethtool_rx_flow_spec *fs; 3033 unsigned long flags; 3034 3035 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3036 3037 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3038 if (item->fs.location == cmd->fs.location) { 3039 /* disable screener regs for the flow entry */ 3040 fs = &(item->fs); 3041 netdev_dbg(netdev, 3042 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3043 fs->flow_type, (int)fs->ring_cookie, fs->location, 3044 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3045 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3046 htons(fs->h_u.tcp_ip4_spec.psrc), 3047 htons(fs->h_u.tcp_ip4_spec.pdst)); 3048 3049 gem_writel_n(bp, SCRT2, fs->location, 0); 3050 3051 list_del(&item->list); 3052 bp->rx_fs_list.count--; 3053 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3054 kfree(item); 3055 return 0; 3056 } 3057 } 3058 3059 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3060 return -EINVAL; 3061 } 3062 3063 static int gem_get_flow_entry(struct net_device *netdev, 3064 struct ethtool_rxnfc *cmd) 3065 { 3066 struct macb *bp = netdev_priv(netdev); 3067 struct ethtool_rx_fs_item *item; 3068 3069 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3070 if (item->fs.location == cmd->fs.location) { 3071 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3072 return 0; 3073 } 3074 } 3075 return -EINVAL; 3076 } 3077 3078 static int gem_get_all_flow_entries(struct net_device *netdev, 3079 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3080 { 3081 struct macb *bp = netdev_priv(netdev); 3082 struct ethtool_rx_fs_item *item; 3083 uint32_t cnt = 0; 3084 3085 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3086 if (cnt == cmd->rule_cnt) 3087 return -EMSGSIZE; 3088 rule_locs[cnt] = item->fs.location; 3089 cnt++; 3090 } 3091 cmd->data = bp->max_tuples; 3092 cmd->rule_cnt = cnt; 3093 3094 return 0; 3095 } 3096 3097 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3098 u32 *rule_locs) 3099 { 3100 struct macb *bp = netdev_priv(netdev); 3101 int ret = 0; 3102 3103 switch (cmd->cmd) { 3104 case ETHTOOL_GRXRINGS: 3105 cmd->data = bp->num_queues; 3106 break; 3107 case ETHTOOL_GRXCLSRLCNT: 3108 cmd->rule_cnt = bp->rx_fs_list.count; 3109 break; 3110 case ETHTOOL_GRXCLSRULE: 3111 ret = gem_get_flow_entry(netdev, cmd); 3112 break; 3113 case ETHTOOL_GRXCLSRLALL: 3114 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3115 break; 3116 default: 3117 netdev_err(netdev, 3118 "Command parameter %d is not supported\n", cmd->cmd); 3119 ret = -EOPNOTSUPP; 3120 } 3121 3122 return ret; 3123 } 3124 3125 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3126 { 3127 struct macb *bp = netdev_priv(netdev); 3128 int ret; 3129 3130 switch (cmd->cmd) { 3131 case ETHTOOL_SRXCLSRLINS: 3132 if ((cmd->fs.location >= bp->max_tuples) 3133 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3134 ret = -EINVAL; 3135 break; 3136 } 3137 ret = gem_add_flow_filter(netdev, cmd); 3138 break; 3139 case ETHTOOL_SRXCLSRLDEL: 3140 ret = gem_del_flow_filter(netdev, cmd); 3141 break; 3142 default: 3143 netdev_err(netdev, 3144 "Command parameter %d is not supported\n", cmd->cmd); 3145 ret = -EOPNOTSUPP; 3146 } 3147 3148 return ret; 3149 } 3150 3151 static const struct ethtool_ops macb_ethtool_ops = { 3152 .get_regs_len = macb_get_regs_len, 3153 .get_regs = macb_get_regs, 3154 .get_link = ethtool_op_get_link, 3155 .get_ts_info = ethtool_op_get_ts_info, 3156 .get_wol = macb_get_wol, 3157 .set_wol = macb_set_wol, 3158 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3159 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3160 .get_ringparam = macb_get_ringparam, 3161 .set_ringparam = macb_set_ringparam, 3162 }; 3163 3164 static const struct ethtool_ops gem_ethtool_ops = { 3165 .get_regs_len = macb_get_regs_len, 3166 .get_regs = macb_get_regs, 3167 .get_link = ethtool_op_get_link, 3168 .get_ts_info = macb_get_ts_info, 3169 .get_ethtool_stats = gem_get_ethtool_stats, 3170 .get_strings = gem_get_ethtool_strings, 3171 .get_sset_count = gem_get_sset_count, 3172 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3173 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3174 .get_ringparam = macb_get_ringparam, 3175 .set_ringparam = macb_set_ringparam, 3176 .get_rxnfc = gem_get_rxnfc, 3177 .set_rxnfc = gem_set_rxnfc, 3178 }; 3179 3180 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3181 { 3182 struct phy_device *phydev = dev->phydev; 3183 struct macb *bp = netdev_priv(dev); 3184 3185 if (!netif_running(dev)) 3186 return -EINVAL; 3187 3188 if (!phydev) 3189 return -ENODEV; 3190 3191 if (!bp->ptp_info) 3192 return phy_mii_ioctl(phydev, rq, cmd); 3193 3194 switch (cmd) { 3195 case SIOCSHWTSTAMP: 3196 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3197 case SIOCGHWTSTAMP: 3198 return bp->ptp_info->get_hwtst(dev, rq); 3199 default: 3200 return phy_mii_ioctl(phydev, rq, cmd); 3201 } 3202 } 3203 3204 static int macb_set_features(struct net_device *netdev, 3205 netdev_features_t features) 3206 { 3207 struct macb *bp = netdev_priv(netdev); 3208 netdev_features_t changed = features ^ netdev->features; 3209 3210 /* TX checksum offload */ 3211 if ((changed & NETIF_F_HW_CSUM) && macb_is_gem(bp)) { 3212 u32 dmacfg; 3213 3214 dmacfg = gem_readl(bp, DMACFG); 3215 if (features & NETIF_F_HW_CSUM) 3216 dmacfg |= GEM_BIT(TXCOEN); 3217 else 3218 dmacfg &= ~GEM_BIT(TXCOEN); 3219 gem_writel(bp, DMACFG, dmacfg); 3220 } 3221 3222 /* RX checksum offload */ 3223 if ((changed & NETIF_F_RXCSUM) && macb_is_gem(bp)) { 3224 u32 netcfg; 3225 3226 netcfg = gem_readl(bp, NCFGR); 3227 if (features & NETIF_F_RXCSUM && 3228 !(netdev->flags & IFF_PROMISC)) 3229 netcfg |= GEM_BIT(RXCOEN); 3230 else 3231 netcfg &= ~GEM_BIT(RXCOEN); 3232 gem_writel(bp, NCFGR, netcfg); 3233 } 3234 3235 /* RX Flow Filters */ 3236 if ((changed & NETIF_F_NTUPLE) && macb_is_gem(bp)) { 3237 bool turn_on = features & NETIF_F_NTUPLE; 3238 3239 gem_enable_flow_filters(bp, turn_on); 3240 } 3241 return 0; 3242 } 3243 3244 static const struct net_device_ops macb_netdev_ops = { 3245 .ndo_open = macb_open, 3246 .ndo_stop = macb_close, 3247 .ndo_start_xmit = macb_start_xmit, 3248 .ndo_set_rx_mode = macb_set_rx_mode, 3249 .ndo_get_stats = macb_get_stats, 3250 .ndo_do_ioctl = macb_ioctl, 3251 .ndo_validate_addr = eth_validate_addr, 3252 .ndo_change_mtu = macb_change_mtu, 3253 .ndo_set_mac_address = eth_mac_addr, 3254 #ifdef CONFIG_NET_POLL_CONTROLLER 3255 .ndo_poll_controller = macb_poll_controller, 3256 #endif 3257 .ndo_set_features = macb_set_features, 3258 .ndo_features_check = macb_features_check, 3259 }; 3260 3261 /* Configure peripheral capabilities according to device tree 3262 * and integration options used 3263 */ 3264 static void macb_configure_caps(struct macb *bp, 3265 const struct macb_config *dt_conf) 3266 { 3267 u32 dcfg; 3268 3269 if (dt_conf) 3270 bp->caps = dt_conf->caps; 3271 3272 if (hw_is_gem(bp->regs, bp->native_io)) { 3273 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3274 3275 dcfg = gem_readl(bp, DCFG1); 3276 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3277 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3278 dcfg = gem_readl(bp, DCFG2); 3279 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3280 bp->caps |= MACB_CAPS_FIFO_MODE; 3281 #ifdef CONFIG_MACB_USE_HWSTAMP 3282 if (gem_has_ptp(bp)) { 3283 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3284 pr_err("GEM doesn't support hardware ptp.\n"); 3285 else { 3286 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3287 bp->ptp_info = &gem_ptp_info; 3288 } 3289 } 3290 #endif 3291 } 3292 3293 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3294 } 3295 3296 static void macb_probe_queues(void __iomem *mem, 3297 bool native_io, 3298 unsigned int *queue_mask, 3299 unsigned int *num_queues) 3300 { 3301 unsigned int hw_q; 3302 3303 *queue_mask = 0x1; 3304 *num_queues = 1; 3305 3306 /* is it macb or gem ? 3307 * 3308 * We need to read directly from the hardware here because 3309 * we are early in the probe process and don't have the 3310 * MACB_CAPS_MACB_IS_GEM flag positioned 3311 */ 3312 if (!hw_is_gem(mem, native_io)) 3313 return; 3314 3315 /* bit 0 is never set but queue 0 always exists */ 3316 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3317 3318 *queue_mask |= 0x1; 3319 3320 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3321 if (*queue_mask & (1 << hw_q)) 3322 (*num_queues)++; 3323 } 3324 3325 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3326 struct clk **hclk, struct clk **tx_clk, 3327 struct clk **rx_clk, struct clk **tsu_clk) 3328 { 3329 struct macb_platform_data *pdata; 3330 int err; 3331 3332 pdata = dev_get_platdata(&pdev->dev); 3333 if (pdata) { 3334 *pclk = pdata->pclk; 3335 *hclk = pdata->hclk; 3336 } else { 3337 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3338 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3339 } 3340 3341 if (IS_ERR_OR_NULL(*pclk)) { 3342 err = PTR_ERR(*pclk); 3343 if (!err) 3344 err = -ENODEV; 3345 3346 dev_err(&pdev->dev, "failed to get macb_clk (%u)\n", err); 3347 return err; 3348 } 3349 3350 if (IS_ERR_OR_NULL(*hclk)) { 3351 err = PTR_ERR(*hclk); 3352 if (!err) 3353 err = -ENODEV; 3354 3355 dev_err(&pdev->dev, "failed to get hclk (%u)\n", err); 3356 return err; 3357 } 3358 3359 *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); 3360 if (IS_ERR(*tx_clk)) 3361 *tx_clk = NULL; 3362 3363 *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); 3364 if (IS_ERR(*rx_clk)) 3365 *rx_clk = NULL; 3366 3367 *tsu_clk = devm_clk_get(&pdev->dev, "tsu_clk"); 3368 if (IS_ERR(*tsu_clk)) 3369 *tsu_clk = NULL; 3370 3371 err = clk_prepare_enable(*pclk); 3372 if (err) { 3373 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3374 return err; 3375 } 3376 3377 err = clk_prepare_enable(*hclk); 3378 if (err) { 3379 dev_err(&pdev->dev, "failed to enable hclk (%u)\n", err); 3380 goto err_disable_pclk; 3381 } 3382 3383 err = clk_prepare_enable(*tx_clk); 3384 if (err) { 3385 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 3386 goto err_disable_hclk; 3387 } 3388 3389 err = clk_prepare_enable(*rx_clk); 3390 if (err) { 3391 dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); 3392 goto err_disable_txclk; 3393 } 3394 3395 err = clk_prepare_enable(*tsu_clk); 3396 if (err) { 3397 dev_err(&pdev->dev, "failed to enable tsu_clk (%u)\n", err); 3398 goto err_disable_rxclk; 3399 } 3400 3401 return 0; 3402 3403 err_disable_rxclk: 3404 clk_disable_unprepare(*rx_clk); 3405 3406 err_disable_txclk: 3407 clk_disable_unprepare(*tx_clk); 3408 3409 err_disable_hclk: 3410 clk_disable_unprepare(*hclk); 3411 3412 err_disable_pclk: 3413 clk_disable_unprepare(*pclk); 3414 3415 return err; 3416 } 3417 3418 static int macb_init(struct platform_device *pdev) 3419 { 3420 struct net_device *dev = platform_get_drvdata(pdev); 3421 unsigned int hw_q, q; 3422 struct macb *bp = netdev_priv(dev); 3423 struct macb_queue *queue; 3424 int err; 3425 u32 val, reg; 3426 3427 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3428 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3429 3430 /* set the queue register mapping once for all: queue0 has a special 3431 * register mapping but we don't want to test the queue index then 3432 * compute the corresponding register offset at run time. 3433 */ 3434 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3435 if (!(bp->queue_mask & (1 << hw_q))) 3436 continue; 3437 3438 queue = &bp->queues[q]; 3439 queue->bp = bp; 3440 netif_napi_add(dev, &queue->napi, macb_poll, 64); 3441 if (hw_q) { 3442 queue->ISR = GEM_ISR(hw_q - 1); 3443 queue->IER = GEM_IER(hw_q - 1); 3444 queue->IDR = GEM_IDR(hw_q - 1); 3445 queue->IMR = GEM_IMR(hw_q - 1); 3446 queue->TBQP = GEM_TBQP(hw_q - 1); 3447 queue->RBQP = GEM_RBQP(hw_q - 1); 3448 queue->RBQS = GEM_RBQS(hw_q - 1); 3449 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3450 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3451 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3452 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3453 } 3454 #endif 3455 } else { 3456 /* queue0 uses legacy registers */ 3457 queue->ISR = MACB_ISR; 3458 queue->IER = MACB_IER; 3459 queue->IDR = MACB_IDR; 3460 queue->IMR = MACB_IMR; 3461 queue->TBQP = MACB_TBQP; 3462 queue->RBQP = MACB_RBQP; 3463 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3464 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3465 queue->TBQPH = MACB_TBQPH; 3466 queue->RBQPH = MACB_RBQPH; 3467 } 3468 #endif 3469 } 3470 3471 /* get irq: here we use the linux queue index, not the hardware 3472 * queue index. the queue irq definitions in the device tree 3473 * must remove the optional gaps that could exist in the 3474 * hardware queue mask. 3475 */ 3476 queue->irq = platform_get_irq(pdev, q); 3477 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3478 IRQF_SHARED, dev->name, queue); 3479 if (err) { 3480 dev_err(&pdev->dev, 3481 "Unable to request IRQ %d (error %d)\n", 3482 queue->irq, err); 3483 return err; 3484 } 3485 3486 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3487 q++; 3488 } 3489 3490 dev->netdev_ops = &macb_netdev_ops; 3491 3492 /* setup appropriated routines according to adapter type */ 3493 if (macb_is_gem(bp)) { 3494 bp->max_tx_length = GEM_MAX_TX_LEN; 3495 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3496 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3497 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3498 bp->macbgem_ops.mog_rx = gem_rx; 3499 dev->ethtool_ops = &gem_ethtool_ops; 3500 } else { 3501 bp->max_tx_length = MACB_MAX_TX_LEN; 3502 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3503 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3504 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3505 bp->macbgem_ops.mog_rx = macb_rx; 3506 dev->ethtool_ops = &macb_ethtool_ops; 3507 } 3508 3509 /* Set features */ 3510 dev->hw_features = NETIF_F_SG; 3511 3512 /* Check LSO capability */ 3513 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3514 dev->hw_features |= MACB_NETIF_LSO; 3515 3516 /* Checksum offload is only available on gem with packet buffer */ 3517 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3518 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3519 if (bp->caps & MACB_CAPS_SG_DISABLED) 3520 dev->hw_features &= ~NETIF_F_SG; 3521 dev->features = dev->hw_features; 3522 3523 /* Check RX Flow Filters support. 3524 * Max Rx flows set by availability of screeners & compare regs: 3525 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3526 */ 3527 reg = gem_readl(bp, DCFG8); 3528 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3529 GEM_BFEXT(T2SCR, reg)); 3530 if (bp->max_tuples > 0) { 3531 /* also needs one ethtype match to check IPv4 */ 3532 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3533 /* program this reg now */ 3534 reg = 0; 3535 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3536 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3537 /* Filtering is supported in hw but don't enable it in kernel now */ 3538 dev->hw_features |= NETIF_F_NTUPLE; 3539 /* init Rx flow definitions */ 3540 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3541 bp->rx_fs_list.count = 0; 3542 spin_lock_init(&bp->rx_fs_lock); 3543 } else 3544 bp->max_tuples = 0; 3545 } 3546 3547 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3548 val = 0; 3549 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3550 val = GEM_BIT(RGMII); 3551 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3552 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3553 val = MACB_BIT(RMII); 3554 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3555 val = MACB_BIT(MII); 3556 3557 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3558 val |= MACB_BIT(CLKEN); 3559 3560 macb_or_gem_writel(bp, USRIO, val); 3561 } 3562 3563 /* Set MII management clock divider */ 3564 val = macb_mdc_clk_div(bp); 3565 val |= macb_dbw(bp); 3566 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3567 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3568 macb_writel(bp, NCFGR, val); 3569 3570 return 0; 3571 } 3572 3573 #if defined(CONFIG_OF) 3574 /* 1518 rounded up */ 3575 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3576 /* max number of receive buffers */ 3577 #define AT91ETHER_MAX_RX_DESCR 9 3578 3579 /* Initialize and start the Receiver and Transmit subsystems */ 3580 static int at91ether_start(struct net_device *dev) 3581 { 3582 struct macb *lp = netdev_priv(dev); 3583 struct macb_queue *q = &lp->queues[0]; 3584 struct macb_dma_desc *desc; 3585 dma_addr_t addr; 3586 u32 ctl; 3587 int i; 3588 3589 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3590 (AT91ETHER_MAX_RX_DESCR * 3591 macb_dma_desc_get_size(lp)), 3592 &q->rx_ring_dma, GFP_KERNEL); 3593 if (!q->rx_ring) 3594 return -ENOMEM; 3595 3596 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3597 AT91ETHER_MAX_RX_DESCR * 3598 AT91ETHER_MAX_RBUFF_SZ, 3599 &q->rx_buffers_dma, GFP_KERNEL); 3600 if (!q->rx_buffers) { 3601 dma_free_coherent(&lp->pdev->dev, 3602 AT91ETHER_MAX_RX_DESCR * 3603 macb_dma_desc_get_size(lp), 3604 q->rx_ring, q->rx_ring_dma); 3605 q->rx_ring = NULL; 3606 return -ENOMEM; 3607 } 3608 3609 addr = q->rx_buffers_dma; 3610 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3611 desc = macb_rx_desc(q, i); 3612 macb_set_addr(lp, desc, addr); 3613 desc->ctrl = 0; 3614 addr += AT91ETHER_MAX_RBUFF_SZ; 3615 } 3616 3617 /* Set the Wrap bit on the last descriptor */ 3618 desc->addr |= MACB_BIT(RX_WRAP); 3619 3620 /* Reset buffer index */ 3621 q->rx_tail = 0; 3622 3623 /* Program address of descriptor list in Rx Buffer Queue register */ 3624 macb_writel(lp, RBQP, q->rx_ring_dma); 3625 3626 /* Enable Receive and Transmit */ 3627 ctl = macb_readl(lp, NCR); 3628 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3629 3630 return 0; 3631 } 3632 3633 /* Open the ethernet interface */ 3634 static int at91ether_open(struct net_device *dev) 3635 { 3636 struct macb *lp = netdev_priv(dev); 3637 u32 ctl; 3638 int ret; 3639 3640 /* Clear internal statistics */ 3641 ctl = macb_readl(lp, NCR); 3642 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3643 3644 macb_set_hwaddr(lp); 3645 3646 ret = at91ether_start(dev); 3647 if (ret) 3648 return ret; 3649 3650 /* Enable MAC interrupts */ 3651 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3652 MACB_BIT(RXUBR) | 3653 MACB_BIT(ISR_TUND) | 3654 MACB_BIT(ISR_RLE) | 3655 MACB_BIT(TCOMP) | 3656 MACB_BIT(ISR_ROVR) | 3657 MACB_BIT(HRESP)); 3658 3659 /* schedule a link state check */ 3660 phy_start(dev->phydev); 3661 3662 netif_start_queue(dev); 3663 3664 return 0; 3665 } 3666 3667 /* Close the interface */ 3668 static int at91ether_close(struct net_device *dev) 3669 { 3670 struct macb *lp = netdev_priv(dev); 3671 struct macb_queue *q = &lp->queues[0]; 3672 u32 ctl; 3673 3674 /* Disable Receiver and Transmitter */ 3675 ctl = macb_readl(lp, NCR); 3676 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3677 3678 /* Disable MAC interrupts */ 3679 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3680 MACB_BIT(RXUBR) | 3681 MACB_BIT(ISR_TUND) | 3682 MACB_BIT(ISR_RLE) | 3683 MACB_BIT(TCOMP) | 3684 MACB_BIT(ISR_ROVR) | 3685 MACB_BIT(HRESP)); 3686 3687 netif_stop_queue(dev); 3688 3689 dma_free_coherent(&lp->pdev->dev, 3690 AT91ETHER_MAX_RX_DESCR * 3691 macb_dma_desc_get_size(lp), 3692 q->rx_ring, q->rx_ring_dma); 3693 q->rx_ring = NULL; 3694 3695 dma_free_coherent(&lp->pdev->dev, 3696 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3697 q->rx_buffers, q->rx_buffers_dma); 3698 q->rx_buffers = NULL; 3699 3700 return 0; 3701 } 3702 3703 /* Transmit packet */ 3704 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3705 struct net_device *dev) 3706 { 3707 struct macb *lp = netdev_priv(dev); 3708 3709 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3710 netif_stop_queue(dev); 3711 3712 /* Store packet information (to free when Tx completed) */ 3713 lp->skb = skb; 3714 lp->skb_length = skb->len; 3715 lp->skb_physaddr = dma_map_single(&lp->pdev->dev, skb->data, 3716 skb->len, DMA_TO_DEVICE); 3717 if (dma_mapping_error(&lp->pdev->dev, lp->skb_physaddr)) { 3718 dev_kfree_skb_any(skb); 3719 dev->stats.tx_dropped++; 3720 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3721 return NETDEV_TX_OK; 3722 } 3723 3724 /* Set address of the data in the Transmit Address register */ 3725 macb_writel(lp, TAR, lp->skb_physaddr); 3726 /* Set length of the packet in the Transmit Control register */ 3727 macb_writel(lp, TCR, skb->len); 3728 3729 } else { 3730 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3731 return NETDEV_TX_BUSY; 3732 } 3733 3734 return NETDEV_TX_OK; 3735 } 3736 3737 /* Extract received frame from buffer descriptors and sent to upper layers. 3738 * (Called from interrupt context) 3739 */ 3740 static void at91ether_rx(struct net_device *dev) 3741 { 3742 struct macb *lp = netdev_priv(dev); 3743 struct macb_queue *q = &lp->queues[0]; 3744 struct macb_dma_desc *desc; 3745 unsigned char *p_recv; 3746 struct sk_buff *skb; 3747 unsigned int pktlen; 3748 3749 desc = macb_rx_desc(q, q->rx_tail); 3750 while (desc->addr & MACB_BIT(RX_USED)) { 3751 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3752 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3753 skb = netdev_alloc_skb(dev, pktlen + 2); 3754 if (skb) { 3755 skb_reserve(skb, 2); 3756 skb_put_data(skb, p_recv, pktlen); 3757 3758 skb->protocol = eth_type_trans(skb, dev); 3759 dev->stats.rx_packets++; 3760 dev->stats.rx_bytes += pktlen; 3761 netif_rx(skb); 3762 } else { 3763 dev->stats.rx_dropped++; 3764 } 3765 3766 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3767 dev->stats.multicast++; 3768 3769 /* reset ownership bit */ 3770 desc->addr &= ~MACB_BIT(RX_USED); 3771 3772 /* wrap after last buffer */ 3773 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3774 q->rx_tail = 0; 3775 else 3776 q->rx_tail++; 3777 3778 desc = macb_rx_desc(q, q->rx_tail); 3779 } 3780 } 3781 3782 /* MAC interrupt handler */ 3783 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3784 { 3785 struct net_device *dev = dev_id; 3786 struct macb *lp = netdev_priv(dev); 3787 u32 intstatus, ctl; 3788 3789 /* MAC Interrupt Status register indicates what interrupts are pending. 3790 * It is automatically cleared once read. 3791 */ 3792 intstatus = macb_readl(lp, ISR); 3793 3794 /* Receive complete */ 3795 if (intstatus & MACB_BIT(RCOMP)) 3796 at91ether_rx(dev); 3797 3798 /* Transmit complete */ 3799 if (intstatus & MACB_BIT(TCOMP)) { 3800 /* The TCOM bit is set even if the transmission failed */ 3801 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3802 dev->stats.tx_errors++; 3803 3804 if (lp->skb) { 3805 dev_consume_skb_irq(lp->skb); 3806 lp->skb = NULL; 3807 dma_unmap_single(&lp->pdev->dev, lp->skb_physaddr, 3808 lp->skb_length, DMA_TO_DEVICE); 3809 dev->stats.tx_packets++; 3810 dev->stats.tx_bytes += lp->skb_length; 3811 } 3812 netif_wake_queue(dev); 3813 } 3814 3815 /* Work-around for EMAC Errata section 41.3.1 */ 3816 if (intstatus & MACB_BIT(RXUBR)) { 3817 ctl = macb_readl(lp, NCR); 3818 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3819 wmb(); 3820 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3821 } 3822 3823 if (intstatus & MACB_BIT(ISR_ROVR)) 3824 netdev_err(dev, "ROVR error\n"); 3825 3826 return IRQ_HANDLED; 3827 } 3828 3829 #ifdef CONFIG_NET_POLL_CONTROLLER 3830 static void at91ether_poll_controller(struct net_device *dev) 3831 { 3832 unsigned long flags; 3833 3834 local_irq_save(flags); 3835 at91ether_interrupt(dev->irq, dev); 3836 local_irq_restore(flags); 3837 } 3838 #endif 3839 3840 static const struct net_device_ops at91ether_netdev_ops = { 3841 .ndo_open = at91ether_open, 3842 .ndo_stop = at91ether_close, 3843 .ndo_start_xmit = at91ether_start_xmit, 3844 .ndo_get_stats = macb_get_stats, 3845 .ndo_set_rx_mode = macb_set_rx_mode, 3846 .ndo_set_mac_address = eth_mac_addr, 3847 .ndo_do_ioctl = macb_ioctl, 3848 .ndo_validate_addr = eth_validate_addr, 3849 #ifdef CONFIG_NET_POLL_CONTROLLER 3850 .ndo_poll_controller = at91ether_poll_controller, 3851 #endif 3852 }; 3853 3854 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3855 struct clk **hclk, struct clk **tx_clk, 3856 struct clk **rx_clk, struct clk **tsu_clk) 3857 { 3858 int err; 3859 3860 *hclk = NULL; 3861 *tx_clk = NULL; 3862 *rx_clk = NULL; 3863 *tsu_clk = NULL; 3864 3865 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3866 if (IS_ERR(*pclk)) 3867 return PTR_ERR(*pclk); 3868 3869 err = clk_prepare_enable(*pclk); 3870 if (err) { 3871 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3872 return err; 3873 } 3874 3875 return 0; 3876 } 3877 3878 static int at91ether_init(struct platform_device *pdev) 3879 { 3880 struct net_device *dev = platform_get_drvdata(pdev); 3881 struct macb *bp = netdev_priv(dev); 3882 int err; 3883 u32 reg; 3884 3885 bp->queues[0].bp = bp; 3886 3887 dev->netdev_ops = &at91ether_netdev_ops; 3888 dev->ethtool_ops = &macb_ethtool_ops; 3889 3890 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3891 0, dev->name, dev); 3892 if (err) 3893 return err; 3894 3895 macb_writel(bp, NCR, 0); 3896 3897 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3898 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3899 reg |= MACB_BIT(RM9200_RMII); 3900 3901 macb_writel(bp, NCFGR, reg); 3902 3903 return 0; 3904 } 3905 3906 static const struct macb_config at91sam9260_config = { 3907 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3908 .clk_init = macb_clk_init, 3909 .init = macb_init, 3910 }; 3911 3912 static const struct macb_config sama5d3macb_config = { 3913 .caps = MACB_CAPS_SG_DISABLED 3914 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3915 .clk_init = macb_clk_init, 3916 .init = macb_init, 3917 }; 3918 3919 static const struct macb_config pc302gem_config = { 3920 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 3921 .dma_burst_length = 16, 3922 .clk_init = macb_clk_init, 3923 .init = macb_init, 3924 }; 3925 3926 static const struct macb_config sama5d2_config = { 3927 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3928 .dma_burst_length = 16, 3929 .clk_init = macb_clk_init, 3930 .init = macb_init, 3931 }; 3932 3933 static const struct macb_config sama5d3_config = { 3934 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 3935 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 3936 .dma_burst_length = 16, 3937 .clk_init = macb_clk_init, 3938 .init = macb_init, 3939 .jumbo_max_len = 10240, 3940 }; 3941 3942 static const struct macb_config sama5d4_config = { 3943 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3944 .dma_burst_length = 4, 3945 .clk_init = macb_clk_init, 3946 .init = macb_init, 3947 }; 3948 3949 static const struct macb_config emac_config = { 3950 .caps = MACB_CAPS_NEEDS_RSTONUBR, 3951 .clk_init = at91ether_clk_init, 3952 .init = at91ether_init, 3953 }; 3954 3955 static const struct macb_config np4_config = { 3956 .caps = MACB_CAPS_USRIO_DISABLED, 3957 .clk_init = macb_clk_init, 3958 .init = macb_init, 3959 }; 3960 3961 static const struct macb_config zynqmp_config = { 3962 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3963 MACB_CAPS_JUMBO | 3964 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 3965 .dma_burst_length = 16, 3966 .clk_init = macb_clk_init, 3967 .init = macb_init, 3968 .jumbo_max_len = 10240, 3969 }; 3970 3971 static const struct macb_config zynq_config = { 3972 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 3973 MACB_CAPS_NEEDS_RSTONUBR, 3974 .dma_burst_length = 16, 3975 .clk_init = macb_clk_init, 3976 .init = macb_init, 3977 }; 3978 3979 static const struct of_device_id macb_dt_ids[] = { 3980 { .compatible = "cdns,at32ap7000-macb" }, 3981 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 3982 { .compatible = "cdns,macb" }, 3983 { .compatible = "cdns,np4-macb", .data = &np4_config }, 3984 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 3985 { .compatible = "cdns,gem", .data = &pc302gem_config }, 3986 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 3987 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 3988 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 3989 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 3990 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 3991 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 3992 { .compatible = "cdns,emac", .data = &emac_config }, 3993 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 3994 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 3995 { /* sentinel */ } 3996 }; 3997 MODULE_DEVICE_TABLE(of, macb_dt_ids); 3998 #endif /* CONFIG_OF */ 3999 4000 static const struct macb_config default_gem_config = { 4001 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4002 MACB_CAPS_JUMBO | 4003 MACB_CAPS_GEM_HAS_PTP, 4004 .dma_burst_length = 16, 4005 .clk_init = macb_clk_init, 4006 .init = macb_init, 4007 .jumbo_max_len = 10240, 4008 }; 4009 4010 static int macb_probe(struct platform_device *pdev) 4011 { 4012 const struct macb_config *macb_config = &default_gem_config; 4013 int (*clk_init)(struct platform_device *, struct clk **, 4014 struct clk **, struct clk **, struct clk **, 4015 struct clk **) = macb_config->clk_init; 4016 int (*init)(struct platform_device *) = macb_config->init; 4017 struct device_node *np = pdev->dev.of_node; 4018 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4019 struct clk *tsu_clk = NULL; 4020 unsigned int queue_mask, num_queues; 4021 bool native_io; 4022 struct phy_device *phydev; 4023 struct net_device *dev; 4024 struct resource *regs; 4025 void __iomem *mem; 4026 const char *mac; 4027 struct macb *bp; 4028 int err, val; 4029 4030 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4031 mem = devm_ioremap_resource(&pdev->dev, regs); 4032 if (IS_ERR(mem)) 4033 return PTR_ERR(mem); 4034 4035 if (np) { 4036 const struct of_device_id *match; 4037 4038 match = of_match_node(macb_dt_ids, np); 4039 if (match && match->data) { 4040 macb_config = match->data; 4041 clk_init = macb_config->clk_init; 4042 init = macb_config->init; 4043 } 4044 } 4045 4046 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4047 if (err) 4048 return err; 4049 4050 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4051 pm_runtime_use_autosuspend(&pdev->dev); 4052 pm_runtime_get_noresume(&pdev->dev); 4053 pm_runtime_set_active(&pdev->dev); 4054 pm_runtime_enable(&pdev->dev); 4055 native_io = hw_is_native_io(mem); 4056 4057 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4058 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4059 if (!dev) { 4060 err = -ENOMEM; 4061 goto err_disable_clocks; 4062 } 4063 4064 dev->base_addr = regs->start; 4065 4066 SET_NETDEV_DEV(dev, &pdev->dev); 4067 4068 bp = netdev_priv(dev); 4069 bp->pdev = pdev; 4070 bp->dev = dev; 4071 bp->regs = mem; 4072 bp->native_io = native_io; 4073 if (native_io) { 4074 bp->macb_reg_readl = hw_readl_native; 4075 bp->macb_reg_writel = hw_writel_native; 4076 } else { 4077 bp->macb_reg_readl = hw_readl; 4078 bp->macb_reg_writel = hw_writel; 4079 } 4080 bp->num_queues = num_queues; 4081 bp->queue_mask = queue_mask; 4082 if (macb_config) 4083 bp->dma_burst_length = macb_config->dma_burst_length; 4084 bp->pclk = pclk; 4085 bp->hclk = hclk; 4086 bp->tx_clk = tx_clk; 4087 bp->rx_clk = rx_clk; 4088 bp->tsu_clk = tsu_clk; 4089 if (macb_config) 4090 bp->jumbo_max_len = macb_config->jumbo_max_len; 4091 4092 bp->wol = 0; 4093 if (of_get_property(np, "magic-packet", NULL)) 4094 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4095 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4096 4097 spin_lock_init(&bp->lock); 4098 4099 /* setup capabilities */ 4100 macb_configure_caps(bp, macb_config); 4101 4102 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4103 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4104 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4105 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4106 } 4107 #endif 4108 platform_set_drvdata(pdev, dev); 4109 4110 dev->irq = platform_get_irq(pdev, 0); 4111 if (dev->irq < 0) { 4112 err = dev->irq; 4113 goto err_out_free_netdev; 4114 } 4115 4116 /* MTU range: 68 - 1500 or 10240 */ 4117 dev->min_mtu = GEM_MTU_MIN_SIZE; 4118 if (bp->caps & MACB_CAPS_JUMBO) 4119 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4120 else 4121 dev->max_mtu = ETH_DATA_LEN; 4122 4123 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4124 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4125 if (val) 4126 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4127 macb_dma_desc_get_size(bp); 4128 4129 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4130 if (val) 4131 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4132 macb_dma_desc_get_size(bp); 4133 } 4134 4135 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4136 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4137 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4138 4139 mac = of_get_mac_address(np); 4140 if (PTR_ERR(mac) == -EPROBE_DEFER) { 4141 err = -EPROBE_DEFER; 4142 goto err_out_free_netdev; 4143 } else if (!IS_ERR(mac)) { 4144 ether_addr_copy(bp->dev->dev_addr, mac); 4145 } else { 4146 macb_get_hwaddr(bp); 4147 } 4148 4149 err = of_get_phy_mode(np); 4150 if (err < 0) 4151 /* not found in DT, MII by default */ 4152 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4153 else 4154 bp->phy_interface = err; 4155 4156 /* IP specific init */ 4157 err = init(pdev); 4158 if (err) 4159 goto err_out_free_netdev; 4160 4161 err = macb_mii_init(bp); 4162 if (err) 4163 goto err_out_free_netdev; 4164 4165 phydev = dev->phydev; 4166 4167 netif_carrier_off(dev); 4168 4169 err = register_netdev(dev); 4170 if (err) { 4171 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4172 goto err_out_unregister_mdio; 4173 } 4174 4175 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4176 (unsigned long)bp); 4177 4178 phy_attached_info(phydev); 4179 4180 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4181 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4182 dev->base_addr, dev->irq, dev->dev_addr); 4183 4184 pm_runtime_mark_last_busy(&bp->pdev->dev); 4185 pm_runtime_put_autosuspend(&bp->pdev->dev); 4186 4187 return 0; 4188 4189 err_out_unregister_mdio: 4190 phy_disconnect(dev->phydev); 4191 mdiobus_unregister(bp->mii_bus); 4192 of_node_put(bp->phy_node); 4193 if (np && of_phy_is_fixed_link(np)) 4194 of_phy_deregister_fixed_link(np); 4195 mdiobus_free(bp->mii_bus); 4196 4197 err_out_free_netdev: 4198 free_netdev(dev); 4199 4200 err_disable_clocks: 4201 clk_disable_unprepare(tx_clk); 4202 clk_disable_unprepare(hclk); 4203 clk_disable_unprepare(pclk); 4204 clk_disable_unprepare(rx_clk); 4205 clk_disable_unprepare(tsu_clk); 4206 pm_runtime_disable(&pdev->dev); 4207 pm_runtime_set_suspended(&pdev->dev); 4208 pm_runtime_dont_use_autosuspend(&pdev->dev); 4209 4210 return err; 4211 } 4212 4213 static int macb_remove(struct platform_device *pdev) 4214 { 4215 struct net_device *dev; 4216 struct macb *bp; 4217 struct device_node *np = pdev->dev.of_node; 4218 4219 dev = platform_get_drvdata(pdev); 4220 4221 if (dev) { 4222 bp = netdev_priv(dev); 4223 if (dev->phydev) 4224 phy_disconnect(dev->phydev); 4225 mdiobus_unregister(bp->mii_bus); 4226 if (np && of_phy_is_fixed_link(np)) 4227 of_phy_deregister_fixed_link(np); 4228 dev->phydev = NULL; 4229 mdiobus_free(bp->mii_bus); 4230 4231 unregister_netdev(dev); 4232 pm_runtime_disable(&pdev->dev); 4233 pm_runtime_dont_use_autosuspend(&pdev->dev); 4234 if (!pm_runtime_suspended(&pdev->dev)) { 4235 clk_disable_unprepare(bp->tx_clk); 4236 clk_disable_unprepare(bp->hclk); 4237 clk_disable_unprepare(bp->pclk); 4238 clk_disable_unprepare(bp->rx_clk); 4239 clk_disable_unprepare(bp->tsu_clk); 4240 pm_runtime_set_suspended(&pdev->dev); 4241 } 4242 of_node_put(bp->phy_node); 4243 free_netdev(dev); 4244 } 4245 4246 return 0; 4247 } 4248 4249 static int __maybe_unused macb_suspend(struct device *dev) 4250 { 4251 struct net_device *netdev = dev_get_drvdata(dev); 4252 struct macb *bp = netdev_priv(netdev); 4253 struct macb_queue *queue = bp->queues; 4254 unsigned long flags; 4255 unsigned int q; 4256 4257 if (!netif_running(netdev)) 4258 return 0; 4259 4260 4261 if (bp->wol & MACB_WOL_ENABLED) { 4262 macb_writel(bp, IER, MACB_BIT(WOL)); 4263 macb_writel(bp, WOL, MACB_BIT(MAG)); 4264 enable_irq_wake(bp->queues[0].irq); 4265 netif_device_detach(netdev); 4266 } else { 4267 netif_device_detach(netdev); 4268 for (q = 0, queue = bp->queues; q < bp->num_queues; 4269 ++q, ++queue) 4270 napi_disable(&queue->napi); 4271 phy_stop(netdev->phydev); 4272 phy_suspend(netdev->phydev); 4273 spin_lock_irqsave(&bp->lock, flags); 4274 macb_reset_hw(bp); 4275 spin_unlock_irqrestore(&bp->lock, flags); 4276 } 4277 4278 netif_carrier_off(netdev); 4279 if (bp->ptp_info) 4280 bp->ptp_info->ptp_remove(netdev); 4281 pm_runtime_force_suspend(dev); 4282 4283 return 0; 4284 } 4285 4286 static int __maybe_unused macb_resume(struct device *dev) 4287 { 4288 struct net_device *netdev = dev_get_drvdata(dev); 4289 struct macb *bp = netdev_priv(netdev); 4290 struct macb_queue *queue = bp->queues; 4291 unsigned int q; 4292 4293 if (!netif_running(netdev)) 4294 return 0; 4295 4296 pm_runtime_force_resume(dev); 4297 4298 if (bp->wol & MACB_WOL_ENABLED) { 4299 macb_writel(bp, IDR, MACB_BIT(WOL)); 4300 macb_writel(bp, WOL, 0); 4301 disable_irq_wake(bp->queues[0].irq); 4302 } else { 4303 macb_writel(bp, NCR, MACB_BIT(MPE)); 4304 for (q = 0, queue = bp->queues; q < bp->num_queues; 4305 ++q, ++queue) 4306 napi_enable(&queue->napi); 4307 phy_resume(netdev->phydev); 4308 phy_init_hw(netdev->phydev); 4309 phy_start(netdev->phydev); 4310 } 4311 4312 bp->macbgem_ops.mog_init_rings(bp); 4313 macb_init_hw(bp); 4314 macb_set_rx_mode(netdev); 4315 netif_device_attach(netdev); 4316 if (bp->ptp_info) 4317 bp->ptp_info->ptp_init(netdev); 4318 4319 return 0; 4320 } 4321 4322 static int __maybe_unused macb_runtime_suspend(struct device *dev) 4323 { 4324 struct net_device *netdev = dev_get_drvdata(dev); 4325 struct macb *bp = netdev_priv(netdev); 4326 4327 if (!(device_may_wakeup(&bp->dev->dev))) { 4328 clk_disable_unprepare(bp->tx_clk); 4329 clk_disable_unprepare(bp->hclk); 4330 clk_disable_unprepare(bp->pclk); 4331 clk_disable_unprepare(bp->rx_clk); 4332 } 4333 clk_disable_unprepare(bp->tsu_clk); 4334 4335 return 0; 4336 } 4337 4338 static int __maybe_unused macb_runtime_resume(struct device *dev) 4339 { 4340 struct net_device *netdev = dev_get_drvdata(dev); 4341 struct macb *bp = netdev_priv(netdev); 4342 4343 if (!(device_may_wakeup(&bp->dev->dev))) { 4344 clk_prepare_enable(bp->pclk); 4345 clk_prepare_enable(bp->hclk); 4346 clk_prepare_enable(bp->tx_clk); 4347 clk_prepare_enable(bp->rx_clk); 4348 } 4349 clk_prepare_enable(bp->tsu_clk); 4350 4351 return 0; 4352 } 4353 4354 static const struct dev_pm_ops macb_pm_ops = { 4355 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 4356 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 4357 }; 4358 4359 static struct platform_driver macb_driver = { 4360 .probe = macb_probe, 4361 .remove = macb_remove, 4362 .driver = { 4363 .name = "macb", 4364 .of_match_table = of_match_ptr(macb_dt_ids), 4365 .pm = &macb_pm_ops, 4366 }, 4367 }; 4368 4369 module_platform_driver(macb_driver); 4370 4371 MODULE_LICENSE("GPL"); 4372 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4373 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4374 MODULE_ALIAS("platform:macb"); 4375