1 /* 2 * Cadence MACB/GEM Ethernet Controller driver 3 * 4 * Copyright (C) 2004-2006 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 #include <linux/clk.h> 13 #include <linux/crc32.h> 14 #include <linux/module.h> 15 #include <linux/moduleparam.h> 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/circ_buf.h> 19 #include <linux/slab.h> 20 #include <linux/init.h> 21 #include <linux/io.h> 22 #include <linux/gpio.h> 23 #include <linux/gpio/consumer.h> 24 #include <linux/interrupt.h> 25 #include <linux/netdevice.h> 26 #include <linux/etherdevice.h> 27 #include <linux/dma-mapping.h> 28 #include <linux/platform_data/macb.h> 29 #include <linux/platform_device.h> 30 #include <linux/phy.h> 31 #include <linux/of.h> 32 #include <linux/of_device.h> 33 #include <linux/of_gpio.h> 34 #include <linux/of_mdio.h> 35 #include <linux/of_net.h> 36 #include <linux/ip.h> 37 #include <linux/udp.h> 38 #include <linux/tcp.h> 39 #include <linux/iopoll.h> 40 #include <linux/pm_runtime.h> 41 #include "macb.h" 42 43 #define MACB_RX_BUFFER_SIZE 128 44 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 45 46 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 47 #define MIN_RX_RING_SIZE 64 48 #define MAX_RX_RING_SIZE 8192 49 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 50 * (bp)->rx_ring_size) 51 52 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 53 #define MIN_TX_RING_SIZE 64 54 #define MAX_TX_RING_SIZE 4096 55 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 56 * (bp)->tx_ring_size) 57 58 /* level of occupied TX descriptors under which we wake up TX process */ 59 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 60 61 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 62 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 63 | MACB_BIT(ISR_RLE) \ 64 | MACB_BIT(TXERR)) 65 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 66 | MACB_BIT(TXUBR)) 67 68 /* Max length of transmit frame must be a multiple of 8 bytes */ 69 #define MACB_TX_LEN_ALIGN 8 70 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 71 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 72 73 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 74 #define MACB_NETIF_LSO NETIF_F_TSO 75 76 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 77 #define MACB_WOL_ENABLED (0x1 << 1) 78 79 /* Graceful stop timeouts in us. We should allow up to 80 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 81 */ 82 #define MACB_HALT_TIMEOUT 1230 83 84 #define MACB_PM_TIMEOUT 100 /* ms */ 85 86 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 87 88 /* DMA buffer descriptor might be different size 89 * depends on hardware configuration: 90 * 91 * 1. dma address width 32 bits: 92 * word 1: 32 bit address of Data Buffer 93 * word 2: control 94 * 95 * 2. dma address width 64 bits: 96 * word 1: 32 bit address of Data Buffer 97 * word 2: control 98 * word 3: upper 32 bit address of Data Buffer 99 * word 4: unused 100 * 101 * 3. dma address width 32 bits with hardware timestamping: 102 * word 1: 32 bit address of Data Buffer 103 * word 2: control 104 * word 3: timestamp word 1 105 * word 4: timestamp word 2 106 * 107 * 4. dma address width 64 bits with hardware timestamping: 108 * word 1: 32 bit address of Data Buffer 109 * word 2: control 110 * word 3: upper 32 bit address of Data Buffer 111 * word 4: unused 112 * word 5: timestamp word 1 113 * word 6: timestamp word 2 114 */ 115 static unsigned int macb_dma_desc_get_size(struct macb *bp) 116 { 117 #ifdef MACB_EXT_DESC 118 unsigned int desc_size; 119 120 switch (bp->hw_dma_cap) { 121 case HW_DMA_CAP_64B: 122 desc_size = sizeof(struct macb_dma_desc) 123 + sizeof(struct macb_dma_desc_64); 124 break; 125 case HW_DMA_CAP_PTP: 126 desc_size = sizeof(struct macb_dma_desc) 127 + sizeof(struct macb_dma_desc_ptp); 128 break; 129 case HW_DMA_CAP_64B_PTP: 130 desc_size = sizeof(struct macb_dma_desc) 131 + sizeof(struct macb_dma_desc_64) 132 + sizeof(struct macb_dma_desc_ptp); 133 break; 134 default: 135 desc_size = sizeof(struct macb_dma_desc); 136 } 137 return desc_size; 138 #endif 139 return sizeof(struct macb_dma_desc); 140 } 141 142 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 143 { 144 #ifdef MACB_EXT_DESC 145 switch (bp->hw_dma_cap) { 146 case HW_DMA_CAP_64B: 147 case HW_DMA_CAP_PTP: 148 desc_idx <<= 1; 149 break; 150 case HW_DMA_CAP_64B_PTP: 151 desc_idx *= 3; 152 break; 153 default: 154 break; 155 } 156 #endif 157 return desc_idx; 158 } 159 160 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 161 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 162 { 163 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 164 return (struct macb_dma_desc_64 *)((void *)desc + sizeof(struct macb_dma_desc)); 165 return NULL; 166 } 167 #endif 168 169 /* Ring buffer accessors */ 170 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 171 { 172 return index & (bp->tx_ring_size - 1); 173 } 174 175 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 176 unsigned int index) 177 { 178 index = macb_tx_ring_wrap(queue->bp, index); 179 index = macb_adj_dma_desc_idx(queue->bp, index); 180 return &queue->tx_ring[index]; 181 } 182 183 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 184 unsigned int index) 185 { 186 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 187 } 188 189 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 190 { 191 dma_addr_t offset; 192 193 offset = macb_tx_ring_wrap(queue->bp, index) * 194 macb_dma_desc_get_size(queue->bp); 195 196 return queue->tx_ring_dma + offset; 197 } 198 199 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 200 { 201 return index & (bp->rx_ring_size - 1); 202 } 203 204 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 205 { 206 index = macb_rx_ring_wrap(queue->bp, index); 207 index = macb_adj_dma_desc_idx(queue->bp, index); 208 return &queue->rx_ring[index]; 209 } 210 211 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 212 { 213 return queue->rx_buffers + queue->bp->rx_buffer_size * 214 macb_rx_ring_wrap(queue->bp, index); 215 } 216 217 /* I/O accessors */ 218 static u32 hw_readl_native(struct macb *bp, int offset) 219 { 220 return __raw_readl(bp->regs + offset); 221 } 222 223 static void hw_writel_native(struct macb *bp, int offset, u32 value) 224 { 225 __raw_writel(value, bp->regs + offset); 226 } 227 228 static u32 hw_readl(struct macb *bp, int offset) 229 { 230 return readl_relaxed(bp->regs + offset); 231 } 232 233 static void hw_writel(struct macb *bp, int offset, u32 value) 234 { 235 writel_relaxed(value, bp->regs + offset); 236 } 237 238 /* Find the CPU endianness by using the loopback bit of NCR register. When the 239 * CPU is in big endian we need to program swapped mode for management 240 * descriptor access. 241 */ 242 static bool hw_is_native_io(void __iomem *addr) 243 { 244 u32 value = MACB_BIT(LLB); 245 246 __raw_writel(value, addr + MACB_NCR); 247 value = __raw_readl(addr + MACB_NCR); 248 249 /* Write 0 back to disable everything */ 250 __raw_writel(0, addr + MACB_NCR); 251 252 return value == MACB_BIT(LLB); 253 } 254 255 static bool hw_is_gem(void __iomem *addr, bool native_io) 256 { 257 u32 id; 258 259 if (native_io) 260 id = __raw_readl(addr + MACB_MID); 261 else 262 id = readl_relaxed(addr + MACB_MID); 263 264 return MACB_BFEXT(IDNUM, id) >= 0x2; 265 } 266 267 static void macb_set_hwaddr(struct macb *bp) 268 { 269 u32 bottom; 270 u16 top; 271 272 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 273 macb_or_gem_writel(bp, SA1B, bottom); 274 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 275 macb_or_gem_writel(bp, SA1T, top); 276 277 /* Clear unused address register sets */ 278 macb_or_gem_writel(bp, SA2B, 0); 279 macb_or_gem_writel(bp, SA2T, 0); 280 macb_or_gem_writel(bp, SA3B, 0); 281 macb_or_gem_writel(bp, SA3T, 0); 282 macb_or_gem_writel(bp, SA4B, 0); 283 macb_or_gem_writel(bp, SA4T, 0); 284 } 285 286 static void macb_get_hwaddr(struct macb *bp) 287 { 288 u32 bottom; 289 u16 top; 290 u8 addr[6]; 291 int i; 292 293 /* Check all 4 address register for valid address */ 294 for (i = 0; i < 4; i++) { 295 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 296 top = macb_or_gem_readl(bp, SA1T + i * 8); 297 298 addr[0] = bottom & 0xff; 299 addr[1] = (bottom >> 8) & 0xff; 300 addr[2] = (bottom >> 16) & 0xff; 301 addr[3] = (bottom >> 24) & 0xff; 302 addr[4] = top & 0xff; 303 addr[5] = (top >> 8) & 0xff; 304 305 if (is_valid_ether_addr(addr)) { 306 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 307 return; 308 } 309 } 310 311 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 312 eth_hw_addr_random(bp->dev); 313 } 314 315 static int macb_mdio_wait_for_idle(struct macb *bp) 316 { 317 u32 val; 318 319 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 320 1, MACB_MDIO_TIMEOUT); 321 } 322 323 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 324 { 325 struct macb *bp = bus->priv; 326 int status; 327 328 status = pm_runtime_get_sync(&bp->pdev->dev); 329 if (status < 0) 330 goto mdio_pm_exit; 331 332 status = macb_mdio_wait_for_idle(bp); 333 if (status < 0) 334 goto mdio_read_exit; 335 336 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 337 | MACB_BF(RW, MACB_MAN_READ) 338 | MACB_BF(PHYA, mii_id) 339 | MACB_BF(REGA, regnum) 340 | MACB_BF(CODE, MACB_MAN_CODE))); 341 342 status = macb_mdio_wait_for_idle(bp); 343 if (status < 0) 344 goto mdio_read_exit; 345 346 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 347 348 mdio_read_exit: 349 pm_runtime_mark_last_busy(&bp->pdev->dev); 350 pm_runtime_put_autosuspend(&bp->pdev->dev); 351 mdio_pm_exit: 352 return status; 353 } 354 355 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 356 u16 value) 357 { 358 struct macb *bp = bus->priv; 359 int status; 360 361 status = pm_runtime_get_sync(&bp->pdev->dev); 362 if (status < 0) 363 goto mdio_pm_exit; 364 365 status = macb_mdio_wait_for_idle(bp); 366 if (status < 0) 367 goto mdio_write_exit; 368 369 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 370 | MACB_BF(RW, MACB_MAN_WRITE) 371 | MACB_BF(PHYA, mii_id) 372 | MACB_BF(REGA, regnum) 373 | MACB_BF(CODE, MACB_MAN_CODE) 374 | MACB_BF(DATA, value))); 375 376 status = macb_mdio_wait_for_idle(bp); 377 if (status < 0) 378 goto mdio_write_exit; 379 380 mdio_write_exit: 381 pm_runtime_mark_last_busy(&bp->pdev->dev); 382 pm_runtime_put_autosuspend(&bp->pdev->dev); 383 mdio_pm_exit: 384 return status; 385 } 386 387 /** 388 * macb_set_tx_clk() - Set a clock to a new frequency 389 * @clk Pointer to the clock to change 390 * @rate New frequency in Hz 391 * @dev Pointer to the struct net_device 392 */ 393 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 394 { 395 long ferr, rate, rate_rounded; 396 397 if (!clk) 398 return; 399 400 switch (speed) { 401 case SPEED_10: 402 rate = 2500000; 403 break; 404 case SPEED_100: 405 rate = 25000000; 406 break; 407 case SPEED_1000: 408 rate = 125000000; 409 break; 410 default: 411 return; 412 } 413 414 rate_rounded = clk_round_rate(clk, rate); 415 if (rate_rounded < 0) 416 return; 417 418 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 419 * is not satisfied. 420 */ 421 ferr = abs(rate_rounded - rate); 422 ferr = DIV_ROUND_UP(ferr, rate / 100000); 423 if (ferr > 5) 424 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 425 rate); 426 427 if (clk_set_rate(clk, rate_rounded)) 428 netdev_err(dev, "adjusting tx_clk failed.\n"); 429 } 430 431 static void macb_handle_link_change(struct net_device *dev) 432 { 433 struct macb *bp = netdev_priv(dev); 434 struct phy_device *phydev = dev->phydev; 435 unsigned long flags; 436 int status_change = 0; 437 438 spin_lock_irqsave(&bp->lock, flags); 439 440 if (phydev->link) { 441 if ((bp->speed != phydev->speed) || 442 (bp->duplex != phydev->duplex)) { 443 u32 reg; 444 445 reg = macb_readl(bp, NCFGR); 446 reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 447 if (macb_is_gem(bp)) 448 reg &= ~GEM_BIT(GBE); 449 450 if (phydev->duplex) 451 reg |= MACB_BIT(FD); 452 if (phydev->speed == SPEED_100) 453 reg |= MACB_BIT(SPD); 454 if (phydev->speed == SPEED_1000 && 455 bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 456 reg |= GEM_BIT(GBE); 457 458 macb_or_gem_writel(bp, NCFGR, reg); 459 460 bp->speed = phydev->speed; 461 bp->duplex = phydev->duplex; 462 status_change = 1; 463 } 464 } 465 466 if (phydev->link != bp->link) { 467 if (!phydev->link) { 468 bp->speed = 0; 469 bp->duplex = -1; 470 } 471 bp->link = phydev->link; 472 473 status_change = 1; 474 } 475 476 spin_unlock_irqrestore(&bp->lock, flags); 477 478 if (status_change) { 479 if (phydev->link) { 480 /* Update the TX clock rate if and only if the link is 481 * up and there has been a link change. 482 */ 483 macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); 484 485 netif_carrier_on(dev); 486 netdev_info(dev, "link up (%d/%s)\n", 487 phydev->speed, 488 phydev->duplex == DUPLEX_FULL ? 489 "Full" : "Half"); 490 } else { 491 netif_carrier_off(dev); 492 netdev_info(dev, "link down\n"); 493 } 494 } 495 } 496 497 /* based on au1000_eth. c*/ 498 static int macb_mii_probe(struct net_device *dev) 499 { 500 struct macb *bp = netdev_priv(dev); 501 struct phy_device *phydev; 502 struct device_node *np; 503 int ret, i; 504 505 np = bp->pdev->dev.of_node; 506 ret = 0; 507 508 if (np) { 509 if (of_phy_is_fixed_link(np)) { 510 bp->phy_node = of_node_get(np); 511 } else { 512 bp->phy_node = of_parse_phandle(np, "phy-handle", 0); 513 /* fallback to standard phy registration if no 514 * phy-handle was found nor any phy found during 515 * dt phy registration 516 */ 517 if (!bp->phy_node && !phy_find_first(bp->mii_bus)) { 518 for (i = 0; i < PHY_MAX_ADDR; i++) { 519 phydev = mdiobus_scan(bp->mii_bus, i); 520 if (IS_ERR(phydev) && 521 PTR_ERR(phydev) != -ENODEV) { 522 ret = PTR_ERR(phydev); 523 break; 524 } 525 } 526 527 if (ret) 528 return -ENODEV; 529 } 530 } 531 } 532 533 if (bp->phy_node) { 534 phydev = of_phy_connect(dev, bp->phy_node, 535 &macb_handle_link_change, 0, 536 bp->phy_interface); 537 if (!phydev) 538 return -ENODEV; 539 } else { 540 phydev = phy_find_first(bp->mii_bus); 541 if (!phydev) { 542 netdev_err(dev, "no PHY found\n"); 543 return -ENXIO; 544 } 545 546 /* attach the mac to the phy */ 547 ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, 548 bp->phy_interface); 549 if (ret) { 550 netdev_err(dev, "Could not attach to PHY\n"); 551 return ret; 552 } 553 } 554 555 /* mask with MAC supported features */ 556 if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 557 phy_set_max_speed(phydev, SPEED_1000); 558 else 559 phy_set_max_speed(phydev, SPEED_100); 560 561 if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) 562 phy_remove_link_mode(phydev, 563 ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 564 565 bp->link = 0; 566 bp->speed = 0; 567 bp->duplex = -1; 568 569 return 0; 570 } 571 572 static int macb_mii_init(struct macb *bp) 573 { 574 struct device_node *np; 575 int err = -ENXIO; 576 577 /* Enable management port */ 578 macb_writel(bp, NCR, MACB_BIT(MPE)); 579 580 bp->mii_bus = mdiobus_alloc(); 581 if (!bp->mii_bus) { 582 err = -ENOMEM; 583 goto err_out; 584 } 585 586 bp->mii_bus->name = "MACB_mii_bus"; 587 bp->mii_bus->read = &macb_mdio_read; 588 bp->mii_bus->write = &macb_mdio_write; 589 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 590 bp->pdev->name, bp->pdev->id); 591 bp->mii_bus->priv = bp; 592 bp->mii_bus->parent = &bp->pdev->dev; 593 594 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 595 596 np = bp->pdev->dev.of_node; 597 if (np && of_phy_is_fixed_link(np)) { 598 if (of_phy_register_fixed_link(np) < 0) { 599 dev_err(&bp->pdev->dev, 600 "broken fixed-link specification %pOF\n", np); 601 goto err_out_free_mdiobus; 602 } 603 604 err = mdiobus_register(bp->mii_bus); 605 } else { 606 err = of_mdiobus_register(bp->mii_bus, np); 607 } 608 609 if (err) 610 goto err_out_free_fixed_link; 611 612 err = macb_mii_probe(bp->dev); 613 if (err) 614 goto err_out_unregister_bus; 615 616 return 0; 617 618 err_out_unregister_bus: 619 mdiobus_unregister(bp->mii_bus); 620 err_out_free_fixed_link: 621 if (np && of_phy_is_fixed_link(np)) 622 of_phy_deregister_fixed_link(np); 623 err_out_free_mdiobus: 624 of_node_put(bp->phy_node); 625 mdiobus_free(bp->mii_bus); 626 err_out: 627 return err; 628 } 629 630 static void macb_update_stats(struct macb *bp) 631 { 632 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 633 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 634 int offset = MACB_PFR; 635 636 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 637 638 for (; p < end; p++, offset += 4) 639 *p += bp->macb_reg_readl(bp, offset); 640 } 641 642 static int macb_halt_tx(struct macb *bp) 643 { 644 unsigned long halt_time, timeout; 645 u32 status; 646 647 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 648 649 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 650 do { 651 halt_time = jiffies; 652 status = macb_readl(bp, TSR); 653 if (!(status & MACB_BIT(TGO))) 654 return 0; 655 656 udelay(250); 657 } while (time_before(halt_time, timeout)); 658 659 return -ETIMEDOUT; 660 } 661 662 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 663 { 664 if (tx_skb->mapping) { 665 if (tx_skb->mapped_as_page) 666 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 667 tx_skb->size, DMA_TO_DEVICE); 668 else 669 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 670 tx_skb->size, DMA_TO_DEVICE); 671 tx_skb->mapping = 0; 672 } 673 674 if (tx_skb->skb) { 675 dev_kfree_skb_any(tx_skb->skb); 676 tx_skb->skb = NULL; 677 } 678 } 679 680 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 681 { 682 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 683 struct macb_dma_desc_64 *desc_64; 684 685 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 686 desc_64 = macb_64b_desc(bp, desc); 687 desc_64->addrh = upper_32_bits(addr); 688 /* The low bits of RX address contain the RX_USED bit, clearing 689 * of which allows packet RX. Make sure the high bits are also 690 * visible to HW at that point. 691 */ 692 dma_wmb(); 693 } 694 #endif 695 desc->addr = lower_32_bits(addr); 696 } 697 698 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 699 { 700 dma_addr_t addr = 0; 701 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 702 struct macb_dma_desc_64 *desc_64; 703 704 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 705 desc_64 = macb_64b_desc(bp, desc); 706 addr = ((u64)(desc_64->addrh) << 32); 707 } 708 #endif 709 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 710 return addr; 711 } 712 713 static void macb_tx_error_task(struct work_struct *work) 714 { 715 struct macb_queue *queue = container_of(work, struct macb_queue, 716 tx_error_task); 717 struct macb *bp = queue->bp; 718 struct macb_tx_skb *tx_skb; 719 struct macb_dma_desc *desc; 720 struct sk_buff *skb; 721 unsigned int tail; 722 unsigned long flags; 723 724 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 725 (unsigned int)(queue - bp->queues), 726 queue->tx_tail, queue->tx_head); 727 728 /* Prevent the queue IRQ handlers from running: each of them may call 729 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 730 * As explained below, we have to halt the transmission before updating 731 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 732 * network engine about the macb/gem being halted. 733 */ 734 spin_lock_irqsave(&bp->lock, flags); 735 736 /* Make sure nobody is trying to queue up new packets */ 737 netif_tx_stop_all_queues(bp->dev); 738 739 /* Stop transmission now 740 * (in case we have just queued new packets) 741 * macb/gem must be halted to write TBQP register 742 */ 743 if (macb_halt_tx(bp)) 744 /* Just complain for now, reinitializing TX path can be good */ 745 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 746 747 /* Treat frames in TX queue including the ones that caused the error. 748 * Free transmit buffers in upper layer. 749 */ 750 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 751 u32 ctrl; 752 753 desc = macb_tx_desc(queue, tail); 754 ctrl = desc->ctrl; 755 tx_skb = macb_tx_skb(queue, tail); 756 skb = tx_skb->skb; 757 758 if (ctrl & MACB_BIT(TX_USED)) { 759 /* skb is set for the last buffer of the frame */ 760 while (!skb) { 761 macb_tx_unmap(bp, tx_skb); 762 tail++; 763 tx_skb = macb_tx_skb(queue, tail); 764 skb = tx_skb->skb; 765 } 766 767 /* ctrl still refers to the first buffer descriptor 768 * since it's the only one written back by the hardware 769 */ 770 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 771 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 772 macb_tx_ring_wrap(bp, tail), 773 skb->data); 774 bp->dev->stats.tx_packets++; 775 queue->stats.tx_packets++; 776 bp->dev->stats.tx_bytes += skb->len; 777 queue->stats.tx_bytes += skb->len; 778 } 779 } else { 780 /* "Buffers exhausted mid-frame" errors may only happen 781 * if the driver is buggy, so complain loudly about 782 * those. Statistics are updated by hardware. 783 */ 784 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 785 netdev_err(bp->dev, 786 "BUG: TX buffers exhausted mid-frame\n"); 787 788 desc->ctrl = ctrl | MACB_BIT(TX_USED); 789 } 790 791 macb_tx_unmap(bp, tx_skb); 792 } 793 794 /* Set end of TX queue */ 795 desc = macb_tx_desc(queue, 0); 796 macb_set_addr(bp, desc, 0); 797 desc->ctrl = MACB_BIT(TX_USED); 798 799 /* Make descriptor updates visible to hardware */ 800 wmb(); 801 802 /* Reinitialize the TX desc queue */ 803 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 804 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 805 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 806 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 807 #endif 808 /* Make TX ring reflect state of hardware */ 809 queue->tx_head = 0; 810 queue->tx_tail = 0; 811 812 /* Housework before enabling TX IRQ */ 813 macb_writel(bp, TSR, macb_readl(bp, TSR)); 814 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 815 816 /* Now we are ready to start transmission again */ 817 netif_tx_start_all_queues(bp->dev); 818 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 819 820 spin_unlock_irqrestore(&bp->lock, flags); 821 } 822 823 static void macb_tx_interrupt(struct macb_queue *queue) 824 { 825 unsigned int tail; 826 unsigned int head; 827 u32 status; 828 struct macb *bp = queue->bp; 829 u16 queue_index = queue - bp->queues; 830 831 status = macb_readl(bp, TSR); 832 macb_writel(bp, TSR, status); 833 834 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 835 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 836 837 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 838 (unsigned long)status); 839 840 head = queue->tx_head; 841 for (tail = queue->tx_tail; tail != head; tail++) { 842 struct macb_tx_skb *tx_skb; 843 struct sk_buff *skb; 844 struct macb_dma_desc *desc; 845 u32 ctrl; 846 847 desc = macb_tx_desc(queue, tail); 848 849 /* Make hw descriptor updates visible to CPU */ 850 rmb(); 851 852 ctrl = desc->ctrl; 853 854 /* TX_USED bit is only set by hardware on the very first buffer 855 * descriptor of the transmitted frame. 856 */ 857 if (!(ctrl & MACB_BIT(TX_USED))) 858 break; 859 860 /* Process all buffers of the current transmitted frame */ 861 for (;; tail++) { 862 tx_skb = macb_tx_skb(queue, tail); 863 skb = tx_skb->skb; 864 865 /* First, update TX stats if needed */ 866 if (skb) { 867 if (unlikely(skb_shinfo(skb)->tx_flags & 868 SKBTX_HW_TSTAMP) && 869 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 870 /* skb now belongs to timestamp buffer 871 * and will be removed later 872 */ 873 tx_skb->skb = NULL; 874 } 875 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 876 macb_tx_ring_wrap(bp, tail), 877 skb->data); 878 bp->dev->stats.tx_packets++; 879 queue->stats.tx_packets++; 880 bp->dev->stats.tx_bytes += skb->len; 881 queue->stats.tx_bytes += skb->len; 882 } 883 884 /* Now we can safely release resources */ 885 macb_tx_unmap(bp, tx_skb); 886 887 /* skb is set only for the last buffer of the frame. 888 * WARNING: at this point skb has been freed by 889 * macb_tx_unmap(). 890 */ 891 if (skb) 892 break; 893 } 894 } 895 896 queue->tx_tail = tail; 897 if (__netif_subqueue_stopped(bp->dev, queue_index) && 898 CIRC_CNT(queue->tx_head, queue->tx_tail, 899 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 900 netif_wake_subqueue(bp->dev, queue_index); 901 } 902 903 static void gem_rx_refill(struct macb_queue *queue) 904 { 905 unsigned int entry; 906 struct sk_buff *skb; 907 dma_addr_t paddr; 908 struct macb *bp = queue->bp; 909 struct macb_dma_desc *desc; 910 911 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 912 bp->rx_ring_size) > 0) { 913 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 914 915 /* Make hw descriptor updates visible to CPU */ 916 rmb(); 917 918 queue->rx_prepared_head++; 919 desc = macb_rx_desc(queue, entry); 920 921 if (!queue->rx_skbuff[entry]) { 922 /* allocate sk_buff for this free entry in ring */ 923 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 924 if (unlikely(!skb)) { 925 netdev_err(bp->dev, 926 "Unable to allocate sk_buff\n"); 927 break; 928 } 929 930 /* now fill corresponding descriptor entry */ 931 paddr = dma_map_single(&bp->pdev->dev, skb->data, 932 bp->rx_buffer_size, 933 DMA_FROM_DEVICE); 934 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 935 dev_kfree_skb(skb); 936 break; 937 } 938 939 queue->rx_skbuff[entry] = skb; 940 941 if (entry == bp->rx_ring_size - 1) 942 paddr |= MACB_BIT(RX_WRAP); 943 desc->ctrl = 0; 944 /* Setting addr clears RX_USED and allows reception, 945 * make sure ctrl is cleared first to avoid a race. 946 */ 947 dma_wmb(); 948 macb_set_addr(bp, desc, paddr); 949 950 /* properly align Ethernet header */ 951 skb_reserve(skb, NET_IP_ALIGN); 952 } else { 953 desc->ctrl = 0; 954 dma_wmb(); 955 desc->addr &= ~MACB_BIT(RX_USED); 956 } 957 } 958 959 /* Make descriptor updates visible to hardware */ 960 wmb(); 961 962 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 963 queue, queue->rx_prepared_head, queue->rx_tail); 964 } 965 966 /* Mark DMA descriptors from begin up to and not including end as unused */ 967 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 968 unsigned int end) 969 { 970 unsigned int frag; 971 972 for (frag = begin; frag != end; frag++) { 973 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 974 975 desc->addr &= ~MACB_BIT(RX_USED); 976 } 977 978 /* Make descriptor updates visible to hardware */ 979 wmb(); 980 981 /* When this happens, the hardware stats registers for 982 * whatever caused this is updated, so we don't have to record 983 * anything. 984 */ 985 } 986 987 static int gem_rx(struct macb_queue *queue, int budget) 988 { 989 struct macb *bp = queue->bp; 990 unsigned int len; 991 unsigned int entry; 992 struct sk_buff *skb; 993 struct macb_dma_desc *desc; 994 int count = 0; 995 996 while (count < budget) { 997 u32 ctrl; 998 dma_addr_t addr; 999 bool rxused; 1000 1001 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1002 desc = macb_rx_desc(queue, entry); 1003 1004 /* Make hw descriptor updates visible to CPU */ 1005 rmb(); 1006 1007 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1008 addr = macb_get_addr(bp, desc); 1009 1010 if (!rxused) 1011 break; 1012 1013 /* Ensure ctrl is at least as up-to-date as rxused */ 1014 dma_rmb(); 1015 1016 ctrl = desc->ctrl; 1017 1018 queue->rx_tail++; 1019 count++; 1020 1021 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1022 netdev_err(bp->dev, 1023 "not whole frame pointed by descriptor\n"); 1024 bp->dev->stats.rx_dropped++; 1025 queue->stats.rx_dropped++; 1026 break; 1027 } 1028 skb = queue->rx_skbuff[entry]; 1029 if (unlikely(!skb)) { 1030 netdev_err(bp->dev, 1031 "inconsistent Rx descriptor chain\n"); 1032 bp->dev->stats.rx_dropped++; 1033 queue->stats.rx_dropped++; 1034 break; 1035 } 1036 /* now everything is ready for receiving packet */ 1037 queue->rx_skbuff[entry] = NULL; 1038 len = ctrl & bp->rx_frm_len_mask; 1039 1040 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1041 1042 skb_put(skb, len); 1043 dma_unmap_single(&bp->pdev->dev, addr, 1044 bp->rx_buffer_size, DMA_FROM_DEVICE); 1045 1046 skb->protocol = eth_type_trans(skb, bp->dev); 1047 skb_checksum_none_assert(skb); 1048 if (bp->dev->features & NETIF_F_RXCSUM && 1049 !(bp->dev->flags & IFF_PROMISC) && 1050 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1051 skb->ip_summed = CHECKSUM_UNNECESSARY; 1052 1053 bp->dev->stats.rx_packets++; 1054 queue->stats.rx_packets++; 1055 bp->dev->stats.rx_bytes += skb->len; 1056 queue->stats.rx_bytes += skb->len; 1057 1058 gem_ptp_do_rxstamp(bp, skb, desc); 1059 1060 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1061 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1062 skb->len, skb->csum); 1063 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1064 skb_mac_header(skb), 16, true); 1065 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1066 skb->data, 32, true); 1067 #endif 1068 1069 netif_receive_skb(skb); 1070 } 1071 1072 gem_rx_refill(queue); 1073 1074 return count; 1075 } 1076 1077 static int macb_rx_frame(struct macb_queue *queue, unsigned int first_frag, 1078 unsigned int last_frag) 1079 { 1080 unsigned int len; 1081 unsigned int frag; 1082 unsigned int offset; 1083 struct sk_buff *skb; 1084 struct macb_dma_desc *desc; 1085 struct macb *bp = queue->bp; 1086 1087 desc = macb_rx_desc(queue, last_frag); 1088 len = desc->ctrl & bp->rx_frm_len_mask; 1089 1090 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1091 macb_rx_ring_wrap(bp, first_frag), 1092 macb_rx_ring_wrap(bp, last_frag), len); 1093 1094 /* The ethernet header starts NET_IP_ALIGN bytes into the 1095 * first buffer. Since the header is 14 bytes, this makes the 1096 * payload word-aligned. 1097 * 1098 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1099 * the two padding bytes into the skb so that we avoid hitting 1100 * the slowpath in memcpy(), and pull them off afterwards. 1101 */ 1102 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1103 if (!skb) { 1104 bp->dev->stats.rx_dropped++; 1105 for (frag = first_frag; ; frag++) { 1106 desc = macb_rx_desc(queue, frag); 1107 desc->addr &= ~MACB_BIT(RX_USED); 1108 if (frag == last_frag) 1109 break; 1110 } 1111 1112 /* Make descriptor updates visible to hardware */ 1113 wmb(); 1114 1115 return 1; 1116 } 1117 1118 offset = 0; 1119 len += NET_IP_ALIGN; 1120 skb_checksum_none_assert(skb); 1121 skb_put(skb, len); 1122 1123 for (frag = first_frag; ; frag++) { 1124 unsigned int frag_len = bp->rx_buffer_size; 1125 1126 if (offset + frag_len > len) { 1127 if (unlikely(frag != last_frag)) { 1128 dev_kfree_skb_any(skb); 1129 return -1; 1130 } 1131 frag_len = len - offset; 1132 } 1133 skb_copy_to_linear_data_offset(skb, offset, 1134 macb_rx_buffer(queue, frag), 1135 frag_len); 1136 offset += bp->rx_buffer_size; 1137 desc = macb_rx_desc(queue, frag); 1138 desc->addr &= ~MACB_BIT(RX_USED); 1139 1140 if (frag == last_frag) 1141 break; 1142 } 1143 1144 /* Make descriptor updates visible to hardware */ 1145 wmb(); 1146 1147 __skb_pull(skb, NET_IP_ALIGN); 1148 skb->protocol = eth_type_trans(skb, bp->dev); 1149 1150 bp->dev->stats.rx_packets++; 1151 bp->dev->stats.rx_bytes += skb->len; 1152 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1153 skb->len, skb->csum); 1154 netif_receive_skb(skb); 1155 1156 return 0; 1157 } 1158 1159 static inline void macb_init_rx_ring(struct macb_queue *queue) 1160 { 1161 struct macb *bp = queue->bp; 1162 dma_addr_t addr; 1163 struct macb_dma_desc *desc = NULL; 1164 int i; 1165 1166 addr = queue->rx_buffers_dma; 1167 for (i = 0; i < bp->rx_ring_size; i++) { 1168 desc = macb_rx_desc(queue, i); 1169 macb_set_addr(bp, desc, addr); 1170 desc->ctrl = 0; 1171 addr += bp->rx_buffer_size; 1172 } 1173 desc->addr |= MACB_BIT(RX_WRAP); 1174 queue->rx_tail = 0; 1175 } 1176 1177 static int macb_rx(struct macb_queue *queue, int budget) 1178 { 1179 struct macb *bp = queue->bp; 1180 bool reset_rx_queue = false; 1181 int received = 0; 1182 unsigned int tail; 1183 int first_frag = -1; 1184 1185 for (tail = queue->rx_tail; budget > 0; tail++) { 1186 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1187 u32 ctrl; 1188 1189 /* Make hw descriptor updates visible to CPU */ 1190 rmb(); 1191 1192 if (!(desc->addr & MACB_BIT(RX_USED))) 1193 break; 1194 1195 /* Ensure ctrl is at least as up-to-date as addr */ 1196 dma_rmb(); 1197 1198 ctrl = desc->ctrl; 1199 1200 if (ctrl & MACB_BIT(RX_SOF)) { 1201 if (first_frag != -1) 1202 discard_partial_frame(queue, first_frag, tail); 1203 first_frag = tail; 1204 } 1205 1206 if (ctrl & MACB_BIT(RX_EOF)) { 1207 int dropped; 1208 1209 if (unlikely(first_frag == -1)) { 1210 reset_rx_queue = true; 1211 continue; 1212 } 1213 1214 dropped = macb_rx_frame(queue, first_frag, tail); 1215 first_frag = -1; 1216 if (unlikely(dropped < 0)) { 1217 reset_rx_queue = true; 1218 continue; 1219 } 1220 if (!dropped) { 1221 received++; 1222 budget--; 1223 } 1224 } 1225 } 1226 1227 if (unlikely(reset_rx_queue)) { 1228 unsigned long flags; 1229 u32 ctrl; 1230 1231 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1232 1233 spin_lock_irqsave(&bp->lock, flags); 1234 1235 ctrl = macb_readl(bp, NCR); 1236 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1237 1238 macb_init_rx_ring(queue); 1239 queue_writel(queue, RBQP, queue->rx_ring_dma); 1240 1241 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1242 1243 spin_unlock_irqrestore(&bp->lock, flags); 1244 return received; 1245 } 1246 1247 if (first_frag != -1) 1248 queue->rx_tail = first_frag; 1249 else 1250 queue->rx_tail = tail; 1251 1252 return received; 1253 } 1254 1255 static int macb_poll(struct napi_struct *napi, int budget) 1256 { 1257 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1258 struct macb *bp = queue->bp; 1259 int work_done; 1260 u32 status; 1261 1262 status = macb_readl(bp, RSR); 1263 macb_writel(bp, RSR, status); 1264 1265 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1266 (unsigned long)status, budget); 1267 1268 work_done = bp->macbgem_ops.mog_rx(queue, budget); 1269 if (work_done < budget) { 1270 napi_complete_done(napi, work_done); 1271 1272 /* Packets received while interrupts were disabled */ 1273 status = macb_readl(bp, RSR); 1274 if (status) { 1275 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1276 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1277 napi_reschedule(napi); 1278 } else { 1279 queue_writel(queue, IER, bp->rx_intr_mask); 1280 } 1281 } 1282 1283 /* TODO: Handle errors */ 1284 1285 return work_done; 1286 } 1287 1288 static void macb_hresp_error_task(unsigned long data) 1289 { 1290 struct macb *bp = (struct macb *)data; 1291 struct net_device *dev = bp->dev; 1292 struct macb_queue *queue = bp->queues; 1293 unsigned int q; 1294 u32 ctrl; 1295 1296 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1297 queue_writel(queue, IDR, bp->rx_intr_mask | 1298 MACB_TX_INT_FLAGS | 1299 MACB_BIT(HRESP)); 1300 } 1301 ctrl = macb_readl(bp, NCR); 1302 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1303 macb_writel(bp, NCR, ctrl); 1304 1305 netif_tx_stop_all_queues(dev); 1306 netif_carrier_off(dev); 1307 1308 bp->macbgem_ops.mog_init_rings(bp); 1309 1310 /* Initialize TX and RX buffers */ 1311 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1312 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 1313 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1314 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1315 queue_writel(queue, RBQPH, 1316 upper_32_bits(queue->rx_ring_dma)); 1317 #endif 1318 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1319 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1320 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1321 queue_writel(queue, TBQPH, 1322 upper_32_bits(queue->tx_ring_dma)); 1323 #endif 1324 1325 /* Enable interrupts */ 1326 queue_writel(queue, IER, 1327 bp->rx_intr_mask | 1328 MACB_TX_INT_FLAGS | 1329 MACB_BIT(HRESP)); 1330 } 1331 1332 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1333 macb_writel(bp, NCR, ctrl); 1334 1335 netif_carrier_on(dev); 1336 netif_tx_start_all_queues(dev); 1337 } 1338 1339 static void macb_tx_restart(struct macb_queue *queue) 1340 { 1341 unsigned int head = queue->tx_head; 1342 unsigned int tail = queue->tx_tail; 1343 struct macb *bp = queue->bp; 1344 1345 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1346 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1347 1348 if (head == tail) 1349 return; 1350 1351 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1352 } 1353 1354 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1355 { 1356 struct macb_queue *queue = dev_id; 1357 struct macb *bp = queue->bp; 1358 struct net_device *dev = bp->dev; 1359 u32 status, ctrl; 1360 1361 status = queue_readl(queue, ISR); 1362 1363 if (unlikely(!status)) 1364 return IRQ_NONE; 1365 1366 spin_lock(&bp->lock); 1367 1368 while (status) { 1369 /* close possible race with dev_close */ 1370 if (unlikely(!netif_running(dev))) { 1371 queue_writel(queue, IDR, -1); 1372 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1373 queue_writel(queue, ISR, -1); 1374 break; 1375 } 1376 1377 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1378 (unsigned int)(queue - bp->queues), 1379 (unsigned long)status); 1380 1381 if (status & bp->rx_intr_mask) { 1382 /* There's no point taking any more interrupts 1383 * until we have processed the buffers. The 1384 * scheduling call may fail if the poll routine 1385 * is already scheduled, so disable interrupts 1386 * now. 1387 */ 1388 queue_writel(queue, IDR, bp->rx_intr_mask); 1389 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1390 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1391 1392 if (napi_schedule_prep(&queue->napi)) { 1393 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1394 __napi_schedule(&queue->napi); 1395 } 1396 } 1397 1398 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1399 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1400 schedule_work(&queue->tx_error_task); 1401 1402 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1403 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1404 1405 break; 1406 } 1407 1408 if (status & MACB_BIT(TCOMP)) 1409 macb_tx_interrupt(queue); 1410 1411 if (status & MACB_BIT(TXUBR)) 1412 macb_tx_restart(queue); 1413 1414 /* Link change detection isn't possible with RMII, so we'll 1415 * add that if/when we get our hands on a full-blown MII PHY. 1416 */ 1417 1418 /* There is a hardware issue under heavy load where DMA can 1419 * stop, this causes endless "used buffer descriptor read" 1420 * interrupts but it can be cleared by re-enabling RX. See 1421 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1422 * section 16.7.4 for details. RXUBR is only enabled for 1423 * these two versions. 1424 */ 1425 if (status & MACB_BIT(RXUBR)) { 1426 ctrl = macb_readl(bp, NCR); 1427 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1428 wmb(); 1429 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1430 1431 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1432 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1433 } 1434 1435 if (status & MACB_BIT(ISR_ROVR)) { 1436 /* We missed at least one packet */ 1437 if (macb_is_gem(bp)) 1438 bp->hw_stats.gem.rx_overruns++; 1439 else 1440 bp->hw_stats.macb.rx_overruns++; 1441 1442 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1443 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1444 } 1445 1446 if (status & MACB_BIT(HRESP)) { 1447 tasklet_schedule(&bp->hresp_err_tasklet); 1448 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1449 1450 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1451 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1452 } 1453 status = queue_readl(queue, ISR); 1454 } 1455 1456 spin_unlock(&bp->lock); 1457 1458 return IRQ_HANDLED; 1459 } 1460 1461 #ifdef CONFIG_NET_POLL_CONTROLLER 1462 /* Polling receive - used by netconsole and other diagnostic tools 1463 * to allow network i/o with interrupts disabled. 1464 */ 1465 static void macb_poll_controller(struct net_device *dev) 1466 { 1467 struct macb *bp = netdev_priv(dev); 1468 struct macb_queue *queue; 1469 unsigned long flags; 1470 unsigned int q; 1471 1472 local_irq_save(flags); 1473 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1474 macb_interrupt(dev->irq, queue); 1475 local_irq_restore(flags); 1476 } 1477 #endif 1478 1479 static unsigned int macb_tx_map(struct macb *bp, 1480 struct macb_queue *queue, 1481 struct sk_buff *skb, 1482 unsigned int hdrlen) 1483 { 1484 dma_addr_t mapping; 1485 unsigned int len, entry, i, tx_head = queue->tx_head; 1486 struct macb_tx_skb *tx_skb = NULL; 1487 struct macb_dma_desc *desc; 1488 unsigned int offset, size, count = 0; 1489 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1490 unsigned int eof = 1, mss_mfs = 0; 1491 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1492 1493 /* LSO */ 1494 if (skb_shinfo(skb)->gso_size != 0) { 1495 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1496 /* UDP - UFO */ 1497 lso_ctrl = MACB_LSO_UFO_ENABLE; 1498 else 1499 /* TCP - TSO */ 1500 lso_ctrl = MACB_LSO_TSO_ENABLE; 1501 } 1502 1503 /* First, map non-paged data */ 1504 len = skb_headlen(skb); 1505 1506 /* first buffer length */ 1507 size = hdrlen; 1508 1509 offset = 0; 1510 while (len) { 1511 entry = macb_tx_ring_wrap(bp, tx_head); 1512 tx_skb = &queue->tx_skb[entry]; 1513 1514 mapping = dma_map_single(&bp->pdev->dev, 1515 skb->data + offset, 1516 size, DMA_TO_DEVICE); 1517 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1518 goto dma_error; 1519 1520 /* Save info to properly release resources */ 1521 tx_skb->skb = NULL; 1522 tx_skb->mapping = mapping; 1523 tx_skb->size = size; 1524 tx_skb->mapped_as_page = false; 1525 1526 len -= size; 1527 offset += size; 1528 count++; 1529 tx_head++; 1530 1531 size = min(len, bp->max_tx_length); 1532 } 1533 1534 /* Then, map paged data from fragments */ 1535 for (f = 0; f < nr_frags; f++) { 1536 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1537 1538 len = skb_frag_size(frag); 1539 offset = 0; 1540 while (len) { 1541 size = min(len, bp->max_tx_length); 1542 entry = macb_tx_ring_wrap(bp, tx_head); 1543 tx_skb = &queue->tx_skb[entry]; 1544 1545 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1546 offset, size, DMA_TO_DEVICE); 1547 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1548 goto dma_error; 1549 1550 /* Save info to properly release resources */ 1551 tx_skb->skb = NULL; 1552 tx_skb->mapping = mapping; 1553 tx_skb->size = size; 1554 tx_skb->mapped_as_page = true; 1555 1556 len -= size; 1557 offset += size; 1558 count++; 1559 tx_head++; 1560 } 1561 } 1562 1563 /* Should never happen */ 1564 if (unlikely(!tx_skb)) { 1565 netdev_err(bp->dev, "BUG! empty skb!\n"); 1566 return 0; 1567 } 1568 1569 /* This is the last buffer of the frame: save socket buffer */ 1570 tx_skb->skb = skb; 1571 1572 /* Update TX ring: update buffer descriptors in reverse order 1573 * to avoid race condition 1574 */ 1575 1576 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1577 * to set the end of TX queue 1578 */ 1579 i = tx_head; 1580 entry = macb_tx_ring_wrap(bp, i); 1581 ctrl = MACB_BIT(TX_USED); 1582 desc = macb_tx_desc(queue, entry); 1583 desc->ctrl = ctrl; 1584 1585 if (lso_ctrl) { 1586 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1587 /* include header and FCS in value given to h/w */ 1588 mss_mfs = skb_shinfo(skb)->gso_size + 1589 skb_transport_offset(skb) + 1590 ETH_FCS_LEN; 1591 else /* TSO */ { 1592 mss_mfs = skb_shinfo(skb)->gso_size; 1593 /* TCP Sequence Number Source Select 1594 * can be set only for TSO 1595 */ 1596 seq_ctrl = 0; 1597 } 1598 } 1599 1600 do { 1601 i--; 1602 entry = macb_tx_ring_wrap(bp, i); 1603 tx_skb = &queue->tx_skb[entry]; 1604 desc = macb_tx_desc(queue, entry); 1605 1606 ctrl = (u32)tx_skb->size; 1607 if (eof) { 1608 ctrl |= MACB_BIT(TX_LAST); 1609 eof = 0; 1610 } 1611 if (unlikely(entry == (bp->tx_ring_size - 1))) 1612 ctrl |= MACB_BIT(TX_WRAP); 1613 1614 /* First descriptor is header descriptor */ 1615 if (i == queue->tx_head) { 1616 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1617 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1618 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1619 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1620 ctrl |= MACB_BIT(TX_NOCRC); 1621 } else 1622 /* Only set MSS/MFS on payload descriptors 1623 * (second or later descriptor) 1624 */ 1625 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1626 1627 /* Set TX buffer descriptor */ 1628 macb_set_addr(bp, desc, tx_skb->mapping); 1629 /* desc->addr must be visible to hardware before clearing 1630 * 'TX_USED' bit in desc->ctrl. 1631 */ 1632 wmb(); 1633 desc->ctrl = ctrl; 1634 } while (i != queue->tx_head); 1635 1636 queue->tx_head = tx_head; 1637 1638 return count; 1639 1640 dma_error: 1641 netdev_err(bp->dev, "TX DMA map failed\n"); 1642 1643 for (i = queue->tx_head; i != tx_head; i++) { 1644 tx_skb = macb_tx_skb(queue, i); 1645 1646 macb_tx_unmap(bp, tx_skb); 1647 } 1648 1649 return 0; 1650 } 1651 1652 static netdev_features_t macb_features_check(struct sk_buff *skb, 1653 struct net_device *dev, 1654 netdev_features_t features) 1655 { 1656 unsigned int nr_frags, f; 1657 unsigned int hdrlen; 1658 1659 /* Validate LSO compatibility */ 1660 1661 /* there is only one buffer */ 1662 if (!skb_is_nonlinear(skb)) 1663 return features; 1664 1665 /* length of header */ 1666 hdrlen = skb_transport_offset(skb); 1667 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1668 hdrlen += tcp_hdrlen(skb); 1669 1670 /* For LSO: 1671 * When software supplies two or more payload buffers all payload buffers 1672 * apart from the last must be a multiple of 8 bytes in size. 1673 */ 1674 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1675 return features & ~MACB_NETIF_LSO; 1676 1677 nr_frags = skb_shinfo(skb)->nr_frags; 1678 /* No need to check last fragment */ 1679 nr_frags--; 1680 for (f = 0; f < nr_frags; f++) { 1681 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1682 1683 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1684 return features & ~MACB_NETIF_LSO; 1685 } 1686 return features; 1687 } 1688 1689 static inline int macb_clear_csum(struct sk_buff *skb) 1690 { 1691 /* no change for packets without checksum offloading */ 1692 if (skb->ip_summed != CHECKSUM_PARTIAL) 1693 return 0; 1694 1695 /* make sure we can modify the header */ 1696 if (unlikely(skb_cow_head(skb, 0))) 1697 return -1; 1698 1699 /* initialize checksum field 1700 * This is required - at least for Zynq, which otherwise calculates 1701 * wrong UDP header checksums for UDP packets with UDP data len <=2 1702 */ 1703 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1704 return 0; 1705 } 1706 1707 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1708 { 1709 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1710 int padlen = ETH_ZLEN - (*skb)->len; 1711 int headroom = skb_headroom(*skb); 1712 int tailroom = skb_tailroom(*skb); 1713 struct sk_buff *nskb; 1714 u32 fcs; 1715 1716 if (!(ndev->features & NETIF_F_HW_CSUM) || 1717 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1718 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1719 return 0; 1720 1721 if (padlen <= 0) { 1722 /* FCS could be appeded to tailroom. */ 1723 if (tailroom >= ETH_FCS_LEN) 1724 goto add_fcs; 1725 /* FCS could be appeded by moving data to headroom. */ 1726 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1727 padlen = 0; 1728 /* No room for FCS, need to reallocate skb. */ 1729 else 1730 padlen = ETH_FCS_LEN; 1731 } else { 1732 /* Add room for FCS. */ 1733 padlen += ETH_FCS_LEN; 1734 } 1735 1736 if (!cloned && headroom + tailroom >= padlen) { 1737 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1738 skb_set_tail_pointer(*skb, (*skb)->len); 1739 } else { 1740 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1741 if (!nskb) 1742 return -ENOMEM; 1743 1744 dev_consume_skb_any(*skb); 1745 *skb = nskb; 1746 } 1747 1748 if (padlen > ETH_FCS_LEN) 1749 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1750 1751 add_fcs: 1752 /* set FCS to packet */ 1753 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1754 fcs = ~fcs; 1755 1756 skb_put_u8(*skb, fcs & 0xff); 1757 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1758 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1759 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1760 1761 return 0; 1762 } 1763 1764 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1765 { 1766 u16 queue_index = skb_get_queue_mapping(skb); 1767 struct macb *bp = netdev_priv(dev); 1768 struct macb_queue *queue = &bp->queues[queue_index]; 1769 unsigned long flags; 1770 unsigned int desc_cnt, nr_frags, frag_size, f; 1771 unsigned int hdrlen; 1772 bool is_lso, is_udp = 0; 1773 netdev_tx_t ret = NETDEV_TX_OK; 1774 1775 if (macb_clear_csum(skb)) { 1776 dev_kfree_skb_any(skb); 1777 return ret; 1778 } 1779 1780 if (macb_pad_and_fcs(&skb, dev)) { 1781 dev_kfree_skb_any(skb); 1782 return ret; 1783 } 1784 1785 is_lso = (skb_shinfo(skb)->gso_size != 0); 1786 1787 if (is_lso) { 1788 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1789 1790 /* length of headers */ 1791 if (is_udp) 1792 /* only queue eth + ip headers separately for UDP */ 1793 hdrlen = skb_transport_offset(skb); 1794 else 1795 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1796 if (skb_headlen(skb) < hdrlen) { 1797 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1798 /* if this is required, would need to copy to single buffer */ 1799 return NETDEV_TX_BUSY; 1800 } 1801 } else 1802 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1803 1804 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1805 netdev_vdbg(bp->dev, 1806 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1807 queue_index, skb->len, skb->head, skb->data, 1808 skb_tail_pointer(skb), skb_end_pointer(skb)); 1809 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1810 skb->data, 16, true); 1811 #endif 1812 1813 /* Count how many TX buffer descriptors are needed to send this 1814 * socket buffer: skb fragments of jumbo frames may need to be 1815 * split into many buffer descriptors. 1816 */ 1817 if (is_lso && (skb_headlen(skb) > hdrlen)) 1818 /* extra header descriptor if also payload in first buffer */ 1819 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1820 else 1821 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1822 nr_frags = skb_shinfo(skb)->nr_frags; 1823 for (f = 0; f < nr_frags; f++) { 1824 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1825 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1826 } 1827 1828 spin_lock_irqsave(&bp->lock, flags); 1829 1830 /* This is a hard error, log it. */ 1831 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1832 bp->tx_ring_size) < desc_cnt) { 1833 netif_stop_subqueue(dev, queue_index); 1834 spin_unlock_irqrestore(&bp->lock, flags); 1835 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1836 queue->tx_head, queue->tx_tail); 1837 return NETDEV_TX_BUSY; 1838 } 1839 1840 /* Map socket buffer for DMA transfer */ 1841 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1842 dev_kfree_skb_any(skb); 1843 goto unlock; 1844 } 1845 1846 /* Make newly initialized descriptor visible to hardware */ 1847 wmb(); 1848 skb_tx_timestamp(skb); 1849 1850 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1851 1852 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1853 netif_stop_subqueue(dev, queue_index); 1854 1855 unlock: 1856 spin_unlock_irqrestore(&bp->lock, flags); 1857 1858 return ret; 1859 } 1860 1861 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1862 { 1863 if (!macb_is_gem(bp)) { 1864 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1865 } else { 1866 bp->rx_buffer_size = size; 1867 1868 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1869 netdev_dbg(bp->dev, 1870 "RX buffer must be multiple of %d bytes, expanding\n", 1871 RX_BUFFER_MULTIPLE); 1872 bp->rx_buffer_size = 1873 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1874 } 1875 } 1876 1877 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1878 bp->dev->mtu, bp->rx_buffer_size); 1879 } 1880 1881 static void gem_free_rx_buffers(struct macb *bp) 1882 { 1883 struct sk_buff *skb; 1884 struct macb_dma_desc *desc; 1885 struct macb_queue *queue; 1886 dma_addr_t addr; 1887 unsigned int q; 1888 int i; 1889 1890 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1891 if (!queue->rx_skbuff) 1892 continue; 1893 1894 for (i = 0; i < bp->rx_ring_size; i++) { 1895 skb = queue->rx_skbuff[i]; 1896 1897 if (!skb) 1898 continue; 1899 1900 desc = macb_rx_desc(queue, i); 1901 addr = macb_get_addr(bp, desc); 1902 1903 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1904 DMA_FROM_DEVICE); 1905 dev_kfree_skb_any(skb); 1906 skb = NULL; 1907 } 1908 1909 kfree(queue->rx_skbuff); 1910 queue->rx_skbuff = NULL; 1911 } 1912 } 1913 1914 static void macb_free_rx_buffers(struct macb *bp) 1915 { 1916 struct macb_queue *queue = &bp->queues[0]; 1917 1918 if (queue->rx_buffers) { 1919 dma_free_coherent(&bp->pdev->dev, 1920 bp->rx_ring_size * bp->rx_buffer_size, 1921 queue->rx_buffers, queue->rx_buffers_dma); 1922 queue->rx_buffers = NULL; 1923 } 1924 } 1925 1926 static void macb_free_consistent(struct macb *bp) 1927 { 1928 struct macb_queue *queue; 1929 unsigned int q; 1930 int size; 1931 1932 bp->macbgem_ops.mog_free_rx_buffers(bp); 1933 1934 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1935 kfree(queue->tx_skb); 1936 queue->tx_skb = NULL; 1937 if (queue->tx_ring) { 1938 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1939 dma_free_coherent(&bp->pdev->dev, size, 1940 queue->tx_ring, queue->tx_ring_dma); 1941 queue->tx_ring = NULL; 1942 } 1943 if (queue->rx_ring) { 1944 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 1945 dma_free_coherent(&bp->pdev->dev, size, 1946 queue->rx_ring, queue->rx_ring_dma); 1947 queue->rx_ring = NULL; 1948 } 1949 } 1950 } 1951 1952 static int gem_alloc_rx_buffers(struct macb *bp) 1953 { 1954 struct macb_queue *queue; 1955 unsigned int q; 1956 int size; 1957 1958 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1959 size = bp->rx_ring_size * sizeof(struct sk_buff *); 1960 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 1961 if (!queue->rx_skbuff) 1962 return -ENOMEM; 1963 else 1964 netdev_dbg(bp->dev, 1965 "Allocated %d RX struct sk_buff entries at %p\n", 1966 bp->rx_ring_size, queue->rx_skbuff); 1967 } 1968 return 0; 1969 } 1970 1971 static int macb_alloc_rx_buffers(struct macb *bp) 1972 { 1973 struct macb_queue *queue = &bp->queues[0]; 1974 int size; 1975 1976 size = bp->rx_ring_size * bp->rx_buffer_size; 1977 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 1978 &queue->rx_buffers_dma, GFP_KERNEL); 1979 if (!queue->rx_buffers) 1980 return -ENOMEM; 1981 1982 netdev_dbg(bp->dev, 1983 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 1984 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 1985 return 0; 1986 } 1987 1988 static int macb_alloc_consistent(struct macb *bp) 1989 { 1990 struct macb_queue *queue; 1991 unsigned int q; 1992 int size; 1993 1994 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1995 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1996 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1997 &queue->tx_ring_dma, 1998 GFP_KERNEL); 1999 if (!queue->tx_ring) 2000 goto out_err; 2001 netdev_dbg(bp->dev, 2002 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2003 q, size, (unsigned long)queue->tx_ring_dma, 2004 queue->tx_ring); 2005 2006 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2007 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2008 if (!queue->tx_skb) 2009 goto out_err; 2010 2011 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2012 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2013 &queue->rx_ring_dma, GFP_KERNEL); 2014 if (!queue->rx_ring) 2015 goto out_err; 2016 netdev_dbg(bp->dev, 2017 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2018 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2019 } 2020 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2021 goto out_err; 2022 2023 return 0; 2024 2025 out_err: 2026 macb_free_consistent(bp); 2027 return -ENOMEM; 2028 } 2029 2030 static void gem_init_rings(struct macb *bp) 2031 { 2032 struct macb_queue *queue; 2033 struct macb_dma_desc *desc = NULL; 2034 unsigned int q; 2035 int i; 2036 2037 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2038 for (i = 0; i < bp->tx_ring_size; i++) { 2039 desc = macb_tx_desc(queue, i); 2040 macb_set_addr(bp, desc, 0); 2041 desc->ctrl = MACB_BIT(TX_USED); 2042 } 2043 desc->ctrl |= MACB_BIT(TX_WRAP); 2044 queue->tx_head = 0; 2045 queue->tx_tail = 0; 2046 2047 queue->rx_tail = 0; 2048 queue->rx_prepared_head = 0; 2049 2050 gem_rx_refill(queue); 2051 } 2052 2053 } 2054 2055 static void macb_init_rings(struct macb *bp) 2056 { 2057 int i; 2058 struct macb_dma_desc *desc = NULL; 2059 2060 macb_init_rx_ring(&bp->queues[0]); 2061 2062 for (i = 0; i < bp->tx_ring_size; i++) { 2063 desc = macb_tx_desc(&bp->queues[0], i); 2064 macb_set_addr(bp, desc, 0); 2065 desc->ctrl = MACB_BIT(TX_USED); 2066 } 2067 bp->queues[0].tx_head = 0; 2068 bp->queues[0].tx_tail = 0; 2069 desc->ctrl |= MACB_BIT(TX_WRAP); 2070 } 2071 2072 static void macb_reset_hw(struct macb *bp) 2073 { 2074 struct macb_queue *queue; 2075 unsigned int q; 2076 u32 ctrl = macb_readl(bp, NCR); 2077 2078 /* Disable RX and TX (XXX: Should we halt the transmission 2079 * more gracefully?) 2080 */ 2081 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2082 2083 /* Clear the stats registers (XXX: Update stats first?) */ 2084 ctrl |= MACB_BIT(CLRSTAT); 2085 2086 macb_writel(bp, NCR, ctrl); 2087 2088 /* Clear all status flags */ 2089 macb_writel(bp, TSR, -1); 2090 macb_writel(bp, RSR, -1); 2091 2092 /* Disable all interrupts */ 2093 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2094 queue_writel(queue, IDR, -1); 2095 queue_readl(queue, ISR); 2096 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2097 queue_writel(queue, ISR, -1); 2098 } 2099 } 2100 2101 static u32 gem_mdc_clk_div(struct macb *bp) 2102 { 2103 u32 config; 2104 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2105 2106 if (pclk_hz <= 20000000) 2107 config = GEM_BF(CLK, GEM_CLK_DIV8); 2108 else if (pclk_hz <= 40000000) 2109 config = GEM_BF(CLK, GEM_CLK_DIV16); 2110 else if (pclk_hz <= 80000000) 2111 config = GEM_BF(CLK, GEM_CLK_DIV32); 2112 else if (pclk_hz <= 120000000) 2113 config = GEM_BF(CLK, GEM_CLK_DIV48); 2114 else if (pclk_hz <= 160000000) 2115 config = GEM_BF(CLK, GEM_CLK_DIV64); 2116 else 2117 config = GEM_BF(CLK, GEM_CLK_DIV96); 2118 2119 return config; 2120 } 2121 2122 static u32 macb_mdc_clk_div(struct macb *bp) 2123 { 2124 u32 config; 2125 unsigned long pclk_hz; 2126 2127 if (macb_is_gem(bp)) 2128 return gem_mdc_clk_div(bp); 2129 2130 pclk_hz = clk_get_rate(bp->pclk); 2131 if (pclk_hz <= 20000000) 2132 config = MACB_BF(CLK, MACB_CLK_DIV8); 2133 else if (pclk_hz <= 40000000) 2134 config = MACB_BF(CLK, MACB_CLK_DIV16); 2135 else if (pclk_hz <= 80000000) 2136 config = MACB_BF(CLK, MACB_CLK_DIV32); 2137 else 2138 config = MACB_BF(CLK, MACB_CLK_DIV64); 2139 2140 return config; 2141 } 2142 2143 /* Get the DMA bus width field of the network configuration register that we 2144 * should program. We find the width from decoding the design configuration 2145 * register to find the maximum supported data bus width. 2146 */ 2147 static u32 macb_dbw(struct macb *bp) 2148 { 2149 if (!macb_is_gem(bp)) 2150 return 0; 2151 2152 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2153 case 4: 2154 return GEM_BF(DBW, GEM_DBW128); 2155 case 2: 2156 return GEM_BF(DBW, GEM_DBW64); 2157 case 1: 2158 default: 2159 return GEM_BF(DBW, GEM_DBW32); 2160 } 2161 } 2162 2163 /* Configure the receive DMA engine 2164 * - use the correct receive buffer size 2165 * - set best burst length for DMA operations 2166 * (if not supported by FIFO, it will fallback to default) 2167 * - set both rx/tx packet buffers to full memory size 2168 * These are configurable parameters for GEM. 2169 */ 2170 static void macb_configure_dma(struct macb *bp) 2171 { 2172 struct macb_queue *queue; 2173 u32 buffer_size; 2174 unsigned int q; 2175 u32 dmacfg; 2176 2177 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2178 if (macb_is_gem(bp)) { 2179 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2180 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2181 if (q) 2182 queue_writel(queue, RBQS, buffer_size); 2183 else 2184 dmacfg |= GEM_BF(RXBS, buffer_size); 2185 } 2186 if (bp->dma_burst_length) 2187 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2188 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2189 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2190 2191 if (bp->native_io) 2192 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2193 else 2194 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2195 2196 if (bp->dev->features & NETIF_F_HW_CSUM) 2197 dmacfg |= GEM_BIT(TXCOEN); 2198 else 2199 dmacfg &= ~GEM_BIT(TXCOEN); 2200 2201 dmacfg &= ~GEM_BIT(ADDR64); 2202 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2203 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2204 dmacfg |= GEM_BIT(ADDR64); 2205 #endif 2206 #ifdef CONFIG_MACB_USE_HWSTAMP 2207 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2208 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2209 #endif 2210 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2211 dmacfg); 2212 gem_writel(bp, DMACFG, dmacfg); 2213 } 2214 } 2215 2216 static void macb_init_hw(struct macb *bp) 2217 { 2218 struct macb_queue *queue; 2219 unsigned int q; 2220 2221 u32 config; 2222 2223 macb_reset_hw(bp); 2224 macb_set_hwaddr(bp); 2225 2226 config = macb_mdc_clk_div(bp); 2227 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2228 config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2229 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2230 config |= MACB_BIT(PAE); /* PAuse Enable */ 2231 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2232 if (bp->caps & MACB_CAPS_JUMBO) 2233 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2234 else 2235 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2236 if (bp->dev->flags & IFF_PROMISC) 2237 config |= MACB_BIT(CAF); /* Copy All Frames */ 2238 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2239 config |= GEM_BIT(RXCOEN); 2240 if (!(bp->dev->flags & IFF_BROADCAST)) 2241 config |= MACB_BIT(NBC); /* No BroadCast */ 2242 config |= macb_dbw(bp); 2243 macb_writel(bp, NCFGR, config); 2244 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2245 gem_writel(bp, JML, bp->jumbo_max_len); 2246 bp->speed = SPEED_10; 2247 bp->duplex = DUPLEX_HALF; 2248 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2249 if (bp->caps & MACB_CAPS_JUMBO) 2250 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2251 2252 macb_configure_dma(bp); 2253 2254 /* Initialize TX and RX buffers */ 2255 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2256 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 2257 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2258 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2259 queue_writel(queue, RBQPH, upper_32_bits(queue->rx_ring_dma)); 2260 #endif 2261 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 2262 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2263 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2264 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 2265 #endif 2266 2267 /* Enable interrupts */ 2268 queue_writel(queue, IER, 2269 bp->rx_intr_mask | 2270 MACB_TX_INT_FLAGS | 2271 MACB_BIT(HRESP)); 2272 } 2273 2274 /* Enable TX and RX */ 2275 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 2276 } 2277 2278 /* The hash address register is 64 bits long and takes up two 2279 * locations in the memory map. The least significant bits are stored 2280 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2281 * 2282 * The unicast hash enable and the multicast hash enable bits in the 2283 * network configuration register enable the reception of hash matched 2284 * frames. The destination address is reduced to a 6 bit index into 2285 * the 64 bit hash register using the following hash function. The 2286 * hash function is an exclusive or of every sixth bit of the 2287 * destination address. 2288 * 2289 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2290 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2291 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2292 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2293 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2294 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2295 * 2296 * da[0] represents the least significant bit of the first byte 2297 * received, that is, the multicast/unicast indicator, and da[47] 2298 * represents the most significant bit of the last byte received. If 2299 * the hash index, hi[n], points to a bit that is set in the hash 2300 * register then the frame will be matched according to whether the 2301 * frame is multicast or unicast. A multicast match will be signalled 2302 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2303 * index points to a bit set in the hash register. A unicast match 2304 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2305 * and the hash index points to a bit set in the hash register. To 2306 * receive all multicast frames, the hash register should be set with 2307 * all ones and the multicast hash enable bit should be set in the 2308 * network configuration register. 2309 */ 2310 2311 static inline int hash_bit_value(int bitnr, __u8 *addr) 2312 { 2313 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2314 return 1; 2315 return 0; 2316 } 2317 2318 /* Return the hash index value for the specified address. */ 2319 static int hash_get_index(__u8 *addr) 2320 { 2321 int i, j, bitval; 2322 int hash_index = 0; 2323 2324 for (j = 0; j < 6; j++) { 2325 for (i = 0, bitval = 0; i < 8; i++) 2326 bitval ^= hash_bit_value(i * 6 + j, addr); 2327 2328 hash_index |= (bitval << j); 2329 } 2330 2331 return hash_index; 2332 } 2333 2334 /* Add multicast addresses to the internal multicast-hash table. */ 2335 static void macb_sethashtable(struct net_device *dev) 2336 { 2337 struct netdev_hw_addr *ha; 2338 unsigned long mc_filter[2]; 2339 unsigned int bitnr; 2340 struct macb *bp = netdev_priv(dev); 2341 2342 mc_filter[0] = 0; 2343 mc_filter[1] = 0; 2344 2345 netdev_for_each_mc_addr(ha, dev) { 2346 bitnr = hash_get_index(ha->addr); 2347 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2348 } 2349 2350 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2351 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2352 } 2353 2354 /* Enable/Disable promiscuous and multicast modes. */ 2355 static void macb_set_rx_mode(struct net_device *dev) 2356 { 2357 unsigned long cfg; 2358 struct macb *bp = netdev_priv(dev); 2359 2360 cfg = macb_readl(bp, NCFGR); 2361 2362 if (dev->flags & IFF_PROMISC) { 2363 /* Enable promiscuous mode */ 2364 cfg |= MACB_BIT(CAF); 2365 2366 /* Disable RX checksum offload */ 2367 if (macb_is_gem(bp)) 2368 cfg &= ~GEM_BIT(RXCOEN); 2369 } else { 2370 /* Disable promiscuous mode */ 2371 cfg &= ~MACB_BIT(CAF); 2372 2373 /* Enable RX checksum offload only if requested */ 2374 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2375 cfg |= GEM_BIT(RXCOEN); 2376 } 2377 2378 if (dev->flags & IFF_ALLMULTI) { 2379 /* Enable all multicast mode */ 2380 macb_or_gem_writel(bp, HRB, -1); 2381 macb_or_gem_writel(bp, HRT, -1); 2382 cfg |= MACB_BIT(NCFGR_MTI); 2383 } else if (!netdev_mc_empty(dev)) { 2384 /* Enable specific multicasts */ 2385 macb_sethashtable(dev); 2386 cfg |= MACB_BIT(NCFGR_MTI); 2387 } else if (dev->flags & (~IFF_ALLMULTI)) { 2388 /* Disable all multicast mode */ 2389 macb_or_gem_writel(bp, HRB, 0); 2390 macb_or_gem_writel(bp, HRT, 0); 2391 cfg &= ~MACB_BIT(NCFGR_MTI); 2392 } 2393 2394 macb_writel(bp, NCFGR, cfg); 2395 } 2396 2397 static int macb_open(struct net_device *dev) 2398 { 2399 struct macb *bp = netdev_priv(dev); 2400 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2401 struct macb_queue *queue; 2402 unsigned int q; 2403 int err; 2404 2405 netdev_dbg(bp->dev, "open\n"); 2406 2407 err = pm_runtime_get_sync(&bp->pdev->dev); 2408 if (err < 0) 2409 goto pm_exit; 2410 2411 /* carrier starts down */ 2412 netif_carrier_off(dev); 2413 2414 /* if the phy is not yet register, retry later*/ 2415 if (!dev->phydev) { 2416 err = -EAGAIN; 2417 goto pm_exit; 2418 } 2419 2420 /* RX buffers initialization */ 2421 macb_init_rx_buffer_size(bp, bufsz); 2422 2423 err = macb_alloc_consistent(bp); 2424 if (err) { 2425 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2426 err); 2427 goto pm_exit; 2428 } 2429 2430 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2431 napi_enable(&queue->napi); 2432 2433 bp->macbgem_ops.mog_init_rings(bp); 2434 macb_init_hw(bp); 2435 2436 /* schedule a link state check */ 2437 phy_start(dev->phydev); 2438 2439 netif_tx_start_all_queues(dev); 2440 2441 if (bp->ptp_info) 2442 bp->ptp_info->ptp_init(dev); 2443 2444 pm_exit: 2445 if (err) { 2446 pm_runtime_put_sync(&bp->pdev->dev); 2447 return err; 2448 } 2449 return 0; 2450 } 2451 2452 static int macb_close(struct net_device *dev) 2453 { 2454 struct macb *bp = netdev_priv(dev); 2455 struct macb_queue *queue; 2456 unsigned long flags; 2457 unsigned int q; 2458 2459 netif_tx_stop_all_queues(dev); 2460 2461 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2462 napi_disable(&queue->napi); 2463 2464 if (dev->phydev) 2465 phy_stop(dev->phydev); 2466 2467 spin_lock_irqsave(&bp->lock, flags); 2468 macb_reset_hw(bp); 2469 netif_carrier_off(dev); 2470 spin_unlock_irqrestore(&bp->lock, flags); 2471 2472 macb_free_consistent(bp); 2473 2474 if (bp->ptp_info) 2475 bp->ptp_info->ptp_remove(dev); 2476 2477 pm_runtime_put(&bp->pdev->dev); 2478 2479 return 0; 2480 } 2481 2482 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2483 { 2484 if (netif_running(dev)) 2485 return -EBUSY; 2486 2487 dev->mtu = new_mtu; 2488 2489 return 0; 2490 } 2491 2492 static void gem_update_stats(struct macb *bp) 2493 { 2494 struct macb_queue *queue; 2495 unsigned int i, q, idx; 2496 unsigned long *stat; 2497 2498 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2499 2500 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2501 u32 offset = gem_statistics[i].offset; 2502 u64 val = bp->macb_reg_readl(bp, offset); 2503 2504 bp->ethtool_stats[i] += val; 2505 *p += val; 2506 2507 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2508 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2509 val = bp->macb_reg_readl(bp, offset + 4); 2510 bp->ethtool_stats[i] += ((u64)val) << 32; 2511 *(++p) += val; 2512 } 2513 } 2514 2515 idx = GEM_STATS_LEN; 2516 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2517 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2518 bp->ethtool_stats[idx++] = *stat; 2519 } 2520 2521 static struct net_device_stats *gem_get_stats(struct macb *bp) 2522 { 2523 struct gem_stats *hwstat = &bp->hw_stats.gem; 2524 struct net_device_stats *nstat = &bp->dev->stats; 2525 2526 gem_update_stats(bp); 2527 2528 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2529 hwstat->rx_alignment_errors + 2530 hwstat->rx_resource_errors + 2531 hwstat->rx_overruns + 2532 hwstat->rx_oversize_frames + 2533 hwstat->rx_jabbers + 2534 hwstat->rx_undersized_frames + 2535 hwstat->rx_length_field_frame_errors); 2536 nstat->tx_errors = (hwstat->tx_late_collisions + 2537 hwstat->tx_excessive_collisions + 2538 hwstat->tx_underrun + 2539 hwstat->tx_carrier_sense_errors); 2540 nstat->multicast = hwstat->rx_multicast_frames; 2541 nstat->collisions = (hwstat->tx_single_collision_frames + 2542 hwstat->tx_multiple_collision_frames + 2543 hwstat->tx_excessive_collisions); 2544 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2545 hwstat->rx_jabbers + 2546 hwstat->rx_undersized_frames + 2547 hwstat->rx_length_field_frame_errors); 2548 nstat->rx_over_errors = hwstat->rx_resource_errors; 2549 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2550 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2551 nstat->rx_fifo_errors = hwstat->rx_overruns; 2552 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2553 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2554 nstat->tx_fifo_errors = hwstat->tx_underrun; 2555 2556 return nstat; 2557 } 2558 2559 static void gem_get_ethtool_stats(struct net_device *dev, 2560 struct ethtool_stats *stats, u64 *data) 2561 { 2562 struct macb *bp; 2563 2564 bp = netdev_priv(dev); 2565 gem_update_stats(bp); 2566 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2567 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2568 } 2569 2570 static int gem_get_sset_count(struct net_device *dev, int sset) 2571 { 2572 struct macb *bp = netdev_priv(dev); 2573 2574 switch (sset) { 2575 case ETH_SS_STATS: 2576 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2577 default: 2578 return -EOPNOTSUPP; 2579 } 2580 } 2581 2582 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2583 { 2584 char stat_string[ETH_GSTRING_LEN]; 2585 struct macb *bp = netdev_priv(dev); 2586 struct macb_queue *queue; 2587 unsigned int i; 2588 unsigned int q; 2589 2590 switch (sset) { 2591 case ETH_SS_STATS: 2592 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2593 memcpy(p, gem_statistics[i].stat_string, 2594 ETH_GSTRING_LEN); 2595 2596 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2597 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2598 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2599 q, queue_statistics[i].stat_string); 2600 memcpy(p, stat_string, ETH_GSTRING_LEN); 2601 } 2602 } 2603 break; 2604 } 2605 } 2606 2607 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2608 { 2609 struct macb *bp = netdev_priv(dev); 2610 struct net_device_stats *nstat = &bp->dev->stats; 2611 struct macb_stats *hwstat = &bp->hw_stats.macb; 2612 2613 if (macb_is_gem(bp)) 2614 return gem_get_stats(bp); 2615 2616 /* read stats from hardware */ 2617 macb_update_stats(bp); 2618 2619 /* Convert HW stats into netdevice stats */ 2620 nstat->rx_errors = (hwstat->rx_fcs_errors + 2621 hwstat->rx_align_errors + 2622 hwstat->rx_resource_errors + 2623 hwstat->rx_overruns + 2624 hwstat->rx_oversize_pkts + 2625 hwstat->rx_jabbers + 2626 hwstat->rx_undersize_pkts + 2627 hwstat->rx_length_mismatch); 2628 nstat->tx_errors = (hwstat->tx_late_cols + 2629 hwstat->tx_excessive_cols + 2630 hwstat->tx_underruns + 2631 hwstat->tx_carrier_errors + 2632 hwstat->sqe_test_errors); 2633 nstat->collisions = (hwstat->tx_single_cols + 2634 hwstat->tx_multiple_cols + 2635 hwstat->tx_excessive_cols); 2636 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2637 hwstat->rx_jabbers + 2638 hwstat->rx_undersize_pkts + 2639 hwstat->rx_length_mismatch); 2640 nstat->rx_over_errors = hwstat->rx_resource_errors + 2641 hwstat->rx_overruns; 2642 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2643 nstat->rx_frame_errors = hwstat->rx_align_errors; 2644 nstat->rx_fifo_errors = hwstat->rx_overruns; 2645 /* XXX: What does "missed" mean? */ 2646 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2647 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2648 nstat->tx_fifo_errors = hwstat->tx_underruns; 2649 /* Don't know about heartbeat or window errors... */ 2650 2651 return nstat; 2652 } 2653 2654 static int macb_get_regs_len(struct net_device *netdev) 2655 { 2656 return MACB_GREGS_NBR * sizeof(u32); 2657 } 2658 2659 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2660 void *p) 2661 { 2662 struct macb *bp = netdev_priv(dev); 2663 unsigned int tail, head; 2664 u32 *regs_buff = p; 2665 2666 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2667 | MACB_GREGS_VERSION; 2668 2669 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2670 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2671 2672 regs_buff[0] = macb_readl(bp, NCR); 2673 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2674 regs_buff[2] = macb_readl(bp, NSR); 2675 regs_buff[3] = macb_readl(bp, TSR); 2676 regs_buff[4] = macb_readl(bp, RBQP); 2677 regs_buff[5] = macb_readl(bp, TBQP); 2678 regs_buff[6] = macb_readl(bp, RSR); 2679 regs_buff[7] = macb_readl(bp, IMR); 2680 2681 regs_buff[8] = tail; 2682 regs_buff[9] = head; 2683 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2684 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2685 2686 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2687 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2688 if (macb_is_gem(bp)) 2689 regs_buff[13] = gem_readl(bp, DMACFG); 2690 } 2691 2692 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2693 { 2694 struct macb *bp = netdev_priv(netdev); 2695 2696 wol->supported = 0; 2697 wol->wolopts = 0; 2698 2699 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2700 wol->supported = WAKE_MAGIC; 2701 2702 if (bp->wol & MACB_WOL_ENABLED) 2703 wol->wolopts |= WAKE_MAGIC; 2704 } 2705 } 2706 2707 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2708 { 2709 struct macb *bp = netdev_priv(netdev); 2710 2711 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2712 (wol->wolopts & ~WAKE_MAGIC)) 2713 return -EOPNOTSUPP; 2714 2715 if (wol->wolopts & WAKE_MAGIC) 2716 bp->wol |= MACB_WOL_ENABLED; 2717 else 2718 bp->wol &= ~MACB_WOL_ENABLED; 2719 2720 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2721 2722 return 0; 2723 } 2724 2725 static void macb_get_ringparam(struct net_device *netdev, 2726 struct ethtool_ringparam *ring) 2727 { 2728 struct macb *bp = netdev_priv(netdev); 2729 2730 ring->rx_max_pending = MAX_RX_RING_SIZE; 2731 ring->tx_max_pending = MAX_TX_RING_SIZE; 2732 2733 ring->rx_pending = bp->rx_ring_size; 2734 ring->tx_pending = bp->tx_ring_size; 2735 } 2736 2737 static int macb_set_ringparam(struct net_device *netdev, 2738 struct ethtool_ringparam *ring) 2739 { 2740 struct macb *bp = netdev_priv(netdev); 2741 u32 new_rx_size, new_tx_size; 2742 unsigned int reset = 0; 2743 2744 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2745 return -EINVAL; 2746 2747 new_rx_size = clamp_t(u32, ring->rx_pending, 2748 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2749 new_rx_size = roundup_pow_of_two(new_rx_size); 2750 2751 new_tx_size = clamp_t(u32, ring->tx_pending, 2752 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2753 new_tx_size = roundup_pow_of_two(new_tx_size); 2754 2755 if ((new_tx_size == bp->tx_ring_size) && 2756 (new_rx_size == bp->rx_ring_size)) { 2757 /* nothing to do */ 2758 return 0; 2759 } 2760 2761 if (netif_running(bp->dev)) { 2762 reset = 1; 2763 macb_close(bp->dev); 2764 } 2765 2766 bp->rx_ring_size = new_rx_size; 2767 bp->tx_ring_size = new_tx_size; 2768 2769 if (reset) 2770 macb_open(bp->dev); 2771 2772 return 0; 2773 } 2774 2775 #ifdef CONFIG_MACB_USE_HWSTAMP 2776 static unsigned int gem_get_tsu_rate(struct macb *bp) 2777 { 2778 struct clk *tsu_clk; 2779 unsigned int tsu_rate; 2780 2781 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2782 if (!IS_ERR(tsu_clk)) 2783 tsu_rate = clk_get_rate(tsu_clk); 2784 /* try pclk instead */ 2785 else if (!IS_ERR(bp->pclk)) { 2786 tsu_clk = bp->pclk; 2787 tsu_rate = clk_get_rate(tsu_clk); 2788 } else 2789 return -ENOTSUPP; 2790 return tsu_rate; 2791 } 2792 2793 static s32 gem_get_ptp_max_adj(void) 2794 { 2795 return 64000000; 2796 } 2797 2798 static int gem_get_ts_info(struct net_device *dev, 2799 struct ethtool_ts_info *info) 2800 { 2801 struct macb *bp = netdev_priv(dev); 2802 2803 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2804 ethtool_op_get_ts_info(dev, info); 2805 return 0; 2806 } 2807 2808 info->so_timestamping = 2809 SOF_TIMESTAMPING_TX_SOFTWARE | 2810 SOF_TIMESTAMPING_RX_SOFTWARE | 2811 SOF_TIMESTAMPING_SOFTWARE | 2812 SOF_TIMESTAMPING_TX_HARDWARE | 2813 SOF_TIMESTAMPING_RX_HARDWARE | 2814 SOF_TIMESTAMPING_RAW_HARDWARE; 2815 info->tx_types = 2816 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2817 (1 << HWTSTAMP_TX_OFF) | 2818 (1 << HWTSTAMP_TX_ON); 2819 info->rx_filters = 2820 (1 << HWTSTAMP_FILTER_NONE) | 2821 (1 << HWTSTAMP_FILTER_ALL); 2822 2823 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2824 2825 return 0; 2826 } 2827 2828 static struct macb_ptp_info gem_ptp_info = { 2829 .ptp_init = gem_ptp_init, 2830 .ptp_remove = gem_ptp_remove, 2831 .get_ptp_max_adj = gem_get_ptp_max_adj, 2832 .get_tsu_rate = gem_get_tsu_rate, 2833 .get_ts_info = gem_get_ts_info, 2834 .get_hwtst = gem_get_hwtst, 2835 .set_hwtst = gem_set_hwtst, 2836 }; 2837 #endif 2838 2839 static int macb_get_ts_info(struct net_device *netdev, 2840 struct ethtool_ts_info *info) 2841 { 2842 struct macb *bp = netdev_priv(netdev); 2843 2844 if (bp->ptp_info) 2845 return bp->ptp_info->get_ts_info(netdev, info); 2846 2847 return ethtool_op_get_ts_info(netdev, info); 2848 } 2849 2850 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2851 { 2852 struct net_device *netdev = bp->dev; 2853 struct ethtool_rx_fs_item *item; 2854 u32 t2_scr; 2855 int num_t2_scr; 2856 2857 if (!(netdev->features & NETIF_F_NTUPLE)) 2858 return; 2859 2860 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2861 2862 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2863 struct ethtool_rx_flow_spec *fs = &item->fs; 2864 struct ethtool_tcpip4_spec *tp4sp_m; 2865 2866 if (fs->location >= num_t2_scr) 2867 continue; 2868 2869 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2870 2871 /* enable/disable screener regs for the flow entry */ 2872 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2873 2874 /* only enable fields with no masking */ 2875 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2876 2877 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2878 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2879 else 2880 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2881 2882 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2883 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2884 else 2885 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2886 2887 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2888 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2889 else 2890 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 2891 2892 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 2893 } 2894 } 2895 2896 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 2897 { 2898 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 2899 uint16_t index = fs->location; 2900 u32 w0, w1, t2_scr; 2901 bool cmp_a = false; 2902 bool cmp_b = false; 2903 bool cmp_c = false; 2904 2905 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 2906 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2907 2908 /* ignore field if any masking set */ 2909 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 2910 /* 1st compare reg - IP source address */ 2911 w0 = 0; 2912 w1 = 0; 2913 w0 = tp4sp_v->ip4src; 2914 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2915 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2916 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 2917 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 2918 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 2919 cmp_a = true; 2920 } 2921 2922 /* ignore field if any masking set */ 2923 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 2924 /* 2nd compare reg - IP destination address */ 2925 w0 = 0; 2926 w1 = 0; 2927 w0 = tp4sp_v->ip4dst; 2928 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2929 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2930 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 2931 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 2932 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 2933 cmp_b = true; 2934 } 2935 2936 /* ignore both port fields if masking set in both */ 2937 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 2938 /* 3rd compare reg - source port, destination port */ 2939 w0 = 0; 2940 w1 = 0; 2941 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 2942 if (tp4sp_m->psrc == tp4sp_m->pdst) { 2943 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 2944 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2945 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2946 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2947 } else { 2948 /* only one port definition */ 2949 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 2950 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 2951 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 2952 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 2953 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2954 } else { /* dst port */ 2955 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2956 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 2957 } 2958 } 2959 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 2960 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 2961 cmp_c = true; 2962 } 2963 2964 t2_scr = 0; 2965 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 2966 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 2967 if (cmp_a) 2968 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 2969 if (cmp_b) 2970 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 2971 if (cmp_c) 2972 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 2973 gem_writel_n(bp, SCRT2, index, t2_scr); 2974 } 2975 2976 static int gem_add_flow_filter(struct net_device *netdev, 2977 struct ethtool_rxnfc *cmd) 2978 { 2979 struct macb *bp = netdev_priv(netdev); 2980 struct ethtool_rx_flow_spec *fs = &cmd->fs; 2981 struct ethtool_rx_fs_item *item, *newfs; 2982 unsigned long flags; 2983 int ret = -EINVAL; 2984 bool added = false; 2985 2986 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 2987 if (newfs == NULL) 2988 return -ENOMEM; 2989 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 2990 2991 netdev_dbg(netdev, 2992 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2993 fs->flow_type, (int)fs->ring_cookie, fs->location, 2994 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2995 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2996 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 2997 2998 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2999 3000 /* find correct place to add in list */ 3001 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3002 if (item->fs.location > newfs->fs.location) { 3003 list_add_tail(&newfs->list, &item->list); 3004 added = true; 3005 break; 3006 } else if (item->fs.location == fs->location) { 3007 netdev_err(netdev, "Rule not added: location %d not free!\n", 3008 fs->location); 3009 ret = -EBUSY; 3010 goto err; 3011 } 3012 } 3013 if (!added) 3014 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3015 3016 gem_prog_cmp_regs(bp, fs); 3017 bp->rx_fs_list.count++; 3018 /* enable filtering if NTUPLE on */ 3019 gem_enable_flow_filters(bp, 1); 3020 3021 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3022 return 0; 3023 3024 err: 3025 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3026 kfree(newfs); 3027 return ret; 3028 } 3029 3030 static int gem_del_flow_filter(struct net_device *netdev, 3031 struct ethtool_rxnfc *cmd) 3032 { 3033 struct macb *bp = netdev_priv(netdev); 3034 struct ethtool_rx_fs_item *item; 3035 struct ethtool_rx_flow_spec *fs; 3036 unsigned long flags; 3037 3038 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3039 3040 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3041 if (item->fs.location == cmd->fs.location) { 3042 /* disable screener regs for the flow entry */ 3043 fs = &(item->fs); 3044 netdev_dbg(netdev, 3045 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3046 fs->flow_type, (int)fs->ring_cookie, fs->location, 3047 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3048 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3049 htons(fs->h_u.tcp_ip4_spec.psrc), 3050 htons(fs->h_u.tcp_ip4_spec.pdst)); 3051 3052 gem_writel_n(bp, SCRT2, fs->location, 0); 3053 3054 list_del(&item->list); 3055 bp->rx_fs_list.count--; 3056 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3057 kfree(item); 3058 return 0; 3059 } 3060 } 3061 3062 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3063 return -EINVAL; 3064 } 3065 3066 static int gem_get_flow_entry(struct net_device *netdev, 3067 struct ethtool_rxnfc *cmd) 3068 { 3069 struct macb *bp = netdev_priv(netdev); 3070 struct ethtool_rx_fs_item *item; 3071 3072 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3073 if (item->fs.location == cmd->fs.location) { 3074 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3075 return 0; 3076 } 3077 } 3078 return -EINVAL; 3079 } 3080 3081 static int gem_get_all_flow_entries(struct net_device *netdev, 3082 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3083 { 3084 struct macb *bp = netdev_priv(netdev); 3085 struct ethtool_rx_fs_item *item; 3086 uint32_t cnt = 0; 3087 3088 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3089 if (cnt == cmd->rule_cnt) 3090 return -EMSGSIZE; 3091 rule_locs[cnt] = item->fs.location; 3092 cnt++; 3093 } 3094 cmd->data = bp->max_tuples; 3095 cmd->rule_cnt = cnt; 3096 3097 return 0; 3098 } 3099 3100 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3101 u32 *rule_locs) 3102 { 3103 struct macb *bp = netdev_priv(netdev); 3104 int ret = 0; 3105 3106 switch (cmd->cmd) { 3107 case ETHTOOL_GRXRINGS: 3108 cmd->data = bp->num_queues; 3109 break; 3110 case ETHTOOL_GRXCLSRLCNT: 3111 cmd->rule_cnt = bp->rx_fs_list.count; 3112 break; 3113 case ETHTOOL_GRXCLSRULE: 3114 ret = gem_get_flow_entry(netdev, cmd); 3115 break; 3116 case ETHTOOL_GRXCLSRLALL: 3117 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3118 break; 3119 default: 3120 netdev_err(netdev, 3121 "Command parameter %d is not supported\n", cmd->cmd); 3122 ret = -EOPNOTSUPP; 3123 } 3124 3125 return ret; 3126 } 3127 3128 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3129 { 3130 struct macb *bp = netdev_priv(netdev); 3131 int ret; 3132 3133 switch (cmd->cmd) { 3134 case ETHTOOL_SRXCLSRLINS: 3135 if ((cmd->fs.location >= bp->max_tuples) 3136 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3137 ret = -EINVAL; 3138 break; 3139 } 3140 ret = gem_add_flow_filter(netdev, cmd); 3141 break; 3142 case ETHTOOL_SRXCLSRLDEL: 3143 ret = gem_del_flow_filter(netdev, cmd); 3144 break; 3145 default: 3146 netdev_err(netdev, 3147 "Command parameter %d is not supported\n", cmd->cmd); 3148 ret = -EOPNOTSUPP; 3149 } 3150 3151 return ret; 3152 } 3153 3154 static const struct ethtool_ops macb_ethtool_ops = { 3155 .get_regs_len = macb_get_regs_len, 3156 .get_regs = macb_get_regs, 3157 .get_link = ethtool_op_get_link, 3158 .get_ts_info = ethtool_op_get_ts_info, 3159 .get_wol = macb_get_wol, 3160 .set_wol = macb_set_wol, 3161 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3162 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3163 .get_ringparam = macb_get_ringparam, 3164 .set_ringparam = macb_set_ringparam, 3165 }; 3166 3167 static const struct ethtool_ops gem_ethtool_ops = { 3168 .get_regs_len = macb_get_regs_len, 3169 .get_regs = macb_get_regs, 3170 .get_link = ethtool_op_get_link, 3171 .get_ts_info = macb_get_ts_info, 3172 .get_ethtool_stats = gem_get_ethtool_stats, 3173 .get_strings = gem_get_ethtool_strings, 3174 .get_sset_count = gem_get_sset_count, 3175 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3176 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3177 .get_ringparam = macb_get_ringparam, 3178 .set_ringparam = macb_set_ringparam, 3179 .get_rxnfc = gem_get_rxnfc, 3180 .set_rxnfc = gem_set_rxnfc, 3181 }; 3182 3183 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3184 { 3185 struct phy_device *phydev = dev->phydev; 3186 struct macb *bp = netdev_priv(dev); 3187 3188 if (!netif_running(dev)) 3189 return -EINVAL; 3190 3191 if (!phydev) 3192 return -ENODEV; 3193 3194 if (!bp->ptp_info) 3195 return phy_mii_ioctl(phydev, rq, cmd); 3196 3197 switch (cmd) { 3198 case SIOCSHWTSTAMP: 3199 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3200 case SIOCGHWTSTAMP: 3201 return bp->ptp_info->get_hwtst(dev, rq); 3202 default: 3203 return phy_mii_ioctl(phydev, rq, cmd); 3204 } 3205 } 3206 3207 static inline void macb_set_txcsum_feature(struct macb *bp, 3208 netdev_features_t features) 3209 { 3210 u32 val; 3211 3212 if (!macb_is_gem(bp)) 3213 return; 3214 3215 val = gem_readl(bp, DMACFG); 3216 if (features & NETIF_F_HW_CSUM) 3217 val |= GEM_BIT(TXCOEN); 3218 else 3219 val &= ~GEM_BIT(TXCOEN); 3220 3221 gem_writel(bp, DMACFG, val); 3222 } 3223 3224 static inline void macb_set_rxcsum_feature(struct macb *bp, 3225 netdev_features_t features) 3226 { 3227 struct net_device *netdev = bp->dev; 3228 u32 val; 3229 3230 if (!macb_is_gem(bp)) 3231 return; 3232 3233 val = gem_readl(bp, NCFGR); 3234 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 3235 val |= GEM_BIT(RXCOEN); 3236 else 3237 val &= ~GEM_BIT(RXCOEN); 3238 3239 gem_writel(bp, NCFGR, val); 3240 } 3241 3242 static inline void macb_set_rxflow_feature(struct macb *bp, 3243 netdev_features_t features) 3244 { 3245 if (!macb_is_gem(bp)) 3246 return; 3247 3248 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 3249 } 3250 3251 static int macb_set_features(struct net_device *netdev, 3252 netdev_features_t features) 3253 { 3254 struct macb *bp = netdev_priv(netdev); 3255 netdev_features_t changed = features ^ netdev->features; 3256 3257 /* TX checksum offload */ 3258 if (changed & NETIF_F_HW_CSUM) 3259 macb_set_txcsum_feature(bp, features); 3260 3261 /* RX checksum offload */ 3262 if (changed & NETIF_F_RXCSUM) 3263 macb_set_rxcsum_feature(bp, features); 3264 3265 /* RX Flow Filters */ 3266 if (changed & NETIF_F_NTUPLE) 3267 macb_set_rxflow_feature(bp, features); 3268 3269 return 0; 3270 } 3271 3272 static void macb_restore_features(struct macb *bp) 3273 { 3274 struct net_device *netdev = bp->dev; 3275 netdev_features_t features = netdev->features; 3276 3277 /* TX checksum offload */ 3278 macb_set_txcsum_feature(bp, features); 3279 3280 /* RX checksum offload */ 3281 macb_set_rxcsum_feature(bp, features); 3282 3283 /* RX Flow Filters */ 3284 macb_set_rxflow_feature(bp, features); 3285 } 3286 3287 static const struct net_device_ops macb_netdev_ops = { 3288 .ndo_open = macb_open, 3289 .ndo_stop = macb_close, 3290 .ndo_start_xmit = macb_start_xmit, 3291 .ndo_set_rx_mode = macb_set_rx_mode, 3292 .ndo_get_stats = macb_get_stats, 3293 .ndo_do_ioctl = macb_ioctl, 3294 .ndo_validate_addr = eth_validate_addr, 3295 .ndo_change_mtu = macb_change_mtu, 3296 .ndo_set_mac_address = eth_mac_addr, 3297 #ifdef CONFIG_NET_POLL_CONTROLLER 3298 .ndo_poll_controller = macb_poll_controller, 3299 #endif 3300 .ndo_set_features = macb_set_features, 3301 .ndo_features_check = macb_features_check, 3302 }; 3303 3304 /* Configure peripheral capabilities according to device tree 3305 * and integration options used 3306 */ 3307 static void macb_configure_caps(struct macb *bp, 3308 const struct macb_config *dt_conf) 3309 { 3310 u32 dcfg; 3311 3312 if (dt_conf) 3313 bp->caps = dt_conf->caps; 3314 3315 if (hw_is_gem(bp->regs, bp->native_io)) { 3316 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3317 3318 dcfg = gem_readl(bp, DCFG1); 3319 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3320 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3321 dcfg = gem_readl(bp, DCFG2); 3322 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3323 bp->caps |= MACB_CAPS_FIFO_MODE; 3324 #ifdef CONFIG_MACB_USE_HWSTAMP 3325 if (gem_has_ptp(bp)) { 3326 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3327 pr_err("GEM doesn't support hardware ptp.\n"); 3328 else { 3329 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3330 bp->ptp_info = &gem_ptp_info; 3331 } 3332 } 3333 #endif 3334 } 3335 3336 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3337 } 3338 3339 static void macb_probe_queues(void __iomem *mem, 3340 bool native_io, 3341 unsigned int *queue_mask, 3342 unsigned int *num_queues) 3343 { 3344 unsigned int hw_q; 3345 3346 *queue_mask = 0x1; 3347 *num_queues = 1; 3348 3349 /* is it macb or gem ? 3350 * 3351 * We need to read directly from the hardware here because 3352 * we are early in the probe process and don't have the 3353 * MACB_CAPS_MACB_IS_GEM flag positioned 3354 */ 3355 if (!hw_is_gem(mem, native_io)) 3356 return; 3357 3358 /* bit 0 is never set but queue 0 always exists */ 3359 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3360 3361 *queue_mask |= 0x1; 3362 3363 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3364 if (*queue_mask & (1 << hw_q)) 3365 (*num_queues)++; 3366 } 3367 3368 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3369 struct clk **hclk, struct clk **tx_clk, 3370 struct clk **rx_clk, struct clk **tsu_clk) 3371 { 3372 struct macb_platform_data *pdata; 3373 int err; 3374 3375 pdata = dev_get_platdata(&pdev->dev); 3376 if (pdata) { 3377 *pclk = pdata->pclk; 3378 *hclk = pdata->hclk; 3379 } else { 3380 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3381 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3382 } 3383 3384 if (IS_ERR_OR_NULL(*pclk)) { 3385 err = PTR_ERR(*pclk); 3386 if (!err) 3387 err = -ENODEV; 3388 3389 dev_err(&pdev->dev, "failed to get macb_clk (%d)\n", err); 3390 return err; 3391 } 3392 3393 if (IS_ERR_OR_NULL(*hclk)) { 3394 err = PTR_ERR(*hclk); 3395 if (!err) 3396 err = -ENODEV; 3397 3398 dev_err(&pdev->dev, "failed to get hclk (%d)\n", err); 3399 return err; 3400 } 3401 3402 *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); 3403 if (IS_ERR(*tx_clk)) 3404 *tx_clk = NULL; 3405 3406 *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); 3407 if (IS_ERR(*rx_clk)) 3408 *rx_clk = NULL; 3409 3410 *tsu_clk = devm_clk_get(&pdev->dev, "tsu_clk"); 3411 if (IS_ERR(*tsu_clk)) 3412 *tsu_clk = NULL; 3413 3414 err = clk_prepare_enable(*pclk); 3415 if (err) { 3416 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3417 return err; 3418 } 3419 3420 err = clk_prepare_enable(*hclk); 3421 if (err) { 3422 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 3423 goto err_disable_pclk; 3424 } 3425 3426 err = clk_prepare_enable(*tx_clk); 3427 if (err) { 3428 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 3429 goto err_disable_hclk; 3430 } 3431 3432 err = clk_prepare_enable(*rx_clk); 3433 if (err) { 3434 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 3435 goto err_disable_txclk; 3436 } 3437 3438 err = clk_prepare_enable(*tsu_clk); 3439 if (err) { 3440 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 3441 goto err_disable_rxclk; 3442 } 3443 3444 return 0; 3445 3446 err_disable_rxclk: 3447 clk_disable_unprepare(*rx_clk); 3448 3449 err_disable_txclk: 3450 clk_disable_unprepare(*tx_clk); 3451 3452 err_disable_hclk: 3453 clk_disable_unprepare(*hclk); 3454 3455 err_disable_pclk: 3456 clk_disable_unprepare(*pclk); 3457 3458 return err; 3459 } 3460 3461 static int macb_init(struct platform_device *pdev) 3462 { 3463 struct net_device *dev = platform_get_drvdata(pdev); 3464 unsigned int hw_q, q; 3465 struct macb *bp = netdev_priv(dev); 3466 struct macb_queue *queue; 3467 int err; 3468 u32 val, reg; 3469 3470 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3471 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3472 3473 /* set the queue register mapping once for all: queue0 has a special 3474 * register mapping but we don't want to test the queue index then 3475 * compute the corresponding register offset at run time. 3476 */ 3477 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3478 if (!(bp->queue_mask & (1 << hw_q))) 3479 continue; 3480 3481 queue = &bp->queues[q]; 3482 queue->bp = bp; 3483 netif_napi_add(dev, &queue->napi, macb_poll, 64); 3484 if (hw_q) { 3485 queue->ISR = GEM_ISR(hw_q - 1); 3486 queue->IER = GEM_IER(hw_q - 1); 3487 queue->IDR = GEM_IDR(hw_q - 1); 3488 queue->IMR = GEM_IMR(hw_q - 1); 3489 queue->TBQP = GEM_TBQP(hw_q - 1); 3490 queue->RBQP = GEM_RBQP(hw_q - 1); 3491 queue->RBQS = GEM_RBQS(hw_q - 1); 3492 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3493 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3494 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3495 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3496 } 3497 #endif 3498 } else { 3499 /* queue0 uses legacy registers */ 3500 queue->ISR = MACB_ISR; 3501 queue->IER = MACB_IER; 3502 queue->IDR = MACB_IDR; 3503 queue->IMR = MACB_IMR; 3504 queue->TBQP = MACB_TBQP; 3505 queue->RBQP = MACB_RBQP; 3506 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3507 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3508 queue->TBQPH = MACB_TBQPH; 3509 queue->RBQPH = MACB_RBQPH; 3510 } 3511 #endif 3512 } 3513 3514 /* get irq: here we use the linux queue index, not the hardware 3515 * queue index. the queue irq definitions in the device tree 3516 * must remove the optional gaps that could exist in the 3517 * hardware queue mask. 3518 */ 3519 queue->irq = platform_get_irq(pdev, q); 3520 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3521 IRQF_SHARED, dev->name, queue); 3522 if (err) { 3523 dev_err(&pdev->dev, 3524 "Unable to request IRQ %d (error %d)\n", 3525 queue->irq, err); 3526 return err; 3527 } 3528 3529 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3530 q++; 3531 } 3532 3533 dev->netdev_ops = &macb_netdev_ops; 3534 3535 /* setup appropriated routines according to adapter type */ 3536 if (macb_is_gem(bp)) { 3537 bp->max_tx_length = GEM_MAX_TX_LEN; 3538 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3539 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3540 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3541 bp->macbgem_ops.mog_rx = gem_rx; 3542 dev->ethtool_ops = &gem_ethtool_ops; 3543 } else { 3544 bp->max_tx_length = MACB_MAX_TX_LEN; 3545 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3546 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3547 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3548 bp->macbgem_ops.mog_rx = macb_rx; 3549 dev->ethtool_ops = &macb_ethtool_ops; 3550 } 3551 3552 /* Set features */ 3553 dev->hw_features = NETIF_F_SG; 3554 3555 /* Check LSO capability */ 3556 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3557 dev->hw_features |= MACB_NETIF_LSO; 3558 3559 /* Checksum offload is only available on gem with packet buffer */ 3560 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3561 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3562 if (bp->caps & MACB_CAPS_SG_DISABLED) 3563 dev->hw_features &= ~NETIF_F_SG; 3564 dev->features = dev->hw_features; 3565 3566 /* Check RX Flow Filters support. 3567 * Max Rx flows set by availability of screeners & compare regs: 3568 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3569 */ 3570 reg = gem_readl(bp, DCFG8); 3571 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3572 GEM_BFEXT(T2SCR, reg)); 3573 if (bp->max_tuples > 0) { 3574 /* also needs one ethtype match to check IPv4 */ 3575 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3576 /* program this reg now */ 3577 reg = 0; 3578 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3579 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3580 /* Filtering is supported in hw but don't enable it in kernel now */ 3581 dev->hw_features |= NETIF_F_NTUPLE; 3582 /* init Rx flow definitions */ 3583 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3584 bp->rx_fs_list.count = 0; 3585 spin_lock_init(&bp->rx_fs_lock); 3586 } else 3587 bp->max_tuples = 0; 3588 } 3589 3590 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3591 val = 0; 3592 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3593 val = GEM_BIT(RGMII); 3594 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3595 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3596 val = MACB_BIT(RMII); 3597 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3598 val = MACB_BIT(MII); 3599 3600 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3601 val |= MACB_BIT(CLKEN); 3602 3603 macb_or_gem_writel(bp, USRIO, val); 3604 } 3605 3606 /* Set MII management clock divider */ 3607 val = macb_mdc_clk_div(bp); 3608 val |= macb_dbw(bp); 3609 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3610 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3611 macb_writel(bp, NCFGR, val); 3612 3613 return 0; 3614 } 3615 3616 #if defined(CONFIG_OF) 3617 /* 1518 rounded up */ 3618 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3619 /* max number of receive buffers */ 3620 #define AT91ETHER_MAX_RX_DESCR 9 3621 3622 /* Initialize and start the Receiver and Transmit subsystems */ 3623 static int at91ether_start(struct net_device *dev) 3624 { 3625 struct macb *lp = netdev_priv(dev); 3626 struct macb_queue *q = &lp->queues[0]; 3627 struct macb_dma_desc *desc; 3628 dma_addr_t addr; 3629 u32 ctl; 3630 int i; 3631 3632 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3633 (AT91ETHER_MAX_RX_DESCR * 3634 macb_dma_desc_get_size(lp)), 3635 &q->rx_ring_dma, GFP_KERNEL); 3636 if (!q->rx_ring) 3637 return -ENOMEM; 3638 3639 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3640 AT91ETHER_MAX_RX_DESCR * 3641 AT91ETHER_MAX_RBUFF_SZ, 3642 &q->rx_buffers_dma, GFP_KERNEL); 3643 if (!q->rx_buffers) { 3644 dma_free_coherent(&lp->pdev->dev, 3645 AT91ETHER_MAX_RX_DESCR * 3646 macb_dma_desc_get_size(lp), 3647 q->rx_ring, q->rx_ring_dma); 3648 q->rx_ring = NULL; 3649 return -ENOMEM; 3650 } 3651 3652 addr = q->rx_buffers_dma; 3653 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3654 desc = macb_rx_desc(q, i); 3655 macb_set_addr(lp, desc, addr); 3656 desc->ctrl = 0; 3657 addr += AT91ETHER_MAX_RBUFF_SZ; 3658 } 3659 3660 /* Set the Wrap bit on the last descriptor */ 3661 desc->addr |= MACB_BIT(RX_WRAP); 3662 3663 /* Reset buffer index */ 3664 q->rx_tail = 0; 3665 3666 /* Program address of descriptor list in Rx Buffer Queue register */ 3667 macb_writel(lp, RBQP, q->rx_ring_dma); 3668 3669 /* Enable Receive and Transmit */ 3670 ctl = macb_readl(lp, NCR); 3671 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3672 3673 return 0; 3674 } 3675 3676 /* Open the ethernet interface */ 3677 static int at91ether_open(struct net_device *dev) 3678 { 3679 struct macb *lp = netdev_priv(dev); 3680 u32 ctl; 3681 int ret; 3682 3683 /* Clear internal statistics */ 3684 ctl = macb_readl(lp, NCR); 3685 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3686 3687 macb_set_hwaddr(lp); 3688 3689 ret = at91ether_start(dev); 3690 if (ret) 3691 return ret; 3692 3693 /* Enable MAC interrupts */ 3694 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3695 MACB_BIT(RXUBR) | 3696 MACB_BIT(ISR_TUND) | 3697 MACB_BIT(ISR_RLE) | 3698 MACB_BIT(TCOMP) | 3699 MACB_BIT(ISR_ROVR) | 3700 MACB_BIT(HRESP)); 3701 3702 /* schedule a link state check */ 3703 phy_start(dev->phydev); 3704 3705 netif_start_queue(dev); 3706 3707 return 0; 3708 } 3709 3710 /* Close the interface */ 3711 static int at91ether_close(struct net_device *dev) 3712 { 3713 struct macb *lp = netdev_priv(dev); 3714 struct macb_queue *q = &lp->queues[0]; 3715 u32 ctl; 3716 3717 /* Disable Receiver and Transmitter */ 3718 ctl = macb_readl(lp, NCR); 3719 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3720 3721 /* Disable MAC interrupts */ 3722 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3723 MACB_BIT(RXUBR) | 3724 MACB_BIT(ISR_TUND) | 3725 MACB_BIT(ISR_RLE) | 3726 MACB_BIT(TCOMP) | 3727 MACB_BIT(ISR_ROVR) | 3728 MACB_BIT(HRESP)); 3729 3730 netif_stop_queue(dev); 3731 3732 dma_free_coherent(&lp->pdev->dev, 3733 AT91ETHER_MAX_RX_DESCR * 3734 macb_dma_desc_get_size(lp), 3735 q->rx_ring, q->rx_ring_dma); 3736 q->rx_ring = NULL; 3737 3738 dma_free_coherent(&lp->pdev->dev, 3739 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3740 q->rx_buffers, q->rx_buffers_dma); 3741 q->rx_buffers = NULL; 3742 3743 return 0; 3744 } 3745 3746 /* Transmit packet */ 3747 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3748 struct net_device *dev) 3749 { 3750 struct macb *lp = netdev_priv(dev); 3751 3752 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3753 netif_stop_queue(dev); 3754 3755 /* Store packet information (to free when Tx completed) */ 3756 lp->skb = skb; 3757 lp->skb_length = skb->len; 3758 lp->skb_physaddr = dma_map_single(&lp->pdev->dev, skb->data, 3759 skb->len, DMA_TO_DEVICE); 3760 if (dma_mapping_error(&lp->pdev->dev, lp->skb_physaddr)) { 3761 dev_kfree_skb_any(skb); 3762 dev->stats.tx_dropped++; 3763 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3764 return NETDEV_TX_OK; 3765 } 3766 3767 /* Set address of the data in the Transmit Address register */ 3768 macb_writel(lp, TAR, lp->skb_physaddr); 3769 /* Set length of the packet in the Transmit Control register */ 3770 macb_writel(lp, TCR, skb->len); 3771 3772 } else { 3773 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3774 return NETDEV_TX_BUSY; 3775 } 3776 3777 return NETDEV_TX_OK; 3778 } 3779 3780 /* Extract received frame from buffer descriptors and sent to upper layers. 3781 * (Called from interrupt context) 3782 */ 3783 static void at91ether_rx(struct net_device *dev) 3784 { 3785 struct macb *lp = netdev_priv(dev); 3786 struct macb_queue *q = &lp->queues[0]; 3787 struct macb_dma_desc *desc; 3788 unsigned char *p_recv; 3789 struct sk_buff *skb; 3790 unsigned int pktlen; 3791 3792 desc = macb_rx_desc(q, q->rx_tail); 3793 while (desc->addr & MACB_BIT(RX_USED)) { 3794 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3795 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3796 skb = netdev_alloc_skb(dev, pktlen + 2); 3797 if (skb) { 3798 skb_reserve(skb, 2); 3799 skb_put_data(skb, p_recv, pktlen); 3800 3801 skb->protocol = eth_type_trans(skb, dev); 3802 dev->stats.rx_packets++; 3803 dev->stats.rx_bytes += pktlen; 3804 netif_rx(skb); 3805 } else { 3806 dev->stats.rx_dropped++; 3807 } 3808 3809 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3810 dev->stats.multicast++; 3811 3812 /* reset ownership bit */ 3813 desc->addr &= ~MACB_BIT(RX_USED); 3814 3815 /* wrap after last buffer */ 3816 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3817 q->rx_tail = 0; 3818 else 3819 q->rx_tail++; 3820 3821 desc = macb_rx_desc(q, q->rx_tail); 3822 } 3823 } 3824 3825 /* MAC interrupt handler */ 3826 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3827 { 3828 struct net_device *dev = dev_id; 3829 struct macb *lp = netdev_priv(dev); 3830 u32 intstatus, ctl; 3831 3832 /* MAC Interrupt Status register indicates what interrupts are pending. 3833 * It is automatically cleared once read. 3834 */ 3835 intstatus = macb_readl(lp, ISR); 3836 3837 /* Receive complete */ 3838 if (intstatus & MACB_BIT(RCOMP)) 3839 at91ether_rx(dev); 3840 3841 /* Transmit complete */ 3842 if (intstatus & MACB_BIT(TCOMP)) { 3843 /* The TCOM bit is set even if the transmission failed */ 3844 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3845 dev->stats.tx_errors++; 3846 3847 if (lp->skb) { 3848 dev_consume_skb_irq(lp->skb); 3849 lp->skb = NULL; 3850 dma_unmap_single(&lp->pdev->dev, lp->skb_physaddr, 3851 lp->skb_length, DMA_TO_DEVICE); 3852 dev->stats.tx_packets++; 3853 dev->stats.tx_bytes += lp->skb_length; 3854 } 3855 netif_wake_queue(dev); 3856 } 3857 3858 /* Work-around for EMAC Errata section 41.3.1 */ 3859 if (intstatus & MACB_BIT(RXUBR)) { 3860 ctl = macb_readl(lp, NCR); 3861 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3862 wmb(); 3863 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3864 } 3865 3866 if (intstatus & MACB_BIT(ISR_ROVR)) 3867 netdev_err(dev, "ROVR error\n"); 3868 3869 return IRQ_HANDLED; 3870 } 3871 3872 #ifdef CONFIG_NET_POLL_CONTROLLER 3873 static void at91ether_poll_controller(struct net_device *dev) 3874 { 3875 unsigned long flags; 3876 3877 local_irq_save(flags); 3878 at91ether_interrupt(dev->irq, dev); 3879 local_irq_restore(flags); 3880 } 3881 #endif 3882 3883 static const struct net_device_ops at91ether_netdev_ops = { 3884 .ndo_open = at91ether_open, 3885 .ndo_stop = at91ether_close, 3886 .ndo_start_xmit = at91ether_start_xmit, 3887 .ndo_get_stats = macb_get_stats, 3888 .ndo_set_rx_mode = macb_set_rx_mode, 3889 .ndo_set_mac_address = eth_mac_addr, 3890 .ndo_do_ioctl = macb_ioctl, 3891 .ndo_validate_addr = eth_validate_addr, 3892 #ifdef CONFIG_NET_POLL_CONTROLLER 3893 .ndo_poll_controller = at91ether_poll_controller, 3894 #endif 3895 }; 3896 3897 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3898 struct clk **hclk, struct clk **tx_clk, 3899 struct clk **rx_clk, struct clk **tsu_clk) 3900 { 3901 int err; 3902 3903 *hclk = NULL; 3904 *tx_clk = NULL; 3905 *rx_clk = NULL; 3906 *tsu_clk = NULL; 3907 3908 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3909 if (IS_ERR(*pclk)) 3910 return PTR_ERR(*pclk); 3911 3912 err = clk_prepare_enable(*pclk); 3913 if (err) { 3914 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3915 return err; 3916 } 3917 3918 return 0; 3919 } 3920 3921 static int at91ether_init(struct platform_device *pdev) 3922 { 3923 struct net_device *dev = platform_get_drvdata(pdev); 3924 struct macb *bp = netdev_priv(dev); 3925 int err; 3926 u32 reg; 3927 3928 bp->queues[0].bp = bp; 3929 3930 dev->netdev_ops = &at91ether_netdev_ops; 3931 dev->ethtool_ops = &macb_ethtool_ops; 3932 3933 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3934 0, dev->name, dev); 3935 if (err) 3936 return err; 3937 3938 macb_writel(bp, NCR, 0); 3939 3940 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3941 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3942 reg |= MACB_BIT(RM9200_RMII); 3943 3944 macb_writel(bp, NCFGR, reg); 3945 3946 return 0; 3947 } 3948 3949 static const struct macb_config at91sam9260_config = { 3950 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3951 .clk_init = macb_clk_init, 3952 .init = macb_init, 3953 }; 3954 3955 static const struct macb_config sama5d3macb_config = { 3956 .caps = MACB_CAPS_SG_DISABLED 3957 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3958 .clk_init = macb_clk_init, 3959 .init = macb_init, 3960 }; 3961 3962 static const struct macb_config pc302gem_config = { 3963 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 3964 .dma_burst_length = 16, 3965 .clk_init = macb_clk_init, 3966 .init = macb_init, 3967 }; 3968 3969 static const struct macb_config sama5d2_config = { 3970 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3971 .dma_burst_length = 16, 3972 .clk_init = macb_clk_init, 3973 .init = macb_init, 3974 }; 3975 3976 static const struct macb_config sama5d3_config = { 3977 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 3978 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 3979 .dma_burst_length = 16, 3980 .clk_init = macb_clk_init, 3981 .init = macb_init, 3982 .jumbo_max_len = 10240, 3983 }; 3984 3985 static const struct macb_config sama5d4_config = { 3986 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3987 .dma_burst_length = 4, 3988 .clk_init = macb_clk_init, 3989 .init = macb_init, 3990 }; 3991 3992 static const struct macb_config emac_config = { 3993 .caps = MACB_CAPS_NEEDS_RSTONUBR, 3994 .clk_init = at91ether_clk_init, 3995 .init = at91ether_init, 3996 }; 3997 3998 static const struct macb_config np4_config = { 3999 .caps = MACB_CAPS_USRIO_DISABLED, 4000 .clk_init = macb_clk_init, 4001 .init = macb_init, 4002 }; 4003 4004 static const struct macb_config zynqmp_config = { 4005 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4006 MACB_CAPS_JUMBO | 4007 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 4008 .dma_burst_length = 16, 4009 .clk_init = macb_clk_init, 4010 .init = macb_init, 4011 .jumbo_max_len = 10240, 4012 }; 4013 4014 static const struct macb_config zynq_config = { 4015 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 4016 MACB_CAPS_NEEDS_RSTONUBR, 4017 .dma_burst_length = 16, 4018 .clk_init = macb_clk_init, 4019 .init = macb_init, 4020 }; 4021 4022 static const struct of_device_id macb_dt_ids[] = { 4023 { .compatible = "cdns,at32ap7000-macb" }, 4024 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 4025 { .compatible = "cdns,macb" }, 4026 { .compatible = "cdns,np4-macb", .data = &np4_config }, 4027 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 4028 { .compatible = "cdns,gem", .data = &pc302gem_config }, 4029 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 4030 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 4031 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 4032 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 4033 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 4034 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 4035 { .compatible = "cdns,emac", .data = &emac_config }, 4036 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 4037 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 4038 { /* sentinel */ } 4039 }; 4040 MODULE_DEVICE_TABLE(of, macb_dt_ids); 4041 #endif /* CONFIG_OF */ 4042 4043 static const struct macb_config default_gem_config = { 4044 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4045 MACB_CAPS_JUMBO | 4046 MACB_CAPS_GEM_HAS_PTP, 4047 .dma_burst_length = 16, 4048 .clk_init = macb_clk_init, 4049 .init = macb_init, 4050 .jumbo_max_len = 10240, 4051 }; 4052 4053 static int macb_probe(struct platform_device *pdev) 4054 { 4055 const struct macb_config *macb_config = &default_gem_config; 4056 int (*clk_init)(struct platform_device *, struct clk **, 4057 struct clk **, struct clk **, struct clk **, 4058 struct clk **) = macb_config->clk_init; 4059 int (*init)(struct platform_device *) = macb_config->init; 4060 struct device_node *np = pdev->dev.of_node; 4061 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4062 struct clk *tsu_clk = NULL; 4063 unsigned int queue_mask, num_queues; 4064 bool native_io; 4065 struct phy_device *phydev; 4066 struct net_device *dev; 4067 struct resource *regs; 4068 void __iomem *mem; 4069 const char *mac; 4070 struct macb *bp; 4071 int err, val; 4072 4073 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4074 mem = devm_ioremap_resource(&pdev->dev, regs); 4075 if (IS_ERR(mem)) 4076 return PTR_ERR(mem); 4077 4078 if (np) { 4079 const struct of_device_id *match; 4080 4081 match = of_match_node(macb_dt_ids, np); 4082 if (match && match->data) { 4083 macb_config = match->data; 4084 clk_init = macb_config->clk_init; 4085 init = macb_config->init; 4086 } 4087 } 4088 4089 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4090 if (err) 4091 return err; 4092 4093 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4094 pm_runtime_use_autosuspend(&pdev->dev); 4095 pm_runtime_get_noresume(&pdev->dev); 4096 pm_runtime_set_active(&pdev->dev); 4097 pm_runtime_enable(&pdev->dev); 4098 native_io = hw_is_native_io(mem); 4099 4100 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4101 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4102 if (!dev) { 4103 err = -ENOMEM; 4104 goto err_disable_clocks; 4105 } 4106 4107 dev->base_addr = regs->start; 4108 4109 SET_NETDEV_DEV(dev, &pdev->dev); 4110 4111 bp = netdev_priv(dev); 4112 bp->pdev = pdev; 4113 bp->dev = dev; 4114 bp->regs = mem; 4115 bp->native_io = native_io; 4116 if (native_io) { 4117 bp->macb_reg_readl = hw_readl_native; 4118 bp->macb_reg_writel = hw_writel_native; 4119 } else { 4120 bp->macb_reg_readl = hw_readl; 4121 bp->macb_reg_writel = hw_writel; 4122 } 4123 bp->num_queues = num_queues; 4124 bp->queue_mask = queue_mask; 4125 if (macb_config) 4126 bp->dma_burst_length = macb_config->dma_burst_length; 4127 bp->pclk = pclk; 4128 bp->hclk = hclk; 4129 bp->tx_clk = tx_clk; 4130 bp->rx_clk = rx_clk; 4131 bp->tsu_clk = tsu_clk; 4132 if (macb_config) 4133 bp->jumbo_max_len = macb_config->jumbo_max_len; 4134 4135 bp->wol = 0; 4136 if (of_get_property(np, "magic-packet", NULL)) 4137 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4138 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4139 4140 spin_lock_init(&bp->lock); 4141 4142 /* setup capabilities */ 4143 macb_configure_caps(bp, macb_config); 4144 4145 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4146 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4147 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4148 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4149 } 4150 #endif 4151 platform_set_drvdata(pdev, dev); 4152 4153 dev->irq = platform_get_irq(pdev, 0); 4154 if (dev->irq < 0) { 4155 err = dev->irq; 4156 goto err_out_free_netdev; 4157 } 4158 4159 /* MTU range: 68 - 1500 or 10240 */ 4160 dev->min_mtu = GEM_MTU_MIN_SIZE; 4161 if (bp->caps & MACB_CAPS_JUMBO) 4162 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4163 else 4164 dev->max_mtu = ETH_DATA_LEN; 4165 4166 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4167 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4168 if (val) 4169 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4170 macb_dma_desc_get_size(bp); 4171 4172 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4173 if (val) 4174 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4175 macb_dma_desc_get_size(bp); 4176 } 4177 4178 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4179 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4180 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4181 4182 mac = of_get_mac_address(np); 4183 if (PTR_ERR(mac) == -EPROBE_DEFER) { 4184 err = -EPROBE_DEFER; 4185 goto err_out_free_netdev; 4186 } else if (!IS_ERR(mac)) { 4187 ether_addr_copy(bp->dev->dev_addr, mac); 4188 } else { 4189 macb_get_hwaddr(bp); 4190 } 4191 4192 err = of_get_phy_mode(np); 4193 if (err < 0) 4194 /* not found in DT, MII by default */ 4195 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4196 else 4197 bp->phy_interface = err; 4198 4199 /* IP specific init */ 4200 err = init(pdev); 4201 if (err) 4202 goto err_out_free_netdev; 4203 4204 err = macb_mii_init(bp); 4205 if (err) 4206 goto err_out_free_netdev; 4207 4208 phydev = dev->phydev; 4209 4210 netif_carrier_off(dev); 4211 4212 err = register_netdev(dev); 4213 if (err) { 4214 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4215 goto err_out_unregister_mdio; 4216 } 4217 4218 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4219 (unsigned long)bp); 4220 4221 phy_attached_info(phydev); 4222 4223 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4224 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4225 dev->base_addr, dev->irq, dev->dev_addr); 4226 4227 pm_runtime_mark_last_busy(&bp->pdev->dev); 4228 pm_runtime_put_autosuspend(&bp->pdev->dev); 4229 4230 return 0; 4231 4232 err_out_unregister_mdio: 4233 phy_disconnect(dev->phydev); 4234 mdiobus_unregister(bp->mii_bus); 4235 of_node_put(bp->phy_node); 4236 if (np && of_phy_is_fixed_link(np)) 4237 of_phy_deregister_fixed_link(np); 4238 mdiobus_free(bp->mii_bus); 4239 4240 err_out_free_netdev: 4241 free_netdev(dev); 4242 4243 err_disable_clocks: 4244 clk_disable_unprepare(tx_clk); 4245 clk_disable_unprepare(hclk); 4246 clk_disable_unprepare(pclk); 4247 clk_disable_unprepare(rx_clk); 4248 clk_disable_unprepare(tsu_clk); 4249 pm_runtime_disable(&pdev->dev); 4250 pm_runtime_set_suspended(&pdev->dev); 4251 pm_runtime_dont_use_autosuspend(&pdev->dev); 4252 4253 return err; 4254 } 4255 4256 static int macb_remove(struct platform_device *pdev) 4257 { 4258 struct net_device *dev; 4259 struct macb *bp; 4260 struct device_node *np = pdev->dev.of_node; 4261 4262 dev = platform_get_drvdata(pdev); 4263 4264 if (dev) { 4265 bp = netdev_priv(dev); 4266 if (dev->phydev) 4267 phy_disconnect(dev->phydev); 4268 mdiobus_unregister(bp->mii_bus); 4269 if (np && of_phy_is_fixed_link(np)) 4270 of_phy_deregister_fixed_link(np); 4271 dev->phydev = NULL; 4272 mdiobus_free(bp->mii_bus); 4273 4274 unregister_netdev(dev); 4275 pm_runtime_disable(&pdev->dev); 4276 pm_runtime_dont_use_autosuspend(&pdev->dev); 4277 if (!pm_runtime_suspended(&pdev->dev)) { 4278 clk_disable_unprepare(bp->tx_clk); 4279 clk_disable_unprepare(bp->hclk); 4280 clk_disable_unprepare(bp->pclk); 4281 clk_disable_unprepare(bp->rx_clk); 4282 clk_disable_unprepare(bp->tsu_clk); 4283 pm_runtime_set_suspended(&pdev->dev); 4284 } 4285 of_node_put(bp->phy_node); 4286 free_netdev(dev); 4287 } 4288 4289 return 0; 4290 } 4291 4292 static int __maybe_unused macb_suspend(struct device *dev) 4293 { 4294 struct net_device *netdev = dev_get_drvdata(dev); 4295 struct macb *bp = netdev_priv(netdev); 4296 struct macb_queue *queue = bp->queues; 4297 unsigned long flags; 4298 unsigned int q; 4299 4300 if (!netif_running(netdev)) 4301 return 0; 4302 4303 4304 if (bp->wol & MACB_WOL_ENABLED) { 4305 macb_writel(bp, IER, MACB_BIT(WOL)); 4306 macb_writel(bp, WOL, MACB_BIT(MAG)); 4307 enable_irq_wake(bp->queues[0].irq); 4308 netif_device_detach(netdev); 4309 } else { 4310 netif_device_detach(netdev); 4311 for (q = 0, queue = bp->queues; q < bp->num_queues; 4312 ++q, ++queue) 4313 napi_disable(&queue->napi); 4314 phy_stop(netdev->phydev); 4315 phy_suspend(netdev->phydev); 4316 spin_lock_irqsave(&bp->lock, flags); 4317 macb_reset_hw(bp); 4318 spin_unlock_irqrestore(&bp->lock, flags); 4319 4320 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4321 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 4322 4323 if (netdev->hw_features & NETIF_F_NTUPLE) 4324 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 4325 } 4326 4327 netif_carrier_off(netdev); 4328 if (bp->ptp_info) 4329 bp->ptp_info->ptp_remove(netdev); 4330 pm_runtime_force_suspend(dev); 4331 4332 return 0; 4333 } 4334 4335 static int __maybe_unused macb_resume(struct device *dev) 4336 { 4337 struct net_device *netdev = dev_get_drvdata(dev); 4338 struct macb *bp = netdev_priv(netdev); 4339 struct macb_queue *queue = bp->queues; 4340 unsigned int q; 4341 4342 if (!netif_running(netdev)) 4343 return 0; 4344 4345 pm_runtime_force_resume(dev); 4346 4347 if (bp->wol & MACB_WOL_ENABLED) { 4348 macb_writel(bp, IDR, MACB_BIT(WOL)); 4349 macb_writel(bp, WOL, 0); 4350 disable_irq_wake(bp->queues[0].irq); 4351 } else { 4352 macb_writel(bp, NCR, MACB_BIT(MPE)); 4353 4354 if (netdev->hw_features & NETIF_F_NTUPLE) 4355 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 4356 4357 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4358 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 4359 4360 for (q = 0, queue = bp->queues; q < bp->num_queues; 4361 ++q, ++queue) 4362 napi_enable(&queue->napi); 4363 phy_resume(netdev->phydev); 4364 phy_init_hw(netdev->phydev); 4365 phy_start(netdev->phydev); 4366 } 4367 4368 bp->macbgem_ops.mog_init_rings(bp); 4369 macb_init_hw(bp); 4370 macb_set_rx_mode(netdev); 4371 macb_restore_features(bp); 4372 netif_device_attach(netdev); 4373 if (bp->ptp_info) 4374 bp->ptp_info->ptp_init(netdev); 4375 4376 return 0; 4377 } 4378 4379 static int __maybe_unused macb_runtime_suspend(struct device *dev) 4380 { 4381 struct net_device *netdev = dev_get_drvdata(dev); 4382 struct macb *bp = netdev_priv(netdev); 4383 4384 if (!(device_may_wakeup(&bp->dev->dev))) { 4385 clk_disable_unprepare(bp->tx_clk); 4386 clk_disable_unprepare(bp->hclk); 4387 clk_disable_unprepare(bp->pclk); 4388 clk_disable_unprepare(bp->rx_clk); 4389 } 4390 clk_disable_unprepare(bp->tsu_clk); 4391 4392 return 0; 4393 } 4394 4395 static int __maybe_unused macb_runtime_resume(struct device *dev) 4396 { 4397 struct net_device *netdev = dev_get_drvdata(dev); 4398 struct macb *bp = netdev_priv(netdev); 4399 4400 if (!(device_may_wakeup(&bp->dev->dev))) { 4401 clk_prepare_enable(bp->pclk); 4402 clk_prepare_enable(bp->hclk); 4403 clk_prepare_enable(bp->tx_clk); 4404 clk_prepare_enable(bp->rx_clk); 4405 } 4406 clk_prepare_enable(bp->tsu_clk); 4407 4408 return 0; 4409 } 4410 4411 static const struct dev_pm_ops macb_pm_ops = { 4412 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 4413 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 4414 }; 4415 4416 static struct platform_driver macb_driver = { 4417 .probe = macb_probe, 4418 .remove = macb_remove, 4419 .driver = { 4420 .name = "macb", 4421 .of_match_table = of_match_ptr(macb_dt_ids), 4422 .pm = &macb_pm_ops, 4423 }, 4424 }; 4425 4426 module_platform_driver(macb_driver); 4427 4428 MODULE_LICENSE("GPL"); 4429 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4430 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4431 MODULE_ALIAS("platform:macb"); 4432