1 /* 2 * Cadence MACB/GEM Ethernet Controller driver 3 * 4 * Copyright (C) 2004-2006 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 #include <linux/clk.h> 13 #include <linux/crc32.h> 14 #include <linux/module.h> 15 #include <linux/moduleparam.h> 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/circ_buf.h> 19 #include <linux/slab.h> 20 #include <linux/init.h> 21 #include <linux/io.h> 22 #include <linux/gpio.h> 23 #include <linux/gpio/consumer.h> 24 #include <linux/interrupt.h> 25 #include <linux/netdevice.h> 26 #include <linux/etherdevice.h> 27 #include <linux/dma-mapping.h> 28 #include <linux/platform_data/macb.h> 29 #include <linux/platform_device.h> 30 #include <linux/phy.h> 31 #include <linux/of.h> 32 #include <linux/of_device.h> 33 #include <linux/of_gpio.h> 34 #include <linux/of_mdio.h> 35 #include <linux/of_net.h> 36 #include <linux/ip.h> 37 #include <linux/udp.h> 38 #include <linux/tcp.h> 39 #include "macb.h" 40 41 #define MACB_RX_BUFFER_SIZE 128 42 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 43 44 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 45 #define MIN_RX_RING_SIZE 64 46 #define MAX_RX_RING_SIZE 8192 47 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 48 * (bp)->rx_ring_size) 49 50 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 51 #define MIN_TX_RING_SIZE 64 52 #define MAX_TX_RING_SIZE 4096 53 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 54 * (bp)->tx_ring_size) 55 56 /* level of occupied TX descriptors under which we wake up TX process */ 57 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 58 59 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(RXUBR) \ 60 | MACB_BIT(ISR_ROVR)) 61 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 62 | MACB_BIT(ISR_RLE) \ 63 | MACB_BIT(TXERR)) 64 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP)) 65 66 /* Max length of transmit frame must be a multiple of 8 bytes */ 67 #define MACB_TX_LEN_ALIGN 8 68 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 69 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 70 71 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 72 #define MACB_NETIF_LSO NETIF_F_TSO 73 74 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 75 #define MACB_WOL_ENABLED (0x1 << 1) 76 77 /* Graceful stop timeouts in us. We should allow up to 78 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 79 */ 80 #define MACB_HALT_TIMEOUT 1230 81 82 /* DMA buffer descriptor might be different size 83 * depends on hardware configuration: 84 * 85 * 1. dma address width 32 bits: 86 * word 1: 32 bit address of Data Buffer 87 * word 2: control 88 * 89 * 2. dma address width 64 bits: 90 * word 1: 32 bit address of Data Buffer 91 * word 2: control 92 * word 3: upper 32 bit address of Data Buffer 93 * word 4: unused 94 * 95 * 3. dma address width 32 bits with hardware timestamping: 96 * word 1: 32 bit address of Data Buffer 97 * word 2: control 98 * word 3: timestamp word 1 99 * word 4: timestamp word 2 100 * 101 * 4. dma address width 64 bits with hardware timestamping: 102 * word 1: 32 bit address of Data Buffer 103 * word 2: control 104 * word 3: upper 32 bit address of Data Buffer 105 * word 4: unused 106 * word 5: timestamp word 1 107 * word 6: timestamp word 2 108 */ 109 static unsigned int macb_dma_desc_get_size(struct macb *bp) 110 { 111 #ifdef MACB_EXT_DESC 112 unsigned int desc_size; 113 114 switch (bp->hw_dma_cap) { 115 case HW_DMA_CAP_64B: 116 desc_size = sizeof(struct macb_dma_desc) 117 + sizeof(struct macb_dma_desc_64); 118 break; 119 case HW_DMA_CAP_PTP: 120 desc_size = sizeof(struct macb_dma_desc) 121 + sizeof(struct macb_dma_desc_ptp); 122 break; 123 case HW_DMA_CAP_64B_PTP: 124 desc_size = sizeof(struct macb_dma_desc) 125 + sizeof(struct macb_dma_desc_64) 126 + sizeof(struct macb_dma_desc_ptp); 127 break; 128 default: 129 desc_size = sizeof(struct macb_dma_desc); 130 } 131 return desc_size; 132 #endif 133 return sizeof(struct macb_dma_desc); 134 } 135 136 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 137 { 138 #ifdef MACB_EXT_DESC 139 switch (bp->hw_dma_cap) { 140 case HW_DMA_CAP_64B: 141 case HW_DMA_CAP_PTP: 142 desc_idx <<= 1; 143 break; 144 case HW_DMA_CAP_64B_PTP: 145 desc_idx *= 3; 146 break; 147 default: 148 break; 149 } 150 #endif 151 return desc_idx; 152 } 153 154 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 155 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 156 { 157 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 158 return (struct macb_dma_desc_64 *)((void *)desc + sizeof(struct macb_dma_desc)); 159 return NULL; 160 } 161 #endif 162 163 /* Ring buffer accessors */ 164 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 165 { 166 return index & (bp->tx_ring_size - 1); 167 } 168 169 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 170 unsigned int index) 171 { 172 index = macb_tx_ring_wrap(queue->bp, index); 173 index = macb_adj_dma_desc_idx(queue->bp, index); 174 return &queue->tx_ring[index]; 175 } 176 177 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 178 unsigned int index) 179 { 180 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 181 } 182 183 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 184 { 185 dma_addr_t offset; 186 187 offset = macb_tx_ring_wrap(queue->bp, index) * 188 macb_dma_desc_get_size(queue->bp); 189 190 return queue->tx_ring_dma + offset; 191 } 192 193 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 194 { 195 return index & (bp->rx_ring_size - 1); 196 } 197 198 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 199 { 200 index = macb_rx_ring_wrap(queue->bp, index); 201 index = macb_adj_dma_desc_idx(queue->bp, index); 202 return &queue->rx_ring[index]; 203 } 204 205 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 206 { 207 return queue->rx_buffers + queue->bp->rx_buffer_size * 208 macb_rx_ring_wrap(queue->bp, index); 209 } 210 211 /* I/O accessors */ 212 static u32 hw_readl_native(struct macb *bp, int offset) 213 { 214 return __raw_readl(bp->regs + offset); 215 } 216 217 static void hw_writel_native(struct macb *bp, int offset, u32 value) 218 { 219 __raw_writel(value, bp->regs + offset); 220 } 221 222 static u32 hw_readl(struct macb *bp, int offset) 223 { 224 return readl_relaxed(bp->regs + offset); 225 } 226 227 static void hw_writel(struct macb *bp, int offset, u32 value) 228 { 229 writel_relaxed(value, bp->regs + offset); 230 } 231 232 /* Find the CPU endianness by using the loopback bit of NCR register. When the 233 * CPU is in big endian we need to program swapped mode for management 234 * descriptor access. 235 */ 236 static bool hw_is_native_io(void __iomem *addr) 237 { 238 u32 value = MACB_BIT(LLB); 239 240 __raw_writel(value, addr + MACB_NCR); 241 value = __raw_readl(addr + MACB_NCR); 242 243 /* Write 0 back to disable everything */ 244 __raw_writel(0, addr + MACB_NCR); 245 246 return value == MACB_BIT(LLB); 247 } 248 249 static bool hw_is_gem(void __iomem *addr, bool native_io) 250 { 251 u32 id; 252 253 if (native_io) 254 id = __raw_readl(addr + MACB_MID); 255 else 256 id = readl_relaxed(addr + MACB_MID); 257 258 return MACB_BFEXT(IDNUM, id) >= 0x2; 259 } 260 261 static void macb_set_hwaddr(struct macb *bp) 262 { 263 u32 bottom; 264 u16 top; 265 266 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 267 macb_or_gem_writel(bp, SA1B, bottom); 268 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 269 macb_or_gem_writel(bp, SA1T, top); 270 271 /* Clear unused address register sets */ 272 macb_or_gem_writel(bp, SA2B, 0); 273 macb_or_gem_writel(bp, SA2T, 0); 274 macb_or_gem_writel(bp, SA3B, 0); 275 macb_or_gem_writel(bp, SA3T, 0); 276 macb_or_gem_writel(bp, SA4B, 0); 277 macb_or_gem_writel(bp, SA4T, 0); 278 } 279 280 static void macb_get_hwaddr(struct macb *bp) 281 { 282 struct macb_platform_data *pdata; 283 u32 bottom; 284 u16 top; 285 u8 addr[6]; 286 int i; 287 288 pdata = dev_get_platdata(&bp->pdev->dev); 289 290 /* Check all 4 address register for valid address */ 291 for (i = 0; i < 4; i++) { 292 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 293 top = macb_or_gem_readl(bp, SA1T + i * 8); 294 295 if (pdata && pdata->rev_eth_addr) { 296 addr[5] = bottom & 0xff; 297 addr[4] = (bottom >> 8) & 0xff; 298 addr[3] = (bottom >> 16) & 0xff; 299 addr[2] = (bottom >> 24) & 0xff; 300 addr[1] = top & 0xff; 301 addr[0] = (top & 0xff00) >> 8; 302 } else { 303 addr[0] = bottom & 0xff; 304 addr[1] = (bottom >> 8) & 0xff; 305 addr[2] = (bottom >> 16) & 0xff; 306 addr[3] = (bottom >> 24) & 0xff; 307 addr[4] = top & 0xff; 308 addr[5] = (top >> 8) & 0xff; 309 } 310 311 if (is_valid_ether_addr(addr)) { 312 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 313 return; 314 } 315 } 316 317 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 318 eth_hw_addr_random(bp->dev); 319 } 320 321 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 322 { 323 struct macb *bp = bus->priv; 324 int value; 325 326 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 327 | MACB_BF(RW, MACB_MAN_READ) 328 | MACB_BF(PHYA, mii_id) 329 | MACB_BF(REGA, regnum) 330 | MACB_BF(CODE, MACB_MAN_CODE))); 331 332 /* wait for end of transfer */ 333 while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) 334 cpu_relax(); 335 336 value = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 337 338 return value; 339 } 340 341 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 342 u16 value) 343 { 344 struct macb *bp = bus->priv; 345 346 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 347 | MACB_BF(RW, MACB_MAN_WRITE) 348 | MACB_BF(PHYA, mii_id) 349 | MACB_BF(REGA, regnum) 350 | MACB_BF(CODE, MACB_MAN_CODE) 351 | MACB_BF(DATA, value))); 352 353 /* wait for end of transfer */ 354 while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) 355 cpu_relax(); 356 357 return 0; 358 } 359 360 /** 361 * macb_set_tx_clk() - Set a clock to a new frequency 362 * @clk Pointer to the clock to change 363 * @rate New frequency in Hz 364 * @dev Pointer to the struct net_device 365 */ 366 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 367 { 368 long ferr, rate, rate_rounded; 369 370 if (!clk) 371 return; 372 373 switch (speed) { 374 case SPEED_10: 375 rate = 2500000; 376 break; 377 case SPEED_100: 378 rate = 25000000; 379 break; 380 case SPEED_1000: 381 rate = 125000000; 382 break; 383 default: 384 return; 385 } 386 387 rate_rounded = clk_round_rate(clk, rate); 388 if (rate_rounded < 0) 389 return; 390 391 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 392 * is not satisfied. 393 */ 394 ferr = abs(rate_rounded - rate); 395 ferr = DIV_ROUND_UP(ferr, rate / 100000); 396 if (ferr > 5) 397 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 398 rate); 399 400 if (clk_set_rate(clk, rate_rounded)) 401 netdev_err(dev, "adjusting tx_clk failed.\n"); 402 } 403 404 static void macb_handle_link_change(struct net_device *dev) 405 { 406 struct macb *bp = netdev_priv(dev); 407 struct phy_device *phydev = dev->phydev; 408 unsigned long flags; 409 int status_change = 0; 410 411 spin_lock_irqsave(&bp->lock, flags); 412 413 if (phydev->link) { 414 if ((bp->speed != phydev->speed) || 415 (bp->duplex != phydev->duplex)) { 416 u32 reg; 417 418 reg = macb_readl(bp, NCFGR); 419 reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 420 if (macb_is_gem(bp)) 421 reg &= ~GEM_BIT(GBE); 422 423 if (phydev->duplex) 424 reg |= MACB_BIT(FD); 425 if (phydev->speed == SPEED_100) 426 reg |= MACB_BIT(SPD); 427 if (phydev->speed == SPEED_1000 && 428 bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 429 reg |= GEM_BIT(GBE); 430 431 macb_or_gem_writel(bp, NCFGR, reg); 432 433 bp->speed = phydev->speed; 434 bp->duplex = phydev->duplex; 435 status_change = 1; 436 } 437 } 438 439 if (phydev->link != bp->link) { 440 if (!phydev->link) { 441 bp->speed = 0; 442 bp->duplex = -1; 443 } 444 bp->link = phydev->link; 445 446 status_change = 1; 447 } 448 449 spin_unlock_irqrestore(&bp->lock, flags); 450 451 if (status_change) { 452 if (phydev->link) { 453 /* Update the TX clock rate if and only if the link is 454 * up and there has been a link change. 455 */ 456 macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); 457 458 netif_carrier_on(dev); 459 netdev_info(dev, "link up (%d/%s)\n", 460 phydev->speed, 461 phydev->duplex == DUPLEX_FULL ? 462 "Full" : "Half"); 463 } else { 464 netif_carrier_off(dev); 465 netdev_info(dev, "link down\n"); 466 } 467 } 468 } 469 470 /* based on au1000_eth. c*/ 471 static int macb_mii_probe(struct net_device *dev) 472 { 473 struct macb *bp = netdev_priv(dev); 474 struct macb_platform_data *pdata; 475 struct phy_device *phydev; 476 struct device_node *np; 477 int phy_irq, ret, i; 478 479 pdata = dev_get_platdata(&bp->pdev->dev); 480 np = bp->pdev->dev.of_node; 481 ret = 0; 482 483 if (np) { 484 if (of_phy_is_fixed_link(np)) { 485 if (of_phy_register_fixed_link(np) < 0) { 486 dev_err(&bp->pdev->dev, 487 "broken fixed-link specification\n"); 488 return -ENODEV; 489 } 490 bp->phy_node = of_node_get(np); 491 } else { 492 bp->phy_node = of_parse_phandle(np, "phy-handle", 0); 493 /* fallback to standard phy registration if no 494 * phy-handle was found nor any phy found during 495 * dt phy registration 496 */ 497 if (!bp->phy_node && !phy_find_first(bp->mii_bus)) { 498 for (i = 0; i < PHY_MAX_ADDR; i++) { 499 struct phy_device *phydev; 500 501 phydev = mdiobus_scan(bp->mii_bus, i); 502 if (IS_ERR(phydev) && 503 PTR_ERR(phydev) != -ENODEV) { 504 ret = PTR_ERR(phydev); 505 break; 506 } 507 } 508 509 if (ret) 510 return -ENODEV; 511 } 512 } 513 } 514 515 if (bp->phy_node) { 516 phydev = of_phy_connect(dev, bp->phy_node, 517 &macb_handle_link_change, 0, 518 bp->phy_interface); 519 if (!phydev) 520 return -ENODEV; 521 } else { 522 phydev = phy_find_first(bp->mii_bus); 523 if (!phydev) { 524 netdev_err(dev, "no PHY found\n"); 525 return -ENXIO; 526 } 527 528 if (pdata) { 529 if (gpio_is_valid(pdata->phy_irq_pin)) { 530 ret = devm_gpio_request(&bp->pdev->dev, 531 pdata->phy_irq_pin, "phy int"); 532 if (!ret) { 533 phy_irq = gpio_to_irq(pdata->phy_irq_pin); 534 phydev->irq = (phy_irq < 0) ? PHY_POLL : phy_irq; 535 } 536 } else { 537 phydev->irq = PHY_POLL; 538 } 539 } 540 541 /* attach the mac to the phy */ 542 ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, 543 bp->phy_interface); 544 if (ret) { 545 netdev_err(dev, "Could not attach to PHY\n"); 546 return ret; 547 } 548 } 549 550 /* mask with MAC supported features */ 551 if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 552 phydev->supported &= PHY_GBIT_FEATURES; 553 else 554 phydev->supported &= PHY_BASIC_FEATURES; 555 556 if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) 557 phydev->supported &= ~SUPPORTED_1000baseT_Half; 558 559 phydev->advertising = phydev->supported; 560 561 bp->link = 0; 562 bp->speed = 0; 563 bp->duplex = -1; 564 565 return 0; 566 } 567 568 static int macb_mii_init(struct macb *bp) 569 { 570 struct macb_platform_data *pdata; 571 struct device_node *np; 572 int err; 573 574 /* Enable management port */ 575 macb_writel(bp, NCR, MACB_BIT(MPE)); 576 577 bp->mii_bus = mdiobus_alloc(); 578 if (!bp->mii_bus) { 579 err = -ENOMEM; 580 goto err_out; 581 } 582 583 bp->mii_bus->name = "MACB_mii_bus"; 584 bp->mii_bus->read = &macb_mdio_read; 585 bp->mii_bus->write = &macb_mdio_write; 586 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 587 bp->pdev->name, bp->pdev->id); 588 bp->mii_bus->priv = bp; 589 bp->mii_bus->parent = &bp->pdev->dev; 590 pdata = dev_get_platdata(&bp->pdev->dev); 591 592 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 593 594 np = bp->pdev->dev.of_node; 595 if (pdata) 596 bp->mii_bus->phy_mask = pdata->phy_mask; 597 598 err = of_mdiobus_register(bp->mii_bus, np); 599 if (err) 600 goto err_out_free_mdiobus; 601 602 err = macb_mii_probe(bp->dev); 603 if (err) 604 goto err_out_unregister_bus; 605 606 return 0; 607 608 err_out_unregister_bus: 609 mdiobus_unregister(bp->mii_bus); 610 if (np && of_phy_is_fixed_link(np)) 611 of_phy_deregister_fixed_link(np); 612 err_out_free_mdiobus: 613 of_node_put(bp->phy_node); 614 mdiobus_free(bp->mii_bus); 615 err_out: 616 return err; 617 } 618 619 static void macb_update_stats(struct macb *bp) 620 { 621 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 622 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 623 int offset = MACB_PFR; 624 625 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 626 627 for (; p < end; p++, offset += 4) 628 *p += bp->macb_reg_readl(bp, offset); 629 } 630 631 static int macb_halt_tx(struct macb *bp) 632 { 633 unsigned long halt_time, timeout; 634 u32 status; 635 636 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 637 638 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 639 do { 640 halt_time = jiffies; 641 status = macb_readl(bp, TSR); 642 if (!(status & MACB_BIT(TGO))) 643 return 0; 644 645 usleep_range(10, 250); 646 } while (time_before(halt_time, timeout)); 647 648 return -ETIMEDOUT; 649 } 650 651 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 652 { 653 if (tx_skb->mapping) { 654 if (tx_skb->mapped_as_page) 655 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 656 tx_skb->size, DMA_TO_DEVICE); 657 else 658 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 659 tx_skb->size, DMA_TO_DEVICE); 660 tx_skb->mapping = 0; 661 } 662 663 if (tx_skb->skb) { 664 dev_kfree_skb_any(tx_skb->skb); 665 tx_skb->skb = NULL; 666 } 667 } 668 669 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 670 { 671 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 672 struct macb_dma_desc_64 *desc_64; 673 674 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 675 desc_64 = macb_64b_desc(bp, desc); 676 desc_64->addrh = upper_32_bits(addr); 677 } 678 #endif 679 desc->addr = lower_32_bits(addr); 680 } 681 682 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 683 { 684 dma_addr_t addr = 0; 685 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 686 struct macb_dma_desc_64 *desc_64; 687 688 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 689 desc_64 = macb_64b_desc(bp, desc); 690 addr = ((u64)(desc_64->addrh) << 32); 691 } 692 #endif 693 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 694 return addr; 695 } 696 697 static void macb_tx_error_task(struct work_struct *work) 698 { 699 struct macb_queue *queue = container_of(work, struct macb_queue, 700 tx_error_task); 701 struct macb *bp = queue->bp; 702 struct macb_tx_skb *tx_skb; 703 struct macb_dma_desc *desc; 704 struct sk_buff *skb; 705 unsigned int tail; 706 unsigned long flags; 707 708 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 709 (unsigned int)(queue - bp->queues), 710 queue->tx_tail, queue->tx_head); 711 712 /* Prevent the queue IRQ handlers from running: each of them may call 713 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 714 * As explained below, we have to halt the transmission before updating 715 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 716 * network engine about the macb/gem being halted. 717 */ 718 spin_lock_irqsave(&bp->lock, flags); 719 720 /* Make sure nobody is trying to queue up new packets */ 721 netif_tx_stop_all_queues(bp->dev); 722 723 /* Stop transmission now 724 * (in case we have just queued new packets) 725 * macb/gem must be halted to write TBQP register 726 */ 727 if (macb_halt_tx(bp)) 728 /* Just complain for now, reinitializing TX path can be good */ 729 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 730 731 /* Treat frames in TX queue including the ones that caused the error. 732 * Free transmit buffers in upper layer. 733 */ 734 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 735 u32 ctrl; 736 737 desc = macb_tx_desc(queue, tail); 738 ctrl = desc->ctrl; 739 tx_skb = macb_tx_skb(queue, tail); 740 skb = tx_skb->skb; 741 742 if (ctrl & MACB_BIT(TX_USED)) { 743 /* skb is set for the last buffer of the frame */ 744 while (!skb) { 745 macb_tx_unmap(bp, tx_skb); 746 tail++; 747 tx_skb = macb_tx_skb(queue, tail); 748 skb = tx_skb->skb; 749 } 750 751 /* ctrl still refers to the first buffer descriptor 752 * since it's the only one written back by the hardware 753 */ 754 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 755 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 756 macb_tx_ring_wrap(bp, tail), 757 skb->data); 758 bp->dev->stats.tx_packets++; 759 queue->stats.tx_packets++; 760 bp->dev->stats.tx_bytes += skb->len; 761 queue->stats.tx_bytes += skb->len; 762 } 763 } else { 764 /* "Buffers exhausted mid-frame" errors may only happen 765 * if the driver is buggy, so complain loudly about 766 * those. Statistics are updated by hardware. 767 */ 768 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 769 netdev_err(bp->dev, 770 "BUG: TX buffers exhausted mid-frame\n"); 771 772 desc->ctrl = ctrl | MACB_BIT(TX_USED); 773 } 774 775 macb_tx_unmap(bp, tx_skb); 776 } 777 778 /* Set end of TX queue */ 779 desc = macb_tx_desc(queue, 0); 780 macb_set_addr(bp, desc, 0); 781 desc->ctrl = MACB_BIT(TX_USED); 782 783 /* Make descriptor updates visible to hardware */ 784 wmb(); 785 786 /* Reinitialize the TX desc queue */ 787 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 788 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 789 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 790 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 791 #endif 792 /* Make TX ring reflect state of hardware */ 793 queue->tx_head = 0; 794 queue->tx_tail = 0; 795 796 /* Housework before enabling TX IRQ */ 797 macb_writel(bp, TSR, macb_readl(bp, TSR)); 798 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 799 800 /* Now we are ready to start transmission again */ 801 netif_tx_start_all_queues(bp->dev); 802 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 803 804 spin_unlock_irqrestore(&bp->lock, flags); 805 } 806 807 static void macb_tx_interrupt(struct macb_queue *queue) 808 { 809 unsigned int tail; 810 unsigned int head; 811 u32 status; 812 struct macb *bp = queue->bp; 813 u16 queue_index = queue - bp->queues; 814 815 status = macb_readl(bp, TSR); 816 macb_writel(bp, TSR, status); 817 818 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 819 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 820 821 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 822 (unsigned long)status); 823 824 head = queue->tx_head; 825 for (tail = queue->tx_tail; tail != head; tail++) { 826 struct macb_tx_skb *tx_skb; 827 struct sk_buff *skb; 828 struct macb_dma_desc *desc; 829 u32 ctrl; 830 831 desc = macb_tx_desc(queue, tail); 832 833 /* Make hw descriptor updates visible to CPU */ 834 rmb(); 835 836 ctrl = desc->ctrl; 837 838 /* TX_USED bit is only set by hardware on the very first buffer 839 * descriptor of the transmitted frame. 840 */ 841 if (!(ctrl & MACB_BIT(TX_USED))) 842 break; 843 844 /* Process all buffers of the current transmitted frame */ 845 for (;; tail++) { 846 tx_skb = macb_tx_skb(queue, tail); 847 skb = tx_skb->skb; 848 849 /* First, update TX stats if needed */ 850 if (skb) { 851 if (gem_ptp_do_txstamp(queue, skb, desc) == 0) { 852 /* skb now belongs to timestamp buffer 853 * and will be removed later 854 */ 855 tx_skb->skb = NULL; 856 } 857 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 858 macb_tx_ring_wrap(bp, tail), 859 skb->data); 860 bp->dev->stats.tx_packets++; 861 queue->stats.tx_packets++; 862 bp->dev->stats.tx_bytes += skb->len; 863 queue->stats.tx_bytes += skb->len; 864 } 865 866 /* Now we can safely release resources */ 867 macb_tx_unmap(bp, tx_skb); 868 869 /* skb is set only for the last buffer of the frame. 870 * WARNING: at this point skb has been freed by 871 * macb_tx_unmap(). 872 */ 873 if (skb) 874 break; 875 } 876 } 877 878 queue->tx_tail = tail; 879 if (__netif_subqueue_stopped(bp->dev, queue_index) && 880 CIRC_CNT(queue->tx_head, queue->tx_tail, 881 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 882 netif_wake_subqueue(bp->dev, queue_index); 883 } 884 885 static void gem_rx_refill(struct macb_queue *queue) 886 { 887 unsigned int entry; 888 struct sk_buff *skb; 889 dma_addr_t paddr; 890 struct macb *bp = queue->bp; 891 struct macb_dma_desc *desc; 892 893 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 894 bp->rx_ring_size) > 0) { 895 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 896 897 /* Make hw descriptor updates visible to CPU */ 898 rmb(); 899 900 queue->rx_prepared_head++; 901 desc = macb_rx_desc(queue, entry); 902 903 if (!queue->rx_skbuff[entry]) { 904 /* allocate sk_buff for this free entry in ring */ 905 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 906 if (unlikely(!skb)) { 907 netdev_err(bp->dev, 908 "Unable to allocate sk_buff\n"); 909 break; 910 } 911 912 /* now fill corresponding descriptor entry */ 913 paddr = dma_map_single(&bp->pdev->dev, skb->data, 914 bp->rx_buffer_size, 915 DMA_FROM_DEVICE); 916 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 917 dev_kfree_skb(skb); 918 break; 919 } 920 921 queue->rx_skbuff[entry] = skb; 922 923 if (entry == bp->rx_ring_size - 1) 924 paddr |= MACB_BIT(RX_WRAP); 925 macb_set_addr(bp, desc, paddr); 926 desc->ctrl = 0; 927 928 /* properly align Ethernet header */ 929 skb_reserve(skb, NET_IP_ALIGN); 930 } else { 931 desc->addr &= ~MACB_BIT(RX_USED); 932 desc->ctrl = 0; 933 } 934 } 935 936 /* Make descriptor updates visible to hardware */ 937 wmb(); 938 939 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 940 queue, queue->rx_prepared_head, queue->rx_tail); 941 } 942 943 /* Mark DMA descriptors from begin up to and not including end as unused */ 944 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 945 unsigned int end) 946 { 947 unsigned int frag; 948 949 for (frag = begin; frag != end; frag++) { 950 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 951 952 desc->addr &= ~MACB_BIT(RX_USED); 953 } 954 955 /* Make descriptor updates visible to hardware */ 956 wmb(); 957 958 /* When this happens, the hardware stats registers for 959 * whatever caused this is updated, so we don't have to record 960 * anything. 961 */ 962 } 963 964 static int gem_rx(struct macb_queue *queue, int budget) 965 { 966 struct macb *bp = queue->bp; 967 unsigned int len; 968 unsigned int entry; 969 struct sk_buff *skb; 970 struct macb_dma_desc *desc; 971 int count = 0; 972 973 while (count < budget) { 974 u32 ctrl; 975 dma_addr_t addr; 976 bool rxused; 977 978 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 979 desc = macb_rx_desc(queue, entry); 980 981 /* Make hw descriptor updates visible to CPU */ 982 rmb(); 983 984 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 985 addr = macb_get_addr(bp, desc); 986 ctrl = desc->ctrl; 987 988 if (!rxused) 989 break; 990 991 queue->rx_tail++; 992 count++; 993 994 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 995 netdev_err(bp->dev, 996 "not whole frame pointed by descriptor\n"); 997 bp->dev->stats.rx_dropped++; 998 queue->stats.rx_dropped++; 999 break; 1000 } 1001 skb = queue->rx_skbuff[entry]; 1002 if (unlikely(!skb)) { 1003 netdev_err(bp->dev, 1004 "inconsistent Rx descriptor chain\n"); 1005 bp->dev->stats.rx_dropped++; 1006 queue->stats.rx_dropped++; 1007 break; 1008 } 1009 /* now everything is ready for receiving packet */ 1010 queue->rx_skbuff[entry] = NULL; 1011 len = ctrl & bp->rx_frm_len_mask; 1012 1013 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1014 1015 skb_put(skb, len); 1016 dma_unmap_single(&bp->pdev->dev, addr, 1017 bp->rx_buffer_size, DMA_FROM_DEVICE); 1018 1019 skb->protocol = eth_type_trans(skb, bp->dev); 1020 skb_checksum_none_assert(skb); 1021 if (bp->dev->features & NETIF_F_RXCSUM && 1022 !(bp->dev->flags & IFF_PROMISC) && 1023 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1024 skb->ip_summed = CHECKSUM_UNNECESSARY; 1025 1026 bp->dev->stats.rx_packets++; 1027 queue->stats.rx_packets++; 1028 bp->dev->stats.rx_bytes += skb->len; 1029 queue->stats.rx_bytes += skb->len; 1030 1031 gem_ptp_do_rxstamp(bp, skb, desc); 1032 1033 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1034 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1035 skb->len, skb->csum); 1036 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1037 skb_mac_header(skb), 16, true); 1038 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1039 skb->data, 32, true); 1040 #endif 1041 1042 netif_receive_skb(skb); 1043 } 1044 1045 gem_rx_refill(queue); 1046 1047 return count; 1048 } 1049 1050 static int macb_rx_frame(struct macb_queue *queue, unsigned int first_frag, 1051 unsigned int last_frag) 1052 { 1053 unsigned int len; 1054 unsigned int frag; 1055 unsigned int offset; 1056 struct sk_buff *skb; 1057 struct macb_dma_desc *desc; 1058 struct macb *bp = queue->bp; 1059 1060 desc = macb_rx_desc(queue, last_frag); 1061 len = desc->ctrl & bp->rx_frm_len_mask; 1062 1063 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1064 macb_rx_ring_wrap(bp, first_frag), 1065 macb_rx_ring_wrap(bp, last_frag), len); 1066 1067 /* The ethernet header starts NET_IP_ALIGN bytes into the 1068 * first buffer. Since the header is 14 bytes, this makes the 1069 * payload word-aligned. 1070 * 1071 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1072 * the two padding bytes into the skb so that we avoid hitting 1073 * the slowpath in memcpy(), and pull them off afterwards. 1074 */ 1075 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1076 if (!skb) { 1077 bp->dev->stats.rx_dropped++; 1078 for (frag = first_frag; ; frag++) { 1079 desc = macb_rx_desc(queue, frag); 1080 desc->addr &= ~MACB_BIT(RX_USED); 1081 if (frag == last_frag) 1082 break; 1083 } 1084 1085 /* Make descriptor updates visible to hardware */ 1086 wmb(); 1087 1088 return 1; 1089 } 1090 1091 offset = 0; 1092 len += NET_IP_ALIGN; 1093 skb_checksum_none_assert(skb); 1094 skb_put(skb, len); 1095 1096 for (frag = first_frag; ; frag++) { 1097 unsigned int frag_len = bp->rx_buffer_size; 1098 1099 if (offset + frag_len > len) { 1100 if (unlikely(frag != last_frag)) { 1101 dev_kfree_skb_any(skb); 1102 return -1; 1103 } 1104 frag_len = len - offset; 1105 } 1106 skb_copy_to_linear_data_offset(skb, offset, 1107 macb_rx_buffer(queue, frag), 1108 frag_len); 1109 offset += bp->rx_buffer_size; 1110 desc = macb_rx_desc(queue, frag); 1111 desc->addr &= ~MACB_BIT(RX_USED); 1112 1113 if (frag == last_frag) 1114 break; 1115 } 1116 1117 /* Make descriptor updates visible to hardware */ 1118 wmb(); 1119 1120 __skb_pull(skb, NET_IP_ALIGN); 1121 skb->protocol = eth_type_trans(skb, bp->dev); 1122 1123 bp->dev->stats.rx_packets++; 1124 bp->dev->stats.rx_bytes += skb->len; 1125 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1126 skb->len, skb->csum); 1127 netif_receive_skb(skb); 1128 1129 return 0; 1130 } 1131 1132 static inline void macb_init_rx_ring(struct macb_queue *queue) 1133 { 1134 struct macb *bp = queue->bp; 1135 dma_addr_t addr; 1136 struct macb_dma_desc *desc = NULL; 1137 int i; 1138 1139 addr = queue->rx_buffers_dma; 1140 for (i = 0; i < bp->rx_ring_size; i++) { 1141 desc = macb_rx_desc(queue, i); 1142 macb_set_addr(bp, desc, addr); 1143 desc->ctrl = 0; 1144 addr += bp->rx_buffer_size; 1145 } 1146 desc->addr |= MACB_BIT(RX_WRAP); 1147 queue->rx_tail = 0; 1148 } 1149 1150 static int macb_rx(struct macb_queue *queue, int budget) 1151 { 1152 struct macb *bp = queue->bp; 1153 bool reset_rx_queue = false; 1154 int received = 0; 1155 unsigned int tail; 1156 int first_frag = -1; 1157 1158 for (tail = queue->rx_tail; budget > 0; tail++) { 1159 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1160 u32 ctrl; 1161 1162 /* Make hw descriptor updates visible to CPU */ 1163 rmb(); 1164 1165 ctrl = desc->ctrl; 1166 1167 if (!(desc->addr & MACB_BIT(RX_USED))) 1168 break; 1169 1170 if (ctrl & MACB_BIT(RX_SOF)) { 1171 if (first_frag != -1) 1172 discard_partial_frame(queue, first_frag, tail); 1173 first_frag = tail; 1174 } 1175 1176 if (ctrl & MACB_BIT(RX_EOF)) { 1177 int dropped; 1178 1179 if (unlikely(first_frag == -1)) { 1180 reset_rx_queue = true; 1181 continue; 1182 } 1183 1184 dropped = macb_rx_frame(queue, first_frag, tail); 1185 first_frag = -1; 1186 if (unlikely(dropped < 0)) { 1187 reset_rx_queue = true; 1188 continue; 1189 } 1190 if (!dropped) { 1191 received++; 1192 budget--; 1193 } 1194 } 1195 } 1196 1197 if (unlikely(reset_rx_queue)) { 1198 unsigned long flags; 1199 u32 ctrl; 1200 1201 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1202 1203 spin_lock_irqsave(&bp->lock, flags); 1204 1205 ctrl = macb_readl(bp, NCR); 1206 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1207 1208 macb_init_rx_ring(queue); 1209 queue_writel(queue, RBQP, queue->rx_ring_dma); 1210 1211 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1212 1213 spin_unlock_irqrestore(&bp->lock, flags); 1214 return received; 1215 } 1216 1217 if (first_frag != -1) 1218 queue->rx_tail = first_frag; 1219 else 1220 queue->rx_tail = tail; 1221 1222 return received; 1223 } 1224 1225 static int macb_poll(struct napi_struct *napi, int budget) 1226 { 1227 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1228 struct macb *bp = queue->bp; 1229 int work_done; 1230 u32 status; 1231 1232 status = macb_readl(bp, RSR); 1233 macb_writel(bp, RSR, status); 1234 1235 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1236 (unsigned long)status, budget); 1237 1238 work_done = bp->macbgem_ops.mog_rx(queue, budget); 1239 if (work_done < budget) { 1240 napi_complete_done(napi, work_done); 1241 1242 /* Packets received while interrupts were disabled */ 1243 status = macb_readl(bp, RSR); 1244 if (status) { 1245 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1246 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1247 napi_reschedule(napi); 1248 } else { 1249 queue_writel(queue, IER, MACB_RX_INT_FLAGS); 1250 } 1251 } 1252 1253 /* TODO: Handle errors */ 1254 1255 return work_done; 1256 } 1257 1258 static void macb_hresp_error_task(unsigned long data) 1259 { 1260 struct macb *bp = (struct macb *)data; 1261 struct net_device *dev = bp->dev; 1262 struct macb_queue *queue = bp->queues; 1263 unsigned int q; 1264 u32 ctrl; 1265 1266 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1267 queue_writel(queue, IDR, MACB_RX_INT_FLAGS | 1268 MACB_TX_INT_FLAGS | 1269 MACB_BIT(HRESP)); 1270 } 1271 ctrl = macb_readl(bp, NCR); 1272 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1273 macb_writel(bp, NCR, ctrl); 1274 1275 netif_tx_stop_all_queues(dev); 1276 netif_carrier_off(dev); 1277 1278 bp->macbgem_ops.mog_init_rings(bp); 1279 1280 /* Initialize TX and RX buffers */ 1281 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1282 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 1283 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1284 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1285 queue_writel(queue, RBQPH, 1286 upper_32_bits(queue->rx_ring_dma)); 1287 #endif 1288 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1289 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1290 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1291 queue_writel(queue, TBQPH, 1292 upper_32_bits(queue->tx_ring_dma)); 1293 #endif 1294 1295 /* Enable interrupts */ 1296 queue_writel(queue, IER, 1297 MACB_RX_INT_FLAGS | 1298 MACB_TX_INT_FLAGS | 1299 MACB_BIT(HRESP)); 1300 } 1301 1302 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1303 macb_writel(bp, NCR, ctrl); 1304 1305 netif_carrier_on(dev); 1306 netif_tx_start_all_queues(dev); 1307 } 1308 1309 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1310 { 1311 struct macb_queue *queue = dev_id; 1312 struct macb *bp = queue->bp; 1313 struct net_device *dev = bp->dev; 1314 u32 status, ctrl; 1315 1316 status = queue_readl(queue, ISR); 1317 1318 if (unlikely(!status)) 1319 return IRQ_NONE; 1320 1321 spin_lock(&bp->lock); 1322 1323 while (status) { 1324 /* close possible race with dev_close */ 1325 if (unlikely(!netif_running(dev))) { 1326 queue_writel(queue, IDR, -1); 1327 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1328 queue_writel(queue, ISR, -1); 1329 break; 1330 } 1331 1332 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1333 (unsigned int)(queue - bp->queues), 1334 (unsigned long)status); 1335 1336 if (status & MACB_RX_INT_FLAGS) { 1337 /* There's no point taking any more interrupts 1338 * until we have processed the buffers. The 1339 * scheduling call may fail if the poll routine 1340 * is already scheduled, so disable interrupts 1341 * now. 1342 */ 1343 queue_writel(queue, IDR, MACB_RX_INT_FLAGS); 1344 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1345 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1346 1347 if (napi_schedule_prep(&queue->napi)) { 1348 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1349 __napi_schedule(&queue->napi); 1350 } 1351 } 1352 1353 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1354 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1355 schedule_work(&queue->tx_error_task); 1356 1357 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1358 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1359 1360 break; 1361 } 1362 1363 if (status & MACB_BIT(TCOMP)) 1364 macb_tx_interrupt(queue); 1365 1366 /* Link change detection isn't possible with RMII, so we'll 1367 * add that if/when we get our hands on a full-blown MII PHY. 1368 */ 1369 1370 /* There is a hardware issue under heavy load where DMA can 1371 * stop, this causes endless "used buffer descriptor read" 1372 * interrupts but it can be cleared by re-enabling RX. See 1373 * the at91 manual, section 41.3.1 or the Zynq manual 1374 * section 16.7.4 for details. 1375 */ 1376 if (status & MACB_BIT(RXUBR)) { 1377 ctrl = macb_readl(bp, NCR); 1378 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1379 wmb(); 1380 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1381 1382 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1383 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1384 } 1385 1386 if (status & MACB_BIT(ISR_ROVR)) { 1387 /* We missed at least one packet */ 1388 if (macb_is_gem(bp)) 1389 bp->hw_stats.gem.rx_overruns++; 1390 else 1391 bp->hw_stats.macb.rx_overruns++; 1392 1393 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1394 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1395 } 1396 1397 if (status & MACB_BIT(HRESP)) { 1398 tasklet_schedule(&bp->hresp_err_tasklet); 1399 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1400 1401 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1402 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1403 } 1404 status = queue_readl(queue, ISR); 1405 } 1406 1407 spin_unlock(&bp->lock); 1408 1409 return IRQ_HANDLED; 1410 } 1411 1412 #ifdef CONFIG_NET_POLL_CONTROLLER 1413 /* Polling receive - used by netconsole and other diagnostic tools 1414 * to allow network i/o with interrupts disabled. 1415 */ 1416 static void macb_poll_controller(struct net_device *dev) 1417 { 1418 struct macb *bp = netdev_priv(dev); 1419 struct macb_queue *queue; 1420 unsigned long flags; 1421 unsigned int q; 1422 1423 local_irq_save(flags); 1424 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1425 macb_interrupt(dev->irq, queue); 1426 local_irq_restore(flags); 1427 } 1428 #endif 1429 1430 static unsigned int macb_tx_map(struct macb *bp, 1431 struct macb_queue *queue, 1432 struct sk_buff *skb, 1433 unsigned int hdrlen) 1434 { 1435 dma_addr_t mapping; 1436 unsigned int len, entry, i, tx_head = queue->tx_head; 1437 struct macb_tx_skb *tx_skb = NULL; 1438 struct macb_dma_desc *desc; 1439 unsigned int offset, size, count = 0; 1440 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1441 unsigned int eof = 1, mss_mfs = 0; 1442 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1443 1444 /* LSO */ 1445 if (skb_shinfo(skb)->gso_size != 0) { 1446 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1447 /* UDP - UFO */ 1448 lso_ctrl = MACB_LSO_UFO_ENABLE; 1449 else 1450 /* TCP - TSO */ 1451 lso_ctrl = MACB_LSO_TSO_ENABLE; 1452 } 1453 1454 /* First, map non-paged data */ 1455 len = skb_headlen(skb); 1456 1457 /* first buffer length */ 1458 size = hdrlen; 1459 1460 offset = 0; 1461 while (len) { 1462 entry = macb_tx_ring_wrap(bp, tx_head); 1463 tx_skb = &queue->tx_skb[entry]; 1464 1465 mapping = dma_map_single(&bp->pdev->dev, 1466 skb->data + offset, 1467 size, DMA_TO_DEVICE); 1468 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1469 goto dma_error; 1470 1471 /* Save info to properly release resources */ 1472 tx_skb->skb = NULL; 1473 tx_skb->mapping = mapping; 1474 tx_skb->size = size; 1475 tx_skb->mapped_as_page = false; 1476 1477 len -= size; 1478 offset += size; 1479 count++; 1480 tx_head++; 1481 1482 size = min(len, bp->max_tx_length); 1483 } 1484 1485 /* Then, map paged data from fragments */ 1486 for (f = 0; f < nr_frags; f++) { 1487 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1488 1489 len = skb_frag_size(frag); 1490 offset = 0; 1491 while (len) { 1492 size = min(len, bp->max_tx_length); 1493 entry = macb_tx_ring_wrap(bp, tx_head); 1494 tx_skb = &queue->tx_skb[entry]; 1495 1496 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1497 offset, size, DMA_TO_DEVICE); 1498 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1499 goto dma_error; 1500 1501 /* Save info to properly release resources */ 1502 tx_skb->skb = NULL; 1503 tx_skb->mapping = mapping; 1504 tx_skb->size = size; 1505 tx_skb->mapped_as_page = true; 1506 1507 len -= size; 1508 offset += size; 1509 count++; 1510 tx_head++; 1511 } 1512 } 1513 1514 /* Should never happen */ 1515 if (unlikely(!tx_skb)) { 1516 netdev_err(bp->dev, "BUG! empty skb!\n"); 1517 return 0; 1518 } 1519 1520 /* This is the last buffer of the frame: save socket buffer */ 1521 tx_skb->skb = skb; 1522 1523 /* Update TX ring: update buffer descriptors in reverse order 1524 * to avoid race condition 1525 */ 1526 1527 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1528 * to set the end of TX queue 1529 */ 1530 i = tx_head; 1531 entry = macb_tx_ring_wrap(bp, i); 1532 ctrl = MACB_BIT(TX_USED); 1533 desc = macb_tx_desc(queue, entry); 1534 desc->ctrl = ctrl; 1535 1536 if (lso_ctrl) { 1537 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1538 /* include header and FCS in value given to h/w */ 1539 mss_mfs = skb_shinfo(skb)->gso_size + 1540 skb_transport_offset(skb) + 1541 ETH_FCS_LEN; 1542 else /* TSO */ { 1543 mss_mfs = skb_shinfo(skb)->gso_size; 1544 /* TCP Sequence Number Source Select 1545 * can be set only for TSO 1546 */ 1547 seq_ctrl = 0; 1548 } 1549 } 1550 1551 do { 1552 i--; 1553 entry = macb_tx_ring_wrap(bp, i); 1554 tx_skb = &queue->tx_skb[entry]; 1555 desc = macb_tx_desc(queue, entry); 1556 1557 ctrl = (u32)tx_skb->size; 1558 if (eof) { 1559 ctrl |= MACB_BIT(TX_LAST); 1560 eof = 0; 1561 } 1562 if (unlikely(entry == (bp->tx_ring_size - 1))) 1563 ctrl |= MACB_BIT(TX_WRAP); 1564 1565 /* First descriptor is header descriptor */ 1566 if (i == queue->tx_head) { 1567 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1568 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1569 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1570 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1571 ctrl |= MACB_BIT(TX_NOCRC); 1572 } else 1573 /* Only set MSS/MFS on payload descriptors 1574 * (second or later descriptor) 1575 */ 1576 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1577 1578 /* Set TX buffer descriptor */ 1579 macb_set_addr(bp, desc, tx_skb->mapping); 1580 /* desc->addr must be visible to hardware before clearing 1581 * 'TX_USED' bit in desc->ctrl. 1582 */ 1583 wmb(); 1584 desc->ctrl = ctrl; 1585 } while (i != queue->tx_head); 1586 1587 queue->tx_head = tx_head; 1588 1589 return count; 1590 1591 dma_error: 1592 netdev_err(bp->dev, "TX DMA map failed\n"); 1593 1594 for (i = queue->tx_head; i != tx_head; i++) { 1595 tx_skb = macb_tx_skb(queue, i); 1596 1597 macb_tx_unmap(bp, tx_skb); 1598 } 1599 1600 return 0; 1601 } 1602 1603 static netdev_features_t macb_features_check(struct sk_buff *skb, 1604 struct net_device *dev, 1605 netdev_features_t features) 1606 { 1607 unsigned int nr_frags, f; 1608 unsigned int hdrlen; 1609 1610 /* Validate LSO compatibility */ 1611 1612 /* there is only one buffer */ 1613 if (!skb_is_nonlinear(skb)) 1614 return features; 1615 1616 /* length of header */ 1617 hdrlen = skb_transport_offset(skb); 1618 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1619 hdrlen += tcp_hdrlen(skb); 1620 1621 /* For LSO: 1622 * When software supplies two or more payload buffers all payload buffers 1623 * apart from the last must be a multiple of 8 bytes in size. 1624 */ 1625 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1626 return features & ~MACB_NETIF_LSO; 1627 1628 nr_frags = skb_shinfo(skb)->nr_frags; 1629 /* No need to check last fragment */ 1630 nr_frags--; 1631 for (f = 0; f < nr_frags; f++) { 1632 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1633 1634 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1635 return features & ~MACB_NETIF_LSO; 1636 } 1637 return features; 1638 } 1639 1640 static inline int macb_clear_csum(struct sk_buff *skb) 1641 { 1642 /* no change for packets without checksum offloading */ 1643 if (skb->ip_summed != CHECKSUM_PARTIAL) 1644 return 0; 1645 1646 /* make sure we can modify the header */ 1647 if (unlikely(skb_cow_head(skb, 0))) 1648 return -1; 1649 1650 /* initialize checksum field 1651 * This is required - at least for Zynq, which otherwise calculates 1652 * wrong UDP header checksums for UDP packets with UDP data len <=2 1653 */ 1654 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1655 return 0; 1656 } 1657 1658 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1659 { 1660 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1661 int padlen = ETH_ZLEN - (*skb)->len; 1662 int headroom = skb_headroom(*skb); 1663 int tailroom = skb_tailroom(*skb); 1664 struct sk_buff *nskb; 1665 u32 fcs; 1666 1667 if (!(ndev->features & NETIF_F_HW_CSUM) || 1668 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1669 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1670 return 0; 1671 1672 if (padlen <= 0) { 1673 /* FCS could be appeded to tailroom. */ 1674 if (tailroom >= ETH_FCS_LEN) 1675 goto add_fcs; 1676 /* FCS could be appeded by moving data to headroom. */ 1677 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1678 padlen = 0; 1679 /* No room for FCS, need to reallocate skb. */ 1680 else 1681 padlen = ETH_FCS_LEN - tailroom; 1682 } else { 1683 /* Add room for FCS. */ 1684 padlen += ETH_FCS_LEN; 1685 } 1686 1687 if (!cloned && headroom + tailroom >= padlen) { 1688 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1689 skb_set_tail_pointer(*skb, (*skb)->len); 1690 } else { 1691 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1692 if (!nskb) 1693 return -ENOMEM; 1694 1695 dev_kfree_skb_any(*skb); 1696 *skb = nskb; 1697 } 1698 1699 if (padlen) { 1700 if (padlen >= ETH_FCS_LEN) 1701 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1702 else 1703 skb_trim(*skb, ETH_FCS_LEN - padlen); 1704 } 1705 1706 add_fcs: 1707 /* set FCS to packet */ 1708 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1709 fcs = ~fcs; 1710 1711 skb_put_u8(*skb, fcs & 0xff); 1712 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1713 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1714 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1715 1716 return 0; 1717 } 1718 1719 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1720 { 1721 u16 queue_index = skb_get_queue_mapping(skb); 1722 struct macb *bp = netdev_priv(dev); 1723 struct macb_queue *queue = &bp->queues[queue_index]; 1724 unsigned long flags; 1725 unsigned int desc_cnt, nr_frags, frag_size, f; 1726 unsigned int hdrlen; 1727 bool is_lso, is_udp = 0; 1728 netdev_tx_t ret = NETDEV_TX_OK; 1729 1730 if (macb_clear_csum(skb)) { 1731 dev_kfree_skb_any(skb); 1732 return ret; 1733 } 1734 1735 if (macb_pad_and_fcs(&skb, dev)) { 1736 dev_kfree_skb_any(skb); 1737 return ret; 1738 } 1739 1740 is_lso = (skb_shinfo(skb)->gso_size != 0); 1741 1742 if (is_lso) { 1743 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1744 1745 /* length of headers */ 1746 if (is_udp) 1747 /* only queue eth + ip headers separately for UDP */ 1748 hdrlen = skb_transport_offset(skb); 1749 else 1750 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1751 if (skb_headlen(skb) < hdrlen) { 1752 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1753 /* if this is required, would need to copy to single buffer */ 1754 return NETDEV_TX_BUSY; 1755 } 1756 } else 1757 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1758 1759 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1760 netdev_vdbg(bp->dev, 1761 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1762 queue_index, skb->len, skb->head, skb->data, 1763 skb_tail_pointer(skb), skb_end_pointer(skb)); 1764 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1765 skb->data, 16, true); 1766 #endif 1767 1768 /* Count how many TX buffer descriptors are needed to send this 1769 * socket buffer: skb fragments of jumbo frames may need to be 1770 * split into many buffer descriptors. 1771 */ 1772 if (is_lso && (skb_headlen(skb) > hdrlen)) 1773 /* extra header descriptor if also payload in first buffer */ 1774 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1775 else 1776 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1777 nr_frags = skb_shinfo(skb)->nr_frags; 1778 for (f = 0; f < nr_frags; f++) { 1779 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1780 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1781 } 1782 1783 spin_lock_irqsave(&bp->lock, flags); 1784 1785 /* This is a hard error, log it. */ 1786 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1787 bp->tx_ring_size) < desc_cnt) { 1788 netif_stop_subqueue(dev, queue_index); 1789 spin_unlock_irqrestore(&bp->lock, flags); 1790 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1791 queue->tx_head, queue->tx_tail); 1792 return NETDEV_TX_BUSY; 1793 } 1794 1795 /* Map socket buffer for DMA transfer */ 1796 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1797 dev_kfree_skb_any(skb); 1798 goto unlock; 1799 } 1800 1801 /* Make newly initialized descriptor visible to hardware */ 1802 wmb(); 1803 skb_tx_timestamp(skb); 1804 1805 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1806 1807 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1808 netif_stop_subqueue(dev, queue_index); 1809 1810 unlock: 1811 spin_unlock_irqrestore(&bp->lock, flags); 1812 1813 return ret; 1814 } 1815 1816 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1817 { 1818 if (!macb_is_gem(bp)) { 1819 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1820 } else { 1821 bp->rx_buffer_size = size; 1822 1823 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1824 netdev_dbg(bp->dev, 1825 "RX buffer must be multiple of %d bytes, expanding\n", 1826 RX_BUFFER_MULTIPLE); 1827 bp->rx_buffer_size = 1828 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1829 } 1830 } 1831 1832 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1833 bp->dev->mtu, bp->rx_buffer_size); 1834 } 1835 1836 static void gem_free_rx_buffers(struct macb *bp) 1837 { 1838 struct sk_buff *skb; 1839 struct macb_dma_desc *desc; 1840 struct macb_queue *queue; 1841 dma_addr_t addr; 1842 unsigned int q; 1843 int i; 1844 1845 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1846 if (!queue->rx_skbuff) 1847 continue; 1848 1849 for (i = 0; i < bp->rx_ring_size; i++) { 1850 skb = queue->rx_skbuff[i]; 1851 1852 if (!skb) 1853 continue; 1854 1855 desc = macb_rx_desc(queue, i); 1856 addr = macb_get_addr(bp, desc); 1857 1858 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1859 DMA_FROM_DEVICE); 1860 dev_kfree_skb_any(skb); 1861 skb = NULL; 1862 } 1863 1864 kfree(queue->rx_skbuff); 1865 queue->rx_skbuff = NULL; 1866 } 1867 } 1868 1869 static void macb_free_rx_buffers(struct macb *bp) 1870 { 1871 struct macb_queue *queue = &bp->queues[0]; 1872 1873 if (queue->rx_buffers) { 1874 dma_free_coherent(&bp->pdev->dev, 1875 bp->rx_ring_size * bp->rx_buffer_size, 1876 queue->rx_buffers, queue->rx_buffers_dma); 1877 queue->rx_buffers = NULL; 1878 } 1879 } 1880 1881 static void macb_free_consistent(struct macb *bp) 1882 { 1883 struct macb_queue *queue; 1884 unsigned int q; 1885 int size; 1886 1887 bp->macbgem_ops.mog_free_rx_buffers(bp); 1888 1889 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1890 kfree(queue->tx_skb); 1891 queue->tx_skb = NULL; 1892 if (queue->tx_ring) { 1893 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1894 dma_free_coherent(&bp->pdev->dev, size, 1895 queue->tx_ring, queue->tx_ring_dma); 1896 queue->tx_ring = NULL; 1897 } 1898 if (queue->rx_ring) { 1899 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 1900 dma_free_coherent(&bp->pdev->dev, size, 1901 queue->rx_ring, queue->rx_ring_dma); 1902 queue->rx_ring = NULL; 1903 } 1904 } 1905 } 1906 1907 static int gem_alloc_rx_buffers(struct macb *bp) 1908 { 1909 struct macb_queue *queue; 1910 unsigned int q; 1911 int size; 1912 1913 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1914 size = bp->rx_ring_size * sizeof(struct sk_buff *); 1915 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 1916 if (!queue->rx_skbuff) 1917 return -ENOMEM; 1918 else 1919 netdev_dbg(bp->dev, 1920 "Allocated %d RX struct sk_buff entries at %p\n", 1921 bp->rx_ring_size, queue->rx_skbuff); 1922 } 1923 return 0; 1924 } 1925 1926 static int macb_alloc_rx_buffers(struct macb *bp) 1927 { 1928 struct macb_queue *queue = &bp->queues[0]; 1929 int size; 1930 1931 size = bp->rx_ring_size * bp->rx_buffer_size; 1932 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 1933 &queue->rx_buffers_dma, GFP_KERNEL); 1934 if (!queue->rx_buffers) 1935 return -ENOMEM; 1936 1937 netdev_dbg(bp->dev, 1938 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 1939 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 1940 return 0; 1941 } 1942 1943 static int macb_alloc_consistent(struct macb *bp) 1944 { 1945 struct macb_queue *queue; 1946 unsigned int q; 1947 int size; 1948 1949 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1950 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1951 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1952 &queue->tx_ring_dma, 1953 GFP_KERNEL); 1954 if (!queue->tx_ring) 1955 goto out_err; 1956 netdev_dbg(bp->dev, 1957 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 1958 q, size, (unsigned long)queue->tx_ring_dma, 1959 queue->tx_ring); 1960 1961 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 1962 queue->tx_skb = kmalloc(size, GFP_KERNEL); 1963 if (!queue->tx_skb) 1964 goto out_err; 1965 1966 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 1967 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1968 &queue->rx_ring_dma, GFP_KERNEL); 1969 if (!queue->rx_ring) 1970 goto out_err; 1971 netdev_dbg(bp->dev, 1972 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 1973 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 1974 } 1975 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 1976 goto out_err; 1977 1978 return 0; 1979 1980 out_err: 1981 macb_free_consistent(bp); 1982 return -ENOMEM; 1983 } 1984 1985 static void gem_init_rings(struct macb *bp) 1986 { 1987 struct macb_queue *queue; 1988 struct macb_dma_desc *desc = NULL; 1989 unsigned int q; 1990 int i; 1991 1992 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1993 for (i = 0; i < bp->tx_ring_size; i++) { 1994 desc = macb_tx_desc(queue, i); 1995 macb_set_addr(bp, desc, 0); 1996 desc->ctrl = MACB_BIT(TX_USED); 1997 } 1998 desc->ctrl |= MACB_BIT(TX_WRAP); 1999 queue->tx_head = 0; 2000 queue->tx_tail = 0; 2001 2002 queue->rx_tail = 0; 2003 queue->rx_prepared_head = 0; 2004 2005 gem_rx_refill(queue); 2006 } 2007 2008 } 2009 2010 static void macb_init_rings(struct macb *bp) 2011 { 2012 int i; 2013 struct macb_dma_desc *desc = NULL; 2014 2015 macb_init_rx_ring(&bp->queues[0]); 2016 2017 for (i = 0; i < bp->tx_ring_size; i++) { 2018 desc = macb_tx_desc(&bp->queues[0], i); 2019 macb_set_addr(bp, desc, 0); 2020 desc->ctrl = MACB_BIT(TX_USED); 2021 } 2022 bp->queues[0].tx_head = 0; 2023 bp->queues[0].tx_tail = 0; 2024 desc->ctrl |= MACB_BIT(TX_WRAP); 2025 } 2026 2027 static void macb_reset_hw(struct macb *bp) 2028 { 2029 struct macb_queue *queue; 2030 unsigned int q; 2031 2032 /* Disable RX and TX (XXX: Should we halt the transmission 2033 * more gracefully?) 2034 */ 2035 macb_writel(bp, NCR, 0); 2036 2037 /* Clear the stats registers (XXX: Update stats first?) */ 2038 macb_writel(bp, NCR, MACB_BIT(CLRSTAT)); 2039 2040 /* Clear all status flags */ 2041 macb_writel(bp, TSR, -1); 2042 macb_writel(bp, RSR, -1); 2043 2044 /* Disable all interrupts */ 2045 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2046 queue_writel(queue, IDR, -1); 2047 queue_readl(queue, ISR); 2048 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2049 queue_writel(queue, ISR, -1); 2050 } 2051 } 2052 2053 static u32 gem_mdc_clk_div(struct macb *bp) 2054 { 2055 u32 config; 2056 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2057 2058 if (pclk_hz <= 20000000) 2059 config = GEM_BF(CLK, GEM_CLK_DIV8); 2060 else if (pclk_hz <= 40000000) 2061 config = GEM_BF(CLK, GEM_CLK_DIV16); 2062 else if (pclk_hz <= 80000000) 2063 config = GEM_BF(CLK, GEM_CLK_DIV32); 2064 else if (pclk_hz <= 120000000) 2065 config = GEM_BF(CLK, GEM_CLK_DIV48); 2066 else if (pclk_hz <= 160000000) 2067 config = GEM_BF(CLK, GEM_CLK_DIV64); 2068 else 2069 config = GEM_BF(CLK, GEM_CLK_DIV96); 2070 2071 return config; 2072 } 2073 2074 static u32 macb_mdc_clk_div(struct macb *bp) 2075 { 2076 u32 config; 2077 unsigned long pclk_hz; 2078 2079 if (macb_is_gem(bp)) 2080 return gem_mdc_clk_div(bp); 2081 2082 pclk_hz = clk_get_rate(bp->pclk); 2083 if (pclk_hz <= 20000000) 2084 config = MACB_BF(CLK, MACB_CLK_DIV8); 2085 else if (pclk_hz <= 40000000) 2086 config = MACB_BF(CLK, MACB_CLK_DIV16); 2087 else if (pclk_hz <= 80000000) 2088 config = MACB_BF(CLK, MACB_CLK_DIV32); 2089 else 2090 config = MACB_BF(CLK, MACB_CLK_DIV64); 2091 2092 return config; 2093 } 2094 2095 /* Get the DMA bus width field of the network configuration register that we 2096 * should program. We find the width from decoding the design configuration 2097 * register to find the maximum supported data bus width. 2098 */ 2099 static u32 macb_dbw(struct macb *bp) 2100 { 2101 if (!macb_is_gem(bp)) 2102 return 0; 2103 2104 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2105 case 4: 2106 return GEM_BF(DBW, GEM_DBW128); 2107 case 2: 2108 return GEM_BF(DBW, GEM_DBW64); 2109 case 1: 2110 default: 2111 return GEM_BF(DBW, GEM_DBW32); 2112 } 2113 } 2114 2115 /* Configure the receive DMA engine 2116 * - use the correct receive buffer size 2117 * - set best burst length for DMA operations 2118 * (if not supported by FIFO, it will fallback to default) 2119 * - set both rx/tx packet buffers to full memory size 2120 * These are configurable parameters for GEM. 2121 */ 2122 static void macb_configure_dma(struct macb *bp) 2123 { 2124 struct macb_queue *queue; 2125 u32 buffer_size; 2126 unsigned int q; 2127 u32 dmacfg; 2128 2129 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2130 if (macb_is_gem(bp)) { 2131 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2132 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2133 if (q) 2134 queue_writel(queue, RBQS, buffer_size); 2135 else 2136 dmacfg |= GEM_BF(RXBS, buffer_size); 2137 } 2138 if (bp->dma_burst_length) 2139 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2140 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2141 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2142 2143 if (bp->native_io) 2144 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2145 else 2146 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2147 2148 if (bp->dev->features & NETIF_F_HW_CSUM) 2149 dmacfg |= GEM_BIT(TXCOEN); 2150 else 2151 dmacfg &= ~GEM_BIT(TXCOEN); 2152 2153 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2154 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2155 dmacfg |= GEM_BIT(ADDR64); 2156 #endif 2157 #ifdef CONFIG_MACB_USE_HWSTAMP 2158 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2159 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2160 #endif 2161 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2162 dmacfg); 2163 gem_writel(bp, DMACFG, dmacfg); 2164 } 2165 } 2166 2167 static void macb_init_hw(struct macb *bp) 2168 { 2169 struct macb_queue *queue; 2170 unsigned int q; 2171 2172 u32 config; 2173 2174 macb_reset_hw(bp); 2175 macb_set_hwaddr(bp); 2176 2177 config = macb_mdc_clk_div(bp); 2178 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2179 config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2180 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2181 config |= MACB_BIT(PAE); /* PAuse Enable */ 2182 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2183 if (bp->caps & MACB_CAPS_JUMBO) 2184 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2185 else 2186 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2187 if (bp->dev->flags & IFF_PROMISC) 2188 config |= MACB_BIT(CAF); /* Copy All Frames */ 2189 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2190 config |= GEM_BIT(RXCOEN); 2191 if (!(bp->dev->flags & IFF_BROADCAST)) 2192 config |= MACB_BIT(NBC); /* No BroadCast */ 2193 config |= macb_dbw(bp); 2194 macb_writel(bp, NCFGR, config); 2195 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2196 gem_writel(bp, JML, bp->jumbo_max_len); 2197 bp->speed = SPEED_10; 2198 bp->duplex = DUPLEX_HALF; 2199 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2200 if (bp->caps & MACB_CAPS_JUMBO) 2201 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2202 2203 macb_configure_dma(bp); 2204 2205 /* Initialize TX and RX buffers */ 2206 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2207 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 2208 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2209 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2210 queue_writel(queue, RBQPH, upper_32_bits(queue->rx_ring_dma)); 2211 #endif 2212 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 2213 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2214 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2215 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 2216 #endif 2217 2218 /* Enable interrupts */ 2219 queue_writel(queue, IER, 2220 MACB_RX_INT_FLAGS | 2221 MACB_TX_INT_FLAGS | 2222 MACB_BIT(HRESP)); 2223 } 2224 2225 /* Enable TX and RX */ 2226 macb_writel(bp, NCR, MACB_BIT(RE) | MACB_BIT(TE) | MACB_BIT(MPE)); 2227 } 2228 2229 /* The hash address register is 64 bits long and takes up two 2230 * locations in the memory map. The least significant bits are stored 2231 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2232 * 2233 * The unicast hash enable and the multicast hash enable bits in the 2234 * network configuration register enable the reception of hash matched 2235 * frames. The destination address is reduced to a 6 bit index into 2236 * the 64 bit hash register using the following hash function. The 2237 * hash function is an exclusive or of every sixth bit of the 2238 * destination address. 2239 * 2240 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2241 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2242 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2243 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2244 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2245 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2246 * 2247 * da[0] represents the least significant bit of the first byte 2248 * received, that is, the multicast/unicast indicator, and da[47] 2249 * represents the most significant bit of the last byte received. If 2250 * the hash index, hi[n], points to a bit that is set in the hash 2251 * register then the frame will be matched according to whether the 2252 * frame is multicast or unicast. A multicast match will be signalled 2253 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2254 * index points to a bit set in the hash register. A unicast match 2255 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2256 * and the hash index points to a bit set in the hash register. To 2257 * receive all multicast frames, the hash register should be set with 2258 * all ones and the multicast hash enable bit should be set in the 2259 * network configuration register. 2260 */ 2261 2262 static inline int hash_bit_value(int bitnr, __u8 *addr) 2263 { 2264 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2265 return 1; 2266 return 0; 2267 } 2268 2269 /* Return the hash index value for the specified address. */ 2270 static int hash_get_index(__u8 *addr) 2271 { 2272 int i, j, bitval; 2273 int hash_index = 0; 2274 2275 for (j = 0; j < 6; j++) { 2276 for (i = 0, bitval = 0; i < 8; i++) 2277 bitval ^= hash_bit_value(i * 6 + j, addr); 2278 2279 hash_index |= (bitval << j); 2280 } 2281 2282 return hash_index; 2283 } 2284 2285 /* Add multicast addresses to the internal multicast-hash table. */ 2286 static void macb_sethashtable(struct net_device *dev) 2287 { 2288 struct netdev_hw_addr *ha; 2289 unsigned long mc_filter[2]; 2290 unsigned int bitnr; 2291 struct macb *bp = netdev_priv(dev); 2292 2293 mc_filter[0] = 0; 2294 mc_filter[1] = 0; 2295 2296 netdev_for_each_mc_addr(ha, dev) { 2297 bitnr = hash_get_index(ha->addr); 2298 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2299 } 2300 2301 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2302 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2303 } 2304 2305 /* Enable/Disable promiscuous and multicast modes. */ 2306 static void macb_set_rx_mode(struct net_device *dev) 2307 { 2308 unsigned long cfg; 2309 struct macb *bp = netdev_priv(dev); 2310 2311 cfg = macb_readl(bp, NCFGR); 2312 2313 if (dev->flags & IFF_PROMISC) { 2314 /* Enable promiscuous mode */ 2315 cfg |= MACB_BIT(CAF); 2316 2317 /* Disable RX checksum offload */ 2318 if (macb_is_gem(bp)) 2319 cfg &= ~GEM_BIT(RXCOEN); 2320 } else { 2321 /* Disable promiscuous mode */ 2322 cfg &= ~MACB_BIT(CAF); 2323 2324 /* Enable RX checksum offload only if requested */ 2325 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2326 cfg |= GEM_BIT(RXCOEN); 2327 } 2328 2329 if (dev->flags & IFF_ALLMULTI) { 2330 /* Enable all multicast mode */ 2331 macb_or_gem_writel(bp, HRB, -1); 2332 macb_or_gem_writel(bp, HRT, -1); 2333 cfg |= MACB_BIT(NCFGR_MTI); 2334 } else if (!netdev_mc_empty(dev)) { 2335 /* Enable specific multicasts */ 2336 macb_sethashtable(dev); 2337 cfg |= MACB_BIT(NCFGR_MTI); 2338 } else if (dev->flags & (~IFF_ALLMULTI)) { 2339 /* Disable all multicast mode */ 2340 macb_or_gem_writel(bp, HRB, 0); 2341 macb_or_gem_writel(bp, HRT, 0); 2342 cfg &= ~MACB_BIT(NCFGR_MTI); 2343 } 2344 2345 macb_writel(bp, NCFGR, cfg); 2346 } 2347 2348 static int macb_open(struct net_device *dev) 2349 { 2350 struct macb *bp = netdev_priv(dev); 2351 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2352 struct macb_queue *queue; 2353 unsigned int q; 2354 int err; 2355 2356 netdev_dbg(bp->dev, "open\n"); 2357 2358 /* carrier starts down */ 2359 netif_carrier_off(dev); 2360 2361 /* if the phy is not yet register, retry later*/ 2362 if (!dev->phydev) 2363 return -EAGAIN; 2364 2365 /* RX buffers initialization */ 2366 macb_init_rx_buffer_size(bp, bufsz); 2367 2368 err = macb_alloc_consistent(bp); 2369 if (err) { 2370 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2371 err); 2372 return err; 2373 } 2374 2375 bp->macbgem_ops.mog_init_rings(bp); 2376 macb_init_hw(bp); 2377 2378 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2379 napi_enable(&queue->napi); 2380 2381 /* schedule a link state check */ 2382 phy_start(dev->phydev); 2383 2384 netif_tx_start_all_queues(dev); 2385 2386 if (bp->ptp_info) 2387 bp->ptp_info->ptp_init(dev); 2388 2389 return 0; 2390 } 2391 2392 static int macb_close(struct net_device *dev) 2393 { 2394 struct macb *bp = netdev_priv(dev); 2395 struct macb_queue *queue; 2396 unsigned long flags; 2397 unsigned int q; 2398 2399 netif_tx_stop_all_queues(dev); 2400 2401 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2402 napi_disable(&queue->napi); 2403 2404 if (dev->phydev) 2405 phy_stop(dev->phydev); 2406 2407 spin_lock_irqsave(&bp->lock, flags); 2408 macb_reset_hw(bp); 2409 netif_carrier_off(dev); 2410 spin_unlock_irqrestore(&bp->lock, flags); 2411 2412 macb_free_consistent(bp); 2413 2414 if (bp->ptp_info) 2415 bp->ptp_info->ptp_remove(dev); 2416 2417 return 0; 2418 } 2419 2420 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2421 { 2422 if (netif_running(dev)) 2423 return -EBUSY; 2424 2425 dev->mtu = new_mtu; 2426 2427 return 0; 2428 } 2429 2430 static void gem_update_stats(struct macb *bp) 2431 { 2432 struct macb_queue *queue; 2433 unsigned int i, q, idx; 2434 unsigned long *stat; 2435 2436 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2437 2438 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2439 u32 offset = gem_statistics[i].offset; 2440 u64 val = bp->macb_reg_readl(bp, offset); 2441 2442 bp->ethtool_stats[i] += val; 2443 *p += val; 2444 2445 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2446 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2447 val = bp->macb_reg_readl(bp, offset + 4); 2448 bp->ethtool_stats[i] += ((u64)val) << 32; 2449 *(++p) += val; 2450 } 2451 } 2452 2453 idx = GEM_STATS_LEN; 2454 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2455 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2456 bp->ethtool_stats[idx++] = *stat; 2457 } 2458 2459 static struct net_device_stats *gem_get_stats(struct macb *bp) 2460 { 2461 struct gem_stats *hwstat = &bp->hw_stats.gem; 2462 struct net_device_stats *nstat = &bp->dev->stats; 2463 2464 gem_update_stats(bp); 2465 2466 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2467 hwstat->rx_alignment_errors + 2468 hwstat->rx_resource_errors + 2469 hwstat->rx_overruns + 2470 hwstat->rx_oversize_frames + 2471 hwstat->rx_jabbers + 2472 hwstat->rx_undersized_frames + 2473 hwstat->rx_length_field_frame_errors); 2474 nstat->tx_errors = (hwstat->tx_late_collisions + 2475 hwstat->tx_excessive_collisions + 2476 hwstat->tx_underrun + 2477 hwstat->tx_carrier_sense_errors); 2478 nstat->multicast = hwstat->rx_multicast_frames; 2479 nstat->collisions = (hwstat->tx_single_collision_frames + 2480 hwstat->tx_multiple_collision_frames + 2481 hwstat->tx_excessive_collisions); 2482 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2483 hwstat->rx_jabbers + 2484 hwstat->rx_undersized_frames + 2485 hwstat->rx_length_field_frame_errors); 2486 nstat->rx_over_errors = hwstat->rx_resource_errors; 2487 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2488 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2489 nstat->rx_fifo_errors = hwstat->rx_overruns; 2490 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2491 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2492 nstat->tx_fifo_errors = hwstat->tx_underrun; 2493 2494 return nstat; 2495 } 2496 2497 static void gem_get_ethtool_stats(struct net_device *dev, 2498 struct ethtool_stats *stats, u64 *data) 2499 { 2500 struct macb *bp; 2501 2502 bp = netdev_priv(dev); 2503 gem_update_stats(bp); 2504 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2505 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2506 } 2507 2508 static int gem_get_sset_count(struct net_device *dev, int sset) 2509 { 2510 struct macb *bp = netdev_priv(dev); 2511 2512 switch (sset) { 2513 case ETH_SS_STATS: 2514 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2515 default: 2516 return -EOPNOTSUPP; 2517 } 2518 } 2519 2520 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2521 { 2522 char stat_string[ETH_GSTRING_LEN]; 2523 struct macb *bp = netdev_priv(dev); 2524 struct macb_queue *queue; 2525 unsigned int i; 2526 unsigned int q; 2527 2528 switch (sset) { 2529 case ETH_SS_STATS: 2530 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2531 memcpy(p, gem_statistics[i].stat_string, 2532 ETH_GSTRING_LEN); 2533 2534 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2535 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2536 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2537 q, queue_statistics[i].stat_string); 2538 memcpy(p, stat_string, ETH_GSTRING_LEN); 2539 } 2540 } 2541 break; 2542 } 2543 } 2544 2545 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2546 { 2547 struct macb *bp = netdev_priv(dev); 2548 struct net_device_stats *nstat = &bp->dev->stats; 2549 struct macb_stats *hwstat = &bp->hw_stats.macb; 2550 2551 if (macb_is_gem(bp)) 2552 return gem_get_stats(bp); 2553 2554 /* read stats from hardware */ 2555 macb_update_stats(bp); 2556 2557 /* Convert HW stats into netdevice stats */ 2558 nstat->rx_errors = (hwstat->rx_fcs_errors + 2559 hwstat->rx_align_errors + 2560 hwstat->rx_resource_errors + 2561 hwstat->rx_overruns + 2562 hwstat->rx_oversize_pkts + 2563 hwstat->rx_jabbers + 2564 hwstat->rx_undersize_pkts + 2565 hwstat->rx_length_mismatch); 2566 nstat->tx_errors = (hwstat->tx_late_cols + 2567 hwstat->tx_excessive_cols + 2568 hwstat->tx_underruns + 2569 hwstat->tx_carrier_errors + 2570 hwstat->sqe_test_errors); 2571 nstat->collisions = (hwstat->tx_single_cols + 2572 hwstat->tx_multiple_cols + 2573 hwstat->tx_excessive_cols); 2574 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2575 hwstat->rx_jabbers + 2576 hwstat->rx_undersize_pkts + 2577 hwstat->rx_length_mismatch); 2578 nstat->rx_over_errors = hwstat->rx_resource_errors + 2579 hwstat->rx_overruns; 2580 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2581 nstat->rx_frame_errors = hwstat->rx_align_errors; 2582 nstat->rx_fifo_errors = hwstat->rx_overruns; 2583 /* XXX: What does "missed" mean? */ 2584 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2585 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2586 nstat->tx_fifo_errors = hwstat->tx_underruns; 2587 /* Don't know about heartbeat or window errors... */ 2588 2589 return nstat; 2590 } 2591 2592 static int macb_get_regs_len(struct net_device *netdev) 2593 { 2594 return MACB_GREGS_NBR * sizeof(u32); 2595 } 2596 2597 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2598 void *p) 2599 { 2600 struct macb *bp = netdev_priv(dev); 2601 unsigned int tail, head; 2602 u32 *regs_buff = p; 2603 2604 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2605 | MACB_GREGS_VERSION; 2606 2607 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2608 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2609 2610 regs_buff[0] = macb_readl(bp, NCR); 2611 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2612 regs_buff[2] = macb_readl(bp, NSR); 2613 regs_buff[3] = macb_readl(bp, TSR); 2614 regs_buff[4] = macb_readl(bp, RBQP); 2615 regs_buff[5] = macb_readl(bp, TBQP); 2616 regs_buff[6] = macb_readl(bp, RSR); 2617 regs_buff[7] = macb_readl(bp, IMR); 2618 2619 regs_buff[8] = tail; 2620 regs_buff[9] = head; 2621 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2622 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2623 2624 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2625 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2626 if (macb_is_gem(bp)) 2627 regs_buff[13] = gem_readl(bp, DMACFG); 2628 } 2629 2630 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2631 { 2632 struct macb *bp = netdev_priv(netdev); 2633 2634 wol->supported = 0; 2635 wol->wolopts = 0; 2636 2637 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2638 wol->supported = WAKE_MAGIC; 2639 2640 if (bp->wol & MACB_WOL_ENABLED) 2641 wol->wolopts |= WAKE_MAGIC; 2642 } 2643 } 2644 2645 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2646 { 2647 struct macb *bp = netdev_priv(netdev); 2648 2649 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2650 (wol->wolopts & ~WAKE_MAGIC)) 2651 return -EOPNOTSUPP; 2652 2653 if (wol->wolopts & WAKE_MAGIC) 2654 bp->wol |= MACB_WOL_ENABLED; 2655 else 2656 bp->wol &= ~MACB_WOL_ENABLED; 2657 2658 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2659 2660 return 0; 2661 } 2662 2663 static void macb_get_ringparam(struct net_device *netdev, 2664 struct ethtool_ringparam *ring) 2665 { 2666 struct macb *bp = netdev_priv(netdev); 2667 2668 ring->rx_max_pending = MAX_RX_RING_SIZE; 2669 ring->tx_max_pending = MAX_TX_RING_SIZE; 2670 2671 ring->rx_pending = bp->rx_ring_size; 2672 ring->tx_pending = bp->tx_ring_size; 2673 } 2674 2675 static int macb_set_ringparam(struct net_device *netdev, 2676 struct ethtool_ringparam *ring) 2677 { 2678 struct macb *bp = netdev_priv(netdev); 2679 u32 new_rx_size, new_tx_size; 2680 unsigned int reset = 0; 2681 2682 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2683 return -EINVAL; 2684 2685 new_rx_size = clamp_t(u32, ring->rx_pending, 2686 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2687 new_rx_size = roundup_pow_of_two(new_rx_size); 2688 2689 new_tx_size = clamp_t(u32, ring->tx_pending, 2690 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2691 new_tx_size = roundup_pow_of_two(new_tx_size); 2692 2693 if ((new_tx_size == bp->tx_ring_size) && 2694 (new_rx_size == bp->rx_ring_size)) { 2695 /* nothing to do */ 2696 return 0; 2697 } 2698 2699 if (netif_running(bp->dev)) { 2700 reset = 1; 2701 macb_close(bp->dev); 2702 } 2703 2704 bp->rx_ring_size = new_rx_size; 2705 bp->tx_ring_size = new_tx_size; 2706 2707 if (reset) 2708 macb_open(bp->dev); 2709 2710 return 0; 2711 } 2712 2713 #ifdef CONFIG_MACB_USE_HWSTAMP 2714 static unsigned int gem_get_tsu_rate(struct macb *bp) 2715 { 2716 struct clk *tsu_clk; 2717 unsigned int tsu_rate; 2718 2719 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2720 if (!IS_ERR(tsu_clk)) 2721 tsu_rate = clk_get_rate(tsu_clk); 2722 /* try pclk instead */ 2723 else if (!IS_ERR(bp->pclk)) { 2724 tsu_clk = bp->pclk; 2725 tsu_rate = clk_get_rate(tsu_clk); 2726 } else 2727 return -ENOTSUPP; 2728 return tsu_rate; 2729 } 2730 2731 static s32 gem_get_ptp_max_adj(void) 2732 { 2733 return 64000000; 2734 } 2735 2736 static int gem_get_ts_info(struct net_device *dev, 2737 struct ethtool_ts_info *info) 2738 { 2739 struct macb *bp = netdev_priv(dev); 2740 2741 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2742 ethtool_op_get_ts_info(dev, info); 2743 return 0; 2744 } 2745 2746 info->so_timestamping = 2747 SOF_TIMESTAMPING_TX_SOFTWARE | 2748 SOF_TIMESTAMPING_RX_SOFTWARE | 2749 SOF_TIMESTAMPING_SOFTWARE | 2750 SOF_TIMESTAMPING_TX_HARDWARE | 2751 SOF_TIMESTAMPING_RX_HARDWARE | 2752 SOF_TIMESTAMPING_RAW_HARDWARE; 2753 info->tx_types = 2754 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2755 (1 << HWTSTAMP_TX_OFF) | 2756 (1 << HWTSTAMP_TX_ON); 2757 info->rx_filters = 2758 (1 << HWTSTAMP_FILTER_NONE) | 2759 (1 << HWTSTAMP_FILTER_ALL); 2760 2761 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2762 2763 return 0; 2764 } 2765 2766 static struct macb_ptp_info gem_ptp_info = { 2767 .ptp_init = gem_ptp_init, 2768 .ptp_remove = gem_ptp_remove, 2769 .get_ptp_max_adj = gem_get_ptp_max_adj, 2770 .get_tsu_rate = gem_get_tsu_rate, 2771 .get_ts_info = gem_get_ts_info, 2772 .get_hwtst = gem_get_hwtst, 2773 .set_hwtst = gem_set_hwtst, 2774 }; 2775 #endif 2776 2777 static int macb_get_ts_info(struct net_device *netdev, 2778 struct ethtool_ts_info *info) 2779 { 2780 struct macb *bp = netdev_priv(netdev); 2781 2782 if (bp->ptp_info) 2783 return bp->ptp_info->get_ts_info(netdev, info); 2784 2785 return ethtool_op_get_ts_info(netdev, info); 2786 } 2787 2788 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2789 { 2790 struct ethtool_rx_fs_item *item; 2791 u32 t2_scr; 2792 int num_t2_scr; 2793 2794 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2795 2796 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2797 struct ethtool_rx_flow_spec *fs = &item->fs; 2798 struct ethtool_tcpip4_spec *tp4sp_m; 2799 2800 if (fs->location >= num_t2_scr) 2801 continue; 2802 2803 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2804 2805 /* enable/disable screener regs for the flow entry */ 2806 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2807 2808 /* only enable fields with no masking */ 2809 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2810 2811 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2812 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2813 else 2814 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2815 2816 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2817 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2818 else 2819 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2820 2821 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2822 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2823 else 2824 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 2825 2826 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 2827 } 2828 } 2829 2830 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 2831 { 2832 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 2833 uint16_t index = fs->location; 2834 u32 w0, w1, t2_scr; 2835 bool cmp_a = false; 2836 bool cmp_b = false; 2837 bool cmp_c = false; 2838 2839 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 2840 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2841 2842 /* ignore field if any masking set */ 2843 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 2844 /* 1st compare reg - IP source address */ 2845 w0 = 0; 2846 w1 = 0; 2847 w0 = tp4sp_v->ip4src; 2848 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2849 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2850 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 2851 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 2852 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 2853 cmp_a = true; 2854 } 2855 2856 /* ignore field if any masking set */ 2857 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 2858 /* 2nd compare reg - IP destination address */ 2859 w0 = 0; 2860 w1 = 0; 2861 w0 = tp4sp_v->ip4dst; 2862 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2863 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2864 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 2865 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 2866 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 2867 cmp_b = true; 2868 } 2869 2870 /* ignore both port fields if masking set in both */ 2871 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 2872 /* 3rd compare reg - source port, destination port */ 2873 w0 = 0; 2874 w1 = 0; 2875 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 2876 if (tp4sp_m->psrc == tp4sp_m->pdst) { 2877 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 2878 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2879 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2880 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2881 } else { 2882 /* only one port definition */ 2883 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 2884 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 2885 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 2886 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 2887 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2888 } else { /* dst port */ 2889 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2890 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 2891 } 2892 } 2893 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 2894 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 2895 cmp_c = true; 2896 } 2897 2898 t2_scr = 0; 2899 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 2900 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 2901 if (cmp_a) 2902 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 2903 if (cmp_b) 2904 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 2905 if (cmp_c) 2906 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 2907 gem_writel_n(bp, SCRT2, index, t2_scr); 2908 } 2909 2910 static int gem_add_flow_filter(struct net_device *netdev, 2911 struct ethtool_rxnfc *cmd) 2912 { 2913 struct macb *bp = netdev_priv(netdev); 2914 struct ethtool_rx_flow_spec *fs = &cmd->fs; 2915 struct ethtool_rx_fs_item *item, *newfs; 2916 unsigned long flags; 2917 int ret = -EINVAL; 2918 bool added = false; 2919 2920 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 2921 if (newfs == NULL) 2922 return -ENOMEM; 2923 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 2924 2925 netdev_dbg(netdev, 2926 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2927 fs->flow_type, (int)fs->ring_cookie, fs->location, 2928 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2929 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2930 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 2931 2932 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2933 2934 /* find correct place to add in list */ 2935 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2936 if (item->fs.location > newfs->fs.location) { 2937 list_add_tail(&newfs->list, &item->list); 2938 added = true; 2939 break; 2940 } else if (item->fs.location == fs->location) { 2941 netdev_err(netdev, "Rule not added: location %d not free!\n", 2942 fs->location); 2943 ret = -EBUSY; 2944 goto err; 2945 } 2946 } 2947 if (!added) 2948 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 2949 2950 gem_prog_cmp_regs(bp, fs); 2951 bp->rx_fs_list.count++; 2952 /* enable filtering if NTUPLE on */ 2953 if (netdev->features & NETIF_F_NTUPLE) 2954 gem_enable_flow_filters(bp, 1); 2955 2956 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2957 return 0; 2958 2959 err: 2960 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2961 kfree(newfs); 2962 return ret; 2963 } 2964 2965 static int gem_del_flow_filter(struct net_device *netdev, 2966 struct ethtool_rxnfc *cmd) 2967 { 2968 struct macb *bp = netdev_priv(netdev); 2969 struct ethtool_rx_fs_item *item; 2970 struct ethtool_rx_flow_spec *fs; 2971 unsigned long flags; 2972 2973 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2974 2975 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2976 if (item->fs.location == cmd->fs.location) { 2977 /* disable screener regs for the flow entry */ 2978 fs = &(item->fs); 2979 netdev_dbg(netdev, 2980 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2981 fs->flow_type, (int)fs->ring_cookie, fs->location, 2982 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2983 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2984 htons(fs->h_u.tcp_ip4_spec.psrc), 2985 htons(fs->h_u.tcp_ip4_spec.pdst)); 2986 2987 gem_writel_n(bp, SCRT2, fs->location, 0); 2988 2989 list_del(&item->list); 2990 bp->rx_fs_list.count--; 2991 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2992 kfree(item); 2993 return 0; 2994 } 2995 } 2996 2997 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2998 return -EINVAL; 2999 } 3000 3001 static int gem_get_flow_entry(struct net_device *netdev, 3002 struct ethtool_rxnfc *cmd) 3003 { 3004 struct macb *bp = netdev_priv(netdev); 3005 struct ethtool_rx_fs_item *item; 3006 3007 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3008 if (item->fs.location == cmd->fs.location) { 3009 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3010 return 0; 3011 } 3012 } 3013 return -EINVAL; 3014 } 3015 3016 static int gem_get_all_flow_entries(struct net_device *netdev, 3017 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3018 { 3019 struct macb *bp = netdev_priv(netdev); 3020 struct ethtool_rx_fs_item *item; 3021 uint32_t cnt = 0; 3022 3023 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3024 if (cnt == cmd->rule_cnt) 3025 return -EMSGSIZE; 3026 rule_locs[cnt] = item->fs.location; 3027 cnt++; 3028 } 3029 cmd->data = bp->max_tuples; 3030 cmd->rule_cnt = cnt; 3031 3032 return 0; 3033 } 3034 3035 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3036 u32 *rule_locs) 3037 { 3038 struct macb *bp = netdev_priv(netdev); 3039 int ret = 0; 3040 3041 switch (cmd->cmd) { 3042 case ETHTOOL_GRXRINGS: 3043 cmd->data = bp->num_queues; 3044 break; 3045 case ETHTOOL_GRXCLSRLCNT: 3046 cmd->rule_cnt = bp->rx_fs_list.count; 3047 break; 3048 case ETHTOOL_GRXCLSRULE: 3049 ret = gem_get_flow_entry(netdev, cmd); 3050 break; 3051 case ETHTOOL_GRXCLSRLALL: 3052 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3053 break; 3054 default: 3055 netdev_err(netdev, 3056 "Command parameter %d is not supported\n", cmd->cmd); 3057 ret = -EOPNOTSUPP; 3058 } 3059 3060 return ret; 3061 } 3062 3063 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3064 { 3065 struct macb *bp = netdev_priv(netdev); 3066 int ret; 3067 3068 switch (cmd->cmd) { 3069 case ETHTOOL_SRXCLSRLINS: 3070 if ((cmd->fs.location >= bp->max_tuples) 3071 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3072 ret = -EINVAL; 3073 break; 3074 } 3075 ret = gem_add_flow_filter(netdev, cmd); 3076 break; 3077 case ETHTOOL_SRXCLSRLDEL: 3078 ret = gem_del_flow_filter(netdev, cmd); 3079 break; 3080 default: 3081 netdev_err(netdev, 3082 "Command parameter %d is not supported\n", cmd->cmd); 3083 ret = -EOPNOTSUPP; 3084 } 3085 3086 return ret; 3087 } 3088 3089 static const struct ethtool_ops macb_ethtool_ops = { 3090 .get_regs_len = macb_get_regs_len, 3091 .get_regs = macb_get_regs, 3092 .get_link = ethtool_op_get_link, 3093 .get_ts_info = ethtool_op_get_ts_info, 3094 .get_wol = macb_get_wol, 3095 .set_wol = macb_set_wol, 3096 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3097 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3098 .get_ringparam = macb_get_ringparam, 3099 .set_ringparam = macb_set_ringparam, 3100 }; 3101 3102 static const struct ethtool_ops gem_ethtool_ops = { 3103 .get_regs_len = macb_get_regs_len, 3104 .get_regs = macb_get_regs, 3105 .get_link = ethtool_op_get_link, 3106 .get_ts_info = macb_get_ts_info, 3107 .get_ethtool_stats = gem_get_ethtool_stats, 3108 .get_strings = gem_get_ethtool_strings, 3109 .get_sset_count = gem_get_sset_count, 3110 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3111 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3112 .get_ringparam = macb_get_ringparam, 3113 .set_ringparam = macb_set_ringparam, 3114 .get_rxnfc = gem_get_rxnfc, 3115 .set_rxnfc = gem_set_rxnfc, 3116 }; 3117 3118 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3119 { 3120 struct phy_device *phydev = dev->phydev; 3121 struct macb *bp = netdev_priv(dev); 3122 3123 if (!netif_running(dev)) 3124 return -EINVAL; 3125 3126 if (!phydev) 3127 return -ENODEV; 3128 3129 if (!bp->ptp_info) 3130 return phy_mii_ioctl(phydev, rq, cmd); 3131 3132 switch (cmd) { 3133 case SIOCSHWTSTAMP: 3134 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3135 case SIOCGHWTSTAMP: 3136 return bp->ptp_info->get_hwtst(dev, rq); 3137 default: 3138 return phy_mii_ioctl(phydev, rq, cmd); 3139 } 3140 } 3141 3142 static int macb_set_features(struct net_device *netdev, 3143 netdev_features_t features) 3144 { 3145 struct macb *bp = netdev_priv(netdev); 3146 netdev_features_t changed = features ^ netdev->features; 3147 3148 /* TX checksum offload */ 3149 if ((changed & NETIF_F_HW_CSUM) && macb_is_gem(bp)) { 3150 u32 dmacfg; 3151 3152 dmacfg = gem_readl(bp, DMACFG); 3153 if (features & NETIF_F_HW_CSUM) 3154 dmacfg |= GEM_BIT(TXCOEN); 3155 else 3156 dmacfg &= ~GEM_BIT(TXCOEN); 3157 gem_writel(bp, DMACFG, dmacfg); 3158 } 3159 3160 /* RX checksum offload */ 3161 if ((changed & NETIF_F_RXCSUM) && macb_is_gem(bp)) { 3162 u32 netcfg; 3163 3164 netcfg = gem_readl(bp, NCFGR); 3165 if (features & NETIF_F_RXCSUM && 3166 !(netdev->flags & IFF_PROMISC)) 3167 netcfg |= GEM_BIT(RXCOEN); 3168 else 3169 netcfg &= ~GEM_BIT(RXCOEN); 3170 gem_writel(bp, NCFGR, netcfg); 3171 } 3172 3173 /* RX Flow Filters */ 3174 if ((changed & NETIF_F_NTUPLE) && macb_is_gem(bp)) { 3175 bool turn_on = features & NETIF_F_NTUPLE; 3176 3177 gem_enable_flow_filters(bp, turn_on); 3178 } 3179 return 0; 3180 } 3181 3182 static const struct net_device_ops macb_netdev_ops = { 3183 .ndo_open = macb_open, 3184 .ndo_stop = macb_close, 3185 .ndo_start_xmit = macb_start_xmit, 3186 .ndo_set_rx_mode = macb_set_rx_mode, 3187 .ndo_get_stats = macb_get_stats, 3188 .ndo_do_ioctl = macb_ioctl, 3189 .ndo_validate_addr = eth_validate_addr, 3190 .ndo_change_mtu = macb_change_mtu, 3191 .ndo_set_mac_address = eth_mac_addr, 3192 #ifdef CONFIG_NET_POLL_CONTROLLER 3193 .ndo_poll_controller = macb_poll_controller, 3194 #endif 3195 .ndo_set_features = macb_set_features, 3196 .ndo_features_check = macb_features_check, 3197 }; 3198 3199 /* Configure peripheral capabilities according to device tree 3200 * and integration options used 3201 */ 3202 static void macb_configure_caps(struct macb *bp, 3203 const struct macb_config *dt_conf) 3204 { 3205 u32 dcfg; 3206 3207 if (dt_conf) 3208 bp->caps = dt_conf->caps; 3209 3210 if (hw_is_gem(bp->regs, bp->native_io)) { 3211 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3212 3213 dcfg = gem_readl(bp, DCFG1); 3214 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3215 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3216 dcfg = gem_readl(bp, DCFG2); 3217 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3218 bp->caps |= MACB_CAPS_FIFO_MODE; 3219 #ifdef CONFIG_MACB_USE_HWSTAMP 3220 if (gem_has_ptp(bp)) { 3221 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3222 pr_err("GEM doesn't support hardware ptp.\n"); 3223 else { 3224 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3225 bp->ptp_info = &gem_ptp_info; 3226 } 3227 } 3228 #endif 3229 } 3230 3231 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3232 } 3233 3234 static void macb_probe_queues(void __iomem *mem, 3235 bool native_io, 3236 unsigned int *queue_mask, 3237 unsigned int *num_queues) 3238 { 3239 unsigned int hw_q; 3240 3241 *queue_mask = 0x1; 3242 *num_queues = 1; 3243 3244 /* is it macb or gem ? 3245 * 3246 * We need to read directly from the hardware here because 3247 * we are early in the probe process and don't have the 3248 * MACB_CAPS_MACB_IS_GEM flag positioned 3249 */ 3250 if (!hw_is_gem(mem, native_io)) 3251 return; 3252 3253 /* bit 0 is never set but queue 0 always exists */ 3254 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3255 3256 *queue_mask |= 0x1; 3257 3258 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3259 if (*queue_mask & (1 << hw_q)) 3260 (*num_queues)++; 3261 } 3262 3263 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3264 struct clk **hclk, struct clk **tx_clk, 3265 struct clk **rx_clk) 3266 { 3267 struct macb_platform_data *pdata; 3268 int err; 3269 3270 pdata = dev_get_platdata(&pdev->dev); 3271 if (pdata) { 3272 *pclk = pdata->pclk; 3273 *hclk = pdata->hclk; 3274 } else { 3275 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3276 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3277 } 3278 3279 if (IS_ERR(*pclk)) { 3280 err = PTR_ERR(*pclk); 3281 dev_err(&pdev->dev, "failed to get macb_clk (%u)\n", err); 3282 return err; 3283 } 3284 3285 if (IS_ERR(*hclk)) { 3286 err = PTR_ERR(*hclk); 3287 dev_err(&pdev->dev, "failed to get hclk (%u)\n", err); 3288 return err; 3289 } 3290 3291 *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); 3292 if (IS_ERR(*tx_clk)) 3293 *tx_clk = NULL; 3294 3295 *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); 3296 if (IS_ERR(*rx_clk)) 3297 *rx_clk = NULL; 3298 3299 err = clk_prepare_enable(*pclk); 3300 if (err) { 3301 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3302 return err; 3303 } 3304 3305 err = clk_prepare_enable(*hclk); 3306 if (err) { 3307 dev_err(&pdev->dev, "failed to enable hclk (%u)\n", err); 3308 goto err_disable_pclk; 3309 } 3310 3311 err = clk_prepare_enable(*tx_clk); 3312 if (err) { 3313 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 3314 goto err_disable_hclk; 3315 } 3316 3317 err = clk_prepare_enable(*rx_clk); 3318 if (err) { 3319 dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); 3320 goto err_disable_txclk; 3321 } 3322 3323 return 0; 3324 3325 err_disable_txclk: 3326 clk_disable_unprepare(*tx_clk); 3327 3328 err_disable_hclk: 3329 clk_disable_unprepare(*hclk); 3330 3331 err_disable_pclk: 3332 clk_disable_unprepare(*pclk); 3333 3334 return err; 3335 } 3336 3337 static int macb_init(struct platform_device *pdev) 3338 { 3339 struct net_device *dev = platform_get_drvdata(pdev); 3340 unsigned int hw_q, q; 3341 struct macb *bp = netdev_priv(dev); 3342 struct macb_queue *queue; 3343 int err; 3344 u32 val, reg; 3345 3346 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3347 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3348 3349 /* set the queue register mapping once for all: queue0 has a special 3350 * register mapping but we don't want to test the queue index then 3351 * compute the corresponding register offset at run time. 3352 */ 3353 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3354 if (!(bp->queue_mask & (1 << hw_q))) 3355 continue; 3356 3357 queue = &bp->queues[q]; 3358 queue->bp = bp; 3359 netif_napi_add(dev, &queue->napi, macb_poll, 64); 3360 if (hw_q) { 3361 queue->ISR = GEM_ISR(hw_q - 1); 3362 queue->IER = GEM_IER(hw_q - 1); 3363 queue->IDR = GEM_IDR(hw_q - 1); 3364 queue->IMR = GEM_IMR(hw_q - 1); 3365 queue->TBQP = GEM_TBQP(hw_q - 1); 3366 queue->RBQP = GEM_RBQP(hw_q - 1); 3367 queue->RBQS = GEM_RBQS(hw_q - 1); 3368 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3369 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3370 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3371 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3372 } 3373 #endif 3374 } else { 3375 /* queue0 uses legacy registers */ 3376 queue->ISR = MACB_ISR; 3377 queue->IER = MACB_IER; 3378 queue->IDR = MACB_IDR; 3379 queue->IMR = MACB_IMR; 3380 queue->TBQP = MACB_TBQP; 3381 queue->RBQP = MACB_RBQP; 3382 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3383 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3384 queue->TBQPH = MACB_TBQPH; 3385 queue->RBQPH = MACB_RBQPH; 3386 } 3387 #endif 3388 } 3389 3390 /* get irq: here we use the linux queue index, not the hardware 3391 * queue index. the queue irq definitions in the device tree 3392 * must remove the optional gaps that could exist in the 3393 * hardware queue mask. 3394 */ 3395 queue->irq = platform_get_irq(pdev, q); 3396 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3397 IRQF_SHARED, dev->name, queue); 3398 if (err) { 3399 dev_err(&pdev->dev, 3400 "Unable to request IRQ %d (error %d)\n", 3401 queue->irq, err); 3402 return err; 3403 } 3404 3405 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3406 q++; 3407 } 3408 3409 dev->netdev_ops = &macb_netdev_ops; 3410 3411 /* setup appropriated routines according to adapter type */ 3412 if (macb_is_gem(bp)) { 3413 bp->max_tx_length = GEM_MAX_TX_LEN; 3414 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3415 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3416 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3417 bp->macbgem_ops.mog_rx = gem_rx; 3418 dev->ethtool_ops = &gem_ethtool_ops; 3419 } else { 3420 bp->max_tx_length = MACB_MAX_TX_LEN; 3421 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3422 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3423 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3424 bp->macbgem_ops.mog_rx = macb_rx; 3425 dev->ethtool_ops = &macb_ethtool_ops; 3426 } 3427 3428 /* Set features */ 3429 dev->hw_features = NETIF_F_SG; 3430 3431 /* Check LSO capability */ 3432 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3433 dev->hw_features |= MACB_NETIF_LSO; 3434 3435 /* Checksum offload is only available on gem with packet buffer */ 3436 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3437 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3438 if (bp->caps & MACB_CAPS_SG_DISABLED) 3439 dev->hw_features &= ~NETIF_F_SG; 3440 dev->features = dev->hw_features; 3441 3442 /* Check RX Flow Filters support. 3443 * Max Rx flows set by availability of screeners & compare regs: 3444 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3445 */ 3446 reg = gem_readl(bp, DCFG8); 3447 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3448 GEM_BFEXT(T2SCR, reg)); 3449 if (bp->max_tuples > 0) { 3450 /* also needs one ethtype match to check IPv4 */ 3451 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3452 /* program this reg now */ 3453 reg = 0; 3454 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3455 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3456 /* Filtering is supported in hw but don't enable it in kernel now */ 3457 dev->hw_features |= NETIF_F_NTUPLE; 3458 /* init Rx flow definitions */ 3459 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3460 bp->rx_fs_list.count = 0; 3461 spin_lock_init(&bp->rx_fs_lock); 3462 } else 3463 bp->max_tuples = 0; 3464 } 3465 3466 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3467 val = 0; 3468 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3469 val = GEM_BIT(RGMII); 3470 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3471 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3472 val = MACB_BIT(RMII); 3473 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3474 val = MACB_BIT(MII); 3475 3476 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3477 val |= MACB_BIT(CLKEN); 3478 3479 macb_or_gem_writel(bp, USRIO, val); 3480 } 3481 3482 /* Set MII management clock divider */ 3483 val = macb_mdc_clk_div(bp); 3484 val |= macb_dbw(bp); 3485 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3486 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3487 macb_writel(bp, NCFGR, val); 3488 3489 return 0; 3490 } 3491 3492 #if defined(CONFIG_OF) 3493 /* 1518 rounded up */ 3494 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3495 /* max number of receive buffers */ 3496 #define AT91ETHER_MAX_RX_DESCR 9 3497 3498 /* Initialize and start the Receiver and Transmit subsystems */ 3499 static int at91ether_start(struct net_device *dev) 3500 { 3501 struct macb *lp = netdev_priv(dev); 3502 struct macb_queue *q = &lp->queues[0]; 3503 struct macb_dma_desc *desc; 3504 dma_addr_t addr; 3505 u32 ctl; 3506 int i; 3507 3508 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3509 (AT91ETHER_MAX_RX_DESCR * 3510 macb_dma_desc_get_size(lp)), 3511 &q->rx_ring_dma, GFP_KERNEL); 3512 if (!q->rx_ring) 3513 return -ENOMEM; 3514 3515 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3516 AT91ETHER_MAX_RX_DESCR * 3517 AT91ETHER_MAX_RBUFF_SZ, 3518 &q->rx_buffers_dma, GFP_KERNEL); 3519 if (!q->rx_buffers) { 3520 dma_free_coherent(&lp->pdev->dev, 3521 AT91ETHER_MAX_RX_DESCR * 3522 macb_dma_desc_get_size(lp), 3523 q->rx_ring, q->rx_ring_dma); 3524 q->rx_ring = NULL; 3525 return -ENOMEM; 3526 } 3527 3528 addr = q->rx_buffers_dma; 3529 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3530 desc = macb_rx_desc(q, i); 3531 macb_set_addr(lp, desc, addr); 3532 desc->ctrl = 0; 3533 addr += AT91ETHER_MAX_RBUFF_SZ; 3534 } 3535 3536 /* Set the Wrap bit on the last descriptor */ 3537 desc->addr |= MACB_BIT(RX_WRAP); 3538 3539 /* Reset buffer index */ 3540 q->rx_tail = 0; 3541 3542 /* Program address of descriptor list in Rx Buffer Queue register */ 3543 macb_writel(lp, RBQP, q->rx_ring_dma); 3544 3545 /* Enable Receive and Transmit */ 3546 ctl = macb_readl(lp, NCR); 3547 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3548 3549 return 0; 3550 } 3551 3552 /* Open the ethernet interface */ 3553 static int at91ether_open(struct net_device *dev) 3554 { 3555 struct macb *lp = netdev_priv(dev); 3556 u32 ctl; 3557 int ret; 3558 3559 /* Clear internal statistics */ 3560 ctl = macb_readl(lp, NCR); 3561 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3562 3563 macb_set_hwaddr(lp); 3564 3565 ret = at91ether_start(dev); 3566 if (ret) 3567 return ret; 3568 3569 /* Enable MAC interrupts */ 3570 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3571 MACB_BIT(RXUBR) | 3572 MACB_BIT(ISR_TUND) | 3573 MACB_BIT(ISR_RLE) | 3574 MACB_BIT(TCOMP) | 3575 MACB_BIT(ISR_ROVR) | 3576 MACB_BIT(HRESP)); 3577 3578 /* schedule a link state check */ 3579 phy_start(dev->phydev); 3580 3581 netif_start_queue(dev); 3582 3583 return 0; 3584 } 3585 3586 /* Close the interface */ 3587 static int at91ether_close(struct net_device *dev) 3588 { 3589 struct macb *lp = netdev_priv(dev); 3590 struct macb_queue *q = &lp->queues[0]; 3591 u32 ctl; 3592 3593 /* Disable Receiver and Transmitter */ 3594 ctl = macb_readl(lp, NCR); 3595 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3596 3597 /* Disable MAC interrupts */ 3598 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3599 MACB_BIT(RXUBR) | 3600 MACB_BIT(ISR_TUND) | 3601 MACB_BIT(ISR_RLE) | 3602 MACB_BIT(TCOMP) | 3603 MACB_BIT(ISR_ROVR) | 3604 MACB_BIT(HRESP)); 3605 3606 netif_stop_queue(dev); 3607 3608 dma_free_coherent(&lp->pdev->dev, 3609 AT91ETHER_MAX_RX_DESCR * 3610 macb_dma_desc_get_size(lp), 3611 q->rx_ring, q->rx_ring_dma); 3612 q->rx_ring = NULL; 3613 3614 dma_free_coherent(&lp->pdev->dev, 3615 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3616 q->rx_buffers, q->rx_buffers_dma); 3617 q->rx_buffers = NULL; 3618 3619 return 0; 3620 } 3621 3622 /* Transmit packet */ 3623 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3624 struct net_device *dev) 3625 { 3626 struct macb *lp = netdev_priv(dev); 3627 3628 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3629 netif_stop_queue(dev); 3630 3631 /* Store packet information (to free when Tx completed) */ 3632 lp->skb = skb; 3633 lp->skb_length = skb->len; 3634 lp->skb_physaddr = dma_map_single(NULL, skb->data, skb->len, 3635 DMA_TO_DEVICE); 3636 if (dma_mapping_error(NULL, lp->skb_physaddr)) { 3637 dev_kfree_skb_any(skb); 3638 dev->stats.tx_dropped++; 3639 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3640 return NETDEV_TX_OK; 3641 } 3642 3643 /* Set address of the data in the Transmit Address register */ 3644 macb_writel(lp, TAR, lp->skb_physaddr); 3645 /* Set length of the packet in the Transmit Control register */ 3646 macb_writel(lp, TCR, skb->len); 3647 3648 } else { 3649 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3650 return NETDEV_TX_BUSY; 3651 } 3652 3653 return NETDEV_TX_OK; 3654 } 3655 3656 /* Extract received frame from buffer descriptors and sent to upper layers. 3657 * (Called from interrupt context) 3658 */ 3659 static void at91ether_rx(struct net_device *dev) 3660 { 3661 struct macb *lp = netdev_priv(dev); 3662 struct macb_queue *q = &lp->queues[0]; 3663 struct macb_dma_desc *desc; 3664 unsigned char *p_recv; 3665 struct sk_buff *skb; 3666 unsigned int pktlen; 3667 3668 desc = macb_rx_desc(q, q->rx_tail); 3669 while (desc->addr & MACB_BIT(RX_USED)) { 3670 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3671 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3672 skb = netdev_alloc_skb(dev, pktlen + 2); 3673 if (skb) { 3674 skb_reserve(skb, 2); 3675 skb_put_data(skb, p_recv, pktlen); 3676 3677 skb->protocol = eth_type_trans(skb, dev); 3678 dev->stats.rx_packets++; 3679 dev->stats.rx_bytes += pktlen; 3680 netif_rx(skb); 3681 } else { 3682 dev->stats.rx_dropped++; 3683 } 3684 3685 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3686 dev->stats.multicast++; 3687 3688 /* reset ownership bit */ 3689 desc->addr &= ~MACB_BIT(RX_USED); 3690 3691 /* wrap after last buffer */ 3692 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3693 q->rx_tail = 0; 3694 else 3695 q->rx_tail++; 3696 3697 desc = macb_rx_desc(q, q->rx_tail); 3698 } 3699 } 3700 3701 /* MAC interrupt handler */ 3702 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3703 { 3704 struct net_device *dev = dev_id; 3705 struct macb *lp = netdev_priv(dev); 3706 u32 intstatus, ctl; 3707 3708 /* MAC Interrupt Status register indicates what interrupts are pending. 3709 * It is automatically cleared once read. 3710 */ 3711 intstatus = macb_readl(lp, ISR); 3712 3713 /* Receive complete */ 3714 if (intstatus & MACB_BIT(RCOMP)) 3715 at91ether_rx(dev); 3716 3717 /* Transmit complete */ 3718 if (intstatus & MACB_BIT(TCOMP)) { 3719 /* The TCOM bit is set even if the transmission failed */ 3720 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3721 dev->stats.tx_errors++; 3722 3723 if (lp->skb) { 3724 dev_kfree_skb_irq(lp->skb); 3725 lp->skb = NULL; 3726 dma_unmap_single(NULL, lp->skb_physaddr, 3727 lp->skb_length, DMA_TO_DEVICE); 3728 dev->stats.tx_packets++; 3729 dev->stats.tx_bytes += lp->skb_length; 3730 } 3731 netif_wake_queue(dev); 3732 } 3733 3734 /* Work-around for EMAC Errata section 41.3.1 */ 3735 if (intstatus & MACB_BIT(RXUBR)) { 3736 ctl = macb_readl(lp, NCR); 3737 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3738 wmb(); 3739 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3740 } 3741 3742 if (intstatus & MACB_BIT(ISR_ROVR)) 3743 netdev_err(dev, "ROVR error\n"); 3744 3745 return IRQ_HANDLED; 3746 } 3747 3748 #ifdef CONFIG_NET_POLL_CONTROLLER 3749 static void at91ether_poll_controller(struct net_device *dev) 3750 { 3751 unsigned long flags; 3752 3753 local_irq_save(flags); 3754 at91ether_interrupt(dev->irq, dev); 3755 local_irq_restore(flags); 3756 } 3757 #endif 3758 3759 static const struct net_device_ops at91ether_netdev_ops = { 3760 .ndo_open = at91ether_open, 3761 .ndo_stop = at91ether_close, 3762 .ndo_start_xmit = at91ether_start_xmit, 3763 .ndo_get_stats = macb_get_stats, 3764 .ndo_set_rx_mode = macb_set_rx_mode, 3765 .ndo_set_mac_address = eth_mac_addr, 3766 .ndo_do_ioctl = macb_ioctl, 3767 .ndo_validate_addr = eth_validate_addr, 3768 #ifdef CONFIG_NET_POLL_CONTROLLER 3769 .ndo_poll_controller = at91ether_poll_controller, 3770 #endif 3771 }; 3772 3773 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3774 struct clk **hclk, struct clk **tx_clk, 3775 struct clk **rx_clk) 3776 { 3777 int err; 3778 3779 *hclk = NULL; 3780 *tx_clk = NULL; 3781 *rx_clk = NULL; 3782 3783 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3784 if (IS_ERR(*pclk)) 3785 return PTR_ERR(*pclk); 3786 3787 err = clk_prepare_enable(*pclk); 3788 if (err) { 3789 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3790 return err; 3791 } 3792 3793 return 0; 3794 } 3795 3796 static int at91ether_init(struct platform_device *pdev) 3797 { 3798 struct net_device *dev = platform_get_drvdata(pdev); 3799 struct macb *bp = netdev_priv(dev); 3800 int err; 3801 u32 reg; 3802 3803 bp->queues[0].bp = bp; 3804 3805 dev->netdev_ops = &at91ether_netdev_ops; 3806 dev->ethtool_ops = &macb_ethtool_ops; 3807 3808 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3809 0, dev->name, dev); 3810 if (err) 3811 return err; 3812 3813 macb_writel(bp, NCR, 0); 3814 3815 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3816 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3817 reg |= MACB_BIT(RM9200_RMII); 3818 3819 macb_writel(bp, NCFGR, reg); 3820 3821 return 0; 3822 } 3823 3824 static const struct macb_config at91sam9260_config = { 3825 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3826 .clk_init = macb_clk_init, 3827 .init = macb_init, 3828 }; 3829 3830 static const struct macb_config pc302gem_config = { 3831 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 3832 .dma_burst_length = 16, 3833 .clk_init = macb_clk_init, 3834 .init = macb_init, 3835 }; 3836 3837 static const struct macb_config sama5d2_config = { 3838 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3839 .dma_burst_length = 16, 3840 .clk_init = macb_clk_init, 3841 .init = macb_init, 3842 }; 3843 3844 static const struct macb_config sama5d3_config = { 3845 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 3846 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 3847 .dma_burst_length = 16, 3848 .clk_init = macb_clk_init, 3849 .init = macb_init, 3850 .jumbo_max_len = 10240, 3851 }; 3852 3853 static const struct macb_config sama5d4_config = { 3854 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3855 .dma_burst_length = 4, 3856 .clk_init = macb_clk_init, 3857 .init = macb_init, 3858 }; 3859 3860 static const struct macb_config emac_config = { 3861 .clk_init = at91ether_clk_init, 3862 .init = at91ether_init, 3863 }; 3864 3865 static const struct macb_config np4_config = { 3866 .caps = MACB_CAPS_USRIO_DISABLED, 3867 .clk_init = macb_clk_init, 3868 .init = macb_init, 3869 }; 3870 3871 static const struct macb_config zynqmp_config = { 3872 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3873 MACB_CAPS_JUMBO | 3874 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 3875 .dma_burst_length = 16, 3876 .clk_init = macb_clk_init, 3877 .init = macb_init, 3878 .jumbo_max_len = 10240, 3879 }; 3880 3881 static const struct macb_config zynq_config = { 3882 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF, 3883 .dma_burst_length = 16, 3884 .clk_init = macb_clk_init, 3885 .init = macb_init, 3886 }; 3887 3888 static const struct of_device_id macb_dt_ids[] = { 3889 { .compatible = "cdns,at32ap7000-macb" }, 3890 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 3891 { .compatible = "cdns,macb" }, 3892 { .compatible = "cdns,np4-macb", .data = &np4_config }, 3893 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 3894 { .compatible = "cdns,gem", .data = &pc302gem_config }, 3895 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 3896 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 3897 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 3898 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 3899 { .compatible = "cdns,emac", .data = &emac_config }, 3900 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 3901 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 3902 { /* sentinel */ } 3903 }; 3904 MODULE_DEVICE_TABLE(of, macb_dt_ids); 3905 #endif /* CONFIG_OF */ 3906 3907 static const struct macb_config default_gem_config = { 3908 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3909 MACB_CAPS_JUMBO | 3910 MACB_CAPS_GEM_HAS_PTP, 3911 .dma_burst_length = 16, 3912 .clk_init = macb_clk_init, 3913 .init = macb_init, 3914 .jumbo_max_len = 10240, 3915 }; 3916 3917 static int macb_probe(struct platform_device *pdev) 3918 { 3919 const struct macb_config *macb_config = &default_gem_config; 3920 int (*clk_init)(struct platform_device *, struct clk **, 3921 struct clk **, struct clk **, struct clk **) 3922 = macb_config->clk_init; 3923 int (*init)(struct platform_device *) = macb_config->init; 3924 struct device_node *np = pdev->dev.of_node; 3925 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 3926 unsigned int queue_mask, num_queues; 3927 struct macb_platform_data *pdata; 3928 bool native_io; 3929 struct phy_device *phydev; 3930 struct net_device *dev; 3931 struct resource *regs; 3932 void __iomem *mem; 3933 const char *mac; 3934 struct macb *bp; 3935 int err, val; 3936 3937 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3938 mem = devm_ioremap_resource(&pdev->dev, regs); 3939 if (IS_ERR(mem)) 3940 return PTR_ERR(mem); 3941 3942 if (np) { 3943 const struct of_device_id *match; 3944 3945 match = of_match_node(macb_dt_ids, np); 3946 if (match && match->data) { 3947 macb_config = match->data; 3948 clk_init = macb_config->clk_init; 3949 init = macb_config->init; 3950 } 3951 } 3952 3953 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk); 3954 if (err) 3955 return err; 3956 3957 native_io = hw_is_native_io(mem); 3958 3959 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 3960 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 3961 if (!dev) { 3962 err = -ENOMEM; 3963 goto err_disable_clocks; 3964 } 3965 3966 dev->base_addr = regs->start; 3967 3968 SET_NETDEV_DEV(dev, &pdev->dev); 3969 3970 bp = netdev_priv(dev); 3971 bp->pdev = pdev; 3972 bp->dev = dev; 3973 bp->regs = mem; 3974 bp->native_io = native_io; 3975 if (native_io) { 3976 bp->macb_reg_readl = hw_readl_native; 3977 bp->macb_reg_writel = hw_writel_native; 3978 } else { 3979 bp->macb_reg_readl = hw_readl; 3980 bp->macb_reg_writel = hw_writel; 3981 } 3982 bp->num_queues = num_queues; 3983 bp->queue_mask = queue_mask; 3984 if (macb_config) 3985 bp->dma_burst_length = macb_config->dma_burst_length; 3986 bp->pclk = pclk; 3987 bp->hclk = hclk; 3988 bp->tx_clk = tx_clk; 3989 bp->rx_clk = rx_clk; 3990 if (macb_config) 3991 bp->jumbo_max_len = macb_config->jumbo_max_len; 3992 3993 bp->wol = 0; 3994 if (of_get_property(np, "magic-packet", NULL)) 3995 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 3996 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 3997 3998 spin_lock_init(&bp->lock); 3999 4000 /* setup capabilities */ 4001 macb_configure_caps(bp, macb_config); 4002 4003 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4004 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4005 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4006 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4007 } 4008 #endif 4009 platform_set_drvdata(pdev, dev); 4010 4011 dev->irq = platform_get_irq(pdev, 0); 4012 if (dev->irq < 0) { 4013 err = dev->irq; 4014 goto err_out_free_netdev; 4015 } 4016 4017 /* MTU range: 68 - 1500 or 10240 */ 4018 dev->min_mtu = GEM_MTU_MIN_SIZE; 4019 if (bp->caps & MACB_CAPS_JUMBO) 4020 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4021 else 4022 dev->max_mtu = ETH_DATA_LEN; 4023 4024 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4025 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4026 if (val) 4027 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4028 macb_dma_desc_get_size(bp); 4029 4030 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4031 if (val) 4032 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4033 macb_dma_desc_get_size(bp); 4034 } 4035 4036 mac = of_get_mac_address(np); 4037 if (mac) { 4038 ether_addr_copy(bp->dev->dev_addr, mac); 4039 } else { 4040 err = of_get_nvmem_mac_address(np, bp->dev->dev_addr); 4041 if (err) { 4042 if (err == -EPROBE_DEFER) 4043 goto err_out_free_netdev; 4044 macb_get_hwaddr(bp); 4045 } 4046 } 4047 4048 err = of_get_phy_mode(np); 4049 if (err < 0) { 4050 pdata = dev_get_platdata(&pdev->dev); 4051 if (pdata && pdata->is_rmii) 4052 bp->phy_interface = PHY_INTERFACE_MODE_RMII; 4053 else 4054 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4055 } else { 4056 bp->phy_interface = err; 4057 } 4058 4059 /* IP specific init */ 4060 err = init(pdev); 4061 if (err) 4062 goto err_out_free_netdev; 4063 4064 err = macb_mii_init(bp); 4065 if (err) 4066 goto err_out_free_netdev; 4067 4068 phydev = dev->phydev; 4069 4070 netif_carrier_off(dev); 4071 4072 err = register_netdev(dev); 4073 if (err) { 4074 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4075 goto err_out_unregister_mdio; 4076 } 4077 4078 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4079 (unsigned long)bp); 4080 4081 phy_attached_info(phydev); 4082 4083 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4084 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4085 dev->base_addr, dev->irq, dev->dev_addr); 4086 4087 return 0; 4088 4089 err_out_unregister_mdio: 4090 phy_disconnect(dev->phydev); 4091 mdiobus_unregister(bp->mii_bus); 4092 of_node_put(bp->phy_node); 4093 if (np && of_phy_is_fixed_link(np)) 4094 of_phy_deregister_fixed_link(np); 4095 mdiobus_free(bp->mii_bus); 4096 4097 err_out_free_netdev: 4098 free_netdev(dev); 4099 4100 err_disable_clocks: 4101 clk_disable_unprepare(tx_clk); 4102 clk_disable_unprepare(hclk); 4103 clk_disable_unprepare(pclk); 4104 clk_disable_unprepare(rx_clk); 4105 4106 return err; 4107 } 4108 4109 static int macb_remove(struct platform_device *pdev) 4110 { 4111 struct net_device *dev; 4112 struct macb *bp; 4113 struct device_node *np = pdev->dev.of_node; 4114 4115 dev = platform_get_drvdata(pdev); 4116 4117 if (dev) { 4118 bp = netdev_priv(dev); 4119 if (dev->phydev) 4120 phy_disconnect(dev->phydev); 4121 mdiobus_unregister(bp->mii_bus); 4122 if (np && of_phy_is_fixed_link(np)) 4123 of_phy_deregister_fixed_link(np); 4124 dev->phydev = NULL; 4125 mdiobus_free(bp->mii_bus); 4126 4127 unregister_netdev(dev); 4128 clk_disable_unprepare(bp->tx_clk); 4129 clk_disable_unprepare(bp->hclk); 4130 clk_disable_unprepare(bp->pclk); 4131 clk_disable_unprepare(bp->rx_clk); 4132 of_node_put(bp->phy_node); 4133 free_netdev(dev); 4134 } 4135 4136 return 0; 4137 } 4138 4139 static int __maybe_unused macb_suspend(struct device *dev) 4140 { 4141 struct platform_device *pdev = to_platform_device(dev); 4142 struct net_device *netdev = platform_get_drvdata(pdev); 4143 struct macb *bp = netdev_priv(netdev); 4144 4145 netif_carrier_off(netdev); 4146 netif_device_detach(netdev); 4147 4148 if (bp->wol & MACB_WOL_ENABLED) { 4149 macb_writel(bp, IER, MACB_BIT(WOL)); 4150 macb_writel(bp, WOL, MACB_BIT(MAG)); 4151 enable_irq_wake(bp->queues[0].irq); 4152 } else { 4153 clk_disable_unprepare(bp->tx_clk); 4154 clk_disable_unprepare(bp->hclk); 4155 clk_disable_unprepare(bp->pclk); 4156 clk_disable_unprepare(bp->rx_clk); 4157 } 4158 4159 return 0; 4160 } 4161 4162 static int __maybe_unused macb_resume(struct device *dev) 4163 { 4164 struct platform_device *pdev = to_platform_device(dev); 4165 struct net_device *netdev = platform_get_drvdata(pdev); 4166 struct macb *bp = netdev_priv(netdev); 4167 4168 if (bp->wol & MACB_WOL_ENABLED) { 4169 macb_writel(bp, IDR, MACB_BIT(WOL)); 4170 macb_writel(bp, WOL, 0); 4171 disable_irq_wake(bp->queues[0].irq); 4172 } else { 4173 clk_prepare_enable(bp->pclk); 4174 clk_prepare_enable(bp->hclk); 4175 clk_prepare_enable(bp->tx_clk); 4176 clk_prepare_enable(bp->rx_clk); 4177 } 4178 4179 netif_device_attach(netdev); 4180 4181 return 0; 4182 } 4183 4184 static SIMPLE_DEV_PM_OPS(macb_pm_ops, macb_suspend, macb_resume); 4185 4186 static struct platform_driver macb_driver = { 4187 .probe = macb_probe, 4188 .remove = macb_remove, 4189 .driver = { 4190 .name = "macb", 4191 .of_match_table = of_match_ptr(macb_dt_ids), 4192 .pm = &macb_pm_ops, 4193 }, 4194 }; 4195 4196 module_platform_driver(macb_driver); 4197 4198 MODULE_LICENSE("GPL"); 4199 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4200 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4201 MODULE_ALIAS("platform:macb"); 4202