1 /* 2 * Cadence MACB/GEM Ethernet Controller driver 3 * 4 * Copyright (C) 2004-2006 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 #include <linux/clk.h> 13 #include <linux/module.h> 14 #include <linux/moduleparam.h> 15 #include <linux/kernel.h> 16 #include <linux/types.h> 17 #include <linux/circ_buf.h> 18 #include <linux/slab.h> 19 #include <linux/init.h> 20 #include <linux/io.h> 21 #include <linux/gpio.h> 22 #include <linux/gpio/consumer.h> 23 #include <linux/interrupt.h> 24 #include <linux/netdevice.h> 25 #include <linux/etherdevice.h> 26 #include <linux/dma-mapping.h> 27 #include <linux/platform_data/macb.h> 28 #include <linux/platform_device.h> 29 #include <linux/phy.h> 30 #include <linux/of.h> 31 #include <linux/of_device.h> 32 #include <linux/of_gpio.h> 33 #include <linux/of_mdio.h> 34 #include <linux/of_net.h> 35 #include <linux/ip.h> 36 #include <linux/udp.h> 37 #include <linux/tcp.h> 38 #include "macb.h" 39 40 #define MACB_RX_BUFFER_SIZE 128 41 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 42 43 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 44 #define MIN_RX_RING_SIZE 64 45 #define MAX_RX_RING_SIZE 8192 46 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 47 * (bp)->rx_ring_size) 48 49 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 50 #define MIN_TX_RING_SIZE 64 51 #define MAX_TX_RING_SIZE 4096 52 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 53 * (bp)->tx_ring_size) 54 55 /* level of occupied TX descriptors under which we wake up TX process */ 56 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 57 58 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(RXUBR) \ 59 | MACB_BIT(ISR_ROVR)) 60 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 61 | MACB_BIT(ISR_RLE) \ 62 | MACB_BIT(TXERR)) 63 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP)) 64 65 /* Max length of transmit frame must be a multiple of 8 bytes */ 66 #define MACB_TX_LEN_ALIGN 8 67 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 68 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 69 70 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 71 #define MACB_NETIF_LSO NETIF_F_TSO 72 73 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 74 #define MACB_WOL_ENABLED (0x1 << 1) 75 76 /* Graceful stop timeouts in us. We should allow up to 77 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 78 */ 79 #define MACB_HALT_TIMEOUT 1230 80 81 /* DMA buffer descriptor might be different size 82 * depends on hardware configuration: 83 * 84 * 1. dma address width 32 bits: 85 * word 1: 32 bit address of Data Buffer 86 * word 2: control 87 * 88 * 2. dma address width 64 bits: 89 * word 1: 32 bit address of Data Buffer 90 * word 2: control 91 * word 3: upper 32 bit address of Data Buffer 92 * word 4: unused 93 * 94 * 3. dma address width 32 bits with hardware timestamping: 95 * word 1: 32 bit address of Data Buffer 96 * word 2: control 97 * word 3: timestamp word 1 98 * word 4: timestamp word 2 99 * 100 * 4. dma address width 64 bits with hardware timestamping: 101 * word 1: 32 bit address of Data Buffer 102 * word 2: control 103 * word 3: upper 32 bit address of Data Buffer 104 * word 4: unused 105 * word 5: timestamp word 1 106 * word 6: timestamp word 2 107 */ 108 static unsigned int macb_dma_desc_get_size(struct macb *bp) 109 { 110 #ifdef MACB_EXT_DESC 111 unsigned int desc_size; 112 113 switch (bp->hw_dma_cap) { 114 case HW_DMA_CAP_64B: 115 desc_size = sizeof(struct macb_dma_desc) 116 + sizeof(struct macb_dma_desc_64); 117 break; 118 case HW_DMA_CAP_PTP: 119 desc_size = sizeof(struct macb_dma_desc) 120 + sizeof(struct macb_dma_desc_ptp); 121 break; 122 case HW_DMA_CAP_64B_PTP: 123 desc_size = sizeof(struct macb_dma_desc) 124 + sizeof(struct macb_dma_desc_64) 125 + sizeof(struct macb_dma_desc_ptp); 126 break; 127 default: 128 desc_size = sizeof(struct macb_dma_desc); 129 } 130 return desc_size; 131 #endif 132 return sizeof(struct macb_dma_desc); 133 } 134 135 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 136 { 137 #ifdef MACB_EXT_DESC 138 switch (bp->hw_dma_cap) { 139 case HW_DMA_CAP_64B: 140 case HW_DMA_CAP_PTP: 141 desc_idx <<= 1; 142 break; 143 case HW_DMA_CAP_64B_PTP: 144 desc_idx *= 3; 145 break; 146 default: 147 break; 148 } 149 #endif 150 return desc_idx; 151 } 152 153 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 154 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 155 { 156 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 157 return (struct macb_dma_desc_64 *)((void *)desc + sizeof(struct macb_dma_desc)); 158 return NULL; 159 } 160 #endif 161 162 /* Ring buffer accessors */ 163 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 164 { 165 return index & (bp->tx_ring_size - 1); 166 } 167 168 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 169 unsigned int index) 170 { 171 index = macb_tx_ring_wrap(queue->bp, index); 172 index = macb_adj_dma_desc_idx(queue->bp, index); 173 return &queue->tx_ring[index]; 174 } 175 176 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 177 unsigned int index) 178 { 179 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 180 } 181 182 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 183 { 184 dma_addr_t offset; 185 186 offset = macb_tx_ring_wrap(queue->bp, index) * 187 macb_dma_desc_get_size(queue->bp); 188 189 return queue->tx_ring_dma + offset; 190 } 191 192 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 193 { 194 return index & (bp->rx_ring_size - 1); 195 } 196 197 static struct macb_dma_desc *macb_rx_desc(struct macb *bp, unsigned int index) 198 { 199 index = macb_rx_ring_wrap(bp, index); 200 index = macb_adj_dma_desc_idx(bp, index); 201 return &bp->rx_ring[index]; 202 } 203 204 static void *macb_rx_buffer(struct macb *bp, unsigned int index) 205 { 206 return bp->rx_buffers + bp->rx_buffer_size * 207 macb_rx_ring_wrap(bp, index); 208 } 209 210 /* I/O accessors */ 211 static u32 hw_readl_native(struct macb *bp, int offset) 212 { 213 return __raw_readl(bp->regs + offset); 214 } 215 216 static void hw_writel_native(struct macb *bp, int offset, u32 value) 217 { 218 __raw_writel(value, bp->regs + offset); 219 } 220 221 static u32 hw_readl(struct macb *bp, int offset) 222 { 223 return readl_relaxed(bp->regs + offset); 224 } 225 226 static void hw_writel(struct macb *bp, int offset, u32 value) 227 { 228 writel_relaxed(value, bp->regs + offset); 229 } 230 231 /* Find the CPU endianness by using the loopback bit of NCR register. When the 232 * CPU is in big endian we need to program swapped mode for management 233 * descriptor access. 234 */ 235 static bool hw_is_native_io(void __iomem *addr) 236 { 237 u32 value = MACB_BIT(LLB); 238 239 __raw_writel(value, addr + MACB_NCR); 240 value = __raw_readl(addr + MACB_NCR); 241 242 /* Write 0 back to disable everything */ 243 __raw_writel(0, addr + MACB_NCR); 244 245 return value == MACB_BIT(LLB); 246 } 247 248 static bool hw_is_gem(void __iomem *addr, bool native_io) 249 { 250 u32 id; 251 252 if (native_io) 253 id = __raw_readl(addr + MACB_MID); 254 else 255 id = readl_relaxed(addr + MACB_MID); 256 257 return MACB_BFEXT(IDNUM, id) >= 0x2; 258 } 259 260 static void macb_set_hwaddr(struct macb *bp) 261 { 262 u32 bottom; 263 u16 top; 264 265 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 266 macb_or_gem_writel(bp, SA1B, bottom); 267 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 268 macb_or_gem_writel(bp, SA1T, top); 269 270 /* Clear unused address register sets */ 271 macb_or_gem_writel(bp, SA2B, 0); 272 macb_or_gem_writel(bp, SA2T, 0); 273 macb_or_gem_writel(bp, SA3B, 0); 274 macb_or_gem_writel(bp, SA3T, 0); 275 macb_or_gem_writel(bp, SA4B, 0); 276 macb_or_gem_writel(bp, SA4T, 0); 277 } 278 279 static void macb_get_hwaddr(struct macb *bp) 280 { 281 struct macb_platform_data *pdata; 282 u32 bottom; 283 u16 top; 284 u8 addr[6]; 285 int i; 286 287 pdata = dev_get_platdata(&bp->pdev->dev); 288 289 /* Check all 4 address register for valid address */ 290 for (i = 0; i < 4; i++) { 291 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 292 top = macb_or_gem_readl(bp, SA1T + i * 8); 293 294 if (pdata && pdata->rev_eth_addr) { 295 addr[5] = bottom & 0xff; 296 addr[4] = (bottom >> 8) & 0xff; 297 addr[3] = (bottom >> 16) & 0xff; 298 addr[2] = (bottom >> 24) & 0xff; 299 addr[1] = top & 0xff; 300 addr[0] = (top & 0xff00) >> 8; 301 } else { 302 addr[0] = bottom & 0xff; 303 addr[1] = (bottom >> 8) & 0xff; 304 addr[2] = (bottom >> 16) & 0xff; 305 addr[3] = (bottom >> 24) & 0xff; 306 addr[4] = top & 0xff; 307 addr[5] = (top >> 8) & 0xff; 308 } 309 310 if (is_valid_ether_addr(addr)) { 311 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 312 return; 313 } 314 } 315 316 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 317 eth_hw_addr_random(bp->dev); 318 } 319 320 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 321 { 322 struct macb *bp = bus->priv; 323 int value; 324 325 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 326 | MACB_BF(RW, MACB_MAN_READ) 327 | MACB_BF(PHYA, mii_id) 328 | MACB_BF(REGA, regnum) 329 | MACB_BF(CODE, MACB_MAN_CODE))); 330 331 /* wait for end of transfer */ 332 while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) 333 cpu_relax(); 334 335 value = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 336 337 return value; 338 } 339 340 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 341 u16 value) 342 { 343 struct macb *bp = bus->priv; 344 345 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 346 | MACB_BF(RW, MACB_MAN_WRITE) 347 | MACB_BF(PHYA, mii_id) 348 | MACB_BF(REGA, regnum) 349 | MACB_BF(CODE, MACB_MAN_CODE) 350 | MACB_BF(DATA, value))); 351 352 /* wait for end of transfer */ 353 while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) 354 cpu_relax(); 355 356 return 0; 357 } 358 359 /** 360 * macb_set_tx_clk() - Set a clock to a new frequency 361 * @clk Pointer to the clock to change 362 * @rate New frequency in Hz 363 * @dev Pointer to the struct net_device 364 */ 365 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 366 { 367 long ferr, rate, rate_rounded; 368 369 if (!clk) 370 return; 371 372 switch (speed) { 373 case SPEED_10: 374 rate = 2500000; 375 break; 376 case SPEED_100: 377 rate = 25000000; 378 break; 379 case SPEED_1000: 380 rate = 125000000; 381 break; 382 default: 383 return; 384 } 385 386 rate_rounded = clk_round_rate(clk, rate); 387 if (rate_rounded < 0) 388 return; 389 390 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 391 * is not satisfied. 392 */ 393 ferr = abs(rate_rounded - rate); 394 ferr = DIV_ROUND_UP(ferr, rate / 100000); 395 if (ferr > 5) 396 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 397 rate); 398 399 if (clk_set_rate(clk, rate_rounded)) 400 netdev_err(dev, "adjusting tx_clk failed.\n"); 401 } 402 403 static void macb_handle_link_change(struct net_device *dev) 404 { 405 struct macb *bp = netdev_priv(dev); 406 struct phy_device *phydev = dev->phydev; 407 unsigned long flags; 408 int status_change = 0; 409 410 spin_lock_irqsave(&bp->lock, flags); 411 412 if (phydev->link) { 413 if ((bp->speed != phydev->speed) || 414 (bp->duplex != phydev->duplex)) { 415 u32 reg; 416 417 reg = macb_readl(bp, NCFGR); 418 reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 419 if (macb_is_gem(bp)) 420 reg &= ~GEM_BIT(GBE); 421 422 if (phydev->duplex) 423 reg |= MACB_BIT(FD); 424 if (phydev->speed == SPEED_100) 425 reg |= MACB_BIT(SPD); 426 if (phydev->speed == SPEED_1000 && 427 bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 428 reg |= GEM_BIT(GBE); 429 430 macb_or_gem_writel(bp, NCFGR, reg); 431 432 bp->speed = phydev->speed; 433 bp->duplex = phydev->duplex; 434 status_change = 1; 435 } 436 } 437 438 if (phydev->link != bp->link) { 439 if (!phydev->link) { 440 bp->speed = 0; 441 bp->duplex = -1; 442 } 443 bp->link = phydev->link; 444 445 status_change = 1; 446 } 447 448 spin_unlock_irqrestore(&bp->lock, flags); 449 450 if (status_change) { 451 if (phydev->link) { 452 /* Update the TX clock rate if and only if the link is 453 * up and there has been a link change. 454 */ 455 macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); 456 457 netif_carrier_on(dev); 458 netdev_info(dev, "link up (%d/%s)\n", 459 phydev->speed, 460 phydev->duplex == DUPLEX_FULL ? 461 "Full" : "Half"); 462 } else { 463 netif_carrier_off(dev); 464 netdev_info(dev, "link down\n"); 465 } 466 } 467 } 468 469 /* based on au1000_eth. c*/ 470 static int macb_mii_probe(struct net_device *dev) 471 { 472 struct macb *bp = netdev_priv(dev); 473 struct macb_platform_data *pdata; 474 struct phy_device *phydev; 475 int phy_irq; 476 int ret; 477 478 if (bp->phy_node) { 479 phydev = of_phy_connect(dev, bp->phy_node, 480 &macb_handle_link_change, 0, 481 bp->phy_interface); 482 if (!phydev) 483 return -ENODEV; 484 } else { 485 phydev = phy_find_first(bp->mii_bus); 486 if (!phydev) { 487 netdev_err(dev, "no PHY found\n"); 488 return -ENXIO; 489 } 490 491 pdata = dev_get_platdata(&bp->pdev->dev); 492 if (pdata) { 493 if (gpio_is_valid(pdata->phy_irq_pin)) { 494 ret = devm_gpio_request(&bp->pdev->dev, 495 pdata->phy_irq_pin, "phy int"); 496 if (!ret) { 497 phy_irq = gpio_to_irq(pdata->phy_irq_pin); 498 phydev->irq = (phy_irq < 0) ? PHY_POLL : phy_irq; 499 } 500 } else { 501 phydev->irq = PHY_POLL; 502 } 503 } 504 505 /* attach the mac to the phy */ 506 ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, 507 bp->phy_interface); 508 if (ret) { 509 netdev_err(dev, "Could not attach to PHY\n"); 510 return ret; 511 } 512 } 513 514 /* mask with MAC supported features */ 515 if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 516 phydev->supported &= PHY_GBIT_FEATURES; 517 else 518 phydev->supported &= PHY_BASIC_FEATURES; 519 520 if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) 521 phydev->supported &= ~SUPPORTED_1000baseT_Half; 522 523 phydev->advertising = phydev->supported; 524 525 bp->link = 0; 526 bp->speed = 0; 527 bp->duplex = -1; 528 529 return 0; 530 } 531 532 static int macb_mii_init(struct macb *bp) 533 { 534 struct macb_platform_data *pdata; 535 struct device_node *np; 536 int err = -ENXIO, i; 537 538 /* Enable management port */ 539 macb_writel(bp, NCR, MACB_BIT(MPE)); 540 541 bp->mii_bus = mdiobus_alloc(); 542 if (!bp->mii_bus) { 543 err = -ENOMEM; 544 goto err_out; 545 } 546 547 bp->mii_bus->name = "MACB_mii_bus"; 548 bp->mii_bus->read = &macb_mdio_read; 549 bp->mii_bus->write = &macb_mdio_write; 550 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 551 bp->pdev->name, bp->pdev->id); 552 bp->mii_bus->priv = bp; 553 bp->mii_bus->parent = &bp->pdev->dev; 554 pdata = dev_get_platdata(&bp->pdev->dev); 555 556 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 557 558 np = bp->pdev->dev.of_node; 559 if (np) { 560 if (of_phy_is_fixed_link(np)) { 561 if (of_phy_register_fixed_link(np) < 0) { 562 dev_err(&bp->pdev->dev, 563 "broken fixed-link specification\n"); 564 goto err_out_unregister_bus; 565 } 566 bp->phy_node = of_node_get(np); 567 568 err = mdiobus_register(bp->mii_bus); 569 } else { 570 /* try dt phy registration */ 571 err = of_mdiobus_register(bp->mii_bus, np); 572 573 /* fallback to standard phy registration if no phy were 574 * found during dt phy registration 575 */ 576 if (!err && !phy_find_first(bp->mii_bus)) { 577 for (i = 0; i < PHY_MAX_ADDR; i++) { 578 struct phy_device *phydev; 579 580 phydev = mdiobus_scan(bp->mii_bus, i); 581 if (IS_ERR(phydev) && 582 PTR_ERR(phydev) != -ENODEV) { 583 err = PTR_ERR(phydev); 584 break; 585 } 586 } 587 588 if (err) 589 goto err_out_unregister_bus; 590 } 591 } 592 } else { 593 for (i = 0; i < PHY_MAX_ADDR; i++) 594 bp->mii_bus->irq[i] = PHY_POLL; 595 596 if (pdata) 597 bp->mii_bus->phy_mask = pdata->phy_mask; 598 599 err = mdiobus_register(bp->mii_bus); 600 } 601 602 if (err) 603 goto err_out_free_mdiobus; 604 605 err = macb_mii_probe(bp->dev); 606 if (err) 607 goto err_out_unregister_bus; 608 609 return 0; 610 611 err_out_unregister_bus: 612 mdiobus_unregister(bp->mii_bus); 613 err_out_free_mdiobus: 614 mdiobus_free(bp->mii_bus); 615 err_out: 616 return err; 617 } 618 619 static void macb_update_stats(struct macb *bp) 620 { 621 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 622 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 623 int offset = MACB_PFR; 624 625 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 626 627 for (; p < end; p++, offset += 4) 628 *p += bp->macb_reg_readl(bp, offset); 629 } 630 631 static int macb_halt_tx(struct macb *bp) 632 { 633 unsigned long halt_time, timeout; 634 u32 status; 635 636 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 637 638 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 639 do { 640 halt_time = jiffies; 641 status = macb_readl(bp, TSR); 642 if (!(status & MACB_BIT(TGO))) 643 return 0; 644 645 usleep_range(10, 250); 646 } while (time_before(halt_time, timeout)); 647 648 return -ETIMEDOUT; 649 } 650 651 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 652 { 653 if (tx_skb->mapping) { 654 if (tx_skb->mapped_as_page) 655 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 656 tx_skb->size, DMA_TO_DEVICE); 657 else 658 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 659 tx_skb->size, DMA_TO_DEVICE); 660 tx_skb->mapping = 0; 661 } 662 663 if (tx_skb->skb) { 664 dev_kfree_skb_any(tx_skb->skb); 665 tx_skb->skb = NULL; 666 } 667 } 668 669 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 670 { 671 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 672 struct macb_dma_desc_64 *desc_64; 673 674 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 675 desc_64 = macb_64b_desc(bp, desc); 676 desc_64->addrh = upper_32_bits(addr); 677 } 678 #endif 679 desc->addr = lower_32_bits(addr); 680 } 681 682 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 683 { 684 dma_addr_t addr = 0; 685 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 686 struct macb_dma_desc_64 *desc_64; 687 688 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 689 desc_64 = macb_64b_desc(bp, desc); 690 addr = ((u64)(desc_64->addrh) << 32); 691 } 692 #endif 693 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 694 return addr; 695 } 696 697 static void macb_tx_error_task(struct work_struct *work) 698 { 699 struct macb_queue *queue = container_of(work, struct macb_queue, 700 tx_error_task); 701 struct macb *bp = queue->bp; 702 struct macb_tx_skb *tx_skb; 703 struct macb_dma_desc *desc; 704 struct sk_buff *skb; 705 unsigned int tail; 706 unsigned long flags; 707 708 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 709 (unsigned int)(queue - bp->queues), 710 queue->tx_tail, queue->tx_head); 711 712 /* Prevent the queue IRQ handlers from running: each of them may call 713 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 714 * As explained below, we have to halt the transmission before updating 715 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 716 * network engine about the macb/gem being halted. 717 */ 718 spin_lock_irqsave(&bp->lock, flags); 719 720 /* Make sure nobody is trying to queue up new packets */ 721 netif_tx_stop_all_queues(bp->dev); 722 723 /* Stop transmission now 724 * (in case we have just queued new packets) 725 * macb/gem must be halted to write TBQP register 726 */ 727 if (macb_halt_tx(bp)) 728 /* Just complain for now, reinitializing TX path can be good */ 729 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 730 731 /* Treat frames in TX queue including the ones that caused the error. 732 * Free transmit buffers in upper layer. 733 */ 734 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 735 u32 ctrl; 736 737 desc = macb_tx_desc(queue, tail); 738 ctrl = desc->ctrl; 739 tx_skb = macb_tx_skb(queue, tail); 740 skb = tx_skb->skb; 741 742 if (ctrl & MACB_BIT(TX_USED)) { 743 /* skb is set for the last buffer of the frame */ 744 while (!skb) { 745 macb_tx_unmap(bp, tx_skb); 746 tail++; 747 tx_skb = macb_tx_skb(queue, tail); 748 skb = tx_skb->skb; 749 } 750 751 /* ctrl still refers to the first buffer descriptor 752 * since it's the only one written back by the hardware 753 */ 754 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 755 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 756 macb_tx_ring_wrap(bp, tail), 757 skb->data); 758 bp->dev->stats.tx_packets++; 759 bp->dev->stats.tx_bytes += skb->len; 760 } 761 } else { 762 /* "Buffers exhausted mid-frame" errors may only happen 763 * if the driver is buggy, so complain loudly about 764 * those. Statistics are updated by hardware. 765 */ 766 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 767 netdev_err(bp->dev, 768 "BUG: TX buffers exhausted mid-frame\n"); 769 770 desc->ctrl = ctrl | MACB_BIT(TX_USED); 771 } 772 773 macb_tx_unmap(bp, tx_skb); 774 } 775 776 /* Set end of TX queue */ 777 desc = macb_tx_desc(queue, 0); 778 macb_set_addr(bp, desc, 0); 779 desc->ctrl = MACB_BIT(TX_USED); 780 781 /* Make descriptor updates visible to hardware */ 782 wmb(); 783 784 /* Reinitialize the TX desc queue */ 785 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 786 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 787 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 788 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 789 #endif 790 /* Make TX ring reflect state of hardware */ 791 queue->tx_head = 0; 792 queue->tx_tail = 0; 793 794 /* Housework before enabling TX IRQ */ 795 macb_writel(bp, TSR, macb_readl(bp, TSR)); 796 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 797 798 /* Now we are ready to start transmission again */ 799 netif_tx_start_all_queues(bp->dev); 800 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 801 802 spin_unlock_irqrestore(&bp->lock, flags); 803 } 804 805 static void macb_tx_interrupt(struct macb_queue *queue) 806 { 807 unsigned int tail; 808 unsigned int head; 809 u32 status; 810 struct macb *bp = queue->bp; 811 u16 queue_index = queue - bp->queues; 812 813 status = macb_readl(bp, TSR); 814 macb_writel(bp, TSR, status); 815 816 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 817 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 818 819 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 820 (unsigned long)status); 821 822 head = queue->tx_head; 823 for (tail = queue->tx_tail; tail != head; tail++) { 824 struct macb_tx_skb *tx_skb; 825 struct sk_buff *skb; 826 struct macb_dma_desc *desc; 827 u32 ctrl; 828 829 desc = macb_tx_desc(queue, tail); 830 831 /* Make hw descriptor updates visible to CPU */ 832 rmb(); 833 834 ctrl = desc->ctrl; 835 836 /* TX_USED bit is only set by hardware on the very first buffer 837 * descriptor of the transmitted frame. 838 */ 839 if (!(ctrl & MACB_BIT(TX_USED))) 840 break; 841 842 /* Process all buffers of the current transmitted frame */ 843 for (;; tail++) { 844 tx_skb = macb_tx_skb(queue, tail); 845 skb = tx_skb->skb; 846 847 /* First, update TX stats if needed */ 848 if (skb) { 849 if (gem_ptp_do_txstamp(queue, skb, desc) == 0) { 850 /* skb now belongs to timestamp buffer 851 * and will be removed later 852 */ 853 tx_skb->skb = NULL; 854 } 855 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 856 macb_tx_ring_wrap(bp, tail), 857 skb->data); 858 bp->dev->stats.tx_packets++; 859 bp->dev->stats.tx_bytes += skb->len; 860 } 861 862 /* Now we can safely release resources */ 863 macb_tx_unmap(bp, tx_skb); 864 865 /* skb is set only for the last buffer of the frame. 866 * WARNING: at this point skb has been freed by 867 * macb_tx_unmap(). 868 */ 869 if (skb) 870 break; 871 } 872 } 873 874 queue->tx_tail = tail; 875 if (__netif_subqueue_stopped(bp->dev, queue_index) && 876 CIRC_CNT(queue->tx_head, queue->tx_tail, 877 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 878 netif_wake_subqueue(bp->dev, queue_index); 879 } 880 881 static void gem_rx_refill(struct macb *bp) 882 { 883 unsigned int entry; 884 struct sk_buff *skb; 885 dma_addr_t paddr; 886 struct macb_dma_desc *desc; 887 888 while (CIRC_SPACE(bp->rx_prepared_head, bp->rx_tail, 889 bp->rx_ring_size) > 0) { 890 entry = macb_rx_ring_wrap(bp, bp->rx_prepared_head); 891 892 /* Make hw descriptor updates visible to CPU */ 893 rmb(); 894 895 bp->rx_prepared_head++; 896 desc = macb_rx_desc(bp, entry); 897 898 if (!bp->rx_skbuff[entry]) { 899 /* allocate sk_buff for this free entry in ring */ 900 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 901 if (unlikely(!skb)) { 902 netdev_err(bp->dev, 903 "Unable to allocate sk_buff\n"); 904 break; 905 } 906 907 /* now fill corresponding descriptor entry */ 908 paddr = dma_map_single(&bp->pdev->dev, skb->data, 909 bp->rx_buffer_size, 910 DMA_FROM_DEVICE); 911 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 912 dev_kfree_skb(skb); 913 break; 914 } 915 916 bp->rx_skbuff[entry] = skb; 917 918 if (entry == bp->rx_ring_size - 1) 919 paddr |= MACB_BIT(RX_WRAP); 920 macb_set_addr(bp, desc, paddr); 921 desc->ctrl = 0; 922 923 /* properly align Ethernet header */ 924 skb_reserve(skb, NET_IP_ALIGN); 925 } else { 926 desc->addr &= ~MACB_BIT(RX_USED); 927 desc->ctrl = 0; 928 } 929 } 930 931 /* Make descriptor updates visible to hardware */ 932 wmb(); 933 934 netdev_vdbg(bp->dev, "rx ring: prepared head %d, tail %d\n", 935 bp->rx_prepared_head, bp->rx_tail); 936 } 937 938 /* Mark DMA descriptors from begin up to and not including end as unused */ 939 static void discard_partial_frame(struct macb *bp, unsigned int begin, 940 unsigned int end) 941 { 942 unsigned int frag; 943 944 for (frag = begin; frag != end; frag++) { 945 struct macb_dma_desc *desc = macb_rx_desc(bp, frag); 946 947 desc->addr &= ~MACB_BIT(RX_USED); 948 } 949 950 /* Make descriptor updates visible to hardware */ 951 wmb(); 952 953 /* When this happens, the hardware stats registers for 954 * whatever caused this is updated, so we don't have to record 955 * anything. 956 */ 957 } 958 959 static int gem_rx(struct macb *bp, int budget) 960 { 961 unsigned int len; 962 unsigned int entry; 963 struct sk_buff *skb; 964 struct macb_dma_desc *desc; 965 int count = 0; 966 967 while (count < budget) { 968 u32 ctrl; 969 dma_addr_t addr; 970 bool rxused; 971 972 entry = macb_rx_ring_wrap(bp, bp->rx_tail); 973 desc = macb_rx_desc(bp, entry); 974 975 /* Make hw descriptor updates visible to CPU */ 976 rmb(); 977 978 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 979 addr = macb_get_addr(bp, desc); 980 ctrl = desc->ctrl; 981 982 if (!rxused) 983 break; 984 985 bp->rx_tail++; 986 count++; 987 988 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 989 netdev_err(bp->dev, 990 "not whole frame pointed by descriptor\n"); 991 bp->dev->stats.rx_dropped++; 992 break; 993 } 994 skb = bp->rx_skbuff[entry]; 995 if (unlikely(!skb)) { 996 netdev_err(bp->dev, 997 "inconsistent Rx descriptor chain\n"); 998 bp->dev->stats.rx_dropped++; 999 break; 1000 } 1001 /* now everything is ready for receiving packet */ 1002 bp->rx_skbuff[entry] = NULL; 1003 len = ctrl & bp->rx_frm_len_mask; 1004 1005 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1006 1007 skb_put(skb, len); 1008 dma_unmap_single(&bp->pdev->dev, addr, 1009 bp->rx_buffer_size, DMA_FROM_DEVICE); 1010 1011 skb->protocol = eth_type_trans(skb, bp->dev); 1012 skb_checksum_none_assert(skb); 1013 if (bp->dev->features & NETIF_F_RXCSUM && 1014 !(bp->dev->flags & IFF_PROMISC) && 1015 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1016 skb->ip_summed = CHECKSUM_UNNECESSARY; 1017 1018 bp->dev->stats.rx_packets++; 1019 bp->dev->stats.rx_bytes += skb->len; 1020 1021 gem_ptp_do_rxstamp(bp, skb, desc); 1022 1023 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1024 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1025 skb->len, skb->csum); 1026 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1027 skb_mac_header(skb), 16, true); 1028 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1029 skb->data, 32, true); 1030 #endif 1031 1032 netif_receive_skb(skb); 1033 } 1034 1035 gem_rx_refill(bp); 1036 1037 return count; 1038 } 1039 1040 static int macb_rx_frame(struct macb *bp, unsigned int first_frag, 1041 unsigned int last_frag) 1042 { 1043 unsigned int len; 1044 unsigned int frag; 1045 unsigned int offset; 1046 struct sk_buff *skb; 1047 struct macb_dma_desc *desc; 1048 1049 desc = macb_rx_desc(bp, last_frag); 1050 len = desc->ctrl & bp->rx_frm_len_mask; 1051 1052 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1053 macb_rx_ring_wrap(bp, first_frag), 1054 macb_rx_ring_wrap(bp, last_frag), len); 1055 1056 /* The ethernet header starts NET_IP_ALIGN bytes into the 1057 * first buffer. Since the header is 14 bytes, this makes the 1058 * payload word-aligned. 1059 * 1060 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1061 * the two padding bytes into the skb so that we avoid hitting 1062 * the slowpath in memcpy(), and pull them off afterwards. 1063 */ 1064 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1065 if (!skb) { 1066 bp->dev->stats.rx_dropped++; 1067 for (frag = first_frag; ; frag++) { 1068 desc = macb_rx_desc(bp, frag); 1069 desc->addr &= ~MACB_BIT(RX_USED); 1070 if (frag == last_frag) 1071 break; 1072 } 1073 1074 /* Make descriptor updates visible to hardware */ 1075 wmb(); 1076 1077 return 1; 1078 } 1079 1080 offset = 0; 1081 len += NET_IP_ALIGN; 1082 skb_checksum_none_assert(skb); 1083 skb_put(skb, len); 1084 1085 for (frag = first_frag; ; frag++) { 1086 unsigned int frag_len = bp->rx_buffer_size; 1087 1088 if (offset + frag_len > len) { 1089 if (unlikely(frag != last_frag)) { 1090 dev_kfree_skb_any(skb); 1091 return -1; 1092 } 1093 frag_len = len - offset; 1094 } 1095 skb_copy_to_linear_data_offset(skb, offset, 1096 macb_rx_buffer(bp, frag), 1097 frag_len); 1098 offset += bp->rx_buffer_size; 1099 desc = macb_rx_desc(bp, frag); 1100 desc->addr &= ~MACB_BIT(RX_USED); 1101 1102 if (frag == last_frag) 1103 break; 1104 } 1105 1106 /* Make descriptor updates visible to hardware */ 1107 wmb(); 1108 1109 __skb_pull(skb, NET_IP_ALIGN); 1110 skb->protocol = eth_type_trans(skb, bp->dev); 1111 1112 bp->dev->stats.rx_packets++; 1113 bp->dev->stats.rx_bytes += skb->len; 1114 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1115 skb->len, skb->csum); 1116 netif_receive_skb(skb); 1117 1118 return 0; 1119 } 1120 1121 static inline void macb_init_rx_ring(struct macb *bp) 1122 { 1123 dma_addr_t addr; 1124 struct macb_dma_desc *desc = NULL; 1125 int i; 1126 1127 addr = bp->rx_buffers_dma; 1128 for (i = 0; i < bp->rx_ring_size; i++) { 1129 desc = macb_rx_desc(bp, i); 1130 macb_set_addr(bp, desc, addr); 1131 desc->ctrl = 0; 1132 addr += bp->rx_buffer_size; 1133 } 1134 desc->addr |= MACB_BIT(RX_WRAP); 1135 bp->rx_tail = 0; 1136 } 1137 1138 static int macb_rx(struct macb *bp, int budget) 1139 { 1140 bool reset_rx_queue = false; 1141 int received = 0; 1142 unsigned int tail; 1143 int first_frag = -1; 1144 1145 for (tail = bp->rx_tail; budget > 0; tail++) { 1146 struct macb_dma_desc *desc = macb_rx_desc(bp, tail); 1147 u32 ctrl; 1148 1149 /* Make hw descriptor updates visible to CPU */ 1150 rmb(); 1151 1152 ctrl = desc->ctrl; 1153 1154 if (!(desc->addr & MACB_BIT(RX_USED))) 1155 break; 1156 1157 if (ctrl & MACB_BIT(RX_SOF)) { 1158 if (first_frag != -1) 1159 discard_partial_frame(bp, first_frag, tail); 1160 first_frag = tail; 1161 } 1162 1163 if (ctrl & MACB_BIT(RX_EOF)) { 1164 int dropped; 1165 1166 if (unlikely(first_frag == -1)) { 1167 reset_rx_queue = true; 1168 continue; 1169 } 1170 1171 dropped = macb_rx_frame(bp, first_frag, tail); 1172 first_frag = -1; 1173 if (unlikely(dropped < 0)) { 1174 reset_rx_queue = true; 1175 continue; 1176 } 1177 if (!dropped) { 1178 received++; 1179 budget--; 1180 } 1181 } 1182 } 1183 1184 if (unlikely(reset_rx_queue)) { 1185 unsigned long flags; 1186 u32 ctrl; 1187 1188 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1189 1190 spin_lock_irqsave(&bp->lock, flags); 1191 1192 ctrl = macb_readl(bp, NCR); 1193 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1194 1195 macb_init_rx_ring(bp); 1196 macb_writel(bp, RBQP, bp->rx_ring_dma); 1197 1198 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1199 1200 spin_unlock_irqrestore(&bp->lock, flags); 1201 return received; 1202 } 1203 1204 if (first_frag != -1) 1205 bp->rx_tail = first_frag; 1206 else 1207 bp->rx_tail = tail; 1208 1209 return received; 1210 } 1211 1212 static int macb_poll(struct napi_struct *napi, int budget) 1213 { 1214 struct macb *bp = container_of(napi, struct macb, napi); 1215 int work_done; 1216 u32 status; 1217 1218 status = macb_readl(bp, RSR); 1219 macb_writel(bp, RSR, status); 1220 1221 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1222 (unsigned long)status, budget); 1223 1224 work_done = bp->macbgem_ops.mog_rx(bp, budget); 1225 if (work_done < budget) { 1226 napi_complete_done(napi, work_done); 1227 1228 /* Packets received while interrupts were disabled */ 1229 status = macb_readl(bp, RSR); 1230 if (status) { 1231 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1232 macb_writel(bp, ISR, MACB_BIT(RCOMP)); 1233 napi_reschedule(napi); 1234 } else { 1235 macb_writel(bp, IER, MACB_RX_INT_FLAGS); 1236 } 1237 } 1238 1239 /* TODO: Handle errors */ 1240 1241 return work_done; 1242 } 1243 1244 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1245 { 1246 struct macb_queue *queue = dev_id; 1247 struct macb *bp = queue->bp; 1248 struct net_device *dev = bp->dev; 1249 u32 status, ctrl; 1250 1251 status = queue_readl(queue, ISR); 1252 1253 if (unlikely(!status)) 1254 return IRQ_NONE; 1255 1256 spin_lock(&bp->lock); 1257 1258 while (status) { 1259 /* close possible race with dev_close */ 1260 if (unlikely(!netif_running(dev))) { 1261 queue_writel(queue, IDR, -1); 1262 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1263 queue_writel(queue, ISR, -1); 1264 break; 1265 } 1266 1267 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1268 (unsigned int)(queue - bp->queues), 1269 (unsigned long)status); 1270 1271 if (status & MACB_RX_INT_FLAGS) { 1272 /* There's no point taking any more interrupts 1273 * until we have processed the buffers. The 1274 * scheduling call may fail if the poll routine 1275 * is already scheduled, so disable interrupts 1276 * now. 1277 */ 1278 queue_writel(queue, IDR, MACB_RX_INT_FLAGS); 1279 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1280 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1281 1282 if (napi_schedule_prep(&bp->napi)) { 1283 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1284 __napi_schedule(&bp->napi); 1285 } 1286 } 1287 1288 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1289 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1290 schedule_work(&queue->tx_error_task); 1291 1292 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1293 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1294 1295 break; 1296 } 1297 1298 if (status & MACB_BIT(TCOMP)) 1299 macb_tx_interrupt(queue); 1300 1301 /* Link change detection isn't possible with RMII, so we'll 1302 * add that if/when we get our hands on a full-blown MII PHY. 1303 */ 1304 1305 /* There is a hardware issue under heavy load where DMA can 1306 * stop, this causes endless "used buffer descriptor read" 1307 * interrupts but it can be cleared by re-enabling RX. See 1308 * the at91 manual, section 41.3.1 or the Zynq manual 1309 * section 16.7.4 for details. 1310 */ 1311 if (status & MACB_BIT(RXUBR)) { 1312 ctrl = macb_readl(bp, NCR); 1313 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1314 wmb(); 1315 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1316 1317 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1318 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1319 } 1320 1321 if (status & MACB_BIT(ISR_ROVR)) { 1322 /* We missed at least one packet */ 1323 if (macb_is_gem(bp)) 1324 bp->hw_stats.gem.rx_overruns++; 1325 else 1326 bp->hw_stats.macb.rx_overruns++; 1327 1328 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1329 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1330 } 1331 1332 if (status & MACB_BIT(HRESP)) { 1333 /* TODO: Reset the hardware, and maybe move the 1334 * netdev_err to a lower-priority context as well 1335 * (work queue?) 1336 */ 1337 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1338 1339 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1340 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1341 } 1342 status = queue_readl(queue, ISR); 1343 } 1344 1345 spin_unlock(&bp->lock); 1346 1347 return IRQ_HANDLED; 1348 } 1349 1350 #ifdef CONFIG_NET_POLL_CONTROLLER 1351 /* Polling receive - used by netconsole and other diagnostic tools 1352 * to allow network i/o with interrupts disabled. 1353 */ 1354 static void macb_poll_controller(struct net_device *dev) 1355 { 1356 struct macb *bp = netdev_priv(dev); 1357 struct macb_queue *queue; 1358 unsigned long flags; 1359 unsigned int q; 1360 1361 local_irq_save(flags); 1362 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1363 macb_interrupt(dev->irq, queue); 1364 local_irq_restore(flags); 1365 } 1366 #endif 1367 1368 static unsigned int macb_tx_map(struct macb *bp, 1369 struct macb_queue *queue, 1370 struct sk_buff *skb, 1371 unsigned int hdrlen) 1372 { 1373 dma_addr_t mapping; 1374 unsigned int len, entry, i, tx_head = queue->tx_head; 1375 struct macb_tx_skb *tx_skb = NULL; 1376 struct macb_dma_desc *desc; 1377 unsigned int offset, size, count = 0; 1378 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1379 unsigned int eof = 1, mss_mfs = 0; 1380 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1381 1382 /* LSO */ 1383 if (skb_shinfo(skb)->gso_size != 0) { 1384 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1385 /* UDP - UFO */ 1386 lso_ctrl = MACB_LSO_UFO_ENABLE; 1387 else 1388 /* TCP - TSO */ 1389 lso_ctrl = MACB_LSO_TSO_ENABLE; 1390 } 1391 1392 /* First, map non-paged data */ 1393 len = skb_headlen(skb); 1394 1395 /* first buffer length */ 1396 size = hdrlen; 1397 1398 offset = 0; 1399 while (len) { 1400 entry = macb_tx_ring_wrap(bp, tx_head); 1401 tx_skb = &queue->tx_skb[entry]; 1402 1403 mapping = dma_map_single(&bp->pdev->dev, 1404 skb->data + offset, 1405 size, DMA_TO_DEVICE); 1406 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1407 goto dma_error; 1408 1409 /* Save info to properly release resources */ 1410 tx_skb->skb = NULL; 1411 tx_skb->mapping = mapping; 1412 tx_skb->size = size; 1413 tx_skb->mapped_as_page = false; 1414 1415 len -= size; 1416 offset += size; 1417 count++; 1418 tx_head++; 1419 1420 size = min(len, bp->max_tx_length); 1421 } 1422 1423 /* Then, map paged data from fragments */ 1424 for (f = 0; f < nr_frags; f++) { 1425 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1426 1427 len = skb_frag_size(frag); 1428 offset = 0; 1429 while (len) { 1430 size = min(len, bp->max_tx_length); 1431 entry = macb_tx_ring_wrap(bp, tx_head); 1432 tx_skb = &queue->tx_skb[entry]; 1433 1434 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1435 offset, size, DMA_TO_DEVICE); 1436 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1437 goto dma_error; 1438 1439 /* Save info to properly release resources */ 1440 tx_skb->skb = NULL; 1441 tx_skb->mapping = mapping; 1442 tx_skb->size = size; 1443 tx_skb->mapped_as_page = true; 1444 1445 len -= size; 1446 offset += size; 1447 count++; 1448 tx_head++; 1449 } 1450 } 1451 1452 /* Should never happen */ 1453 if (unlikely(!tx_skb)) { 1454 netdev_err(bp->dev, "BUG! empty skb!\n"); 1455 return 0; 1456 } 1457 1458 /* This is the last buffer of the frame: save socket buffer */ 1459 tx_skb->skb = skb; 1460 1461 /* Update TX ring: update buffer descriptors in reverse order 1462 * to avoid race condition 1463 */ 1464 1465 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1466 * to set the end of TX queue 1467 */ 1468 i = tx_head; 1469 entry = macb_tx_ring_wrap(bp, i); 1470 ctrl = MACB_BIT(TX_USED); 1471 desc = macb_tx_desc(queue, entry); 1472 desc->ctrl = ctrl; 1473 1474 if (lso_ctrl) { 1475 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1476 /* include header and FCS in value given to h/w */ 1477 mss_mfs = skb_shinfo(skb)->gso_size + 1478 skb_transport_offset(skb) + 1479 ETH_FCS_LEN; 1480 else /* TSO */ { 1481 mss_mfs = skb_shinfo(skb)->gso_size; 1482 /* TCP Sequence Number Source Select 1483 * can be set only for TSO 1484 */ 1485 seq_ctrl = 0; 1486 } 1487 } 1488 1489 do { 1490 i--; 1491 entry = macb_tx_ring_wrap(bp, i); 1492 tx_skb = &queue->tx_skb[entry]; 1493 desc = macb_tx_desc(queue, entry); 1494 1495 ctrl = (u32)tx_skb->size; 1496 if (eof) { 1497 ctrl |= MACB_BIT(TX_LAST); 1498 eof = 0; 1499 } 1500 if (unlikely(entry == (bp->tx_ring_size - 1))) 1501 ctrl |= MACB_BIT(TX_WRAP); 1502 1503 /* First descriptor is header descriptor */ 1504 if (i == queue->tx_head) { 1505 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1506 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1507 } else 1508 /* Only set MSS/MFS on payload descriptors 1509 * (second or later descriptor) 1510 */ 1511 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1512 1513 /* Set TX buffer descriptor */ 1514 macb_set_addr(bp, desc, tx_skb->mapping); 1515 /* desc->addr must be visible to hardware before clearing 1516 * 'TX_USED' bit in desc->ctrl. 1517 */ 1518 wmb(); 1519 desc->ctrl = ctrl; 1520 } while (i != queue->tx_head); 1521 1522 queue->tx_head = tx_head; 1523 1524 return count; 1525 1526 dma_error: 1527 netdev_err(bp->dev, "TX DMA map failed\n"); 1528 1529 for (i = queue->tx_head; i != tx_head; i++) { 1530 tx_skb = macb_tx_skb(queue, i); 1531 1532 macb_tx_unmap(bp, tx_skb); 1533 } 1534 1535 return 0; 1536 } 1537 1538 static netdev_features_t macb_features_check(struct sk_buff *skb, 1539 struct net_device *dev, 1540 netdev_features_t features) 1541 { 1542 unsigned int nr_frags, f; 1543 unsigned int hdrlen; 1544 1545 /* Validate LSO compatibility */ 1546 1547 /* there is only one buffer */ 1548 if (!skb_is_nonlinear(skb)) 1549 return features; 1550 1551 /* length of header */ 1552 hdrlen = skb_transport_offset(skb); 1553 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1554 hdrlen += tcp_hdrlen(skb); 1555 1556 /* For LSO: 1557 * When software supplies two or more payload buffers all payload buffers 1558 * apart from the last must be a multiple of 8 bytes in size. 1559 */ 1560 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1561 return features & ~MACB_NETIF_LSO; 1562 1563 nr_frags = skb_shinfo(skb)->nr_frags; 1564 /* No need to check last fragment */ 1565 nr_frags--; 1566 for (f = 0; f < nr_frags; f++) { 1567 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1568 1569 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1570 return features & ~MACB_NETIF_LSO; 1571 } 1572 return features; 1573 } 1574 1575 static inline int macb_clear_csum(struct sk_buff *skb) 1576 { 1577 /* no change for packets without checksum offloading */ 1578 if (skb->ip_summed != CHECKSUM_PARTIAL) 1579 return 0; 1580 1581 /* make sure we can modify the header */ 1582 if (unlikely(skb_cow_head(skb, 0))) 1583 return -1; 1584 1585 /* initialize checksum field 1586 * This is required - at least for Zynq, which otherwise calculates 1587 * wrong UDP header checksums for UDP packets with UDP data len <=2 1588 */ 1589 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1590 return 0; 1591 } 1592 1593 static int macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1594 { 1595 u16 queue_index = skb_get_queue_mapping(skb); 1596 struct macb *bp = netdev_priv(dev); 1597 struct macb_queue *queue = &bp->queues[queue_index]; 1598 unsigned long flags; 1599 unsigned int desc_cnt, nr_frags, frag_size, f; 1600 unsigned int hdrlen; 1601 bool is_lso, is_udp = 0; 1602 1603 is_lso = (skb_shinfo(skb)->gso_size != 0); 1604 1605 if (is_lso) { 1606 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1607 1608 /* length of headers */ 1609 if (is_udp) 1610 /* only queue eth + ip headers separately for UDP */ 1611 hdrlen = skb_transport_offset(skb); 1612 else 1613 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1614 if (skb_headlen(skb) < hdrlen) { 1615 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1616 /* if this is required, would need to copy to single buffer */ 1617 return NETDEV_TX_BUSY; 1618 } 1619 } else 1620 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1621 1622 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1623 netdev_vdbg(bp->dev, 1624 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1625 queue_index, skb->len, skb->head, skb->data, 1626 skb_tail_pointer(skb), skb_end_pointer(skb)); 1627 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1628 skb->data, 16, true); 1629 #endif 1630 1631 /* Count how many TX buffer descriptors are needed to send this 1632 * socket buffer: skb fragments of jumbo frames may need to be 1633 * split into many buffer descriptors. 1634 */ 1635 if (is_lso && (skb_headlen(skb) > hdrlen)) 1636 /* extra header descriptor if also payload in first buffer */ 1637 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1638 else 1639 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1640 nr_frags = skb_shinfo(skb)->nr_frags; 1641 for (f = 0; f < nr_frags; f++) { 1642 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1643 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1644 } 1645 1646 spin_lock_irqsave(&bp->lock, flags); 1647 1648 /* This is a hard error, log it. */ 1649 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1650 bp->tx_ring_size) < desc_cnt) { 1651 netif_stop_subqueue(dev, queue_index); 1652 spin_unlock_irqrestore(&bp->lock, flags); 1653 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1654 queue->tx_head, queue->tx_tail); 1655 return NETDEV_TX_BUSY; 1656 } 1657 1658 if (macb_clear_csum(skb)) { 1659 dev_kfree_skb_any(skb); 1660 goto unlock; 1661 } 1662 1663 /* Map socket buffer for DMA transfer */ 1664 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1665 dev_kfree_skb_any(skb); 1666 goto unlock; 1667 } 1668 1669 /* Make newly initialized descriptor visible to hardware */ 1670 wmb(); 1671 skb_tx_timestamp(skb); 1672 1673 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1674 1675 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1676 netif_stop_subqueue(dev, queue_index); 1677 1678 unlock: 1679 spin_unlock_irqrestore(&bp->lock, flags); 1680 1681 return NETDEV_TX_OK; 1682 } 1683 1684 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1685 { 1686 if (!macb_is_gem(bp)) { 1687 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1688 } else { 1689 bp->rx_buffer_size = size; 1690 1691 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1692 netdev_dbg(bp->dev, 1693 "RX buffer must be multiple of %d bytes, expanding\n", 1694 RX_BUFFER_MULTIPLE); 1695 bp->rx_buffer_size = 1696 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1697 } 1698 } 1699 1700 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1701 bp->dev->mtu, bp->rx_buffer_size); 1702 } 1703 1704 static void gem_free_rx_buffers(struct macb *bp) 1705 { 1706 struct sk_buff *skb; 1707 struct macb_dma_desc *desc; 1708 dma_addr_t addr; 1709 int i; 1710 1711 if (!bp->rx_skbuff) 1712 return; 1713 1714 for (i = 0; i < bp->rx_ring_size; i++) { 1715 skb = bp->rx_skbuff[i]; 1716 1717 if (!skb) 1718 continue; 1719 1720 desc = macb_rx_desc(bp, i); 1721 addr = macb_get_addr(bp, desc); 1722 1723 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1724 DMA_FROM_DEVICE); 1725 dev_kfree_skb_any(skb); 1726 skb = NULL; 1727 } 1728 1729 kfree(bp->rx_skbuff); 1730 bp->rx_skbuff = NULL; 1731 } 1732 1733 static void macb_free_rx_buffers(struct macb *bp) 1734 { 1735 if (bp->rx_buffers) { 1736 dma_free_coherent(&bp->pdev->dev, 1737 bp->rx_ring_size * bp->rx_buffer_size, 1738 bp->rx_buffers, bp->rx_buffers_dma); 1739 bp->rx_buffers = NULL; 1740 } 1741 } 1742 1743 static void macb_free_consistent(struct macb *bp) 1744 { 1745 struct macb_queue *queue; 1746 unsigned int q; 1747 1748 bp->macbgem_ops.mog_free_rx_buffers(bp); 1749 if (bp->rx_ring) { 1750 dma_free_coherent(&bp->pdev->dev, RX_RING_BYTES(bp), 1751 bp->rx_ring, bp->rx_ring_dma); 1752 bp->rx_ring = NULL; 1753 } 1754 1755 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1756 kfree(queue->tx_skb); 1757 queue->tx_skb = NULL; 1758 if (queue->tx_ring) { 1759 dma_free_coherent(&bp->pdev->dev, TX_RING_BYTES(bp), 1760 queue->tx_ring, queue->tx_ring_dma); 1761 queue->tx_ring = NULL; 1762 } 1763 } 1764 } 1765 1766 static int gem_alloc_rx_buffers(struct macb *bp) 1767 { 1768 int size; 1769 1770 size = bp->rx_ring_size * sizeof(struct sk_buff *); 1771 bp->rx_skbuff = kzalloc(size, GFP_KERNEL); 1772 if (!bp->rx_skbuff) 1773 return -ENOMEM; 1774 else 1775 netdev_dbg(bp->dev, 1776 "Allocated %d RX struct sk_buff entries at %p\n", 1777 bp->rx_ring_size, bp->rx_skbuff); 1778 return 0; 1779 } 1780 1781 static int macb_alloc_rx_buffers(struct macb *bp) 1782 { 1783 int size; 1784 1785 size = bp->rx_ring_size * bp->rx_buffer_size; 1786 bp->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 1787 &bp->rx_buffers_dma, GFP_KERNEL); 1788 if (!bp->rx_buffers) 1789 return -ENOMEM; 1790 1791 netdev_dbg(bp->dev, 1792 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 1793 size, (unsigned long)bp->rx_buffers_dma, bp->rx_buffers); 1794 return 0; 1795 } 1796 1797 static int macb_alloc_consistent(struct macb *bp) 1798 { 1799 struct macb_queue *queue; 1800 unsigned int q; 1801 int size; 1802 1803 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1804 size = TX_RING_BYTES(bp); 1805 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1806 &queue->tx_ring_dma, 1807 GFP_KERNEL); 1808 if (!queue->tx_ring) 1809 goto out_err; 1810 netdev_dbg(bp->dev, 1811 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 1812 q, size, (unsigned long)queue->tx_ring_dma, 1813 queue->tx_ring); 1814 1815 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 1816 queue->tx_skb = kmalloc(size, GFP_KERNEL); 1817 if (!queue->tx_skb) 1818 goto out_err; 1819 } 1820 1821 size = RX_RING_BYTES(bp); 1822 bp->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1823 &bp->rx_ring_dma, GFP_KERNEL); 1824 if (!bp->rx_ring) 1825 goto out_err; 1826 netdev_dbg(bp->dev, 1827 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 1828 size, (unsigned long)bp->rx_ring_dma, bp->rx_ring); 1829 1830 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 1831 goto out_err; 1832 1833 return 0; 1834 1835 out_err: 1836 macb_free_consistent(bp); 1837 return -ENOMEM; 1838 } 1839 1840 static void gem_init_rings(struct macb *bp) 1841 { 1842 struct macb_queue *queue; 1843 struct macb_dma_desc *desc = NULL; 1844 unsigned int q; 1845 int i; 1846 1847 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1848 for (i = 0; i < bp->tx_ring_size; i++) { 1849 desc = macb_tx_desc(queue, i); 1850 macb_set_addr(bp, desc, 0); 1851 desc->ctrl = MACB_BIT(TX_USED); 1852 } 1853 desc->ctrl |= MACB_BIT(TX_WRAP); 1854 queue->tx_head = 0; 1855 queue->tx_tail = 0; 1856 } 1857 1858 bp->rx_tail = 0; 1859 bp->rx_prepared_head = 0; 1860 1861 gem_rx_refill(bp); 1862 } 1863 1864 static void macb_init_rings(struct macb *bp) 1865 { 1866 int i; 1867 struct macb_dma_desc *desc = NULL; 1868 1869 macb_init_rx_ring(bp); 1870 1871 for (i = 0; i < bp->tx_ring_size; i++) { 1872 desc = macb_tx_desc(&bp->queues[0], i); 1873 macb_set_addr(bp, desc, 0); 1874 desc->ctrl = MACB_BIT(TX_USED); 1875 } 1876 bp->queues[0].tx_head = 0; 1877 bp->queues[0].tx_tail = 0; 1878 desc->ctrl |= MACB_BIT(TX_WRAP); 1879 } 1880 1881 static void macb_reset_hw(struct macb *bp) 1882 { 1883 struct macb_queue *queue; 1884 unsigned int q; 1885 1886 /* Disable RX and TX (XXX: Should we halt the transmission 1887 * more gracefully?) 1888 */ 1889 macb_writel(bp, NCR, 0); 1890 1891 /* Clear the stats registers (XXX: Update stats first?) */ 1892 macb_writel(bp, NCR, MACB_BIT(CLRSTAT)); 1893 1894 /* Clear all status flags */ 1895 macb_writel(bp, TSR, -1); 1896 macb_writel(bp, RSR, -1); 1897 1898 /* Disable all interrupts */ 1899 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1900 queue_writel(queue, IDR, -1); 1901 queue_readl(queue, ISR); 1902 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1903 queue_writel(queue, ISR, -1); 1904 } 1905 } 1906 1907 static u32 gem_mdc_clk_div(struct macb *bp) 1908 { 1909 u32 config; 1910 unsigned long pclk_hz = clk_get_rate(bp->pclk); 1911 1912 if (pclk_hz <= 20000000) 1913 config = GEM_BF(CLK, GEM_CLK_DIV8); 1914 else if (pclk_hz <= 40000000) 1915 config = GEM_BF(CLK, GEM_CLK_DIV16); 1916 else if (pclk_hz <= 80000000) 1917 config = GEM_BF(CLK, GEM_CLK_DIV32); 1918 else if (pclk_hz <= 120000000) 1919 config = GEM_BF(CLK, GEM_CLK_DIV48); 1920 else if (pclk_hz <= 160000000) 1921 config = GEM_BF(CLK, GEM_CLK_DIV64); 1922 else 1923 config = GEM_BF(CLK, GEM_CLK_DIV96); 1924 1925 return config; 1926 } 1927 1928 static u32 macb_mdc_clk_div(struct macb *bp) 1929 { 1930 u32 config; 1931 unsigned long pclk_hz; 1932 1933 if (macb_is_gem(bp)) 1934 return gem_mdc_clk_div(bp); 1935 1936 pclk_hz = clk_get_rate(bp->pclk); 1937 if (pclk_hz <= 20000000) 1938 config = MACB_BF(CLK, MACB_CLK_DIV8); 1939 else if (pclk_hz <= 40000000) 1940 config = MACB_BF(CLK, MACB_CLK_DIV16); 1941 else if (pclk_hz <= 80000000) 1942 config = MACB_BF(CLK, MACB_CLK_DIV32); 1943 else 1944 config = MACB_BF(CLK, MACB_CLK_DIV64); 1945 1946 return config; 1947 } 1948 1949 /* Get the DMA bus width field of the network configuration register that we 1950 * should program. We find the width from decoding the design configuration 1951 * register to find the maximum supported data bus width. 1952 */ 1953 static u32 macb_dbw(struct macb *bp) 1954 { 1955 if (!macb_is_gem(bp)) 1956 return 0; 1957 1958 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 1959 case 4: 1960 return GEM_BF(DBW, GEM_DBW128); 1961 case 2: 1962 return GEM_BF(DBW, GEM_DBW64); 1963 case 1: 1964 default: 1965 return GEM_BF(DBW, GEM_DBW32); 1966 } 1967 } 1968 1969 /* Configure the receive DMA engine 1970 * - use the correct receive buffer size 1971 * - set best burst length for DMA operations 1972 * (if not supported by FIFO, it will fallback to default) 1973 * - set both rx/tx packet buffers to full memory size 1974 * These are configurable parameters for GEM. 1975 */ 1976 static void macb_configure_dma(struct macb *bp) 1977 { 1978 u32 dmacfg; 1979 1980 if (macb_is_gem(bp)) { 1981 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 1982 dmacfg |= GEM_BF(RXBS, bp->rx_buffer_size / RX_BUFFER_MULTIPLE); 1983 if (bp->dma_burst_length) 1984 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 1985 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 1986 dmacfg &= ~GEM_BIT(ENDIA_PKT); 1987 1988 if (bp->native_io) 1989 dmacfg &= ~GEM_BIT(ENDIA_DESC); 1990 else 1991 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 1992 1993 if (bp->dev->features & NETIF_F_HW_CSUM) 1994 dmacfg |= GEM_BIT(TXCOEN); 1995 else 1996 dmacfg &= ~GEM_BIT(TXCOEN); 1997 1998 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1999 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2000 dmacfg |= GEM_BIT(ADDR64); 2001 #endif 2002 #ifdef CONFIG_MACB_USE_HWSTAMP 2003 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2004 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2005 #endif 2006 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2007 dmacfg); 2008 gem_writel(bp, DMACFG, dmacfg); 2009 } 2010 } 2011 2012 static void macb_init_hw(struct macb *bp) 2013 { 2014 struct macb_queue *queue; 2015 unsigned int q; 2016 2017 u32 config; 2018 2019 macb_reset_hw(bp); 2020 macb_set_hwaddr(bp); 2021 2022 config = macb_mdc_clk_div(bp); 2023 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2024 config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2025 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2026 config |= MACB_BIT(PAE); /* PAuse Enable */ 2027 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2028 if (bp->caps & MACB_CAPS_JUMBO) 2029 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2030 else 2031 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2032 if (bp->dev->flags & IFF_PROMISC) 2033 config |= MACB_BIT(CAF); /* Copy All Frames */ 2034 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2035 config |= GEM_BIT(RXCOEN); 2036 if (!(bp->dev->flags & IFF_BROADCAST)) 2037 config |= MACB_BIT(NBC); /* No BroadCast */ 2038 config |= macb_dbw(bp); 2039 macb_writel(bp, NCFGR, config); 2040 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2041 gem_writel(bp, JML, bp->jumbo_max_len); 2042 bp->speed = SPEED_10; 2043 bp->duplex = DUPLEX_HALF; 2044 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2045 if (bp->caps & MACB_CAPS_JUMBO) 2046 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2047 2048 macb_configure_dma(bp); 2049 2050 /* Initialize TX and RX buffers */ 2051 macb_writel(bp, RBQP, lower_32_bits(bp->rx_ring_dma)); 2052 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2053 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2054 macb_writel(bp, RBQPH, upper_32_bits(bp->rx_ring_dma)); 2055 #endif 2056 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2057 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 2058 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2059 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2060 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 2061 #endif 2062 2063 /* Enable interrupts */ 2064 queue_writel(queue, IER, 2065 MACB_RX_INT_FLAGS | 2066 MACB_TX_INT_FLAGS | 2067 MACB_BIT(HRESP)); 2068 } 2069 2070 /* Enable TX and RX */ 2071 macb_writel(bp, NCR, MACB_BIT(RE) | MACB_BIT(TE) | MACB_BIT(MPE)); 2072 } 2073 2074 /* The hash address register is 64 bits long and takes up two 2075 * locations in the memory map. The least significant bits are stored 2076 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2077 * 2078 * The unicast hash enable and the multicast hash enable bits in the 2079 * network configuration register enable the reception of hash matched 2080 * frames. The destination address is reduced to a 6 bit index into 2081 * the 64 bit hash register using the following hash function. The 2082 * hash function is an exclusive or of every sixth bit of the 2083 * destination address. 2084 * 2085 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2086 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2087 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2088 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2089 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2090 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2091 * 2092 * da[0] represents the least significant bit of the first byte 2093 * received, that is, the multicast/unicast indicator, and da[47] 2094 * represents the most significant bit of the last byte received. If 2095 * the hash index, hi[n], points to a bit that is set in the hash 2096 * register then the frame will be matched according to whether the 2097 * frame is multicast or unicast. A multicast match will be signalled 2098 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2099 * index points to a bit set in the hash register. A unicast match 2100 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2101 * and the hash index points to a bit set in the hash register. To 2102 * receive all multicast frames, the hash register should be set with 2103 * all ones and the multicast hash enable bit should be set in the 2104 * network configuration register. 2105 */ 2106 2107 static inline int hash_bit_value(int bitnr, __u8 *addr) 2108 { 2109 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2110 return 1; 2111 return 0; 2112 } 2113 2114 /* Return the hash index value for the specified address. */ 2115 static int hash_get_index(__u8 *addr) 2116 { 2117 int i, j, bitval; 2118 int hash_index = 0; 2119 2120 for (j = 0; j < 6; j++) { 2121 for (i = 0, bitval = 0; i < 8; i++) 2122 bitval ^= hash_bit_value(i * 6 + j, addr); 2123 2124 hash_index |= (bitval << j); 2125 } 2126 2127 return hash_index; 2128 } 2129 2130 /* Add multicast addresses to the internal multicast-hash table. */ 2131 static void macb_sethashtable(struct net_device *dev) 2132 { 2133 struct netdev_hw_addr *ha; 2134 unsigned long mc_filter[2]; 2135 unsigned int bitnr; 2136 struct macb *bp = netdev_priv(dev); 2137 2138 mc_filter[0] = 0; 2139 mc_filter[1] = 0; 2140 2141 netdev_for_each_mc_addr(ha, dev) { 2142 bitnr = hash_get_index(ha->addr); 2143 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2144 } 2145 2146 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2147 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2148 } 2149 2150 /* Enable/Disable promiscuous and multicast modes. */ 2151 static void macb_set_rx_mode(struct net_device *dev) 2152 { 2153 unsigned long cfg; 2154 struct macb *bp = netdev_priv(dev); 2155 2156 cfg = macb_readl(bp, NCFGR); 2157 2158 if (dev->flags & IFF_PROMISC) { 2159 /* Enable promiscuous mode */ 2160 cfg |= MACB_BIT(CAF); 2161 2162 /* Disable RX checksum offload */ 2163 if (macb_is_gem(bp)) 2164 cfg &= ~GEM_BIT(RXCOEN); 2165 } else { 2166 /* Disable promiscuous mode */ 2167 cfg &= ~MACB_BIT(CAF); 2168 2169 /* Enable RX checksum offload only if requested */ 2170 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2171 cfg |= GEM_BIT(RXCOEN); 2172 } 2173 2174 if (dev->flags & IFF_ALLMULTI) { 2175 /* Enable all multicast mode */ 2176 macb_or_gem_writel(bp, HRB, -1); 2177 macb_or_gem_writel(bp, HRT, -1); 2178 cfg |= MACB_BIT(NCFGR_MTI); 2179 } else if (!netdev_mc_empty(dev)) { 2180 /* Enable specific multicasts */ 2181 macb_sethashtable(dev); 2182 cfg |= MACB_BIT(NCFGR_MTI); 2183 } else if (dev->flags & (~IFF_ALLMULTI)) { 2184 /* Disable all multicast mode */ 2185 macb_or_gem_writel(bp, HRB, 0); 2186 macb_or_gem_writel(bp, HRT, 0); 2187 cfg &= ~MACB_BIT(NCFGR_MTI); 2188 } 2189 2190 macb_writel(bp, NCFGR, cfg); 2191 } 2192 2193 static int macb_open(struct net_device *dev) 2194 { 2195 struct macb *bp = netdev_priv(dev); 2196 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2197 int err; 2198 2199 netdev_dbg(bp->dev, "open\n"); 2200 2201 /* carrier starts down */ 2202 netif_carrier_off(dev); 2203 2204 /* if the phy is not yet register, retry later*/ 2205 if (!dev->phydev) 2206 return -EAGAIN; 2207 2208 /* RX buffers initialization */ 2209 macb_init_rx_buffer_size(bp, bufsz); 2210 2211 err = macb_alloc_consistent(bp); 2212 if (err) { 2213 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2214 err); 2215 return err; 2216 } 2217 2218 napi_enable(&bp->napi); 2219 2220 bp->macbgem_ops.mog_init_rings(bp); 2221 macb_init_hw(bp); 2222 2223 /* schedule a link state check */ 2224 phy_start(dev->phydev); 2225 2226 netif_tx_start_all_queues(dev); 2227 2228 if (bp->ptp_info) 2229 bp->ptp_info->ptp_init(dev); 2230 2231 return 0; 2232 } 2233 2234 static int macb_close(struct net_device *dev) 2235 { 2236 struct macb *bp = netdev_priv(dev); 2237 unsigned long flags; 2238 2239 netif_tx_stop_all_queues(dev); 2240 napi_disable(&bp->napi); 2241 2242 if (dev->phydev) 2243 phy_stop(dev->phydev); 2244 2245 spin_lock_irqsave(&bp->lock, flags); 2246 macb_reset_hw(bp); 2247 netif_carrier_off(dev); 2248 spin_unlock_irqrestore(&bp->lock, flags); 2249 2250 macb_free_consistent(bp); 2251 2252 if (bp->ptp_info) 2253 bp->ptp_info->ptp_remove(dev); 2254 2255 return 0; 2256 } 2257 2258 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2259 { 2260 if (netif_running(dev)) 2261 return -EBUSY; 2262 2263 dev->mtu = new_mtu; 2264 2265 return 0; 2266 } 2267 2268 static void gem_update_stats(struct macb *bp) 2269 { 2270 unsigned int i; 2271 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2272 2273 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2274 u32 offset = gem_statistics[i].offset; 2275 u64 val = bp->macb_reg_readl(bp, offset); 2276 2277 bp->ethtool_stats[i] += val; 2278 *p += val; 2279 2280 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2281 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2282 val = bp->macb_reg_readl(bp, offset + 4); 2283 bp->ethtool_stats[i] += ((u64)val) << 32; 2284 *(++p) += val; 2285 } 2286 } 2287 } 2288 2289 static struct net_device_stats *gem_get_stats(struct macb *bp) 2290 { 2291 struct gem_stats *hwstat = &bp->hw_stats.gem; 2292 struct net_device_stats *nstat = &bp->dev->stats; 2293 2294 gem_update_stats(bp); 2295 2296 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2297 hwstat->rx_alignment_errors + 2298 hwstat->rx_resource_errors + 2299 hwstat->rx_overruns + 2300 hwstat->rx_oversize_frames + 2301 hwstat->rx_jabbers + 2302 hwstat->rx_undersized_frames + 2303 hwstat->rx_length_field_frame_errors); 2304 nstat->tx_errors = (hwstat->tx_late_collisions + 2305 hwstat->tx_excessive_collisions + 2306 hwstat->tx_underrun + 2307 hwstat->tx_carrier_sense_errors); 2308 nstat->multicast = hwstat->rx_multicast_frames; 2309 nstat->collisions = (hwstat->tx_single_collision_frames + 2310 hwstat->tx_multiple_collision_frames + 2311 hwstat->tx_excessive_collisions); 2312 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2313 hwstat->rx_jabbers + 2314 hwstat->rx_undersized_frames + 2315 hwstat->rx_length_field_frame_errors); 2316 nstat->rx_over_errors = hwstat->rx_resource_errors; 2317 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2318 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2319 nstat->rx_fifo_errors = hwstat->rx_overruns; 2320 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2321 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2322 nstat->tx_fifo_errors = hwstat->tx_underrun; 2323 2324 return nstat; 2325 } 2326 2327 static void gem_get_ethtool_stats(struct net_device *dev, 2328 struct ethtool_stats *stats, u64 *data) 2329 { 2330 struct macb *bp; 2331 2332 bp = netdev_priv(dev); 2333 gem_update_stats(bp); 2334 memcpy(data, &bp->ethtool_stats, sizeof(u64) * GEM_STATS_LEN); 2335 } 2336 2337 static int gem_get_sset_count(struct net_device *dev, int sset) 2338 { 2339 switch (sset) { 2340 case ETH_SS_STATS: 2341 return GEM_STATS_LEN; 2342 default: 2343 return -EOPNOTSUPP; 2344 } 2345 } 2346 2347 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2348 { 2349 unsigned int i; 2350 2351 switch (sset) { 2352 case ETH_SS_STATS: 2353 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2354 memcpy(p, gem_statistics[i].stat_string, 2355 ETH_GSTRING_LEN); 2356 break; 2357 } 2358 } 2359 2360 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2361 { 2362 struct macb *bp = netdev_priv(dev); 2363 struct net_device_stats *nstat = &bp->dev->stats; 2364 struct macb_stats *hwstat = &bp->hw_stats.macb; 2365 2366 if (macb_is_gem(bp)) 2367 return gem_get_stats(bp); 2368 2369 /* read stats from hardware */ 2370 macb_update_stats(bp); 2371 2372 /* Convert HW stats into netdevice stats */ 2373 nstat->rx_errors = (hwstat->rx_fcs_errors + 2374 hwstat->rx_align_errors + 2375 hwstat->rx_resource_errors + 2376 hwstat->rx_overruns + 2377 hwstat->rx_oversize_pkts + 2378 hwstat->rx_jabbers + 2379 hwstat->rx_undersize_pkts + 2380 hwstat->rx_length_mismatch); 2381 nstat->tx_errors = (hwstat->tx_late_cols + 2382 hwstat->tx_excessive_cols + 2383 hwstat->tx_underruns + 2384 hwstat->tx_carrier_errors + 2385 hwstat->sqe_test_errors); 2386 nstat->collisions = (hwstat->tx_single_cols + 2387 hwstat->tx_multiple_cols + 2388 hwstat->tx_excessive_cols); 2389 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2390 hwstat->rx_jabbers + 2391 hwstat->rx_undersize_pkts + 2392 hwstat->rx_length_mismatch); 2393 nstat->rx_over_errors = hwstat->rx_resource_errors + 2394 hwstat->rx_overruns; 2395 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2396 nstat->rx_frame_errors = hwstat->rx_align_errors; 2397 nstat->rx_fifo_errors = hwstat->rx_overruns; 2398 /* XXX: What does "missed" mean? */ 2399 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2400 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2401 nstat->tx_fifo_errors = hwstat->tx_underruns; 2402 /* Don't know about heartbeat or window errors... */ 2403 2404 return nstat; 2405 } 2406 2407 static int macb_get_regs_len(struct net_device *netdev) 2408 { 2409 return MACB_GREGS_NBR * sizeof(u32); 2410 } 2411 2412 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2413 void *p) 2414 { 2415 struct macb *bp = netdev_priv(dev); 2416 unsigned int tail, head; 2417 u32 *regs_buff = p; 2418 2419 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2420 | MACB_GREGS_VERSION; 2421 2422 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2423 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2424 2425 regs_buff[0] = macb_readl(bp, NCR); 2426 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2427 regs_buff[2] = macb_readl(bp, NSR); 2428 regs_buff[3] = macb_readl(bp, TSR); 2429 regs_buff[4] = macb_readl(bp, RBQP); 2430 regs_buff[5] = macb_readl(bp, TBQP); 2431 regs_buff[6] = macb_readl(bp, RSR); 2432 regs_buff[7] = macb_readl(bp, IMR); 2433 2434 regs_buff[8] = tail; 2435 regs_buff[9] = head; 2436 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2437 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2438 2439 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2440 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2441 if (macb_is_gem(bp)) 2442 regs_buff[13] = gem_readl(bp, DMACFG); 2443 } 2444 2445 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2446 { 2447 struct macb *bp = netdev_priv(netdev); 2448 2449 wol->supported = 0; 2450 wol->wolopts = 0; 2451 2452 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2453 wol->supported = WAKE_MAGIC; 2454 2455 if (bp->wol & MACB_WOL_ENABLED) 2456 wol->wolopts |= WAKE_MAGIC; 2457 } 2458 } 2459 2460 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2461 { 2462 struct macb *bp = netdev_priv(netdev); 2463 2464 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2465 (wol->wolopts & ~WAKE_MAGIC)) 2466 return -EOPNOTSUPP; 2467 2468 if (wol->wolopts & WAKE_MAGIC) 2469 bp->wol |= MACB_WOL_ENABLED; 2470 else 2471 bp->wol &= ~MACB_WOL_ENABLED; 2472 2473 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2474 2475 return 0; 2476 } 2477 2478 static void macb_get_ringparam(struct net_device *netdev, 2479 struct ethtool_ringparam *ring) 2480 { 2481 struct macb *bp = netdev_priv(netdev); 2482 2483 ring->rx_max_pending = MAX_RX_RING_SIZE; 2484 ring->tx_max_pending = MAX_TX_RING_SIZE; 2485 2486 ring->rx_pending = bp->rx_ring_size; 2487 ring->tx_pending = bp->tx_ring_size; 2488 } 2489 2490 static int macb_set_ringparam(struct net_device *netdev, 2491 struct ethtool_ringparam *ring) 2492 { 2493 struct macb *bp = netdev_priv(netdev); 2494 u32 new_rx_size, new_tx_size; 2495 unsigned int reset = 0; 2496 2497 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2498 return -EINVAL; 2499 2500 new_rx_size = clamp_t(u32, ring->rx_pending, 2501 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2502 new_rx_size = roundup_pow_of_two(new_rx_size); 2503 2504 new_tx_size = clamp_t(u32, ring->tx_pending, 2505 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2506 new_tx_size = roundup_pow_of_two(new_tx_size); 2507 2508 if ((new_tx_size == bp->tx_ring_size) && 2509 (new_rx_size == bp->rx_ring_size)) { 2510 /* nothing to do */ 2511 return 0; 2512 } 2513 2514 if (netif_running(bp->dev)) { 2515 reset = 1; 2516 macb_close(bp->dev); 2517 } 2518 2519 bp->rx_ring_size = new_rx_size; 2520 bp->tx_ring_size = new_tx_size; 2521 2522 if (reset) 2523 macb_open(bp->dev); 2524 2525 return 0; 2526 } 2527 2528 #ifdef CONFIG_MACB_USE_HWSTAMP 2529 static unsigned int gem_get_tsu_rate(struct macb *bp) 2530 { 2531 struct clk *tsu_clk; 2532 unsigned int tsu_rate; 2533 2534 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2535 if (!IS_ERR(tsu_clk)) 2536 tsu_rate = clk_get_rate(tsu_clk); 2537 /* try pclk instead */ 2538 else if (!IS_ERR(bp->pclk)) { 2539 tsu_clk = bp->pclk; 2540 tsu_rate = clk_get_rate(tsu_clk); 2541 } else 2542 return -ENOTSUPP; 2543 return tsu_rate; 2544 } 2545 2546 static s32 gem_get_ptp_max_adj(void) 2547 { 2548 return 64000000; 2549 } 2550 2551 static int gem_get_ts_info(struct net_device *dev, 2552 struct ethtool_ts_info *info) 2553 { 2554 struct macb *bp = netdev_priv(dev); 2555 2556 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2557 ethtool_op_get_ts_info(dev, info); 2558 return 0; 2559 } 2560 2561 info->so_timestamping = 2562 SOF_TIMESTAMPING_TX_SOFTWARE | 2563 SOF_TIMESTAMPING_RX_SOFTWARE | 2564 SOF_TIMESTAMPING_SOFTWARE | 2565 SOF_TIMESTAMPING_TX_HARDWARE | 2566 SOF_TIMESTAMPING_RX_HARDWARE | 2567 SOF_TIMESTAMPING_RAW_HARDWARE; 2568 info->tx_types = 2569 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2570 (1 << HWTSTAMP_TX_OFF) | 2571 (1 << HWTSTAMP_TX_ON); 2572 info->rx_filters = 2573 (1 << HWTSTAMP_FILTER_NONE) | 2574 (1 << HWTSTAMP_FILTER_ALL); 2575 2576 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2577 2578 return 0; 2579 } 2580 2581 static struct macb_ptp_info gem_ptp_info = { 2582 .ptp_init = gem_ptp_init, 2583 .ptp_remove = gem_ptp_remove, 2584 .get_ptp_max_adj = gem_get_ptp_max_adj, 2585 .get_tsu_rate = gem_get_tsu_rate, 2586 .get_ts_info = gem_get_ts_info, 2587 .get_hwtst = gem_get_hwtst, 2588 .set_hwtst = gem_set_hwtst, 2589 }; 2590 #endif 2591 2592 static int macb_get_ts_info(struct net_device *netdev, 2593 struct ethtool_ts_info *info) 2594 { 2595 struct macb *bp = netdev_priv(netdev); 2596 2597 if (bp->ptp_info) 2598 return bp->ptp_info->get_ts_info(netdev, info); 2599 2600 return ethtool_op_get_ts_info(netdev, info); 2601 } 2602 2603 static const struct ethtool_ops macb_ethtool_ops = { 2604 .get_regs_len = macb_get_regs_len, 2605 .get_regs = macb_get_regs, 2606 .get_link = ethtool_op_get_link, 2607 .get_ts_info = ethtool_op_get_ts_info, 2608 .get_wol = macb_get_wol, 2609 .set_wol = macb_set_wol, 2610 .get_link_ksettings = phy_ethtool_get_link_ksettings, 2611 .set_link_ksettings = phy_ethtool_set_link_ksettings, 2612 .get_ringparam = macb_get_ringparam, 2613 .set_ringparam = macb_set_ringparam, 2614 }; 2615 2616 static const struct ethtool_ops gem_ethtool_ops = { 2617 .get_regs_len = macb_get_regs_len, 2618 .get_regs = macb_get_regs, 2619 .get_link = ethtool_op_get_link, 2620 .get_ts_info = macb_get_ts_info, 2621 .get_ethtool_stats = gem_get_ethtool_stats, 2622 .get_strings = gem_get_ethtool_strings, 2623 .get_sset_count = gem_get_sset_count, 2624 .get_link_ksettings = phy_ethtool_get_link_ksettings, 2625 .set_link_ksettings = phy_ethtool_set_link_ksettings, 2626 .get_ringparam = macb_get_ringparam, 2627 .set_ringparam = macb_set_ringparam, 2628 }; 2629 2630 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 2631 { 2632 struct phy_device *phydev = dev->phydev; 2633 struct macb *bp = netdev_priv(dev); 2634 2635 if (!netif_running(dev)) 2636 return -EINVAL; 2637 2638 if (!phydev) 2639 return -ENODEV; 2640 2641 if (!bp->ptp_info) 2642 return phy_mii_ioctl(phydev, rq, cmd); 2643 2644 switch (cmd) { 2645 case SIOCSHWTSTAMP: 2646 return bp->ptp_info->set_hwtst(dev, rq, cmd); 2647 case SIOCGHWTSTAMP: 2648 return bp->ptp_info->get_hwtst(dev, rq); 2649 default: 2650 return phy_mii_ioctl(phydev, rq, cmd); 2651 } 2652 } 2653 2654 static int macb_set_features(struct net_device *netdev, 2655 netdev_features_t features) 2656 { 2657 struct macb *bp = netdev_priv(netdev); 2658 netdev_features_t changed = features ^ netdev->features; 2659 2660 /* TX checksum offload */ 2661 if ((changed & NETIF_F_HW_CSUM) && macb_is_gem(bp)) { 2662 u32 dmacfg; 2663 2664 dmacfg = gem_readl(bp, DMACFG); 2665 if (features & NETIF_F_HW_CSUM) 2666 dmacfg |= GEM_BIT(TXCOEN); 2667 else 2668 dmacfg &= ~GEM_BIT(TXCOEN); 2669 gem_writel(bp, DMACFG, dmacfg); 2670 } 2671 2672 /* RX checksum offload */ 2673 if ((changed & NETIF_F_RXCSUM) && macb_is_gem(bp)) { 2674 u32 netcfg; 2675 2676 netcfg = gem_readl(bp, NCFGR); 2677 if (features & NETIF_F_RXCSUM && 2678 !(netdev->flags & IFF_PROMISC)) 2679 netcfg |= GEM_BIT(RXCOEN); 2680 else 2681 netcfg &= ~GEM_BIT(RXCOEN); 2682 gem_writel(bp, NCFGR, netcfg); 2683 } 2684 2685 return 0; 2686 } 2687 2688 static const struct net_device_ops macb_netdev_ops = { 2689 .ndo_open = macb_open, 2690 .ndo_stop = macb_close, 2691 .ndo_start_xmit = macb_start_xmit, 2692 .ndo_set_rx_mode = macb_set_rx_mode, 2693 .ndo_get_stats = macb_get_stats, 2694 .ndo_do_ioctl = macb_ioctl, 2695 .ndo_validate_addr = eth_validate_addr, 2696 .ndo_change_mtu = macb_change_mtu, 2697 .ndo_set_mac_address = eth_mac_addr, 2698 #ifdef CONFIG_NET_POLL_CONTROLLER 2699 .ndo_poll_controller = macb_poll_controller, 2700 #endif 2701 .ndo_set_features = macb_set_features, 2702 .ndo_features_check = macb_features_check, 2703 }; 2704 2705 /* Configure peripheral capabilities according to device tree 2706 * and integration options used 2707 */ 2708 static void macb_configure_caps(struct macb *bp, 2709 const struct macb_config *dt_conf) 2710 { 2711 u32 dcfg; 2712 2713 if (dt_conf) 2714 bp->caps = dt_conf->caps; 2715 2716 if (hw_is_gem(bp->regs, bp->native_io)) { 2717 bp->caps |= MACB_CAPS_MACB_IS_GEM; 2718 2719 dcfg = gem_readl(bp, DCFG1); 2720 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 2721 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 2722 dcfg = gem_readl(bp, DCFG2); 2723 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 2724 bp->caps |= MACB_CAPS_FIFO_MODE; 2725 #ifdef CONFIG_MACB_USE_HWSTAMP 2726 if (gem_has_ptp(bp)) { 2727 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 2728 pr_err("GEM doesn't support hardware ptp.\n"); 2729 else { 2730 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 2731 bp->ptp_info = &gem_ptp_info; 2732 } 2733 } 2734 #endif 2735 } 2736 2737 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 2738 } 2739 2740 static void macb_probe_queues(void __iomem *mem, 2741 bool native_io, 2742 unsigned int *queue_mask, 2743 unsigned int *num_queues) 2744 { 2745 unsigned int hw_q; 2746 2747 *queue_mask = 0x1; 2748 *num_queues = 1; 2749 2750 /* is it macb or gem ? 2751 * 2752 * We need to read directly from the hardware here because 2753 * we are early in the probe process and don't have the 2754 * MACB_CAPS_MACB_IS_GEM flag positioned 2755 */ 2756 if (!hw_is_gem(mem, native_io)) 2757 return; 2758 2759 /* bit 0 is never set but queue 0 always exists */ 2760 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 2761 2762 *queue_mask |= 0x1; 2763 2764 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 2765 if (*queue_mask & (1 << hw_q)) 2766 (*num_queues)++; 2767 } 2768 2769 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 2770 struct clk **hclk, struct clk **tx_clk, 2771 struct clk **rx_clk) 2772 { 2773 struct macb_platform_data *pdata; 2774 int err; 2775 2776 pdata = dev_get_platdata(&pdev->dev); 2777 if (pdata) { 2778 *pclk = pdata->pclk; 2779 *hclk = pdata->hclk; 2780 } else { 2781 *pclk = devm_clk_get(&pdev->dev, "pclk"); 2782 *hclk = devm_clk_get(&pdev->dev, "hclk"); 2783 } 2784 2785 if (IS_ERR(*pclk)) { 2786 err = PTR_ERR(*pclk); 2787 dev_err(&pdev->dev, "failed to get macb_clk (%u)\n", err); 2788 return err; 2789 } 2790 2791 if (IS_ERR(*hclk)) { 2792 err = PTR_ERR(*hclk); 2793 dev_err(&pdev->dev, "failed to get hclk (%u)\n", err); 2794 return err; 2795 } 2796 2797 *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); 2798 if (IS_ERR(*tx_clk)) 2799 *tx_clk = NULL; 2800 2801 *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); 2802 if (IS_ERR(*rx_clk)) 2803 *rx_clk = NULL; 2804 2805 err = clk_prepare_enable(*pclk); 2806 if (err) { 2807 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 2808 return err; 2809 } 2810 2811 err = clk_prepare_enable(*hclk); 2812 if (err) { 2813 dev_err(&pdev->dev, "failed to enable hclk (%u)\n", err); 2814 goto err_disable_pclk; 2815 } 2816 2817 err = clk_prepare_enable(*tx_clk); 2818 if (err) { 2819 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 2820 goto err_disable_hclk; 2821 } 2822 2823 err = clk_prepare_enable(*rx_clk); 2824 if (err) { 2825 dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); 2826 goto err_disable_txclk; 2827 } 2828 2829 return 0; 2830 2831 err_disable_txclk: 2832 clk_disable_unprepare(*tx_clk); 2833 2834 err_disable_hclk: 2835 clk_disable_unprepare(*hclk); 2836 2837 err_disable_pclk: 2838 clk_disable_unprepare(*pclk); 2839 2840 return err; 2841 } 2842 2843 static int macb_init(struct platform_device *pdev) 2844 { 2845 struct net_device *dev = platform_get_drvdata(pdev); 2846 unsigned int hw_q, q; 2847 struct macb *bp = netdev_priv(dev); 2848 struct macb_queue *queue; 2849 int err; 2850 u32 val; 2851 2852 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 2853 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 2854 2855 /* set the queue register mapping once for all: queue0 has a special 2856 * register mapping but we don't want to test the queue index then 2857 * compute the corresponding register offset at run time. 2858 */ 2859 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 2860 if (!(bp->queue_mask & (1 << hw_q))) 2861 continue; 2862 2863 queue = &bp->queues[q]; 2864 queue->bp = bp; 2865 if (hw_q) { 2866 queue->ISR = GEM_ISR(hw_q - 1); 2867 queue->IER = GEM_IER(hw_q - 1); 2868 queue->IDR = GEM_IDR(hw_q - 1); 2869 queue->IMR = GEM_IMR(hw_q - 1); 2870 queue->TBQP = GEM_TBQP(hw_q - 1); 2871 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2872 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2873 queue->TBQPH = GEM_TBQPH(hw_q - 1); 2874 #endif 2875 } else { 2876 /* queue0 uses legacy registers */ 2877 queue->ISR = MACB_ISR; 2878 queue->IER = MACB_IER; 2879 queue->IDR = MACB_IDR; 2880 queue->IMR = MACB_IMR; 2881 queue->TBQP = MACB_TBQP; 2882 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2883 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2884 queue->TBQPH = MACB_TBQPH; 2885 #endif 2886 } 2887 2888 /* get irq: here we use the linux queue index, not the hardware 2889 * queue index. the queue irq definitions in the device tree 2890 * must remove the optional gaps that could exist in the 2891 * hardware queue mask. 2892 */ 2893 queue->irq = platform_get_irq(pdev, q); 2894 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 2895 IRQF_SHARED, dev->name, queue); 2896 if (err) { 2897 dev_err(&pdev->dev, 2898 "Unable to request IRQ %d (error %d)\n", 2899 queue->irq, err); 2900 return err; 2901 } 2902 2903 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 2904 q++; 2905 } 2906 2907 dev->netdev_ops = &macb_netdev_ops; 2908 netif_napi_add(dev, &bp->napi, macb_poll, 64); 2909 2910 /* setup appropriated routines according to adapter type */ 2911 if (macb_is_gem(bp)) { 2912 bp->max_tx_length = GEM_MAX_TX_LEN; 2913 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 2914 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 2915 bp->macbgem_ops.mog_init_rings = gem_init_rings; 2916 bp->macbgem_ops.mog_rx = gem_rx; 2917 dev->ethtool_ops = &gem_ethtool_ops; 2918 } else { 2919 bp->max_tx_length = MACB_MAX_TX_LEN; 2920 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 2921 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 2922 bp->macbgem_ops.mog_init_rings = macb_init_rings; 2923 bp->macbgem_ops.mog_rx = macb_rx; 2924 dev->ethtool_ops = &macb_ethtool_ops; 2925 } 2926 2927 /* Set features */ 2928 dev->hw_features = NETIF_F_SG; 2929 2930 /* Check LSO capability */ 2931 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 2932 dev->hw_features |= MACB_NETIF_LSO; 2933 2934 /* Checksum offload is only available on gem with packet buffer */ 2935 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 2936 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 2937 if (bp->caps & MACB_CAPS_SG_DISABLED) 2938 dev->hw_features &= ~NETIF_F_SG; 2939 dev->features = dev->hw_features; 2940 2941 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 2942 val = 0; 2943 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 2944 val = GEM_BIT(RGMII); 2945 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 2946 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 2947 val = MACB_BIT(RMII); 2948 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 2949 val = MACB_BIT(MII); 2950 2951 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 2952 val |= MACB_BIT(CLKEN); 2953 2954 macb_or_gem_writel(bp, USRIO, val); 2955 } 2956 2957 /* Set MII management clock divider */ 2958 val = macb_mdc_clk_div(bp); 2959 val |= macb_dbw(bp); 2960 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2961 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2962 macb_writel(bp, NCFGR, val); 2963 2964 return 0; 2965 } 2966 2967 #if defined(CONFIG_OF) 2968 /* 1518 rounded up */ 2969 #define AT91ETHER_MAX_RBUFF_SZ 0x600 2970 /* max number of receive buffers */ 2971 #define AT91ETHER_MAX_RX_DESCR 9 2972 2973 /* Initialize and start the Receiver and Transmit subsystems */ 2974 static int at91ether_start(struct net_device *dev) 2975 { 2976 struct macb *lp = netdev_priv(dev); 2977 struct macb_dma_desc *desc; 2978 dma_addr_t addr; 2979 u32 ctl; 2980 int i; 2981 2982 lp->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 2983 (AT91ETHER_MAX_RX_DESCR * 2984 macb_dma_desc_get_size(lp)), 2985 &lp->rx_ring_dma, GFP_KERNEL); 2986 if (!lp->rx_ring) 2987 return -ENOMEM; 2988 2989 lp->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 2990 AT91ETHER_MAX_RX_DESCR * 2991 AT91ETHER_MAX_RBUFF_SZ, 2992 &lp->rx_buffers_dma, GFP_KERNEL); 2993 if (!lp->rx_buffers) { 2994 dma_free_coherent(&lp->pdev->dev, 2995 AT91ETHER_MAX_RX_DESCR * 2996 macb_dma_desc_get_size(lp), 2997 lp->rx_ring, lp->rx_ring_dma); 2998 lp->rx_ring = NULL; 2999 return -ENOMEM; 3000 } 3001 3002 addr = lp->rx_buffers_dma; 3003 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3004 desc = macb_rx_desc(lp, i); 3005 macb_set_addr(lp, desc, addr); 3006 desc->ctrl = 0; 3007 addr += AT91ETHER_MAX_RBUFF_SZ; 3008 } 3009 3010 /* Set the Wrap bit on the last descriptor */ 3011 desc->addr |= MACB_BIT(RX_WRAP); 3012 3013 /* Reset buffer index */ 3014 lp->rx_tail = 0; 3015 3016 /* Program address of descriptor list in Rx Buffer Queue register */ 3017 macb_writel(lp, RBQP, lp->rx_ring_dma); 3018 3019 /* Enable Receive and Transmit */ 3020 ctl = macb_readl(lp, NCR); 3021 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3022 3023 return 0; 3024 } 3025 3026 /* Open the ethernet interface */ 3027 static int at91ether_open(struct net_device *dev) 3028 { 3029 struct macb *lp = netdev_priv(dev); 3030 u32 ctl; 3031 int ret; 3032 3033 /* Clear internal statistics */ 3034 ctl = macb_readl(lp, NCR); 3035 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3036 3037 macb_set_hwaddr(lp); 3038 3039 ret = at91ether_start(dev); 3040 if (ret) 3041 return ret; 3042 3043 /* Enable MAC interrupts */ 3044 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3045 MACB_BIT(RXUBR) | 3046 MACB_BIT(ISR_TUND) | 3047 MACB_BIT(ISR_RLE) | 3048 MACB_BIT(TCOMP) | 3049 MACB_BIT(ISR_ROVR) | 3050 MACB_BIT(HRESP)); 3051 3052 /* schedule a link state check */ 3053 phy_start(dev->phydev); 3054 3055 netif_start_queue(dev); 3056 3057 return 0; 3058 } 3059 3060 /* Close the interface */ 3061 static int at91ether_close(struct net_device *dev) 3062 { 3063 struct macb *lp = netdev_priv(dev); 3064 u32 ctl; 3065 3066 /* Disable Receiver and Transmitter */ 3067 ctl = macb_readl(lp, NCR); 3068 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3069 3070 /* Disable MAC interrupts */ 3071 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3072 MACB_BIT(RXUBR) | 3073 MACB_BIT(ISR_TUND) | 3074 MACB_BIT(ISR_RLE) | 3075 MACB_BIT(TCOMP) | 3076 MACB_BIT(ISR_ROVR) | 3077 MACB_BIT(HRESP)); 3078 3079 netif_stop_queue(dev); 3080 3081 dma_free_coherent(&lp->pdev->dev, 3082 AT91ETHER_MAX_RX_DESCR * 3083 macb_dma_desc_get_size(lp), 3084 lp->rx_ring, lp->rx_ring_dma); 3085 lp->rx_ring = NULL; 3086 3087 dma_free_coherent(&lp->pdev->dev, 3088 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3089 lp->rx_buffers, lp->rx_buffers_dma); 3090 lp->rx_buffers = NULL; 3091 3092 return 0; 3093 } 3094 3095 /* Transmit packet */ 3096 static int at91ether_start_xmit(struct sk_buff *skb, struct net_device *dev) 3097 { 3098 struct macb *lp = netdev_priv(dev); 3099 3100 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3101 netif_stop_queue(dev); 3102 3103 /* Store packet information (to free when Tx completed) */ 3104 lp->skb = skb; 3105 lp->skb_length = skb->len; 3106 lp->skb_physaddr = dma_map_single(NULL, skb->data, skb->len, 3107 DMA_TO_DEVICE); 3108 if (dma_mapping_error(NULL, lp->skb_physaddr)) { 3109 dev_kfree_skb_any(skb); 3110 dev->stats.tx_dropped++; 3111 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3112 return NETDEV_TX_OK; 3113 } 3114 3115 /* Set address of the data in the Transmit Address register */ 3116 macb_writel(lp, TAR, lp->skb_physaddr); 3117 /* Set length of the packet in the Transmit Control register */ 3118 macb_writel(lp, TCR, skb->len); 3119 3120 } else { 3121 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3122 return NETDEV_TX_BUSY; 3123 } 3124 3125 return NETDEV_TX_OK; 3126 } 3127 3128 /* Extract received frame from buffer descriptors and sent to upper layers. 3129 * (Called from interrupt context) 3130 */ 3131 static void at91ether_rx(struct net_device *dev) 3132 { 3133 struct macb *lp = netdev_priv(dev); 3134 struct macb_dma_desc *desc; 3135 unsigned char *p_recv; 3136 struct sk_buff *skb; 3137 unsigned int pktlen; 3138 3139 desc = macb_rx_desc(lp, lp->rx_tail); 3140 while (desc->addr & MACB_BIT(RX_USED)) { 3141 p_recv = lp->rx_buffers + lp->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3142 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3143 skb = netdev_alloc_skb(dev, pktlen + 2); 3144 if (skb) { 3145 skb_reserve(skb, 2); 3146 skb_put_data(skb, p_recv, pktlen); 3147 3148 skb->protocol = eth_type_trans(skb, dev); 3149 dev->stats.rx_packets++; 3150 dev->stats.rx_bytes += pktlen; 3151 netif_rx(skb); 3152 } else { 3153 dev->stats.rx_dropped++; 3154 } 3155 3156 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3157 dev->stats.multicast++; 3158 3159 /* reset ownership bit */ 3160 desc->addr &= ~MACB_BIT(RX_USED); 3161 3162 /* wrap after last buffer */ 3163 if (lp->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3164 lp->rx_tail = 0; 3165 else 3166 lp->rx_tail++; 3167 3168 desc = macb_rx_desc(lp, lp->rx_tail); 3169 } 3170 } 3171 3172 /* MAC interrupt handler */ 3173 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3174 { 3175 struct net_device *dev = dev_id; 3176 struct macb *lp = netdev_priv(dev); 3177 u32 intstatus, ctl; 3178 3179 /* MAC Interrupt Status register indicates what interrupts are pending. 3180 * It is automatically cleared once read. 3181 */ 3182 intstatus = macb_readl(lp, ISR); 3183 3184 /* Receive complete */ 3185 if (intstatus & MACB_BIT(RCOMP)) 3186 at91ether_rx(dev); 3187 3188 /* Transmit complete */ 3189 if (intstatus & MACB_BIT(TCOMP)) { 3190 /* The TCOM bit is set even if the transmission failed */ 3191 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3192 dev->stats.tx_errors++; 3193 3194 if (lp->skb) { 3195 dev_kfree_skb_irq(lp->skb); 3196 lp->skb = NULL; 3197 dma_unmap_single(NULL, lp->skb_physaddr, 3198 lp->skb_length, DMA_TO_DEVICE); 3199 dev->stats.tx_packets++; 3200 dev->stats.tx_bytes += lp->skb_length; 3201 } 3202 netif_wake_queue(dev); 3203 } 3204 3205 /* Work-around for EMAC Errata section 41.3.1 */ 3206 if (intstatus & MACB_BIT(RXUBR)) { 3207 ctl = macb_readl(lp, NCR); 3208 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3209 wmb(); 3210 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3211 } 3212 3213 if (intstatus & MACB_BIT(ISR_ROVR)) 3214 netdev_err(dev, "ROVR error\n"); 3215 3216 return IRQ_HANDLED; 3217 } 3218 3219 #ifdef CONFIG_NET_POLL_CONTROLLER 3220 static void at91ether_poll_controller(struct net_device *dev) 3221 { 3222 unsigned long flags; 3223 3224 local_irq_save(flags); 3225 at91ether_interrupt(dev->irq, dev); 3226 local_irq_restore(flags); 3227 } 3228 #endif 3229 3230 static const struct net_device_ops at91ether_netdev_ops = { 3231 .ndo_open = at91ether_open, 3232 .ndo_stop = at91ether_close, 3233 .ndo_start_xmit = at91ether_start_xmit, 3234 .ndo_get_stats = macb_get_stats, 3235 .ndo_set_rx_mode = macb_set_rx_mode, 3236 .ndo_set_mac_address = eth_mac_addr, 3237 .ndo_do_ioctl = macb_ioctl, 3238 .ndo_validate_addr = eth_validate_addr, 3239 #ifdef CONFIG_NET_POLL_CONTROLLER 3240 .ndo_poll_controller = at91ether_poll_controller, 3241 #endif 3242 }; 3243 3244 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3245 struct clk **hclk, struct clk **tx_clk, 3246 struct clk **rx_clk) 3247 { 3248 int err; 3249 3250 *hclk = NULL; 3251 *tx_clk = NULL; 3252 *rx_clk = NULL; 3253 3254 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3255 if (IS_ERR(*pclk)) 3256 return PTR_ERR(*pclk); 3257 3258 err = clk_prepare_enable(*pclk); 3259 if (err) { 3260 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3261 return err; 3262 } 3263 3264 return 0; 3265 } 3266 3267 static int at91ether_init(struct platform_device *pdev) 3268 { 3269 struct net_device *dev = platform_get_drvdata(pdev); 3270 struct macb *bp = netdev_priv(dev); 3271 int err; 3272 u32 reg; 3273 3274 dev->netdev_ops = &at91ether_netdev_ops; 3275 dev->ethtool_ops = &macb_ethtool_ops; 3276 3277 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3278 0, dev->name, dev); 3279 if (err) 3280 return err; 3281 3282 macb_writel(bp, NCR, 0); 3283 3284 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3285 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3286 reg |= MACB_BIT(RM9200_RMII); 3287 3288 macb_writel(bp, NCFGR, reg); 3289 3290 return 0; 3291 } 3292 3293 static const struct macb_config at91sam9260_config = { 3294 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3295 .clk_init = macb_clk_init, 3296 .init = macb_init, 3297 }; 3298 3299 static const struct macb_config pc302gem_config = { 3300 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 3301 .dma_burst_length = 16, 3302 .clk_init = macb_clk_init, 3303 .init = macb_init, 3304 }; 3305 3306 static const struct macb_config sama5d2_config = { 3307 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3308 .dma_burst_length = 16, 3309 .clk_init = macb_clk_init, 3310 .init = macb_init, 3311 }; 3312 3313 static const struct macb_config sama5d3_config = { 3314 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 3315 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 3316 .dma_burst_length = 16, 3317 .clk_init = macb_clk_init, 3318 .init = macb_init, 3319 .jumbo_max_len = 10240, 3320 }; 3321 3322 static const struct macb_config sama5d4_config = { 3323 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3324 .dma_burst_length = 4, 3325 .clk_init = macb_clk_init, 3326 .init = macb_init, 3327 }; 3328 3329 static const struct macb_config emac_config = { 3330 .clk_init = at91ether_clk_init, 3331 .init = at91ether_init, 3332 }; 3333 3334 static const struct macb_config np4_config = { 3335 .caps = MACB_CAPS_USRIO_DISABLED, 3336 .clk_init = macb_clk_init, 3337 .init = macb_init, 3338 }; 3339 3340 static const struct macb_config zynqmp_config = { 3341 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3342 MACB_CAPS_JUMBO | 3343 MACB_CAPS_GEM_HAS_PTP, 3344 .dma_burst_length = 16, 3345 .clk_init = macb_clk_init, 3346 .init = macb_init, 3347 .jumbo_max_len = 10240, 3348 }; 3349 3350 static const struct macb_config zynq_config = { 3351 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF, 3352 .dma_burst_length = 16, 3353 .clk_init = macb_clk_init, 3354 .init = macb_init, 3355 }; 3356 3357 static const struct of_device_id macb_dt_ids[] = { 3358 { .compatible = "cdns,at32ap7000-macb" }, 3359 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 3360 { .compatible = "cdns,macb" }, 3361 { .compatible = "cdns,np4-macb", .data = &np4_config }, 3362 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 3363 { .compatible = "cdns,gem", .data = &pc302gem_config }, 3364 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 3365 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 3366 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 3367 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 3368 { .compatible = "cdns,emac", .data = &emac_config }, 3369 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 3370 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 3371 { /* sentinel */ } 3372 }; 3373 MODULE_DEVICE_TABLE(of, macb_dt_ids); 3374 #endif /* CONFIG_OF */ 3375 3376 static const struct macb_config default_gem_config = { 3377 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3378 MACB_CAPS_JUMBO | 3379 MACB_CAPS_GEM_HAS_PTP, 3380 .dma_burst_length = 16, 3381 .clk_init = macb_clk_init, 3382 .init = macb_init, 3383 .jumbo_max_len = 10240, 3384 }; 3385 3386 static int macb_probe(struct platform_device *pdev) 3387 { 3388 const struct macb_config *macb_config = &default_gem_config; 3389 int (*clk_init)(struct platform_device *, struct clk **, 3390 struct clk **, struct clk **, struct clk **) 3391 = macb_config->clk_init; 3392 int (*init)(struct platform_device *) = macb_config->init; 3393 struct device_node *np = pdev->dev.of_node; 3394 struct device_node *phy_node; 3395 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 3396 unsigned int queue_mask, num_queues; 3397 struct macb_platform_data *pdata; 3398 bool native_io; 3399 struct phy_device *phydev; 3400 struct net_device *dev; 3401 struct resource *regs; 3402 void __iomem *mem; 3403 const char *mac; 3404 struct macb *bp; 3405 int err; 3406 3407 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3408 mem = devm_ioremap_resource(&pdev->dev, regs); 3409 if (IS_ERR(mem)) 3410 return PTR_ERR(mem); 3411 3412 if (np) { 3413 const struct of_device_id *match; 3414 3415 match = of_match_node(macb_dt_ids, np); 3416 if (match && match->data) { 3417 macb_config = match->data; 3418 clk_init = macb_config->clk_init; 3419 init = macb_config->init; 3420 } 3421 } 3422 3423 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk); 3424 if (err) 3425 return err; 3426 3427 native_io = hw_is_native_io(mem); 3428 3429 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 3430 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 3431 if (!dev) { 3432 err = -ENOMEM; 3433 goto err_disable_clocks; 3434 } 3435 3436 dev->base_addr = regs->start; 3437 3438 SET_NETDEV_DEV(dev, &pdev->dev); 3439 3440 bp = netdev_priv(dev); 3441 bp->pdev = pdev; 3442 bp->dev = dev; 3443 bp->regs = mem; 3444 bp->native_io = native_io; 3445 if (native_io) { 3446 bp->macb_reg_readl = hw_readl_native; 3447 bp->macb_reg_writel = hw_writel_native; 3448 } else { 3449 bp->macb_reg_readl = hw_readl; 3450 bp->macb_reg_writel = hw_writel; 3451 } 3452 bp->num_queues = num_queues; 3453 bp->queue_mask = queue_mask; 3454 if (macb_config) 3455 bp->dma_burst_length = macb_config->dma_burst_length; 3456 bp->pclk = pclk; 3457 bp->hclk = hclk; 3458 bp->tx_clk = tx_clk; 3459 bp->rx_clk = rx_clk; 3460 if (macb_config) 3461 bp->jumbo_max_len = macb_config->jumbo_max_len; 3462 3463 bp->wol = 0; 3464 if (of_get_property(np, "magic-packet", NULL)) 3465 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 3466 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 3467 3468 spin_lock_init(&bp->lock); 3469 3470 /* setup capabilities */ 3471 macb_configure_caps(bp, macb_config); 3472 3473 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3474 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 3475 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 3476 bp->hw_dma_cap |= HW_DMA_CAP_64B; 3477 } 3478 #endif 3479 platform_set_drvdata(pdev, dev); 3480 3481 dev->irq = platform_get_irq(pdev, 0); 3482 if (dev->irq < 0) { 3483 err = dev->irq; 3484 goto err_out_free_netdev; 3485 } 3486 3487 /* MTU range: 68 - 1500 or 10240 */ 3488 dev->min_mtu = GEM_MTU_MIN_SIZE; 3489 if (bp->caps & MACB_CAPS_JUMBO) 3490 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 3491 else 3492 dev->max_mtu = ETH_DATA_LEN; 3493 3494 mac = of_get_mac_address(np); 3495 if (mac) 3496 ether_addr_copy(bp->dev->dev_addr, mac); 3497 else 3498 macb_get_hwaddr(bp); 3499 3500 /* Power up the PHY if there is a GPIO reset */ 3501 phy_node = of_get_next_available_child(np, NULL); 3502 if (phy_node) { 3503 int gpio = of_get_named_gpio(phy_node, "reset-gpios", 0); 3504 3505 if (gpio_is_valid(gpio)) { 3506 bp->reset_gpio = gpio_to_desc(gpio); 3507 gpiod_direction_output(bp->reset_gpio, 1); 3508 } 3509 } 3510 of_node_put(phy_node); 3511 3512 err = of_get_phy_mode(np); 3513 if (err < 0) { 3514 pdata = dev_get_platdata(&pdev->dev); 3515 if (pdata && pdata->is_rmii) 3516 bp->phy_interface = PHY_INTERFACE_MODE_RMII; 3517 else 3518 bp->phy_interface = PHY_INTERFACE_MODE_MII; 3519 } else { 3520 bp->phy_interface = err; 3521 } 3522 3523 /* IP specific init */ 3524 err = init(pdev); 3525 if (err) 3526 goto err_out_free_netdev; 3527 3528 err = macb_mii_init(bp); 3529 if (err) 3530 goto err_out_free_netdev; 3531 3532 phydev = dev->phydev; 3533 3534 netif_carrier_off(dev); 3535 3536 err = register_netdev(dev); 3537 if (err) { 3538 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 3539 goto err_out_unregister_mdio; 3540 } 3541 3542 phy_attached_info(phydev); 3543 3544 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 3545 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 3546 dev->base_addr, dev->irq, dev->dev_addr); 3547 3548 return 0; 3549 3550 err_out_unregister_mdio: 3551 phy_disconnect(dev->phydev); 3552 mdiobus_unregister(bp->mii_bus); 3553 mdiobus_free(bp->mii_bus); 3554 3555 /* Shutdown the PHY if there is a GPIO reset */ 3556 if (bp->reset_gpio) 3557 gpiod_set_value(bp->reset_gpio, 0); 3558 3559 err_out_free_netdev: 3560 free_netdev(dev); 3561 3562 err_disable_clocks: 3563 clk_disable_unprepare(tx_clk); 3564 clk_disable_unprepare(hclk); 3565 clk_disable_unprepare(pclk); 3566 clk_disable_unprepare(rx_clk); 3567 3568 return err; 3569 } 3570 3571 static int macb_remove(struct platform_device *pdev) 3572 { 3573 struct net_device *dev; 3574 struct macb *bp; 3575 3576 dev = platform_get_drvdata(pdev); 3577 3578 if (dev) { 3579 bp = netdev_priv(dev); 3580 if (dev->phydev) 3581 phy_disconnect(dev->phydev); 3582 mdiobus_unregister(bp->mii_bus); 3583 dev->phydev = NULL; 3584 mdiobus_free(bp->mii_bus); 3585 3586 /* Shutdown the PHY if there is a GPIO reset */ 3587 if (bp->reset_gpio) 3588 gpiod_set_value(bp->reset_gpio, 0); 3589 3590 unregister_netdev(dev); 3591 clk_disable_unprepare(bp->tx_clk); 3592 clk_disable_unprepare(bp->hclk); 3593 clk_disable_unprepare(bp->pclk); 3594 clk_disable_unprepare(bp->rx_clk); 3595 of_node_put(bp->phy_node); 3596 free_netdev(dev); 3597 } 3598 3599 return 0; 3600 } 3601 3602 static int __maybe_unused macb_suspend(struct device *dev) 3603 { 3604 struct platform_device *pdev = to_platform_device(dev); 3605 struct net_device *netdev = platform_get_drvdata(pdev); 3606 struct macb *bp = netdev_priv(netdev); 3607 3608 netif_carrier_off(netdev); 3609 netif_device_detach(netdev); 3610 3611 if (bp->wol & MACB_WOL_ENABLED) { 3612 macb_writel(bp, IER, MACB_BIT(WOL)); 3613 macb_writel(bp, WOL, MACB_BIT(MAG)); 3614 enable_irq_wake(bp->queues[0].irq); 3615 } else { 3616 clk_disable_unprepare(bp->tx_clk); 3617 clk_disable_unprepare(bp->hclk); 3618 clk_disable_unprepare(bp->pclk); 3619 clk_disable_unprepare(bp->rx_clk); 3620 } 3621 3622 return 0; 3623 } 3624 3625 static int __maybe_unused macb_resume(struct device *dev) 3626 { 3627 struct platform_device *pdev = to_platform_device(dev); 3628 struct net_device *netdev = platform_get_drvdata(pdev); 3629 struct macb *bp = netdev_priv(netdev); 3630 3631 if (bp->wol & MACB_WOL_ENABLED) { 3632 macb_writel(bp, IDR, MACB_BIT(WOL)); 3633 macb_writel(bp, WOL, 0); 3634 disable_irq_wake(bp->queues[0].irq); 3635 } else { 3636 clk_prepare_enable(bp->pclk); 3637 clk_prepare_enable(bp->hclk); 3638 clk_prepare_enable(bp->tx_clk); 3639 clk_prepare_enable(bp->rx_clk); 3640 } 3641 3642 netif_device_attach(netdev); 3643 3644 return 0; 3645 } 3646 3647 static SIMPLE_DEV_PM_OPS(macb_pm_ops, macb_suspend, macb_resume); 3648 3649 static struct platform_driver macb_driver = { 3650 .probe = macb_probe, 3651 .remove = macb_remove, 3652 .driver = { 3653 .name = "macb", 3654 .of_match_table = of_match_ptr(macb_dt_ids), 3655 .pm = &macb_pm_ops, 3656 }, 3657 }; 3658 3659 module_platform_driver(macb_driver); 3660 3661 MODULE_LICENSE("GPL"); 3662 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 3663 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 3664 MODULE_ALIAS("platform:macb"); 3665