1 /* 2 * Cadence MACB/GEM Ethernet Controller driver 3 * 4 * Copyright (C) 2004-2006 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 #include <linux/clk.h> 13 #include <linux/crc32.h> 14 #include <linux/module.h> 15 #include <linux/moduleparam.h> 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/circ_buf.h> 19 #include <linux/slab.h> 20 #include <linux/init.h> 21 #include <linux/io.h> 22 #include <linux/gpio.h> 23 #include <linux/gpio/consumer.h> 24 #include <linux/interrupt.h> 25 #include <linux/netdevice.h> 26 #include <linux/etherdevice.h> 27 #include <linux/dma-mapping.h> 28 #include <linux/platform_data/macb.h> 29 #include <linux/platform_device.h> 30 #include <linux/phy.h> 31 #include <linux/of.h> 32 #include <linux/of_device.h> 33 #include <linux/of_gpio.h> 34 #include <linux/of_mdio.h> 35 #include <linux/of_net.h> 36 #include <linux/ip.h> 37 #include <linux/udp.h> 38 #include <linux/tcp.h> 39 #include "macb.h" 40 41 #define MACB_RX_BUFFER_SIZE 128 42 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 43 44 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 45 #define MIN_RX_RING_SIZE 64 46 #define MAX_RX_RING_SIZE 8192 47 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 48 * (bp)->rx_ring_size) 49 50 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 51 #define MIN_TX_RING_SIZE 64 52 #define MAX_TX_RING_SIZE 4096 53 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 54 * (bp)->tx_ring_size) 55 56 /* level of occupied TX descriptors under which we wake up TX process */ 57 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 58 59 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(RXUBR) \ 60 | MACB_BIT(ISR_ROVR)) 61 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 62 | MACB_BIT(ISR_RLE) \ 63 | MACB_BIT(TXERR)) 64 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP)) 65 66 /* Max length of transmit frame must be a multiple of 8 bytes */ 67 #define MACB_TX_LEN_ALIGN 8 68 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 69 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 70 71 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 72 #define MACB_NETIF_LSO NETIF_F_TSO 73 74 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 75 #define MACB_WOL_ENABLED (0x1 << 1) 76 77 /* Graceful stop timeouts in us. We should allow up to 78 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 79 */ 80 #define MACB_HALT_TIMEOUT 1230 81 82 /* DMA buffer descriptor might be different size 83 * depends on hardware configuration: 84 * 85 * 1. dma address width 32 bits: 86 * word 1: 32 bit address of Data Buffer 87 * word 2: control 88 * 89 * 2. dma address width 64 bits: 90 * word 1: 32 bit address of Data Buffer 91 * word 2: control 92 * word 3: upper 32 bit address of Data Buffer 93 * word 4: unused 94 * 95 * 3. dma address width 32 bits with hardware timestamping: 96 * word 1: 32 bit address of Data Buffer 97 * word 2: control 98 * word 3: timestamp word 1 99 * word 4: timestamp word 2 100 * 101 * 4. dma address width 64 bits with hardware timestamping: 102 * word 1: 32 bit address of Data Buffer 103 * word 2: control 104 * word 3: upper 32 bit address of Data Buffer 105 * word 4: unused 106 * word 5: timestamp word 1 107 * word 6: timestamp word 2 108 */ 109 static unsigned int macb_dma_desc_get_size(struct macb *bp) 110 { 111 #ifdef MACB_EXT_DESC 112 unsigned int desc_size; 113 114 switch (bp->hw_dma_cap) { 115 case HW_DMA_CAP_64B: 116 desc_size = sizeof(struct macb_dma_desc) 117 + sizeof(struct macb_dma_desc_64); 118 break; 119 case HW_DMA_CAP_PTP: 120 desc_size = sizeof(struct macb_dma_desc) 121 + sizeof(struct macb_dma_desc_ptp); 122 break; 123 case HW_DMA_CAP_64B_PTP: 124 desc_size = sizeof(struct macb_dma_desc) 125 + sizeof(struct macb_dma_desc_64) 126 + sizeof(struct macb_dma_desc_ptp); 127 break; 128 default: 129 desc_size = sizeof(struct macb_dma_desc); 130 } 131 return desc_size; 132 #endif 133 return sizeof(struct macb_dma_desc); 134 } 135 136 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 137 { 138 #ifdef MACB_EXT_DESC 139 switch (bp->hw_dma_cap) { 140 case HW_DMA_CAP_64B: 141 case HW_DMA_CAP_PTP: 142 desc_idx <<= 1; 143 break; 144 case HW_DMA_CAP_64B_PTP: 145 desc_idx *= 3; 146 break; 147 default: 148 break; 149 } 150 #endif 151 return desc_idx; 152 } 153 154 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 155 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 156 { 157 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 158 return (struct macb_dma_desc_64 *)((void *)desc + sizeof(struct macb_dma_desc)); 159 return NULL; 160 } 161 #endif 162 163 /* Ring buffer accessors */ 164 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 165 { 166 return index & (bp->tx_ring_size - 1); 167 } 168 169 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 170 unsigned int index) 171 { 172 index = macb_tx_ring_wrap(queue->bp, index); 173 index = macb_adj_dma_desc_idx(queue->bp, index); 174 return &queue->tx_ring[index]; 175 } 176 177 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 178 unsigned int index) 179 { 180 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 181 } 182 183 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 184 { 185 dma_addr_t offset; 186 187 offset = macb_tx_ring_wrap(queue->bp, index) * 188 macb_dma_desc_get_size(queue->bp); 189 190 return queue->tx_ring_dma + offset; 191 } 192 193 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 194 { 195 return index & (bp->rx_ring_size - 1); 196 } 197 198 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 199 { 200 index = macb_rx_ring_wrap(queue->bp, index); 201 index = macb_adj_dma_desc_idx(queue->bp, index); 202 return &queue->rx_ring[index]; 203 } 204 205 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 206 { 207 return queue->rx_buffers + queue->bp->rx_buffer_size * 208 macb_rx_ring_wrap(queue->bp, index); 209 } 210 211 /* I/O accessors */ 212 static u32 hw_readl_native(struct macb *bp, int offset) 213 { 214 return __raw_readl(bp->regs + offset); 215 } 216 217 static void hw_writel_native(struct macb *bp, int offset, u32 value) 218 { 219 __raw_writel(value, bp->regs + offset); 220 } 221 222 static u32 hw_readl(struct macb *bp, int offset) 223 { 224 return readl_relaxed(bp->regs + offset); 225 } 226 227 static void hw_writel(struct macb *bp, int offset, u32 value) 228 { 229 writel_relaxed(value, bp->regs + offset); 230 } 231 232 /* Find the CPU endianness by using the loopback bit of NCR register. When the 233 * CPU is in big endian we need to program swapped mode for management 234 * descriptor access. 235 */ 236 static bool hw_is_native_io(void __iomem *addr) 237 { 238 u32 value = MACB_BIT(LLB); 239 240 __raw_writel(value, addr + MACB_NCR); 241 value = __raw_readl(addr + MACB_NCR); 242 243 /* Write 0 back to disable everything */ 244 __raw_writel(0, addr + MACB_NCR); 245 246 return value == MACB_BIT(LLB); 247 } 248 249 static bool hw_is_gem(void __iomem *addr, bool native_io) 250 { 251 u32 id; 252 253 if (native_io) 254 id = __raw_readl(addr + MACB_MID); 255 else 256 id = readl_relaxed(addr + MACB_MID); 257 258 return MACB_BFEXT(IDNUM, id) >= 0x2; 259 } 260 261 static void macb_set_hwaddr(struct macb *bp) 262 { 263 u32 bottom; 264 u16 top; 265 266 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 267 macb_or_gem_writel(bp, SA1B, bottom); 268 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 269 macb_or_gem_writel(bp, SA1T, top); 270 271 /* Clear unused address register sets */ 272 macb_or_gem_writel(bp, SA2B, 0); 273 macb_or_gem_writel(bp, SA2T, 0); 274 macb_or_gem_writel(bp, SA3B, 0); 275 macb_or_gem_writel(bp, SA3T, 0); 276 macb_or_gem_writel(bp, SA4B, 0); 277 macb_or_gem_writel(bp, SA4T, 0); 278 } 279 280 static void macb_get_hwaddr(struct macb *bp) 281 { 282 struct macb_platform_data *pdata; 283 u32 bottom; 284 u16 top; 285 u8 addr[6]; 286 int i; 287 288 pdata = dev_get_platdata(&bp->pdev->dev); 289 290 /* Check all 4 address register for valid address */ 291 for (i = 0; i < 4; i++) { 292 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 293 top = macb_or_gem_readl(bp, SA1T + i * 8); 294 295 if (pdata && pdata->rev_eth_addr) { 296 addr[5] = bottom & 0xff; 297 addr[4] = (bottom >> 8) & 0xff; 298 addr[3] = (bottom >> 16) & 0xff; 299 addr[2] = (bottom >> 24) & 0xff; 300 addr[1] = top & 0xff; 301 addr[0] = (top & 0xff00) >> 8; 302 } else { 303 addr[0] = bottom & 0xff; 304 addr[1] = (bottom >> 8) & 0xff; 305 addr[2] = (bottom >> 16) & 0xff; 306 addr[3] = (bottom >> 24) & 0xff; 307 addr[4] = top & 0xff; 308 addr[5] = (top >> 8) & 0xff; 309 } 310 311 if (is_valid_ether_addr(addr)) { 312 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 313 return; 314 } 315 } 316 317 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 318 eth_hw_addr_random(bp->dev); 319 } 320 321 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 322 { 323 struct macb *bp = bus->priv; 324 int value; 325 326 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 327 | MACB_BF(RW, MACB_MAN_READ) 328 | MACB_BF(PHYA, mii_id) 329 | MACB_BF(REGA, regnum) 330 | MACB_BF(CODE, MACB_MAN_CODE))); 331 332 /* wait for end of transfer */ 333 while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) 334 cpu_relax(); 335 336 value = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 337 338 return value; 339 } 340 341 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 342 u16 value) 343 { 344 struct macb *bp = bus->priv; 345 346 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 347 | MACB_BF(RW, MACB_MAN_WRITE) 348 | MACB_BF(PHYA, mii_id) 349 | MACB_BF(REGA, regnum) 350 | MACB_BF(CODE, MACB_MAN_CODE) 351 | MACB_BF(DATA, value))); 352 353 /* wait for end of transfer */ 354 while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) 355 cpu_relax(); 356 357 return 0; 358 } 359 360 /** 361 * macb_set_tx_clk() - Set a clock to a new frequency 362 * @clk Pointer to the clock to change 363 * @rate New frequency in Hz 364 * @dev Pointer to the struct net_device 365 */ 366 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 367 { 368 long ferr, rate, rate_rounded; 369 370 if (!clk) 371 return; 372 373 switch (speed) { 374 case SPEED_10: 375 rate = 2500000; 376 break; 377 case SPEED_100: 378 rate = 25000000; 379 break; 380 case SPEED_1000: 381 rate = 125000000; 382 break; 383 default: 384 return; 385 } 386 387 rate_rounded = clk_round_rate(clk, rate); 388 if (rate_rounded < 0) 389 return; 390 391 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 392 * is not satisfied. 393 */ 394 ferr = abs(rate_rounded - rate); 395 ferr = DIV_ROUND_UP(ferr, rate / 100000); 396 if (ferr > 5) 397 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 398 rate); 399 400 if (clk_set_rate(clk, rate_rounded)) 401 netdev_err(dev, "adjusting tx_clk failed.\n"); 402 } 403 404 static void macb_handle_link_change(struct net_device *dev) 405 { 406 struct macb *bp = netdev_priv(dev); 407 struct phy_device *phydev = dev->phydev; 408 unsigned long flags; 409 int status_change = 0; 410 411 spin_lock_irqsave(&bp->lock, flags); 412 413 if (phydev->link) { 414 if ((bp->speed != phydev->speed) || 415 (bp->duplex != phydev->duplex)) { 416 u32 reg; 417 418 reg = macb_readl(bp, NCFGR); 419 reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 420 if (macb_is_gem(bp)) 421 reg &= ~GEM_BIT(GBE); 422 423 if (phydev->duplex) 424 reg |= MACB_BIT(FD); 425 if (phydev->speed == SPEED_100) 426 reg |= MACB_BIT(SPD); 427 if (phydev->speed == SPEED_1000 && 428 bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 429 reg |= GEM_BIT(GBE); 430 431 macb_or_gem_writel(bp, NCFGR, reg); 432 433 bp->speed = phydev->speed; 434 bp->duplex = phydev->duplex; 435 status_change = 1; 436 } 437 } 438 439 if (phydev->link != bp->link) { 440 if (!phydev->link) { 441 bp->speed = 0; 442 bp->duplex = -1; 443 } 444 bp->link = phydev->link; 445 446 status_change = 1; 447 } 448 449 spin_unlock_irqrestore(&bp->lock, flags); 450 451 if (status_change) { 452 if (phydev->link) { 453 /* Update the TX clock rate if and only if the link is 454 * up and there has been a link change. 455 */ 456 macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); 457 458 netif_carrier_on(dev); 459 netdev_info(dev, "link up (%d/%s)\n", 460 phydev->speed, 461 phydev->duplex == DUPLEX_FULL ? 462 "Full" : "Half"); 463 } else { 464 netif_carrier_off(dev); 465 netdev_info(dev, "link down\n"); 466 } 467 } 468 } 469 470 /* based on au1000_eth. c*/ 471 static int macb_mii_probe(struct net_device *dev) 472 { 473 struct macb *bp = netdev_priv(dev); 474 struct macb_platform_data *pdata; 475 struct phy_device *phydev; 476 struct device_node *np; 477 int phy_irq, ret, i; 478 479 pdata = dev_get_platdata(&bp->pdev->dev); 480 np = bp->pdev->dev.of_node; 481 ret = 0; 482 483 if (np) { 484 if (of_phy_is_fixed_link(np)) { 485 bp->phy_node = of_node_get(np); 486 } else { 487 bp->phy_node = of_parse_phandle(np, "phy-handle", 0); 488 /* fallback to standard phy registration if no 489 * phy-handle was found nor any phy found during 490 * dt phy registration 491 */ 492 if (!bp->phy_node && !phy_find_first(bp->mii_bus)) { 493 for (i = 0; i < PHY_MAX_ADDR; i++) { 494 struct phy_device *phydev; 495 496 phydev = mdiobus_scan(bp->mii_bus, i); 497 if (IS_ERR(phydev) && 498 PTR_ERR(phydev) != -ENODEV) { 499 ret = PTR_ERR(phydev); 500 break; 501 } 502 } 503 504 if (ret) 505 return -ENODEV; 506 } 507 } 508 } 509 510 if (bp->phy_node) { 511 phydev = of_phy_connect(dev, bp->phy_node, 512 &macb_handle_link_change, 0, 513 bp->phy_interface); 514 if (!phydev) 515 return -ENODEV; 516 } else { 517 phydev = phy_find_first(bp->mii_bus); 518 if (!phydev) { 519 netdev_err(dev, "no PHY found\n"); 520 return -ENXIO; 521 } 522 523 if (pdata) { 524 if (gpio_is_valid(pdata->phy_irq_pin)) { 525 ret = devm_gpio_request(&bp->pdev->dev, 526 pdata->phy_irq_pin, "phy int"); 527 if (!ret) { 528 phy_irq = gpio_to_irq(pdata->phy_irq_pin); 529 phydev->irq = (phy_irq < 0) ? PHY_POLL : phy_irq; 530 } 531 } else { 532 phydev->irq = PHY_POLL; 533 } 534 } 535 536 /* attach the mac to the phy */ 537 ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, 538 bp->phy_interface); 539 if (ret) { 540 netdev_err(dev, "Could not attach to PHY\n"); 541 return ret; 542 } 543 } 544 545 /* mask with MAC supported features */ 546 if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 547 phy_set_max_speed(phydev, SPEED_1000); 548 else 549 phy_set_max_speed(phydev, SPEED_100); 550 551 if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) 552 phy_remove_link_mode(phydev, 553 ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 554 555 bp->link = 0; 556 bp->speed = 0; 557 bp->duplex = -1; 558 559 return 0; 560 } 561 562 static int macb_mii_init(struct macb *bp) 563 { 564 struct macb_platform_data *pdata; 565 struct device_node *np; 566 int err = -ENXIO; 567 568 /* Enable management port */ 569 macb_writel(bp, NCR, MACB_BIT(MPE)); 570 571 bp->mii_bus = mdiobus_alloc(); 572 if (!bp->mii_bus) { 573 err = -ENOMEM; 574 goto err_out; 575 } 576 577 bp->mii_bus->name = "MACB_mii_bus"; 578 bp->mii_bus->read = &macb_mdio_read; 579 bp->mii_bus->write = &macb_mdio_write; 580 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 581 bp->pdev->name, bp->pdev->id); 582 bp->mii_bus->priv = bp; 583 bp->mii_bus->parent = &bp->pdev->dev; 584 pdata = dev_get_platdata(&bp->pdev->dev); 585 586 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 587 588 np = bp->pdev->dev.of_node; 589 if (np && of_phy_is_fixed_link(np)) { 590 if (of_phy_register_fixed_link(np) < 0) { 591 dev_err(&bp->pdev->dev, 592 "broken fixed-link specification %pOF\n", np); 593 goto err_out_free_mdiobus; 594 } 595 596 err = mdiobus_register(bp->mii_bus); 597 } else { 598 if (pdata) 599 bp->mii_bus->phy_mask = pdata->phy_mask; 600 601 err = of_mdiobus_register(bp->mii_bus, np); 602 } 603 604 if (err) 605 goto err_out_free_fixed_link; 606 607 err = macb_mii_probe(bp->dev); 608 if (err) 609 goto err_out_unregister_bus; 610 611 return 0; 612 613 err_out_unregister_bus: 614 mdiobus_unregister(bp->mii_bus); 615 err_out_free_fixed_link: 616 if (np && of_phy_is_fixed_link(np)) 617 of_phy_deregister_fixed_link(np); 618 err_out_free_mdiobus: 619 of_node_put(bp->phy_node); 620 mdiobus_free(bp->mii_bus); 621 err_out: 622 return err; 623 } 624 625 static void macb_update_stats(struct macb *bp) 626 { 627 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 628 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 629 int offset = MACB_PFR; 630 631 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 632 633 for (; p < end; p++, offset += 4) 634 *p += bp->macb_reg_readl(bp, offset); 635 } 636 637 static int macb_halt_tx(struct macb *bp) 638 { 639 unsigned long halt_time, timeout; 640 u32 status; 641 642 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 643 644 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 645 do { 646 halt_time = jiffies; 647 status = macb_readl(bp, TSR); 648 if (!(status & MACB_BIT(TGO))) 649 return 0; 650 651 udelay(250); 652 } while (time_before(halt_time, timeout)); 653 654 return -ETIMEDOUT; 655 } 656 657 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 658 { 659 if (tx_skb->mapping) { 660 if (tx_skb->mapped_as_page) 661 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 662 tx_skb->size, DMA_TO_DEVICE); 663 else 664 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 665 tx_skb->size, DMA_TO_DEVICE); 666 tx_skb->mapping = 0; 667 } 668 669 if (tx_skb->skb) { 670 dev_kfree_skb_any(tx_skb->skb); 671 tx_skb->skb = NULL; 672 } 673 } 674 675 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 676 { 677 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 678 struct macb_dma_desc_64 *desc_64; 679 680 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 681 desc_64 = macb_64b_desc(bp, desc); 682 desc_64->addrh = upper_32_bits(addr); 683 } 684 #endif 685 desc->addr = lower_32_bits(addr); 686 } 687 688 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 689 { 690 dma_addr_t addr = 0; 691 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 692 struct macb_dma_desc_64 *desc_64; 693 694 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 695 desc_64 = macb_64b_desc(bp, desc); 696 addr = ((u64)(desc_64->addrh) << 32); 697 } 698 #endif 699 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 700 return addr; 701 } 702 703 static void macb_tx_error_task(struct work_struct *work) 704 { 705 struct macb_queue *queue = container_of(work, struct macb_queue, 706 tx_error_task); 707 struct macb *bp = queue->bp; 708 struct macb_tx_skb *tx_skb; 709 struct macb_dma_desc *desc; 710 struct sk_buff *skb; 711 unsigned int tail; 712 unsigned long flags; 713 714 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 715 (unsigned int)(queue - bp->queues), 716 queue->tx_tail, queue->tx_head); 717 718 /* Prevent the queue IRQ handlers from running: each of them may call 719 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 720 * As explained below, we have to halt the transmission before updating 721 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 722 * network engine about the macb/gem being halted. 723 */ 724 spin_lock_irqsave(&bp->lock, flags); 725 726 /* Make sure nobody is trying to queue up new packets */ 727 netif_tx_stop_all_queues(bp->dev); 728 729 /* Stop transmission now 730 * (in case we have just queued new packets) 731 * macb/gem must be halted to write TBQP register 732 */ 733 if (macb_halt_tx(bp)) 734 /* Just complain for now, reinitializing TX path can be good */ 735 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 736 737 /* Treat frames in TX queue including the ones that caused the error. 738 * Free transmit buffers in upper layer. 739 */ 740 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 741 u32 ctrl; 742 743 desc = macb_tx_desc(queue, tail); 744 ctrl = desc->ctrl; 745 tx_skb = macb_tx_skb(queue, tail); 746 skb = tx_skb->skb; 747 748 if (ctrl & MACB_BIT(TX_USED)) { 749 /* skb is set for the last buffer of the frame */ 750 while (!skb) { 751 macb_tx_unmap(bp, tx_skb); 752 tail++; 753 tx_skb = macb_tx_skb(queue, tail); 754 skb = tx_skb->skb; 755 } 756 757 /* ctrl still refers to the first buffer descriptor 758 * since it's the only one written back by the hardware 759 */ 760 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 761 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 762 macb_tx_ring_wrap(bp, tail), 763 skb->data); 764 bp->dev->stats.tx_packets++; 765 queue->stats.tx_packets++; 766 bp->dev->stats.tx_bytes += skb->len; 767 queue->stats.tx_bytes += skb->len; 768 } 769 } else { 770 /* "Buffers exhausted mid-frame" errors may only happen 771 * if the driver is buggy, so complain loudly about 772 * those. Statistics are updated by hardware. 773 */ 774 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 775 netdev_err(bp->dev, 776 "BUG: TX buffers exhausted mid-frame\n"); 777 778 desc->ctrl = ctrl | MACB_BIT(TX_USED); 779 } 780 781 macb_tx_unmap(bp, tx_skb); 782 } 783 784 /* Set end of TX queue */ 785 desc = macb_tx_desc(queue, 0); 786 macb_set_addr(bp, desc, 0); 787 desc->ctrl = MACB_BIT(TX_USED); 788 789 /* Make descriptor updates visible to hardware */ 790 wmb(); 791 792 /* Reinitialize the TX desc queue */ 793 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 794 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 795 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 796 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 797 #endif 798 /* Make TX ring reflect state of hardware */ 799 queue->tx_head = 0; 800 queue->tx_tail = 0; 801 802 /* Housework before enabling TX IRQ */ 803 macb_writel(bp, TSR, macb_readl(bp, TSR)); 804 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 805 806 /* Now we are ready to start transmission again */ 807 netif_tx_start_all_queues(bp->dev); 808 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 809 810 spin_unlock_irqrestore(&bp->lock, flags); 811 } 812 813 static void macb_tx_interrupt(struct macb_queue *queue) 814 { 815 unsigned int tail; 816 unsigned int head; 817 u32 status; 818 struct macb *bp = queue->bp; 819 u16 queue_index = queue - bp->queues; 820 821 status = macb_readl(bp, TSR); 822 macb_writel(bp, TSR, status); 823 824 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 825 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 826 827 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 828 (unsigned long)status); 829 830 head = queue->tx_head; 831 for (tail = queue->tx_tail; tail != head; tail++) { 832 struct macb_tx_skb *tx_skb; 833 struct sk_buff *skb; 834 struct macb_dma_desc *desc; 835 u32 ctrl; 836 837 desc = macb_tx_desc(queue, tail); 838 839 /* Make hw descriptor updates visible to CPU */ 840 rmb(); 841 842 ctrl = desc->ctrl; 843 844 /* TX_USED bit is only set by hardware on the very first buffer 845 * descriptor of the transmitted frame. 846 */ 847 if (!(ctrl & MACB_BIT(TX_USED))) 848 break; 849 850 /* Process all buffers of the current transmitted frame */ 851 for (;; tail++) { 852 tx_skb = macb_tx_skb(queue, tail); 853 skb = tx_skb->skb; 854 855 /* First, update TX stats if needed */ 856 if (skb) { 857 if (gem_ptp_do_txstamp(queue, skb, desc) == 0) { 858 /* skb now belongs to timestamp buffer 859 * and will be removed later 860 */ 861 tx_skb->skb = NULL; 862 } 863 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 864 macb_tx_ring_wrap(bp, tail), 865 skb->data); 866 bp->dev->stats.tx_packets++; 867 queue->stats.tx_packets++; 868 bp->dev->stats.tx_bytes += skb->len; 869 queue->stats.tx_bytes += skb->len; 870 } 871 872 /* Now we can safely release resources */ 873 macb_tx_unmap(bp, tx_skb); 874 875 /* skb is set only for the last buffer of the frame. 876 * WARNING: at this point skb has been freed by 877 * macb_tx_unmap(). 878 */ 879 if (skb) 880 break; 881 } 882 } 883 884 queue->tx_tail = tail; 885 if (__netif_subqueue_stopped(bp->dev, queue_index) && 886 CIRC_CNT(queue->tx_head, queue->tx_tail, 887 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 888 netif_wake_subqueue(bp->dev, queue_index); 889 } 890 891 static void gem_rx_refill(struct macb_queue *queue) 892 { 893 unsigned int entry; 894 struct sk_buff *skb; 895 dma_addr_t paddr; 896 struct macb *bp = queue->bp; 897 struct macb_dma_desc *desc; 898 899 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 900 bp->rx_ring_size) > 0) { 901 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 902 903 /* Make hw descriptor updates visible to CPU */ 904 rmb(); 905 906 queue->rx_prepared_head++; 907 desc = macb_rx_desc(queue, entry); 908 909 if (!queue->rx_skbuff[entry]) { 910 /* allocate sk_buff for this free entry in ring */ 911 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 912 if (unlikely(!skb)) { 913 netdev_err(bp->dev, 914 "Unable to allocate sk_buff\n"); 915 break; 916 } 917 918 /* now fill corresponding descriptor entry */ 919 paddr = dma_map_single(&bp->pdev->dev, skb->data, 920 bp->rx_buffer_size, 921 DMA_FROM_DEVICE); 922 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 923 dev_kfree_skb(skb); 924 break; 925 } 926 927 queue->rx_skbuff[entry] = skb; 928 929 if (entry == bp->rx_ring_size - 1) 930 paddr |= MACB_BIT(RX_WRAP); 931 macb_set_addr(bp, desc, paddr); 932 desc->ctrl = 0; 933 934 /* properly align Ethernet header */ 935 skb_reserve(skb, NET_IP_ALIGN); 936 } else { 937 desc->addr &= ~MACB_BIT(RX_USED); 938 desc->ctrl = 0; 939 } 940 } 941 942 /* Make descriptor updates visible to hardware */ 943 wmb(); 944 945 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 946 queue, queue->rx_prepared_head, queue->rx_tail); 947 } 948 949 /* Mark DMA descriptors from begin up to and not including end as unused */ 950 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 951 unsigned int end) 952 { 953 unsigned int frag; 954 955 for (frag = begin; frag != end; frag++) { 956 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 957 958 desc->addr &= ~MACB_BIT(RX_USED); 959 } 960 961 /* Make descriptor updates visible to hardware */ 962 wmb(); 963 964 /* When this happens, the hardware stats registers for 965 * whatever caused this is updated, so we don't have to record 966 * anything. 967 */ 968 } 969 970 static int gem_rx(struct macb_queue *queue, int budget) 971 { 972 struct macb *bp = queue->bp; 973 unsigned int len; 974 unsigned int entry; 975 struct sk_buff *skb; 976 struct macb_dma_desc *desc; 977 int count = 0; 978 979 while (count < budget) { 980 u32 ctrl; 981 dma_addr_t addr; 982 bool rxused; 983 984 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 985 desc = macb_rx_desc(queue, entry); 986 987 /* Make hw descriptor updates visible to CPU */ 988 rmb(); 989 990 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 991 addr = macb_get_addr(bp, desc); 992 ctrl = desc->ctrl; 993 994 if (!rxused) 995 break; 996 997 queue->rx_tail++; 998 count++; 999 1000 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1001 netdev_err(bp->dev, 1002 "not whole frame pointed by descriptor\n"); 1003 bp->dev->stats.rx_dropped++; 1004 queue->stats.rx_dropped++; 1005 break; 1006 } 1007 skb = queue->rx_skbuff[entry]; 1008 if (unlikely(!skb)) { 1009 netdev_err(bp->dev, 1010 "inconsistent Rx descriptor chain\n"); 1011 bp->dev->stats.rx_dropped++; 1012 queue->stats.rx_dropped++; 1013 break; 1014 } 1015 /* now everything is ready for receiving packet */ 1016 queue->rx_skbuff[entry] = NULL; 1017 len = ctrl & bp->rx_frm_len_mask; 1018 1019 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1020 1021 skb_put(skb, len); 1022 dma_unmap_single(&bp->pdev->dev, addr, 1023 bp->rx_buffer_size, DMA_FROM_DEVICE); 1024 1025 skb->protocol = eth_type_trans(skb, bp->dev); 1026 skb_checksum_none_assert(skb); 1027 if (bp->dev->features & NETIF_F_RXCSUM && 1028 !(bp->dev->flags & IFF_PROMISC) && 1029 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1030 skb->ip_summed = CHECKSUM_UNNECESSARY; 1031 1032 bp->dev->stats.rx_packets++; 1033 queue->stats.rx_packets++; 1034 bp->dev->stats.rx_bytes += skb->len; 1035 queue->stats.rx_bytes += skb->len; 1036 1037 gem_ptp_do_rxstamp(bp, skb, desc); 1038 1039 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1040 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1041 skb->len, skb->csum); 1042 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1043 skb_mac_header(skb), 16, true); 1044 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1045 skb->data, 32, true); 1046 #endif 1047 1048 netif_receive_skb(skb); 1049 } 1050 1051 gem_rx_refill(queue); 1052 1053 return count; 1054 } 1055 1056 static int macb_rx_frame(struct macb_queue *queue, unsigned int first_frag, 1057 unsigned int last_frag) 1058 { 1059 unsigned int len; 1060 unsigned int frag; 1061 unsigned int offset; 1062 struct sk_buff *skb; 1063 struct macb_dma_desc *desc; 1064 struct macb *bp = queue->bp; 1065 1066 desc = macb_rx_desc(queue, last_frag); 1067 len = desc->ctrl & bp->rx_frm_len_mask; 1068 1069 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1070 macb_rx_ring_wrap(bp, first_frag), 1071 macb_rx_ring_wrap(bp, last_frag), len); 1072 1073 /* The ethernet header starts NET_IP_ALIGN bytes into the 1074 * first buffer. Since the header is 14 bytes, this makes the 1075 * payload word-aligned. 1076 * 1077 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1078 * the two padding bytes into the skb so that we avoid hitting 1079 * the slowpath in memcpy(), and pull them off afterwards. 1080 */ 1081 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1082 if (!skb) { 1083 bp->dev->stats.rx_dropped++; 1084 for (frag = first_frag; ; frag++) { 1085 desc = macb_rx_desc(queue, frag); 1086 desc->addr &= ~MACB_BIT(RX_USED); 1087 if (frag == last_frag) 1088 break; 1089 } 1090 1091 /* Make descriptor updates visible to hardware */ 1092 wmb(); 1093 1094 return 1; 1095 } 1096 1097 offset = 0; 1098 len += NET_IP_ALIGN; 1099 skb_checksum_none_assert(skb); 1100 skb_put(skb, len); 1101 1102 for (frag = first_frag; ; frag++) { 1103 unsigned int frag_len = bp->rx_buffer_size; 1104 1105 if (offset + frag_len > len) { 1106 if (unlikely(frag != last_frag)) { 1107 dev_kfree_skb_any(skb); 1108 return -1; 1109 } 1110 frag_len = len - offset; 1111 } 1112 skb_copy_to_linear_data_offset(skb, offset, 1113 macb_rx_buffer(queue, frag), 1114 frag_len); 1115 offset += bp->rx_buffer_size; 1116 desc = macb_rx_desc(queue, frag); 1117 desc->addr &= ~MACB_BIT(RX_USED); 1118 1119 if (frag == last_frag) 1120 break; 1121 } 1122 1123 /* Make descriptor updates visible to hardware */ 1124 wmb(); 1125 1126 __skb_pull(skb, NET_IP_ALIGN); 1127 skb->protocol = eth_type_trans(skb, bp->dev); 1128 1129 bp->dev->stats.rx_packets++; 1130 bp->dev->stats.rx_bytes += skb->len; 1131 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1132 skb->len, skb->csum); 1133 netif_receive_skb(skb); 1134 1135 return 0; 1136 } 1137 1138 static inline void macb_init_rx_ring(struct macb_queue *queue) 1139 { 1140 struct macb *bp = queue->bp; 1141 dma_addr_t addr; 1142 struct macb_dma_desc *desc = NULL; 1143 int i; 1144 1145 addr = queue->rx_buffers_dma; 1146 for (i = 0; i < bp->rx_ring_size; i++) { 1147 desc = macb_rx_desc(queue, i); 1148 macb_set_addr(bp, desc, addr); 1149 desc->ctrl = 0; 1150 addr += bp->rx_buffer_size; 1151 } 1152 desc->addr |= MACB_BIT(RX_WRAP); 1153 queue->rx_tail = 0; 1154 } 1155 1156 static int macb_rx(struct macb_queue *queue, int budget) 1157 { 1158 struct macb *bp = queue->bp; 1159 bool reset_rx_queue = false; 1160 int received = 0; 1161 unsigned int tail; 1162 int first_frag = -1; 1163 1164 for (tail = queue->rx_tail; budget > 0; tail++) { 1165 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1166 u32 ctrl; 1167 1168 /* Make hw descriptor updates visible to CPU */ 1169 rmb(); 1170 1171 ctrl = desc->ctrl; 1172 1173 if (!(desc->addr & MACB_BIT(RX_USED))) 1174 break; 1175 1176 if (ctrl & MACB_BIT(RX_SOF)) { 1177 if (first_frag != -1) 1178 discard_partial_frame(queue, first_frag, tail); 1179 first_frag = tail; 1180 } 1181 1182 if (ctrl & MACB_BIT(RX_EOF)) { 1183 int dropped; 1184 1185 if (unlikely(first_frag == -1)) { 1186 reset_rx_queue = true; 1187 continue; 1188 } 1189 1190 dropped = macb_rx_frame(queue, first_frag, tail); 1191 first_frag = -1; 1192 if (unlikely(dropped < 0)) { 1193 reset_rx_queue = true; 1194 continue; 1195 } 1196 if (!dropped) { 1197 received++; 1198 budget--; 1199 } 1200 } 1201 } 1202 1203 if (unlikely(reset_rx_queue)) { 1204 unsigned long flags; 1205 u32 ctrl; 1206 1207 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1208 1209 spin_lock_irqsave(&bp->lock, flags); 1210 1211 ctrl = macb_readl(bp, NCR); 1212 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1213 1214 macb_init_rx_ring(queue); 1215 queue_writel(queue, RBQP, queue->rx_ring_dma); 1216 1217 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1218 1219 spin_unlock_irqrestore(&bp->lock, flags); 1220 return received; 1221 } 1222 1223 if (first_frag != -1) 1224 queue->rx_tail = first_frag; 1225 else 1226 queue->rx_tail = tail; 1227 1228 return received; 1229 } 1230 1231 static int macb_poll(struct napi_struct *napi, int budget) 1232 { 1233 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1234 struct macb *bp = queue->bp; 1235 int work_done; 1236 u32 status; 1237 1238 status = macb_readl(bp, RSR); 1239 macb_writel(bp, RSR, status); 1240 1241 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1242 (unsigned long)status, budget); 1243 1244 work_done = bp->macbgem_ops.mog_rx(queue, budget); 1245 if (work_done < budget) { 1246 napi_complete_done(napi, work_done); 1247 1248 /* Packets received while interrupts were disabled */ 1249 status = macb_readl(bp, RSR); 1250 if (status) { 1251 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1252 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1253 napi_reschedule(napi); 1254 } else { 1255 queue_writel(queue, IER, MACB_RX_INT_FLAGS); 1256 } 1257 } 1258 1259 /* TODO: Handle errors */ 1260 1261 return work_done; 1262 } 1263 1264 static void macb_hresp_error_task(unsigned long data) 1265 { 1266 struct macb *bp = (struct macb *)data; 1267 struct net_device *dev = bp->dev; 1268 struct macb_queue *queue = bp->queues; 1269 unsigned int q; 1270 u32 ctrl; 1271 1272 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1273 queue_writel(queue, IDR, MACB_RX_INT_FLAGS | 1274 MACB_TX_INT_FLAGS | 1275 MACB_BIT(HRESP)); 1276 } 1277 ctrl = macb_readl(bp, NCR); 1278 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1279 macb_writel(bp, NCR, ctrl); 1280 1281 netif_tx_stop_all_queues(dev); 1282 netif_carrier_off(dev); 1283 1284 bp->macbgem_ops.mog_init_rings(bp); 1285 1286 /* Initialize TX and RX buffers */ 1287 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1288 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 1289 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1290 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1291 queue_writel(queue, RBQPH, 1292 upper_32_bits(queue->rx_ring_dma)); 1293 #endif 1294 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1295 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1296 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1297 queue_writel(queue, TBQPH, 1298 upper_32_bits(queue->tx_ring_dma)); 1299 #endif 1300 1301 /* Enable interrupts */ 1302 queue_writel(queue, IER, 1303 MACB_RX_INT_FLAGS | 1304 MACB_TX_INT_FLAGS | 1305 MACB_BIT(HRESP)); 1306 } 1307 1308 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1309 macb_writel(bp, NCR, ctrl); 1310 1311 netif_carrier_on(dev); 1312 netif_tx_start_all_queues(dev); 1313 } 1314 1315 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1316 { 1317 struct macb_queue *queue = dev_id; 1318 struct macb *bp = queue->bp; 1319 struct net_device *dev = bp->dev; 1320 u32 status, ctrl; 1321 1322 status = queue_readl(queue, ISR); 1323 1324 if (unlikely(!status)) 1325 return IRQ_NONE; 1326 1327 spin_lock(&bp->lock); 1328 1329 while (status) { 1330 /* close possible race with dev_close */ 1331 if (unlikely(!netif_running(dev))) { 1332 queue_writel(queue, IDR, -1); 1333 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1334 queue_writel(queue, ISR, -1); 1335 break; 1336 } 1337 1338 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1339 (unsigned int)(queue - bp->queues), 1340 (unsigned long)status); 1341 1342 if (status & MACB_RX_INT_FLAGS) { 1343 /* There's no point taking any more interrupts 1344 * until we have processed the buffers. The 1345 * scheduling call may fail if the poll routine 1346 * is already scheduled, so disable interrupts 1347 * now. 1348 */ 1349 queue_writel(queue, IDR, MACB_RX_INT_FLAGS); 1350 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1351 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1352 1353 if (napi_schedule_prep(&queue->napi)) { 1354 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1355 __napi_schedule(&queue->napi); 1356 } 1357 } 1358 1359 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1360 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1361 schedule_work(&queue->tx_error_task); 1362 1363 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1364 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1365 1366 break; 1367 } 1368 1369 if (status & MACB_BIT(TCOMP)) 1370 macb_tx_interrupt(queue); 1371 1372 /* Link change detection isn't possible with RMII, so we'll 1373 * add that if/when we get our hands on a full-blown MII PHY. 1374 */ 1375 1376 /* There is a hardware issue under heavy load where DMA can 1377 * stop, this causes endless "used buffer descriptor read" 1378 * interrupts but it can be cleared by re-enabling RX. See 1379 * the at91 manual, section 41.3.1 or the Zynq manual 1380 * section 16.7.4 for details. 1381 */ 1382 if (status & MACB_BIT(RXUBR)) { 1383 ctrl = macb_readl(bp, NCR); 1384 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1385 wmb(); 1386 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1387 1388 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1389 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1390 } 1391 1392 if (status & MACB_BIT(ISR_ROVR)) { 1393 /* We missed at least one packet */ 1394 if (macb_is_gem(bp)) 1395 bp->hw_stats.gem.rx_overruns++; 1396 else 1397 bp->hw_stats.macb.rx_overruns++; 1398 1399 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1400 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1401 } 1402 1403 if (status & MACB_BIT(HRESP)) { 1404 tasklet_schedule(&bp->hresp_err_tasklet); 1405 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1406 1407 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1408 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1409 } 1410 status = queue_readl(queue, ISR); 1411 } 1412 1413 spin_unlock(&bp->lock); 1414 1415 return IRQ_HANDLED; 1416 } 1417 1418 #ifdef CONFIG_NET_POLL_CONTROLLER 1419 /* Polling receive - used by netconsole and other diagnostic tools 1420 * to allow network i/o with interrupts disabled. 1421 */ 1422 static void macb_poll_controller(struct net_device *dev) 1423 { 1424 struct macb *bp = netdev_priv(dev); 1425 struct macb_queue *queue; 1426 unsigned long flags; 1427 unsigned int q; 1428 1429 local_irq_save(flags); 1430 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1431 macb_interrupt(dev->irq, queue); 1432 local_irq_restore(flags); 1433 } 1434 #endif 1435 1436 static unsigned int macb_tx_map(struct macb *bp, 1437 struct macb_queue *queue, 1438 struct sk_buff *skb, 1439 unsigned int hdrlen) 1440 { 1441 dma_addr_t mapping; 1442 unsigned int len, entry, i, tx_head = queue->tx_head; 1443 struct macb_tx_skb *tx_skb = NULL; 1444 struct macb_dma_desc *desc; 1445 unsigned int offset, size, count = 0; 1446 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1447 unsigned int eof = 1, mss_mfs = 0; 1448 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1449 1450 /* LSO */ 1451 if (skb_shinfo(skb)->gso_size != 0) { 1452 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1453 /* UDP - UFO */ 1454 lso_ctrl = MACB_LSO_UFO_ENABLE; 1455 else 1456 /* TCP - TSO */ 1457 lso_ctrl = MACB_LSO_TSO_ENABLE; 1458 } 1459 1460 /* First, map non-paged data */ 1461 len = skb_headlen(skb); 1462 1463 /* first buffer length */ 1464 size = hdrlen; 1465 1466 offset = 0; 1467 while (len) { 1468 entry = macb_tx_ring_wrap(bp, tx_head); 1469 tx_skb = &queue->tx_skb[entry]; 1470 1471 mapping = dma_map_single(&bp->pdev->dev, 1472 skb->data + offset, 1473 size, DMA_TO_DEVICE); 1474 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1475 goto dma_error; 1476 1477 /* Save info to properly release resources */ 1478 tx_skb->skb = NULL; 1479 tx_skb->mapping = mapping; 1480 tx_skb->size = size; 1481 tx_skb->mapped_as_page = false; 1482 1483 len -= size; 1484 offset += size; 1485 count++; 1486 tx_head++; 1487 1488 size = min(len, bp->max_tx_length); 1489 } 1490 1491 /* Then, map paged data from fragments */ 1492 for (f = 0; f < nr_frags; f++) { 1493 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1494 1495 len = skb_frag_size(frag); 1496 offset = 0; 1497 while (len) { 1498 size = min(len, bp->max_tx_length); 1499 entry = macb_tx_ring_wrap(bp, tx_head); 1500 tx_skb = &queue->tx_skb[entry]; 1501 1502 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1503 offset, size, DMA_TO_DEVICE); 1504 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1505 goto dma_error; 1506 1507 /* Save info to properly release resources */ 1508 tx_skb->skb = NULL; 1509 tx_skb->mapping = mapping; 1510 tx_skb->size = size; 1511 tx_skb->mapped_as_page = true; 1512 1513 len -= size; 1514 offset += size; 1515 count++; 1516 tx_head++; 1517 } 1518 } 1519 1520 /* Should never happen */ 1521 if (unlikely(!tx_skb)) { 1522 netdev_err(bp->dev, "BUG! empty skb!\n"); 1523 return 0; 1524 } 1525 1526 /* This is the last buffer of the frame: save socket buffer */ 1527 tx_skb->skb = skb; 1528 1529 /* Update TX ring: update buffer descriptors in reverse order 1530 * to avoid race condition 1531 */ 1532 1533 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1534 * to set the end of TX queue 1535 */ 1536 i = tx_head; 1537 entry = macb_tx_ring_wrap(bp, i); 1538 ctrl = MACB_BIT(TX_USED); 1539 desc = macb_tx_desc(queue, entry); 1540 desc->ctrl = ctrl; 1541 1542 if (lso_ctrl) { 1543 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1544 /* include header and FCS in value given to h/w */ 1545 mss_mfs = skb_shinfo(skb)->gso_size + 1546 skb_transport_offset(skb) + 1547 ETH_FCS_LEN; 1548 else /* TSO */ { 1549 mss_mfs = skb_shinfo(skb)->gso_size; 1550 /* TCP Sequence Number Source Select 1551 * can be set only for TSO 1552 */ 1553 seq_ctrl = 0; 1554 } 1555 } 1556 1557 do { 1558 i--; 1559 entry = macb_tx_ring_wrap(bp, i); 1560 tx_skb = &queue->tx_skb[entry]; 1561 desc = macb_tx_desc(queue, entry); 1562 1563 ctrl = (u32)tx_skb->size; 1564 if (eof) { 1565 ctrl |= MACB_BIT(TX_LAST); 1566 eof = 0; 1567 } 1568 if (unlikely(entry == (bp->tx_ring_size - 1))) 1569 ctrl |= MACB_BIT(TX_WRAP); 1570 1571 /* First descriptor is header descriptor */ 1572 if (i == queue->tx_head) { 1573 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1574 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1575 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1576 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1577 ctrl |= MACB_BIT(TX_NOCRC); 1578 } else 1579 /* Only set MSS/MFS on payload descriptors 1580 * (second or later descriptor) 1581 */ 1582 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1583 1584 /* Set TX buffer descriptor */ 1585 macb_set_addr(bp, desc, tx_skb->mapping); 1586 /* desc->addr must be visible to hardware before clearing 1587 * 'TX_USED' bit in desc->ctrl. 1588 */ 1589 wmb(); 1590 desc->ctrl = ctrl; 1591 } while (i != queue->tx_head); 1592 1593 queue->tx_head = tx_head; 1594 1595 return count; 1596 1597 dma_error: 1598 netdev_err(bp->dev, "TX DMA map failed\n"); 1599 1600 for (i = queue->tx_head; i != tx_head; i++) { 1601 tx_skb = macb_tx_skb(queue, i); 1602 1603 macb_tx_unmap(bp, tx_skb); 1604 } 1605 1606 return 0; 1607 } 1608 1609 static netdev_features_t macb_features_check(struct sk_buff *skb, 1610 struct net_device *dev, 1611 netdev_features_t features) 1612 { 1613 unsigned int nr_frags, f; 1614 unsigned int hdrlen; 1615 1616 /* Validate LSO compatibility */ 1617 1618 /* there is only one buffer */ 1619 if (!skb_is_nonlinear(skb)) 1620 return features; 1621 1622 /* length of header */ 1623 hdrlen = skb_transport_offset(skb); 1624 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1625 hdrlen += tcp_hdrlen(skb); 1626 1627 /* For LSO: 1628 * When software supplies two or more payload buffers all payload buffers 1629 * apart from the last must be a multiple of 8 bytes in size. 1630 */ 1631 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1632 return features & ~MACB_NETIF_LSO; 1633 1634 nr_frags = skb_shinfo(skb)->nr_frags; 1635 /* No need to check last fragment */ 1636 nr_frags--; 1637 for (f = 0; f < nr_frags; f++) { 1638 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1639 1640 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1641 return features & ~MACB_NETIF_LSO; 1642 } 1643 return features; 1644 } 1645 1646 static inline int macb_clear_csum(struct sk_buff *skb) 1647 { 1648 /* no change for packets without checksum offloading */ 1649 if (skb->ip_summed != CHECKSUM_PARTIAL) 1650 return 0; 1651 1652 /* make sure we can modify the header */ 1653 if (unlikely(skb_cow_head(skb, 0))) 1654 return -1; 1655 1656 /* initialize checksum field 1657 * This is required - at least for Zynq, which otherwise calculates 1658 * wrong UDP header checksums for UDP packets with UDP data len <=2 1659 */ 1660 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1661 return 0; 1662 } 1663 1664 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1665 { 1666 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1667 int padlen = ETH_ZLEN - (*skb)->len; 1668 int headroom = skb_headroom(*skb); 1669 int tailroom = skb_tailroom(*skb); 1670 struct sk_buff *nskb; 1671 u32 fcs; 1672 1673 if (!(ndev->features & NETIF_F_HW_CSUM) || 1674 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1675 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1676 return 0; 1677 1678 if (padlen <= 0) { 1679 /* FCS could be appeded to tailroom. */ 1680 if (tailroom >= ETH_FCS_LEN) 1681 goto add_fcs; 1682 /* FCS could be appeded by moving data to headroom. */ 1683 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1684 padlen = 0; 1685 /* No room for FCS, need to reallocate skb. */ 1686 else 1687 padlen = ETH_FCS_LEN - tailroom; 1688 } else { 1689 /* Add room for FCS. */ 1690 padlen += ETH_FCS_LEN; 1691 } 1692 1693 if (!cloned && headroom + tailroom >= padlen) { 1694 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1695 skb_set_tail_pointer(*skb, (*skb)->len); 1696 } else { 1697 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1698 if (!nskb) 1699 return -ENOMEM; 1700 1701 dev_kfree_skb_any(*skb); 1702 *skb = nskb; 1703 } 1704 1705 if (padlen) { 1706 if (padlen >= ETH_FCS_LEN) 1707 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1708 else 1709 skb_trim(*skb, ETH_FCS_LEN - padlen); 1710 } 1711 1712 add_fcs: 1713 /* set FCS to packet */ 1714 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1715 fcs = ~fcs; 1716 1717 skb_put_u8(*skb, fcs & 0xff); 1718 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1719 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1720 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1721 1722 return 0; 1723 } 1724 1725 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1726 { 1727 u16 queue_index = skb_get_queue_mapping(skb); 1728 struct macb *bp = netdev_priv(dev); 1729 struct macb_queue *queue = &bp->queues[queue_index]; 1730 unsigned long flags; 1731 unsigned int desc_cnt, nr_frags, frag_size, f; 1732 unsigned int hdrlen; 1733 bool is_lso, is_udp = 0; 1734 netdev_tx_t ret = NETDEV_TX_OK; 1735 1736 if (macb_clear_csum(skb)) { 1737 dev_kfree_skb_any(skb); 1738 return ret; 1739 } 1740 1741 if (macb_pad_and_fcs(&skb, dev)) { 1742 dev_kfree_skb_any(skb); 1743 return ret; 1744 } 1745 1746 is_lso = (skb_shinfo(skb)->gso_size != 0); 1747 1748 if (is_lso) { 1749 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1750 1751 /* length of headers */ 1752 if (is_udp) 1753 /* only queue eth + ip headers separately for UDP */ 1754 hdrlen = skb_transport_offset(skb); 1755 else 1756 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1757 if (skb_headlen(skb) < hdrlen) { 1758 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1759 /* if this is required, would need to copy to single buffer */ 1760 return NETDEV_TX_BUSY; 1761 } 1762 } else 1763 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1764 1765 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1766 netdev_vdbg(bp->dev, 1767 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1768 queue_index, skb->len, skb->head, skb->data, 1769 skb_tail_pointer(skb), skb_end_pointer(skb)); 1770 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1771 skb->data, 16, true); 1772 #endif 1773 1774 /* Count how many TX buffer descriptors are needed to send this 1775 * socket buffer: skb fragments of jumbo frames may need to be 1776 * split into many buffer descriptors. 1777 */ 1778 if (is_lso && (skb_headlen(skb) > hdrlen)) 1779 /* extra header descriptor if also payload in first buffer */ 1780 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1781 else 1782 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1783 nr_frags = skb_shinfo(skb)->nr_frags; 1784 for (f = 0; f < nr_frags; f++) { 1785 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1786 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1787 } 1788 1789 spin_lock_irqsave(&bp->lock, flags); 1790 1791 /* This is a hard error, log it. */ 1792 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1793 bp->tx_ring_size) < desc_cnt) { 1794 netif_stop_subqueue(dev, queue_index); 1795 spin_unlock_irqrestore(&bp->lock, flags); 1796 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1797 queue->tx_head, queue->tx_tail); 1798 return NETDEV_TX_BUSY; 1799 } 1800 1801 /* Map socket buffer for DMA transfer */ 1802 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1803 dev_kfree_skb_any(skb); 1804 goto unlock; 1805 } 1806 1807 /* Make newly initialized descriptor visible to hardware */ 1808 wmb(); 1809 skb_tx_timestamp(skb); 1810 1811 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1812 1813 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1814 netif_stop_subqueue(dev, queue_index); 1815 1816 unlock: 1817 spin_unlock_irqrestore(&bp->lock, flags); 1818 1819 return ret; 1820 } 1821 1822 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1823 { 1824 if (!macb_is_gem(bp)) { 1825 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1826 } else { 1827 bp->rx_buffer_size = size; 1828 1829 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1830 netdev_dbg(bp->dev, 1831 "RX buffer must be multiple of %d bytes, expanding\n", 1832 RX_BUFFER_MULTIPLE); 1833 bp->rx_buffer_size = 1834 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1835 } 1836 } 1837 1838 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1839 bp->dev->mtu, bp->rx_buffer_size); 1840 } 1841 1842 static void gem_free_rx_buffers(struct macb *bp) 1843 { 1844 struct sk_buff *skb; 1845 struct macb_dma_desc *desc; 1846 struct macb_queue *queue; 1847 dma_addr_t addr; 1848 unsigned int q; 1849 int i; 1850 1851 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1852 if (!queue->rx_skbuff) 1853 continue; 1854 1855 for (i = 0; i < bp->rx_ring_size; i++) { 1856 skb = queue->rx_skbuff[i]; 1857 1858 if (!skb) 1859 continue; 1860 1861 desc = macb_rx_desc(queue, i); 1862 addr = macb_get_addr(bp, desc); 1863 1864 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1865 DMA_FROM_DEVICE); 1866 dev_kfree_skb_any(skb); 1867 skb = NULL; 1868 } 1869 1870 kfree(queue->rx_skbuff); 1871 queue->rx_skbuff = NULL; 1872 } 1873 } 1874 1875 static void macb_free_rx_buffers(struct macb *bp) 1876 { 1877 struct macb_queue *queue = &bp->queues[0]; 1878 1879 if (queue->rx_buffers) { 1880 dma_free_coherent(&bp->pdev->dev, 1881 bp->rx_ring_size * bp->rx_buffer_size, 1882 queue->rx_buffers, queue->rx_buffers_dma); 1883 queue->rx_buffers = NULL; 1884 } 1885 } 1886 1887 static void macb_free_consistent(struct macb *bp) 1888 { 1889 struct macb_queue *queue; 1890 unsigned int q; 1891 int size; 1892 1893 bp->macbgem_ops.mog_free_rx_buffers(bp); 1894 1895 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1896 kfree(queue->tx_skb); 1897 queue->tx_skb = NULL; 1898 if (queue->tx_ring) { 1899 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1900 dma_free_coherent(&bp->pdev->dev, size, 1901 queue->tx_ring, queue->tx_ring_dma); 1902 queue->tx_ring = NULL; 1903 } 1904 if (queue->rx_ring) { 1905 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 1906 dma_free_coherent(&bp->pdev->dev, size, 1907 queue->rx_ring, queue->rx_ring_dma); 1908 queue->rx_ring = NULL; 1909 } 1910 } 1911 } 1912 1913 static int gem_alloc_rx_buffers(struct macb *bp) 1914 { 1915 struct macb_queue *queue; 1916 unsigned int q; 1917 int size; 1918 1919 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1920 size = bp->rx_ring_size * sizeof(struct sk_buff *); 1921 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 1922 if (!queue->rx_skbuff) 1923 return -ENOMEM; 1924 else 1925 netdev_dbg(bp->dev, 1926 "Allocated %d RX struct sk_buff entries at %p\n", 1927 bp->rx_ring_size, queue->rx_skbuff); 1928 } 1929 return 0; 1930 } 1931 1932 static int macb_alloc_rx_buffers(struct macb *bp) 1933 { 1934 struct macb_queue *queue = &bp->queues[0]; 1935 int size; 1936 1937 size = bp->rx_ring_size * bp->rx_buffer_size; 1938 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 1939 &queue->rx_buffers_dma, GFP_KERNEL); 1940 if (!queue->rx_buffers) 1941 return -ENOMEM; 1942 1943 netdev_dbg(bp->dev, 1944 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 1945 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 1946 return 0; 1947 } 1948 1949 static int macb_alloc_consistent(struct macb *bp) 1950 { 1951 struct macb_queue *queue; 1952 unsigned int q; 1953 int size; 1954 1955 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1956 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 1957 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1958 &queue->tx_ring_dma, 1959 GFP_KERNEL); 1960 if (!queue->tx_ring) 1961 goto out_err; 1962 netdev_dbg(bp->dev, 1963 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 1964 q, size, (unsigned long)queue->tx_ring_dma, 1965 queue->tx_ring); 1966 1967 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 1968 queue->tx_skb = kmalloc(size, GFP_KERNEL); 1969 if (!queue->tx_skb) 1970 goto out_err; 1971 1972 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 1973 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1974 &queue->rx_ring_dma, GFP_KERNEL); 1975 if (!queue->rx_ring) 1976 goto out_err; 1977 netdev_dbg(bp->dev, 1978 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 1979 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 1980 } 1981 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 1982 goto out_err; 1983 1984 return 0; 1985 1986 out_err: 1987 macb_free_consistent(bp); 1988 return -ENOMEM; 1989 } 1990 1991 static void gem_init_rings(struct macb *bp) 1992 { 1993 struct macb_queue *queue; 1994 struct macb_dma_desc *desc = NULL; 1995 unsigned int q; 1996 int i; 1997 1998 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1999 for (i = 0; i < bp->tx_ring_size; i++) { 2000 desc = macb_tx_desc(queue, i); 2001 macb_set_addr(bp, desc, 0); 2002 desc->ctrl = MACB_BIT(TX_USED); 2003 } 2004 desc->ctrl |= MACB_BIT(TX_WRAP); 2005 queue->tx_head = 0; 2006 queue->tx_tail = 0; 2007 2008 queue->rx_tail = 0; 2009 queue->rx_prepared_head = 0; 2010 2011 gem_rx_refill(queue); 2012 } 2013 2014 } 2015 2016 static void macb_init_rings(struct macb *bp) 2017 { 2018 int i; 2019 struct macb_dma_desc *desc = NULL; 2020 2021 macb_init_rx_ring(&bp->queues[0]); 2022 2023 for (i = 0; i < bp->tx_ring_size; i++) { 2024 desc = macb_tx_desc(&bp->queues[0], i); 2025 macb_set_addr(bp, desc, 0); 2026 desc->ctrl = MACB_BIT(TX_USED); 2027 } 2028 bp->queues[0].tx_head = 0; 2029 bp->queues[0].tx_tail = 0; 2030 desc->ctrl |= MACB_BIT(TX_WRAP); 2031 } 2032 2033 static void macb_reset_hw(struct macb *bp) 2034 { 2035 struct macb_queue *queue; 2036 unsigned int q; 2037 u32 ctrl = macb_readl(bp, NCR); 2038 2039 /* Disable RX and TX (XXX: Should we halt the transmission 2040 * more gracefully?) 2041 */ 2042 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2043 2044 /* Clear the stats registers (XXX: Update stats first?) */ 2045 ctrl |= MACB_BIT(CLRSTAT); 2046 2047 macb_writel(bp, NCR, ctrl); 2048 2049 /* Clear all status flags */ 2050 macb_writel(bp, TSR, -1); 2051 macb_writel(bp, RSR, -1); 2052 2053 /* Disable all interrupts */ 2054 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2055 queue_writel(queue, IDR, -1); 2056 queue_readl(queue, ISR); 2057 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2058 queue_writel(queue, ISR, -1); 2059 } 2060 } 2061 2062 static u32 gem_mdc_clk_div(struct macb *bp) 2063 { 2064 u32 config; 2065 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2066 2067 if (pclk_hz <= 20000000) 2068 config = GEM_BF(CLK, GEM_CLK_DIV8); 2069 else if (pclk_hz <= 40000000) 2070 config = GEM_BF(CLK, GEM_CLK_DIV16); 2071 else if (pclk_hz <= 80000000) 2072 config = GEM_BF(CLK, GEM_CLK_DIV32); 2073 else if (pclk_hz <= 120000000) 2074 config = GEM_BF(CLK, GEM_CLK_DIV48); 2075 else if (pclk_hz <= 160000000) 2076 config = GEM_BF(CLK, GEM_CLK_DIV64); 2077 else 2078 config = GEM_BF(CLK, GEM_CLK_DIV96); 2079 2080 return config; 2081 } 2082 2083 static u32 macb_mdc_clk_div(struct macb *bp) 2084 { 2085 u32 config; 2086 unsigned long pclk_hz; 2087 2088 if (macb_is_gem(bp)) 2089 return gem_mdc_clk_div(bp); 2090 2091 pclk_hz = clk_get_rate(bp->pclk); 2092 if (pclk_hz <= 20000000) 2093 config = MACB_BF(CLK, MACB_CLK_DIV8); 2094 else if (pclk_hz <= 40000000) 2095 config = MACB_BF(CLK, MACB_CLK_DIV16); 2096 else if (pclk_hz <= 80000000) 2097 config = MACB_BF(CLK, MACB_CLK_DIV32); 2098 else 2099 config = MACB_BF(CLK, MACB_CLK_DIV64); 2100 2101 return config; 2102 } 2103 2104 /* Get the DMA bus width field of the network configuration register that we 2105 * should program. We find the width from decoding the design configuration 2106 * register to find the maximum supported data bus width. 2107 */ 2108 static u32 macb_dbw(struct macb *bp) 2109 { 2110 if (!macb_is_gem(bp)) 2111 return 0; 2112 2113 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2114 case 4: 2115 return GEM_BF(DBW, GEM_DBW128); 2116 case 2: 2117 return GEM_BF(DBW, GEM_DBW64); 2118 case 1: 2119 default: 2120 return GEM_BF(DBW, GEM_DBW32); 2121 } 2122 } 2123 2124 /* Configure the receive DMA engine 2125 * - use the correct receive buffer size 2126 * - set best burst length for DMA operations 2127 * (if not supported by FIFO, it will fallback to default) 2128 * - set both rx/tx packet buffers to full memory size 2129 * These are configurable parameters for GEM. 2130 */ 2131 static void macb_configure_dma(struct macb *bp) 2132 { 2133 struct macb_queue *queue; 2134 u32 buffer_size; 2135 unsigned int q; 2136 u32 dmacfg; 2137 2138 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2139 if (macb_is_gem(bp)) { 2140 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2141 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2142 if (q) 2143 queue_writel(queue, RBQS, buffer_size); 2144 else 2145 dmacfg |= GEM_BF(RXBS, buffer_size); 2146 } 2147 if (bp->dma_burst_length) 2148 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2149 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2150 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2151 2152 if (bp->native_io) 2153 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2154 else 2155 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2156 2157 if (bp->dev->features & NETIF_F_HW_CSUM) 2158 dmacfg |= GEM_BIT(TXCOEN); 2159 else 2160 dmacfg &= ~GEM_BIT(TXCOEN); 2161 2162 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2163 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2164 dmacfg |= GEM_BIT(ADDR64); 2165 #endif 2166 #ifdef CONFIG_MACB_USE_HWSTAMP 2167 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2168 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2169 #endif 2170 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2171 dmacfg); 2172 gem_writel(bp, DMACFG, dmacfg); 2173 } 2174 } 2175 2176 static void macb_init_hw(struct macb *bp) 2177 { 2178 struct macb_queue *queue; 2179 unsigned int q; 2180 2181 u32 config; 2182 2183 macb_reset_hw(bp); 2184 macb_set_hwaddr(bp); 2185 2186 config = macb_mdc_clk_div(bp); 2187 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2188 config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2189 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2190 config |= MACB_BIT(PAE); /* PAuse Enable */ 2191 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2192 if (bp->caps & MACB_CAPS_JUMBO) 2193 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2194 else 2195 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2196 if (bp->dev->flags & IFF_PROMISC) 2197 config |= MACB_BIT(CAF); /* Copy All Frames */ 2198 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2199 config |= GEM_BIT(RXCOEN); 2200 if (!(bp->dev->flags & IFF_BROADCAST)) 2201 config |= MACB_BIT(NBC); /* No BroadCast */ 2202 config |= macb_dbw(bp); 2203 macb_writel(bp, NCFGR, config); 2204 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2205 gem_writel(bp, JML, bp->jumbo_max_len); 2206 bp->speed = SPEED_10; 2207 bp->duplex = DUPLEX_HALF; 2208 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2209 if (bp->caps & MACB_CAPS_JUMBO) 2210 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2211 2212 macb_configure_dma(bp); 2213 2214 /* Initialize TX and RX buffers */ 2215 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2216 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 2217 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2218 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2219 queue_writel(queue, RBQPH, upper_32_bits(queue->rx_ring_dma)); 2220 #endif 2221 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 2222 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2223 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2224 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 2225 #endif 2226 2227 /* Enable interrupts */ 2228 queue_writel(queue, IER, 2229 MACB_RX_INT_FLAGS | 2230 MACB_TX_INT_FLAGS | 2231 MACB_BIT(HRESP)); 2232 } 2233 2234 /* Enable TX and RX */ 2235 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 2236 } 2237 2238 /* The hash address register is 64 bits long and takes up two 2239 * locations in the memory map. The least significant bits are stored 2240 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2241 * 2242 * The unicast hash enable and the multicast hash enable bits in the 2243 * network configuration register enable the reception of hash matched 2244 * frames. The destination address is reduced to a 6 bit index into 2245 * the 64 bit hash register using the following hash function. The 2246 * hash function is an exclusive or of every sixth bit of the 2247 * destination address. 2248 * 2249 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2250 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2251 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2252 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2253 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2254 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2255 * 2256 * da[0] represents the least significant bit of the first byte 2257 * received, that is, the multicast/unicast indicator, and da[47] 2258 * represents the most significant bit of the last byte received. If 2259 * the hash index, hi[n], points to a bit that is set in the hash 2260 * register then the frame will be matched according to whether the 2261 * frame is multicast or unicast. A multicast match will be signalled 2262 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2263 * index points to a bit set in the hash register. A unicast match 2264 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2265 * and the hash index points to a bit set in the hash register. To 2266 * receive all multicast frames, the hash register should be set with 2267 * all ones and the multicast hash enable bit should be set in the 2268 * network configuration register. 2269 */ 2270 2271 static inline int hash_bit_value(int bitnr, __u8 *addr) 2272 { 2273 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2274 return 1; 2275 return 0; 2276 } 2277 2278 /* Return the hash index value for the specified address. */ 2279 static int hash_get_index(__u8 *addr) 2280 { 2281 int i, j, bitval; 2282 int hash_index = 0; 2283 2284 for (j = 0; j < 6; j++) { 2285 for (i = 0, bitval = 0; i < 8; i++) 2286 bitval ^= hash_bit_value(i * 6 + j, addr); 2287 2288 hash_index |= (bitval << j); 2289 } 2290 2291 return hash_index; 2292 } 2293 2294 /* Add multicast addresses to the internal multicast-hash table. */ 2295 static void macb_sethashtable(struct net_device *dev) 2296 { 2297 struct netdev_hw_addr *ha; 2298 unsigned long mc_filter[2]; 2299 unsigned int bitnr; 2300 struct macb *bp = netdev_priv(dev); 2301 2302 mc_filter[0] = 0; 2303 mc_filter[1] = 0; 2304 2305 netdev_for_each_mc_addr(ha, dev) { 2306 bitnr = hash_get_index(ha->addr); 2307 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2308 } 2309 2310 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2311 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2312 } 2313 2314 /* Enable/Disable promiscuous and multicast modes. */ 2315 static void macb_set_rx_mode(struct net_device *dev) 2316 { 2317 unsigned long cfg; 2318 struct macb *bp = netdev_priv(dev); 2319 2320 cfg = macb_readl(bp, NCFGR); 2321 2322 if (dev->flags & IFF_PROMISC) { 2323 /* Enable promiscuous mode */ 2324 cfg |= MACB_BIT(CAF); 2325 2326 /* Disable RX checksum offload */ 2327 if (macb_is_gem(bp)) 2328 cfg &= ~GEM_BIT(RXCOEN); 2329 } else { 2330 /* Disable promiscuous mode */ 2331 cfg &= ~MACB_BIT(CAF); 2332 2333 /* Enable RX checksum offload only if requested */ 2334 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2335 cfg |= GEM_BIT(RXCOEN); 2336 } 2337 2338 if (dev->flags & IFF_ALLMULTI) { 2339 /* Enable all multicast mode */ 2340 macb_or_gem_writel(bp, HRB, -1); 2341 macb_or_gem_writel(bp, HRT, -1); 2342 cfg |= MACB_BIT(NCFGR_MTI); 2343 } else if (!netdev_mc_empty(dev)) { 2344 /* Enable specific multicasts */ 2345 macb_sethashtable(dev); 2346 cfg |= MACB_BIT(NCFGR_MTI); 2347 } else if (dev->flags & (~IFF_ALLMULTI)) { 2348 /* Disable all multicast mode */ 2349 macb_or_gem_writel(bp, HRB, 0); 2350 macb_or_gem_writel(bp, HRT, 0); 2351 cfg &= ~MACB_BIT(NCFGR_MTI); 2352 } 2353 2354 macb_writel(bp, NCFGR, cfg); 2355 } 2356 2357 static int macb_open(struct net_device *dev) 2358 { 2359 struct macb *bp = netdev_priv(dev); 2360 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2361 struct macb_queue *queue; 2362 unsigned int q; 2363 int err; 2364 2365 netdev_dbg(bp->dev, "open\n"); 2366 2367 /* carrier starts down */ 2368 netif_carrier_off(dev); 2369 2370 /* if the phy is not yet register, retry later*/ 2371 if (!dev->phydev) 2372 return -EAGAIN; 2373 2374 /* RX buffers initialization */ 2375 macb_init_rx_buffer_size(bp, bufsz); 2376 2377 err = macb_alloc_consistent(bp); 2378 if (err) { 2379 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2380 err); 2381 return err; 2382 } 2383 2384 bp->macbgem_ops.mog_init_rings(bp); 2385 macb_init_hw(bp); 2386 2387 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2388 napi_enable(&queue->napi); 2389 2390 /* schedule a link state check */ 2391 phy_start(dev->phydev); 2392 2393 netif_tx_start_all_queues(dev); 2394 2395 if (bp->ptp_info) 2396 bp->ptp_info->ptp_init(dev); 2397 2398 return 0; 2399 } 2400 2401 static int macb_close(struct net_device *dev) 2402 { 2403 struct macb *bp = netdev_priv(dev); 2404 struct macb_queue *queue; 2405 unsigned long flags; 2406 unsigned int q; 2407 2408 netif_tx_stop_all_queues(dev); 2409 2410 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2411 napi_disable(&queue->napi); 2412 2413 if (dev->phydev) 2414 phy_stop(dev->phydev); 2415 2416 spin_lock_irqsave(&bp->lock, flags); 2417 macb_reset_hw(bp); 2418 netif_carrier_off(dev); 2419 spin_unlock_irqrestore(&bp->lock, flags); 2420 2421 macb_free_consistent(bp); 2422 2423 if (bp->ptp_info) 2424 bp->ptp_info->ptp_remove(dev); 2425 2426 return 0; 2427 } 2428 2429 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2430 { 2431 if (netif_running(dev)) 2432 return -EBUSY; 2433 2434 dev->mtu = new_mtu; 2435 2436 return 0; 2437 } 2438 2439 static void gem_update_stats(struct macb *bp) 2440 { 2441 struct macb_queue *queue; 2442 unsigned int i, q, idx; 2443 unsigned long *stat; 2444 2445 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2446 2447 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2448 u32 offset = gem_statistics[i].offset; 2449 u64 val = bp->macb_reg_readl(bp, offset); 2450 2451 bp->ethtool_stats[i] += val; 2452 *p += val; 2453 2454 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2455 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2456 val = bp->macb_reg_readl(bp, offset + 4); 2457 bp->ethtool_stats[i] += ((u64)val) << 32; 2458 *(++p) += val; 2459 } 2460 } 2461 2462 idx = GEM_STATS_LEN; 2463 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2464 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2465 bp->ethtool_stats[idx++] = *stat; 2466 } 2467 2468 static struct net_device_stats *gem_get_stats(struct macb *bp) 2469 { 2470 struct gem_stats *hwstat = &bp->hw_stats.gem; 2471 struct net_device_stats *nstat = &bp->dev->stats; 2472 2473 gem_update_stats(bp); 2474 2475 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2476 hwstat->rx_alignment_errors + 2477 hwstat->rx_resource_errors + 2478 hwstat->rx_overruns + 2479 hwstat->rx_oversize_frames + 2480 hwstat->rx_jabbers + 2481 hwstat->rx_undersized_frames + 2482 hwstat->rx_length_field_frame_errors); 2483 nstat->tx_errors = (hwstat->tx_late_collisions + 2484 hwstat->tx_excessive_collisions + 2485 hwstat->tx_underrun + 2486 hwstat->tx_carrier_sense_errors); 2487 nstat->multicast = hwstat->rx_multicast_frames; 2488 nstat->collisions = (hwstat->tx_single_collision_frames + 2489 hwstat->tx_multiple_collision_frames + 2490 hwstat->tx_excessive_collisions); 2491 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2492 hwstat->rx_jabbers + 2493 hwstat->rx_undersized_frames + 2494 hwstat->rx_length_field_frame_errors); 2495 nstat->rx_over_errors = hwstat->rx_resource_errors; 2496 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2497 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2498 nstat->rx_fifo_errors = hwstat->rx_overruns; 2499 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2500 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2501 nstat->tx_fifo_errors = hwstat->tx_underrun; 2502 2503 return nstat; 2504 } 2505 2506 static void gem_get_ethtool_stats(struct net_device *dev, 2507 struct ethtool_stats *stats, u64 *data) 2508 { 2509 struct macb *bp; 2510 2511 bp = netdev_priv(dev); 2512 gem_update_stats(bp); 2513 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2514 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2515 } 2516 2517 static int gem_get_sset_count(struct net_device *dev, int sset) 2518 { 2519 struct macb *bp = netdev_priv(dev); 2520 2521 switch (sset) { 2522 case ETH_SS_STATS: 2523 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2524 default: 2525 return -EOPNOTSUPP; 2526 } 2527 } 2528 2529 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2530 { 2531 char stat_string[ETH_GSTRING_LEN]; 2532 struct macb *bp = netdev_priv(dev); 2533 struct macb_queue *queue; 2534 unsigned int i; 2535 unsigned int q; 2536 2537 switch (sset) { 2538 case ETH_SS_STATS: 2539 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2540 memcpy(p, gem_statistics[i].stat_string, 2541 ETH_GSTRING_LEN); 2542 2543 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2544 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2545 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2546 q, queue_statistics[i].stat_string); 2547 memcpy(p, stat_string, ETH_GSTRING_LEN); 2548 } 2549 } 2550 break; 2551 } 2552 } 2553 2554 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2555 { 2556 struct macb *bp = netdev_priv(dev); 2557 struct net_device_stats *nstat = &bp->dev->stats; 2558 struct macb_stats *hwstat = &bp->hw_stats.macb; 2559 2560 if (macb_is_gem(bp)) 2561 return gem_get_stats(bp); 2562 2563 /* read stats from hardware */ 2564 macb_update_stats(bp); 2565 2566 /* Convert HW stats into netdevice stats */ 2567 nstat->rx_errors = (hwstat->rx_fcs_errors + 2568 hwstat->rx_align_errors + 2569 hwstat->rx_resource_errors + 2570 hwstat->rx_overruns + 2571 hwstat->rx_oversize_pkts + 2572 hwstat->rx_jabbers + 2573 hwstat->rx_undersize_pkts + 2574 hwstat->rx_length_mismatch); 2575 nstat->tx_errors = (hwstat->tx_late_cols + 2576 hwstat->tx_excessive_cols + 2577 hwstat->tx_underruns + 2578 hwstat->tx_carrier_errors + 2579 hwstat->sqe_test_errors); 2580 nstat->collisions = (hwstat->tx_single_cols + 2581 hwstat->tx_multiple_cols + 2582 hwstat->tx_excessive_cols); 2583 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2584 hwstat->rx_jabbers + 2585 hwstat->rx_undersize_pkts + 2586 hwstat->rx_length_mismatch); 2587 nstat->rx_over_errors = hwstat->rx_resource_errors + 2588 hwstat->rx_overruns; 2589 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2590 nstat->rx_frame_errors = hwstat->rx_align_errors; 2591 nstat->rx_fifo_errors = hwstat->rx_overruns; 2592 /* XXX: What does "missed" mean? */ 2593 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2594 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2595 nstat->tx_fifo_errors = hwstat->tx_underruns; 2596 /* Don't know about heartbeat or window errors... */ 2597 2598 return nstat; 2599 } 2600 2601 static int macb_get_regs_len(struct net_device *netdev) 2602 { 2603 return MACB_GREGS_NBR * sizeof(u32); 2604 } 2605 2606 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2607 void *p) 2608 { 2609 struct macb *bp = netdev_priv(dev); 2610 unsigned int tail, head; 2611 u32 *regs_buff = p; 2612 2613 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2614 | MACB_GREGS_VERSION; 2615 2616 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2617 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2618 2619 regs_buff[0] = macb_readl(bp, NCR); 2620 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2621 regs_buff[2] = macb_readl(bp, NSR); 2622 regs_buff[3] = macb_readl(bp, TSR); 2623 regs_buff[4] = macb_readl(bp, RBQP); 2624 regs_buff[5] = macb_readl(bp, TBQP); 2625 regs_buff[6] = macb_readl(bp, RSR); 2626 regs_buff[7] = macb_readl(bp, IMR); 2627 2628 regs_buff[8] = tail; 2629 regs_buff[9] = head; 2630 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2631 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2632 2633 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2634 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2635 if (macb_is_gem(bp)) 2636 regs_buff[13] = gem_readl(bp, DMACFG); 2637 } 2638 2639 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2640 { 2641 struct macb *bp = netdev_priv(netdev); 2642 2643 wol->supported = 0; 2644 wol->wolopts = 0; 2645 2646 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2647 wol->supported = WAKE_MAGIC; 2648 2649 if (bp->wol & MACB_WOL_ENABLED) 2650 wol->wolopts |= WAKE_MAGIC; 2651 } 2652 } 2653 2654 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2655 { 2656 struct macb *bp = netdev_priv(netdev); 2657 2658 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2659 (wol->wolopts & ~WAKE_MAGIC)) 2660 return -EOPNOTSUPP; 2661 2662 if (wol->wolopts & WAKE_MAGIC) 2663 bp->wol |= MACB_WOL_ENABLED; 2664 else 2665 bp->wol &= ~MACB_WOL_ENABLED; 2666 2667 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2668 2669 return 0; 2670 } 2671 2672 static void macb_get_ringparam(struct net_device *netdev, 2673 struct ethtool_ringparam *ring) 2674 { 2675 struct macb *bp = netdev_priv(netdev); 2676 2677 ring->rx_max_pending = MAX_RX_RING_SIZE; 2678 ring->tx_max_pending = MAX_TX_RING_SIZE; 2679 2680 ring->rx_pending = bp->rx_ring_size; 2681 ring->tx_pending = bp->tx_ring_size; 2682 } 2683 2684 static int macb_set_ringparam(struct net_device *netdev, 2685 struct ethtool_ringparam *ring) 2686 { 2687 struct macb *bp = netdev_priv(netdev); 2688 u32 new_rx_size, new_tx_size; 2689 unsigned int reset = 0; 2690 2691 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2692 return -EINVAL; 2693 2694 new_rx_size = clamp_t(u32, ring->rx_pending, 2695 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2696 new_rx_size = roundup_pow_of_two(new_rx_size); 2697 2698 new_tx_size = clamp_t(u32, ring->tx_pending, 2699 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2700 new_tx_size = roundup_pow_of_two(new_tx_size); 2701 2702 if ((new_tx_size == bp->tx_ring_size) && 2703 (new_rx_size == bp->rx_ring_size)) { 2704 /* nothing to do */ 2705 return 0; 2706 } 2707 2708 if (netif_running(bp->dev)) { 2709 reset = 1; 2710 macb_close(bp->dev); 2711 } 2712 2713 bp->rx_ring_size = new_rx_size; 2714 bp->tx_ring_size = new_tx_size; 2715 2716 if (reset) 2717 macb_open(bp->dev); 2718 2719 return 0; 2720 } 2721 2722 #ifdef CONFIG_MACB_USE_HWSTAMP 2723 static unsigned int gem_get_tsu_rate(struct macb *bp) 2724 { 2725 struct clk *tsu_clk; 2726 unsigned int tsu_rate; 2727 2728 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2729 if (!IS_ERR(tsu_clk)) 2730 tsu_rate = clk_get_rate(tsu_clk); 2731 /* try pclk instead */ 2732 else if (!IS_ERR(bp->pclk)) { 2733 tsu_clk = bp->pclk; 2734 tsu_rate = clk_get_rate(tsu_clk); 2735 } else 2736 return -ENOTSUPP; 2737 return tsu_rate; 2738 } 2739 2740 static s32 gem_get_ptp_max_adj(void) 2741 { 2742 return 64000000; 2743 } 2744 2745 static int gem_get_ts_info(struct net_device *dev, 2746 struct ethtool_ts_info *info) 2747 { 2748 struct macb *bp = netdev_priv(dev); 2749 2750 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2751 ethtool_op_get_ts_info(dev, info); 2752 return 0; 2753 } 2754 2755 info->so_timestamping = 2756 SOF_TIMESTAMPING_TX_SOFTWARE | 2757 SOF_TIMESTAMPING_RX_SOFTWARE | 2758 SOF_TIMESTAMPING_SOFTWARE | 2759 SOF_TIMESTAMPING_TX_HARDWARE | 2760 SOF_TIMESTAMPING_RX_HARDWARE | 2761 SOF_TIMESTAMPING_RAW_HARDWARE; 2762 info->tx_types = 2763 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2764 (1 << HWTSTAMP_TX_OFF) | 2765 (1 << HWTSTAMP_TX_ON); 2766 info->rx_filters = 2767 (1 << HWTSTAMP_FILTER_NONE) | 2768 (1 << HWTSTAMP_FILTER_ALL); 2769 2770 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2771 2772 return 0; 2773 } 2774 2775 static struct macb_ptp_info gem_ptp_info = { 2776 .ptp_init = gem_ptp_init, 2777 .ptp_remove = gem_ptp_remove, 2778 .get_ptp_max_adj = gem_get_ptp_max_adj, 2779 .get_tsu_rate = gem_get_tsu_rate, 2780 .get_ts_info = gem_get_ts_info, 2781 .get_hwtst = gem_get_hwtst, 2782 .set_hwtst = gem_set_hwtst, 2783 }; 2784 #endif 2785 2786 static int macb_get_ts_info(struct net_device *netdev, 2787 struct ethtool_ts_info *info) 2788 { 2789 struct macb *bp = netdev_priv(netdev); 2790 2791 if (bp->ptp_info) 2792 return bp->ptp_info->get_ts_info(netdev, info); 2793 2794 return ethtool_op_get_ts_info(netdev, info); 2795 } 2796 2797 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2798 { 2799 struct ethtool_rx_fs_item *item; 2800 u32 t2_scr; 2801 int num_t2_scr; 2802 2803 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2804 2805 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2806 struct ethtool_rx_flow_spec *fs = &item->fs; 2807 struct ethtool_tcpip4_spec *tp4sp_m; 2808 2809 if (fs->location >= num_t2_scr) 2810 continue; 2811 2812 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2813 2814 /* enable/disable screener regs for the flow entry */ 2815 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2816 2817 /* only enable fields with no masking */ 2818 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2819 2820 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2821 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2822 else 2823 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2824 2825 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2826 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2827 else 2828 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2829 2830 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2831 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2832 else 2833 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 2834 2835 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 2836 } 2837 } 2838 2839 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 2840 { 2841 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 2842 uint16_t index = fs->location; 2843 u32 w0, w1, t2_scr; 2844 bool cmp_a = false; 2845 bool cmp_b = false; 2846 bool cmp_c = false; 2847 2848 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 2849 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2850 2851 /* ignore field if any masking set */ 2852 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 2853 /* 1st compare reg - IP source address */ 2854 w0 = 0; 2855 w1 = 0; 2856 w0 = tp4sp_v->ip4src; 2857 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2858 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2859 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 2860 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 2861 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 2862 cmp_a = true; 2863 } 2864 2865 /* ignore field if any masking set */ 2866 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 2867 /* 2nd compare reg - IP destination address */ 2868 w0 = 0; 2869 w1 = 0; 2870 w0 = tp4sp_v->ip4dst; 2871 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2872 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2873 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 2874 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 2875 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 2876 cmp_b = true; 2877 } 2878 2879 /* ignore both port fields if masking set in both */ 2880 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 2881 /* 3rd compare reg - source port, destination port */ 2882 w0 = 0; 2883 w1 = 0; 2884 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 2885 if (tp4sp_m->psrc == tp4sp_m->pdst) { 2886 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 2887 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2888 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2889 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2890 } else { 2891 /* only one port definition */ 2892 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 2893 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 2894 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 2895 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 2896 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2897 } else { /* dst port */ 2898 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2899 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 2900 } 2901 } 2902 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 2903 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 2904 cmp_c = true; 2905 } 2906 2907 t2_scr = 0; 2908 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 2909 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 2910 if (cmp_a) 2911 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 2912 if (cmp_b) 2913 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 2914 if (cmp_c) 2915 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 2916 gem_writel_n(bp, SCRT2, index, t2_scr); 2917 } 2918 2919 static int gem_add_flow_filter(struct net_device *netdev, 2920 struct ethtool_rxnfc *cmd) 2921 { 2922 struct macb *bp = netdev_priv(netdev); 2923 struct ethtool_rx_flow_spec *fs = &cmd->fs; 2924 struct ethtool_rx_fs_item *item, *newfs; 2925 unsigned long flags; 2926 int ret = -EINVAL; 2927 bool added = false; 2928 2929 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 2930 if (newfs == NULL) 2931 return -ENOMEM; 2932 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 2933 2934 netdev_dbg(netdev, 2935 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2936 fs->flow_type, (int)fs->ring_cookie, fs->location, 2937 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2938 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2939 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 2940 2941 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2942 2943 /* find correct place to add in list */ 2944 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2945 if (item->fs.location > newfs->fs.location) { 2946 list_add_tail(&newfs->list, &item->list); 2947 added = true; 2948 break; 2949 } else if (item->fs.location == fs->location) { 2950 netdev_err(netdev, "Rule not added: location %d not free!\n", 2951 fs->location); 2952 ret = -EBUSY; 2953 goto err; 2954 } 2955 } 2956 if (!added) 2957 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 2958 2959 gem_prog_cmp_regs(bp, fs); 2960 bp->rx_fs_list.count++; 2961 /* enable filtering if NTUPLE on */ 2962 if (netdev->features & NETIF_F_NTUPLE) 2963 gem_enable_flow_filters(bp, 1); 2964 2965 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2966 return 0; 2967 2968 err: 2969 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2970 kfree(newfs); 2971 return ret; 2972 } 2973 2974 static int gem_del_flow_filter(struct net_device *netdev, 2975 struct ethtool_rxnfc *cmd) 2976 { 2977 struct macb *bp = netdev_priv(netdev); 2978 struct ethtool_rx_fs_item *item; 2979 struct ethtool_rx_flow_spec *fs; 2980 unsigned long flags; 2981 2982 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2983 2984 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2985 if (item->fs.location == cmd->fs.location) { 2986 /* disable screener regs for the flow entry */ 2987 fs = &(item->fs); 2988 netdev_dbg(netdev, 2989 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2990 fs->flow_type, (int)fs->ring_cookie, fs->location, 2991 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2992 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2993 htons(fs->h_u.tcp_ip4_spec.psrc), 2994 htons(fs->h_u.tcp_ip4_spec.pdst)); 2995 2996 gem_writel_n(bp, SCRT2, fs->location, 0); 2997 2998 list_del(&item->list); 2999 bp->rx_fs_list.count--; 3000 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3001 kfree(item); 3002 return 0; 3003 } 3004 } 3005 3006 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3007 return -EINVAL; 3008 } 3009 3010 static int gem_get_flow_entry(struct net_device *netdev, 3011 struct ethtool_rxnfc *cmd) 3012 { 3013 struct macb *bp = netdev_priv(netdev); 3014 struct ethtool_rx_fs_item *item; 3015 3016 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3017 if (item->fs.location == cmd->fs.location) { 3018 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3019 return 0; 3020 } 3021 } 3022 return -EINVAL; 3023 } 3024 3025 static int gem_get_all_flow_entries(struct net_device *netdev, 3026 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3027 { 3028 struct macb *bp = netdev_priv(netdev); 3029 struct ethtool_rx_fs_item *item; 3030 uint32_t cnt = 0; 3031 3032 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3033 if (cnt == cmd->rule_cnt) 3034 return -EMSGSIZE; 3035 rule_locs[cnt] = item->fs.location; 3036 cnt++; 3037 } 3038 cmd->data = bp->max_tuples; 3039 cmd->rule_cnt = cnt; 3040 3041 return 0; 3042 } 3043 3044 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3045 u32 *rule_locs) 3046 { 3047 struct macb *bp = netdev_priv(netdev); 3048 int ret = 0; 3049 3050 switch (cmd->cmd) { 3051 case ETHTOOL_GRXRINGS: 3052 cmd->data = bp->num_queues; 3053 break; 3054 case ETHTOOL_GRXCLSRLCNT: 3055 cmd->rule_cnt = bp->rx_fs_list.count; 3056 break; 3057 case ETHTOOL_GRXCLSRULE: 3058 ret = gem_get_flow_entry(netdev, cmd); 3059 break; 3060 case ETHTOOL_GRXCLSRLALL: 3061 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3062 break; 3063 default: 3064 netdev_err(netdev, 3065 "Command parameter %d is not supported\n", cmd->cmd); 3066 ret = -EOPNOTSUPP; 3067 } 3068 3069 return ret; 3070 } 3071 3072 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3073 { 3074 struct macb *bp = netdev_priv(netdev); 3075 int ret; 3076 3077 switch (cmd->cmd) { 3078 case ETHTOOL_SRXCLSRLINS: 3079 if ((cmd->fs.location >= bp->max_tuples) 3080 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3081 ret = -EINVAL; 3082 break; 3083 } 3084 ret = gem_add_flow_filter(netdev, cmd); 3085 break; 3086 case ETHTOOL_SRXCLSRLDEL: 3087 ret = gem_del_flow_filter(netdev, cmd); 3088 break; 3089 default: 3090 netdev_err(netdev, 3091 "Command parameter %d is not supported\n", cmd->cmd); 3092 ret = -EOPNOTSUPP; 3093 } 3094 3095 return ret; 3096 } 3097 3098 static const struct ethtool_ops macb_ethtool_ops = { 3099 .get_regs_len = macb_get_regs_len, 3100 .get_regs = macb_get_regs, 3101 .get_link = ethtool_op_get_link, 3102 .get_ts_info = ethtool_op_get_ts_info, 3103 .get_wol = macb_get_wol, 3104 .set_wol = macb_set_wol, 3105 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3106 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3107 .get_ringparam = macb_get_ringparam, 3108 .set_ringparam = macb_set_ringparam, 3109 }; 3110 3111 static const struct ethtool_ops gem_ethtool_ops = { 3112 .get_regs_len = macb_get_regs_len, 3113 .get_regs = macb_get_regs, 3114 .get_link = ethtool_op_get_link, 3115 .get_ts_info = macb_get_ts_info, 3116 .get_ethtool_stats = gem_get_ethtool_stats, 3117 .get_strings = gem_get_ethtool_strings, 3118 .get_sset_count = gem_get_sset_count, 3119 .get_link_ksettings = phy_ethtool_get_link_ksettings, 3120 .set_link_ksettings = phy_ethtool_set_link_ksettings, 3121 .get_ringparam = macb_get_ringparam, 3122 .set_ringparam = macb_set_ringparam, 3123 .get_rxnfc = gem_get_rxnfc, 3124 .set_rxnfc = gem_set_rxnfc, 3125 }; 3126 3127 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3128 { 3129 struct phy_device *phydev = dev->phydev; 3130 struct macb *bp = netdev_priv(dev); 3131 3132 if (!netif_running(dev)) 3133 return -EINVAL; 3134 3135 if (!phydev) 3136 return -ENODEV; 3137 3138 if (!bp->ptp_info) 3139 return phy_mii_ioctl(phydev, rq, cmd); 3140 3141 switch (cmd) { 3142 case SIOCSHWTSTAMP: 3143 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3144 case SIOCGHWTSTAMP: 3145 return bp->ptp_info->get_hwtst(dev, rq); 3146 default: 3147 return phy_mii_ioctl(phydev, rq, cmd); 3148 } 3149 } 3150 3151 static int macb_set_features(struct net_device *netdev, 3152 netdev_features_t features) 3153 { 3154 struct macb *bp = netdev_priv(netdev); 3155 netdev_features_t changed = features ^ netdev->features; 3156 3157 /* TX checksum offload */ 3158 if ((changed & NETIF_F_HW_CSUM) && macb_is_gem(bp)) { 3159 u32 dmacfg; 3160 3161 dmacfg = gem_readl(bp, DMACFG); 3162 if (features & NETIF_F_HW_CSUM) 3163 dmacfg |= GEM_BIT(TXCOEN); 3164 else 3165 dmacfg &= ~GEM_BIT(TXCOEN); 3166 gem_writel(bp, DMACFG, dmacfg); 3167 } 3168 3169 /* RX checksum offload */ 3170 if ((changed & NETIF_F_RXCSUM) && macb_is_gem(bp)) { 3171 u32 netcfg; 3172 3173 netcfg = gem_readl(bp, NCFGR); 3174 if (features & NETIF_F_RXCSUM && 3175 !(netdev->flags & IFF_PROMISC)) 3176 netcfg |= GEM_BIT(RXCOEN); 3177 else 3178 netcfg &= ~GEM_BIT(RXCOEN); 3179 gem_writel(bp, NCFGR, netcfg); 3180 } 3181 3182 /* RX Flow Filters */ 3183 if ((changed & NETIF_F_NTUPLE) && macb_is_gem(bp)) { 3184 bool turn_on = features & NETIF_F_NTUPLE; 3185 3186 gem_enable_flow_filters(bp, turn_on); 3187 } 3188 return 0; 3189 } 3190 3191 static const struct net_device_ops macb_netdev_ops = { 3192 .ndo_open = macb_open, 3193 .ndo_stop = macb_close, 3194 .ndo_start_xmit = macb_start_xmit, 3195 .ndo_set_rx_mode = macb_set_rx_mode, 3196 .ndo_get_stats = macb_get_stats, 3197 .ndo_do_ioctl = macb_ioctl, 3198 .ndo_validate_addr = eth_validate_addr, 3199 .ndo_change_mtu = macb_change_mtu, 3200 .ndo_set_mac_address = eth_mac_addr, 3201 #ifdef CONFIG_NET_POLL_CONTROLLER 3202 .ndo_poll_controller = macb_poll_controller, 3203 #endif 3204 .ndo_set_features = macb_set_features, 3205 .ndo_features_check = macb_features_check, 3206 }; 3207 3208 /* Configure peripheral capabilities according to device tree 3209 * and integration options used 3210 */ 3211 static void macb_configure_caps(struct macb *bp, 3212 const struct macb_config *dt_conf) 3213 { 3214 u32 dcfg; 3215 3216 if (dt_conf) 3217 bp->caps = dt_conf->caps; 3218 3219 if (hw_is_gem(bp->regs, bp->native_io)) { 3220 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3221 3222 dcfg = gem_readl(bp, DCFG1); 3223 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3224 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3225 dcfg = gem_readl(bp, DCFG2); 3226 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3227 bp->caps |= MACB_CAPS_FIFO_MODE; 3228 #ifdef CONFIG_MACB_USE_HWSTAMP 3229 if (gem_has_ptp(bp)) { 3230 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3231 pr_err("GEM doesn't support hardware ptp.\n"); 3232 else { 3233 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3234 bp->ptp_info = &gem_ptp_info; 3235 } 3236 } 3237 #endif 3238 } 3239 3240 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3241 } 3242 3243 static void macb_probe_queues(void __iomem *mem, 3244 bool native_io, 3245 unsigned int *queue_mask, 3246 unsigned int *num_queues) 3247 { 3248 unsigned int hw_q; 3249 3250 *queue_mask = 0x1; 3251 *num_queues = 1; 3252 3253 /* is it macb or gem ? 3254 * 3255 * We need to read directly from the hardware here because 3256 * we are early in the probe process and don't have the 3257 * MACB_CAPS_MACB_IS_GEM flag positioned 3258 */ 3259 if (!hw_is_gem(mem, native_io)) 3260 return; 3261 3262 /* bit 0 is never set but queue 0 always exists */ 3263 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3264 3265 *queue_mask |= 0x1; 3266 3267 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3268 if (*queue_mask & (1 << hw_q)) 3269 (*num_queues)++; 3270 } 3271 3272 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3273 struct clk **hclk, struct clk **tx_clk, 3274 struct clk **rx_clk) 3275 { 3276 struct macb_platform_data *pdata; 3277 int err; 3278 3279 pdata = dev_get_platdata(&pdev->dev); 3280 if (pdata) { 3281 *pclk = pdata->pclk; 3282 *hclk = pdata->hclk; 3283 } else { 3284 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3285 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3286 } 3287 3288 if (IS_ERR(*pclk)) { 3289 err = PTR_ERR(*pclk); 3290 dev_err(&pdev->dev, "failed to get macb_clk (%u)\n", err); 3291 return err; 3292 } 3293 3294 if (IS_ERR(*hclk)) { 3295 err = PTR_ERR(*hclk); 3296 dev_err(&pdev->dev, "failed to get hclk (%u)\n", err); 3297 return err; 3298 } 3299 3300 *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); 3301 if (IS_ERR(*tx_clk)) 3302 *tx_clk = NULL; 3303 3304 *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); 3305 if (IS_ERR(*rx_clk)) 3306 *rx_clk = NULL; 3307 3308 err = clk_prepare_enable(*pclk); 3309 if (err) { 3310 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3311 return err; 3312 } 3313 3314 err = clk_prepare_enable(*hclk); 3315 if (err) { 3316 dev_err(&pdev->dev, "failed to enable hclk (%u)\n", err); 3317 goto err_disable_pclk; 3318 } 3319 3320 err = clk_prepare_enable(*tx_clk); 3321 if (err) { 3322 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 3323 goto err_disable_hclk; 3324 } 3325 3326 err = clk_prepare_enable(*rx_clk); 3327 if (err) { 3328 dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); 3329 goto err_disable_txclk; 3330 } 3331 3332 return 0; 3333 3334 err_disable_txclk: 3335 clk_disable_unprepare(*tx_clk); 3336 3337 err_disable_hclk: 3338 clk_disable_unprepare(*hclk); 3339 3340 err_disable_pclk: 3341 clk_disable_unprepare(*pclk); 3342 3343 return err; 3344 } 3345 3346 static int macb_init(struct platform_device *pdev) 3347 { 3348 struct net_device *dev = platform_get_drvdata(pdev); 3349 unsigned int hw_q, q; 3350 struct macb *bp = netdev_priv(dev); 3351 struct macb_queue *queue; 3352 int err; 3353 u32 val, reg; 3354 3355 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3356 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3357 3358 /* set the queue register mapping once for all: queue0 has a special 3359 * register mapping but we don't want to test the queue index then 3360 * compute the corresponding register offset at run time. 3361 */ 3362 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3363 if (!(bp->queue_mask & (1 << hw_q))) 3364 continue; 3365 3366 queue = &bp->queues[q]; 3367 queue->bp = bp; 3368 netif_napi_add(dev, &queue->napi, macb_poll, 64); 3369 if (hw_q) { 3370 queue->ISR = GEM_ISR(hw_q - 1); 3371 queue->IER = GEM_IER(hw_q - 1); 3372 queue->IDR = GEM_IDR(hw_q - 1); 3373 queue->IMR = GEM_IMR(hw_q - 1); 3374 queue->TBQP = GEM_TBQP(hw_q - 1); 3375 queue->RBQP = GEM_RBQP(hw_q - 1); 3376 queue->RBQS = GEM_RBQS(hw_q - 1); 3377 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3378 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3379 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3380 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3381 } 3382 #endif 3383 } else { 3384 /* queue0 uses legacy registers */ 3385 queue->ISR = MACB_ISR; 3386 queue->IER = MACB_IER; 3387 queue->IDR = MACB_IDR; 3388 queue->IMR = MACB_IMR; 3389 queue->TBQP = MACB_TBQP; 3390 queue->RBQP = MACB_RBQP; 3391 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3392 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3393 queue->TBQPH = MACB_TBQPH; 3394 queue->RBQPH = MACB_RBQPH; 3395 } 3396 #endif 3397 } 3398 3399 /* get irq: here we use the linux queue index, not the hardware 3400 * queue index. the queue irq definitions in the device tree 3401 * must remove the optional gaps that could exist in the 3402 * hardware queue mask. 3403 */ 3404 queue->irq = platform_get_irq(pdev, q); 3405 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3406 IRQF_SHARED, dev->name, queue); 3407 if (err) { 3408 dev_err(&pdev->dev, 3409 "Unable to request IRQ %d (error %d)\n", 3410 queue->irq, err); 3411 return err; 3412 } 3413 3414 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3415 q++; 3416 } 3417 3418 dev->netdev_ops = &macb_netdev_ops; 3419 3420 /* setup appropriated routines according to adapter type */ 3421 if (macb_is_gem(bp)) { 3422 bp->max_tx_length = GEM_MAX_TX_LEN; 3423 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3424 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3425 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3426 bp->macbgem_ops.mog_rx = gem_rx; 3427 dev->ethtool_ops = &gem_ethtool_ops; 3428 } else { 3429 bp->max_tx_length = MACB_MAX_TX_LEN; 3430 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3431 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3432 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3433 bp->macbgem_ops.mog_rx = macb_rx; 3434 dev->ethtool_ops = &macb_ethtool_ops; 3435 } 3436 3437 /* Set features */ 3438 dev->hw_features = NETIF_F_SG; 3439 3440 /* Check LSO capability */ 3441 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3442 dev->hw_features |= MACB_NETIF_LSO; 3443 3444 /* Checksum offload is only available on gem with packet buffer */ 3445 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3446 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3447 if (bp->caps & MACB_CAPS_SG_DISABLED) 3448 dev->hw_features &= ~NETIF_F_SG; 3449 dev->features = dev->hw_features; 3450 3451 /* Check RX Flow Filters support. 3452 * Max Rx flows set by availability of screeners & compare regs: 3453 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3454 */ 3455 reg = gem_readl(bp, DCFG8); 3456 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3457 GEM_BFEXT(T2SCR, reg)); 3458 if (bp->max_tuples > 0) { 3459 /* also needs one ethtype match to check IPv4 */ 3460 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3461 /* program this reg now */ 3462 reg = 0; 3463 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3464 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3465 /* Filtering is supported in hw but don't enable it in kernel now */ 3466 dev->hw_features |= NETIF_F_NTUPLE; 3467 /* init Rx flow definitions */ 3468 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3469 bp->rx_fs_list.count = 0; 3470 spin_lock_init(&bp->rx_fs_lock); 3471 } else 3472 bp->max_tuples = 0; 3473 } 3474 3475 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3476 val = 0; 3477 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3478 val = GEM_BIT(RGMII); 3479 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3480 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3481 val = MACB_BIT(RMII); 3482 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3483 val = MACB_BIT(MII); 3484 3485 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3486 val |= MACB_BIT(CLKEN); 3487 3488 macb_or_gem_writel(bp, USRIO, val); 3489 } 3490 3491 /* Set MII management clock divider */ 3492 val = macb_mdc_clk_div(bp); 3493 val |= macb_dbw(bp); 3494 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3495 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3496 macb_writel(bp, NCFGR, val); 3497 3498 return 0; 3499 } 3500 3501 #if defined(CONFIG_OF) 3502 /* 1518 rounded up */ 3503 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3504 /* max number of receive buffers */ 3505 #define AT91ETHER_MAX_RX_DESCR 9 3506 3507 /* Initialize and start the Receiver and Transmit subsystems */ 3508 static int at91ether_start(struct net_device *dev) 3509 { 3510 struct macb *lp = netdev_priv(dev); 3511 struct macb_queue *q = &lp->queues[0]; 3512 struct macb_dma_desc *desc; 3513 dma_addr_t addr; 3514 u32 ctl; 3515 int i; 3516 3517 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3518 (AT91ETHER_MAX_RX_DESCR * 3519 macb_dma_desc_get_size(lp)), 3520 &q->rx_ring_dma, GFP_KERNEL); 3521 if (!q->rx_ring) 3522 return -ENOMEM; 3523 3524 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3525 AT91ETHER_MAX_RX_DESCR * 3526 AT91ETHER_MAX_RBUFF_SZ, 3527 &q->rx_buffers_dma, GFP_KERNEL); 3528 if (!q->rx_buffers) { 3529 dma_free_coherent(&lp->pdev->dev, 3530 AT91ETHER_MAX_RX_DESCR * 3531 macb_dma_desc_get_size(lp), 3532 q->rx_ring, q->rx_ring_dma); 3533 q->rx_ring = NULL; 3534 return -ENOMEM; 3535 } 3536 3537 addr = q->rx_buffers_dma; 3538 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3539 desc = macb_rx_desc(q, i); 3540 macb_set_addr(lp, desc, addr); 3541 desc->ctrl = 0; 3542 addr += AT91ETHER_MAX_RBUFF_SZ; 3543 } 3544 3545 /* Set the Wrap bit on the last descriptor */ 3546 desc->addr |= MACB_BIT(RX_WRAP); 3547 3548 /* Reset buffer index */ 3549 q->rx_tail = 0; 3550 3551 /* Program address of descriptor list in Rx Buffer Queue register */ 3552 macb_writel(lp, RBQP, q->rx_ring_dma); 3553 3554 /* Enable Receive and Transmit */ 3555 ctl = macb_readl(lp, NCR); 3556 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3557 3558 return 0; 3559 } 3560 3561 /* Open the ethernet interface */ 3562 static int at91ether_open(struct net_device *dev) 3563 { 3564 struct macb *lp = netdev_priv(dev); 3565 u32 ctl; 3566 int ret; 3567 3568 /* Clear internal statistics */ 3569 ctl = macb_readl(lp, NCR); 3570 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3571 3572 macb_set_hwaddr(lp); 3573 3574 ret = at91ether_start(dev); 3575 if (ret) 3576 return ret; 3577 3578 /* Enable MAC interrupts */ 3579 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3580 MACB_BIT(RXUBR) | 3581 MACB_BIT(ISR_TUND) | 3582 MACB_BIT(ISR_RLE) | 3583 MACB_BIT(TCOMP) | 3584 MACB_BIT(ISR_ROVR) | 3585 MACB_BIT(HRESP)); 3586 3587 /* schedule a link state check */ 3588 phy_start(dev->phydev); 3589 3590 netif_start_queue(dev); 3591 3592 return 0; 3593 } 3594 3595 /* Close the interface */ 3596 static int at91ether_close(struct net_device *dev) 3597 { 3598 struct macb *lp = netdev_priv(dev); 3599 struct macb_queue *q = &lp->queues[0]; 3600 u32 ctl; 3601 3602 /* Disable Receiver and Transmitter */ 3603 ctl = macb_readl(lp, NCR); 3604 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3605 3606 /* Disable MAC interrupts */ 3607 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3608 MACB_BIT(RXUBR) | 3609 MACB_BIT(ISR_TUND) | 3610 MACB_BIT(ISR_RLE) | 3611 MACB_BIT(TCOMP) | 3612 MACB_BIT(ISR_ROVR) | 3613 MACB_BIT(HRESP)); 3614 3615 netif_stop_queue(dev); 3616 3617 dma_free_coherent(&lp->pdev->dev, 3618 AT91ETHER_MAX_RX_DESCR * 3619 macb_dma_desc_get_size(lp), 3620 q->rx_ring, q->rx_ring_dma); 3621 q->rx_ring = NULL; 3622 3623 dma_free_coherent(&lp->pdev->dev, 3624 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3625 q->rx_buffers, q->rx_buffers_dma); 3626 q->rx_buffers = NULL; 3627 3628 return 0; 3629 } 3630 3631 /* Transmit packet */ 3632 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3633 struct net_device *dev) 3634 { 3635 struct macb *lp = netdev_priv(dev); 3636 3637 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3638 netif_stop_queue(dev); 3639 3640 /* Store packet information (to free when Tx completed) */ 3641 lp->skb = skb; 3642 lp->skb_length = skb->len; 3643 lp->skb_physaddr = dma_map_single(NULL, skb->data, skb->len, 3644 DMA_TO_DEVICE); 3645 if (dma_mapping_error(NULL, lp->skb_physaddr)) { 3646 dev_kfree_skb_any(skb); 3647 dev->stats.tx_dropped++; 3648 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3649 return NETDEV_TX_OK; 3650 } 3651 3652 /* Set address of the data in the Transmit Address register */ 3653 macb_writel(lp, TAR, lp->skb_physaddr); 3654 /* Set length of the packet in the Transmit Control register */ 3655 macb_writel(lp, TCR, skb->len); 3656 3657 } else { 3658 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3659 return NETDEV_TX_BUSY; 3660 } 3661 3662 return NETDEV_TX_OK; 3663 } 3664 3665 /* Extract received frame from buffer descriptors and sent to upper layers. 3666 * (Called from interrupt context) 3667 */ 3668 static void at91ether_rx(struct net_device *dev) 3669 { 3670 struct macb *lp = netdev_priv(dev); 3671 struct macb_queue *q = &lp->queues[0]; 3672 struct macb_dma_desc *desc; 3673 unsigned char *p_recv; 3674 struct sk_buff *skb; 3675 unsigned int pktlen; 3676 3677 desc = macb_rx_desc(q, q->rx_tail); 3678 while (desc->addr & MACB_BIT(RX_USED)) { 3679 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3680 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3681 skb = netdev_alloc_skb(dev, pktlen + 2); 3682 if (skb) { 3683 skb_reserve(skb, 2); 3684 skb_put_data(skb, p_recv, pktlen); 3685 3686 skb->protocol = eth_type_trans(skb, dev); 3687 dev->stats.rx_packets++; 3688 dev->stats.rx_bytes += pktlen; 3689 netif_rx(skb); 3690 } else { 3691 dev->stats.rx_dropped++; 3692 } 3693 3694 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3695 dev->stats.multicast++; 3696 3697 /* reset ownership bit */ 3698 desc->addr &= ~MACB_BIT(RX_USED); 3699 3700 /* wrap after last buffer */ 3701 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3702 q->rx_tail = 0; 3703 else 3704 q->rx_tail++; 3705 3706 desc = macb_rx_desc(q, q->rx_tail); 3707 } 3708 } 3709 3710 /* MAC interrupt handler */ 3711 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3712 { 3713 struct net_device *dev = dev_id; 3714 struct macb *lp = netdev_priv(dev); 3715 u32 intstatus, ctl; 3716 3717 /* MAC Interrupt Status register indicates what interrupts are pending. 3718 * It is automatically cleared once read. 3719 */ 3720 intstatus = macb_readl(lp, ISR); 3721 3722 /* Receive complete */ 3723 if (intstatus & MACB_BIT(RCOMP)) 3724 at91ether_rx(dev); 3725 3726 /* Transmit complete */ 3727 if (intstatus & MACB_BIT(TCOMP)) { 3728 /* The TCOM bit is set even if the transmission failed */ 3729 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3730 dev->stats.tx_errors++; 3731 3732 if (lp->skb) { 3733 dev_kfree_skb_irq(lp->skb); 3734 lp->skb = NULL; 3735 dma_unmap_single(NULL, lp->skb_physaddr, 3736 lp->skb_length, DMA_TO_DEVICE); 3737 dev->stats.tx_packets++; 3738 dev->stats.tx_bytes += lp->skb_length; 3739 } 3740 netif_wake_queue(dev); 3741 } 3742 3743 /* Work-around for EMAC Errata section 41.3.1 */ 3744 if (intstatus & MACB_BIT(RXUBR)) { 3745 ctl = macb_readl(lp, NCR); 3746 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3747 wmb(); 3748 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3749 } 3750 3751 if (intstatus & MACB_BIT(ISR_ROVR)) 3752 netdev_err(dev, "ROVR error\n"); 3753 3754 return IRQ_HANDLED; 3755 } 3756 3757 #ifdef CONFIG_NET_POLL_CONTROLLER 3758 static void at91ether_poll_controller(struct net_device *dev) 3759 { 3760 unsigned long flags; 3761 3762 local_irq_save(flags); 3763 at91ether_interrupt(dev->irq, dev); 3764 local_irq_restore(flags); 3765 } 3766 #endif 3767 3768 static const struct net_device_ops at91ether_netdev_ops = { 3769 .ndo_open = at91ether_open, 3770 .ndo_stop = at91ether_close, 3771 .ndo_start_xmit = at91ether_start_xmit, 3772 .ndo_get_stats = macb_get_stats, 3773 .ndo_set_rx_mode = macb_set_rx_mode, 3774 .ndo_set_mac_address = eth_mac_addr, 3775 .ndo_do_ioctl = macb_ioctl, 3776 .ndo_validate_addr = eth_validate_addr, 3777 #ifdef CONFIG_NET_POLL_CONTROLLER 3778 .ndo_poll_controller = at91ether_poll_controller, 3779 #endif 3780 }; 3781 3782 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3783 struct clk **hclk, struct clk **tx_clk, 3784 struct clk **rx_clk) 3785 { 3786 int err; 3787 3788 *hclk = NULL; 3789 *tx_clk = NULL; 3790 *rx_clk = NULL; 3791 3792 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3793 if (IS_ERR(*pclk)) 3794 return PTR_ERR(*pclk); 3795 3796 err = clk_prepare_enable(*pclk); 3797 if (err) { 3798 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3799 return err; 3800 } 3801 3802 return 0; 3803 } 3804 3805 static int at91ether_init(struct platform_device *pdev) 3806 { 3807 struct net_device *dev = platform_get_drvdata(pdev); 3808 struct macb *bp = netdev_priv(dev); 3809 int err; 3810 u32 reg; 3811 3812 bp->queues[0].bp = bp; 3813 3814 dev->netdev_ops = &at91ether_netdev_ops; 3815 dev->ethtool_ops = &macb_ethtool_ops; 3816 3817 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3818 0, dev->name, dev); 3819 if (err) 3820 return err; 3821 3822 macb_writel(bp, NCR, 0); 3823 3824 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3825 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3826 reg |= MACB_BIT(RM9200_RMII); 3827 3828 macb_writel(bp, NCFGR, reg); 3829 3830 return 0; 3831 } 3832 3833 static const struct macb_config at91sam9260_config = { 3834 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3835 .clk_init = macb_clk_init, 3836 .init = macb_init, 3837 }; 3838 3839 static const struct macb_config pc302gem_config = { 3840 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 3841 .dma_burst_length = 16, 3842 .clk_init = macb_clk_init, 3843 .init = macb_init, 3844 }; 3845 3846 static const struct macb_config sama5d2_config = { 3847 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3848 .dma_burst_length = 16, 3849 .clk_init = macb_clk_init, 3850 .init = macb_init, 3851 }; 3852 3853 static const struct macb_config sama5d3_config = { 3854 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 3855 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 3856 .dma_burst_length = 16, 3857 .clk_init = macb_clk_init, 3858 .init = macb_init, 3859 .jumbo_max_len = 10240, 3860 }; 3861 3862 static const struct macb_config sama5d4_config = { 3863 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3864 .dma_burst_length = 4, 3865 .clk_init = macb_clk_init, 3866 .init = macb_init, 3867 }; 3868 3869 static const struct macb_config emac_config = { 3870 .clk_init = at91ether_clk_init, 3871 .init = at91ether_init, 3872 }; 3873 3874 static const struct macb_config np4_config = { 3875 .caps = MACB_CAPS_USRIO_DISABLED, 3876 .clk_init = macb_clk_init, 3877 .init = macb_init, 3878 }; 3879 3880 static const struct macb_config zynqmp_config = { 3881 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3882 MACB_CAPS_JUMBO | 3883 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 3884 .dma_burst_length = 16, 3885 .clk_init = macb_clk_init, 3886 .init = macb_init, 3887 .jumbo_max_len = 10240, 3888 }; 3889 3890 static const struct macb_config zynq_config = { 3891 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF, 3892 .dma_burst_length = 16, 3893 .clk_init = macb_clk_init, 3894 .init = macb_init, 3895 }; 3896 3897 static const struct of_device_id macb_dt_ids[] = { 3898 { .compatible = "cdns,at32ap7000-macb" }, 3899 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 3900 { .compatible = "cdns,macb" }, 3901 { .compatible = "cdns,np4-macb", .data = &np4_config }, 3902 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 3903 { .compatible = "cdns,gem", .data = &pc302gem_config }, 3904 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 3905 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 3906 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 3907 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 3908 { .compatible = "cdns,emac", .data = &emac_config }, 3909 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 3910 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 3911 { /* sentinel */ } 3912 }; 3913 MODULE_DEVICE_TABLE(of, macb_dt_ids); 3914 #endif /* CONFIG_OF */ 3915 3916 static const struct macb_config default_gem_config = { 3917 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3918 MACB_CAPS_JUMBO | 3919 MACB_CAPS_GEM_HAS_PTP, 3920 .dma_burst_length = 16, 3921 .clk_init = macb_clk_init, 3922 .init = macb_init, 3923 .jumbo_max_len = 10240, 3924 }; 3925 3926 static int macb_probe(struct platform_device *pdev) 3927 { 3928 const struct macb_config *macb_config = &default_gem_config; 3929 int (*clk_init)(struct platform_device *, struct clk **, 3930 struct clk **, struct clk **, struct clk **) 3931 = macb_config->clk_init; 3932 int (*init)(struct platform_device *) = macb_config->init; 3933 struct device_node *np = pdev->dev.of_node; 3934 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 3935 unsigned int queue_mask, num_queues; 3936 struct macb_platform_data *pdata; 3937 bool native_io; 3938 struct phy_device *phydev; 3939 struct net_device *dev; 3940 struct resource *regs; 3941 void __iomem *mem; 3942 const char *mac; 3943 struct macb *bp; 3944 int err, val; 3945 3946 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3947 mem = devm_ioremap_resource(&pdev->dev, regs); 3948 if (IS_ERR(mem)) 3949 return PTR_ERR(mem); 3950 3951 if (np) { 3952 const struct of_device_id *match; 3953 3954 match = of_match_node(macb_dt_ids, np); 3955 if (match && match->data) { 3956 macb_config = match->data; 3957 clk_init = macb_config->clk_init; 3958 init = macb_config->init; 3959 } 3960 } 3961 3962 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk); 3963 if (err) 3964 return err; 3965 3966 native_io = hw_is_native_io(mem); 3967 3968 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 3969 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 3970 if (!dev) { 3971 err = -ENOMEM; 3972 goto err_disable_clocks; 3973 } 3974 3975 dev->base_addr = regs->start; 3976 3977 SET_NETDEV_DEV(dev, &pdev->dev); 3978 3979 bp = netdev_priv(dev); 3980 bp->pdev = pdev; 3981 bp->dev = dev; 3982 bp->regs = mem; 3983 bp->native_io = native_io; 3984 if (native_io) { 3985 bp->macb_reg_readl = hw_readl_native; 3986 bp->macb_reg_writel = hw_writel_native; 3987 } else { 3988 bp->macb_reg_readl = hw_readl; 3989 bp->macb_reg_writel = hw_writel; 3990 } 3991 bp->num_queues = num_queues; 3992 bp->queue_mask = queue_mask; 3993 if (macb_config) 3994 bp->dma_burst_length = macb_config->dma_burst_length; 3995 bp->pclk = pclk; 3996 bp->hclk = hclk; 3997 bp->tx_clk = tx_clk; 3998 bp->rx_clk = rx_clk; 3999 if (macb_config) 4000 bp->jumbo_max_len = macb_config->jumbo_max_len; 4001 4002 bp->wol = 0; 4003 if (of_get_property(np, "magic-packet", NULL)) 4004 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4005 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4006 4007 spin_lock_init(&bp->lock); 4008 4009 /* setup capabilities */ 4010 macb_configure_caps(bp, macb_config); 4011 4012 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4013 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4014 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4015 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4016 } 4017 #endif 4018 platform_set_drvdata(pdev, dev); 4019 4020 dev->irq = platform_get_irq(pdev, 0); 4021 if (dev->irq < 0) { 4022 err = dev->irq; 4023 goto err_out_free_netdev; 4024 } 4025 4026 /* MTU range: 68 - 1500 or 10240 */ 4027 dev->min_mtu = GEM_MTU_MIN_SIZE; 4028 if (bp->caps & MACB_CAPS_JUMBO) 4029 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4030 else 4031 dev->max_mtu = ETH_DATA_LEN; 4032 4033 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4034 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4035 if (val) 4036 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4037 macb_dma_desc_get_size(bp); 4038 4039 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4040 if (val) 4041 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4042 macb_dma_desc_get_size(bp); 4043 } 4044 4045 mac = of_get_mac_address(np); 4046 if (mac) { 4047 ether_addr_copy(bp->dev->dev_addr, mac); 4048 } else { 4049 err = of_get_nvmem_mac_address(np, bp->dev->dev_addr); 4050 if (err) { 4051 if (err == -EPROBE_DEFER) 4052 goto err_out_free_netdev; 4053 macb_get_hwaddr(bp); 4054 } 4055 } 4056 4057 err = of_get_phy_mode(np); 4058 if (err < 0) { 4059 pdata = dev_get_platdata(&pdev->dev); 4060 if (pdata && pdata->is_rmii) 4061 bp->phy_interface = PHY_INTERFACE_MODE_RMII; 4062 else 4063 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4064 } else { 4065 bp->phy_interface = err; 4066 } 4067 4068 /* IP specific init */ 4069 err = init(pdev); 4070 if (err) 4071 goto err_out_free_netdev; 4072 4073 err = macb_mii_init(bp); 4074 if (err) 4075 goto err_out_free_netdev; 4076 4077 phydev = dev->phydev; 4078 4079 netif_carrier_off(dev); 4080 4081 err = register_netdev(dev); 4082 if (err) { 4083 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4084 goto err_out_unregister_mdio; 4085 } 4086 4087 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4088 (unsigned long)bp); 4089 4090 phy_attached_info(phydev); 4091 4092 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4093 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4094 dev->base_addr, dev->irq, dev->dev_addr); 4095 4096 return 0; 4097 4098 err_out_unregister_mdio: 4099 phy_disconnect(dev->phydev); 4100 mdiobus_unregister(bp->mii_bus); 4101 of_node_put(bp->phy_node); 4102 if (np && of_phy_is_fixed_link(np)) 4103 of_phy_deregister_fixed_link(np); 4104 mdiobus_free(bp->mii_bus); 4105 4106 err_out_free_netdev: 4107 free_netdev(dev); 4108 4109 err_disable_clocks: 4110 clk_disable_unprepare(tx_clk); 4111 clk_disable_unprepare(hclk); 4112 clk_disable_unprepare(pclk); 4113 clk_disable_unprepare(rx_clk); 4114 4115 return err; 4116 } 4117 4118 static int macb_remove(struct platform_device *pdev) 4119 { 4120 struct net_device *dev; 4121 struct macb *bp; 4122 struct device_node *np = pdev->dev.of_node; 4123 4124 dev = platform_get_drvdata(pdev); 4125 4126 if (dev) { 4127 bp = netdev_priv(dev); 4128 if (dev->phydev) 4129 phy_disconnect(dev->phydev); 4130 mdiobus_unregister(bp->mii_bus); 4131 if (np && of_phy_is_fixed_link(np)) 4132 of_phy_deregister_fixed_link(np); 4133 dev->phydev = NULL; 4134 mdiobus_free(bp->mii_bus); 4135 4136 unregister_netdev(dev); 4137 clk_disable_unprepare(bp->tx_clk); 4138 clk_disable_unprepare(bp->hclk); 4139 clk_disable_unprepare(bp->pclk); 4140 clk_disable_unprepare(bp->rx_clk); 4141 of_node_put(bp->phy_node); 4142 free_netdev(dev); 4143 } 4144 4145 return 0; 4146 } 4147 4148 static int __maybe_unused macb_suspend(struct device *dev) 4149 { 4150 struct platform_device *pdev = to_platform_device(dev); 4151 struct net_device *netdev = platform_get_drvdata(pdev); 4152 struct macb *bp = netdev_priv(netdev); 4153 4154 netif_carrier_off(netdev); 4155 netif_device_detach(netdev); 4156 4157 if (bp->wol & MACB_WOL_ENABLED) { 4158 macb_writel(bp, IER, MACB_BIT(WOL)); 4159 macb_writel(bp, WOL, MACB_BIT(MAG)); 4160 enable_irq_wake(bp->queues[0].irq); 4161 } else { 4162 clk_disable_unprepare(bp->tx_clk); 4163 clk_disable_unprepare(bp->hclk); 4164 clk_disable_unprepare(bp->pclk); 4165 clk_disable_unprepare(bp->rx_clk); 4166 } 4167 4168 return 0; 4169 } 4170 4171 static int __maybe_unused macb_resume(struct device *dev) 4172 { 4173 struct platform_device *pdev = to_platform_device(dev); 4174 struct net_device *netdev = platform_get_drvdata(pdev); 4175 struct macb *bp = netdev_priv(netdev); 4176 4177 if (bp->wol & MACB_WOL_ENABLED) { 4178 macb_writel(bp, IDR, MACB_BIT(WOL)); 4179 macb_writel(bp, WOL, 0); 4180 disable_irq_wake(bp->queues[0].irq); 4181 } else { 4182 clk_prepare_enable(bp->pclk); 4183 clk_prepare_enable(bp->hclk); 4184 clk_prepare_enable(bp->tx_clk); 4185 clk_prepare_enable(bp->rx_clk); 4186 } 4187 4188 netif_device_attach(netdev); 4189 4190 return 0; 4191 } 4192 4193 static SIMPLE_DEV_PM_OPS(macb_pm_ops, macb_suspend, macb_resume); 4194 4195 static struct platform_driver macb_driver = { 4196 .probe = macb_probe, 4197 .remove = macb_remove, 4198 .driver = { 4199 .name = "macb", 4200 .of_match_table = of_match_ptr(macb_dt_ids), 4201 .pm = &macb_pm_ops, 4202 }, 4203 }; 4204 4205 module_platform_driver(macb_driver); 4206 4207 MODULE_LICENSE("GPL"); 4208 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4209 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4210 MODULE_ALIAS("platform:macb"); 4211