1 /* 2 * Cadence MACB/GEM Ethernet Controller driver 3 * 4 * Copyright (C) 2004-2006 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 #include <linux/clk.h> 13 #include <linux/module.h> 14 #include <linux/moduleparam.h> 15 #include <linux/kernel.h> 16 #include <linux/types.h> 17 #include <linux/circ_buf.h> 18 #include <linux/slab.h> 19 #include <linux/init.h> 20 #include <linux/io.h> 21 #include <linux/gpio.h> 22 #include <linux/gpio/consumer.h> 23 #include <linux/interrupt.h> 24 #include <linux/netdevice.h> 25 #include <linux/etherdevice.h> 26 #include <linux/dma-mapping.h> 27 #include <linux/platform_data/macb.h> 28 #include <linux/platform_device.h> 29 #include <linux/phy.h> 30 #include <linux/of.h> 31 #include <linux/of_device.h> 32 #include <linux/of_gpio.h> 33 #include <linux/of_mdio.h> 34 #include <linux/of_net.h> 35 #include <linux/ip.h> 36 #include <linux/udp.h> 37 #include <linux/tcp.h> 38 #include "macb.h" 39 40 #define MACB_RX_BUFFER_SIZE 128 41 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 42 43 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 44 #define MIN_RX_RING_SIZE 64 45 #define MAX_RX_RING_SIZE 8192 46 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 47 * (bp)->rx_ring_size) 48 49 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 50 #define MIN_TX_RING_SIZE 64 51 #define MAX_TX_RING_SIZE 4096 52 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 53 * (bp)->tx_ring_size) 54 55 /* level of occupied TX descriptors under which we wake up TX process */ 56 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 57 58 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(RXUBR) \ 59 | MACB_BIT(ISR_ROVR)) 60 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 61 | MACB_BIT(ISR_RLE) \ 62 | MACB_BIT(TXERR)) 63 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP)) 64 65 /* Max length of transmit frame must be a multiple of 8 bytes */ 66 #define MACB_TX_LEN_ALIGN 8 67 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 68 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 69 70 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 71 #define MACB_NETIF_LSO NETIF_F_TSO 72 73 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 74 #define MACB_WOL_ENABLED (0x1 << 1) 75 76 /* Graceful stop timeouts in us. We should allow up to 77 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 78 */ 79 #define MACB_HALT_TIMEOUT 1230 80 81 /* DMA buffer descriptor might be different size 82 * depends on hardware configuration: 83 * 84 * 1. dma address width 32 bits: 85 * word 1: 32 bit address of Data Buffer 86 * word 2: control 87 * 88 * 2. dma address width 64 bits: 89 * word 1: 32 bit address of Data Buffer 90 * word 2: control 91 * word 3: upper 32 bit address of Data Buffer 92 * word 4: unused 93 * 94 * 3. dma address width 32 bits with hardware timestamping: 95 * word 1: 32 bit address of Data Buffer 96 * word 2: control 97 * word 3: timestamp word 1 98 * word 4: timestamp word 2 99 * 100 * 4. dma address width 64 bits with hardware timestamping: 101 * word 1: 32 bit address of Data Buffer 102 * word 2: control 103 * word 3: upper 32 bit address of Data Buffer 104 * word 4: unused 105 * word 5: timestamp word 1 106 * word 6: timestamp word 2 107 */ 108 static unsigned int macb_dma_desc_get_size(struct macb *bp) 109 { 110 #ifdef MACB_EXT_DESC 111 unsigned int desc_size; 112 113 switch (bp->hw_dma_cap) { 114 case HW_DMA_CAP_64B: 115 desc_size = sizeof(struct macb_dma_desc) 116 + sizeof(struct macb_dma_desc_64); 117 break; 118 case HW_DMA_CAP_PTP: 119 desc_size = sizeof(struct macb_dma_desc) 120 + sizeof(struct macb_dma_desc_ptp); 121 break; 122 case HW_DMA_CAP_64B_PTP: 123 desc_size = sizeof(struct macb_dma_desc) 124 + sizeof(struct macb_dma_desc_64) 125 + sizeof(struct macb_dma_desc_ptp); 126 break; 127 default: 128 desc_size = sizeof(struct macb_dma_desc); 129 } 130 return desc_size; 131 #endif 132 return sizeof(struct macb_dma_desc); 133 } 134 135 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 136 { 137 #ifdef MACB_EXT_DESC 138 switch (bp->hw_dma_cap) { 139 case HW_DMA_CAP_64B: 140 case HW_DMA_CAP_PTP: 141 desc_idx <<= 1; 142 break; 143 case HW_DMA_CAP_64B_PTP: 144 desc_idx *= 3; 145 break; 146 default: 147 break; 148 } 149 #endif 150 return desc_idx; 151 } 152 153 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 154 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 155 { 156 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 157 return (struct macb_dma_desc_64 *)((void *)desc + sizeof(struct macb_dma_desc)); 158 return NULL; 159 } 160 #endif 161 162 /* Ring buffer accessors */ 163 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 164 { 165 return index & (bp->tx_ring_size - 1); 166 } 167 168 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 169 unsigned int index) 170 { 171 index = macb_tx_ring_wrap(queue->bp, index); 172 index = macb_adj_dma_desc_idx(queue->bp, index); 173 return &queue->tx_ring[index]; 174 } 175 176 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 177 unsigned int index) 178 { 179 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 180 } 181 182 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 183 { 184 dma_addr_t offset; 185 186 offset = macb_tx_ring_wrap(queue->bp, index) * 187 macb_dma_desc_get_size(queue->bp); 188 189 return queue->tx_ring_dma + offset; 190 } 191 192 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 193 { 194 return index & (bp->rx_ring_size - 1); 195 } 196 197 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 198 { 199 index = macb_rx_ring_wrap(queue->bp, index); 200 index = macb_adj_dma_desc_idx(queue->bp, index); 201 return &queue->rx_ring[index]; 202 } 203 204 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 205 { 206 return queue->rx_buffers + queue->bp->rx_buffer_size * 207 macb_rx_ring_wrap(queue->bp, index); 208 } 209 210 /* I/O accessors */ 211 static u32 hw_readl_native(struct macb *bp, int offset) 212 { 213 return __raw_readl(bp->regs + offset); 214 } 215 216 static void hw_writel_native(struct macb *bp, int offset, u32 value) 217 { 218 __raw_writel(value, bp->regs + offset); 219 } 220 221 static u32 hw_readl(struct macb *bp, int offset) 222 { 223 return readl_relaxed(bp->regs + offset); 224 } 225 226 static void hw_writel(struct macb *bp, int offset, u32 value) 227 { 228 writel_relaxed(value, bp->regs + offset); 229 } 230 231 /* Find the CPU endianness by using the loopback bit of NCR register. When the 232 * CPU is in big endian we need to program swapped mode for management 233 * descriptor access. 234 */ 235 static bool hw_is_native_io(void __iomem *addr) 236 { 237 u32 value = MACB_BIT(LLB); 238 239 __raw_writel(value, addr + MACB_NCR); 240 value = __raw_readl(addr + MACB_NCR); 241 242 /* Write 0 back to disable everything */ 243 __raw_writel(0, addr + MACB_NCR); 244 245 return value == MACB_BIT(LLB); 246 } 247 248 static bool hw_is_gem(void __iomem *addr, bool native_io) 249 { 250 u32 id; 251 252 if (native_io) 253 id = __raw_readl(addr + MACB_MID); 254 else 255 id = readl_relaxed(addr + MACB_MID); 256 257 return MACB_BFEXT(IDNUM, id) >= 0x2; 258 } 259 260 static void macb_set_hwaddr(struct macb *bp) 261 { 262 u32 bottom; 263 u16 top; 264 265 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 266 macb_or_gem_writel(bp, SA1B, bottom); 267 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 268 macb_or_gem_writel(bp, SA1T, top); 269 270 /* Clear unused address register sets */ 271 macb_or_gem_writel(bp, SA2B, 0); 272 macb_or_gem_writel(bp, SA2T, 0); 273 macb_or_gem_writel(bp, SA3B, 0); 274 macb_or_gem_writel(bp, SA3T, 0); 275 macb_or_gem_writel(bp, SA4B, 0); 276 macb_or_gem_writel(bp, SA4T, 0); 277 } 278 279 static void macb_get_hwaddr(struct macb *bp) 280 { 281 struct macb_platform_data *pdata; 282 u32 bottom; 283 u16 top; 284 u8 addr[6]; 285 int i; 286 287 pdata = dev_get_platdata(&bp->pdev->dev); 288 289 /* Check all 4 address register for valid address */ 290 for (i = 0; i < 4; i++) { 291 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 292 top = macb_or_gem_readl(bp, SA1T + i * 8); 293 294 if (pdata && pdata->rev_eth_addr) { 295 addr[5] = bottom & 0xff; 296 addr[4] = (bottom >> 8) & 0xff; 297 addr[3] = (bottom >> 16) & 0xff; 298 addr[2] = (bottom >> 24) & 0xff; 299 addr[1] = top & 0xff; 300 addr[0] = (top & 0xff00) >> 8; 301 } else { 302 addr[0] = bottom & 0xff; 303 addr[1] = (bottom >> 8) & 0xff; 304 addr[2] = (bottom >> 16) & 0xff; 305 addr[3] = (bottom >> 24) & 0xff; 306 addr[4] = top & 0xff; 307 addr[5] = (top >> 8) & 0xff; 308 } 309 310 if (is_valid_ether_addr(addr)) { 311 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 312 return; 313 } 314 } 315 316 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 317 eth_hw_addr_random(bp->dev); 318 } 319 320 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 321 { 322 struct macb *bp = bus->priv; 323 int value; 324 325 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 326 | MACB_BF(RW, MACB_MAN_READ) 327 | MACB_BF(PHYA, mii_id) 328 | MACB_BF(REGA, regnum) 329 | MACB_BF(CODE, MACB_MAN_CODE))); 330 331 /* wait for end of transfer */ 332 while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) 333 cpu_relax(); 334 335 value = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 336 337 return value; 338 } 339 340 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 341 u16 value) 342 { 343 struct macb *bp = bus->priv; 344 345 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_SOF) 346 | MACB_BF(RW, MACB_MAN_WRITE) 347 | MACB_BF(PHYA, mii_id) 348 | MACB_BF(REGA, regnum) 349 | MACB_BF(CODE, MACB_MAN_CODE) 350 | MACB_BF(DATA, value))); 351 352 /* wait for end of transfer */ 353 while (!MACB_BFEXT(IDLE, macb_readl(bp, NSR))) 354 cpu_relax(); 355 356 return 0; 357 } 358 359 /** 360 * macb_set_tx_clk() - Set a clock to a new frequency 361 * @clk Pointer to the clock to change 362 * @rate New frequency in Hz 363 * @dev Pointer to the struct net_device 364 */ 365 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 366 { 367 long ferr, rate, rate_rounded; 368 369 if (!clk) 370 return; 371 372 switch (speed) { 373 case SPEED_10: 374 rate = 2500000; 375 break; 376 case SPEED_100: 377 rate = 25000000; 378 break; 379 case SPEED_1000: 380 rate = 125000000; 381 break; 382 default: 383 return; 384 } 385 386 rate_rounded = clk_round_rate(clk, rate); 387 if (rate_rounded < 0) 388 return; 389 390 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 391 * is not satisfied. 392 */ 393 ferr = abs(rate_rounded - rate); 394 ferr = DIV_ROUND_UP(ferr, rate / 100000); 395 if (ferr > 5) 396 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 397 rate); 398 399 if (clk_set_rate(clk, rate_rounded)) 400 netdev_err(dev, "adjusting tx_clk failed.\n"); 401 } 402 403 static void macb_handle_link_change(struct net_device *dev) 404 { 405 struct macb *bp = netdev_priv(dev); 406 struct phy_device *phydev = dev->phydev; 407 unsigned long flags; 408 int status_change = 0; 409 410 spin_lock_irqsave(&bp->lock, flags); 411 412 if (phydev->link) { 413 if ((bp->speed != phydev->speed) || 414 (bp->duplex != phydev->duplex)) { 415 u32 reg; 416 417 reg = macb_readl(bp, NCFGR); 418 reg &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 419 if (macb_is_gem(bp)) 420 reg &= ~GEM_BIT(GBE); 421 422 if (phydev->duplex) 423 reg |= MACB_BIT(FD); 424 if (phydev->speed == SPEED_100) 425 reg |= MACB_BIT(SPD); 426 if (phydev->speed == SPEED_1000 && 427 bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 428 reg |= GEM_BIT(GBE); 429 430 macb_or_gem_writel(bp, NCFGR, reg); 431 432 bp->speed = phydev->speed; 433 bp->duplex = phydev->duplex; 434 status_change = 1; 435 } 436 } 437 438 if (phydev->link != bp->link) { 439 if (!phydev->link) { 440 bp->speed = 0; 441 bp->duplex = -1; 442 } 443 bp->link = phydev->link; 444 445 status_change = 1; 446 } 447 448 spin_unlock_irqrestore(&bp->lock, flags); 449 450 if (status_change) { 451 if (phydev->link) { 452 /* Update the TX clock rate if and only if the link is 453 * up and there has been a link change. 454 */ 455 macb_set_tx_clk(bp->tx_clk, phydev->speed, dev); 456 457 netif_carrier_on(dev); 458 netdev_info(dev, "link up (%d/%s)\n", 459 phydev->speed, 460 phydev->duplex == DUPLEX_FULL ? 461 "Full" : "Half"); 462 } else { 463 netif_carrier_off(dev); 464 netdev_info(dev, "link down\n"); 465 } 466 } 467 } 468 469 /* based on au1000_eth. c*/ 470 static int macb_mii_probe(struct net_device *dev) 471 { 472 struct macb *bp = netdev_priv(dev); 473 struct macb_platform_data *pdata; 474 struct phy_device *phydev; 475 int phy_irq; 476 int ret; 477 478 if (bp->phy_node) { 479 phydev = of_phy_connect(dev, bp->phy_node, 480 &macb_handle_link_change, 0, 481 bp->phy_interface); 482 if (!phydev) 483 return -ENODEV; 484 } else { 485 phydev = phy_find_first(bp->mii_bus); 486 if (!phydev) { 487 netdev_err(dev, "no PHY found\n"); 488 return -ENXIO; 489 } 490 491 pdata = dev_get_platdata(&bp->pdev->dev); 492 if (pdata) { 493 if (gpio_is_valid(pdata->phy_irq_pin)) { 494 ret = devm_gpio_request(&bp->pdev->dev, 495 pdata->phy_irq_pin, "phy int"); 496 if (!ret) { 497 phy_irq = gpio_to_irq(pdata->phy_irq_pin); 498 phydev->irq = (phy_irq < 0) ? PHY_POLL : phy_irq; 499 } 500 } else { 501 phydev->irq = PHY_POLL; 502 } 503 } 504 505 /* attach the mac to the phy */ 506 ret = phy_connect_direct(dev, phydev, &macb_handle_link_change, 507 bp->phy_interface); 508 if (ret) { 509 netdev_err(dev, "Could not attach to PHY\n"); 510 return ret; 511 } 512 } 513 514 /* mask with MAC supported features */ 515 if (macb_is_gem(bp) && bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE) 516 phydev->supported &= PHY_GBIT_FEATURES; 517 else 518 phydev->supported &= PHY_BASIC_FEATURES; 519 520 if (bp->caps & MACB_CAPS_NO_GIGABIT_HALF) 521 phydev->supported &= ~SUPPORTED_1000baseT_Half; 522 523 phydev->advertising = phydev->supported; 524 525 bp->link = 0; 526 bp->speed = 0; 527 bp->duplex = -1; 528 529 return 0; 530 } 531 532 static int macb_mii_init(struct macb *bp) 533 { 534 struct macb_platform_data *pdata; 535 struct device_node *np; 536 int err = -ENXIO, i; 537 538 /* Enable management port */ 539 macb_writel(bp, NCR, MACB_BIT(MPE)); 540 541 bp->mii_bus = mdiobus_alloc(); 542 if (!bp->mii_bus) { 543 err = -ENOMEM; 544 goto err_out; 545 } 546 547 bp->mii_bus->name = "MACB_mii_bus"; 548 bp->mii_bus->read = &macb_mdio_read; 549 bp->mii_bus->write = &macb_mdio_write; 550 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 551 bp->pdev->name, bp->pdev->id); 552 bp->mii_bus->priv = bp; 553 bp->mii_bus->parent = &bp->pdev->dev; 554 pdata = dev_get_platdata(&bp->pdev->dev); 555 556 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 557 558 np = bp->pdev->dev.of_node; 559 if (np) { 560 if (of_phy_is_fixed_link(np)) { 561 if (of_phy_register_fixed_link(np) < 0) { 562 dev_err(&bp->pdev->dev, 563 "broken fixed-link specification\n"); 564 goto err_out_unregister_bus; 565 } 566 bp->phy_node = of_node_get(np); 567 568 err = mdiobus_register(bp->mii_bus); 569 } else { 570 /* try dt phy registration */ 571 err = of_mdiobus_register(bp->mii_bus, np); 572 573 /* fallback to standard phy registration if no phy were 574 * found during dt phy registration 575 */ 576 if (!err && !phy_find_first(bp->mii_bus)) { 577 for (i = 0; i < PHY_MAX_ADDR; i++) { 578 struct phy_device *phydev; 579 580 phydev = mdiobus_scan(bp->mii_bus, i); 581 if (IS_ERR(phydev) && 582 PTR_ERR(phydev) != -ENODEV) { 583 err = PTR_ERR(phydev); 584 break; 585 } 586 } 587 588 if (err) 589 goto err_out_unregister_bus; 590 } 591 } 592 } else { 593 for (i = 0; i < PHY_MAX_ADDR; i++) 594 bp->mii_bus->irq[i] = PHY_POLL; 595 596 if (pdata) 597 bp->mii_bus->phy_mask = pdata->phy_mask; 598 599 err = mdiobus_register(bp->mii_bus); 600 } 601 602 if (err) 603 goto err_out_free_mdiobus; 604 605 err = macb_mii_probe(bp->dev); 606 if (err) 607 goto err_out_unregister_bus; 608 609 return 0; 610 611 err_out_unregister_bus: 612 mdiobus_unregister(bp->mii_bus); 613 err_out_free_mdiobus: 614 of_node_put(bp->phy_node); 615 if (np && of_phy_is_fixed_link(np)) 616 of_phy_deregister_fixed_link(np); 617 mdiobus_free(bp->mii_bus); 618 err_out: 619 return err; 620 } 621 622 static void macb_update_stats(struct macb *bp) 623 { 624 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 625 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 626 int offset = MACB_PFR; 627 628 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 629 630 for (; p < end; p++, offset += 4) 631 *p += bp->macb_reg_readl(bp, offset); 632 } 633 634 static int macb_halt_tx(struct macb *bp) 635 { 636 unsigned long halt_time, timeout; 637 u32 status; 638 639 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 640 641 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 642 do { 643 halt_time = jiffies; 644 status = macb_readl(bp, TSR); 645 if (!(status & MACB_BIT(TGO))) 646 return 0; 647 648 usleep_range(10, 250); 649 } while (time_before(halt_time, timeout)); 650 651 return -ETIMEDOUT; 652 } 653 654 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 655 { 656 if (tx_skb->mapping) { 657 if (tx_skb->mapped_as_page) 658 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 659 tx_skb->size, DMA_TO_DEVICE); 660 else 661 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 662 tx_skb->size, DMA_TO_DEVICE); 663 tx_skb->mapping = 0; 664 } 665 666 if (tx_skb->skb) { 667 dev_kfree_skb_any(tx_skb->skb); 668 tx_skb->skb = NULL; 669 } 670 } 671 672 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 673 { 674 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 675 struct macb_dma_desc_64 *desc_64; 676 677 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 678 desc_64 = macb_64b_desc(bp, desc); 679 desc_64->addrh = upper_32_bits(addr); 680 } 681 #endif 682 desc->addr = lower_32_bits(addr); 683 } 684 685 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 686 { 687 dma_addr_t addr = 0; 688 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 689 struct macb_dma_desc_64 *desc_64; 690 691 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 692 desc_64 = macb_64b_desc(bp, desc); 693 addr = ((u64)(desc_64->addrh) << 32); 694 } 695 #endif 696 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 697 return addr; 698 } 699 700 static void macb_tx_error_task(struct work_struct *work) 701 { 702 struct macb_queue *queue = container_of(work, struct macb_queue, 703 tx_error_task); 704 struct macb *bp = queue->bp; 705 struct macb_tx_skb *tx_skb; 706 struct macb_dma_desc *desc; 707 struct sk_buff *skb; 708 unsigned int tail; 709 unsigned long flags; 710 711 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 712 (unsigned int)(queue - bp->queues), 713 queue->tx_tail, queue->tx_head); 714 715 /* Prevent the queue IRQ handlers from running: each of them may call 716 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 717 * As explained below, we have to halt the transmission before updating 718 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 719 * network engine about the macb/gem being halted. 720 */ 721 spin_lock_irqsave(&bp->lock, flags); 722 723 /* Make sure nobody is trying to queue up new packets */ 724 netif_tx_stop_all_queues(bp->dev); 725 726 /* Stop transmission now 727 * (in case we have just queued new packets) 728 * macb/gem must be halted to write TBQP register 729 */ 730 if (macb_halt_tx(bp)) 731 /* Just complain for now, reinitializing TX path can be good */ 732 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 733 734 /* Treat frames in TX queue including the ones that caused the error. 735 * Free transmit buffers in upper layer. 736 */ 737 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 738 u32 ctrl; 739 740 desc = macb_tx_desc(queue, tail); 741 ctrl = desc->ctrl; 742 tx_skb = macb_tx_skb(queue, tail); 743 skb = tx_skb->skb; 744 745 if (ctrl & MACB_BIT(TX_USED)) { 746 /* skb is set for the last buffer of the frame */ 747 while (!skb) { 748 macb_tx_unmap(bp, tx_skb); 749 tail++; 750 tx_skb = macb_tx_skb(queue, tail); 751 skb = tx_skb->skb; 752 } 753 754 /* ctrl still refers to the first buffer descriptor 755 * since it's the only one written back by the hardware 756 */ 757 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 758 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 759 macb_tx_ring_wrap(bp, tail), 760 skb->data); 761 bp->dev->stats.tx_packets++; 762 queue->stats.tx_packets++; 763 bp->dev->stats.tx_bytes += skb->len; 764 queue->stats.tx_bytes += skb->len; 765 } 766 } else { 767 /* "Buffers exhausted mid-frame" errors may only happen 768 * if the driver is buggy, so complain loudly about 769 * those. Statistics are updated by hardware. 770 */ 771 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 772 netdev_err(bp->dev, 773 "BUG: TX buffers exhausted mid-frame\n"); 774 775 desc->ctrl = ctrl | MACB_BIT(TX_USED); 776 } 777 778 macb_tx_unmap(bp, tx_skb); 779 } 780 781 /* Set end of TX queue */ 782 desc = macb_tx_desc(queue, 0); 783 macb_set_addr(bp, desc, 0); 784 desc->ctrl = MACB_BIT(TX_USED); 785 786 /* Make descriptor updates visible to hardware */ 787 wmb(); 788 789 /* Reinitialize the TX desc queue */ 790 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 791 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 792 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 793 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 794 #endif 795 /* Make TX ring reflect state of hardware */ 796 queue->tx_head = 0; 797 queue->tx_tail = 0; 798 799 /* Housework before enabling TX IRQ */ 800 macb_writel(bp, TSR, macb_readl(bp, TSR)); 801 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 802 803 /* Now we are ready to start transmission again */ 804 netif_tx_start_all_queues(bp->dev); 805 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 806 807 spin_unlock_irqrestore(&bp->lock, flags); 808 } 809 810 static void macb_tx_interrupt(struct macb_queue *queue) 811 { 812 unsigned int tail; 813 unsigned int head; 814 u32 status; 815 struct macb *bp = queue->bp; 816 u16 queue_index = queue - bp->queues; 817 818 status = macb_readl(bp, TSR); 819 macb_writel(bp, TSR, status); 820 821 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 822 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 823 824 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 825 (unsigned long)status); 826 827 head = queue->tx_head; 828 for (tail = queue->tx_tail; tail != head; tail++) { 829 struct macb_tx_skb *tx_skb; 830 struct sk_buff *skb; 831 struct macb_dma_desc *desc; 832 u32 ctrl; 833 834 desc = macb_tx_desc(queue, tail); 835 836 /* Make hw descriptor updates visible to CPU */ 837 rmb(); 838 839 ctrl = desc->ctrl; 840 841 /* TX_USED bit is only set by hardware on the very first buffer 842 * descriptor of the transmitted frame. 843 */ 844 if (!(ctrl & MACB_BIT(TX_USED))) 845 break; 846 847 /* Process all buffers of the current transmitted frame */ 848 for (;; tail++) { 849 tx_skb = macb_tx_skb(queue, tail); 850 skb = tx_skb->skb; 851 852 /* First, update TX stats if needed */ 853 if (skb) { 854 if (gem_ptp_do_txstamp(queue, skb, desc) == 0) { 855 /* skb now belongs to timestamp buffer 856 * and will be removed later 857 */ 858 tx_skb->skb = NULL; 859 } 860 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 861 macb_tx_ring_wrap(bp, tail), 862 skb->data); 863 bp->dev->stats.tx_packets++; 864 queue->stats.tx_packets++; 865 bp->dev->stats.tx_bytes += skb->len; 866 queue->stats.tx_bytes += skb->len; 867 } 868 869 /* Now we can safely release resources */ 870 macb_tx_unmap(bp, tx_skb); 871 872 /* skb is set only for the last buffer of the frame. 873 * WARNING: at this point skb has been freed by 874 * macb_tx_unmap(). 875 */ 876 if (skb) 877 break; 878 } 879 } 880 881 queue->tx_tail = tail; 882 if (__netif_subqueue_stopped(bp->dev, queue_index) && 883 CIRC_CNT(queue->tx_head, queue->tx_tail, 884 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 885 netif_wake_subqueue(bp->dev, queue_index); 886 } 887 888 static void gem_rx_refill(struct macb_queue *queue) 889 { 890 unsigned int entry; 891 struct sk_buff *skb; 892 dma_addr_t paddr; 893 struct macb *bp = queue->bp; 894 struct macb_dma_desc *desc; 895 896 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 897 bp->rx_ring_size) > 0) { 898 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 899 900 /* Make hw descriptor updates visible to CPU */ 901 rmb(); 902 903 queue->rx_prepared_head++; 904 desc = macb_rx_desc(queue, entry); 905 906 if (!queue->rx_skbuff[entry]) { 907 /* allocate sk_buff for this free entry in ring */ 908 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 909 if (unlikely(!skb)) { 910 netdev_err(bp->dev, 911 "Unable to allocate sk_buff\n"); 912 break; 913 } 914 915 /* now fill corresponding descriptor entry */ 916 paddr = dma_map_single(&bp->pdev->dev, skb->data, 917 bp->rx_buffer_size, 918 DMA_FROM_DEVICE); 919 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 920 dev_kfree_skb(skb); 921 break; 922 } 923 924 queue->rx_skbuff[entry] = skb; 925 926 if (entry == bp->rx_ring_size - 1) 927 paddr |= MACB_BIT(RX_WRAP); 928 macb_set_addr(bp, desc, paddr); 929 desc->ctrl = 0; 930 931 /* properly align Ethernet header */ 932 skb_reserve(skb, NET_IP_ALIGN); 933 } else { 934 desc->addr &= ~MACB_BIT(RX_USED); 935 desc->ctrl = 0; 936 } 937 } 938 939 /* Make descriptor updates visible to hardware */ 940 wmb(); 941 942 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 943 queue, queue->rx_prepared_head, queue->rx_tail); 944 } 945 946 /* Mark DMA descriptors from begin up to and not including end as unused */ 947 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 948 unsigned int end) 949 { 950 unsigned int frag; 951 952 for (frag = begin; frag != end; frag++) { 953 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 954 955 desc->addr &= ~MACB_BIT(RX_USED); 956 } 957 958 /* Make descriptor updates visible to hardware */ 959 wmb(); 960 961 /* When this happens, the hardware stats registers for 962 * whatever caused this is updated, so we don't have to record 963 * anything. 964 */ 965 } 966 967 static int gem_rx(struct macb_queue *queue, int budget) 968 { 969 struct macb *bp = queue->bp; 970 unsigned int len; 971 unsigned int entry; 972 struct sk_buff *skb; 973 struct macb_dma_desc *desc; 974 int count = 0; 975 976 while (count < budget) { 977 u32 ctrl; 978 dma_addr_t addr; 979 bool rxused; 980 981 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 982 desc = macb_rx_desc(queue, entry); 983 984 /* Make hw descriptor updates visible to CPU */ 985 rmb(); 986 987 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 988 addr = macb_get_addr(bp, desc); 989 ctrl = desc->ctrl; 990 991 if (!rxused) 992 break; 993 994 queue->rx_tail++; 995 count++; 996 997 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 998 netdev_err(bp->dev, 999 "not whole frame pointed by descriptor\n"); 1000 bp->dev->stats.rx_dropped++; 1001 queue->stats.rx_dropped++; 1002 break; 1003 } 1004 skb = queue->rx_skbuff[entry]; 1005 if (unlikely(!skb)) { 1006 netdev_err(bp->dev, 1007 "inconsistent Rx descriptor chain\n"); 1008 bp->dev->stats.rx_dropped++; 1009 queue->stats.rx_dropped++; 1010 break; 1011 } 1012 /* now everything is ready for receiving packet */ 1013 queue->rx_skbuff[entry] = NULL; 1014 len = ctrl & bp->rx_frm_len_mask; 1015 1016 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1017 1018 skb_put(skb, len); 1019 dma_unmap_single(&bp->pdev->dev, addr, 1020 bp->rx_buffer_size, DMA_FROM_DEVICE); 1021 1022 skb->protocol = eth_type_trans(skb, bp->dev); 1023 skb_checksum_none_assert(skb); 1024 if (bp->dev->features & NETIF_F_RXCSUM && 1025 !(bp->dev->flags & IFF_PROMISC) && 1026 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1027 skb->ip_summed = CHECKSUM_UNNECESSARY; 1028 1029 bp->dev->stats.rx_packets++; 1030 queue->stats.rx_packets++; 1031 bp->dev->stats.rx_bytes += skb->len; 1032 queue->stats.rx_bytes += skb->len; 1033 1034 gem_ptp_do_rxstamp(bp, skb, desc); 1035 1036 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1037 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1038 skb->len, skb->csum); 1039 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1040 skb_mac_header(skb), 16, true); 1041 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1042 skb->data, 32, true); 1043 #endif 1044 1045 netif_receive_skb(skb); 1046 } 1047 1048 gem_rx_refill(queue); 1049 1050 return count; 1051 } 1052 1053 static int macb_rx_frame(struct macb_queue *queue, unsigned int first_frag, 1054 unsigned int last_frag) 1055 { 1056 unsigned int len; 1057 unsigned int frag; 1058 unsigned int offset; 1059 struct sk_buff *skb; 1060 struct macb_dma_desc *desc; 1061 struct macb *bp = queue->bp; 1062 1063 desc = macb_rx_desc(queue, last_frag); 1064 len = desc->ctrl & bp->rx_frm_len_mask; 1065 1066 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1067 macb_rx_ring_wrap(bp, first_frag), 1068 macb_rx_ring_wrap(bp, last_frag), len); 1069 1070 /* The ethernet header starts NET_IP_ALIGN bytes into the 1071 * first buffer. Since the header is 14 bytes, this makes the 1072 * payload word-aligned. 1073 * 1074 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1075 * the two padding bytes into the skb so that we avoid hitting 1076 * the slowpath in memcpy(), and pull them off afterwards. 1077 */ 1078 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1079 if (!skb) { 1080 bp->dev->stats.rx_dropped++; 1081 for (frag = first_frag; ; frag++) { 1082 desc = macb_rx_desc(queue, frag); 1083 desc->addr &= ~MACB_BIT(RX_USED); 1084 if (frag == last_frag) 1085 break; 1086 } 1087 1088 /* Make descriptor updates visible to hardware */ 1089 wmb(); 1090 1091 return 1; 1092 } 1093 1094 offset = 0; 1095 len += NET_IP_ALIGN; 1096 skb_checksum_none_assert(skb); 1097 skb_put(skb, len); 1098 1099 for (frag = first_frag; ; frag++) { 1100 unsigned int frag_len = bp->rx_buffer_size; 1101 1102 if (offset + frag_len > len) { 1103 if (unlikely(frag != last_frag)) { 1104 dev_kfree_skb_any(skb); 1105 return -1; 1106 } 1107 frag_len = len - offset; 1108 } 1109 skb_copy_to_linear_data_offset(skb, offset, 1110 macb_rx_buffer(queue, frag), 1111 frag_len); 1112 offset += bp->rx_buffer_size; 1113 desc = macb_rx_desc(queue, frag); 1114 desc->addr &= ~MACB_BIT(RX_USED); 1115 1116 if (frag == last_frag) 1117 break; 1118 } 1119 1120 /* Make descriptor updates visible to hardware */ 1121 wmb(); 1122 1123 __skb_pull(skb, NET_IP_ALIGN); 1124 skb->protocol = eth_type_trans(skb, bp->dev); 1125 1126 bp->dev->stats.rx_packets++; 1127 bp->dev->stats.rx_bytes += skb->len; 1128 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1129 skb->len, skb->csum); 1130 netif_receive_skb(skb); 1131 1132 return 0; 1133 } 1134 1135 static inline void macb_init_rx_ring(struct macb_queue *queue) 1136 { 1137 struct macb *bp = queue->bp; 1138 dma_addr_t addr; 1139 struct macb_dma_desc *desc = NULL; 1140 int i; 1141 1142 addr = queue->rx_buffers_dma; 1143 for (i = 0; i < bp->rx_ring_size; i++) { 1144 desc = macb_rx_desc(queue, i); 1145 macb_set_addr(bp, desc, addr); 1146 desc->ctrl = 0; 1147 addr += bp->rx_buffer_size; 1148 } 1149 desc->addr |= MACB_BIT(RX_WRAP); 1150 queue->rx_tail = 0; 1151 } 1152 1153 static int macb_rx(struct macb_queue *queue, int budget) 1154 { 1155 struct macb *bp = queue->bp; 1156 bool reset_rx_queue = false; 1157 int received = 0; 1158 unsigned int tail; 1159 int first_frag = -1; 1160 1161 for (tail = queue->rx_tail; budget > 0; tail++) { 1162 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1163 u32 ctrl; 1164 1165 /* Make hw descriptor updates visible to CPU */ 1166 rmb(); 1167 1168 ctrl = desc->ctrl; 1169 1170 if (!(desc->addr & MACB_BIT(RX_USED))) 1171 break; 1172 1173 if (ctrl & MACB_BIT(RX_SOF)) { 1174 if (first_frag != -1) 1175 discard_partial_frame(queue, first_frag, tail); 1176 first_frag = tail; 1177 } 1178 1179 if (ctrl & MACB_BIT(RX_EOF)) { 1180 int dropped; 1181 1182 if (unlikely(first_frag == -1)) { 1183 reset_rx_queue = true; 1184 continue; 1185 } 1186 1187 dropped = macb_rx_frame(queue, first_frag, tail); 1188 first_frag = -1; 1189 if (unlikely(dropped < 0)) { 1190 reset_rx_queue = true; 1191 continue; 1192 } 1193 if (!dropped) { 1194 received++; 1195 budget--; 1196 } 1197 } 1198 } 1199 1200 if (unlikely(reset_rx_queue)) { 1201 unsigned long flags; 1202 u32 ctrl; 1203 1204 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1205 1206 spin_lock_irqsave(&bp->lock, flags); 1207 1208 ctrl = macb_readl(bp, NCR); 1209 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1210 1211 macb_init_rx_ring(queue); 1212 queue_writel(queue, RBQP, queue->rx_ring_dma); 1213 1214 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1215 1216 spin_unlock_irqrestore(&bp->lock, flags); 1217 return received; 1218 } 1219 1220 if (first_frag != -1) 1221 queue->rx_tail = first_frag; 1222 else 1223 queue->rx_tail = tail; 1224 1225 return received; 1226 } 1227 1228 static int macb_poll(struct napi_struct *napi, int budget) 1229 { 1230 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1231 struct macb *bp = queue->bp; 1232 int work_done; 1233 u32 status; 1234 1235 status = macb_readl(bp, RSR); 1236 macb_writel(bp, RSR, status); 1237 1238 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1239 (unsigned long)status, budget); 1240 1241 work_done = bp->macbgem_ops.mog_rx(queue, budget); 1242 if (work_done < budget) { 1243 napi_complete_done(napi, work_done); 1244 1245 /* Packets received while interrupts were disabled */ 1246 status = macb_readl(bp, RSR); 1247 if (status) { 1248 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1249 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1250 napi_reschedule(napi); 1251 } else { 1252 queue_writel(queue, IER, MACB_RX_INT_FLAGS); 1253 } 1254 } 1255 1256 /* TODO: Handle errors */ 1257 1258 return work_done; 1259 } 1260 1261 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1262 { 1263 struct macb_queue *queue = dev_id; 1264 struct macb *bp = queue->bp; 1265 struct net_device *dev = bp->dev; 1266 u32 status, ctrl; 1267 1268 status = queue_readl(queue, ISR); 1269 1270 if (unlikely(!status)) 1271 return IRQ_NONE; 1272 1273 spin_lock(&bp->lock); 1274 1275 while (status) { 1276 /* close possible race with dev_close */ 1277 if (unlikely(!netif_running(dev))) { 1278 queue_writel(queue, IDR, -1); 1279 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1280 queue_writel(queue, ISR, -1); 1281 break; 1282 } 1283 1284 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1285 (unsigned int)(queue - bp->queues), 1286 (unsigned long)status); 1287 1288 if (status & MACB_RX_INT_FLAGS) { 1289 /* There's no point taking any more interrupts 1290 * until we have processed the buffers. The 1291 * scheduling call may fail if the poll routine 1292 * is already scheduled, so disable interrupts 1293 * now. 1294 */ 1295 queue_writel(queue, IDR, MACB_RX_INT_FLAGS); 1296 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1297 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1298 1299 if (napi_schedule_prep(&queue->napi)) { 1300 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1301 __napi_schedule(&queue->napi); 1302 } 1303 } 1304 1305 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1306 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1307 schedule_work(&queue->tx_error_task); 1308 1309 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1310 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1311 1312 break; 1313 } 1314 1315 if (status & MACB_BIT(TCOMP)) 1316 macb_tx_interrupt(queue); 1317 1318 /* Link change detection isn't possible with RMII, so we'll 1319 * add that if/when we get our hands on a full-blown MII PHY. 1320 */ 1321 1322 /* There is a hardware issue under heavy load where DMA can 1323 * stop, this causes endless "used buffer descriptor read" 1324 * interrupts but it can be cleared by re-enabling RX. See 1325 * the at91 manual, section 41.3.1 or the Zynq manual 1326 * section 16.7.4 for details. 1327 */ 1328 if (status & MACB_BIT(RXUBR)) { 1329 ctrl = macb_readl(bp, NCR); 1330 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1331 wmb(); 1332 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1333 1334 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1335 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1336 } 1337 1338 if (status & MACB_BIT(ISR_ROVR)) { 1339 /* We missed at least one packet */ 1340 if (macb_is_gem(bp)) 1341 bp->hw_stats.gem.rx_overruns++; 1342 else 1343 bp->hw_stats.macb.rx_overruns++; 1344 1345 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1346 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1347 } 1348 1349 if (status & MACB_BIT(HRESP)) { 1350 /* TODO: Reset the hardware, and maybe move the 1351 * netdev_err to a lower-priority context as well 1352 * (work queue?) 1353 */ 1354 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1355 1356 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1357 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1358 } 1359 status = queue_readl(queue, ISR); 1360 } 1361 1362 spin_unlock(&bp->lock); 1363 1364 return IRQ_HANDLED; 1365 } 1366 1367 #ifdef CONFIG_NET_POLL_CONTROLLER 1368 /* Polling receive - used by netconsole and other diagnostic tools 1369 * to allow network i/o with interrupts disabled. 1370 */ 1371 static void macb_poll_controller(struct net_device *dev) 1372 { 1373 struct macb *bp = netdev_priv(dev); 1374 struct macb_queue *queue; 1375 unsigned long flags; 1376 unsigned int q; 1377 1378 local_irq_save(flags); 1379 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1380 macb_interrupt(dev->irq, queue); 1381 local_irq_restore(flags); 1382 } 1383 #endif 1384 1385 static unsigned int macb_tx_map(struct macb *bp, 1386 struct macb_queue *queue, 1387 struct sk_buff *skb, 1388 unsigned int hdrlen) 1389 { 1390 dma_addr_t mapping; 1391 unsigned int len, entry, i, tx_head = queue->tx_head; 1392 struct macb_tx_skb *tx_skb = NULL; 1393 struct macb_dma_desc *desc; 1394 unsigned int offset, size, count = 0; 1395 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1396 unsigned int eof = 1, mss_mfs = 0; 1397 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1398 1399 /* LSO */ 1400 if (skb_shinfo(skb)->gso_size != 0) { 1401 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1402 /* UDP - UFO */ 1403 lso_ctrl = MACB_LSO_UFO_ENABLE; 1404 else 1405 /* TCP - TSO */ 1406 lso_ctrl = MACB_LSO_TSO_ENABLE; 1407 } 1408 1409 /* First, map non-paged data */ 1410 len = skb_headlen(skb); 1411 1412 /* first buffer length */ 1413 size = hdrlen; 1414 1415 offset = 0; 1416 while (len) { 1417 entry = macb_tx_ring_wrap(bp, tx_head); 1418 tx_skb = &queue->tx_skb[entry]; 1419 1420 mapping = dma_map_single(&bp->pdev->dev, 1421 skb->data + offset, 1422 size, DMA_TO_DEVICE); 1423 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1424 goto dma_error; 1425 1426 /* Save info to properly release resources */ 1427 tx_skb->skb = NULL; 1428 tx_skb->mapping = mapping; 1429 tx_skb->size = size; 1430 tx_skb->mapped_as_page = false; 1431 1432 len -= size; 1433 offset += size; 1434 count++; 1435 tx_head++; 1436 1437 size = min(len, bp->max_tx_length); 1438 } 1439 1440 /* Then, map paged data from fragments */ 1441 for (f = 0; f < nr_frags; f++) { 1442 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1443 1444 len = skb_frag_size(frag); 1445 offset = 0; 1446 while (len) { 1447 size = min(len, bp->max_tx_length); 1448 entry = macb_tx_ring_wrap(bp, tx_head); 1449 tx_skb = &queue->tx_skb[entry]; 1450 1451 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1452 offset, size, DMA_TO_DEVICE); 1453 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1454 goto dma_error; 1455 1456 /* Save info to properly release resources */ 1457 tx_skb->skb = NULL; 1458 tx_skb->mapping = mapping; 1459 tx_skb->size = size; 1460 tx_skb->mapped_as_page = true; 1461 1462 len -= size; 1463 offset += size; 1464 count++; 1465 tx_head++; 1466 } 1467 } 1468 1469 /* Should never happen */ 1470 if (unlikely(!tx_skb)) { 1471 netdev_err(bp->dev, "BUG! empty skb!\n"); 1472 return 0; 1473 } 1474 1475 /* This is the last buffer of the frame: save socket buffer */ 1476 tx_skb->skb = skb; 1477 1478 /* Update TX ring: update buffer descriptors in reverse order 1479 * to avoid race condition 1480 */ 1481 1482 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1483 * to set the end of TX queue 1484 */ 1485 i = tx_head; 1486 entry = macb_tx_ring_wrap(bp, i); 1487 ctrl = MACB_BIT(TX_USED); 1488 desc = macb_tx_desc(queue, entry); 1489 desc->ctrl = ctrl; 1490 1491 if (lso_ctrl) { 1492 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1493 /* include header and FCS in value given to h/w */ 1494 mss_mfs = skb_shinfo(skb)->gso_size + 1495 skb_transport_offset(skb) + 1496 ETH_FCS_LEN; 1497 else /* TSO */ { 1498 mss_mfs = skb_shinfo(skb)->gso_size; 1499 /* TCP Sequence Number Source Select 1500 * can be set only for TSO 1501 */ 1502 seq_ctrl = 0; 1503 } 1504 } 1505 1506 do { 1507 i--; 1508 entry = macb_tx_ring_wrap(bp, i); 1509 tx_skb = &queue->tx_skb[entry]; 1510 desc = macb_tx_desc(queue, entry); 1511 1512 ctrl = (u32)tx_skb->size; 1513 if (eof) { 1514 ctrl |= MACB_BIT(TX_LAST); 1515 eof = 0; 1516 } 1517 if (unlikely(entry == (bp->tx_ring_size - 1))) 1518 ctrl |= MACB_BIT(TX_WRAP); 1519 1520 /* First descriptor is header descriptor */ 1521 if (i == queue->tx_head) { 1522 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1523 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1524 } else 1525 /* Only set MSS/MFS on payload descriptors 1526 * (second or later descriptor) 1527 */ 1528 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1529 1530 /* Set TX buffer descriptor */ 1531 macb_set_addr(bp, desc, tx_skb->mapping); 1532 /* desc->addr must be visible to hardware before clearing 1533 * 'TX_USED' bit in desc->ctrl. 1534 */ 1535 wmb(); 1536 desc->ctrl = ctrl; 1537 } while (i != queue->tx_head); 1538 1539 queue->tx_head = tx_head; 1540 1541 return count; 1542 1543 dma_error: 1544 netdev_err(bp->dev, "TX DMA map failed\n"); 1545 1546 for (i = queue->tx_head; i != tx_head; i++) { 1547 tx_skb = macb_tx_skb(queue, i); 1548 1549 macb_tx_unmap(bp, tx_skb); 1550 } 1551 1552 return 0; 1553 } 1554 1555 static netdev_features_t macb_features_check(struct sk_buff *skb, 1556 struct net_device *dev, 1557 netdev_features_t features) 1558 { 1559 unsigned int nr_frags, f; 1560 unsigned int hdrlen; 1561 1562 /* Validate LSO compatibility */ 1563 1564 /* there is only one buffer */ 1565 if (!skb_is_nonlinear(skb)) 1566 return features; 1567 1568 /* length of header */ 1569 hdrlen = skb_transport_offset(skb); 1570 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 1571 hdrlen += tcp_hdrlen(skb); 1572 1573 /* For LSO: 1574 * When software supplies two or more payload buffers all payload buffers 1575 * apart from the last must be a multiple of 8 bytes in size. 1576 */ 1577 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1578 return features & ~MACB_NETIF_LSO; 1579 1580 nr_frags = skb_shinfo(skb)->nr_frags; 1581 /* No need to check last fragment */ 1582 nr_frags--; 1583 for (f = 0; f < nr_frags; f++) { 1584 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1585 1586 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1587 return features & ~MACB_NETIF_LSO; 1588 } 1589 return features; 1590 } 1591 1592 static inline int macb_clear_csum(struct sk_buff *skb) 1593 { 1594 /* no change for packets without checksum offloading */ 1595 if (skb->ip_summed != CHECKSUM_PARTIAL) 1596 return 0; 1597 1598 /* make sure we can modify the header */ 1599 if (unlikely(skb_cow_head(skb, 0))) 1600 return -1; 1601 1602 /* initialize checksum field 1603 * This is required - at least for Zynq, which otherwise calculates 1604 * wrong UDP header checksums for UDP packets with UDP data len <=2 1605 */ 1606 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1607 return 0; 1608 } 1609 1610 static int macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1611 { 1612 u16 queue_index = skb_get_queue_mapping(skb); 1613 struct macb *bp = netdev_priv(dev); 1614 struct macb_queue *queue = &bp->queues[queue_index]; 1615 unsigned long flags; 1616 unsigned int desc_cnt, nr_frags, frag_size, f; 1617 unsigned int hdrlen; 1618 bool is_lso, is_udp = 0; 1619 1620 is_lso = (skb_shinfo(skb)->gso_size != 0); 1621 1622 if (is_lso) { 1623 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP); 1624 1625 /* length of headers */ 1626 if (is_udp) 1627 /* only queue eth + ip headers separately for UDP */ 1628 hdrlen = skb_transport_offset(skb); 1629 else 1630 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 1631 if (skb_headlen(skb) < hdrlen) { 1632 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 1633 /* if this is required, would need to copy to single buffer */ 1634 return NETDEV_TX_BUSY; 1635 } 1636 } else 1637 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 1638 1639 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1640 netdev_vdbg(bp->dev, 1641 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 1642 queue_index, skb->len, skb->head, skb->data, 1643 skb_tail_pointer(skb), skb_end_pointer(skb)); 1644 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 1645 skb->data, 16, true); 1646 #endif 1647 1648 /* Count how many TX buffer descriptors are needed to send this 1649 * socket buffer: skb fragments of jumbo frames may need to be 1650 * split into many buffer descriptors. 1651 */ 1652 if (is_lso && (skb_headlen(skb) > hdrlen)) 1653 /* extra header descriptor if also payload in first buffer */ 1654 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 1655 else 1656 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 1657 nr_frags = skb_shinfo(skb)->nr_frags; 1658 for (f = 0; f < nr_frags; f++) { 1659 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 1660 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 1661 } 1662 1663 spin_lock_irqsave(&bp->lock, flags); 1664 1665 /* This is a hard error, log it. */ 1666 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 1667 bp->tx_ring_size) < desc_cnt) { 1668 netif_stop_subqueue(dev, queue_index); 1669 spin_unlock_irqrestore(&bp->lock, flags); 1670 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 1671 queue->tx_head, queue->tx_tail); 1672 return NETDEV_TX_BUSY; 1673 } 1674 1675 if (macb_clear_csum(skb)) { 1676 dev_kfree_skb_any(skb); 1677 goto unlock; 1678 } 1679 1680 /* Map socket buffer for DMA transfer */ 1681 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 1682 dev_kfree_skb_any(skb); 1683 goto unlock; 1684 } 1685 1686 /* Make newly initialized descriptor visible to hardware */ 1687 wmb(); 1688 skb_tx_timestamp(skb); 1689 1690 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1691 1692 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 1693 netif_stop_subqueue(dev, queue_index); 1694 1695 unlock: 1696 spin_unlock_irqrestore(&bp->lock, flags); 1697 1698 return NETDEV_TX_OK; 1699 } 1700 1701 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 1702 { 1703 if (!macb_is_gem(bp)) { 1704 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 1705 } else { 1706 bp->rx_buffer_size = size; 1707 1708 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 1709 netdev_dbg(bp->dev, 1710 "RX buffer must be multiple of %d bytes, expanding\n", 1711 RX_BUFFER_MULTIPLE); 1712 bp->rx_buffer_size = 1713 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 1714 } 1715 } 1716 1717 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 1718 bp->dev->mtu, bp->rx_buffer_size); 1719 } 1720 1721 static void gem_free_rx_buffers(struct macb *bp) 1722 { 1723 struct sk_buff *skb; 1724 struct macb_dma_desc *desc; 1725 struct macb_queue *queue; 1726 dma_addr_t addr; 1727 unsigned int q; 1728 int i; 1729 1730 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1731 if (!queue->rx_skbuff) 1732 continue; 1733 1734 for (i = 0; i < bp->rx_ring_size; i++) { 1735 skb = queue->rx_skbuff[i]; 1736 1737 if (!skb) 1738 continue; 1739 1740 desc = macb_rx_desc(queue, i); 1741 addr = macb_get_addr(bp, desc); 1742 1743 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 1744 DMA_FROM_DEVICE); 1745 dev_kfree_skb_any(skb); 1746 skb = NULL; 1747 } 1748 1749 kfree(queue->rx_skbuff); 1750 queue->rx_skbuff = NULL; 1751 } 1752 } 1753 1754 static void macb_free_rx_buffers(struct macb *bp) 1755 { 1756 struct macb_queue *queue = &bp->queues[0]; 1757 1758 if (queue->rx_buffers) { 1759 dma_free_coherent(&bp->pdev->dev, 1760 bp->rx_ring_size * bp->rx_buffer_size, 1761 queue->rx_buffers, queue->rx_buffers_dma); 1762 queue->rx_buffers = NULL; 1763 } 1764 } 1765 1766 static void macb_free_consistent(struct macb *bp) 1767 { 1768 struct macb_queue *queue; 1769 unsigned int q; 1770 1771 queue = &bp->queues[0]; 1772 bp->macbgem_ops.mog_free_rx_buffers(bp); 1773 if (queue->rx_ring) { 1774 dma_free_coherent(&bp->pdev->dev, RX_RING_BYTES(bp), 1775 queue->rx_ring, queue->rx_ring_dma); 1776 queue->rx_ring = NULL; 1777 } 1778 1779 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1780 kfree(queue->tx_skb); 1781 queue->tx_skb = NULL; 1782 if (queue->tx_ring) { 1783 dma_free_coherent(&bp->pdev->dev, TX_RING_BYTES(bp), 1784 queue->tx_ring, queue->tx_ring_dma); 1785 queue->tx_ring = NULL; 1786 } 1787 } 1788 } 1789 1790 static int gem_alloc_rx_buffers(struct macb *bp) 1791 { 1792 struct macb_queue *queue; 1793 unsigned int q; 1794 int size; 1795 1796 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1797 size = bp->rx_ring_size * sizeof(struct sk_buff *); 1798 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 1799 if (!queue->rx_skbuff) 1800 return -ENOMEM; 1801 else 1802 netdev_dbg(bp->dev, 1803 "Allocated %d RX struct sk_buff entries at %p\n", 1804 bp->rx_ring_size, queue->rx_skbuff); 1805 } 1806 return 0; 1807 } 1808 1809 static int macb_alloc_rx_buffers(struct macb *bp) 1810 { 1811 struct macb_queue *queue = &bp->queues[0]; 1812 int size; 1813 1814 size = bp->rx_ring_size * bp->rx_buffer_size; 1815 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 1816 &queue->rx_buffers_dma, GFP_KERNEL); 1817 if (!queue->rx_buffers) 1818 return -ENOMEM; 1819 1820 netdev_dbg(bp->dev, 1821 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 1822 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 1823 return 0; 1824 } 1825 1826 static int macb_alloc_consistent(struct macb *bp) 1827 { 1828 struct macb_queue *queue; 1829 unsigned int q; 1830 int size; 1831 1832 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1833 size = TX_RING_BYTES(bp); 1834 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1835 &queue->tx_ring_dma, 1836 GFP_KERNEL); 1837 if (!queue->tx_ring) 1838 goto out_err; 1839 netdev_dbg(bp->dev, 1840 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 1841 q, size, (unsigned long)queue->tx_ring_dma, 1842 queue->tx_ring); 1843 1844 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 1845 queue->tx_skb = kmalloc(size, GFP_KERNEL); 1846 if (!queue->tx_skb) 1847 goto out_err; 1848 1849 size = RX_RING_BYTES(bp); 1850 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 1851 &queue->rx_ring_dma, GFP_KERNEL); 1852 if (!queue->rx_ring) 1853 goto out_err; 1854 netdev_dbg(bp->dev, 1855 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 1856 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 1857 } 1858 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 1859 goto out_err; 1860 1861 return 0; 1862 1863 out_err: 1864 macb_free_consistent(bp); 1865 return -ENOMEM; 1866 } 1867 1868 static void gem_init_rings(struct macb *bp) 1869 { 1870 struct macb_queue *queue; 1871 struct macb_dma_desc *desc = NULL; 1872 unsigned int q; 1873 int i; 1874 1875 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1876 for (i = 0; i < bp->tx_ring_size; i++) { 1877 desc = macb_tx_desc(queue, i); 1878 macb_set_addr(bp, desc, 0); 1879 desc->ctrl = MACB_BIT(TX_USED); 1880 } 1881 desc->ctrl |= MACB_BIT(TX_WRAP); 1882 queue->tx_head = 0; 1883 queue->tx_tail = 0; 1884 1885 queue->rx_tail = 0; 1886 queue->rx_prepared_head = 0; 1887 1888 gem_rx_refill(queue); 1889 } 1890 1891 } 1892 1893 static void macb_init_rings(struct macb *bp) 1894 { 1895 int i; 1896 struct macb_dma_desc *desc = NULL; 1897 1898 macb_init_rx_ring(&bp->queues[0]); 1899 1900 for (i = 0; i < bp->tx_ring_size; i++) { 1901 desc = macb_tx_desc(&bp->queues[0], i); 1902 macb_set_addr(bp, desc, 0); 1903 desc->ctrl = MACB_BIT(TX_USED); 1904 } 1905 bp->queues[0].tx_head = 0; 1906 bp->queues[0].tx_tail = 0; 1907 desc->ctrl |= MACB_BIT(TX_WRAP); 1908 } 1909 1910 static void macb_reset_hw(struct macb *bp) 1911 { 1912 struct macb_queue *queue; 1913 unsigned int q; 1914 1915 /* Disable RX and TX (XXX: Should we halt the transmission 1916 * more gracefully?) 1917 */ 1918 macb_writel(bp, NCR, 0); 1919 1920 /* Clear the stats registers (XXX: Update stats first?) */ 1921 macb_writel(bp, NCR, MACB_BIT(CLRSTAT)); 1922 1923 /* Clear all status flags */ 1924 macb_writel(bp, TSR, -1); 1925 macb_writel(bp, RSR, -1); 1926 1927 /* Disable all interrupts */ 1928 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1929 queue_writel(queue, IDR, -1); 1930 queue_readl(queue, ISR); 1931 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1932 queue_writel(queue, ISR, -1); 1933 } 1934 } 1935 1936 static u32 gem_mdc_clk_div(struct macb *bp) 1937 { 1938 u32 config; 1939 unsigned long pclk_hz = clk_get_rate(bp->pclk); 1940 1941 if (pclk_hz <= 20000000) 1942 config = GEM_BF(CLK, GEM_CLK_DIV8); 1943 else if (pclk_hz <= 40000000) 1944 config = GEM_BF(CLK, GEM_CLK_DIV16); 1945 else if (pclk_hz <= 80000000) 1946 config = GEM_BF(CLK, GEM_CLK_DIV32); 1947 else if (pclk_hz <= 120000000) 1948 config = GEM_BF(CLK, GEM_CLK_DIV48); 1949 else if (pclk_hz <= 160000000) 1950 config = GEM_BF(CLK, GEM_CLK_DIV64); 1951 else 1952 config = GEM_BF(CLK, GEM_CLK_DIV96); 1953 1954 return config; 1955 } 1956 1957 static u32 macb_mdc_clk_div(struct macb *bp) 1958 { 1959 u32 config; 1960 unsigned long pclk_hz; 1961 1962 if (macb_is_gem(bp)) 1963 return gem_mdc_clk_div(bp); 1964 1965 pclk_hz = clk_get_rate(bp->pclk); 1966 if (pclk_hz <= 20000000) 1967 config = MACB_BF(CLK, MACB_CLK_DIV8); 1968 else if (pclk_hz <= 40000000) 1969 config = MACB_BF(CLK, MACB_CLK_DIV16); 1970 else if (pclk_hz <= 80000000) 1971 config = MACB_BF(CLK, MACB_CLK_DIV32); 1972 else 1973 config = MACB_BF(CLK, MACB_CLK_DIV64); 1974 1975 return config; 1976 } 1977 1978 /* Get the DMA bus width field of the network configuration register that we 1979 * should program. We find the width from decoding the design configuration 1980 * register to find the maximum supported data bus width. 1981 */ 1982 static u32 macb_dbw(struct macb *bp) 1983 { 1984 if (!macb_is_gem(bp)) 1985 return 0; 1986 1987 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 1988 case 4: 1989 return GEM_BF(DBW, GEM_DBW128); 1990 case 2: 1991 return GEM_BF(DBW, GEM_DBW64); 1992 case 1: 1993 default: 1994 return GEM_BF(DBW, GEM_DBW32); 1995 } 1996 } 1997 1998 /* Configure the receive DMA engine 1999 * - use the correct receive buffer size 2000 * - set best burst length for DMA operations 2001 * (if not supported by FIFO, it will fallback to default) 2002 * - set both rx/tx packet buffers to full memory size 2003 * These are configurable parameters for GEM. 2004 */ 2005 static void macb_configure_dma(struct macb *bp) 2006 { 2007 struct macb_queue *queue; 2008 u32 buffer_size; 2009 unsigned int q; 2010 u32 dmacfg; 2011 2012 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2013 if (macb_is_gem(bp)) { 2014 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2015 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2016 if (q) 2017 queue_writel(queue, RBQS, buffer_size); 2018 else 2019 dmacfg |= GEM_BF(RXBS, buffer_size); 2020 } 2021 if (bp->dma_burst_length) 2022 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2023 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2024 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2025 2026 if (bp->native_io) 2027 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2028 else 2029 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2030 2031 if (bp->dev->features & NETIF_F_HW_CSUM) 2032 dmacfg |= GEM_BIT(TXCOEN); 2033 else 2034 dmacfg &= ~GEM_BIT(TXCOEN); 2035 2036 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2037 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2038 dmacfg |= GEM_BIT(ADDR64); 2039 #endif 2040 #ifdef CONFIG_MACB_USE_HWSTAMP 2041 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2042 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2043 #endif 2044 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2045 dmacfg); 2046 gem_writel(bp, DMACFG, dmacfg); 2047 } 2048 } 2049 2050 static void macb_init_hw(struct macb *bp) 2051 { 2052 struct macb_queue *queue; 2053 unsigned int q; 2054 2055 u32 config; 2056 2057 macb_reset_hw(bp); 2058 macb_set_hwaddr(bp); 2059 2060 config = macb_mdc_clk_div(bp); 2061 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 2062 config |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 2063 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2064 config |= MACB_BIT(PAE); /* PAuse Enable */ 2065 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2066 if (bp->caps & MACB_CAPS_JUMBO) 2067 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2068 else 2069 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2070 if (bp->dev->flags & IFF_PROMISC) 2071 config |= MACB_BIT(CAF); /* Copy All Frames */ 2072 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2073 config |= GEM_BIT(RXCOEN); 2074 if (!(bp->dev->flags & IFF_BROADCAST)) 2075 config |= MACB_BIT(NBC); /* No BroadCast */ 2076 config |= macb_dbw(bp); 2077 macb_writel(bp, NCFGR, config); 2078 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2079 gem_writel(bp, JML, bp->jumbo_max_len); 2080 bp->speed = SPEED_10; 2081 bp->duplex = DUPLEX_HALF; 2082 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2083 if (bp->caps & MACB_CAPS_JUMBO) 2084 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2085 2086 macb_configure_dma(bp); 2087 2088 /* Initialize TX and RX buffers */ 2089 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2090 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 2091 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2092 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2093 queue_writel(queue, RBQPH, upper_32_bits(queue->rx_ring_dma)); 2094 #endif 2095 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 2096 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2097 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2098 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 2099 #endif 2100 2101 /* Enable interrupts */ 2102 queue_writel(queue, IER, 2103 MACB_RX_INT_FLAGS | 2104 MACB_TX_INT_FLAGS | 2105 MACB_BIT(HRESP)); 2106 } 2107 2108 /* Enable TX and RX */ 2109 macb_writel(bp, NCR, MACB_BIT(RE) | MACB_BIT(TE) | MACB_BIT(MPE)); 2110 } 2111 2112 /* The hash address register is 64 bits long and takes up two 2113 * locations in the memory map. The least significant bits are stored 2114 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2115 * 2116 * The unicast hash enable and the multicast hash enable bits in the 2117 * network configuration register enable the reception of hash matched 2118 * frames. The destination address is reduced to a 6 bit index into 2119 * the 64 bit hash register using the following hash function. The 2120 * hash function is an exclusive or of every sixth bit of the 2121 * destination address. 2122 * 2123 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2124 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2125 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2126 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2127 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2128 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2129 * 2130 * da[0] represents the least significant bit of the first byte 2131 * received, that is, the multicast/unicast indicator, and da[47] 2132 * represents the most significant bit of the last byte received. If 2133 * the hash index, hi[n], points to a bit that is set in the hash 2134 * register then the frame will be matched according to whether the 2135 * frame is multicast or unicast. A multicast match will be signalled 2136 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2137 * index points to a bit set in the hash register. A unicast match 2138 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2139 * and the hash index points to a bit set in the hash register. To 2140 * receive all multicast frames, the hash register should be set with 2141 * all ones and the multicast hash enable bit should be set in the 2142 * network configuration register. 2143 */ 2144 2145 static inline int hash_bit_value(int bitnr, __u8 *addr) 2146 { 2147 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2148 return 1; 2149 return 0; 2150 } 2151 2152 /* Return the hash index value for the specified address. */ 2153 static int hash_get_index(__u8 *addr) 2154 { 2155 int i, j, bitval; 2156 int hash_index = 0; 2157 2158 for (j = 0; j < 6; j++) { 2159 for (i = 0, bitval = 0; i < 8; i++) 2160 bitval ^= hash_bit_value(i * 6 + j, addr); 2161 2162 hash_index |= (bitval << j); 2163 } 2164 2165 return hash_index; 2166 } 2167 2168 /* Add multicast addresses to the internal multicast-hash table. */ 2169 static void macb_sethashtable(struct net_device *dev) 2170 { 2171 struct netdev_hw_addr *ha; 2172 unsigned long mc_filter[2]; 2173 unsigned int bitnr; 2174 struct macb *bp = netdev_priv(dev); 2175 2176 mc_filter[0] = 0; 2177 mc_filter[1] = 0; 2178 2179 netdev_for_each_mc_addr(ha, dev) { 2180 bitnr = hash_get_index(ha->addr); 2181 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2182 } 2183 2184 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2185 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2186 } 2187 2188 /* Enable/Disable promiscuous and multicast modes. */ 2189 static void macb_set_rx_mode(struct net_device *dev) 2190 { 2191 unsigned long cfg; 2192 struct macb *bp = netdev_priv(dev); 2193 2194 cfg = macb_readl(bp, NCFGR); 2195 2196 if (dev->flags & IFF_PROMISC) { 2197 /* Enable promiscuous mode */ 2198 cfg |= MACB_BIT(CAF); 2199 2200 /* Disable RX checksum offload */ 2201 if (macb_is_gem(bp)) 2202 cfg &= ~GEM_BIT(RXCOEN); 2203 } else { 2204 /* Disable promiscuous mode */ 2205 cfg &= ~MACB_BIT(CAF); 2206 2207 /* Enable RX checksum offload only if requested */ 2208 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2209 cfg |= GEM_BIT(RXCOEN); 2210 } 2211 2212 if (dev->flags & IFF_ALLMULTI) { 2213 /* Enable all multicast mode */ 2214 macb_or_gem_writel(bp, HRB, -1); 2215 macb_or_gem_writel(bp, HRT, -1); 2216 cfg |= MACB_BIT(NCFGR_MTI); 2217 } else if (!netdev_mc_empty(dev)) { 2218 /* Enable specific multicasts */ 2219 macb_sethashtable(dev); 2220 cfg |= MACB_BIT(NCFGR_MTI); 2221 } else if (dev->flags & (~IFF_ALLMULTI)) { 2222 /* Disable all multicast mode */ 2223 macb_or_gem_writel(bp, HRB, 0); 2224 macb_or_gem_writel(bp, HRT, 0); 2225 cfg &= ~MACB_BIT(NCFGR_MTI); 2226 } 2227 2228 macb_writel(bp, NCFGR, cfg); 2229 } 2230 2231 static int macb_open(struct net_device *dev) 2232 { 2233 struct macb *bp = netdev_priv(dev); 2234 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2235 struct macb_queue *queue; 2236 unsigned int q; 2237 int err; 2238 2239 netdev_dbg(bp->dev, "open\n"); 2240 2241 /* carrier starts down */ 2242 netif_carrier_off(dev); 2243 2244 /* if the phy is not yet register, retry later*/ 2245 if (!dev->phydev) 2246 return -EAGAIN; 2247 2248 /* RX buffers initialization */ 2249 macb_init_rx_buffer_size(bp, bufsz); 2250 2251 err = macb_alloc_consistent(bp); 2252 if (err) { 2253 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2254 err); 2255 return err; 2256 } 2257 2258 bp->macbgem_ops.mog_init_rings(bp); 2259 macb_init_hw(bp); 2260 2261 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2262 napi_enable(&queue->napi); 2263 2264 /* schedule a link state check */ 2265 phy_start(dev->phydev); 2266 2267 netif_tx_start_all_queues(dev); 2268 2269 if (bp->ptp_info) 2270 bp->ptp_info->ptp_init(dev); 2271 2272 return 0; 2273 } 2274 2275 static int macb_close(struct net_device *dev) 2276 { 2277 struct macb *bp = netdev_priv(dev); 2278 struct macb_queue *queue; 2279 unsigned long flags; 2280 unsigned int q; 2281 2282 netif_tx_stop_all_queues(dev); 2283 2284 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2285 napi_disable(&queue->napi); 2286 2287 if (dev->phydev) 2288 phy_stop(dev->phydev); 2289 2290 spin_lock_irqsave(&bp->lock, flags); 2291 macb_reset_hw(bp); 2292 netif_carrier_off(dev); 2293 spin_unlock_irqrestore(&bp->lock, flags); 2294 2295 macb_free_consistent(bp); 2296 2297 if (bp->ptp_info) 2298 bp->ptp_info->ptp_remove(dev); 2299 2300 return 0; 2301 } 2302 2303 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2304 { 2305 if (netif_running(dev)) 2306 return -EBUSY; 2307 2308 dev->mtu = new_mtu; 2309 2310 return 0; 2311 } 2312 2313 static void gem_update_stats(struct macb *bp) 2314 { 2315 struct macb_queue *queue; 2316 unsigned int i, q, idx; 2317 unsigned long *stat; 2318 2319 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2320 2321 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2322 u32 offset = gem_statistics[i].offset; 2323 u64 val = bp->macb_reg_readl(bp, offset); 2324 2325 bp->ethtool_stats[i] += val; 2326 *p += val; 2327 2328 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2329 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2330 val = bp->macb_reg_readl(bp, offset + 4); 2331 bp->ethtool_stats[i] += ((u64)val) << 32; 2332 *(++p) += val; 2333 } 2334 } 2335 2336 idx = GEM_STATS_LEN; 2337 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2338 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2339 bp->ethtool_stats[idx++] = *stat; 2340 } 2341 2342 static struct net_device_stats *gem_get_stats(struct macb *bp) 2343 { 2344 struct gem_stats *hwstat = &bp->hw_stats.gem; 2345 struct net_device_stats *nstat = &bp->dev->stats; 2346 2347 gem_update_stats(bp); 2348 2349 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2350 hwstat->rx_alignment_errors + 2351 hwstat->rx_resource_errors + 2352 hwstat->rx_overruns + 2353 hwstat->rx_oversize_frames + 2354 hwstat->rx_jabbers + 2355 hwstat->rx_undersized_frames + 2356 hwstat->rx_length_field_frame_errors); 2357 nstat->tx_errors = (hwstat->tx_late_collisions + 2358 hwstat->tx_excessive_collisions + 2359 hwstat->tx_underrun + 2360 hwstat->tx_carrier_sense_errors); 2361 nstat->multicast = hwstat->rx_multicast_frames; 2362 nstat->collisions = (hwstat->tx_single_collision_frames + 2363 hwstat->tx_multiple_collision_frames + 2364 hwstat->tx_excessive_collisions); 2365 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2366 hwstat->rx_jabbers + 2367 hwstat->rx_undersized_frames + 2368 hwstat->rx_length_field_frame_errors); 2369 nstat->rx_over_errors = hwstat->rx_resource_errors; 2370 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2371 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2372 nstat->rx_fifo_errors = hwstat->rx_overruns; 2373 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2374 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2375 nstat->tx_fifo_errors = hwstat->tx_underrun; 2376 2377 return nstat; 2378 } 2379 2380 static void gem_get_ethtool_stats(struct net_device *dev, 2381 struct ethtool_stats *stats, u64 *data) 2382 { 2383 struct macb *bp; 2384 2385 bp = netdev_priv(dev); 2386 gem_update_stats(bp); 2387 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2388 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2389 } 2390 2391 static int gem_get_sset_count(struct net_device *dev, int sset) 2392 { 2393 struct macb *bp = netdev_priv(dev); 2394 2395 switch (sset) { 2396 case ETH_SS_STATS: 2397 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2398 default: 2399 return -EOPNOTSUPP; 2400 } 2401 } 2402 2403 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2404 { 2405 char stat_string[ETH_GSTRING_LEN]; 2406 struct macb *bp = netdev_priv(dev); 2407 struct macb_queue *queue; 2408 unsigned int i; 2409 unsigned int q; 2410 2411 switch (sset) { 2412 case ETH_SS_STATS: 2413 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2414 memcpy(p, gem_statistics[i].stat_string, 2415 ETH_GSTRING_LEN); 2416 2417 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2418 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2419 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2420 q, queue_statistics[i].stat_string); 2421 memcpy(p, stat_string, ETH_GSTRING_LEN); 2422 } 2423 } 2424 break; 2425 } 2426 } 2427 2428 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2429 { 2430 struct macb *bp = netdev_priv(dev); 2431 struct net_device_stats *nstat = &bp->dev->stats; 2432 struct macb_stats *hwstat = &bp->hw_stats.macb; 2433 2434 if (macb_is_gem(bp)) 2435 return gem_get_stats(bp); 2436 2437 /* read stats from hardware */ 2438 macb_update_stats(bp); 2439 2440 /* Convert HW stats into netdevice stats */ 2441 nstat->rx_errors = (hwstat->rx_fcs_errors + 2442 hwstat->rx_align_errors + 2443 hwstat->rx_resource_errors + 2444 hwstat->rx_overruns + 2445 hwstat->rx_oversize_pkts + 2446 hwstat->rx_jabbers + 2447 hwstat->rx_undersize_pkts + 2448 hwstat->rx_length_mismatch); 2449 nstat->tx_errors = (hwstat->tx_late_cols + 2450 hwstat->tx_excessive_cols + 2451 hwstat->tx_underruns + 2452 hwstat->tx_carrier_errors + 2453 hwstat->sqe_test_errors); 2454 nstat->collisions = (hwstat->tx_single_cols + 2455 hwstat->tx_multiple_cols + 2456 hwstat->tx_excessive_cols); 2457 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2458 hwstat->rx_jabbers + 2459 hwstat->rx_undersize_pkts + 2460 hwstat->rx_length_mismatch); 2461 nstat->rx_over_errors = hwstat->rx_resource_errors + 2462 hwstat->rx_overruns; 2463 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2464 nstat->rx_frame_errors = hwstat->rx_align_errors; 2465 nstat->rx_fifo_errors = hwstat->rx_overruns; 2466 /* XXX: What does "missed" mean? */ 2467 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2468 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2469 nstat->tx_fifo_errors = hwstat->tx_underruns; 2470 /* Don't know about heartbeat or window errors... */ 2471 2472 return nstat; 2473 } 2474 2475 static int macb_get_regs_len(struct net_device *netdev) 2476 { 2477 return MACB_GREGS_NBR * sizeof(u32); 2478 } 2479 2480 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2481 void *p) 2482 { 2483 struct macb *bp = netdev_priv(dev); 2484 unsigned int tail, head; 2485 u32 *regs_buff = p; 2486 2487 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2488 | MACB_GREGS_VERSION; 2489 2490 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2491 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2492 2493 regs_buff[0] = macb_readl(bp, NCR); 2494 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2495 regs_buff[2] = macb_readl(bp, NSR); 2496 regs_buff[3] = macb_readl(bp, TSR); 2497 regs_buff[4] = macb_readl(bp, RBQP); 2498 regs_buff[5] = macb_readl(bp, TBQP); 2499 regs_buff[6] = macb_readl(bp, RSR); 2500 regs_buff[7] = macb_readl(bp, IMR); 2501 2502 regs_buff[8] = tail; 2503 regs_buff[9] = head; 2504 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2505 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2506 2507 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2508 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2509 if (macb_is_gem(bp)) 2510 regs_buff[13] = gem_readl(bp, DMACFG); 2511 } 2512 2513 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2514 { 2515 struct macb *bp = netdev_priv(netdev); 2516 2517 wol->supported = 0; 2518 wol->wolopts = 0; 2519 2520 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2521 wol->supported = WAKE_MAGIC; 2522 2523 if (bp->wol & MACB_WOL_ENABLED) 2524 wol->wolopts |= WAKE_MAGIC; 2525 } 2526 } 2527 2528 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2529 { 2530 struct macb *bp = netdev_priv(netdev); 2531 2532 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2533 (wol->wolopts & ~WAKE_MAGIC)) 2534 return -EOPNOTSUPP; 2535 2536 if (wol->wolopts & WAKE_MAGIC) 2537 bp->wol |= MACB_WOL_ENABLED; 2538 else 2539 bp->wol &= ~MACB_WOL_ENABLED; 2540 2541 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2542 2543 return 0; 2544 } 2545 2546 static void macb_get_ringparam(struct net_device *netdev, 2547 struct ethtool_ringparam *ring) 2548 { 2549 struct macb *bp = netdev_priv(netdev); 2550 2551 ring->rx_max_pending = MAX_RX_RING_SIZE; 2552 ring->tx_max_pending = MAX_TX_RING_SIZE; 2553 2554 ring->rx_pending = bp->rx_ring_size; 2555 ring->tx_pending = bp->tx_ring_size; 2556 } 2557 2558 static int macb_set_ringparam(struct net_device *netdev, 2559 struct ethtool_ringparam *ring) 2560 { 2561 struct macb *bp = netdev_priv(netdev); 2562 u32 new_rx_size, new_tx_size; 2563 unsigned int reset = 0; 2564 2565 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2566 return -EINVAL; 2567 2568 new_rx_size = clamp_t(u32, ring->rx_pending, 2569 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2570 new_rx_size = roundup_pow_of_two(new_rx_size); 2571 2572 new_tx_size = clamp_t(u32, ring->tx_pending, 2573 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2574 new_tx_size = roundup_pow_of_two(new_tx_size); 2575 2576 if ((new_tx_size == bp->tx_ring_size) && 2577 (new_rx_size == bp->rx_ring_size)) { 2578 /* nothing to do */ 2579 return 0; 2580 } 2581 2582 if (netif_running(bp->dev)) { 2583 reset = 1; 2584 macb_close(bp->dev); 2585 } 2586 2587 bp->rx_ring_size = new_rx_size; 2588 bp->tx_ring_size = new_tx_size; 2589 2590 if (reset) 2591 macb_open(bp->dev); 2592 2593 return 0; 2594 } 2595 2596 #ifdef CONFIG_MACB_USE_HWSTAMP 2597 static unsigned int gem_get_tsu_rate(struct macb *bp) 2598 { 2599 struct clk *tsu_clk; 2600 unsigned int tsu_rate; 2601 2602 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2603 if (!IS_ERR(tsu_clk)) 2604 tsu_rate = clk_get_rate(tsu_clk); 2605 /* try pclk instead */ 2606 else if (!IS_ERR(bp->pclk)) { 2607 tsu_clk = bp->pclk; 2608 tsu_rate = clk_get_rate(tsu_clk); 2609 } else 2610 return -ENOTSUPP; 2611 return tsu_rate; 2612 } 2613 2614 static s32 gem_get_ptp_max_adj(void) 2615 { 2616 return 64000000; 2617 } 2618 2619 static int gem_get_ts_info(struct net_device *dev, 2620 struct ethtool_ts_info *info) 2621 { 2622 struct macb *bp = netdev_priv(dev); 2623 2624 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 2625 ethtool_op_get_ts_info(dev, info); 2626 return 0; 2627 } 2628 2629 info->so_timestamping = 2630 SOF_TIMESTAMPING_TX_SOFTWARE | 2631 SOF_TIMESTAMPING_RX_SOFTWARE | 2632 SOF_TIMESTAMPING_SOFTWARE | 2633 SOF_TIMESTAMPING_TX_HARDWARE | 2634 SOF_TIMESTAMPING_RX_HARDWARE | 2635 SOF_TIMESTAMPING_RAW_HARDWARE; 2636 info->tx_types = 2637 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 2638 (1 << HWTSTAMP_TX_OFF) | 2639 (1 << HWTSTAMP_TX_ON); 2640 info->rx_filters = 2641 (1 << HWTSTAMP_FILTER_NONE) | 2642 (1 << HWTSTAMP_FILTER_ALL); 2643 2644 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 2645 2646 return 0; 2647 } 2648 2649 static struct macb_ptp_info gem_ptp_info = { 2650 .ptp_init = gem_ptp_init, 2651 .ptp_remove = gem_ptp_remove, 2652 .get_ptp_max_adj = gem_get_ptp_max_adj, 2653 .get_tsu_rate = gem_get_tsu_rate, 2654 .get_ts_info = gem_get_ts_info, 2655 .get_hwtst = gem_get_hwtst, 2656 .set_hwtst = gem_set_hwtst, 2657 }; 2658 #endif 2659 2660 static int macb_get_ts_info(struct net_device *netdev, 2661 struct ethtool_ts_info *info) 2662 { 2663 struct macb *bp = netdev_priv(netdev); 2664 2665 if (bp->ptp_info) 2666 return bp->ptp_info->get_ts_info(netdev, info); 2667 2668 return ethtool_op_get_ts_info(netdev, info); 2669 } 2670 2671 static void gem_enable_flow_filters(struct macb *bp, bool enable) 2672 { 2673 struct ethtool_rx_fs_item *item; 2674 u32 t2_scr; 2675 int num_t2_scr; 2676 2677 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 2678 2679 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2680 struct ethtool_rx_flow_spec *fs = &item->fs; 2681 struct ethtool_tcpip4_spec *tp4sp_m; 2682 2683 if (fs->location >= num_t2_scr) 2684 continue; 2685 2686 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 2687 2688 /* enable/disable screener regs for the flow entry */ 2689 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 2690 2691 /* only enable fields with no masking */ 2692 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2693 2694 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 2695 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 2696 else 2697 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 2698 2699 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 2700 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 2701 else 2702 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 2703 2704 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 2705 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 2706 else 2707 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 2708 2709 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 2710 } 2711 } 2712 2713 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 2714 { 2715 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 2716 uint16_t index = fs->location; 2717 u32 w0, w1, t2_scr; 2718 bool cmp_a = false; 2719 bool cmp_b = false; 2720 bool cmp_c = false; 2721 2722 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 2723 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 2724 2725 /* ignore field if any masking set */ 2726 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 2727 /* 1st compare reg - IP source address */ 2728 w0 = 0; 2729 w1 = 0; 2730 w0 = tp4sp_v->ip4src; 2731 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2732 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2733 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 2734 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 2735 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 2736 cmp_a = true; 2737 } 2738 2739 /* ignore field if any masking set */ 2740 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 2741 /* 2nd compare reg - IP destination address */ 2742 w0 = 0; 2743 w1 = 0; 2744 w0 = tp4sp_v->ip4dst; 2745 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2746 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 2747 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 2748 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 2749 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 2750 cmp_b = true; 2751 } 2752 2753 /* ignore both port fields if masking set in both */ 2754 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 2755 /* 3rd compare reg - source port, destination port */ 2756 w0 = 0; 2757 w1 = 0; 2758 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 2759 if (tp4sp_m->psrc == tp4sp_m->pdst) { 2760 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 2761 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2762 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 2763 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2764 } else { 2765 /* only one port definition */ 2766 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 2767 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 2768 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 2769 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 2770 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 2771 } else { /* dst port */ 2772 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 2773 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 2774 } 2775 } 2776 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 2777 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 2778 cmp_c = true; 2779 } 2780 2781 t2_scr = 0; 2782 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 2783 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 2784 if (cmp_a) 2785 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 2786 if (cmp_b) 2787 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 2788 if (cmp_c) 2789 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 2790 gem_writel_n(bp, SCRT2, index, t2_scr); 2791 } 2792 2793 static int gem_add_flow_filter(struct net_device *netdev, 2794 struct ethtool_rxnfc *cmd) 2795 { 2796 struct macb *bp = netdev_priv(netdev); 2797 struct ethtool_rx_flow_spec *fs = &cmd->fs; 2798 struct ethtool_rx_fs_item *item, *newfs; 2799 unsigned long flags; 2800 int ret = -EINVAL; 2801 bool added = false; 2802 2803 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 2804 if (newfs == NULL) 2805 return -ENOMEM; 2806 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 2807 2808 netdev_dbg(netdev, 2809 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2810 fs->flow_type, (int)fs->ring_cookie, fs->location, 2811 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2812 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2813 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 2814 2815 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2816 2817 /* find correct place to add in list */ 2818 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2819 if (item->fs.location > newfs->fs.location) { 2820 list_add_tail(&newfs->list, &item->list); 2821 added = true; 2822 break; 2823 } else if (item->fs.location == fs->location) { 2824 netdev_err(netdev, "Rule not added: location %d not free!\n", 2825 fs->location); 2826 ret = -EBUSY; 2827 goto err; 2828 } 2829 } 2830 if (!added) 2831 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 2832 2833 gem_prog_cmp_regs(bp, fs); 2834 bp->rx_fs_list.count++; 2835 /* enable filtering if NTUPLE on */ 2836 if (netdev->features & NETIF_F_NTUPLE) 2837 gem_enable_flow_filters(bp, 1); 2838 2839 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2840 return 0; 2841 2842 err: 2843 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2844 kfree(newfs); 2845 return ret; 2846 } 2847 2848 static int gem_del_flow_filter(struct net_device *netdev, 2849 struct ethtool_rxnfc *cmd) 2850 { 2851 struct macb *bp = netdev_priv(netdev); 2852 struct ethtool_rx_fs_item *item; 2853 struct ethtool_rx_flow_spec *fs; 2854 unsigned long flags; 2855 2856 spin_lock_irqsave(&bp->rx_fs_lock, flags); 2857 2858 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2859 if (item->fs.location == cmd->fs.location) { 2860 /* disable screener regs for the flow entry */ 2861 fs = &(item->fs); 2862 netdev_dbg(netdev, 2863 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 2864 fs->flow_type, (int)fs->ring_cookie, fs->location, 2865 htonl(fs->h_u.tcp_ip4_spec.ip4src), 2866 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 2867 htons(fs->h_u.tcp_ip4_spec.psrc), 2868 htons(fs->h_u.tcp_ip4_spec.pdst)); 2869 2870 gem_writel_n(bp, SCRT2, fs->location, 0); 2871 2872 list_del(&item->list); 2873 bp->rx_fs_list.count--; 2874 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2875 kfree(item); 2876 return 0; 2877 } 2878 } 2879 2880 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 2881 return -EINVAL; 2882 } 2883 2884 static int gem_get_flow_entry(struct net_device *netdev, 2885 struct ethtool_rxnfc *cmd) 2886 { 2887 struct macb *bp = netdev_priv(netdev); 2888 struct ethtool_rx_fs_item *item; 2889 2890 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2891 if (item->fs.location == cmd->fs.location) { 2892 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 2893 return 0; 2894 } 2895 } 2896 return -EINVAL; 2897 } 2898 2899 static int gem_get_all_flow_entries(struct net_device *netdev, 2900 struct ethtool_rxnfc *cmd, u32 *rule_locs) 2901 { 2902 struct macb *bp = netdev_priv(netdev); 2903 struct ethtool_rx_fs_item *item; 2904 uint32_t cnt = 0; 2905 2906 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 2907 if (cnt == cmd->rule_cnt) 2908 return -EMSGSIZE; 2909 rule_locs[cnt] = item->fs.location; 2910 cnt++; 2911 } 2912 cmd->data = bp->max_tuples; 2913 cmd->rule_cnt = cnt; 2914 2915 return 0; 2916 } 2917 2918 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 2919 u32 *rule_locs) 2920 { 2921 struct macb *bp = netdev_priv(netdev); 2922 int ret = 0; 2923 2924 switch (cmd->cmd) { 2925 case ETHTOOL_GRXRINGS: 2926 cmd->data = bp->num_queues; 2927 break; 2928 case ETHTOOL_GRXCLSRLCNT: 2929 cmd->rule_cnt = bp->rx_fs_list.count; 2930 break; 2931 case ETHTOOL_GRXCLSRULE: 2932 ret = gem_get_flow_entry(netdev, cmd); 2933 break; 2934 case ETHTOOL_GRXCLSRLALL: 2935 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 2936 break; 2937 default: 2938 netdev_err(netdev, 2939 "Command parameter %d is not supported\n", cmd->cmd); 2940 ret = -EOPNOTSUPP; 2941 } 2942 2943 return ret; 2944 } 2945 2946 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 2947 { 2948 struct macb *bp = netdev_priv(netdev); 2949 int ret; 2950 2951 switch (cmd->cmd) { 2952 case ETHTOOL_SRXCLSRLINS: 2953 if ((cmd->fs.location >= bp->max_tuples) 2954 || (cmd->fs.ring_cookie >= bp->num_queues)) { 2955 ret = -EINVAL; 2956 break; 2957 } 2958 ret = gem_add_flow_filter(netdev, cmd); 2959 break; 2960 case ETHTOOL_SRXCLSRLDEL: 2961 ret = gem_del_flow_filter(netdev, cmd); 2962 break; 2963 default: 2964 netdev_err(netdev, 2965 "Command parameter %d is not supported\n", cmd->cmd); 2966 ret = -EOPNOTSUPP; 2967 } 2968 2969 return ret; 2970 } 2971 2972 static const struct ethtool_ops macb_ethtool_ops = { 2973 .get_regs_len = macb_get_regs_len, 2974 .get_regs = macb_get_regs, 2975 .get_link = ethtool_op_get_link, 2976 .get_ts_info = ethtool_op_get_ts_info, 2977 .get_wol = macb_get_wol, 2978 .set_wol = macb_set_wol, 2979 .get_link_ksettings = phy_ethtool_get_link_ksettings, 2980 .set_link_ksettings = phy_ethtool_set_link_ksettings, 2981 .get_ringparam = macb_get_ringparam, 2982 .set_ringparam = macb_set_ringparam, 2983 }; 2984 2985 static const struct ethtool_ops gem_ethtool_ops = { 2986 .get_regs_len = macb_get_regs_len, 2987 .get_regs = macb_get_regs, 2988 .get_link = ethtool_op_get_link, 2989 .get_ts_info = macb_get_ts_info, 2990 .get_ethtool_stats = gem_get_ethtool_stats, 2991 .get_strings = gem_get_ethtool_strings, 2992 .get_sset_count = gem_get_sset_count, 2993 .get_link_ksettings = phy_ethtool_get_link_ksettings, 2994 .set_link_ksettings = phy_ethtool_set_link_ksettings, 2995 .get_ringparam = macb_get_ringparam, 2996 .set_ringparam = macb_set_ringparam, 2997 .get_rxnfc = gem_get_rxnfc, 2998 .set_rxnfc = gem_set_rxnfc, 2999 }; 3000 3001 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3002 { 3003 struct phy_device *phydev = dev->phydev; 3004 struct macb *bp = netdev_priv(dev); 3005 3006 if (!netif_running(dev)) 3007 return -EINVAL; 3008 3009 if (!phydev) 3010 return -ENODEV; 3011 3012 if (!bp->ptp_info) 3013 return phy_mii_ioctl(phydev, rq, cmd); 3014 3015 switch (cmd) { 3016 case SIOCSHWTSTAMP: 3017 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3018 case SIOCGHWTSTAMP: 3019 return bp->ptp_info->get_hwtst(dev, rq); 3020 default: 3021 return phy_mii_ioctl(phydev, rq, cmd); 3022 } 3023 } 3024 3025 static int macb_set_features(struct net_device *netdev, 3026 netdev_features_t features) 3027 { 3028 struct macb *bp = netdev_priv(netdev); 3029 netdev_features_t changed = features ^ netdev->features; 3030 3031 /* TX checksum offload */ 3032 if ((changed & NETIF_F_HW_CSUM) && macb_is_gem(bp)) { 3033 u32 dmacfg; 3034 3035 dmacfg = gem_readl(bp, DMACFG); 3036 if (features & NETIF_F_HW_CSUM) 3037 dmacfg |= GEM_BIT(TXCOEN); 3038 else 3039 dmacfg &= ~GEM_BIT(TXCOEN); 3040 gem_writel(bp, DMACFG, dmacfg); 3041 } 3042 3043 /* RX checksum offload */ 3044 if ((changed & NETIF_F_RXCSUM) && macb_is_gem(bp)) { 3045 u32 netcfg; 3046 3047 netcfg = gem_readl(bp, NCFGR); 3048 if (features & NETIF_F_RXCSUM && 3049 !(netdev->flags & IFF_PROMISC)) 3050 netcfg |= GEM_BIT(RXCOEN); 3051 else 3052 netcfg &= ~GEM_BIT(RXCOEN); 3053 gem_writel(bp, NCFGR, netcfg); 3054 } 3055 3056 /* RX Flow Filters */ 3057 if ((changed & NETIF_F_NTUPLE) && macb_is_gem(bp)) { 3058 bool turn_on = features & NETIF_F_NTUPLE; 3059 3060 gem_enable_flow_filters(bp, turn_on); 3061 } 3062 return 0; 3063 } 3064 3065 static const struct net_device_ops macb_netdev_ops = { 3066 .ndo_open = macb_open, 3067 .ndo_stop = macb_close, 3068 .ndo_start_xmit = macb_start_xmit, 3069 .ndo_set_rx_mode = macb_set_rx_mode, 3070 .ndo_get_stats = macb_get_stats, 3071 .ndo_do_ioctl = macb_ioctl, 3072 .ndo_validate_addr = eth_validate_addr, 3073 .ndo_change_mtu = macb_change_mtu, 3074 .ndo_set_mac_address = eth_mac_addr, 3075 #ifdef CONFIG_NET_POLL_CONTROLLER 3076 .ndo_poll_controller = macb_poll_controller, 3077 #endif 3078 .ndo_set_features = macb_set_features, 3079 .ndo_features_check = macb_features_check, 3080 }; 3081 3082 /* Configure peripheral capabilities according to device tree 3083 * and integration options used 3084 */ 3085 static void macb_configure_caps(struct macb *bp, 3086 const struct macb_config *dt_conf) 3087 { 3088 u32 dcfg; 3089 3090 if (dt_conf) 3091 bp->caps = dt_conf->caps; 3092 3093 if (hw_is_gem(bp->regs, bp->native_io)) { 3094 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3095 3096 dcfg = gem_readl(bp, DCFG1); 3097 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3098 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3099 dcfg = gem_readl(bp, DCFG2); 3100 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3101 bp->caps |= MACB_CAPS_FIFO_MODE; 3102 #ifdef CONFIG_MACB_USE_HWSTAMP 3103 if (gem_has_ptp(bp)) { 3104 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3105 pr_err("GEM doesn't support hardware ptp.\n"); 3106 else { 3107 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3108 bp->ptp_info = &gem_ptp_info; 3109 } 3110 } 3111 #endif 3112 } 3113 3114 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3115 } 3116 3117 static void macb_probe_queues(void __iomem *mem, 3118 bool native_io, 3119 unsigned int *queue_mask, 3120 unsigned int *num_queues) 3121 { 3122 unsigned int hw_q; 3123 3124 *queue_mask = 0x1; 3125 *num_queues = 1; 3126 3127 /* is it macb or gem ? 3128 * 3129 * We need to read directly from the hardware here because 3130 * we are early in the probe process and don't have the 3131 * MACB_CAPS_MACB_IS_GEM flag positioned 3132 */ 3133 if (!hw_is_gem(mem, native_io)) 3134 return; 3135 3136 /* bit 0 is never set but queue 0 always exists */ 3137 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff; 3138 3139 *queue_mask |= 0x1; 3140 3141 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q) 3142 if (*queue_mask & (1 << hw_q)) 3143 (*num_queues)++; 3144 } 3145 3146 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3147 struct clk **hclk, struct clk **tx_clk, 3148 struct clk **rx_clk) 3149 { 3150 struct macb_platform_data *pdata; 3151 int err; 3152 3153 pdata = dev_get_platdata(&pdev->dev); 3154 if (pdata) { 3155 *pclk = pdata->pclk; 3156 *hclk = pdata->hclk; 3157 } else { 3158 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3159 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3160 } 3161 3162 if (IS_ERR(*pclk)) { 3163 err = PTR_ERR(*pclk); 3164 dev_err(&pdev->dev, "failed to get macb_clk (%u)\n", err); 3165 return err; 3166 } 3167 3168 if (IS_ERR(*hclk)) { 3169 err = PTR_ERR(*hclk); 3170 dev_err(&pdev->dev, "failed to get hclk (%u)\n", err); 3171 return err; 3172 } 3173 3174 *tx_clk = devm_clk_get(&pdev->dev, "tx_clk"); 3175 if (IS_ERR(*tx_clk)) 3176 *tx_clk = NULL; 3177 3178 *rx_clk = devm_clk_get(&pdev->dev, "rx_clk"); 3179 if (IS_ERR(*rx_clk)) 3180 *rx_clk = NULL; 3181 3182 err = clk_prepare_enable(*pclk); 3183 if (err) { 3184 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3185 return err; 3186 } 3187 3188 err = clk_prepare_enable(*hclk); 3189 if (err) { 3190 dev_err(&pdev->dev, "failed to enable hclk (%u)\n", err); 3191 goto err_disable_pclk; 3192 } 3193 3194 err = clk_prepare_enable(*tx_clk); 3195 if (err) { 3196 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 3197 goto err_disable_hclk; 3198 } 3199 3200 err = clk_prepare_enable(*rx_clk); 3201 if (err) { 3202 dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); 3203 goto err_disable_txclk; 3204 } 3205 3206 return 0; 3207 3208 err_disable_txclk: 3209 clk_disable_unprepare(*tx_clk); 3210 3211 err_disable_hclk: 3212 clk_disable_unprepare(*hclk); 3213 3214 err_disable_pclk: 3215 clk_disable_unprepare(*pclk); 3216 3217 return err; 3218 } 3219 3220 static int macb_init(struct platform_device *pdev) 3221 { 3222 struct net_device *dev = platform_get_drvdata(pdev); 3223 unsigned int hw_q, q; 3224 struct macb *bp = netdev_priv(dev); 3225 struct macb_queue *queue; 3226 int err; 3227 u32 val, reg; 3228 3229 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3230 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3231 3232 /* set the queue register mapping once for all: queue0 has a special 3233 * register mapping but we don't want to test the queue index then 3234 * compute the corresponding register offset at run time. 3235 */ 3236 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3237 if (!(bp->queue_mask & (1 << hw_q))) 3238 continue; 3239 3240 queue = &bp->queues[q]; 3241 queue->bp = bp; 3242 netif_napi_add(dev, &queue->napi, macb_poll, 64); 3243 if (hw_q) { 3244 queue->ISR = GEM_ISR(hw_q - 1); 3245 queue->IER = GEM_IER(hw_q - 1); 3246 queue->IDR = GEM_IDR(hw_q - 1); 3247 queue->IMR = GEM_IMR(hw_q - 1); 3248 queue->TBQP = GEM_TBQP(hw_q - 1); 3249 queue->RBQP = GEM_RBQP(hw_q - 1); 3250 queue->RBQS = GEM_RBQS(hw_q - 1); 3251 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3252 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3253 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3254 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3255 } 3256 #endif 3257 } else { 3258 /* queue0 uses legacy registers */ 3259 queue->ISR = MACB_ISR; 3260 queue->IER = MACB_IER; 3261 queue->IDR = MACB_IDR; 3262 queue->IMR = MACB_IMR; 3263 queue->TBQP = MACB_TBQP; 3264 queue->RBQP = MACB_RBQP; 3265 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3266 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3267 queue->TBQPH = MACB_TBQPH; 3268 queue->RBQPH = MACB_RBQPH; 3269 } 3270 #endif 3271 } 3272 3273 /* get irq: here we use the linux queue index, not the hardware 3274 * queue index. the queue irq definitions in the device tree 3275 * must remove the optional gaps that could exist in the 3276 * hardware queue mask. 3277 */ 3278 queue->irq = platform_get_irq(pdev, q); 3279 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3280 IRQF_SHARED, dev->name, queue); 3281 if (err) { 3282 dev_err(&pdev->dev, 3283 "Unable to request IRQ %d (error %d)\n", 3284 queue->irq, err); 3285 return err; 3286 } 3287 3288 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3289 q++; 3290 } 3291 3292 dev->netdev_ops = &macb_netdev_ops; 3293 3294 /* setup appropriated routines according to adapter type */ 3295 if (macb_is_gem(bp)) { 3296 bp->max_tx_length = GEM_MAX_TX_LEN; 3297 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3298 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3299 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3300 bp->macbgem_ops.mog_rx = gem_rx; 3301 dev->ethtool_ops = &gem_ethtool_ops; 3302 } else { 3303 bp->max_tx_length = MACB_MAX_TX_LEN; 3304 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3305 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3306 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3307 bp->macbgem_ops.mog_rx = macb_rx; 3308 dev->ethtool_ops = &macb_ethtool_ops; 3309 } 3310 3311 /* Set features */ 3312 dev->hw_features = NETIF_F_SG; 3313 3314 /* Check LSO capability */ 3315 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3316 dev->hw_features |= MACB_NETIF_LSO; 3317 3318 /* Checksum offload is only available on gem with packet buffer */ 3319 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3320 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3321 if (bp->caps & MACB_CAPS_SG_DISABLED) 3322 dev->hw_features &= ~NETIF_F_SG; 3323 dev->features = dev->hw_features; 3324 3325 /* Check RX Flow Filters support. 3326 * Max Rx flows set by availability of screeners & compare regs: 3327 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3328 */ 3329 reg = gem_readl(bp, DCFG8); 3330 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3331 GEM_BFEXT(T2SCR, reg)); 3332 if (bp->max_tuples > 0) { 3333 /* also needs one ethtype match to check IPv4 */ 3334 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3335 /* program this reg now */ 3336 reg = 0; 3337 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3338 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3339 /* Filtering is supported in hw but don't enable it in kernel now */ 3340 dev->hw_features |= NETIF_F_NTUPLE; 3341 /* init Rx flow definitions */ 3342 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3343 bp->rx_fs_list.count = 0; 3344 spin_lock_init(&bp->rx_fs_lock); 3345 } else 3346 bp->max_tuples = 0; 3347 } 3348 3349 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3350 val = 0; 3351 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII) 3352 val = GEM_BIT(RGMII); 3353 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3354 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3355 val = MACB_BIT(RMII); 3356 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3357 val = MACB_BIT(MII); 3358 3359 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3360 val |= MACB_BIT(CLKEN); 3361 3362 macb_or_gem_writel(bp, USRIO, val); 3363 } 3364 3365 /* Set MII management clock divider */ 3366 val = macb_mdc_clk_div(bp); 3367 val |= macb_dbw(bp); 3368 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3369 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3370 macb_writel(bp, NCFGR, val); 3371 3372 return 0; 3373 } 3374 3375 #if defined(CONFIG_OF) 3376 /* 1518 rounded up */ 3377 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3378 /* max number of receive buffers */ 3379 #define AT91ETHER_MAX_RX_DESCR 9 3380 3381 /* Initialize and start the Receiver and Transmit subsystems */ 3382 static int at91ether_start(struct net_device *dev) 3383 { 3384 struct macb *lp = netdev_priv(dev); 3385 struct macb_queue *q = &lp->queues[0]; 3386 struct macb_dma_desc *desc; 3387 dma_addr_t addr; 3388 u32 ctl; 3389 int i; 3390 3391 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3392 (AT91ETHER_MAX_RX_DESCR * 3393 macb_dma_desc_get_size(lp)), 3394 &q->rx_ring_dma, GFP_KERNEL); 3395 if (!q->rx_ring) 3396 return -ENOMEM; 3397 3398 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3399 AT91ETHER_MAX_RX_DESCR * 3400 AT91ETHER_MAX_RBUFF_SZ, 3401 &q->rx_buffers_dma, GFP_KERNEL); 3402 if (!q->rx_buffers) { 3403 dma_free_coherent(&lp->pdev->dev, 3404 AT91ETHER_MAX_RX_DESCR * 3405 macb_dma_desc_get_size(lp), 3406 q->rx_ring, q->rx_ring_dma); 3407 q->rx_ring = NULL; 3408 return -ENOMEM; 3409 } 3410 3411 addr = q->rx_buffers_dma; 3412 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3413 desc = macb_rx_desc(q, i); 3414 macb_set_addr(lp, desc, addr); 3415 desc->ctrl = 0; 3416 addr += AT91ETHER_MAX_RBUFF_SZ; 3417 } 3418 3419 /* Set the Wrap bit on the last descriptor */ 3420 desc->addr |= MACB_BIT(RX_WRAP); 3421 3422 /* Reset buffer index */ 3423 q->rx_tail = 0; 3424 3425 /* Program address of descriptor list in Rx Buffer Queue register */ 3426 macb_writel(lp, RBQP, q->rx_ring_dma); 3427 3428 /* Enable Receive and Transmit */ 3429 ctl = macb_readl(lp, NCR); 3430 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3431 3432 return 0; 3433 } 3434 3435 /* Open the ethernet interface */ 3436 static int at91ether_open(struct net_device *dev) 3437 { 3438 struct macb *lp = netdev_priv(dev); 3439 u32 ctl; 3440 int ret; 3441 3442 /* Clear internal statistics */ 3443 ctl = macb_readl(lp, NCR); 3444 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3445 3446 macb_set_hwaddr(lp); 3447 3448 ret = at91ether_start(dev); 3449 if (ret) 3450 return ret; 3451 3452 /* Enable MAC interrupts */ 3453 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3454 MACB_BIT(RXUBR) | 3455 MACB_BIT(ISR_TUND) | 3456 MACB_BIT(ISR_RLE) | 3457 MACB_BIT(TCOMP) | 3458 MACB_BIT(ISR_ROVR) | 3459 MACB_BIT(HRESP)); 3460 3461 /* schedule a link state check */ 3462 phy_start(dev->phydev); 3463 3464 netif_start_queue(dev); 3465 3466 return 0; 3467 } 3468 3469 /* Close the interface */ 3470 static int at91ether_close(struct net_device *dev) 3471 { 3472 struct macb *lp = netdev_priv(dev); 3473 struct macb_queue *q = &lp->queues[0]; 3474 u32 ctl; 3475 3476 /* Disable Receiver and Transmitter */ 3477 ctl = macb_readl(lp, NCR); 3478 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3479 3480 /* Disable MAC interrupts */ 3481 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3482 MACB_BIT(RXUBR) | 3483 MACB_BIT(ISR_TUND) | 3484 MACB_BIT(ISR_RLE) | 3485 MACB_BIT(TCOMP) | 3486 MACB_BIT(ISR_ROVR) | 3487 MACB_BIT(HRESP)); 3488 3489 netif_stop_queue(dev); 3490 3491 dma_free_coherent(&lp->pdev->dev, 3492 AT91ETHER_MAX_RX_DESCR * 3493 macb_dma_desc_get_size(lp), 3494 q->rx_ring, q->rx_ring_dma); 3495 q->rx_ring = NULL; 3496 3497 dma_free_coherent(&lp->pdev->dev, 3498 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ, 3499 q->rx_buffers, q->rx_buffers_dma); 3500 q->rx_buffers = NULL; 3501 3502 return 0; 3503 } 3504 3505 /* Transmit packet */ 3506 static int at91ether_start_xmit(struct sk_buff *skb, struct net_device *dev) 3507 { 3508 struct macb *lp = netdev_priv(dev); 3509 3510 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 3511 netif_stop_queue(dev); 3512 3513 /* Store packet information (to free when Tx completed) */ 3514 lp->skb = skb; 3515 lp->skb_length = skb->len; 3516 lp->skb_physaddr = dma_map_single(NULL, skb->data, skb->len, 3517 DMA_TO_DEVICE); 3518 if (dma_mapping_error(NULL, lp->skb_physaddr)) { 3519 dev_kfree_skb_any(skb); 3520 dev->stats.tx_dropped++; 3521 netdev_err(dev, "%s: DMA mapping error\n", __func__); 3522 return NETDEV_TX_OK; 3523 } 3524 3525 /* Set address of the data in the Transmit Address register */ 3526 macb_writel(lp, TAR, lp->skb_physaddr); 3527 /* Set length of the packet in the Transmit Control register */ 3528 macb_writel(lp, TCR, skb->len); 3529 3530 } else { 3531 netdev_err(dev, "%s called, but device is busy!\n", __func__); 3532 return NETDEV_TX_BUSY; 3533 } 3534 3535 return NETDEV_TX_OK; 3536 } 3537 3538 /* Extract received frame from buffer descriptors and sent to upper layers. 3539 * (Called from interrupt context) 3540 */ 3541 static void at91ether_rx(struct net_device *dev) 3542 { 3543 struct macb *lp = netdev_priv(dev); 3544 struct macb_queue *q = &lp->queues[0]; 3545 struct macb_dma_desc *desc; 3546 unsigned char *p_recv; 3547 struct sk_buff *skb; 3548 unsigned int pktlen; 3549 3550 desc = macb_rx_desc(q, q->rx_tail); 3551 while (desc->addr & MACB_BIT(RX_USED)) { 3552 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 3553 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 3554 skb = netdev_alloc_skb(dev, pktlen + 2); 3555 if (skb) { 3556 skb_reserve(skb, 2); 3557 skb_put_data(skb, p_recv, pktlen); 3558 3559 skb->protocol = eth_type_trans(skb, dev); 3560 dev->stats.rx_packets++; 3561 dev->stats.rx_bytes += pktlen; 3562 netif_rx(skb); 3563 } else { 3564 dev->stats.rx_dropped++; 3565 } 3566 3567 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 3568 dev->stats.multicast++; 3569 3570 /* reset ownership bit */ 3571 desc->addr &= ~MACB_BIT(RX_USED); 3572 3573 /* wrap after last buffer */ 3574 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 3575 q->rx_tail = 0; 3576 else 3577 q->rx_tail++; 3578 3579 desc = macb_rx_desc(q, q->rx_tail); 3580 } 3581 } 3582 3583 /* MAC interrupt handler */ 3584 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 3585 { 3586 struct net_device *dev = dev_id; 3587 struct macb *lp = netdev_priv(dev); 3588 u32 intstatus, ctl; 3589 3590 /* MAC Interrupt Status register indicates what interrupts are pending. 3591 * It is automatically cleared once read. 3592 */ 3593 intstatus = macb_readl(lp, ISR); 3594 3595 /* Receive complete */ 3596 if (intstatus & MACB_BIT(RCOMP)) 3597 at91ether_rx(dev); 3598 3599 /* Transmit complete */ 3600 if (intstatus & MACB_BIT(TCOMP)) { 3601 /* The TCOM bit is set even if the transmission failed */ 3602 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 3603 dev->stats.tx_errors++; 3604 3605 if (lp->skb) { 3606 dev_kfree_skb_irq(lp->skb); 3607 lp->skb = NULL; 3608 dma_unmap_single(NULL, lp->skb_physaddr, 3609 lp->skb_length, DMA_TO_DEVICE); 3610 dev->stats.tx_packets++; 3611 dev->stats.tx_bytes += lp->skb_length; 3612 } 3613 netif_wake_queue(dev); 3614 } 3615 3616 /* Work-around for EMAC Errata section 41.3.1 */ 3617 if (intstatus & MACB_BIT(RXUBR)) { 3618 ctl = macb_readl(lp, NCR); 3619 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 3620 wmb(); 3621 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 3622 } 3623 3624 if (intstatus & MACB_BIT(ISR_ROVR)) 3625 netdev_err(dev, "ROVR error\n"); 3626 3627 return IRQ_HANDLED; 3628 } 3629 3630 #ifdef CONFIG_NET_POLL_CONTROLLER 3631 static void at91ether_poll_controller(struct net_device *dev) 3632 { 3633 unsigned long flags; 3634 3635 local_irq_save(flags); 3636 at91ether_interrupt(dev->irq, dev); 3637 local_irq_restore(flags); 3638 } 3639 #endif 3640 3641 static const struct net_device_ops at91ether_netdev_ops = { 3642 .ndo_open = at91ether_open, 3643 .ndo_stop = at91ether_close, 3644 .ndo_start_xmit = at91ether_start_xmit, 3645 .ndo_get_stats = macb_get_stats, 3646 .ndo_set_rx_mode = macb_set_rx_mode, 3647 .ndo_set_mac_address = eth_mac_addr, 3648 .ndo_do_ioctl = macb_ioctl, 3649 .ndo_validate_addr = eth_validate_addr, 3650 #ifdef CONFIG_NET_POLL_CONTROLLER 3651 .ndo_poll_controller = at91ether_poll_controller, 3652 #endif 3653 }; 3654 3655 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 3656 struct clk **hclk, struct clk **tx_clk, 3657 struct clk **rx_clk) 3658 { 3659 int err; 3660 3661 *hclk = NULL; 3662 *tx_clk = NULL; 3663 *rx_clk = NULL; 3664 3665 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 3666 if (IS_ERR(*pclk)) 3667 return PTR_ERR(*pclk); 3668 3669 err = clk_prepare_enable(*pclk); 3670 if (err) { 3671 dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err); 3672 return err; 3673 } 3674 3675 return 0; 3676 } 3677 3678 static int at91ether_init(struct platform_device *pdev) 3679 { 3680 struct net_device *dev = platform_get_drvdata(pdev); 3681 struct macb *bp = netdev_priv(dev); 3682 int err; 3683 u32 reg; 3684 3685 dev->netdev_ops = &at91ether_netdev_ops; 3686 dev->ethtool_ops = &macb_ethtool_ops; 3687 3688 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 3689 0, dev->name, dev); 3690 if (err) 3691 return err; 3692 3693 macb_writel(bp, NCR, 0); 3694 3695 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG); 3696 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) 3697 reg |= MACB_BIT(RM9200_RMII); 3698 3699 macb_writel(bp, NCFGR, reg); 3700 3701 return 0; 3702 } 3703 3704 static const struct macb_config at91sam9260_config = { 3705 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3706 .clk_init = macb_clk_init, 3707 .init = macb_init, 3708 }; 3709 3710 static const struct macb_config pc302gem_config = { 3711 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 3712 .dma_burst_length = 16, 3713 .clk_init = macb_clk_init, 3714 .init = macb_init, 3715 }; 3716 3717 static const struct macb_config sama5d2_config = { 3718 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3719 .dma_burst_length = 16, 3720 .clk_init = macb_clk_init, 3721 .init = macb_init, 3722 }; 3723 3724 static const struct macb_config sama5d3_config = { 3725 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 3726 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 3727 .dma_burst_length = 16, 3728 .clk_init = macb_clk_init, 3729 .init = macb_init, 3730 .jumbo_max_len = 10240, 3731 }; 3732 3733 static const struct macb_config sama5d4_config = { 3734 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 3735 .dma_burst_length = 4, 3736 .clk_init = macb_clk_init, 3737 .init = macb_init, 3738 }; 3739 3740 static const struct macb_config emac_config = { 3741 .clk_init = at91ether_clk_init, 3742 .init = at91ether_init, 3743 }; 3744 3745 static const struct macb_config np4_config = { 3746 .caps = MACB_CAPS_USRIO_DISABLED, 3747 .clk_init = macb_clk_init, 3748 .init = macb_init, 3749 }; 3750 3751 static const struct macb_config zynqmp_config = { 3752 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3753 MACB_CAPS_JUMBO | 3754 MACB_CAPS_GEM_HAS_PTP, 3755 .dma_burst_length = 16, 3756 .clk_init = macb_clk_init, 3757 .init = macb_init, 3758 .jumbo_max_len = 10240, 3759 }; 3760 3761 static const struct macb_config zynq_config = { 3762 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF, 3763 .dma_burst_length = 16, 3764 .clk_init = macb_clk_init, 3765 .init = macb_init, 3766 }; 3767 3768 static const struct of_device_id macb_dt_ids[] = { 3769 { .compatible = "cdns,at32ap7000-macb" }, 3770 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 3771 { .compatible = "cdns,macb" }, 3772 { .compatible = "cdns,np4-macb", .data = &np4_config }, 3773 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 3774 { .compatible = "cdns,gem", .data = &pc302gem_config }, 3775 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 3776 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 3777 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 3778 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 3779 { .compatible = "cdns,emac", .data = &emac_config }, 3780 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 3781 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 3782 { /* sentinel */ } 3783 }; 3784 MODULE_DEVICE_TABLE(of, macb_dt_ids); 3785 #endif /* CONFIG_OF */ 3786 3787 static const struct macb_config default_gem_config = { 3788 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 3789 MACB_CAPS_JUMBO | 3790 MACB_CAPS_GEM_HAS_PTP, 3791 .dma_burst_length = 16, 3792 .clk_init = macb_clk_init, 3793 .init = macb_init, 3794 .jumbo_max_len = 10240, 3795 }; 3796 3797 static int macb_probe(struct platform_device *pdev) 3798 { 3799 const struct macb_config *macb_config = &default_gem_config; 3800 int (*clk_init)(struct platform_device *, struct clk **, 3801 struct clk **, struct clk **, struct clk **) 3802 = macb_config->clk_init; 3803 int (*init)(struct platform_device *) = macb_config->init; 3804 struct device_node *np = pdev->dev.of_node; 3805 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 3806 unsigned int queue_mask, num_queues; 3807 struct macb_platform_data *pdata; 3808 bool native_io; 3809 struct phy_device *phydev; 3810 struct net_device *dev; 3811 struct resource *regs; 3812 void __iomem *mem; 3813 const char *mac; 3814 struct macb *bp; 3815 int err; 3816 3817 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 3818 mem = devm_ioremap_resource(&pdev->dev, regs); 3819 if (IS_ERR(mem)) 3820 return PTR_ERR(mem); 3821 3822 if (np) { 3823 const struct of_device_id *match; 3824 3825 match = of_match_node(macb_dt_ids, np); 3826 if (match && match->data) { 3827 macb_config = match->data; 3828 clk_init = macb_config->clk_init; 3829 init = macb_config->init; 3830 } 3831 } 3832 3833 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk); 3834 if (err) 3835 return err; 3836 3837 native_io = hw_is_native_io(mem); 3838 3839 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 3840 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 3841 if (!dev) { 3842 err = -ENOMEM; 3843 goto err_disable_clocks; 3844 } 3845 3846 dev->base_addr = regs->start; 3847 3848 SET_NETDEV_DEV(dev, &pdev->dev); 3849 3850 bp = netdev_priv(dev); 3851 bp->pdev = pdev; 3852 bp->dev = dev; 3853 bp->regs = mem; 3854 bp->native_io = native_io; 3855 if (native_io) { 3856 bp->macb_reg_readl = hw_readl_native; 3857 bp->macb_reg_writel = hw_writel_native; 3858 } else { 3859 bp->macb_reg_readl = hw_readl; 3860 bp->macb_reg_writel = hw_writel; 3861 } 3862 bp->num_queues = num_queues; 3863 bp->queue_mask = queue_mask; 3864 if (macb_config) 3865 bp->dma_burst_length = macb_config->dma_burst_length; 3866 bp->pclk = pclk; 3867 bp->hclk = hclk; 3868 bp->tx_clk = tx_clk; 3869 bp->rx_clk = rx_clk; 3870 if (macb_config) 3871 bp->jumbo_max_len = macb_config->jumbo_max_len; 3872 3873 bp->wol = 0; 3874 if (of_get_property(np, "magic-packet", NULL)) 3875 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 3876 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 3877 3878 spin_lock_init(&bp->lock); 3879 3880 /* setup capabilities */ 3881 macb_configure_caps(bp, macb_config); 3882 3883 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3884 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 3885 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 3886 bp->hw_dma_cap |= HW_DMA_CAP_64B; 3887 } 3888 #endif 3889 platform_set_drvdata(pdev, dev); 3890 3891 dev->irq = platform_get_irq(pdev, 0); 3892 if (dev->irq < 0) { 3893 err = dev->irq; 3894 goto err_out_free_netdev; 3895 } 3896 3897 /* MTU range: 68 - 1500 or 10240 */ 3898 dev->min_mtu = GEM_MTU_MIN_SIZE; 3899 if (bp->caps & MACB_CAPS_JUMBO) 3900 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 3901 else 3902 dev->max_mtu = ETH_DATA_LEN; 3903 3904 mac = of_get_mac_address(np); 3905 if (mac) 3906 ether_addr_copy(bp->dev->dev_addr, mac); 3907 else 3908 macb_get_hwaddr(bp); 3909 3910 err = of_get_phy_mode(np); 3911 if (err < 0) { 3912 pdata = dev_get_platdata(&pdev->dev); 3913 if (pdata && pdata->is_rmii) 3914 bp->phy_interface = PHY_INTERFACE_MODE_RMII; 3915 else 3916 bp->phy_interface = PHY_INTERFACE_MODE_MII; 3917 } else { 3918 bp->phy_interface = err; 3919 } 3920 3921 /* IP specific init */ 3922 err = init(pdev); 3923 if (err) 3924 goto err_out_free_netdev; 3925 3926 err = macb_mii_init(bp); 3927 if (err) 3928 goto err_out_free_netdev; 3929 3930 phydev = dev->phydev; 3931 3932 netif_carrier_off(dev); 3933 3934 err = register_netdev(dev); 3935 if (err) { 3936 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 3937 goto err_out_unregister_mdio; 3938 } 3939 3940 phy_attached_info(phydev); 3941 3942 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 3943 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 3944 dev->base_addr, dev->irq, dev->dev_addr); 3945 3946 return 0; 3947 3948 err_out_unregister_mdio: 3949 phy_disconnect(dev->phydev); 3950 mdiobus_unregister(bp->mii_bus); 3951 of_node_put(bp->phy_node); 3952 if (np && of_phy_is_fixed_link(np)) 3953 of_phy_deregister_fixed_link(np); 3954 mdiobus_free(bp->mii_bus); 3955 3956 err_out_free_netdev: 3957 free_netdev(dev); 3958 3959 err_disable_clocks: 3960 clk_disable_unprepare(tx_clk); 3961 clk_disable_unprepare(hclk); 3962 clk_disable_unprepare(pclk); 3963 clk_disable_unprepare(rx_clk); 3964 3965 return err; 3966 } 3967 3968 static int macb_remove(struct platform_device *pdev) 3969 { 3970 struct net_device *dev; 3971 struct macb *bp; 3972 struct device_node *np = pdev->dev.of_node; 3973 3974 dev = platform_get_drvdata(pdev); 3975 3976 if (dev) { 3977 bp = netdev_priv(dev); 3978 if (dev->phydev) 3979 phy_disconnect(dev->phydev); 3980 mdiobus_unregister(bp->mii_bus); 3981 if (np && of_phy_is_fixed_link(np)) 3982 of_phy_deregister_fixed_link(np); 3983 dev->phydev = NULL; 3984 mdiobus_free(bp->mii_bus); 3985 3986 unregister_netdev(dev); 3987 clk_disable_unprepare(bp->tx_clk); 3988 clk_disable_unprepare(bp->hclk); 3989 clk_disable_unprepare(bp->pclk); 3990 clk_disable_unprepare(bp->rx_clk); 3991 of_node_put(bp->phy_node); 3992 free_netdev(dev); 3993 } 3994 3995 return 0; 3996 } 3997 3998 static int __maybe_unused macb_suspend(struct device *dev) 3999 { 4000 struct platform_device *pdev = to_platform_device(dev); 4001 struct net_device *netdev = platform_get_drvdata(pdev); 4002 struct macb *bp = netdev_priv(netdev); 4003 4004 netif_carrier_off(netdev); 4005 netif_device_detach(netdev); 4006 4007 if (bp->wol & MACB_WOL_ENABLED) { 4008 macb_writel(bp, IER, MACB_BIT(WOL)); 4009 macb_writel(bp, WOL, MACB_BIT(MAG)); 4010 enable_irq_wake(bp->queues[0].irq); 4011 } else { 4012 clk_disable_unprepare(bp->tx_clk); 4013 clk_disable_unprepare(bp->hclk); 4014 clk_disable_unprepare(bp->pclk); 4015 clk_disable_unprepare(bp->rx_clk); 4016 } 4017 4018 return 0; 4019 } 4020 4021 static int __maybe_unused macb_resume(struct device *dev) 4022 { 4023 struct platform_device *pdev = to_platform_device(dev); 4024 struct net_device *netdev = platform_get_drvdata(pdev); 4025 struct macb *bp = netdev_priv(netdev); 4026 4027 if (bp->wol & MACB_WOL_ENABLED) { 4028 macb_writel(bp, IDR, MACB_BIT(WOL)); 4029 macb_writel(bp, WOL, 0); 4030 disable_irq_wake(bp->queues[0].irq); 4031 } else { 4032 clk_prepare_enable(bp->pclk); 4033 clk_prepare_enable(bp->hclk); 4034 clk_prepare_enable(bp->tx_clk); 4035 clk_prepare_enable(bp->rx_clk); 4036 } 4037 4038 netif_device_attach(netdev); 4039 4040 return 0; 4041 } 4042 4043 static SIMPLE_DEV_PM_OPS(macb_pm_ops, macb_suspend, macb_resume); 4044 4045 static struct platform_driver macb_driver = { 4046 .probe = macb_probe, 4047 .remove = macb_remove, 4048 .driver = { 4049 .name = "macb", 4050 .of_match_table = of_match_ptr(macb_dt_ids), 4051 .pm = &macb_pm_ops, 4052 }, 4053 }; 4054 4055 module_platform_driver(macb_driver); 4056 4057 MODULE_LICENSE("GPL"); 4058 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4059 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4060 MODULE_ALIAS("platform:macb"); 4061