1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/clk.h> 10 #include <linux/clk-provider.h> 11 #include <linux/crc32.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/kernel.h> 15 #include <linux/types.h> 16 #include <linux/circ_buf.h> 17 #include <linux/slab.h> 18 #include <linux/init.h> 19 #include <linux/io.h> 20 #include <linux/gpio.h> 21 #include <linux/gpio/consumer.h> 22 #include <linux/interrupt.h> 23 #include <linux/netdevice.h> 24 #include <linux/etherdevice.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/platform_data/macb.h> 27 #include <linux/platform_device.h> 28 #include <linux/phylink.h> 29 #include <linux/of.h> 30 #include <linux/of_device.h> 31 #include <linux/of_gpio.h> 32 #include <linux/of_mdio.h> 33 #include <linux/of_net.h> 34 #include <linux/ip.h> 35 #include <linux/udp.h> 36 #include <linux/tcp.h> 37 #include <linux/iopoll.h> 38 #include <linux/pm_runtime.h> 39 #include "macb.h" 40 41 /* This structure is only used for MACB on SiFive FU540 devices */ 42 struct sifive_fu540_macb_mgmt { 43 void __iomem *reg; 44 unsigned long rate; 45 struct clk_hw hw; 46 }; 47 48 #define MACB_RX_BUFFER_SIZE 128 49 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 50 51 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 52 #define MIN_RX_RING_SIZE 64 53 #define MAX_RX_RING_SIZE 8192 54 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 55 * (bp)->rx_ring_size) 56 57 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 58 #define MIN_TX_RING_SIZE 64 59 #define MAX_TX_RING_SIZE 4096 60 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 61 * (bp)->tx_ring_size) 62 63 /* level of occupied TX descriptors under which we wake up TX process */ 64 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 65 66 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 67 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 68 | MACB_BIT(ISR_RLE) \ 69 | MACB_BIT(TXERR)) 70 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 71 | MACB_BIT(TXUBR)) 72 73 /* Max length of transmit frame must be a multiple of 8 bytes */ 74 #define MACB_TX_LEN_ALIGN 8 75 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 76 /* Limit maximum TX length as per Cadence TSO errata. This is to avoid a 77 * false amba_error in TX path from the DMA assuming there is not enough 78 * space in the SRAM (16KB) even when there is. 79 */ 80 #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) 81 82 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 83 #define MACB_NETIF_LSO NETIF_F_TSO 84 85 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 86 #define MACB_WOL_ENABLED (0x1 << 1) 87 88 /* Graceful stop timeouts in us. We should allow up to 89 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 90 */ 91 #define MACB_HALT_TIMEOUT 1230 92 93 #define MACB_PM_TIMEOUT 100 /* ms */ 94 95 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 96 97 /* DMA buffer descriptor might be different size 98 * depends on hardware configuration: 99 * 100 * 1. dma address width 32 bits: 101 * word 1: 32 bit address of Data Buffer 102 * word 2: control 103 * 104 * 2. dma address width 64 bits: 105 * word 1: 32 bit address of Data Buffer 106 * word 2: control 107 * word 3: upper 32 bit address of Data Buffer 108 * word 4: unused 109 * 110 * 3. dma address width 32 bits with hardware timestamping: 111 * word 1: 32 bit address of Data Buffer 112 * word 2: control 113 * word 3: timestamp word 1 114 * word 4: timestamp word 2 115 * 116 * 4. dma address width 64 bits with hardware timestamping: 117 * word 1: 32 bit address of Data Buffer 118 * word 2: control 119 * word 3: upper 32 bit address of Data Buffer 120 * word 4: unused 121 * word 5: timestamp word 1 122 * word 6: timestamp word 2 123 */ 124 static unsigned int macb_dma_desc_get_size(struct macb *bp) 125 { 126 #ifdef MACB_EXT_DESC 127 unsigned int desc_size; 128 129 switch (bp->hw_dma_cap) { 130 case HW_DMA_CAP_64B: 131 desc_size = sizeof(struct macb_dma_desc) 132 + sizeof(struct macb_dma_desc_64); 133 break; 134 case HW_DMA_CAP_PTP: 135 desc_size = sizeof(struct macb_dma_desc) 136 + sizeof(struct macb_dma_desc_ptp); 137 break; 138 case HW_DMA_CAP_64B_PTP: 139 desc_size = sizeof(struct macb_dma_desc) 140 + sizeof(struct macb_dma_desc_64) 141 + sizeof(struct macb_dma_desc_ptp); 142 break; 143 default: 144 desc_size = sizeof(struct macb_dma_desc); 145 } 146 return desc_size; 147 #endif 148 return sizeof(struct macb_dma_desc); 149 } 150 151 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 152 { 153 #ifdef MACB_EXT_DESC 154 switch (bp->hw_dma_cap) { 155 case HW_DMA_CAP_64B: 156 case HW_DMA_CAP_PTP: 157 desc_idx <<= 1; 158 break; 159 case HW_DMA_CAP_64B_PTP: 160 desc_idx *= 3; 161 break; 162 default: 163 break; 164 } 165 #endif 166 return desc_idx; 167 } 168 169 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 170 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 171 { 172 return (struct macb_dma_desc_64 *)((void *)desc 173 + sizeof(struct macb_dma_desc)); 174 } 175 #endif 176 177 /* Ring buffer accessors */ 178 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 179 { 180 return index & (bp->tx_ring_size - 1); 181 } 182 183 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 184 unsigned int index) 185 { 186 index = macb_tx_ring_wrap(queue->bp, index); 187 index = macb_adj_dma_desc_idx(queue->bp, index); 188 return &queue->tx_ring[index]; 189 } 190 191 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 192 unsigned int index) 193 { 194 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 195 } 196 197 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 198 { 199 dma_addr_t offset; 200 201 offset = macb_tx_ring_wrap(queue->bp, index) * 202 macb_dma_desc_get_size(queue->bp); 203 204 return queue->tx_ring_dma + offset; 205 } 206 207 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 208 { 209 return index & (bp->rx_ring_size - 1); 210 } 211 212 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 213 { 214 index = macb_rx_ring_wrap(queue->bp, index); 215 index = macb_adj_dma_desc_idx(queue->bp, index); 216 return &queue->rx_ring[index]; 217 } 218 219 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 220 { 221 return queue->rx_buffers + queue->bp->rx_buffer_size * 222 macb_rx_ring_wrap(queue->bp, index); 223 } 224 225 /* I/O accessors */ 226 static u32 hw_readl_native(struct macb *bp, int offset) 227 { 228 return __raw_readl(bp->regs + offset); 229 } 230 231 static void hw_writel_native(struct macb *bp, int offset, u32 value) 232 { 233 __raw_writel(value, bp->regs + offset); 234 } 235 236 static u32 hw_readl(struct macb *bp, int offset) 237 { 238 return readl_relaxed(bp->regs + offset); 239 } 240 241 static void hw_writel(struct macb *bp, int offset, u32 value) 242 { 243 writel_relaxed(value, bp->regs + offset); 244 } 245 246 /* Find the CPU endianness by using the loopback bit of NCR register. When the 247 * CPU is in big endian we need to program swapped mode for management 248 * descriptor access. 249 */ 250 static bool hw_is_native_io(void __iomem *addr) 251 { 252 u32 value = MACB_BIT(LLB); 253 254 __raw_writel(value, addr + MACB_NCR); 255 value = __raw_readl(addr + MACB_NCR); 256 257 /* Write 0 back to disable everything */ 258 __raw_writel(0, addr + MACB_NCR); 259 260 return value == MACB_BIT(LLB); 261 } 262 263 static bool hw_is_gem(void __iomem *addr, bool native_io) 264 { 265 u32 id; 266 267 if (native_io) 268 id = __raw_readl(addr + MACB_MID); 269 else 270 id = readl_relaxed(addr + MACB_MID); 271 272 return MACB_BFEXT(IDNUM, id) >= 0x2; 273 } 274 275 static void macb_set_hwaddr(struct macb *bp) 276 { 277 u32 bottom; 278 u16 top; 279 280 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 281 macb_or_gem_writel(bp, SA1B, bottom); 282 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 283 macb_or_gem_writel(bp, SA1T, top); 284 285 /* Clear unused address register sets */ 286 macb_or_gem_writel(bp, SA2B, 0); 287 macb_or_gem_writel(bp, SA2T, 0); 288 macb_or_gem_writel(bp, SA3B, 0); 289 macb_or_gem_writel(bp, SA3T, 0); 290 macb_or_gem_writel(bp, SA4B, 0); 291 macb_or_gem_writel(bp, SA4T, 0); 292 } 293 294 static void macb_get_hwaddr(struct macb *bp) 295 { 296 u32 bottom; 297 u16 top; 298 u8 addr[6]; 299 int i; 300 301 /* Check all 4 address register for valid address */ 302 for (i = 0; i < 4; i++) { 303 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 304 top = macb_or_gem_readl(bp, SA1T + i * 8); 305 306 addr[0] = bottom & 0xff; 307 addr[1] = (bottom >> 8) & 0xff; 308 addr[2] = (bottom >> 16) & 0xff; 309 addr[3] = (bottom >> 24) & 0xff; 310 addr[4] = top & 0xff; 311 addr[5] = (top >> 8) & 0xff; 312 313 if (is_valid_ether_addr(addr)) { 314 memcpy(bp->dev->dev_addr, addr, sizeof(addr)); 315 return; 316 } 317 } 318 319 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 320 eth_hw_addr_random(bp->dev); 321 } 322 323 static int macb_mdio_wait_for_idle(struct macb *bp) 324 { 325 u32 val; 326 327 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 328 1, MACB_MDIO_TIMEOUT); 329 } 330 331 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 332 { 333 struct macb *bp = bus->priv; 334 int status; 335 336 status = pm_runtime_get_sync(&bp->pdev->dev); 337 if (status < 0) { 338 pm_runtime_put_noidle(&bp->pdev->dev); 339 goto mdio_pm_exit; 340 } 341 342 status = macb_mdio_wait_for_idle(bp); 343 if (status < 0) 344 goto mdio_read_exit; 345 346 if (regnum & MII_ADDR_C45) { 347 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 348 | MACB_BF(RW, MACB_MAN_C45_ADDR) 349 | MACB_BF(PHYA, mii_id) 350 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 351 | MACB_BF(DATA, regnum & 0xFFFF) 352 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 353 354 status = macb_mdio_wait_for_idle(bp); 355 if (status < 0) 356 goto mdio_read_exit; 357 358 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 359 | MACB_BF(RW, MACB_MAN_C45_READ) 360 | MACB_BF(PHYA, mii_id) 361 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 362 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 363 } else { 364 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 365 | MACB_BF(RW, MACB_MAN_C22_READ) 366 | MACB_BF(PHYA, mii_id) 367 | MACB_BF(REGA, regnum) 368 | MACB_BF(CODE, MACB_MAN_C22_CODE))); 369 } 370 371 status = macb_mdio_wait_for_idle(bp); 372 if (status < 0) 373 goto mdio_read_exit; 374 375 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 376 377 mdio_read_exit: 378 pm_runtime_mark_last_busy(&bp->pdev->dev); 379 pm_runtime_put_autosuspend(&bp->pdev->dev); 380 mdio_pm_exit: 381 return status; 382 } 383 384 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 385 u16 value) 386 { 387 struct macb *bp = bus->priv; 388 int status; 389 390 status = pm_runtime_get_sync(&bp->pdev->dev); 391 if (status < 0) { 392 pm_runtime_put_noidle(&bp->pdev->dev); 393 goto mdio_pm_exit; 394 } 395 396 status = macb_mdio_wait_for_idle(bp); 397 if (status < 0) 398 goto mdio_write_exit; 399 400 if (regnum & MII_ADDR_C45) { 401 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 402 | MACB_BF(RW, MACB_MAN_C45_ADDR) 403 | MACB_BF(PHYA, mii_id) 404 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 405 | MACB_BF(DATA, regnum & 0xFFFF) 406 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 407 408 status = macb_mdio_wait_for_idle(bp); 409 if (status < 0) 410 goto mdio_write_exit; 411 412 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 413 | MACB_BF(RW, MACB_MAN_C45_WRITE) 414 | MACB_BF(PHYA, mii_id) 415 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 416 | MACB_BF(CODE, MACB_MAN_C45_CODE) 417 | MACB_BF(DATA, value))); 418 } else { 419 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 420 | MACB_BF(RW, MACB_MAN_C22_WRITE) 421 | MACB_BF(PHYA, mii_id) 422 | MACB_BF(REGA, regnum) 423 | MACB_BF(CODE, MACB_MAN_C22_CODE) 424 | MACB_BF(DATA, value))); 425 } 426 427 status = macb_mdio_wait_for_idle(bp); 428 if (status < 0) 429 goto mdio_write_exit; 430 431 mdio_write_exit: 432 pm_runtime_mark_last_busy(&bp->pdev->dev); 433 pm_runtime_put_autosuspend(&bp->pdev->dev); 434 mdio_pm_exit: 435 return status; 436 } 437 438 static void macb_init_buffers(struct macb *bp) 439 { 440 struct macb_queue *queue; 441 unsigned int q; 442 443 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 444 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 445 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 446 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 447 queue_writel(queue, RBQPH, 448 upper_32_bits(queue->rx_ring_dma)); 449 #endif 450 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 451 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 452 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 453 queue_writel(queue, TBQPH, 454 upper_32_bits(queue->tx_ring_dma)); 455 #endif 456 } 457 } 458 459 /** 460 * macb_set_tx_clk() - Set a clock to a new frequency 461 * @clk Pointer to the clock to change 462 * @rate New frequency in Hz 463 * @dev Pointer to the struct net_device 464 */ 465 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev) 466 { 467 long ferr, rate, rate_rounded; 468 469 if (!clk) 470 return; 471 472 switch (speed) { 473 case SPEED_10: 474 rate = 2500000; 475 break; 476 case SPEED_100: 477 rate = 25000000; 478 break; 479 case SPEED_1000: 480 rate = 125000000; 481 break; 482 default: 483 return; 484 } 485 486 rate_rounded = clk_round_rate(clk, rate); 487 if (rate_rounded < 0) 488 return; 489 490 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 491 * is not satisfied. 492 */ 493 ferr = abs(rate_rounded - rate); 494 ferr = DIV_ROUND_UP(ferr, rate / 100000); 495 if (ferr > 5) 496 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n", 497 rate); 498 499 if (clk_set_rate(clk, rate_rounded)) 500 netdev_err(dev, "adjusting tx_clk failed.\n"); 501 } 502 503 static void macb_validate(struct phylink_config *config, 504 unsigned long *supported, 505 struct phylink_link_state *state) 506 { 507 struct net_device *ndev = to_net_dev(config->dev); 508 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; 509 struct macb *bp = netdev_priv(ndev); 510 511 /* We only support MII, RMII, GMII, RGMII & SGMII. */ 512 if (state->interface != PHY_INTERFACE_MODE_NA && 513 state->interface != PHY_INTERFACE_MODE_MII && 514 state->interface != PHY_INTERFACE_MODE_RMII && 515 state->interface != PHY_INTERFACE_MODE_GMII && 516 state->interface != PHY_INTERFACE_MODE_SGMII && 517 !phy_interface_mode_is_rgmii(state->interface)) { 518 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); 519 return; 520 } 521 522 if (!macb_is_gem(bp) && 523 (state->interface == PHY_INTERFACE_MODE_GMII || 524 phy_interface_mode_is_rgmii(state->interface))) { 525 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); 526 return; 527 } 528 529 phylink_set_port_modes(mask); 530 phylink_set(mask, Autoneg); 531 phylink_set(mask, Asym_Pause); 532 533 phylink_set(mask, 10baseT_Half); 534 phylink_set(mask, 10baseT_Full); 535 phylink_set(mask, 100baseT_Half); 536 phylink_set(mask, 100baseT_Full); 537 538 if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && 539 (state->interface == PHY_INTERFACE_MODE_NA || 540 state->interface == PHY_INTERFACE_MODE_GMII || 541 state->interface == PHY_INTERFACE_MODE_SGMII || 542 phy_interface_mode_is_rgmii(state->interface))) { 543 phylink_set(mask, 1000baseT_Full); 544 phylink_set(mask, 1000baseX_Full); 545 546 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) 547 phylink_set(mask, 1000baseT_Half); 548 } 549 550 bitmap_and(supported, supported, mask, __ETHTOOL_LINK_MODE_MASK_NBITS); 551 bitmap_and(state->advertising, state->advertising, mask, 552 __ETHTOOL_LINK_MODE_MASK_NBITS); 553 } 554 555 static void macb_mac_pcs_get_state(struct phylink_config *config, 556 struct phylink_link_state *state) 557 { 558 state->link = 0; 559 } 560 561 static void macb_mac_an_restart(struct phylink_config *config) 562 { 563 /* Not supported */ 564 } 565 566 static void macb_mac_config(struct phylink_config *config, unsigned int mode, 567 const struct phylink_link_state *state) 568 { 569 struct net_device *ndev = to_net_dev(config->dev); 570 struct macb *bp = netdev_priv(ndev); 571 unsigned long flags; 572 u32 old_ctrl, ctrl; 573 574 spin_lock_irqsave(&bp->lock, flags); 575 576 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); 577 578 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { 579 if (state->interface == PHY_INTERFACE_MODE_RMII) 580 ctrl |= MACB_BIT(RM9200_RMII); 581 } else if (macb_is_gem(bp)) { 582 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); 583 584 if (state->interface == PHY_INTERFACE_MODE_SGMII) 585 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 586 } 587 588 /* Apply the new configuration, if any */ 589 if (old_ctrl ^ ctrl) 590 macb_or_gem_writel(bp, NCFGR, ctrl); 591 592 spin_unlock_irqrestore(&bp->lock, flags); 593 } 594 595 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, 596 phy_interface_t interface) 597 { 598 struct net_device *ndev = to_net_dev(config->dev); 599 struct macb *bp = netdev_priv(ndev); 600 struct macb_queue *queue; 601 unsigned int q; 602 u32 ctrl; 603 604 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 605 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 606 queue_writel(queue, IDR, 607 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 608 609 /* Disable Rx and Tx */ 610 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); 611 macb_writel(bp, NCR, ctrl); 612 613 netif_tx_stop_all_queues(ndev); 614 } 615 616 static void macb_mac_link_up(struct phylink_config *config, 617 struct phy_device *phy, 618 unsigned int mode, phy_interface_t interface, 619 int speed, int duplex, 620 bool tx_pause, bool rx_pause) 621 { 622 struct net_device *ndev = to_net_dev(config->dev); 623 struct macb *bp = netdev_priv(ndev); 624 struct macb_queue *queue; 625 unsigned long flags; 626 unsigned int q; 627 u32 ctrl; 628 629 spin_lock_irqsave(&bp->lock, flags); 630 631 ctrl = macb_or_gem_readl(bp, NCFGR); 632 633 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 634 635 if (speed == SPEED_100) 636 ctrl |= MACB_BIT(SPD); 637 638 if (duplex) 639 ctrl |= MACB_BIT(FD); 640 641 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 642 ctrl &= ~MACB_BIT(PAE); 643 if (macb_is_gem(bp)) { 644 ctrl &= ~GEM_BIT(GBE); 645 646 if (speed == SPEED_1000) 647 ctrl |= GEM_BIT(GBE); 648 } 649 650 /* We do not support MLO_PAUSE_RX yet */ 651 if (tx_pause) 652 ctrl |= MACB_BIT(PAE); 653 654 macb_set_tx_clk(bp->tx_clk, speed, ndev); 655 656 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down 657 * cleared the pipeline and control registers. 658 */ 659 bp->macbgem_ops.mog_init_rings(bp); 660 macb_init_buffers(bp); 661 662 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 663 queue_writel(queue, IER, 664 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 665 } 666 667 macb_or_gem_writel(bp, NCFGR, ctrl); 668 669 spin_unlock_irqrestore(&bp->lock, flags); 670 671 /* Enable Rx and Tx */ 672 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 673 674 netif_tx_wake_all_queues(ndev); 675 } 676 677 static const struct phylink_mac_ops macb_phylink_ops = { 678 .validate = macb_validate, 679 .mac_pcs_get_state = macb_mac_pcs_get_state, 680 .mac_an_restart = macb_mac_an_restart, 681 .mac_config = macb_mac_config, 682 .mac_link_down = macb_mac_link_down, 683 .mac_link_up = macb_mac_link_up, 684 }; 685 686 static bool macb_phy_handle_exists(struct device_node *dn) 687 { 688 dn = of_parse_phandle(dn, "phy-handle", 0); 689 of_node_put(dn); 690 return dn != NULL; 691 } 692 693 static int macb_phylink_connect(struct macb *bp) 694 { 695 struct device_node *dn = bp->pdev->dev.of_node; 696 struct net_device *dev = bp->dev; 697 struct phy_device *phydev; 698 int ret; 699 700 if (dn) 701 ret = phylink_of_phy_connect(bp->phylink, dn, 0); 702 703 if (!dn || (ret && !macb_phy_handle_exists(dn))) { 704 phydev = phy_find_first(bp->mii_bus); 705 if (!phydev) { 706 netdev_err(dev, "no PHY found\n"); 707 return -ENXIO; 708 } 709 710 /* attach the mac to the phy */ 711 ret = phylink_connect_phy(bp->phylink, phydev); 712 } 713 714 if (ret) { 715 netdev_err(dev, "Could not attach PHY (%d)\n", ret); 716 return ret; 717 } 718 719 phylink_start(bp->phylink); 720 721 return 0; 722 } 723 724 /* based on au1000_eth. c*/ 725 static int macb_mii_probe(struct net_device *dev) 726 { 727 struct macb *bp = netdev_priv(dev); 728 729 bp->phylink_config.dev = &dev->dev; 730 bp->phylink_config.type = PHYLINK_NETDEV; 731 732 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, 733 bp->phy_interface, &macb_phylink_ops); 734 if (IS_ERR(bp->phylink)) { 735 netdev_err(dev, "Could not create a phylink instance (%ld)\n", 736 PTR_ERR(bp->phylink)); 737 return PTR_ERR(bp->phylink); 738 } 739 740 return 0; 741 } 742 743 static int macb_mdiobus_register(struct macb *bp) 744 { 745 struct device_node *child, *np = bp->pdev->dev.of_node; 746 747 if (of_phy_is_fixed_link(np)) 748 return mdiobus_register(bp->mii_bus); 749 750 /* Only create the PHY from the device tree if at least one PHY is 751 * described. Otherwise scan the entire MDIO bus. We do this to support 752 * old device tree that did not follow the best practices and did not 753 * describe their network PHYs. 754 */ 755 for_each_available_child_of_node(np, child) 756 if (of_mdiobus_child_is_phy(child)) { 757 /* The loop increments the child refcount, 758 * decrement it before returning. 759 */ 760 of_node_put(child); 761 762 return of_mdiobus_register(bp->mii_bus, np); 763 } 764 765 return mdiobus_register(bp->mii_bus); 766 } 767 768 static int macb_mii_init(struct macb *bp) 769 { 770 int err = -ENXIO; 771 772 /* Enable management port */ 773 macb_writel(bp, NCR, MACB_BIT(MPE)); 774 775 bp->mii_bus = mdiobus_alloc(); 776 if (!bp->mii_bus) { 777 err = -ENOMEM; 778 goto err_out; 779 } 780 781 bp->mii_bus->name = "MACB_mii_bus"; 782 bp->mii_bus->read = &macb_mdio_read; 783 bp->mii_bus->write = &macb_mdio_write; 784 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 785 bp->pdev->name, bp->pdev->id); 786 bp->mii_bus->priv = bp; 787 bp->mii_bus->parent = &bp->pdev->dev; 788 789 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 790 791 err = macb_mdiobus_register(bp); 792 if (err) 793 goto err_out_free_mdiobus; 794 795 err = macb_mii_probe(bp->dev); 796 if (err) 797 goto err_out_unregister_bus; 798 799 return 0; 800 801 err_out_unregister_bus: 802 mdiobus_unregister(bp->mii_bus); 803 err_out_free_mdiobus: 804 mdiobus_free(bp->mii_bus); 805 err_out: 806 return err; 807 } 808 809 static void macb_update_stats(struct macb *bp) 810 { 811 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 812 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 813 int offset = MACB_PFR; 814 815 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 816 817 for (; p < end; p++, offset += 4) 818 *p += bp->macb_reg_readl(bp, offset); 819 } 820 821 static int macb_halt_tx(struct macb *bp) 822 { 823 unsigned long halt_time, timeout; 824 u32 status; 825 826 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 827 828 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 829 do { 830 halt_time = jiffies; 831 status = macb_readl(bp, TSR); 832 if (!(status & MACB_BIT(TGO))) 833 return 0; 834 835 udelay(250); 836 } while (time_before(halt_time, timeout)); 837 838 return -ETIMEDOUT; 839 } 840 841 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 842 { 843 if (tx_skb->mapping) { 844 if (tx_skb->mapped_as_page) 845 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 846 tx_skb->size, DMA_TO_DEVICE); 847 else 848 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 849 tx_skb->size, DMA_TO_DEVICE); 850 tx_skb->mapping = 0; 851 } 852 853 if (tx_skb->skb) { 854 dev_kfree_skb_any(tx_skb->skb); 855 tx_skb->skb = NULL; 856 } 857 } 858 859 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 860 { 861 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 862 struct macb_dma_desc_64 *desc_64; 863 864 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 865 desc_64 = macb_64b_desc(bp, desc); 866 desc_64->addrh = upper_32_bits(addr); 867 /* The low bits of RX address contain the RX_USED bit, clearing 868 * of which allows packet RX. Make sure the high bits are also 869 * visible to HW at that point. 870 */ 871 dma_wmb(); 872 } 873 #endif 874 desc->addr = lower_32_bits(addr); 875 } 876 877 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 878 { 879 dma_addr_t addr = 0; 880 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 881 struct macb_dma_desc_64 *desc_64; 882 883 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 884 desc_64 = macb_64b_desc(bp, desc); 885 addr = ((u64)(desc_64->addrh) << 32); 886 } 887 #endif 888 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 889 return addr; 890 } 891 892 static void macb_tx_error_task(struct work_struct *work) 893 { 894 struct macb_queue *queue = container_of(work, struct macb_queue, 895 tx_error_task); 896 struct macb *bp = queue->bp; 897 struct macb_tx_skb *tx_skb; 898 struct macb_dma_desc *desc; 899 struct sk_buff *skb; 900 unsigned int tail; 901 unsigned long flags; 902 903 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 904 (unsigned int)(queue - bp->queues), 905 queue->tx_tail, queue->tx_head); 906 907 /* Prevent the queue IRQ handlers from running: each of them may call 908 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 909 * As explained below, we have to halt the transmission before updating 910 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 911 * network engine about the macb/gem being halted. 912 */ 913 spin_lock_irqsave(&bp->lock, flags); 914 915 /* Make sure nobody is trying to queue up new packets */ 916 netif_tx_stop_all_queues(bp->dev); 917 918 /* Stop transmission now 919 * (in case we have just queued new packets) 920 * macb/gem must be halted to write TBQP register 921 */ 922 if (macb_halt_tx(bp)) 923 /* Just complain for now, reinitializing TX path can be good */ 924 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 925 926 /* Treat frames in TX queue including the ones that caused the error. 927 * Free transmit buffers in upper layer. 928 */ 929 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 930 u32 ctrl; 931 932 desc = macb_tx_desc(queue, tail); 933 ctrl = desc->ctrl; 934 tx_skb = macb_tx_skb(queue, tail); 935 skb = tx_skb->skb; 936 937 if (ctrl & MACB_BIT(TX_USED)) { 938 /* skb is set for the last buffer of the frame */ 939 while (!skb) { 940 macb_tx_unmap(bp, tx_skb); 941 tail++; 942 tx_skb = macb_tx_skb(queue, tail); 943 skb = tx_skb->skb; 944 } 945 946 /* ctrl still refers to the first buffer descriptor 947 * since it's the only one written back by the hardware 948 */ 949 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 950 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 951 macb_tx_ring_wrap(bp, tail), 952 skb->data); 953 bp->dev->stats.tx_packets++; 954 queue->stats.tx_packets++; 955 bp->dev->stats.tx_bytes += skb->len; 956 queue->stats.tx_bytes += skb->len; 957 } 958 } else { 959 /* "Buffers exhausted mid-frame" errors may only happen 960 * if the driver is buggy, so complain loudly about 961 * those. Statistics are updated by hardware. 962 */ 963 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 964 netdev_err(bp->dev, 965 "BUG: TX buffers exhausted mid-frame\n"); 966 967 desc->ctrl = ctrl | MACB_BIT(TX_USED); 968 } 969 970 macb_tx_unmap(bp, tx_skb); 971 } 972 973 /* Set end of TX queue */ 974 desc = macb_tx_desc(queue, 0); 975 macb_set_addr(bp, desc, 0); 976 desc->ctrl = MACB_BIT(TX_USED); 977 978 /* Make descriptor updates visible to hardware */ 979 wmb(); 980 981 /* Reinitialize the TX desc queue */ 982 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 983 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 984 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 985 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 986 #endif 987 /* Make TX ring reflect state of hardware */ 988 queue->tx_head = 0; 989 queue->tx_tail = 0; 990 991 /* Housework before enabling TX IRQ */ 992 macb_writel(bp, TSR, macb_readl(bp, TSR)); 993 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 994 995 /* Now we are ready to start transmission again */ 996 netif_tx_start_all_queues(bp->dev); 997 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 998 999 spin_unlock_irqrestore(&bp->lock, flags); 1000 } 1001 1002 static void macb_tx_interrupt(struct macb_queue *queue) 1003 { 1004 unsigned int tail; 1005 unsigned int head; 1006 u32 status; 1007 struct macb *bp = queue->bp; 1008 u16 queue_index = queue - bp->queues; 1009 1010 status = macb_readl(bp, TSR); 1011 macb_writel(bp, TSR, status); 1012 1013 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1014 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 1015 1016 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 1017 (unsigned long)status); 1018 1019 head = queue->tx_head; 1020 for (tail = queue->tx_tail; tail != head; tail++) { 1021 struct macb_tx_skb *tx_skb; 1022 struct sk_buff *skb; 1023 struct macb_dma_desc *desc; 1024 u32 ctrl; 1025 1026 desc = macb_tx_desc(queue, tail); 1027 1028 /* Make hw descriptor updates visible to CPU */ 1029 rmb(); 1030 1031 ctrl = desc->ctrl; 1032 1033 /* TX_USED bit is only set by hardware on the very first buffer 1034 * descriptor of the transmitted frame. 1035 */ 1036 if (!(ctrl & MACB_BIT(TX_USED))) 1037 break; 1038 1039 /* Process all buffers of the current transmitted frame */ 1040 for (;; tail++) { 1041 tx_skb = macb_tx_skb(queue, tail); 1042 skb = tx_skb->skb; 1043 1044 /* First, update TX stats if needed */ 1045 if (skb) { 1046 if (unlikely(skb_shinfo(skb)->tx_flags & 1047 SKBTX_HW_TSTAMP) && 1048 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 1049 /* skb now belongs to timestamp buffer 1050 * and will be removed later 1051 */ 1052 tx_skb->skb = NULL; 1053 } 1054 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 1055 macb_tx_ring_wrap(bp, tail), 1056 skb->data); 1057 bp->dev->stats.tx_packets++; 1058 queue->stats.tx_packets++; 1059 bp->dev->stats.tx_bytes += skb->len; 1060 queue->stats.tx_bytes += skb->len; 1061 } 1062 1063 /* Now we can safely release resources */ 1064 macb_tx_unmap(bp, tx_skb); 1065 1066 /* skb is set only for the last buffer of the frame. 1067 * WARNING: at this point skb has been freed by 1068 * macb_tx_unmap(). 1069 */ 1070 if (skb) 1071 break; 1072 } 1073 } 1074 1075 queue->tx_tail = tail; 1076 if (__netif_subqueue_stopped(bp->dev, queue_index) && 1077 CIRC_CNT(queue->tx_head, queue->tx_tail, 1078 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 1079 netif_wake_subqueue(bp->dev, queue_index); 1080 } 1081 1082 static void gem_rx_refill(struct macb_queue *queue) 1083 { 1084 unsigned int entry; 1085 struct sk_buff *skb; 1086 dma_addr_t paddr; 1087 struct macb *bp = queue->bp; 1088 struct macb_dma_desc *desc; 1089 1090 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 1091 bp->rx_ring_size) > 0) { 1092 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 1093 1094 /* Make hw descriptor updates visible to CPU */ 1095 rmb(); 1096 1097 queue->rx_prepared_head++; 1098 desc = macb_rx_desc(queue, entry); 1099 1100 if (!queue->rx_skbuff[entry]) { 1101 /* allocate sk_buff for this free entry in ring */ 1102 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 1103 if (unlikely(!skb)) { 1104 netdev_err(bp->dev, 1105 "Unable to allocate sk_buff\n"); 1106 break; 1107 } 1108 1109 /* now fill corresponding descriptor entry */ 1110 paddr = dma_map_single(&bp->pdev->dev, skb->data, 1111 bp->rx_buffer_size, 1112 DMA_FROM_DEVICE); 1113 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 1114 dev_kfree_skb(skb); 1115 break; 1116 } 1117 1118 queue->rx_skbuff[entry] = skb; 1119 1120 if (entry == bp->rx_ring_size - 1) 1121 paddr |= MACB_BIT(RX_WRAP); 1122 desc->ctrl = 0; 1123 /* Setting addr clears RX_USED and allows reception, 1124 * make sure ctrl is cleared first to avoid a race. 1125 */ 1126 dma_wmb(); 1127 macb_set_addr(bp, desc, paddr); 1128 1129 /* properly align Ethernet header */ 1130 skb_reserve(skb, NET_IP_ALIGN); 1131 } else { 1132 desc->ctrl = 0; 1133 dma_wmb(); 1134 desc->addr &= ~MACB_BIT(RX_USED); 1135 } 1136 } 1137 1138 /* Make descriptor updates visible to hardware */ 1139 wmb(); 1140 1141 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 1142 queue, queue->rx_prepared_head, queue->rx_tail); 1143 } 1144 1145 /* Mark DMA descriptors from begin up to and not including end as unused */ 1146 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 1147 unsigned int end) 1148 { 1149 unsigned int frag; 1150 1151 for (frag = begin; frag != end; frag++) { 1152 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 1153 1154 desc->addr &= ~MACB_BIT(RX_USED); 1155 } 1156 1157 /* Make descriptor updates visible to hardware */ 1158 wmb(); 1159 1160 /* When this happens, the hardware stats registers for 1161 * whatever caused this is updated, so we don't have to record 1162 * anything. 1163 */ 1164 } 1165 1166 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 1167 int budget) 1168 { 1169 struct macb *bp = queue->bp; 1170 unsigned int len; 1171 unsigned int entry; 1172 struct sk_buff *skb; 1173 struct macb_dma_desc *desc; 1174 int count = 0; 1175 1176 while (count < budget) { 1177 u32 ctrl; 1178 dma_addr_t addr; 1179 bool rxused; 1180 1181 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1182 desc = macb_rx_desc(queue, entry); 1183 1184 /* Make hw descriptor updates visible to CPU */ 1185 rmb(); 1186 1187 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1188 addr = macb_get_addr(bp, desc); 1189 1190 if (!rxused) 1191 break; 1192 1193 /* Ensure ctrl is at least as up-to-date as rxused */ 1194 dma_rmb(); 1195 1196 ctrl = desc->ctrl; 1197 1198 queue->rx_tail++; 1199 count++; 1200 1201 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1202 netdev_err(bp->dev, 1203 "not whole frame pointed by descriptor\n"); 1204 bp->dev->stats.rx_dropped++; 1205 queue->stats.rx_dropped++; 1206 break; 1207 } 1208 skb = queue->rx_skbuff[entry]; 1209 if (unlikely(!skb)) { 1210 netdev_err(bp->dev, 1211 "inconsistent Rx descriptor chain\n"); 1212 bp->dev->stats.rx_dropped++; 1213 queue->stats.rx_dropped++; 1214 break; 1215 } 1216 /* now everything is ready for receiving packet */ 1217 queue->rx_skbuff[entry] = NULL; 1218 len = ctrl & bp->rx_frm_len_mask; 1219 1220 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1221 1222 skb_put(skb, len); 1223 dma_unmap_single(&bp->pdev->dev, addr, 1224 bp->rx_buffer_size, DMA_FROM_DEVICE); 1225 1226 skb->protocol = eth_type_trans(skb, bp->dev); 1227 skb_checksum_none_assert(skb); 1228 if (bp->dev->features & NETIF_F_RXCSUM && 1229 !(bp->dev->flags & IFF_PROMISC) && 1230 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1231 skb->ip_summed = CHECKSUM_UNNECESSARY; 1232 1233 bp->dev->stats.rx_packets++; 1234 queue->stats.rx_packets++; 1235 bp->dev->stats.rx_bytes += skb->len; 1236 queue->stats.rx_bytes += skb->len; 1237 1238 gem_ptp_do_rxstamp(bp, skb, desc); 1239 1240 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1241 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1242 skb->len, skb->csum); 1243 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1244 skb_mac_header(skb), 16, true); 1245 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1246 skb->data, 32, true); 1247 #endif 1248 1249 napi_gro_receive(napi, skb); 1250 } 1251 1252 gem_rx_refill(queue); 1253 1254 return count; 1255 } 1256 1257 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1258 unsigned int first_frag, unsigned int last_frag) 1259 { 1260 unsigned int len; 1261 unsigned int frag; 1262 unsigned int offset; 1263 struct sk_buff *skb; 1264 struct macb_dma_desc *desc; 1265 struct macb *bp = queue->bp; 1266 1267 desc = macb_rx_desc(queue, last_frag); 1268 len = desc->ctrl & bp->rx_frm_len_mask; 1269 1270 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1271 macb_rx_ring_wrap(bp, first_frag), 1272 macb_rx_ring_wrap(bp, last_frag), len); 1273 1274 /* The ethernet header starts NET_IP_ALIGN bytes into the 1275 * first buffer. Since the header is 14 bytes, this makes the 1276 * payload word-aligned. 1277 * 1278 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1279 * the two padding bytes into the skb so that we avoid hitting 1280 * the slowpath in memcpy(), and pull them off afterwards. 1281 */ 1282 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1283 if (!skb) { 1284 bp->dev->stats.rx_dropped++; 1285 for (frag = first_frag; ; frag++) { 1286 desc = macb_rx_desc(queue, frag); 1287 desc->addr &= ~MACB_BIT(RX_USED); 1288 if (frag == last_frag) 1289 break; 1290 } 1291 1292 /* Make descriptor updates visible to hardware */ 1293 wmb(); 1294 1295 return 1; 1296 } 1297 1298 offset = 0; 1299 len += NET_IP_ALIGN; 1300 skb_checksum_none_assert(skb); 1301 skb_put(skb, len); 1302 1303 for (frag = first_frag; ; frag++) { 1304 unsigned int frag_len = bp->rx_buffer_size; 1305 1306 if (offset + frag_len > len) { 1307 if (unlikely(frag != last_frag)) { 1308 dev_kfree_skb_any(skb); 1309 return -1; 1310 } 1311 frag_len = len - offset; 1312 } 1313 skb_copy_to_linear_data_offset(skb, offset, 1314 macb_rx_buffer(queue, frag), 1315 frag_len); 1316 offset += bp->rx_buffer_size; 1317 desc = macb_rx_desc(queue, frag); 1318 desc->addr &= ~MACB_BIT(RX_USED); 1319 1320 if (frag == last_frag) 1321 break; 1322 } 1323 1324 /* Make descriptor updates visible to hardware */ 1325 wmb(); 1326 1327 __skb_pull(skb, NET_IP_ALIGN); 1328 skb->protocol = eth_type_trans(skb, bp->dev); 1329 1330 bp->dev->stats.rx_packets++; 1331 bp->dev->stats.rx_bytes += skb->len; 1332 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1333 skb->len, skb->csum); 1334 napi_gro_receive(napi, skb); 1335 1336 return 0; 1337 } 1338 1339 static inline void macb_init_rx_ring(struct macb_queue *queue) 1340 { 1341 struct macb *bp = queue->bp; 1342 dma_addr_t addr; 1343 struct macb_dma_desc *desc = NULL; 1344 int i; 1345 1346 addr = queue->rx_buffers_dma; 1347 for (i = 0; i < bp->rx_ring_size; i++) { 1348 desc = macb_rx_desc(queue, i); 1349 macb_set_addr(bp, desc, addr); 1350 desc->ctrl = 0; 1351 addr += bp->rx_buffer_size; 1352 } 1353 desc->addr |= MACB_BIT(RX_WRAP); 1354 queue->rx_tail = 0; 1355 } 1356 1357 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1358 int budget) 1359 { 1360 struct macb *bp = queue->bp; 1361 bool reset_rx_queue = false; 1362 int received = 0; 1363 unsigned int tail; 1364 int first_frag = -1; 1365 1366 for (tail = queue->rx_tail; budget > 0; tail++) { 1367 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1368 u32 ctrl; 1369 1370 /* Make hw descriptor updates visible to CPU */ 1371 rmb(); 1372 1373 if (!(desc->addr & MACB_BIT(RX_USED))) 1374 break; 1375 1376 /* Ensure ctrl is at least as up-to-date as addr */ 1377 dma_rmb(); 1378 1379 ctrl = desc->ctrl; 1380 1381 if (ctrl & MACB_BIT(RX_SOF)) { 1382 if (first_frag != -1) 1383 discard_partial_frame(queue, first_frag, tail); 1384 first_frag = tail; 1385 } 1386 1387 if (ctrl & MACB_BIT(RX_EOF)) { 1388 int dropped; 1389 1390 if (unlikely(first_frag == -1)) { 1391 reset_rx_queue = true; 1392 continue; 1393 } 1394 1395 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1396 first_frag = -1; 1397 if (unlikely(dropped < 0)) { 1398 reset_rx_queue = true; 1399 continue; 1400 } 1401 if (!dropped) { 1402 received++; 1403 budget--; 1404 } 1405 } 1406 } 1407 1408 if (unlikely(reset_rx_queue)) { 1409 unsigned long flags; 1410 u32 ctrl; 1411 1412 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1413 1414 spin_lock_irqsave(&bp->lock, flags); 1415 1416 ctrl = macb_readl(bp, NCR); 1417 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1418 1419 macb_init_rx_ring(queue); 1420 queue_writel(queue, RBQP, queue->rx_ring_dma); 1421 1422 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1423 1424 spin_unlock_irqrestore(&bp->lock, flags); 1425 return received; 1426 } 1427 1428 if (first_frag != -1) 1429 queue->rx_tail = first_frag; 1430 else 1431 queue->rx_tail = tail; 1432 1433 return received; 1434 } 1435 1436 static int macb_poll(struct napi_struct *napi, int budget) 1437 { 1438 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1439 struct macb *bp = queue->bp; 1440 int work_done; 1441 u32 status; 1442 1443 status = macb_readl(bp, RSR); 1444 macb_writel(bp, RSR, status); 1445 1446 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1447 (unsigned long)status, budget); 1448 1449 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 1450 if (work_done < budget) { 1451 napi_complete_done(napi, work_done); 1452 1453 /* Packets received while interrupts were disabled */ 1454 status = macb_readl(bp, RSR); 1455 if (status) { 1456 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1457 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1458 napi_reschedule(napi); 1459 } else { 1460 queue_writel(queue, IER, bp->rx_intr_mask); 1461 } 1462 } 1463 1464 /* TODO: Handle errors */ 1465 1466 return work_done; 1467 } 1468 1469 static void macb_hresp_error_task(unsigned long data) 1470 { 1471 struct macb *bp = (struct macb *)data; 1472 struct net_device *dev = bp->dev; 1473 struct macb_queue *queue; 1474 unsigned int q; 1475 u32 ctrl; 1476 1477 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1478 queue_writel(queue, IDR, bp->rx_intr_mask | 1479 MACB_TX_INT_FLAGS | 1480 MACB_BIT(HRESP)); 1481 } 1482 ctrl = macb_readl(bp, NCR); 1483 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1484 macb_writel(bp, NCR, ctrl); 1485 1486 netif_tx_stop_all_queues(dev); 1487 netif_carrier_off(dev); 1488 1489 bp->macbgem_ops.mog_init_rings(bp); 1490 1491 /* Initialize TX and RX buffers */ 1492 macb_init_buffers(bp); 1493 1494 /* Enable interrupts */ 1495 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1496 queue_writel(queue, IER, 1497 bp->rx_intr_mask | 1498 MACB_TX_INT_FLAGS | 1499 MACB_BIT(HRESP)); 1500 1501 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1502 macb_writel(bp, NCR, ctrl); 1503 1504 netif_carrier_on(dev); 1505 netif_tx_start_all_queues(dev); 1506 } 1507 1508 static void macb_tx_restart(struct macb_queue *queue) 1509 { 1510 unsigned int head = queue->tx_head; 1511 unsigned int tail = queue->tx_tail; 1512 struct macb *bp = queue->bp; 1513 1514 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1515 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1516 1517 if (head == tail) 1518 return; 1519 1520 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1521 } 1522 1523 static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) 1524 { 1525 struct macb_queue *queue = dev_id; 1526 struct macb *bp = queue->bp; 1527 u32 status; 1528 1529 status = queue_readl(queue, ISR); 1530 1531 if (unlikely(!status)) 1532 return IRQ_NONE; 1533 1534 spin_lock(&bp->lock); 1535 1536 if (status & MACB_BIT(WOL)) { 1537 queue_writel(queue, IDR, MACB_BIT(WOL)); 1538 macb_writel(bp, WOL, 0); 1539 netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", 1540 (unsigned int)(queue - bp->queues), 1541 (unsigned long)status); 1542 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1543 queue_writel(queue, ISR, MACB_BIT(WOL)); 1544 pm_wakeup_event(&bp->pdev->dev, 0); 1545 } 1546 1547 spin_unlock(&bp->lock); 1548 1549 return IRQ_HANDLED; 1550 } 1551 1552 static irqreturn_t gem_wol_interrupt(int irq, void *dev_id) 1553 { 1554 struct macb_queue *queue = dev_id; 1555 struct macb *bp = queue->bp; 1556 u32 status; 1557 1558 status = queue_readl(queue, ISR); 1559 1560 if (unlikely(!status)) 1561 return IRQ_NONE; 1562 1563 spin_lock(&bp->lock); 1564 1565 if (status & GEM_BIT(WOL)) { 1566 queue_writel(queue, IDR, GEM_BIT(WOL)); 1567 gem_writel(bp, WOL, 0); 1568 netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", 1569 (unsigned int)(queue - bp->queues), 1570 (unsigned long)status); 1571 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1572 queue_writel(queue, ISR, GEM_BIT(WOL)); 1573 pm_wakeup_event(&bp->pdev->dev, 0); 1574 } 1575 1576 spin_unlock(&bp->lock); 1577 1578 return IRQ_HANDLED; 1579 } 1580 1581 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1582 { 1583 struct macb_queue *queue = dev_id; 1584 struct macb *bp = queue->bp; 1585 struct net_device *dev = bp->dev; 1586 u32 status, ctrl; 1587 1588 status = queue_readl(queue, ISR); 1589 1590 if (unlikely(!status)) 1591 return IRQ_NONE; 1592 1593 spin_lock(&bp->lock); 1594 1595 while (status) { 1596 /* close possible race with dev_close */ 1597 if (unlikely(!netif_running(dev))) { 1598 queue_writel(queue, IDR, -1); 1599 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1600 queue_writel(queue, ISR, -1); 1601 break; 1602 } 1603 1604 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1605 (unsigned int)(queue - bp->queues), 1606 (unsigned long)status); 1607 1608 if (status & bp->rx_intr_mask) { 1609 /* There's no point taking any more interrupts 1610 * until we have processed the buffers. The 1611 * scheduling call may fail if the poll routine 1612 * is already scheduled, so disable interrupts 1613 * now. 1614 */ 1615 queue_writel(queue, IDR, bp->rx_intr_mask); 1616 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1617 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1618 1619 if (napi_schedule_prep(&queue->napi)) { 1620 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1621 __napi_schedule(&queue->napi); 1622 } 1623 } 1624 1625 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1626 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1627 schedule_work(&queue->tx_error_task); 1628 1629 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1630 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1631 1632 break; 1633 } 1634 1635 if (status & MACB_BIT(TCOMP)) 1636 macb_tx_interrupt(queue); 1637 1638 if (status & MACB_BIT(TXUBR)) 1639 macb_tx_restart(queue); 1640 1641 /* Link change detection isn't possible with RMII, so we'll 1642 * add that if/when we get our hands on a full-blown MII PHY. 1643 */ 1644 1645 /* There is a hardware issue under heavy load where DMA can 1646 * stop, this causes endless "used buffer descriptor read" 1647 * interrupts but it can be cleared by re-enabling RX. See 1648 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1649 * section 16.7.4 for details. RXUBR is only enabled for 1650 * these two versions. 1651 */ 1652 if (status & MACB_BIT(RXUBR)) { 1653 ctrl = macb_readl(bp, NCR); 1654 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1655 wmb(); 1656 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1657 1658 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1659 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1660 } 1661 1662 if (status & MACB_BIT(ISR_ROVR)) { 1663 /* We missed at least one packet */ 1664 if (macb_is_gem(bp)) 1665 bp->hw_stats.gem.rx_overruns++; 1666 else 1667 bp->hw_stats.macb.rx_overruns++; 1668 1669 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1670 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1671 } 1672 1673 if (status & MACB_BIT(HRESP)) { 1674 tasklet_schedule(&bp->hresp_err_tasklet); 1675 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1676 1677 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1678 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1679 } 1680 status = queue_readl(queue, ISR); 1681 } 1682 1683 spin_unlock(&bp->lock); 1684 1685 return IRQ_HANDLED; 1686 } 1687 1688 #ifdef CONFIG_NET_POLL_CONTROLLER 1689 /* Polling receive - used by netconsole and other diagnostic tools 1690 * to allow network i/o with interrupts disabled. 1691 */ 1692 static void macb_poll_controller(struct net_device *dev) 1693 { 1694 struct macb *bp = netdev_priv(dev); 1695 struct macb_queue *queue; 1696 unsigned long flags; 1697 unsigned int q; 1698 1699 local_irq_save(flags); 1700 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1701 macb_interrupt(dev->irq, queue); 1702 local_irq_restore(flags); 1703 } 1704 #endif 1705 1706 static unsigned int macb_tx_map(struct macb *bp, 1707 struct macb_queue *queue, 1708 struct sk_buff *skb, 1709 unsigned int hdrlen) 1710 { 1711 dma_addr_t mapping; 1712 unsigned int len, entry, i, tx_head = queue->tx_head; 1713 struct macb_tx_skb *tx_skb = NULL; 1714 struct macb_dma_desc *desc; 1715 unsigned int offset, size, count = 0; 1716 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1717 unsigned int eof = 1, mss_mfs = 0; 1718 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1719 1720 /* LSO */ 1721 if (skb_shinfo(skb)->gso_size != 0) { 1722 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1723 /* UDP - UFO */ 1724 lso_ctrl = MACB_LSO_UFO_ENABLE; 1725 else 1726 /* TCP - TSO */ 1727 lso_ctrl = MACB_LSO_TSO_ENABLE; 1728 } 1729 1730 /* First, map non-paged data */ 1731 len = skb_headlen(skb); 1732 1733 /* first buffer length */ 1734 size = hdrlen; 1735 1736 offset = 0; 1737 while (len) { 1738 entry = macb_tx_ring_wrap(bp, tx_head); 1739 tx_skb = &queue->tx_skb[entry]; 1740 1741 mapping = dma_map_single(&bp->pdev->dev, 1742 skb->data + offset, 1743 size, DMA_TO_DEVICE); 1744 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1745 goto dma_error; 1746 1747 /* Save info to properly release resources */ 1748 tx_skb->skb = NULL; 1749 tx_skb->mapping = mapping; 1750 tx_skb->size = size; 1751 tx_skb->mapped_as_page = false; 1752 1753 len -= size; 1754 offset += size; 1755 count++; 1756 tx_head++; 1757 1758 size = min(len, bp->max_tx_length); 1759 } 1760 1761 /* Then, map paged data from fragments */ 1762 for (f = 0; f < nr_frags; f++) { 1763 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1764 1765 len = skb_frag_size(frag); 1766 offset = 0; 1767 while (len) { 1768 size = min(len, bp->max_tx_length); 1769 entry = macb_tx_ring_wrap(bp, tx_head); 1770 tx_skb = &queue->tx_skb[entry]; 1771 1772 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1773 offset, size, DMA_TO_DEVICE); 1774 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1775 goto dma_error; 1776 1777 /* Save info to properly release resources */ 1778 tx_skb->skb = NULL; 1779 tx_skb->mapping = mapping; 1780 tx_skb->size = size; 1781 tx_skb->mapped_as_page = true; 1782 1783 len -= size; 1784 offset += size; 1785 count++; 1786 tx_head++; 1787 } 1788 } 1789 1790 /* Should never happen */ 1791 if (unlikely(!tx_skb)) { 1792 netdev_err(bp->dev, "BUG! empty skb!\n"); 1793 return 0; 1794 } 1795 1796 /* This is the last buffer of the frame: save socket buffer */ 1797 tx_skb->skb = skb; 1798 1799 /* Update TX ring: update buffer descriptors in reverse order 1800 * to avoid race condition 1801 */ 1802 1803 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1804 * to set the end of TX queue 1805 */ 1806 i = tx_head; 1807 entry = macb_tx_ring_wrap(bp, i); 1808 ctrl = MACB_BIT(TX_USED); 1809 desc = macb_tx_desc(queue, entry); 1810 desc->ctrl = ctrl; 1811 1812 if (lso_ctrl) { 1813 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1814 /* include header and FCS in value given to h/w */ 1815 mss_mfs = skb_shinfo(skb)->gso_size + 1816 skb_transport_offset(skb) + 1817 ETH_FCS_LEN; 1818 else /* TSO */ { 1819 mss_mfs = skb_shinfo(skb)->gso_size; 1820 /* TCP Sequence Number Source Select 1821 * can be set only for TSO 1822 */ 1823 seq_ctrl = 0; 1824 } 1825 } 1826 1827 do { 1828 i--; 1829 entry = macb_tx_ring_wrap(bp, i); 1830 tx_skb = &queue->tx_skb[entry]; 1831 desc = macb_tx_desc(queue, entry); 1832 1833 ctrl = (u32)tx_skb->size; 1834 if (eof) { 1835 ctrl |= MACB_BIT(TX_LAST); 1836 eof = 0; 1837 } 1838 if (unlikely(entry == (bp->tx_ring_size - 1))) 1839 ctrl |= MACB_BIT(TX_WRAP); 1840 1841 /* First descriptor is header descriptor */ 1842 if (i == queue->tx_head) { 1843 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 1844 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 1845 if ((bp->dev->features & NETIF_F_HW_CSUM) && 1846 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 1847 ctrl |= MACB_BIT(TX_NOCRC); 1848 } else 1849 /* Only set MSS/MFS on payload descriptors 1850 * (second or later descriptor) 1851 */ 1852 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 1853 1854 /* Set TX buffer descriptor */ 1855 macb_set_addr(bp, desc, tx_skb->mapping); 1856 /* desc->addr must be visible to hardware before clearing 1857 * 'TX_USED' bit in desc->ctrl. 1858 */ 1859 wmb(); 1860 desc->ctrl = ctrl; 1861 } while (i != queue->tx_head); 1862 1863 queue->tx_head = tx_head; 1864 1865 return count; 1866 1867 dma_error: 1868 netdev_err(bp->dev, "TX DMA map failed\n"); 1869 1870 for (i = queue->tx_head; i != tx_head; i++) { 1871 tx_skb = macb_tx_skb(queue, i); 1872 1873 macb_tx_unmap(bp, tx_skb); 1874 } 1875 1876 return 0; 1877 } 1878 1879 static netdev_features_t macb_features_check(struct sk_buff *skb, 1880 struct net_device *dev, 1881 netdev_features_t features) 1882 { 1883 unsigned int nr_frags, f; 1884 unsigned int hdrlen; 1885 1886 /* Validate LSO compatibility */ 1887 1888 /* there is only one buffer or protocol is not UDP */ 1889 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) 1890 return features; 1891 1892 /* length of header */ 1893 hdrlen = skb_transport_offset(skb); 1894 1895 /* For UFO only: 1896 * When software supplies two or more payload buffers all payload buffers 1897 * apart from the last must be a multiple of 8 bytes in size. 1898 */ 1899 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 1900 return features & ~MACB_NETIF_LSO; 1901 1902 nr_frags = skb_shinfo(skb)->nr_frags; 1903 /* No need to check last fragment */ 1904 nr_frags--; 1905 for (f = 0; f < nr_frags; f++) { 1906 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1907 1908 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 1909 return features & ~MACB_NETIF_LSO; 1910 } 1911 return features; 1912 } 1913 1914 static inline int macb_clear_csum(struct sk_buff *skb) 1915 { 1916 /* no change for packets without checksum offloading */ 1917 if (skb->ip_summed != CHECKSUM_PARTIAL) 1918 return 0; 1919 1920 /* make sure we can modify the header */ 1921 if (unlikely(skb_cow_head(skb, 0))) 1922 return -1; 1923 1924 /* initialize checksum field 1925 * This is required - at least for Zynq, which otherwise calculates 1926 * wrong UDP header checksums for UDP packets with UDP data len <=2 1927 */ 1928 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 1929 return 0; 1930 } 1931 1932 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 1933 { 1934 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb); 1935 int padlen = ETH_ZLEN - (*skb)->len; 1936 int headroom = skb_headroom(*skb); 1937 int tailroom = skb_tailroom(*skb); 1938 struct sk_buff *nskb; 1939 u32 fcs; 1940 1941 if (!(ndev->features & NETIF_F_HW_CSUM) || 1942 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 1943 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 1944 return 0; 1945 1946 if (padlen <= 0) { 1947 /* FCS could be appeded to tailroom. */ 1948 if (tailroom >= ETH_FCS_LEN) 1949 goto add_fcs; 1950 /* FCS could be appeded by moving data to headroom. */ 1951 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 1952 padlen = 0; 1953 /* No room for FCS, need to reallocate skb. */ 1954 else 1955 padlen = ETH_FCS_LEN; 1956 } else { 1957 /* Add room for FCS. */ 1958 padlen += ETH_FCS_LEN; 1959 } 1960 1961 if (!cloned && headroom + tailroom >= padlen) { 1962 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 1963 skb_set_tail_pointer(*skb, (*skb)->len); 1964 } else { 1965 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 1966 if (!nskb) 1967 return -ENOMEM; 1968 1969 dev_consume_skb_any(*skb); 1970 *skb = nskb; 1971 } 1972 1973 if (padlen > ETH_FCS_LEN) 1974 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 1975 1976 add_fcs: 1977 /* set FCS to packet */ 1978 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 1979 fcs = ~fcs; 1980 1981 skb_put_u8(*skb, fcs & 0xff); 1982 skb_put_u8(*skb, (fcs >> 8) & 0xff); 1983 skb_put_u8(*skb, (fcs >> 16) & 0xff); 1984 skb_put_u8(*skb, (fcs >> 24) & 0xff); 1985 1986 return 0; 1987 } 1988 1989 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 1990 { 1991 u16 queue_index = skb_get_queue_mapping(skb); 1992 struct macb *bp = netdev_priv(dev); 1993 struct macb_queue *queue = &bp->queues[queue_index]; 1994 unsigned long flags; 1995 unsigned int desc_cnt, nr_frags, frag_size, f; 1996 unsigned int hdrlen; 1997 bool is_lso; 1998 netdev_tx_t ret = NETDEV_TX_OK; 1999 2000 if (macb_clear_csum(skb)) { 2001 dev_kfree_skb_any(skb); 2002 return ret; 2003 } 2004 2005 if (macb_pad_and_fcs(&skb, dev)) { 2006 dev_kfree_skb_any(skb); 2007 return ret; 2008 } 2009 2010 is_lso = (skb_shinfo(skb)->gso_size != 0); 2011 2012 if (is_lso) { 2013 /* length of headers */ 2014 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2015 /* only queue eth + ip headers separately for UDP */ 2016 hdrlen = skb_transport_offset(skb); 2017 else 2018 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 2019 if (skb_headlen(skb) < hdrlen) { 2020 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 2021 /* if this is required, would need to copy to single buffer */ 2022 return NETDEV_TX_BUSY; 2023 } 2024 } else 2025 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 2026 2027 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 2028 netdev_vdbg(bp->dev, 2029 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 2030 queue_index, skb->len, skb->head, skb->data, 2031 skb_tail_pointer(skb), skb_end_pointer(skb)); 2032 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 2033 skb->data, 16, true); 2034 #endif 2035 2036 /* Count how many TX buffer descriptors are needed to send this 2037 * socket buffer: skb fragments of jumbo frames may need to be 2038 * split into many buffer descriptors. 2039 */ 2040 if (is_lso && (skb_headlen(skb) > hdrlen)) 2041 /* extra header descriptor if also payload in first buffer */ 2042 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 2043 else 2044 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 2045 nr_frags = skb_shinfo(skb)->nr_frags; 2046 for (f = 0; f < nr_frags; f++) { 2047 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 2048 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 2049 } 2050 2051 spin_lock_irqsave(&bp->lock, flags); 2052 2053 /* This is a hard error, log it. */ 2054 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 2055 bp->tx_ring_size) < desc_cnt) { 2056 netif_stop_subqueue(dev, queue_index); 2057 spin_unlock_irqrestore(&bp->lock, flags); 2058 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 2059 queue->tx_head, queue->tx_tail); 2060 return NETDEV_TX_BUSY; 2061 } 2062 2063 /* Map socket buffer for DMA transfer */ 2064 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 2065 dev_kfree_skb_any(skb); 2066 goto unlock; 2067 } 2068 2069 /* Make newly initialized descriptor visible to hardware */ 2070 wmb(); 2071 skb_tx_timestamp(skb); 2072 2073 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 2074 2075 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 2076 netif_stop_subqueue(dev, queue_index); 2077 2078 unlock: 2079 spin_unlock_irqrestore(&bp->lock, flags); 2080 2081 return ret; 2082 } 2083 2084 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 2085 { 2086 if (!macb_is_gem(bp)) { 2087 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 2088 } else { 2089 bp->rx_buffer_size = size; 2090 2091 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 2092 netdev_dbg(bp->dev, 2093 "RX buffer must be multiple of %d bytes, expanding\n", 2094 RX_BUFFER_MULTIPLE); 2095 bp->rx_buffer_size = 2096 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 2097 } 2098 } 2099 2100 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 2101 bp->dev->mtu, bp->rx_buffer_size); 2102 } 2103 2104 static void gem_free_rx_buffers(struct macb *bp) 2105 { 2106 struct sk_buff *skb; 2107 struct macb_dma_desc *desc; 2108 struct macb_queue *queue; 2109 dma_addr_t addr; 2110 unsigned int q; 2111 int i; 2112 2113 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2114 if (!queue->rx_skbuff) 2115 continue; 2116 2117 for (i = 0; i < bp->rx_ring_size; i++) { 2118 skb = queue->rx_skbuff[i]; 2119 2120 if (!skb) 2121 continue; 2122 2123 desc = macb_rx_desc(queue, i); 2124 addr = macb_get_addr(bp, desc); 2125 2126 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 2127 DMA_FROM_DEVICE); 2128 dev_kfree_skb_any(skb); 2129 skb = NULL; 2130 } 2131 2132 kfree(queue->rx_skbuff); 2133 queue->rx_skbuff = NULL; 2134 } 2135 } 2136 2137 static void macb_free_rx_buffers(struct macb *bp) 2138 { 2139 struct macb_queue *queue = &bp->queues[0]; 2140 2141 if (queue->rx_buffers) { 2142 dma_free_coherent(&bp->pdev->dev, 2143 bp->rx_ring_size * bp->rx_buffer_size, 2144 queue->rx_buffers, queue->rx_buffers_dma); 2145 queue->rx_buffers = NULL; 2146 } 2147 } 2148 2149 static void macb_free_consistent(struct macb *bp) 2150 { 2151 struct macb_queue *queue; 2152 unsigned int q; 2153 int size; 2154 2155 bp->macbgem_ops.mog_free_rx_buffers(bp); 2156 2157 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2158 kfree(queue->tx_skb); 2159 queue->tx_skb = NULL; 2160 if (queue->tx_ring) { 2161 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2162 dma_free_coherent(&bp->pdev->dev, size, 2163 queue->tx_ring, queue->tx_ring_dma); 2164 queue->tx_ring = NULL; 2165 } 2166 if (queue->rx_ring) { 2167 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2168 dma_free_coherent(&bp->pdev->dev, size, 2169 queue->rx_ring, queue->rx_ring_dma); 2170 queue->rx_ring = NULL; 2171 } 2172 } 2173 } 2174 2175 static int gem_alloc_rx_buffers(struct macb *bp) 2176 { 2177 struct macb_queue *queue; 2178 unsigned int q; 2179 int size; 2180 2181 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2182 size = bp->rx_ring_size * sizeof(struct sk_buff *); 2183 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 2184 if (!queue->rx_skbuff) 2185 return -ENOMEM; 2186 else 2187 netdev_dbg(bp->dev, 2188 "Allocated %d RX struct sk_buff entries at %p\n", 2189 bp->rx_ring_size, queue->rx_skbuff); 2190 } 2191 return 0; 2192 } 2193 2194 static int macb_alloc_rx_buffers(struct macb *bp) 2195 { 2196 struct macb_queue *queue = &bp->queues[0]; 2197 int size; 2198 2199 size = bp->rx_ring_size * bp->rx_buffer_size; 2200 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 2201 &queue->rx_buffers_dma, GFP_KERNEL); 2202 if (!queue->rx_buffers) 2203 return -ENOMEM; 2204 2205 netdev_dbg(bp->dev, 2206 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 2207 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 2208 return 0; 2209 } 2210 2211 static int macb_alloc_consistent(struct macb *bp) 2212 { 2213 struct macb_queue *queue; 2214 unsigned int q; 2215 int size; 2216 2217 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2218 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2219 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2220 &queue->tx_ring_dma, 2221 GFP_KERNEL); 2222 if (!queue->tx_ring) 2223 goto out_err; 2224 netdev_dbg(bp->dev, 2225 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2226 q, size, (unsigned long)queue->tx_ring_dma, 2227 queue->tx_ring); 2228 2229 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2230 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2231 if (!queue->tx_skb) 2232 goto out_err; 2233 2234 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2235 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2236 &queue->rx_ring_dma, GFP_KERNEL); 2237 if (!queue->rx_ring) 2238 goto out_err; 2239 netdev_dbg(bp->dev, 2240 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2241 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2242 } 2243 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2244 goto out_err; 2245 2246 return 0; 2247 2248 out_err: 2249 macb_free_consistent(bp); 2250 return -ENOMEM; 2251 } 2252 2253 static void gem_init_rings(struct macb *bp) 2254 { 2255 struct macb_queue *queue; 2256 struct macb_dma_desc *desc = NULL; 2257 unsigned int q; 2258 int i; 2259 2260 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2261 for (i = 0; i < bp->tx_ring_size; i++) { 2262 desc = macb_tx_desc(queue, i); 2263 macb_set_addr(bp, desc, 0); 2264 desc->ctrl = MACB_BIT(TX_USED); 2265 } 2266 desc->ctrl |= MACB_BIT(TX_WRAP); 2267 queue->tx_head = 0; 2268 queue->tx_tail = 0; 2269 2270 queue->rx_tail = 0; 2271 queue->rx_prepared_head = 0; 2272 2273 gem_rx_refill(queue); 2274 } 2275 2276 } 2277 2278 static void macb_init_rings(struct macb *bp) 2279 { 2280 int i; 2281 struct macb_dma_desc *desc = NULL; 2282 2283 macb_init_rx_ring(&bp->queues[0]); 2284 2285 for (i = 0; i < bp->tx_ring_size; i++) { 2286 desc = macb_tx_desc(&bp->queues[0], i); 2287 macb_set_addr(bp, desc, 0); 2288 desc->ctrl = MACB_BIT(TX_USED); 2289 } 2290 bp->queues[0].tx_head = 0; 2291 bp->queues[0].tx_tail = 0; 2292 desc->ctrl |= MACB_BIT(TX_WRAP); 2293 } 2294 2295 static void macb_reset_hw(struct macb *bp) 2296 { 2297 struct macb_queue *queue; 2298 unsigned int q; 2299 u32 ctrl = macb_readl(bp, NCR); 2300 2301 /* Disable RX and TX (XXX: Should we halt the transmission 2302 * more gracefully?) 2303 */ 2304 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2305 2306 /* Clear the stats registers (XXX: Update stats first?) */ 2307 ctrl |= MACB_BIT(CLRSTAT); 2308 2309 macb_writel(bp, NCR, ctrl); 2310 2311 /* Clear all status flags */ 2312 macb_writel(bp, TSR, -1); 2313 macb_writel(bp, RSR, -1); 2314 2315 /* Disable all interrupts */ 2316 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2317 queue_writel(queue, IDR, -1); 2318 queue_readl(queue, ISR); 2319 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2320 queue_writel(queue, ISR, -1); 2321 } 2322 } 2323 2324 static u32 gem_mdc_clk_div(struct macb *bp) 2325 { 2326 u32 config; 2327 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2328 2329 if (pclk_hz <= 20000000) 2330 config = GEM_BF(CLK, GEM_CLK_DIV8); 2331 else if (pclk_hz <= 40000000) 2332 config = GEM_BF(CLK, GEM_CLK_DIV16); 2333 else if (pclk_hz <= 80000000) 2334 config = GEM_BF(CLK, GEM_CLK_DIV32); 2335 else if (pclk_hz <= 120000000) 2336 config = GEM_BF(CLK, GEM_CLK_DIV48); 2337 else if (pclk_hz <= 160000000) 2338 config = GEM_BF(CLK, GEM_CLK_DIV64); 2339 else 2340 config = GEM_BF(CLK, GEM_CLK_DIV96); 2341 2342 return config; 2343 } 2344 2345 static u32 macb_mdc_clk_div(struct macb *bp) 2346 { 2347 u32 config; 2348 unsigned long pclk_hz; 2349 2350 if (macb_is_gem(bp)) 2351 return gem_mdc_clk_div(bp); 2352 2353 pclk_hz = clk_get_rate(bp->pclk); 2354 if (pclk_hz <= 20000000) 2355 config = MACB_BF(CLK, MACB_CLK_DIV8); 2356 else if (pclk_hz <= 40000000) 2357 config = MACB_BF(CLK, MACB_CLK_DIV16); 2358 else if (pclk_hz <= 80000000) 2359 config = MACB_BF(CLK, MACB_CLK_DIV32); 2360 else 2361 config = MACB_BF(CLK, MACB_CLK_DIV64); 2362 2363 return config; 2364 } 2365 2366 /* Get the DMA bus width field of the network configuration register that we 2367 * should program. We find the width from decoding the design configuration 2368 * register to find the maximum supported data bus width. 2369 */ 2370 static u32 macb_dbw(struct macb *bp) 2371 { 2372 if (!macb_is_gem(bp)) 2373 return 0; 2374 2375 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2376 case 4: 2377 return GEM_BF(DBW, GEM_DBW128); 2378 case 2: 2379 return GEM_BF(DBW, GEM_DBW64); 2380 case 1: 2381 default: 2382 return GEM_BF(DBW, GEM_DBW32); 2383 } 2384 } 2385 2386 /* Configure the receive DMA engine 2387 * - use the correct receive buffer size 2388 * - set best burst length for DMA operations 2389 * (if not supported by FIFO, it will fallback to default) 2390 * - set both rx/tx packet buffers to full memory size 2391 * These are configurable parameters for GEM. 2392 */ 2393 static void macb_configure_dma(struct macb *bp) 2394 { 2395 struct macb_queue *queue; 2396 u32 buffer_size; 2397 unsigned int q; 2398 u32 dmacfg; 2399 2400 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2401 if (macb_is_gem(bp)) { 2402 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2403 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2404 if (q) 2405 queue_writel(queue, RBQS, buffer_size); 2406 else 2407 dmacfg |= GEM_BF(RXBS, buffer_size); 2408 } 2409 if (bp->dma_burst_length) 2410 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2411 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2412 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2413 2414 if (bp->native_io) 2415 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2416 else 2417 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2418 2419 if (bp->dev->features & NETIF_F_HW_CSUM) 2420 dmacfg |= GEM_BIT(TXCOEN); 2421 else 2422 dmacfg &= ~GEM_BIT(TXCOEN); 2423 2424 dmacfg &= ~GEM_BIT(ADDR64); 2425 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2426 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2427 dmacfg |= GEM_BIT(ADDR64); 2428 #endif 2429 #ifdef CONFIG_MACB_USE_HWSTAMP 2430 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2431 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2432 #endif 2433 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2434 dmacfg); 2435 gem_writel(bp, DMACFG, dmacfg); 2436 } 2437 } 2438 2439 static void macb_init_hw(struct macb *bp) 2440 { 2441 u32 config; 2442 2443 macb_reset_hw(bp); 2444 macb_set_hwaddr(bp); 2445 2446 config = macb_mdc_clk_div(bp); 2447 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2448 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2449 if (bp->caps & MACB_CAPS_JUMBO) 2450 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2451 else 2452 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2453 if (bp->dev->flags & IFF_PROMISC) 2454 config |= MACB_BIT(CAF); /* Copy All Frames */ 2455 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2456 config |= GEM_BIT(RXCOEN); 2457 if (!(bp->dev->flags & IFF_BROADCAST)) 2458 config |= MACB_BIT(NBC); /* No BroadCast */ 2459 config |= macb_dbw(bp); 2460 macb_writel(bp, NCFGR, config); 2461 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2462 gem_writel(bp, JML, bp->jumbo_max_len); 2463 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2464 if (bp->caps & MACB_CAPS_JUMBO) 2465 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2466 2467 macb_configure_dma(bp); 2468 } 2469 2470 /* The hash address register is 64 bits long and takes up two 2471 * locations in the memory map. The least significant bits are stored 2472 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2473 * 2474 * The unicast hash enable and the multicast hash enable bits in the 2475 * network configuration register enable the reception of hash matched 2476 * frames. The destination address is reduced to a 6 bit index into 2477 * the 64 bit hash register using the following hash function. The 2478 * hash function is an exclusive or of every sixth bit of the 2479 * destination address. 2480 * 2481 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2482 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2483 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2484 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2485 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2486 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2487 * 2488 * da[0] represents the least significant bit of the first byte 2489 * received, that is, the multicast/unicast indicator, and da[47] 2490 * represents the most significant bit of the last byte received. If 2491 * the hash index, hi[n], points to a bit that is set in the hash 2492 * register then the frame will be matched according to whether the 2493 * frame is multicast or unicast. A multicast match will be signalled 2494 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2495 * index points to a bit set in the hash register. A unicast match 2496 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2497 * and the hash index points to a bit set in the hash register. To 2498 * receive all multicast frames, the hash register should be set with 2499 * all ones and the multicast hash enable bit should be set in the 2500 * network configuration register. 2501 */ 2502 2503 static inline int hash_bit_value(int bitnr, __u8 *addr) 2504 { 2505 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2506 return 1; 2507 return 0; 2508 } 2509 2510 /* Return the hash index value for the specified address. */ 2511 static int hash_get_index(__u8 *addr) 2512 { 2513 int i, j, bitval; 2514 int hash_index = 0; 2515 2516 for (j = 0; j < 6; j++) { 2517 for (i = 0, bitval = 0; i < 8; i++) 2518 bitval ^= hash_bit_value(i * 6 + j, addr); 2519 2520 hash_index |= (bitval << j); 2521 } 2522 2523 return hash_index; 2524 } 2525 2526 /* Add multicast addresses to the internal multicast-hash table. */ 2527 static void macb_sethashtable(struct net_device *dev) 2528 { 2529 struct netdev_hw_addr *ha; 2530 unsigned long mc_filter[2]; 2531 unsigned int bitnr; 2532 struct macb *bp = netdev_priv(dev); 2533 2534 mc_filter[0] = 0; 2535 mc_filter[1] = 0; 2536 2537 netdev_for_each_mc_addr(ha, dev) { 2538 bitnr = hash_get_index(ha->addr); 2539 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2540 } 2541 2542 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2543 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2544 } 2545 2546 /* Enable/Disable promiscuous and multicast modes. */ 2547 static void macb_set_rx_mode(struct net_device *dev) 2548 { 2549 unsigned long cfg; 2550 struct macb *bp = netdev_priv(dev); 2551 2552 cfg = macb_readl(bp, NCFGR); 2553 2554 if (dev->flags & IFF_PROMISC) { 2555 /* Enable promiscuous mode */ 2556 cfg |= MACB_BIT(CAF); 2557 2558 /* Disable RX checksum offload */ 2559 if (macb_is_gem(bp)) 2560 cfg &= ~GEM_BIT(RXCOEN); 2561 } else { 2562 /* Disable promiscuous mode */ 2563 cfg &= ~MACB_BIT(CAF); 2564 2565 /* Enable RX checksum offload only if requested */ 2566 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2567 cfg |= GEM_BIT(RXCOEN); 2568 } 2569 2570 if (dev->flags & IFF_ALLMULTI) { 2571 /* Enable all multicast mode */ 2572 macb_or_gem_writel(bp, HRB, -1); 2573 macb_or_gem_writel(bp, HRT, -1); 2574 cfg |= MACB_BIT(NCFGR_MTI); 2575 } else if (!netdev_mc_empty(dev)) { 2576 /* Enable specific multicasts */ 2577 macb_sethashtable(dev); 2578 cfg |= MACB_BIT(NCFGR_MTI); 2579 } else if (dev->flags & (~IFF_ALLMULTI)) { 2580 /* Disable all multicast mode */ 2581 macb_or_gem_writel(bp, HRB, 0); 2582 macb_or_gem_writel(bp, HRT, 0); 2583 cfg &= ~MACB_BIT(NCFGR_MTI); 2584 } 2585 2586 macb_writel(bp, NCFGR, cfg); 2587 } 2588 2589 static int macb_open(struct net_device *dev) 2590 { 2591 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2592 struct macb *bp = netdev_priv(dev); 2593 struct macb_queue *queue; 2594 unsigned int q; 2595 int err; 2596 2597 netdev_dbg(bp->dev, "open\n"); 2598 2599 err = pm_runtime_get_sync(&bp->pdev->dev); 2600 if (err < 0) 2601 goto pm_exit; 2602 2603 /* RX buffers initialization */ 2604 macb_init_rx_buffer_size(bp, bufsz); 2605 2606 err = macb_alloc_consistent(bp); 2607 if (err) { 2608 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2609 err); 2610 goto pm_exit; 2611 } 2612 2613 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2614 napi_enable(&queue->napi); 2615 2616 macb_init_hw(bp); 2617 2618 err = macb_phylink_connect(bp); 2619 if (err) 2620 goto reset_hw; 2621 2622 netif_tx_start_all_queues(dev); 2623 2624 if (bp->ptp_info) 2625 bp->ptp_info->ptp_init(dev); 2626 2627 return 0; 2628 2629 reset_hw: 2630 macb_reset_hw(bp); 2631 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2632 napi_disable(&queue->napi); 2633 macb_free_consistent(bp); 2634 pm_exit: 2635 pm_runtime_put_sync(&bp->pdev->dev); 2636 return err; 2637 } 2638 2639 static int macb_close(struct net_device *dev) 2640 { 2641 struct macb *bp = netdev_priv(dev); 2642 struct macb_queue *queue; 2643 unsigned long flags; 2644 unsigned int q; 2645 2646 netif_tx_stop_all_queues(dev); 2647 2648 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2649 napi_disable(&queue->napi); 2650 2651 phylink_stop(bp->phylink); 2652 phylink_disconnect_phy(bp->phylink); 2653 2654 spin_lock_irqsave(&bp->lock, flags); 2655 macb_reset_hw(bp); 2656 netif_carrier_off(dev); 2657 spin_unlock_irqrestore(&bp->lock, flags); 2658 2659 macb_free_consistent(bp); 2660 2661 if (bp->ptp_info) 2662 bp->ptp_info->ptp_remove(dev); 2663 2664 pm_runtime_put(&bp->pdev->dev); 2665 2666 return 0; 2667 } 2668 2669 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2670 { 2671 if (netif_running(dev)) 2672 return -EBUSY; 2673 2674 dev->mtu = new_mtu; 2675 2676 return 0; 2677 } 2678 2679 static void gem_update_stats(struct macb *bp) 2680 { 2681 struct macb_queue *queue; 2682 unsigned int i, q, idx; 2683 unsigned long *stat; 2684 2685 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2686 2687 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2688 u32 offset = gem_statistics[i].offset; 2689 u64 val = bp->macb_reg_readl(bp, offset); 2690 2691 bp->ethtool_stats[i] += val; 2692 *p += val; 2693 2694 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2695 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2696 val = bp->macb_reg_readl(bp, offset + 4); 2697 bp->ethtool_stats[i] += ((u64)val) << 32; 2698 *(++p) += val; 2699 } 2700 } 2701 2702 idx = GEM_STATS_LEN; 2703 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2704 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2705 bp->ethtool_stats[idx++] = *stat; 2706 } 2707 2708 static struct net_device_stats *gem_get_stats(struct macb *bp) 2709 { 2710 struct gem_stats *hwstat = &bp->hw_stats.gem; 2711 struct net_device_stats *nstat = &bp->dev->stats; 2712 2713 gem_update_stats(bp); 2714 2715 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2716 hwstat->rx_alignment_errors + 2717 hwstat->rx_resource_errors + 2718 hwstat->rx_overruns + 2719 hwstat->rx_oversize_frames + 2720 hwstat->rx_jabbers + 2721 hwstat->rx_undersized_frames + 2722 hwstat->rx_length_field_frame_errors); 2723 nstat->tx_errors = (hwstat->tx_late_collisions + 2724 hwstat->tx_excessive_collisions + 2725 hwstat->tx_underrun + 2726 hwstat->tx_carrier_sense_errors); 2727 nstat->multicast = hwstat->rx_multicast_frames; 2728 nstat->collisions = (hwstat->tx_single_collision_frames + 2729 hwstat->tx_multiple_collision_frames + 2730 hwstat->tx_excessive_collisions); 2731 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2732 hwstat->rx_jabbers + 2733 hwstat->rx_undersized_frames + 2734 hwstat->rx_length_field_frame_errors); 2735 nstat->rx_over_errors = hwstat->rx_resource_errors; 2736 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2737 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2738 nstat->rx_fifo_errors = hwstat->rx_overruns; 2739 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2740 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2741 nstat->tx_fifo_errors = hwstat->tx_underrun; 2742 2743 return nstat; 2744 } 2745 2746 static void gem_get_ethtool_stats(struct net_device *dev, 2747 struct ethtool_stats *stats, u64 *data) 2748 { 2749 struct macb *bp; 2750 2751 bp = netdev_priv(dev); 2752 gem_update_stats(bp); 2753 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2754 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2755 } 2756 2757 static int gem_get_sset_count(struct net_device *dev, int sset) 2758 { 2759 struct macb *bp = netdev_priv(dev); 2760 2761 switch (sset) { 2762 case ETH_SS_STATS: 2763 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2764 default: 2765 return -EOPNOTSUPP; 2766 } 2767 } 2768 2769 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2770 { 2771 char stat_string[ETH_GSTRING_LEN]; 2772 struct macb *bp = netdev_priv(dev); 2773 struct macb_queue *queue; 2774 unsigned int i; 2775 unsigned int q; 2776 2777 switch (sset) { 2778 case ETH_SS_STATS: 2779 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2780 memcpy(p, gem_statistics[i].stat_string, 2781 ETH_GSTRING_LEN); 2782 2783 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2784 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2785 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2786 q, queue_statistics[i].stat_string); 2787 memcpy(p, stat_string, ETH_GSTRING_LEN); 2788 } 2789 } 2790 break; 2791 } 2792 } 2793 2794 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2795 { 2796 struct macb *bp = netdev_priv(dev); 2797 struct net_device_stats *nstat = &bp->dev->stats; 2798 struct macb_stats *hwstat = &bp->hw_stats.macb; 2799 2800 if (macb_is_gem(bp)) 2801 return gem_get_stats(bp); 2802 2803 /* read stats from hardware */ 2804 macb_update_stats(bp); 2805 2806 /* Convert HW stats into netdevice stats */ 2807 nstat->rx_errors = (hwstat->rx_fcs_errors + 2808 hwstat->rx_align_errors + 2809 hwstat->rx_resource_errors + 2810 hwstat->rx_overruns + 2811 hwstat->rx_oversize_pkts + 2812 hwstat->rx_jabbers + 2813 hwstat->rx_undersize_pkts + 2814 hwstat->rx_length_mismatch); 2815 nstat->tx_errors = (hwstat->tx_late_cols + 2816 hwstat->tx_excessive_cols + 2817 hwstat->tx_underruns + 2818 hwstat->tx_carrier_errors + 2819 hwstat->sqe_test_errors); 2820 nstat->collisions = (hwstat->tx_single_cols + 2821 hwstat->tx_multiple_cols + 2822 hwstat->tx_excessive_cols); 2823 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2824 hwstat->rx_jabbers + 2825 hwstat->rx_undersize_pkts + 2826 hwstat->rx_length_mismatch); 2827 nstat->rx_over_errors = hwstat->rx_resource_errors + 2828 hwstat->rx_overruns; 2829 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2830 nstat->rx_frame_errors = hwstat->rx_align_errors; 2831 nstat->rx_fifo_errors = hwstat->rx_overruns; 2832 /* XXX: What does "missed" mean? */ 2833 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 2834 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 2835 nstat->tx_fifo_errors = hwstat->tx_underruns; 2836 /* Don't know about heartbeat or window errors... */ 2837 2838 return nstat; 2839 } 2840 2841 static int macb_get_regs_len(struct net_device *netdev) 2842 { 2843 return MACB_GREGS_NBR * sizeof(u32); 2844 } 2845 2846 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 2847 void *p) 2848 { 2849 struct macb *bp = netdev_priv(dev); 2850 unsigned int tail, head; 2851 u32 *regs_buff = p; 2852 2853 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 2854 | MACB_GREGS_VERSION; 2855 2856 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 2857 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 2858 2859 regs_buff[0] = macb_readl(bp, NCR); 2860 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 2861 regs_buff[2] = macb_readl(bp, NSR); 2862 regs_buff[3] = macb_readl(bp, TSR); 2863 regs_buff[4] = macb_readl(bp, RBQP); 2864 regs_buff[5] = macb_readl(bp, TBQP); 2865 regs_buff[6] = macb_readl(bp, RSR); 2866 regs_buff[7] = macb_readl(bp, IMR); 2867 2868 regs_buff[8] = tail; 2869 regs_buff[9] = head; 2870 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 2871 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 2872 2873 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 2874 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 2875 if (macb_is_gem(bp)) 2876 regs_buff[13] = gem_readl(bp, DMACFG); 2877 } 2878 2879 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2880 { 2881 struct macb *bp = netdev_priv(netdev); 2882 2883 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 2884 phylink_ethtool_get_wol(bp->phylink, wol); 2885 wol->supported |= WAKE_MAGIC; 2886 2887 if (bp->wol & MACB_WOL_ENABLED) 2888 wol->wolopts |= WAKE_MAGIC; 2889 } 2890 } 2891 2892 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 2893 { 2894 struct macb *bp = netdev_priv(netdev); 2895 int ret; 2896 2897 /* Pass the order to phylink layer */ 2898 ret = phylink_ethtool_set_wol(bp->phylink, wol); 2899 /* Don't manage WoL on MAC if handled by the PHY 2900 * or if there's a failure in talking to the PHY 2901 */ 2902 if (!ret || ret != -EOPNOTSUPP) 2903 return ret; 2904 2905 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 2906 (wol->wolopts & ~WAKE_MAGIC)) 2907 return -EOPNOTSUPP; 2908 2909 if (wol->wolopts & WAKE_MAGIC) 2910 bp->wol |= MACB_WOL_ENABLED; 2911 else 2912 bp->wol &= ~MACB_WOL_ENABLED; 2913 2914 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 2915 2916 return 0; 2917 } 2918 2919 static int macb_get_link_ksettings(struct net_device *netdev, 2920 struct ethtool_link_ksettings *kset) 2921 { 2922 struct macb *bp = netdev_priv(netdev); 2923 2924 return phylink_ethtool_ksettings_get(bp->phylink, kset); 2925 } 2926 2927 static int macb_set_link_ksettings(struct net_device *netdev, 2928 const struct ethtool_link_ksettings *kset) 2929 { 2930 struct macb *bp = netdev_priv(netdev); 2931 2932 return phylink_ethtool_ksettings_set(bp->phylink, kset); 2933 } 2934 2935 static void macb_get_ringparam(struct net_device *netdev, 2936 struct ethtool_ringparam *ring) 2937 { 2938 struct macb *bp = netdev_priv(netdev); 2939 2940 ring->rx_max_pending = MAX_RX_RING_SIZE; 2941 ring->tx_max_pending = MAX_TX_RING_SIZE; 2942 2943 ring->rx_pending = bp->rx_ring_size; 2944 ring->tx_pending = bp->tx_ring_size; 2945 } 2946 2947 static int macb_set_ringparam(struct net_device *netdev, 2948 struct ethtool_ringparam *ring) 2949 { 2950 struct macb *bp = netdev_priv(netdev); 2951 u32 new_rx_size, new_tx_size; 2952 unsigned int reset = 0; 2953 2954 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 2955 return -EINVAL; 2956 2957 new_rx_size = clamp_t(u32, ring->rx_pending, 2958 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 2959 new_rx_size = roundup_pow_of_two(new_rx_size); 2960 2961 new_tx_size = clamp_t(u32, ring->tx_pending, 2962 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 2963 new_tx_size = roundup_pow_of_two(new_tx_size); 2964 2965 if ((new_tx_size == bp->tx_ring_size) && 2966 (new_rx_size == bp->rx_ring_size)) { 2967 /* nothing to do */ 2968 return 0; 2969 } 2970 2971 if (netif_running(bp->dev)) { 2972 reset = 1; 2973 macb_close(bp->dev); 2974 } 2975 2976 bp->rx_ring_size = new_rx_size; 2977 bp->tx_ring_size = new_tx_size; 2978 2979 if (reset) 2980 macb_open(bp->dev); 2981 2982 return 0; 2983 } 2984 2985 #ifdef CONFIG_MACB_USE_HWSTAMP 2986 static unsigned int gem_get_tsu_rate(struct macb *bp) 2987 { 2988 struct clk *tsu_clk; 2989 unsigned int tsu_rate; 2990 2991 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 2992 if (!IS_ERR(tsu_clk)) 2993 tsu_rate = clk_get_rate(tsu_clk); 2994 /* try pclk instead */ 2995 else if (!IS_ERR(bp->pclk)) { 2996 tsu_clk = bp->pclk; 2997 tsu_rate = clk_get_rate(tsu_clk); 2998 } else 2999 return -ENOTSUPP; 3000 return tsu_rate; 3001 } 3002 3003 static s32 gem_get_ptp_max_adj(void) 3004 { 3005 return 64000000; 3006 } 3007 3008 static int gem_get_ts_info(struct net_device *dev, 3009 struct ethtool_ts_info *info) 3010 { 3011 struct macb *bp = netdev_priv(dev); 3012 3013 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 3014 ethtool_op_get_ts_info(dev, info); 3015 return 0; 3016 } 3017 3018 info->so_timestamping = 3019 SOF_TIMESTAMPING_TX_SOFTWARE | 3020 SOF_TIMESTAMPING_RX_SOFTWARE | 3021 SOF_TIMESTAMPING_SOFTWARE | 3022 SOF_TIMESTAMPING_TX_HARDWARE | 3023 SOF_TIMESTAMPING_RX_HARDWARE | 3024 SOF_TIMESTAMPING_RAW_HARDWARE; 3025 info->tx_types = 3026 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 3027 (1 << HWTSTAMP_TX_OFF) | 3028 (1 << HWTSTAMP_TX_ON); 3029 info->rx_filters = 3030 (1 << HWTSTAMP_FILTER_NONE) | 3031 (1 << HWTSTAMP_FILTER_ALL); 3032 3033 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 3034 3035 return 0; 3036 } 3037 3038 static struct macb_ptp_info gem_ptp_info = { 3039 .ptp_init = gem_ptp_init, 3040 .ptp_remove = gem_ptp_remove, 3041 .get_ptp_max_adj = gem_get_ptp_max_adj, 3042 .get_tsu_rate = gem_get_tsu_rate, 3043 .get_ts_info = gem_get_ts_info, 3044 .get_hwtst = gem_get_hwtst, 3045 .set_hwtst = gem_set_hwtst, 3046 }; 3047 #endif 3048 3049 static int macb_get_ts_info(struct net_device *netdev, 3050 struct ethtool_ts_info *info) 3051 { 3052 struct macb *bp = netdev_priv(netdev); 3053 3054 if (bp->ptp_info) 3055 return bp->ptp_info->get_ts_info(netdev, info); 3056 3057 return ethtool_op_get_ts_info(netdev, info); 3058 } 3059 3060 static void gem_enable_flow_filters(struct macb *bp, bool enable) 3061 { 3062 struct net_device *netdev = bp->dev; 3063 struct ethtool_rx_fs_item *item; 3064 u32 t2_scr; 3065 int num_t2_scr; 3066 3067 if (!(netdev->features & NETIF_F_NTUPLE)) 3068 return; 3069 3070 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 3071 3072 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3073 struct ethtool_rx_flow_spec *fs = &item->fs; 3074 struct ethtool_tcpip4_spec *tp4sp_m; 3075 3076 if (fs->location >= num_t2_scr) 3077 continue; 3078 3079 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 3080 3081 /* enable/disable screener regs for the flow entry */ 3082 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 3083 3084 /* only enable fields with no masking */ 3085 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3086 3087 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 3088 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 3089 else 3090 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 3091 3092 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 3093 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 3094 else 3095 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 3096 3097 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 3098 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 3099 else 3100 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 3101 3102 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 3103 } 3104 } 3105 3106 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 3107 { 3108 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 3109 uint16_t index = fs->location; 3110 u32 w0, w1, t2_scr; 3111 bool cmp_a = false; 3112 bool cmp_b = false; 3113 bool cmp_c = false; 3114 3115 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 3116 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3117 3118 /* ignore field if any masking set */ 3119 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 3120 /* 1st compare reg - IP source address */ 3121 w0 = 0; 3122 w1 = 0; 3123 w0 = tp4sp_v->ip4src; 3124 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3125 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3126 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 3127 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 3128 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 3129 cmp_a = true; 3130 } 3131 3132 /* ignore field if any masking set */ 3133 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 3134 /* 2nd compare reg - IP destination address */ 3135 w0 = 0; 3136 w1 = 0; 3137 w0 = tp4sp_v->ip4dst; 3138 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3139 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3140 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 3141 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 3142 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 3143 cmp_b = true; 3144 } 3145 3146 /* ignore both port fields if masking set in both */ 3147 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 3148 /* 3rd compare reg - source port, destination port */ 3149 w0 = 0; 3150 w1 = 0; 3151 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 3152 if (tp4sp_m->psrc == tp4sp_m->pdst) { 3153 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 3154 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3155 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3156 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3157 } else { 3158 /* only one port definition */ 3159 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 3160 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 3161 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 3162 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 3163 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3164 } else { /* dst port */ 3165 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3166 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 3167 } 3168 } 3169 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 3170 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 3171 cmp_c = true; 3172 } 3173 3174 t2_scr = 0; 3175 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 3176 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 3177 if (cmp_a) 3178 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 3179 if (cmp_b) 3180 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 3181 if (cmp_c) 3182 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 3183 gem_writel_n(bp, SCRT2, index, t2_scr); 3184 } 3185 3186 static int gem_add_flow_filter(struct net_device *netdev, 3187 struct ethtool_rxnfc *cmd) 3188 { 3189 struct macb *bp = netdev_priv(netdev); 3190 struct ethtool_rx_flow_spec *fs = &cmd->fs; 3191 struct ethtool_rx_fs_item *item, *newfs; 3192 unsigned long flags; 3193 int ret = -EINVAL; 3194 bool added = false; 3195 3196 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 3197 if (newfs == NULL) 3198 return -ENOMEM; 3199 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 3200 3201 netdev_dbg(netdev, 3202 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3203 fs->flow_type, (int)fs->ring_cookie, fs->location, 3204 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3205 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3206 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 3207 3208 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3209 3210 /* find correct place to add in list */ 3211 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3212 if (item->fs.location > newfs->fs.location) { 3213 list_add_tail(&newfs->list, &item->list); 3214 added = true; 3215 break; 3216 } else if (item->fs.location == fs->location) { 3217 netdev_err(netdev, "Rule not added: location %d not free!\n", 3218 fs->location); 3219 ret = -EBUSY; 3220 goto err; 3221 } 3222 } 3223 if (!added) 3224 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3225 3226 gem_prog_cmp_regs(bp, fs); 3227 bp->rx_fs_list.count++; 3228 /* enable filtering if NTUPLE on */ 3229 gem_enable_flow_filters(bp, 1); 3230 3231 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3232 return 0; 3233 3234 err: 3235 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3236 kfree(newfs); 3237 return ret; 3238 } 3239 3240 static int gem_del_flow_filter(struct net_device *netdev, 3241 struct ethtool_rxnfc *cmd) 3242 { 3243 struct macb *bp = netdev_priv(netdev); 3244 struct ethtool_rx_fs_item *item; 3245 struct ethtool_rx_flow_spec *fs; 3246 unsigned long flags; 3247 3248 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3249 3250 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3251 if (item->fs.location == cmd->fs.location) { 3252 /* disable screener regs for the flow entry */ 3253 fs = &(item->fs); 3254 netdev_dbg(netdev, 3255 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3256 fs->flow_type, (int)fs->ring_cookie, fs->location, 3257 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3258 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3259 htons(fs->h_u.tcp_ip4_spec.psrc), 3260 htons(fs->h_u.tcp_ip4_spec.pdst)); 3261 3262 gem_writel_n(bp, SCRT2, fs->location, 0); 3263 3264 list_del(&item->list); 3265 bp->rx_fs_list.count--; 3266 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3267 kfree(item); 3268 return 0; 3269 } 3270 } 3271 3272 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3273 return -EINVAL; 3274 } 3275 3276 static int gem_get_flow_entry(struct net_device *netdev, 3277 struct ethtool_rxnfc *cmd) 3278 { 3279 struct macb *bp = netdev_priv(netdev); 3280 struct ethtool_rx_fs_item *item; 3281 3282 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3283 if (item->fs.location == cmd->fs.location) { 3284 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3285 return 0; 3286 } 3287 } 3288 return -EINVAL; 3289 } 3290 3291 static int gem_get_all_flow_entries(struct net_device *netdev, 3292 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3293 { 3294 struct macb *bp = netdev_priv(netdev); 3295 struct ethtool_rx_fs_item *item; 3296 uint32_t cnt = 0; 3297 3298 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3299 if (cnt == cmd->rule_cnt) 3300 return -EMSGSIZE; 3301 rule_locs[cnt] = item->fs.location; 3302 cnt++; 3303 } 3304 cmd->data = bp->max_tuples; 3305 cmd->rule_cnt = cnt; 3306 3307 return 0; 3308 } 3309 3310 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3311 u32 *rule_locs) 3312 { 3313 struct macb *bp = netdev_priv(netdev); 3314 int ret = 0; 3315 3316 switch (cmd->cmd) { 3317 case ETHTOOL_GRXRINGS: 3318 cmd->data = bp->num_queues; 3319 break; 3320 case ETHTOOL_GRXCLSRLCNT: 3321 cmd->rule_cnt = bp->rx_fs_list.count; 3322 break; 3323 case ETHTOOL_GRXCLSRULE: 3324 ret = gem_get_flow_entry(netdev, cmd); 3325 break; 3326 case ETHTOOL_GRXCLSRLALL: 3327 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3328 break; 3329 default: 3330 netdev_err(netdev, 3331 "Command parameter %d is not supported\n", cmd->cmd); 3332 ret = -EOPNOTSUPP; 3333 } 3334 3335 return ret; 3336 } 3337 3338 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3339 { 3340 struct macb *bp = netdev_priv(netdev); 3341 int ret; 3342 3343 switch (cmd->cmd) { 3344 case ETHTOOL_SRXCLSRLINS: 3345 if ((cmd->fs.location >= bp->max_tuples) 3346 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3347 ret = -EINVAL; 3348 break; 3349 } 3350 ret = gem_add_flow_filter(netdev, cmd); 3351 break; 3352 case ETHTOOL_SRXCLSRLDEL: 3353 ret = gem_del_flow_filter(netdev, cmd); 3354 break; 3355 default: 3356 netdev_err(netdev, 3357 "Command parameter %d is not supported\n", cmd->cmd); 3358 ret = -EOPNOTSUPP; 3359 } 3360 3361 return ret; 3362 } 3363 3364 static const struct ethtool_ops macb_ethtool_ops = { 3365 .get_regs_len = macb_get_regs_len, 3366 .get_regs = macb_get_regs, 3367 .get_link = ethtool_op_get_link, 3368 .get_ts_info = ethtool_op_get_ts_info, 3369 .get_wol = macb_get_wol, 3370 .set_wol = macb_set_wol, 3371 .get_link_ksettings = macb_get_link_ksettings, 3372 .set_link_ksettings = macb_set_link_ksettings, 3373 .get_ringparam = macb_get_ringparam, 3374 .set_ringparam = macb_set_ringparam, 3375 }; 3376 3377 static const struct ethtool_ops gem_ethtool_ops = { 3378 .get_regs_len = macb_get_regs_len, 3379 .get_regs = macb_get_regs, 3380 .get_wol = macb_get_wol, 3381 .set_wol = macb_set_wol, 3382 .get_link = ethtool_op_get_link, 3383 .get_ts_info = macb_get_ts_info, 3384 .get_ethtool_stats = gem_get_ethtool_stats, 3385 .get_strings = gem_get_ethtool_strings, 3386 .get_sset_count = gem_get_sset_count, 3387 .get_link_ksettings = macb_get_link_ksettings, 3388 .set_link_ksettings = macb_set_link_ksettings, 3389 .get_ringparam = macb_get_ringparam, 3390 .set_ringparam = macb_set_ringparam, 3391 .get_rxnfc = gem_get_rxnfc, 3392 .set_rxnfc = gem_set_rxnfc, 3393 }; 3394 3395 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3396 { 3397 struct macb *bp = netdev_priv(dev); 3398 3399 if (!netif_running(dev)) 3400 return -EINVAL; 3401 3402 if (bp->ptp_info) { 3403 switch (cmd) { 3404 case SIOCSHWTSTAMP: 3405 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3406 case SIOCGHWTSTAMP: 3407 return bp->ptp_info->get_hwtst(dev, rq); 3408 } 3409 } 3410 3411 return phylink_mii_ioctl(bp->phylink, rq, cmd); 3412 } 3413 3414 static inline void macb_set_txcsum_feature(struct macb *bp, 3415 netdev_features_t features) 3416 { 3417 u32 val; 3418 3419 if (!macb_is_gem(bp)) 3420 return; 3421 3422 val = gem_readl(bp, DMACFG); 3423 if (features & NETIF_F_HW_CSUM) 3424 val |= GEM_BIT(TXCOEN); 3425 else 3426 val &= ~GEM_BIT(TXCOEN); 3427 3428 gem_writel(bp, DMACFG, val); 3429 } 3430 3431 static inline void macb_set_rxcsum_feature(struct macb *bp, 3432 netdev_features_t features) 3433 { 3434 struct net_device *netdev = bp->dev; 3435 u32 val; 3436 3437 if (!macb_is_gem(bp)) 3438 return; 3439 3440 val = gem_readl(bp, NCFGR); 3441 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 3442 val |= GEM_BIT(RXCOEN); 3443 else 3444 val &= ~GEM_BIT(RXCOEN); 3445 3446 gem_writel(bp, NCFGR, val); 3447 } 3448 3449 static inline void macb_set_rxflow_feature(struct macb *bp, 3450 netdev_features_t features) 3451 { 3452 if (!macb_is_gem(bp)) 3453 return; 3454 3455 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 3456 } 3457 3458 static int macb_set_features(struct net_device *netdev, 3459 netdev_features_t features) 3460 { 3461 struct macb *bp = netdev_priv(netdev); 3462 netdev_features_t changed = features ^ netdev->features; 3463 3464 /* TX checksum offload */ 3465 if (changed & NETIF_F_HW_CSUM) 3466 macb_set_txcsum_feature(bp, features); 3467 3468 /* RX checksum offload */ 3469 if (changed & NETIF_F_RXCSUM) 3470 macb_set_rxcsum_feature(bp, features); 3471 3472 /* RX Flow Filters */ 3473 if (changed & NETIF_F_NTUPLE) 3474 macb_set_rxflow_feature(bp, features); 3475 3476 return 0; 3477 } 3478 3479 static void macb_restore_features(struct macb *bp) 3480 { 3481 struct net_device *netdev = bp->dev; 3482 netdev_features_t features = netdev->features; 3483 3484 /* TX checksum offload */ 3485 macb_set_txcsum_feature(bp, features); 3486 3487 /* RX checksum offload */ 3488 macb_set_rxcsum_feature(bp, features); 3489 3490 /* RX Flow Filters */ 3491 macb_set_rxflow_feature(bp, features); 3492 } 3493 3494 static const struct net_device_ops macb_netdev_ops = { 3495 .ndo_open = macb_open, 3496 .ndo_stop = macb_close, 3497 .ndo_start_xmit = macb_start_xmit, 3498 .ndo_set_rx_mode = macb_set_rx_mode, 3499 .ndo_get_stats = macb_get_stats, 3500 .ndo_do_ioctl = macb_ioctl, 3501 .ndo_validate_addr = eth_validate_addr, 3502 .ndo_change_mtu = macb_change_mtu, 3503 .ndo_set_mac_address = eth_mac_addr, 3504 #ifdef CONFIG_NET_POLL_CONTROLLER 3505 .ndo_poll_controller = macb_poll_controller, 3506 #endif 3507 .ndo_set_features = macb_set_features, 3508 .ndo_features_check = macb_features_check, 3509 }; 3510 3511 /* Configure peripheral capabilities according to device tree 3512 * and integration options used 3513 */ 3514 static void macb_configure_caps(struct macb *bp, 3515 const struct macb_config *dt_conf) 3516 { 3517 u32 dcfg; 3518 3519 if (dt_conf) 3520 bp->caps = dt_conf->caps; 3521 3522 if (hw_is_gem(bp->regs, bp->native_io)) { 3523 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3524 3525 dcfg = gem_readl(bp, DCFG1); 3526 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3527 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3528 dcfg = gem_readl(bp, DCFG2); 3529 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3530 bp->caps |= MACB_CAPS_FIFO_MODE; 3531 #ifdef CONFIG_MACB_USE_HWSTAMP 3532 if (gem_has_ptp(bp)) { 3533 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3534 dev_err(&bp->pdev->dev, 3535 "GEM doesn't support hardware ptp.\n"); 3536 else { 3537 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3538 bp->ptp_info = &gem_ptp_info; 3539 } 3540 } 3541 #endif 3542 } 3543 3544 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3545 } 3546 3547 static void macb_probe_queues(void __iomem *mem, 3548 bool native_io, 3549 unsigned int *queue_mask, 3550 unsigned int *num_queues) 3551 { 3552 *queue_mask = 0x1; 3553 *num_queues = 1; 3554 3555 /* is it macb or gem ? 3556 * 3557 * We need to read directly from the hardware here because 3558 * we are early in the probe process and don't have the 3559 * MACB_CAPS_MACB_IS_GEM flag positioned 3560 */ 3561 if (!hw_is_gem(mem, native_io)) 3562 return; 3563 3564 /* bit 0 is never set but queue 0 always exists */ 3565 *queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff; 3566 *num_queues = hweight32(*queue_mask); 3567 } 3568 3569 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3570 struct clk **hclk, struct clk **tx_clk, 3571 struct clk **rx_clk, struct clk **tsu_clk) 3572 { 3573 struct macb_platform_data *pdata; 3574 int err; 3575 3576 pdata = dev_get_platdata(&pdev->dev); 3577 if (pdata) { 3578 *pclk = pdata->pclk; 3579 *hclk = pdata->hclk; 3580 } else { 3581 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3582 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3583 } 3584 3585 if (IS_ERR_OR_NULL(*pclk)) { 3586 err = PTR_ERR(*pclk); 3587 if (!err) 3588 err = -ENODEV; 3589 3590 dev_err(&pdev->dev, "failed to get macb_clk (%d)\n", err); 3591 return err; 3592 } 3593 3594 if (IS_ERR_OR_NULL(*hclk)) { 3595 err = PTR_ERR(*hclk); 3596 if (!err) 3597 err = -ENODEV; 3598 3599 dev_err(&pdev->dev, "failed to get hclk (%d)\n", err); 3600 return err; 3601 } 3602 3603 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); 3604 if (IS_ERR(*tx_clk)) 3605 return PTR_ERR(*tx_clk); 3606 3607 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); 3608 if (IS_ERR(*rx_clk)) 3609 return PTR_ERR(*rx_clk); 3610 3611 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); 3612 if (IS_ERR(*tsu_clk)) 3613 return PTR_ERR(*tsu_clk); 3614 3615 err = clk_prepare_enable(*pclk); 3616 if (err) { 3617 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3618 return err; 3619 } 3620 3621 err = clk_prepare_enable(*hclk); 3622 if (err) { 3623 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 3624 goto err_disable_pclk; 3625 } 3626 3627 err = clk_prepare_enable(*tx_clk); 3628 if (err) { 3629 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 3630 goto err_disable_hclk; 3631 } 3632 3633 err = clk_prepare_enable(*rx_clk); 3634 if (err) { 3635 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 3636 goto err_disable_txclk; 3637 } 3638 3639 err = clk_prepare_enable(*tsu_clk); 3640 if (err) { 3641 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 3642 goto err_disable_rxclk; 3643 } 3644 3645 return 0; 3646 3647 err_disable_rxclk: 3648 clk_disable_unprepare(*rx_clk); 3649 3650 err_disable_txclk: 3651 clk_disable_unprepare(*tx_clk); 3652 3653 err_disable_hclk: 3654 clk_disable_unprepare(*hclk); 3655 3656 err_disable_pclk: 3657 clk_disable_unprepare(*pclk); 3658 3659 return err; 3660 } 3661 3662 static int macb_init(struct platform_device *pdev) 3663 { 3664 struct net_device *dev = platform_get_drvdata(pdev); 3665 unsigned int hw_q, q; 3666 struct macb *bp = netdev_priv(dev); 3667 struct macb_queue *queue; 3668 int err; 3669 u32 val, reg; 3670 3671 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3672 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3673 3674 /* set the queue register mapping once for all: queue0 has a special 3675 * register mapping but we don't want to test the queue index then 3676 * compute the corresponding register offset at run time. 3677 */ 3678 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3679 if (!(bp->queue_mask & (1 << hw_q))) 3680 continue; 3681 3682 queue = &bp->queues[q]; 3683 queue->bp = bp; 3684 netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT); 3685 if (hw_q) { 3686 queue->ISR = GEM_ISR(hw_q - 1); 3687 queue->IER = GEM_IER(hw_q - 1); 3688 queue->IDR = GEM_IDR(hw_q - 1); 3689 queue->IMR = GEM_IMR(hw_q - 1); 3690 queue->TBQP = GEM_TBQP(hw_q - 1); 3691 queue->RBQP = GEM_RBQP(hw_q - 1); 3692 queue->RBQS = GEM_RBQS(hw_q - 1); 3693 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3694 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3695 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3696 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3697 } 3698 #endif 3699 } else { 3700 /* queue0 uses legacy registers */ 3701 queue->ISR = MACB_ISR; 3702 queue->IER = MACB_IER; 3703 queue->IDR = MACB_IDR; 3704 queue->IMR = MACB_IMR; 3705 queue->TBQP = MACB_TBQP; 3706 queue->RBQP = MACB_RBQP; 3707 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3708 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3709 queue->TBQPH = MACB_TBQPH; 3710 queue->RBQPH = MACB_RBQPH; 3711 } 3712 #endif 3713 } 3714 3715 /* get irq: here we use the linux queue index, not the hardware 3716 * queue index. the queue irq definitions in the device tree 3717 * must remove the optional gaps that could exist in the 3718 * hardware queue mask. 3719 */ 3720 queue->irq = platform_get_irq(pdev, q); 3721 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3722 IRQF_SHARED, dev->name, queue); 3723 if (err) { 3724 dev_err(&pdev->dev, 3725 "Unable to request IRQ %d (error %d)\n", 3726 queue->irq, err); 3727 return err; 3728 } 3729 3730 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3731 q++; 3732 } 3733 3734 dev->netdev_ops = &macb_netdev_ops; 3735 3736 /* setup appropriated routines according to adapter type */ 3737 if (macb_is_gem(bp)) { 3738 bp->max_tx_length = GEM_MAX_TX_LEN; 3739 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3740 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3741 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3742 bp->macbgem_ops.mog_rx = gem_rx; 3743 dev->ethtool_ops = &gem_ethtool_ops; 3744 } else { 3745 bp->max_tx_length = MACB_MAX_TX_LEN; 3746 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3747 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3748 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3749 bp->macbgem_ops.mog_rx = macb_rx; 3750 dev->ethtool_ops = &macb_ethtool_ops; 3751 } 3752 3753 /* Set features */ 3754 dev->hw_features = NETIF_F_SG; 3755 3756 /* Check LSO capability */ 3757 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3758 dev->hw_features |= MACB_NETIF_LSO; 3759 3760 /* Checksum offload is only available on gem with packet buffer */ 3761 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3762 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3763 if (bp->caps & MACB_CAPS_SG_DISABLED) 3764 dev->hw_features &= ~NETIF_F_SG; 3765 dev->features = dev->hw_features; 3766 3767 /* Check RX Flow Filters support. 3768 * Max Rx flows set by availability of screeners & compare regs: 3769 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3770 */ 3771 reg = gem_readl(bp, DCFG8); 3772 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3773 GEM_BFEXT(T2SCR, reg)); 3774 if (bp->max_tuples > 0) { 3775 /* also needs one ethtype match to check IPv4 */ 3776 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3777 /* program this reg now */ 3778 reg = 0; 3779 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3780 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3781 /* Filtering is supported in hw but don't enable it in kernel now */ 3782 dev->hw_features |= NETIF_F_NTUPLE; 3783 /* init Rx flow definitions */ 3784 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3785 bp->rx_fs_list.count = 0; 3786 spin_lock_init(&bp->rx_fs_lock); 3787 } else 3788 bp->max_tuples = 0; 3789 } 3790 3791 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3792 val = 0; 3793 if (phy_interface_mode_is_rgmii(bp->phy_interface)) 3794 val = GEM_BIT(RGMII); 3795 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3796 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3797 val = MACB_BIT(RMII); 3798 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3799 val = MACB_BIT(MII); 3800 3801 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3802 val |= MACB_BIT(CLKEN); 3803 3804 macb_or_gem_writel(bp, USRIO, val); 3805 } 3806 3807 /* Set MII management clock divider */ 3808 val = macb_mdc_clk_div(bp); 3809 val |= macb_dbw(bp); 3810 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3811 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3812 macb_writel(bp, NCFGR, val); 3813 3814 return 0; 3815 } 3816 3817 #if defined(CONFIG_OF) 3818 /* 1518 rounded up */ 3819 #define AT91ETHER_MAX_RBUFF_SZ 0x600 3820 /* max number of receive buffers */ 3821 #define AT91ETHER_MAX_RX_DESCR 9 3822 3823 static struct sifive_fu540_macb_mgmt *mgmt; 3824 3825 static int at91ether_alloc_coherent(struct macb *lp) 3826 { 3827 struct macb_queue *q = &lp->queues[0]; 3828 3829 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 3830 (AT91ETHER_MAX_RX_DESCR * 3831 macb_dma_desc_get_size(lp)), 3832 &q->rx_ring_dma, GFP_KERNEL); 3833 if (!q->rx_ring) 3834 return -ENOMEM; 3835 3836 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 3837 AT91ETHER_MAX_RX_DESCR * 3838 AT91ETHER_MAX_RBUFF_SZ, 3839 &q->rx_buffers_dma, GFP_KERNEL); 3840 if (!q->rx_buffers) { 3841 dma_free_coherent(&lp->pdev->dev, 3842 AT91ETHER_MAX_RX_DESCR * 3843 macb_dma_desc_get_size(lp), 3844 q->rx_ring, q->rx_ring_dma); 3845 q->rx_ring = NULL; 3846 return -ENOMEM; 3847 } 3848 3849 return 0; 3850 } 3851 3852 static void at91ether_free_coherent(struct macb *lp) 3853 { 3854 struct macb_queue *q = &lp->queues[0]; 3855 3856 if (q->rx_ring) { 3857 dma_free_coherent(&lp->pdev->dev, 3858 AT91ETHER_MAX_RX_DESCR * 3859 macb_dma_desc_get_size(lp), 3860 q->rx_ring, q->rx_ring_dma); 3861 q->rx_ring = NULL; 3862 } 3863 3864 if (q->rx_buffers) { 3865 dma_free_coherent(&lp->pdev->dev, 3866 AT91ETHER_MAX_RX_DESCR * 3867 AT91ETHER_MAX_RBUFF_SZ, 3868 q->rx_buffers, q->rx_buffers_dma); 3869 q->rx_buffers = NULL; 3870 } 3871 } 3872 3873 /* Initialize and start the Receiver and Transmit subsystems */ 3874 static int at91ether_start(struct macb *lp) 3875 { 3876 struct macb_queue *q = &lp->queues[0]; 3877 struct macb_dma_desc *desc; 3878 dma_addr_t addr; 3879 u32 ctl; 3880 int i, ret; 3881 3882 ret = at91ether_alloc_coherent(lp); 3883 if (ret) 3884 return ret; 3885 3886 addr = q->rx_buffers_dma; 3887 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 3888 desc = macb_rx_desc(q, i); 3889 macb_set_addr(lp, desc, addr); 3890 desc->ctrl = 0; 3891 addr += AT91ETHER_MAX_RBUFF_SZ; 3892 } 3893 3894 /* Set the Wrap bit on the last descriptor */ 3895 desc->addr |= MACB_BIT(RX_WRAP); 3896 3897 /* Reset buffer index */ 3898 q->rx_tail = 0; 3899 3900 /* Program address of descriptor list in Rx Buffer Queue register */ 3901 macb_writel(lp, RBQP, q->rx_ring_dma); 3902 3903 /* Enable Receive and Transmit */ 3904 ctl = macb_readl(lp, NCR); 3905 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 3906 3907 /* Enable MAC interrupts */ 3908 macb_writel(lp, IER, MACB_BIT(RCOMP) | 3909 MACB_BIT(RXUBR) | 3910 MACB_BIT(ISR_TUND) | 3911 MACB_BIT(ISR_RLE) | 3912 MACB_BIT(TCOMP) | 3913 MACB_BIT(ISR_ROVR) | 3914 MACB_BIT(HRESP)); 3915 3916 return 0; 3917 } 3918 3919 static void at91ether_stop(struct macb *lp) 3920 { 3921 u32 ctl; 3922 3923 /* Disable MAC interrupts */ 3924 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 3925 MACB_BIT(RXUBR) | 3926 MACB_BIT(ISR_TUND) | 3927 MACB_BIT(ISR_RLE) | 3928 MACB_BIT(TCOMP) | 3929 MACB_BIT(ISR_ROVR) | 3930 MACB_BIT(HRESP)); 3931 3932 /* Disable Receiver and Transmitter */ 3933 ctl = macb_readl(lp, NCR); 3934 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 3935 3936 /* Free resources. */ 3937 at91ether_free_coherent(lp); 3938 } 3939 3940 /* Open the ethernet interface */ 3941 static int at91ether_open(struct net_device *dev) 3942 { 3943 struct macb *lp = netdev_priv(dev); 3944 u32 ctl; 3945 int ret; 3946 3947 ret = pm_runtime_get_sync(&lp->pdev->dev); 3948 if (ret < 0) { 3949 pm_runtime_put_noidle(&lp->pdev->dev); 3950 return ret; 3951 } 3952 3953 /* Clear internal statistics */ 3954 ctl = macb_readl(lp, NCR); 3955 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 3956 3957 macb_set_hwaddr(lp); 3958 3959 ret = at91ether_start(lp); 3960 if (ret) 3961 goto pm_exit; 3962 3963 ret = macb_phylink_connect(lp); 3964 if (ret) 3965 goto stop; 3966 3967 netif_start_queue(dev); 3968 3969 return 0; 3970 3971 stop: 3972 at91ether_stop(lp); 3973 pm_exit: 3974 pm_runtime_put_sync(&lp->pdev->dev); 3975 return ret; 3976 } 3977 3978 /* Close the interface */ 3979 static int at91ether_close(struct net_device *dev) 3980 { 3981 struct macb *lp = netdev_priv(dev); 3982 3983 netif_stop_queue(dev); 3984 3985 phylink_stop(lp->phylink); 3986 phylink_disconnect_phy(lp->phylink); 3987 3988 at91ether_stop(lp); 3989 3990 return pm_runtime_put(&lp->pdev->dev); 3991 } 3992 3993 /* Transmit packet */ 3994 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 3995 struct net_device *dev) 3996 { 3997 struct macb *lp = netdev_priv(dev); 3998 3999 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 4000 netif_stop_queue(dev); 4001 4002 /* Store packet information (to free when Tx completed) */ 4003 lp->skb = skb; 4004 lp->skb_length = skb->len; 4005 lp->skb_physaddr = dma_map_single(&lp->pdev->dev, skb->data, 4006 skb->len, DMA_TO_DEVICE); 4007 if (dma_mapping_error(&lp->pdev->dev, lp->skb_physaddr)) { 4008 dev_kfree_skb_any(skb); 4009 dev->stats.tx_dropped++; 4010 netdev_err(dev, "%s: DMA mapping error\n", __func__); 4011 return NETDEV_TX_OK; 4012 } 4013 4014 /* Set address of the data in the Transmit Address register */ 4015 macb_writel(lp, TAR, lp->skb_physaddr); 4016 /* Set length of the packet in the Transmit Control register */ 4017 macb_writel(lp, TCR, skb->len); 4018 4019 } else { 4020 netdev_err(dev, "%s called, but device is busy!\n", __func__); 4021 return NETDEV_TX_BUSY; 4022 } 4023 4024 return NETDEV_TX_OK; 4025 } 4026 4027 /* Extract received frame from buffer descriptors and sent to upper layers. 4028 * (Called from interrupt context) 4029 */ 4030 static void at91ether_rx(struct net_device *dev) 4031 { 4032 struct macb *lp = netdev_priv(dev); 4033 struct macb_queue *q = &lp->queues[0]; 4034 struct macb_dma_desc *desc; 4035 unsigned char *p_recv; 4036 struct sk_buff *skb; 4037 unsigned int pktlen; 4038 4039 desc = macb_rx_desc(q, q->rx_tail); 4040 while (desc->addr & MACB_BIT(RX_USED)) { 4041 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 4042 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 4043 skb = netdev_alloc_skb(dev, pktlen + 2); 4044 if (skb) { 4045 skb_reserve(skb, 2); 4046 skb_put_data(skb, p_recv, pktlen); 4047 4048 skb->protocol = eth_type_trans(skb, dev); 4049 dev->stats.rx_packets++; 4050 dev->stats.rx_bytes += pktlen; 4051 netif_rx(skb); 4052 } else { 4053 dev->stats.rx_dropped++; 4054 } 4055 4056 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 4057 dev->stats.multicast++; 4058 4059 /* reset ownership bit */ 4060 desc->addr &= ~MACB_BIT(RX_USED); 4061 4062 /* wrap after last buffer */ 4063 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 4064 q->rx_tail = 0; 4065 else 4066 q->rx_tail++; 4067 4068 desc = macb_rx_desc(q, q->rx_tail); 4069 } 4070 } 4071 4072 /* MAC interrupt handler */ 4073 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 4074 { 4075 struct net_device *dev = dev_id; 4076 struct macb *lp = netdev_priv(dev); 4077 u32 intstatus, ctl; 4078 4079 /* MAC Interrupt Status register indicates what interrupts are pending. 4080 * It is automatically cleared once read. 4081 */ 4082 intstatus = macb_readl(lp, ISR); 4083 4084 /* Receive complete */ 4085 if (intstatus & MACB_BIT(RCOMP)) 4086 at91ether_rx(dev); 4087 4088 /* Transmit complete */ 4089 if (intstatus & MACB_BIT(TCOMP)) { 4090 /* The TCOM bit is set even if the transmission failed */ 4091 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 4092 dev->stats.tx_errors++; 4093 4094 if (lp->skb) { 4095 dev_consume_skb_irq(lp->skb); 4096 lp->skb = NULL; 4097 dma_unmap_single(&lp->pdev->dev, lp->skb_physaddr, 4098 lp->skb_length, DMA_TO_DEVICE); 4099 dev->stats.tx_packets++; 4100 dev->stats.tx_bytes += lp->skb_length; 4101 } 4102 netif_wake_queue(dev); 4103 } 4104 4105 /* Work-around for EMAC Errata section 41.3.1 */ 4106 if (intstatus & MACB_BIT(RXUBR)) { 4107 ctl = macb_readl(lp, NCR); 4108 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 4109 wmb(); 4110 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 4111 } 4112 4113 if (intstatus & MACB_BIT(ISR_ROVR)) 4114 netdev_err(dev, "ROVR error\n"); 4115 4116 return IRQ_HANDLED; 4117 } 4118 4119 #ifdef CONFIG_NET_POLL_CONTROLLER 4120 static void at91ether_poll_controller(struct net_device *dev) 4121 { 4122 unsigned long flags; 4123 4124 local_irq_save(flags); 4125 at91ether_interrupt(dev->irq, dev); 4126 local_irq_restore(flags); 4127 } 4128 #endif 4129 4130 static const struct net_device_ops at91ether_netdev_ops = { 4131 .ndo_open = at91ether_open, 4132 .ndo_stop = at91ether_close, 4133 .ndo_start_xmit = at91ether_start_xmit, 4134 .ndo_get_stats = macb_get_stats, 4135 .ndo_set_rx_mode = macb_set_rx_mode, 4136 .ndo_set_mac_address = eth_mac_addr, 4137 .ndo_do_ioctl = macb_ioctl, 4138 .ndo_validate_addr = eth_validate_addr, 4139 #ifdef CONFIG_NET_POLL_CONTROLLER 4140 .ndo_poll_controller = at91ether_poll_controller, 4141 #endif 4142 }; 4143 4144 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 4145 struct clk **hclk, struct clk **tx_clk, 4146 struct clk **rx_clk, struct clk **tsu_clk) 4147 { 4148 int err; 4149 4150 *hclk = NULL; 4151 *tx_clk = NULL; 4152 *rx_clk = NULL; 4153 *tsu_clk = NULL; 4154 4155 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 4156 if (IS_ERR(*pclk)) 4157 return PTR_ERR(*pclk); 4158 4159 err = clk_prepare_enable(*pclk); 4160 if (err) { 4161 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 4162 return err; 4163 } 4164 4165 return 0; 4166 } 4167 4168 static int at91ether_init(struct platform_device *pdev) 4169 { 4170 struct net_device *dev = platform_get_drvdata(pdev); 4171 struct macb *bp = netdev_priv(dev); 4172 int err; 4173 4174 bp->queues[0].bp = bp; 4175 4176 dev->netdev_ops = &at91ether_netdev_ops; 4177 dev->ethtool_ops = &macb_ethtool_ops; 4178 4179 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 4180 0, dev->name, dev); 4181 if (err) 4182 return err; 4183 4184 macb_writel(bp, NCR, 0); 4185 4186 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); 4187 4188 return 0; 4189 } 4190 4191 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 4192 unsigned long parent_rate) 4193 { 4194 return mgmt->rate; 4195 } 4196 4197 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, 4198 unsigned long *parent_rate) 4199 { 4200 if (WARN_ON(rate < 2500000)) 4201 return 2500000; 4202 else if (rate == 2500000) 4203 return 2500000; 4204 else if (WARN_ON(rate < 13750000)) 4205 return 2500000; 4206 else if (WARN_ON(rate < 25000000)) 4207 return 25000000; 4208 else if (rate == 25000000) 4209 return 25000000; 4210 else if (WARN_ON(rate < 75000000)) 4211 return 25000000; 4212 else if (WARN_ON(rate < 125000000)) 4213 return 125000000; 4214 else if (rate == 125000000) 4215 return 125000000; 4216 4217 WARN_ON(rate > 125000000); 4218 4219 return 125000000; 4220 } 4221 4222 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 4223 unsigned long parent_rate) 4224 { 4225 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); 4226 if (rate != 125000000) 4227 iowrite32(1, mgmt->reg); 4228 else 4229 iowrite32(0, mgmt->reg); 4230 mgmt->rate = rate; 4231 4232 return 0; 4233 } 4234 4235 static const struct clk_ops fu540_c000_ops = { 4236 .recalc_rate = fu540_macb_tx_recalc_rate, 4237 .round_rate = fu540_macb_tx_round_rate, 4238 .set_rate = fu540_macb_tx_set_rate, 4239 }; 4240 4241 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 4242 struct clk **hclk, struct clk **tx_clk, 4243 struct clk **rx_clk, struct clk **tsu_clk) 4244 { 4245 struct clk_init_data init; 4246 int err = 0; 4247 4248 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 4249 if (err) 4250 return err; 4251 4252 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 4253 if (!mgmt) 4254 return -ENOMEM; 4255 4256 init.name = "sifive-gemgxl-mgmt"; 4257 init.ops = &fu540_c000_ops; 4258 init.flags = 0; 4259 init.num_parents = 0; 4260 4261 mgmt->rate = 0; 4262 mgmt->hw.init = &init; 4263 4264 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); 4265 if (IS_ERR(*tx_clk)) 4266 return PTR_ERR(*tx_clk); 4267 4268 err = clk_prepare_enable(*tx_clk); 4269 if (err) 4270 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 4271 else 4272 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 4273 4274 return 0; 4275 } 4276 4277 static int fu540_c000_init(struct platform_device *pdev) 4278 { 4279 mgmt->reg = devm_platform_ioremap_resource(pdev, 1); 4280 if (IS_ERR(mgmt->reg)) 4281 return PTR_ERR(mgmt->reg); 4282 4283 return macb_init(pdev); 4284 } 4285 4286 static const struct macb_config fu540_c000_config = { 4287 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 4288 MACB_CAPS_GEM_HAS_PTP, 4289 .dma_burst_length = 16, 4290 .clk_init = fu540_c000_clk_init, 4291 .init = fu540_c000_init, 4292 .jumbo_max_len = 10240, 4293 }; 4294 4295 static const struct macb_config at91sam9260_config = { 4296 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4297 .clk_init = macb_clk_init, 4298 .init = macb_init, 4299 }; 4300 4301 static const struct macb_config sama5d3macb_config = { 4302 .caps = MACB_CAPS_SG_DISABLED 4303 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4304 .clk_init = macb_clk_init, 4305 .init = macb_init, 4306 }; 4307 4308 static const struct macb_config pc302gem_config = { 4309 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 4310 .dma_burst_length = 16, 4311 .clk_init = macb_clk_init, 4312 .init = macb_init, 4313 }; 4314 4315 static const struct macb_config sama5d2_config = { 4316 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4317 .dma_burst_length = 16, 4318 .clk_init = macb_clk_init, 4319 .init = macb_init, 4320 }; 4321 4322 static const struct macb_config sama5d3_config = { 4323 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 4324 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 4325 .dma_burst_length = 16, 4326 .clk_init = macb_clk_init, 4327 .init = macb_init, 4328 .jumbo_max_len = 10240, 4329 }; 4330 4331 static const struct macb_config sama5d4_config = { 4332 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4333 .dma_burst_length = 4, 4334 .clk_init = macb_clk_init, 4335 .init = macb_init, 4336 }; 4337 4338 static const struct macb_config emac_config = { 4339 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC, 4340 .clk_init = at91ether_clk_init, 4341 .init = at91ether_init, 4342 }; 4343 4344 static const struct macb_config np4_config = { 4345 .caps = MACB_CAPS_USRIO_DISABLED, 4346 .clk_init = macb_clk_init, 4347 .init = macb_init, 4348 }; 4349 4350 static const struct macb_config zynqmp_config = { 4351 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4352 MACB_CAPS_JUMBO | 4353 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 4354 .dma_burst_length = 16, 4355 .clk_init = macb_clk_init, 4356 .init = macb_init, 4357 .jumbo_max_len = 10240, 4358 }; 4359 4360 static const struct macb_config zynq_config = { 4361 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 4362 MACB_CAPS_NEEDS_RSTONUBR, 4363 .dma_burst_length = 16, 4364 .clk_init = macb_clk_init, 4365 .init = macb_init, 4366 }; 4367 4368 static const struct of_device_id macb_dt_ids[] = { 4369 { .compatible = "cdns,at32ap7000-macb" }, 4370 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 4371 { .compatible = "cdns,macb" }, 4372 { .compatible = "cdns,np4-macb", .data = &np4_config }, 4373 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 4374 { .compatible = "cdns,gem", .data = &pc302gem_config }, 4375 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 4376 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 4377 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 4378 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 4379 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 4380 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 4381 { .compatible = "cdns,emac", .data = &emac_config }, 4382 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 4383 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 4384 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 4385 { /* sentinel */ } 4386 }; 4387 MODULE_DEVICE_TABLE(of, macb_dt_ids); 4388 #endif /* CONFIG_OF */ 4389 4390 static const struct macb_config default_gem_config = { 4391 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4392 MACB_CAPS_JUMBO | 4393 MACB_CAPS_GEM_HAS_PTP, 4394 .dma_burst_length = 16, 4395 .clk_init = macb_clk_init, 4396 .init = macb_init, 4397 .jumbo_max_len = 10240, 4398 }; 4399 4400 static int macb_probe(struct platform_device *pdev) 4401 { 4402 const struct macb_config *macb_config = &default_gem_config; 4403 int (*clk_init)(struct platform_device *, struct clk **, 4404 struct clk **, struct clk **, struct clk **, 4405 struct clk **) = macb_config->clk_init; 4406 int (*init)(struct platform_device *) = macb_config->init; 4407 struct device_node *np = pdev->dev.of_node; 4408 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4409 struct clk *tsu_clk = NULL; 4410 unsigned int queue_mask, num_queues; 4411 bool native_io; 4412 phy_interface_t interface; 4413 struct net_device *dev; 4414 struct resource *regs; 4415 void __iomem *mem; 4416 const char *mac; 4417 struct macb *bp; 4418 int err, val; 4419 4420 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4421 mem = devm_ioremap_resource(&pdev->dev, regs); 4422 if (IS_ERR(mem)) 4423 return PTR_ERR(mem); 4424 4425 if (np) { 4426 const struct of_device_id *match; 4427 4428 match = of_match_node(macb_dt_ids, np); 4429 if (match && match->data) { 4430 macb_config = match->data; 4431 clk_init = macb_config->clk_init; 4432 init = macb_config->init; 4433 } 4434 } 4435 4436 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4437 if (err) 4438 return err; 4439 4440 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4441 pm_runtime_use_autosuspend(&pdev->dev); 4442 pm_runtime_get_noresume(&pdev->dev); 4443 pm_runtime_set_active(&pdev->dev); 4444 pm_runtime_enable(&pdev->dev); 4445 native_io = hw_is_native_io(mem); 4446 4447 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4448 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4449 if (!dev) { 4450 err = -ENOMEM; 4451 goto err_disable_clocks; 4452 } 4453 4454 dev->base_addr = regs->start; 4455 4456 SET_NETDEV_DEV(dev, &pdev->dev); 4457 4458 bp = netdev_priv(dev); 4459 bp->pdev = pdev; 4460 bp->dev = dev; 4461 bp->regs = mem; 4462 bp->native_io = native_io; 4463 if (native_io) { 4464 bp->macb_reg_readl = hw_readl_native; 4465 bp->macb_reg_writel = hw_writel_native; 4466 } else { 4467 bp->macb_reg_readl = hw_readl; 4468 bp->macb_reg_writel = hw_writel; 4469 } 4470 bp->num_queues = num_queues; 4471 bp->queue_mask = queue_mask; 4472 if (macb_config) 4473 bp->dma_burst_length = macb_config->dma_burst_length; 4474 bp->pclk = pclk; 4475 bp->hclk = hclk; 4476 bp->tx_clk = tx_clk; 4477 bp->rx_clk = rx_clk; 4478 bp->tsu_clk = tsu_clk; 4479 if (macb_config) 4480 bp->jumbo_max_len = macb_config->jumbo_max_len; 4481 4482 bp->wol = 0; 4483 if (of_get_property(np, "magic-packet", NULL)) 4484 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4485 device_set_wakeup_capable(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4486 4487 spin_lock_init(&bp->lock); 4488 4489 /* setup capabilities */ 4490 macb_configure_caps(bp, macb_config); 4491 4492 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4493 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4494 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4495 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4496 } 4497 #endif 4498 platform_set_drvdata(pdev, dev); 4499 4500 dev->irq = platform_get_irq(pdev, 0); 4501 if (dev->irq < 0) { 4502 err = dev->irq; 4503 goto err_out_free_netdev; 4504 } 4505 4506 /* MTU range: 68 - 1500 or 10240 */ 4507 dev->min_mtu = GEM_MTU_MIN_SIZE; 4508 if (bp->caps & MACB_CAPS_JUMBO) 4509 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4510 else 4511 dev->max_mtu = ETH_DATA_LEN; 4512 4513 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4514 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4515 if (val) 4516 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4517 macb_dma_desc_get_size(bp); 4518 4519 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4520 if (val) 4521 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4522 macb_dma_desc_get_size(bp); 4523 } 4524 4525 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4526 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4527 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4528 4529 mac = of_get_mac_address(np); 4530 if (PTR_ERR(mac) == -EPROBE_DEFER) { 4531 err = -EPROBE_DEFER; 4532 goto err_out_free_netdev; 4533 } else if (!IS_ERR_OR_NULL(mac)) { 4534 ether_addr_copy(bp->dev->dev_addr, mac); 4535 } else { 4536 macb_get_hwaddr(bp); 4537 } 4538 4539 err = of_get_phy_mode(np, &interface); 4540 if (err) 4541 /* not found in DT, MII by default */ 4542 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4543 else 4544 bp->phy_interface = interface; 4545 4546 /* IP specific init */ 4547 err = init(pdev); 4548 if (err) 4549 goto err_out_free_netdev; 4550 4551 err = macb_mii_init(bp); 4552 if (err) 4553 goto err_out_free_netdev; 4554 4555 netif_carrier_off(dev); 4556 4557 err = register_netdev(dev); 4558 if (err) { 4559 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4560 goto err_out_unregister_mdio; 4561 } 4562 4563 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task, 4564 (unsigned long)bp); 4565 4566 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4567 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4568 dev->base_addr, dev->irq, dev->dev_addr); 4569 4570 pm_runtime_mark_last_busy(&bp->pdev->dev); 4571 pm_runtime_put_autosuspend(&bp->pdev->dev); 4572 4573 return 0; 4574 4575 err_out_unregister_mdio: 4576 mdiobus_unregister(bp->mii_bus); 4577 mdiobus_free(bp->mii_bus); 4578 4579 err_out_free_netdev: 4580 free_netdev(dev); 4581 4582 err_disable_clocks: 4583 clk_disable_unprepare(tx_clk); 4584 clk_disable_unprepare(hclk); 4585 clk_disable_unprepare(pclk); 4586 clk_disable_unprepare(rx_clk); 4587 clk_disable_unprepare(tsu_clk); 4588 pm_runtime_disable(&pdev->dev); 4589 pm_runtime_set_suspended(&pdev->dev); 4590 pm_runtime_dont_use_autosuspend(&pdev->dev); 4591 4592 return err; 4593 } 4594 4595 static int macb_remove(struct platform_device *pdev) 4596 { 4597 struct net_device *dev; 4598 struct macb *bp; 4599 4600 dev = platform_get_drvdata(pdev); 4601 4602 if (dev) { 4603 bp = netdev_priv(dev); 4604 mdiobus_unregister(bp->mii_bus); 4605 mdiobus_free(bp->mii_bus); 4606 4607 unregister_netdev(dev); 4608 tasklet_kill(&bp->hresp_err_tasklet); 4609 pm_runtime_disable(&pdev->dev); 4610 pm_runtime_dont_use_autosuspend(&pdev->dev); 4611 if (!pm_runtime_suspended(&pdev->dev)) { 4612 clk_disable_unprepare(bp->tx_clk); 4613 clk_disable_unprepare(bp->hclk); 4614 clk_disable_unprepare(bp->pclk); 4615 clk_disable_unprepare(bp->rx_clk); 4616 clk_disable_unprepare(bp->tsu_clk); 4617 pm_runtime_set_suspended(&pdev->dev); 4618 } 4619 phylink_destroy(bp->phylink); 4620 free_netdev(dev); 4621 } 4622 4623 return 0; 4624 } 4625 4626 static int __maybe_unused macb_suspend(struct device *dev) 4627 { 4628 struct net_device *netdev = dev_get_drvdata(dev); 4629 struct macb *bp = netdev_priv(netdev); 4630 struct macb_queue *queue = bp->queues; 4631 unsigned long flags; 4632 unsigned int q; 4633 int err; 4634 4635 if (!netif_running(netdev)) 4636 return 0; 4637 4638 if (bp->wol & MACB_WOL_ENABLED) { 4639 spin_lock_irqsave(&bp->lock, flags); 4640 /* Flush all status bits */ 4641 macb_writel(bp, TSR, -1); 4642 macb_writel(bp, RSR, -1); 4643 for (q = 0, queue = bp->queues; q < bp->num_queues; 4644 ++q, ++queue) { 4645 /* Disable all interrupts */ 4646 queue_writel(queue, IDR, -1); 4647 queue_readl(queue, ISR); 4648 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 4649 queue_writel(queue, ISR, -1); 4650 } 4651 /* Change interrupt handler and 4652 * Enable WoL IRQ on queue 0 4653 */ 4654 devm_free_irq(dev, bp->queues[0].irq, bp->queues); 4655 if (macb_is_gem(bp)) { 4656 err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, 4657 IRQF_SHARED, netdev->name, bp->queues); 4658 if (err) { 4659 dev_err(dev, 4660 "Unable to request IRQ %d (error %d)\n", 4661 bp->queues[0].irq, err); 4662 spin_unlock_irqrestore(&bp->lock, flags); 4663 return err; 4664 } 4665 queue_writel(bp->queues, IER, GEM_BIT(WOL)); 4666 gem_writel(bp, WOL, MACB_BIT(MAG)); 4667 } else { 4668 err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, 4669 IRQF_SHARED, netdev->name, bp->queues); 4670 if (err) { 4671 dev_err(dev, 4672 "Unable to request IRQ %d (error %d)\n", 4673 bp->queues[0].irq, err); 4674 spin_unlock_irqrestore(&bp->lock, flags); 4675 return err; 4676 } 4677 queue_writel(bp->queues, IER, MACB_BIT(WOL)); 4678 macb_writel(bp, WOL, MACB_BIT(MAG)); 4679 } 4680 spin_unlock_irqrestore(&bp->lock, flags); 4681 4682 enable_irq_wake(bp->queues[0].irq); 4683 } 4684 4685 netif_device_detach(netdev); 4686 for (q = 0, queue = bp->queues; q < bp->num_queues; 4687 ++q, ++queue) 4688 napi_disable(&queue->napi); 4689 4690 if (!(bp->wol & MACB_WOL_ENABLED)) { 4691 rtnl_lock(); 4692 phylink_stop(bp->phylink); 4693 rtnl_unlock(); 4694 spin_lock_irqsave(&bp->lock, flags); 4695 macb_reset_hw(bp); 4696 spin_unlock_irqrestore(&bp->lock, flags); 4697 } 4698 4699 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4700 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 4701 4702 if (netdev->hw_features & NETIF_F_NTUPLE) 4703 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 4704 4705 if (bp->ptp_info) 4706 bp->ptp_info->ptp_remove(netdev); 4707 if (!device_may_wakeup(dev)) 4708 pm_runtime_force_suspend(dev); 4709 4710 return 0; 4711 } 4712 4713 static int __maybe_unused macb_resume(struct device *dev) 4714 { 4715 struct net_device *netdev = dev_get_drvdata(dev); 4716 struct macb *bp = netdev_priv(netdev); 4717 struct macb_queue *queue = bp->queues; 4718 unsigned long flags; 4719 unsigned int q; 4720 int err; 4721 4722 if (!netif_running(netdev)) 4723 return 0; 4724 4725 if (!device_may_wakeup(dev)) 4726 pm_runtime_force_resume(dev); 4727 4728 if (bp->wol & MACB_WOL_ENABLED) { 4729 spin_lock_irqsave(&bp->lock, flags); 4730 /* Disable WoL */ 4731 if (macb_is_gem(bp)) { 4732 queue_writel(bp->queues, IDR, GEM_BIT(WOL)); 4733 gem_writel(bp, WOL, 0); 4734 } else { 4735 queue_writel(bp->queues, IDR, MACB_BIT(WOL)); 4736 macb_writel(bp, WOL, 0); 4737 } 4738 /* Clear ISR on queue 0 */ 4739 queue_readl(bp->queues, ISR); 4740 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 4741 queue_writel(bp->queues, ISR, -1); 4742 /* Replace interrupt handler on queue 0 */ 4743 devm_free_irq(dev, bp->queues[0].irq, bp->queues); 4744 err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, 4745 IRQF_SHARED, netdev->name, bp->queues); 4746 if (err) { 4747 dev_err(dev, 4748 "Unable to request IRQ %d (error %d)\n", 4749 bp->queues[0].irq, err); 4750 spin_unlock_irqrestore(&bp->lock, flags); 4751 return err; 4752 } 4753 spin_unlock_irqrestore(&bp->lock, flags); 4754 4755 disable_irq_wake(bp->queues[0].irq); 4756 4757 /* Now make sure we disable phy before moving 4758 * to common restore path 4759 */ 4760 rtnl_lock(); 4761 phylink_stop(bp->phylink); 4762 rtnl_unlock(); 4763 } 4764 4765 for (q = 0, queue = bp->queues; q < bp->num_queues; 4766 ++q, ++queue) 4767 napi_enable(&queue->napi); 4768 4769 if (netdev->hw_features & NETIF_F_NTUPLE) 4770 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 4771 4772 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4773 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 4774 4775 macb_writel(bp, NCR, MACB_BIT(MPE)); 4776 macb_init_hw(bp); 4777 macb_set_rx_mode(netdev); 4778 macb_restore_features(bp); 4779 rtnl_lock(); 4780 phylink_start(bp->phylink); 4781 rtnl_unlock(); 4782 4783 netif_device_attach(netdev); 4784 if (bp->ptp_info) 4785 bp->ptp_info->ptp_init(netdev); 4786 4787 return 0; 4788 } 4789 4790 static int __maybe_unused macb_runtime_suspend(struct device *dev) 4791 { 4792 struct net_device *netdev = dev_get_drvdata(dev); 4793 struct macb *bp = netdev_priv(netdev); 4794 4795 if (!(device_may_wakeup(dev))) { 4796 clk_disable_unprepare(bp->tx_clk); 4797 clk_disable_unprepare(bp->hclk); 4798 clk_disable_unprepare(bp->pclk); 4799 clk_disable_unprepare(bp->rx_clk); 4800 } 4801 clk_disable_unprepare(bp->tsu_clk); 4802 4803 return 0; 4804 } 4805 4806 static int __maybe_unused macb_runtime_resume(struct device *dev) 4807 { 4808 struct net_device *netdev = dev_get_drvdata(dev); 4809 struct macb *bp = netdev_priv(netdev); 4810 4811 if (!(device_may_wakeup(dev))) { 4812 clk_prepare_enable(bp->pclk); 4813 clk_prepare_enable(bp->hclk); 4814 clk_prepare_enable(bp->tx_clk); 4815 clk_prepare_enable(bp->rx_clk); 4816 } 4817 clk_prepare_enable(bp->tsu_clk); 4818 4819 return 0; 4820 } 4821 4822 static const struct dev_pm_ops macb_pm_ops = { 4823 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 4824 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 4825 }; 4826 4827 static struct platform_driver macb_driver = { 4828 .probe = macb_probe, 4829 .remove = macb_remove, 4830 .driver = { 4831 .name = "macb", 4832 .of_match_table = of_match_ptr(macb_dt_ids), 4833 .pm = &macb_pm_ops, 4834 }, 4835 }; 4836 4837 module_platform_driver(macb_driver); 4838 4839 MODULE_LICENSE("GPL"); 4840 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 4841 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 4842 MODULE_ALIAS("platform:macb"); 4843