1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #include <linux/clk.h> 10 #include <linux/clk-provider.h> 11 #include <linux/crc32.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/kernel.h> 15 #include <linux/types.h> 16 #include <linux/circ_buf.h> 17 #include <linux/slab.h> 18 #include <linux/init.h> 19 #include <linux/io.h> 20 #include <linux/gpio.h> 21 #include <linux/gpio/consumer.h> 22 #include <linux/interrupt.h> 23 #include <linux/netdevice.h> 24 #include <linux/etherdevice.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/platform_device.h> 27 #include <linux/phylink.h> 28 #include <linux/of.h> 29 #include <linux/of_device.h> 30 #include <linux/of_gpio.h> 31 #include <linux/of_mdio.h> 32 #include <linux/of_net.h> 33 #include <linux/ip.h> 34 #include <linux/udp.h> 35 #include <linux/tcp.h> 36 #include <linux/iopoll.h> 37 #include <linux/pm_runtime.h> 38 #include "macb.h" 39 40 /* This structure is only used for MACB on SiFive FU540 devices */ 41 struct sifive_fu540_macb_mgmt { 42 void __iomem *reg; 43 unsigned long rate; 44 struct clk_hw hw; 45 }; 46 47 #define MACB_RX_BUFFER_SIZE 128 48 #define RX_BUFFER_MULTIPLE 64 /* bytes */ 49 50 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 51 #define MIN_RX_RING_SIZE 64 52 #define MAX_RX_RING_SIZE 8192 53 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 54 * (bp)->rx_ring_size) 55 56 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 57 #define MIN_TX_RING_SIZE 64 58 #define MAX_TX_RING_SIZE 4096 59 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ 60 * (bp)->tx_ring_size) 61 62 /* level of occupied TX descriptors under which we wake up TX process */ 63 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 64 65 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 66 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 67 | MACB_BIT(ISR_RLE) \ 68 | MACB_BIT(TXERR)) 69 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 70 | MACB_BIT(TXUBR)) 71 72 /* Max length of transmit frame must be a multiple of 8 bytes */ 73 #define MACB_TX_LEN_ALIGN 8 74 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 75 /* Limit maximum TX length as per Cadence TSO errata. This is to avoid a 76 * false amba_error in TX path from the DMA assuming there is not enough 77 * space in the SRAM (16KB) even when there is. 78 */ 79 #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) 80 81 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU 82 #define MACB_NETIF_LSO NETIF_F_TSO 83 84 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) 85 #define MACB_WOL_ENABLED (0x1 << 1) 86 87 #define HS_SPEED_10000M 4 88 #define MACB_SERDES_RATE_10G 1 89 90 /* Graceful stop timeouts in us. We should allow up to 91 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 92 */ 93 #define MACB_HALT_TIMEOUT 1230 94 95 #define MACB_PM_TIMEOUT 100 /* ms */ 96 97 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 98 99 /* DMA buffer descriptor might be different size 100 * depends on hardware configuration: 101 * 102 * 1. dma address width 32 bits: 103 * word 1: 32 bit address of Data Buffer 104 * word 2: control 105 * 106 * 2. dma address width 64 bits: 107 * word 1: 32 bit address of Data Buffer 108 * word 2: control 109 * word 3: upper 32 bit address of Data Buffer 110 * word 4: unused 111 * 112 * 3. dma address width 32 bits with hardware timestamping: 113 * word 1: 32 bit address of Data Buffer 114 * word 2: control 115 * word 3: timestamp word 1 116 * word 4: timestamp word 2 117 * 118 * 4. dma address width 64 bits with hardware timestamping: 119 * word 1: 32 bit address of Data Buffer 120 * word 2: control 121 * word 3: upper 32 bit address of Data Buffer 122 * word 4: unused 123 * word 5: timestamp word 1 124 * word 6: timestamp word 2 125 */ 126 static unsigned int macb_dma_desc_get_size(struct macb *bp) 127 { 128 #ifdef MACB_EXT_DESC 129 unsigned int desc_size; 130 131 switch (bp->hw_dma_cap) { 132 case HW_DMA_CAP_64B: 133 desc_size = sizeof(struct macb_dma_desc) 134 + sizeof(struct macb_dma_desc_64); 135 break; 136 case HW_DMA_CAP_PTP: 137 desc_size = sizeof(struct macb_dma_desc) 138 + sizeof(struct macb_dma_desc_ptp); 139 break; 140 case HW_DMA_CAP_64B_PTP: 141 desc_size = sizeof(struct macb_dma_desc) 142 + sizeof(struct macb_dma_desc_64) 143 + sizeof(struct macb_dma_desc_ptp); 144 break; 145 default: 146 desc_size = sizeof(struct macb_dma_desc); 147 } 148 return desc_size; 149 #endif 150 return sizeof(struct macb_dma_desc); 151 } 152 153 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 154 { 155 #ifdef MACB_EXT_DESC 156 switch (bp->hw_dma_cap) { 157 case HW_DMA_CAP_64B: 158 case HW_DMA_CAP_PTP: 159 desc_idx <<= 1; 160 break; 161 case HW_DMA_CAP_64B_PTP: 162 desc_idx *= 3; 163 break; 164 default: 165 break; 166 } 167 #endif 168 return desc_idx; 169 } 170 171 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 172 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 173 { 174 return (struct macb_dma_desc_64 *)((void *)desc 175 + sizeof(struct macb_dma_desc)); 176 } 177 #endif 178 179 /* Ring buffer accessors */ 180 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 181 { 182 return index & (bp->tx_ring_size - 1); 183 } 184 185 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 186 unsigned int index) 187 { 188 index = macb_tx_ring_wrap(queue->bp, index); 189 index = macb_adj_dma_desc_idx(queue->bp, index); 190 return &queue->tx_ring[index]; 191 } 192 193 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 194 unsigned int index) 195 { 196 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 197 } 198 199 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 200 { 201 dma_addr_t offset; 202 203 offset = macb_tx_ring_wrap(queue->bp, index) * 204 macb_dma_desc_get_size(queue->bp); 205 206 return queue->tx_ring_dma + offset; 207 } 208 209 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 210 { 211 return index & (bp->rx_ring_size - 1); 212 } 213 214 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 215 { 216 index = macb_rx_ring_wrap(queue->bp, index); 217 index = macb_adj_dma_desc_idx(queue->bp, index); 218 return &queue->rx_ring[index]; 219 } 220 221 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 222 { 223 return queue->rx_buffers + queue->bp->rx_buffer_size * 224 macb_rx_ring_wrap(queue->bp, index); 225 } 226 227 /* I/O accessors */ 228 static u32 hw_readl_native(struct macb *bp, int offset) 229 { 230 return __raw_readl(bp->regs + offset); 231 } 232 233 static void hw_writel_native(struct macb *bp, int offset, u32 value) 234 { 235 __raw_writel(value, bp->regs + offset); 236 } 237 238 static u32 hw_readl(struct macb *bp, int offset) 239 { 240 return readl_relaxed(bp->regs + offset); 241 } 242 243 static void hw_writel(struct macb *bp, int offset, u32 value) 244 { 245 writel_relaxed(value, bp->regs + offset); 246 } 247 248 /* Find the CPU endianness by using the loopback bit of NCR register. When the 249 * CPU is in big endian we need to program swapped mode for management 250 * descriptor access. 251 */ 252 static bool hw_is_native_io(void __iomem *addr) 253 { 254 u32 value = MACB_BIT(LLB); 255 256 __raw_writel(value, addr + MACB_NCR); 257 value = __raw_readl(addr + MACB_NCR); 258 259 /* Write 0 back to disable everything */ 260 __raw_writel(0, addr + MACB_NCR); 261 262 return value == MACB_BIT(LLB); 263 } 264 265 static bool hw_is_gem(void __iomem *addr, bool native_io) 266 { 267 u32 id; 268 269 if (native_io) 270 id = __raw_readl(addr + MACB_MID); 271 else 272 id = readl_relaxed(addr + MACB_MID); 273 274 return MACB_BFEXT(IDNUM, id) >= 0x2; 275 } 276 277 static void macb_set_hwaddr(struct macb *bp) 278 { 279 u32 bottom; 280 u16 top; 281 282 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr)); 283 macb_or_gem_writel(bp, SA1B, bottom); 284 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4))); 285 macb_or_gem_writel(bp, SA1T, top); 286 287 /* Clear unused address register sets */ 288 macb_or_gem_writel(bp, SA2B, 0); 289 macb_or_gem_writel(bp, SA2T, 0); 290 macb_or_gem_writel(bp, SA3B, 0); 291 macb_or_gem_writel(bp, SA3T, 0); 292 macb_or_gem_writel(bp, SA4B, 0); 293 macb_or_gem_writel(bp, SA4T, 0); 294 } 295 296 static void macb_get_hwaddr(struct macb *bp) 297 { 298 u32 bottom; 299 u16 top; 300 u8 addr[6]; 301 int i; 302 303 /* Check all 4 address register for valid address */ 304 for (i = 0; i < 4; i++) { 305 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 306 top = macb_or_gem_readl(bp, SA1T + i * 8); 307 308 addr[0] = bottom & 0xff; 309 addr[1] = (bottom >> 8) & 0xff; 310 addr[2] = (bottom >> 16) & 0xff; 311 addr[3] = (bottom >> 24) & 0xff; 312 addr[4] = top & 0xff; 313 addr[5] = (top >> 8) & 0xff; 314 315 if (is_valid_ether_addr(addr)) { 316 eth_hw_addr_set(bp->dev, addr); 317 return; 318 } 319 } 320 321 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 322 eth_hw_addr_random(bp->dev); 323 } 324 325 static int macb_mdio_wait_for_idle(struct macb *bp) 326 { 327 u32 val; 328 329 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 330 1, MACB_MDIO_TIMEOUT); 331 } 332 333 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) 334 { 335 struct macb *bp = bus->priv; 336 int status; 337 338 status = pm_runtime_get_sync(&bp->pdev->dev); 339 if (status < 0) { 340 pm_runtime_put_noidle(&bp->pdev->dev); 341 goto mdio_pm_exit; 342 } 343 344 status = macb_mdio_wait_for_idle(bp); 345 if (status < 0) 346 goto mdio_read_exit; 347 348 if (regnum & MII_ADDR_C45) { 349 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 350 | MACB_BF(RW, MACB_MAN_C45_ADDR) 351 | MACB_BF(PHYA, mii_id) 352 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 353 | MACB_BF(DATA, regnum & 0xFFFF) 354 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 355 356 status = macb_mdio_wait_for_idle(bp); 357 if (status < 0) 358 goto mdio_read_exit; 359 360 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 361 | MACB_BF(RW, MACB_MAN_C45_READ) 362 | MACB_BF(PHYA, mii_id) 363 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 364 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 365 } else { 366 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 367 | MACB_BF(RW, MACB_MAN_C22_READ) 368 | MACB_BF(PHYA, mii_id) 369 | MACB_BF(REGA, regnum) 370 | MACB_BF(CODE, MACB_MAN_C22_CODE))); 371 } 372 373 status = macb_mdio_wait_for_idle(bp); 374 if (status < 0) 375 goto mdio_read_exit; 376 377 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 378 379 mdio_read_exit: 380 pm_runtime_mark_last_busy(&bp->pdev->dev); 381 pm_runtime_put_autosuspend(&bp->pdev->dev); 382 mdio_pm_exit: 383 return status; 384 } 385 386 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, 387 u16 value) 388 { 389 struct macb *bp = bus->priv; 390 int status; 391 392 status = pm_runtime_get_sync(&bp->pdev->dev); 393 if (status < 0) { 394 pm_runtime_put_noidle(&bp->pdev->dev); 395 goto mdio_pm_exit; 396 } 397 398 status = macb_mdio_wait_for_idle(bp); 399 if (status < 0) 400 goto mdio_write_exit; 401 402 if (regnum & MII_ADDR_C45) { 403 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 404 | MACB_BF(RW, MACB_MAN_C45_ADDR) 405 | MACB_BF(PHYA, mii_id) 406 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 407 | MACB_BF(DATA, regnum & 0xFFFF) 408 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 409 410 status = macb_mdio_wait_for_idle(bp); 411 if (status < 0) 412 goto mdio_write_exit; 413 414 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 415 | MACB_BF(RW, MACB_MAN_C45_WRITE) 416 | MACB_BF(PHYA, mii_id) 417 | MACB_BF(REGA, (regnum >> 16) & 0x1F) 418 | MACB_BF(CODE, MACB_MAN_C45_CODE) 419 | MACB_BF(DATA, value))); 420 } else { 421 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 422 | MACB_BF(RW, MACB_MAN_C22_WRITE) 423 | MACB_BF(PHYA, mii_id) 424 | MACB_BF(REGA, regnum) 425 | MACB_BF(CODE, MACB_MAN_C22_CODE) 426 | MACB_BF(DATA, value))); 427 } 428 429 status = macb_mdio_wait_for_idle(bp); 430 if (status < 0) 431 goto mdio_write_exit; 432 433 mdio_write_exit: 434 pm_runtime_mark_last_busy(&bp->pdev->dev); 435 pm_runtime_put_autosuspend(&bp->pdev->dev); 436 mdio_pm_exit: 437 return status; 438 } 439 440 static void macb_init_buffers(struct macb *bp) 441 { 442 struct macb_queue *queue; 443 unsigned int q; 444 445 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 446 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 447 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 448 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 449 queue_writel(queue, RBQPH, 450 upper_32_bits(queue->rx_ring_dma)); 451 #endif 452 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 453 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 454 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 455 queue_writel(queue, TBQPH, 456 upper_32_bits(queue->tx_ring_dma)); 457 #endif 458 } 459 } 460 461 /** 462 * macb_set_tx_clk() - Set a clock to a new frequency 463 * @bp: pointer to struct macb 464 * @speed: New frequency in Hz 465 */ 466 static void macb_set_tx_clk(struct macb *bp, int speed) 467 { 468 long ferr, rate, rate_rounded; 469 470 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) 471 return; 472 473 /* In case of MII the PHY is the clock master */ 474 if (bp->phy_interface == PHY_INTERFACE_MODE_MII) 475 return; 476 477 switch (speed) { 478 case SPEED_10: 479 rate = 2500000; 480 break; 481 case SPEED_100: 482 rate = 25000000; 483 break; 484 case SPEED_1000: 485 rate = 125000000; 486 break; 487 default: 488 return; 489 } 490 491 rate_rounded = clk_round_rate(bp->tx_clk, rate); 492 if (rate_rounded < 0) 493 return; 494 495 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 496 * is not satisfied. 497 */ 498 ferr = abs(rate_rounded - rate); 499 ferr = DIV_ROUND_UP(ferr, rate / 100000); 500 if (ferr > 5) 501 netdev_warn(bp->dev, 502 "unable to generate target frequency: %ld Hz\n", 503 rate); 504 505 if (clk_set_rate(bp->tx_clk, rate_rounded)) 506 netdev_err(bp->dev, "adjusting tx_clk failed.\n"); 507 } 508 509 static void macb_validate(struct phylink_config *config, 510 unsigned long *supported, 511 struct phylink_link_state *state) 512 { 513 struct net_device *ndev = to_net_dev(config->dev); 514 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; 515 struct macb *bp = netdev_priv(ndev); 516 517 /* We only support MII, RMII, GMII, RGMII & SGMII. */ 518 if (state->interface != PHY_INTERFACE_MODE_NA && 519 state->interface != PHY_INTERFACE_MODE_MII && 520 state->interface != PHY_INTERFACE_MODE_RMII && 521 state->interface != PHY_INTERFACE_MODE_GMII && 522 state->interface != PHY_INTERFACE_MODE_SGMII && 523 state->interface != PHY_INTERFACE_MODE_10GBASER && 524 !phy_interface_mode_is_rgmii(state->interface)) { 525 linkmode_zero(supported); 526 return; 527 } 528 529 if (!macb_is_gem(bp) && 530 (state->interface == PHY_INTERFACE_MODE_GMII || 531 phy_interface_mode_is_rgmii(state->interface))) { 532 linkmode_zero(supported); 533 return; 534 } 535 536 if (state->interface == PHY_INTERFACE_MODE_10GBASER && 537 !(bp->caps & MACB_CAPS_HIGH_SPEED && 538 bp->caps & MACB_CAPS_PCS)) { 539 linkmode_zero(supported); 540 return; 541 } 542 543 phylink_set_port_modes(mask); 544 phylink_set(mask, Autoneg); 545 phylink_set(mask, Asym_Pause); 546 547 if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && 548 (state->interface == PHY_INTERFACE_MODE_NA || 549 state->interface == PHY_INTERFACE_MODE_10GBASER)) { 550 phylink_set_10g_modes(mask); 551 phylink_set(mask, 10000baseKR_Full); 552 if (state->interface != PHY_INTERFACE_MODE_NA) 553 goto out; 554 } 555 556 phylink_set(mask, 10baseT_Half); 557 phylink_set(mask, 10baseT_Full); 558 phylink_set(mask, 100baseT_Half); 559 phylink_set(mask, 100baseT_Full); 560 561 if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && 562 (state->interface == PHY_INTERFACE_MODE_NA || 563 state->interface == PHY_INTERFACE_MODE_GMII || 564 state->interface == PHY_INTERFACE_MODE_SGMII || 565 phy_interface_mode_is_rgmii(state->interface))) { 566 phylink_set(mask, 1000baseT_Full); 567 phylink_set(mask, 1000baseX_Full); 568 569 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) 570 phylink_set(mask, 1000baseT_Half); 571 } 572 out: 573 linkmode_and(supported, supported, mask); 574 linkmode_and(state->advertising, state->advertising, mask); 575 } 576 577 static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, 578 phy_interface_t interface, int speed, 579 int duplex) 580 { 581 struct macb *bp = container_of(pcs, struct macb, phylink_pcs); 582 u32 config; 583 584 config = gem_readl(bp, USX_CONTROL); 585 config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config); 586 config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config); 587 config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS)); 588 config |= GEM_BIT(TX_EN); 589 gem_writel(bp, USX_CONTROL, config); 590 } 591 592 static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, 593 struct phylink_link_state *state) 594 { 595 struct macb *bp = container_of(pcs, struct macb, phylink_pcs); 596 u32 val; 597 598 state->speed = SPEED_10000; 599 state->duplex = 1; 600 state->an_complete = 1; 601 602 val = gem_readl(bp, USX_STATUS); 603 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); 604 val = gem_readl(bp, NCFGR); 605 if (val & GEM_BIT(PAE)) 606 state->pause = MLO_PAUSE_RX; 607 } 608 609 static int macb_usx_pcs_config(struct phylink_pcs *pcs, 610 unsigned int mode, 611 phy_interface_t interface, 612 const unsigned long *advertising, 613 bool permit_pause_to_mac) 614 { 615 struct macb *bp = container_of(pcs, struct macb, phylink_pcs); 616 617 gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | 618 GEM_BIT(SIGNAL_OK)); 619 620 return 0; 621 } 622 623 static void macb_pcs_get_state(struct phylink_pcs *pcs, 624 struct phylink_link_state *state) 625 { 626 state->link = 0; 627 } 628 629 static void macb_pcs_an_restart(struct phylink_pcs *pcs) 630 { 631 /* Not supported */ 632 } 633 634 static int macb_pcs_config(struct phylink_pcs *pcs, 635 unsigned int mode, 636 phy_interface_t interface, 637 const unsigned long *advertising, 638 bool permit_pause_to_mac) 639 { 640 return 0; 641 } 642 643 static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { 644 .pcs_get_state = macb_usx_pcs_get_state, 645 .pcs_config = macb_usx_pcs_config, 646 .pcs_link_up = macb_usx_pcs_link_up, 647 }; 648 649 static const struct phylink_pcs_ops macb_phylink_pcs_ops = { 650 .pcs_get_state = macb_pcs_get_state, 651 .pcs_an_restart = macb_pcs_an_restart, 652 .pcs_config = macb_pcs_config, 653 }; 654 655 static void macb_mac_config(struct phylink_config *config, unsigned int mode, 656 const struct phylink_link_state *state) 657 { 658 struct net_device *ndev = to_net_dev(config->dev); 659 struct macb *bp = netdev_priv(ndev); 660 unsigned long flags; 661 u32 old_ctrl, ctrl; 662 u32 old_ncr, ncr; 663 664 spin_lock_irqsave(&bp->lock, flags); 665 666 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); 667 old_ncr = ncr = macb_or_gem_readl(bp, NCR); 668 669 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { 670 if (state->interface == PHY_INTERFACE_MODE_RMII) 671 ctrl |= MACB_BIT(RM9200_RMII); 672 } else if (macb_is_gem(bp)) { 673 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); 674 ncr &= ~GEM_BIT(ENABLE_HS_MAC); 675 676 if (state->interface == PHY_INTERFACE_MODE_SGMII) { 677 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 678 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) { 679 ctrl |= GEM_BIT(PCSSEL); 680 ncr |= GEM_BIT(ENABLE_HS_MAC); 681 } else if (bp->caps & MACB_CAPS_MIIONRGMII && 682 bp->phy_interface == PHY_INTERFACE_MODE_MII) { 683 ncr |= MACB_BIT(MIIONRGMII); 684 } 685 } 686 687 /* Apply the new configuration, if any */ 688 if (old_ctrl ^ ctrl) 689 macb_or_gem_writel(bp, NCFGR, ctrl); 690 691 if (old_ncr ^ ncr) 692 macb_or_gem_writel(bp, NCR, ncr); 693 694 /* Disable AN for SGMII fixed link configuration, enable otherwise. 695 * Must be written after PCSSEL is set in NCFGR, 696 * otherwise writes will not take effect. 697 */ 698 if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) { 699 u32 pcsctrl, old_pcsctrl; 700 701 old_pcsctrl = gem_readl(bp, PCSCNTRL); 702 if (mode == MLO_AN_FIXED) 703 pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); 704 else 705 pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG); 706 if (old_pcsctrl != pcsctrl) 707 gem_writel(bp, PCSCNTRL, pcsctrl); 708 } 709 710 spin_unlock_irqrestore(&bp->lock, flags); 711 } 712 713 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, 714 phy_interface_t interface) 715 { 716 struct net_device *ndev = to_net_dev(config->dev); 717 struct macb *bp = netdev_priv(ndev); 718 struct macb_queue *queue; 719 unsigned int q; 720 u32 ctrl; 721 722 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 723 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 724 queue_writel(queue, IDR, 725 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 726 727 /* Disable Rx and Tx */ 728 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); 729 macb_writel(bp, NCR, ctrl); 730 731 netif_tx_stop_all_queues(ndev); 732 } 733 734 static void macb_mac_link_up(struct phylink_config *config, 735 struct phy_device *phy, 736 unsigned int mode, phy_interface_t interface, 737 int speed, int duplex, 738 bool tx_pause, bool rx_pause) 739 { 740 struct net_device *ndev = to_net_dev(config->dev); 741 struct macb *bp = netdev_priv(ndev); 742 struct macb_queue *queue; 743 unsigned long flags; 744 unsigned int q; 745 u32 ctrl; 746 747 spin_lock_irqsave(&bp->lock, flags); 748 749 ctrl = macb_or_gem_readl(bp, NCFGR); 750 751 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 752 753 if (speed == SPEED_100) 754 ctrl |= MACB_BIT(SPD); 755 756 if (duplex) 757 ctrl |= MACB_BIT(FD); 758 759 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 760 ctrl &= ~MACB_BIT(PAE); 761 if (macb_is_gem(bp)) { 762 ctrl &= ~GEM_BIT(GBE); 763 764 if (speed == SPEED_1000) 765 ctrl |= GEM_BIT(GBE); 766 } 767 768 if (rx_pause) 769 ctrl |= MACB_BIT(PAE); 770 771 macb_set_tx_clk(bp, speed); 772 773 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down 774 * cleared the pipeline and control registers. 775 */ 776 bp->macbgem_ops.mog_init_rings(bp); 777 macb_init_buffers(bp); 778 779 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 780 queue_writel(queue, IER, 781 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 782 } 783 784 macb_or_gem_writel(bp, NCFGR, ctrl); 785 786 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER) 787 gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, 788 gem_readl(bp, HS_MAC_CONFIG))); 789 790 spin_unlock_irqrestore(&bp->lock, flags); 791 792 /* Enable Rx and Tx */ 793 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE)); 794 795 netif_tx_wake_all_queues(ndev); 796 } 797 798 static int macb_mac_prepare(struct phylink_config *config, unsigned int mode, 799 phy_interface_t interface) 800 { 801 struct net_device *ndev = to_net_dev(config->dev); 802 struct macb *bp = netdev_priv(ndev); 803 804 if (interface == PHY_INTERFACE_MODE_10GBASER) 805 bp->phylink_pcs.ops = &macb_phylink_usx_pcs_ops; 806 else if (interface == PHY_INTERFACE_MODE_SGMII) 807 bp->phylink_pcs.ops = &macb_phylink_pcs_ops; 808 else 809 bp->phylink_pcs.ops = NULL; 810 811 if (bp->phylink_pcs.ops) 812 phylink_set_pcs(bp->phylink, &bp->phylink_pcs); 813 814 return 0; 815 } 816 817 static const struct phylink_mac_ops macb_phylink_ops = { 818 .validate = macb_validate, 819 .mac_prepare = macb_mac_prepare, 820 .mac_config = macb_mac_config, 821 .mac_link_down = macb_mac_link_down, 822 .mac_link_up = macb_mac_link_up, 823 }; 824 825 static bool macb_phy_handle_exists(struct device_node *dn) 826 { 827 dn = of_parse_phandle(dn, "phy-handle", 0); 828 of_node_put(dn); 829 return dn != NULL; 830 } 831 832 static int macb_phylink_connect(struct macb *bp) 833 { 834 struct device_node *dn = bp->pdev->dev.of_node; 835 struct net_device *dev = bp->dev; 836 struct phy_device *phydev; 837 int ret; 838 839 if (dn) 840 ret = phylink_of_phy_connect(bp->phylink, dn, 0); 841 842 if (!dn || (ret && !macb_phy_handle_exists(dn))) { 843 phydev = phy_find_first(bp->mii_bus); 844 if (!phydev) { 845 netdev_err(dev, "no PHY found\n"); 846 return -ENXIO; 847 } 848 849 /* attach the mac to the phy */ 850 ret = phylink_connect_phy(bp->phylink, phydev); 851 } 852 853 if (ret) { 854 netdev_err(dev, "Could not attach PHY (%d)\n", ret); 855 return ret; 856 } 857 858 phylink_start(bp->phylink); 859 860 return 0; 861 } 862 863 static void macb_get_pcs_fixed_state(struct phylink_config *config, 864 struct phylink_link_state *state) 865 { 866 struct net_device *ndev = to_net_dev(config->dev); 867 struct macb *bp = netdev_priv(ndev); 868 869 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; 870 } 871 872 /* based on au1000_eth. c*/ 873 static int macb_mii_probe(struct net_device *dev) 874 { 875 struct macb *bp = netdev_priv(dev); 876 877 bp->phylink_config.dev = &dev->dev; 878 bp->phylink_config.type = PHYLINK_NETDEV; 879 880 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { 881 bp->phylink_config.poll_fixed_state = true; 882 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; 883 } 884 885 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, 886 bp->phy_interface, &macb_phylink_ops); 887 if (IS_ERR(bp->phylink)) { 888 netdev_err(dev, "Could not create a phylink instance (%ld)\n", 889 PTR_ERR(bp->phylink)); 890 return PTR_ERR(bp->phylink); 891 } 892 893 return 0; 894 } 895 896 static int macb_mdiobus_register(struct macb *bp) 897 { 898 struct device_node *child, *np = bp->pdev->dev.of_node; 899 900 /* If we have a child named mdio, probe it instead of looking for PHYs 901 * directly under the MAC node 902 */ 903 child = of_get_child_by_name(np, "mdio"); 904 if (child) { 905 int ret = of_mdiobus_register(bp->mii_bus, child); 906 907 of_node_put(child); 908 return ret; 909 } 910 911 if (of_phy_is_fixed_link(np)) 912 return mdiobus_register(bp->mii_bus); 913 914 /* Only create the PHY from the device tree if at least one PHY is 915 * described. Otherwise scan the entire MDIO bus. We do this to support 916 * old device tree that did not follow the best practices and did not 917 * describe their network PHYs. 918 */ 919 for_each_available_child_of_node(np, child) 920 if (of_mdiobus_child_is_phy(child)) { 921 /* The loop increments the child refcount, 922 * decrement it before returning. 923 */ 924 of_node_put(child); 925 926 return of_mdiobus_register(bp->mii_bus, np); 927 } 928 929 return mdiobus_register(bp->mii_bus); 930 } 931 932 static int macb_mii_init(struct macb *bp) 933 { 934 int err = -ENXIO; 935 936 /* Enable management port */ 937 macb_writel(bp, NCR, MACB_BIT(MPE)); 938 939 bp->mii_bus = mdiobus_alloc(); 940 if (!bp->mii_bus) { 941 err = -ENOMEM; 942 goto err_out; 943 } 944 945 bp->mii_bus->name = "MACB_mii_bus"; 946 bp->mii_bus->read = &macb_mdio_read; 947 bp->mii_bus->write = &macb_mdio_write; 948 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 949 bp->pdev->name, bp->pdev->id); 950 bp->mii_bus->priv = bp; 951 bp->mii_bus->parent = &bp->pdev->dev; 952 953 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 954 955 err = macb_mdiobus_register(bp); 956 if (err) 957 goto err_out_free_mdiobus; 958 959 err = macb_mii_probe(bp->dev); 960 if (err) 961 goto err_out_unregister_bus; 962 963 return 0; 964 965 err_out_unregister_bus: 966 mdiobus_unregister(bp->mii_bus); 967 err_out_free_mdiobus: 968 mdiobus_free(bp->mii_bus); 969 err_out: 970 return err; 971 } 972 973 static void macb_update_stats(struct macb *bp) 974 { 975 u32 *p = &bp->hw_stats.macb.rx_pause_frames; 976 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 977 int offset = MACB_PFR; 978 979 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 980 981 for (; p < end; p++, offset += 4) 982 *p += bp->macb_reg_readl(bp, offset); 983 } 984 985 static int macb_halt_tx(struct macb *bp) 986 { 987 unsigned long halt_time, timeout; 988 u32 status; 989 990 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 991 992 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT); 993 do { 994 halt_time = jiffies; 995 status = macb_readl(bp, TSR); 996 if (!(status & MACB_BIT(TGO))) 997 return 0; 998 999 udelay(250); 1000 } while (time_before(halt_time, timeout)); 1001 1002 return -ETIMEDOUT; 1003 } 1004 1005 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) 1006 { 1007 if (tx_skb->mapping) { 1008 if (tx_skb->mapped_as_page) 1009 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 1010 tx_skb->size, DMA_TO_DEVICE); 1011 else 1012 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 1013 tx_skb->size, DMA_TO_DEVICE); 1014 tx_skb->mapping = 0; 1015 } 1016 1017 if (tx_skb->skb) { 1018 dev_kfree_skb_any(tx_skb->skb); 1019 tx_skb->skb = NULL; 1020 } 1021 } 1022 1023 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 1024 { 1025 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1026 struct macb_dma_desc_64 *desc_64; 1027 1028 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 1029 desc_64 = macb_64b_desc(bp, desc); 1030 desc_64->addrh = upper_32_bits(addr); 1031 /* The low bits of RX address contain the RX_USED bit, clearing 1032 * of which allows packet RX. Make sure the high bits are also 1033 * visible to HW at that point. 1034 */ 1035 dma_wmb(); 1036 } 1037 #endif 1038 desc->addr = lower_32_bits(addr); 1039 } 1040 1041 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 1042 { 1043 dma_addr_t addr = 0; 1044 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1045 struct macb_dma_desc_64 *desc_64; 1046 1047 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 1048 desc_64 = macb_64b_desc(bp, desc); 1049 addr = ((u64)(desc_64->addrh) << 32); 1050 } 1051 #endif 1052 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 1053 return addr; 1054 } 1055 1056 static void macb_tx_error_task(struct work_struct *work) 1057 { 1058 struct macb_queue *queue = container_of(work, struct macb_queue, 1059 tx_error_task); 1060 struct macb *bp = queue->bp; 1061 struct macb_tx_skb *tx_skb; 1062 struct macb_dma_desc *desc; 1063 struct sk_buff *skb; 1064 unsigned int tail; 1065 unsigned long flags; 1066 1067 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 1068 (unsigned int)(queue - bp->queues), 1069 queue->tx_tail, queue->tx_head); 1070 1071 /* Prevent the queue IRQ handlers from running: each of them may call 1072 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). 1073 * As explained below, we have to halt the transmission before updating 1074 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 1075 * network engine about the macb/gem being halted. 1076 */ 1077 spin_lock_irqsave(&bp->lock, flags); 1078 1079 /* Make sure nobody is trying to queue up new packets */ 1080 netif_tx_stop_all_queues(bp->dev); 1081 1082 /* Stop transmission now 1083 * (in case we have just queued new packets) 1084 * macb/gem must be halted to write TBQP register 1085 */ 1086 if (macb_halt_tx(bp)) 1087 /* Just complain for now, reinitializing TX path can be good */ 1088 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 1089 1090 /* Treat frames in TX queue including the ones that caused the error. 1091 * Free transmit buffers in upper layer. 1092 */ 1093 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 1094 u32 ctrl; 1095 1096 desc = macb_tx_desc(queue, tail); 1097 ctrl = desc->ctrl; 1098 tx_skb = macb_tx_skb(queue, tail); 1099 skb = tx_skb->skb; 1100 1101 if (ctrl & MACB_BIT(TX_USED)) { 1102 /* skb is set for the last buffer of the frame */ 1103 while (!skb) { 1104 macb_tx_unmap(bp, tx_skb); 1105 tail++; 1106 tx_skb = macb_tx_skb(queue, tail); 1107 skb = tx_skb->skb; 1108 } 1109 1110 /* ctrl still refers to the first buffer descriptor 1111 * since it's the only one written back by the hardware 1112 */ 1113 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 1114 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 1115 macb_tx_ring_wrap(bp, tail), 1116 skb->data); 1117 bp->dev->stats.tx_packets++; 1118 queue->stats.tx_packets++; 1119 bp->dev->stats.tx_bytes += skb->len; 1120 queue->stats.tx_bytes += skb->len; 1121 } 1122 } else { 1123 /* "Buffers exhausted mid-frame" errors may only happen 1124 * if the driver is buggy, so complain loudly about 1125 * those. Statistics are updated by hardware. 1126 */ 1127 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 1128 netdev_err(bp->dev, 1129 "BUG: TX buffers exhausted mid-frame\n"); 1130 1131 desc->ctrl = ctrl | MACB_BIT(TX_USED); 1132 } 1133 1134 macb_tx_unmap(bp, tx_skb); 1135 } 1136 1137 /* Set end of TX queue */ 1138 desc = macb_tx_desc(queue, 0); 1139 macb_set_addr(bp, desc, 0); 1140 desc->ctrl = MACB_BIT(TX_USED); 1141 1142 /* Make descriptor updates visible to hardware */ 1143 wmb(); 1144 1145 /* Reinitialize the TX desc queue */ 1146 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1147 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 1148 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 1149 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); 1150 #endif 1151 /* Make TX ring reflect state of hardware */ 1152 queue->tx_head = 0; 1153 queue->tx_tail = 0; 1154 1155 /* Housework before enabling TX IRQ */ 1156 macb_writel(bp, TSR, macb_readl(bp, TSR)); 1157 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 1158 1159 /* Now we are ready to start transmission again */ 1160 netif_tx_start_all_queues(bp->dev); 1161 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1162 1163 spin_unlock_irqrestore(&bp->lock, flags); 1164 } 1165 1166 static void macb_tx_interrupt(struct macb_queue *queue) 1167 { 1168 unsigned int tail; 1169 unsigned int head; 1170 u32 status; 1171 struct macb *bp = queue->bp; 1172 u16 queue_index = queue - bp->queues; 1173 1174 status = macb_readl(bp, TSR); 1175 macb_writel(bp, TSR, status); 1176 1177 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1178 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 1179 1180 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", 1181 (unsigned long)status); 1182 1183 head = queue->tx_head; 1184 for (tail = queue->tx_tail; tail != head; tail++) { 1185 struct macb_tx_skb *tx_skb; 1186 struct sk_buff *skb; 1187 struct macb_dma_desc *desc; 1188 u32 ctrl; 1189 1190 desc = macb_tx_desc(queue, tail); 1191 1192 /* Make hw descriptor updates visible to CPU */ 1193 rmb(); 1194 1195 ctrl = desc->ctrl; 1196 1197 /* TX_USED bit is only set by hardware on the very first buffer 1198 * descriptor of the transmitted frame. 1199 */ 1200 if (!(ctrl & MACB_BIT(TX_USED))) 1201 break; 1202 1203 /* Process all buffers of the current transmitted frame */ 1204 for (;; tail++) { 1205 tx_skb = macb_tx_skb(queue, tail); 1206 skb = tx_skb->skb; 1207 1208 /* First, update TX stats if needed */ 1209 if (skb) { 1210 if (unlikely(skb_shinfo(skb)->tx_flags & 1211 SKBTX_HW_TSTAMP) && 1212 gem_ptp_do_txstamp(queue, skb, desc) == 0) { 1213 /* skb now belongs to timestamp buffer 1214 * and will be removed later 1215 */ 1216 tx_skb->skb = NULL; 1217 } 1218 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 1219 macb_tx_ring_wrap(bp, tail), 1220 skb->data); 1221 bp->dev->stats.tx_packets++; 1222 queue->stats.tx_packets++; 1223 bp->dev->stats.tx_bytes += skb->len; 1224 queue->stats.tx_bytes += skb->len; 1225 } 1226 1227 /* Now we can safely release resources */ 1228 macb_tx_unmap(bp, tx_skb); 1229 1230 /* skb is set only for the last buffer of the frame. 1231 * WARNING: at this point skb has been freed by 1232 * macb_tx_unmap(). 1233 */ 1234 if (skb) 1235 break; 1236 } 1237 } 1238 1239 queue->tx_tail = tail; 1240 if (__netif_subqueue_stopped(bp->dev, queue_index) && 1241 CIRC_CNT(queue->tx_head, queue->tx_tail, 1242 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 1243 netif_wake_subqueue(bp->dev, queue_index); 1244 } 1245 1246 static void gem_rx_refill(struct macb_queue *queue) 1247 { 1248 unsigned int entry; 1249 struct sk_buff *skb; 1250 dma_addr_t paddr; 1251 struct macb *bp = queue->bp; 1252 struct macb_dma_desc *desc; 1253 1254 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 1255 bp->rx_ring_size) > 0) { 1256 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 1257 1258 /* Make hw descriptor updates visible to CPU */ 1259 rmb(); 1260 1261 queue->rx_prepared_head++; 1262 desc = macb_rx_desc(queue, entry); 1263 1264 if (!queue->rx_skbuff[entry]) { 1265 /* allocate sk_buff for this free entry in ring */ 1266 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 1267 if (unlikely(!skb)) { 1268 netdev_err(bp->dev, 1269 "Unable to allocate sk_buff\n"); 1270 break; 1271 } 1272 1273 /* now fill corresponding descriptor entry */ 1274 paddr = dma_map_single(&bp->pdev->dev, skb->data, 1275 bp->rx_buffer_size, 1276 DMA_FROM_DEVICE); 1277 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 1278 dev_kfree_skb(skb); 1279 break; 1280 } 1281 1282 queue->rx_skbuff[entry] = skb; 1283 1284 if (entry == bp->rx_ring_size - 1) 1285 paddr |= MACB_BIT(RX_WRAP); 1286 desc->ctrl = 0; 1287 /* Setting addr clears RX_USED and allows reception, 1288 * make sure ctrl is cleared first to avoid a race. 1289 */ 1290 dma_wmb(); 1291 macb_set_addr(bp, desc, paddr); 1292 1293 /* properly align Ethernet header */ 1294 skb_reserve(skb, NET_IP_ALIGN); 1295 } else { 1296 desc->ctrl = 0; 1297 dma_wmb(); 1298 desc->addr &= ~MACB_BIT(RX_USED); 1299 } 1300 } 1301 1302 /* Make descriptor updates visible to hardware */ 1303 wmb(); 1304 1305 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 1306 queue, queue->rx_prepared_head, queue->rx_tail); 1307 } 1308 1309 /* Mark DMA descriptors from begin up to and not including end as unused */ 1310 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 1311 unsigned int end) 1312 { 1313 unsigned int frag; 1314 1315 for (frag = begin; frag != end; frag++) { 1316 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 1317 1318 desc->addr &= ~MACB_BIT(RX_USED); 1319 } 1320 1321 /* Make descriptor updates visible to hardware */ 1322 wmb(); 1323 1324 /* When this happens, the hardware stats registers for 1325 * whatever caused this is updated, so we don't have to record 1326 * anything. 1327 */ 1328 } 1329 1330 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 1331 int budget) 1332 { 1333 struct macb *bp = queue->bp; 1334 unsigned int len; 1335 unsigned int entry; 1336 struct sk_buff *skb; 1337 struct macb_dma_desc *desc; 1338 int count = 0; 1339 1340 while (count < budget) { 1341 u32 ctrl; 1342 dma_addr_t addr; 1343 bool rxused; 1344 1345 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1346 desc = macb_rx_desc(queue, entry); 1347 1348 /* Make hw descriptor updates visible to CPU */ 1349 rmb(); 1350 1351 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1352 addr = macb_get_addr(bp, desc); 1353 1354 if (!rxused) 1355 break; 1356 1357 /* Ensure ctrl is at least as up-to-date as rxused */ 1358 dma_rmb(); 1359 1360 ctrl = desc->ctrl; 1361 1362 queue->rx_tail++; 1363 count++; 1364 1365 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1366 netdev_err(bp->dev, 1367 "not whole frame pointed by descriptor\n"); 1368 bp->dev->stats.rx_dropped++; 1369 queue->stats.rx_dropped++; 1370 break; 1371 } 1372 skb = queue->rx_skbuff[entry]; 1373 if (unlikely(!skb)) { 1374 netdev_err(bp->dev, 1375 "inconsistent Rx descriptor chain\n"); 1376 bp->dev->stats.rx_dropped++; 1377 queue->stats.rx_dropped++; 1378 break; 1379 } 1380 /* now everything is ready for receiving packet */ 1381 queue->rx_skbuff[entry] = NULL; 1382 len = ctrl & bp->rx_frm_len_mask; 1383 1384 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1385 1386 skb_put(skb, len); 1387 dma_unmap_single(&bp->pdev->dev, addr, 1388 bp->rx_buffer_size, DMA_FROM_DEVICE); 1389 1390 skb->protocol = eth_type_trans(skb, bp->dev); 1391 skb_checksum_none_assert(skb); 1392 if (bp->dev->features & NETIF_F_RXCSUM && 1393 !(bp->dev->flags & IFF_PROMISC) && 1394 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1395 skb->ip_summed = CHECKSUM_UNNECESSARY; 1396 1397 bp->dev->stats.rx_packets++; 1398 queue->stats.rx_packets++; 1399 bp->dev->stats.rx_bytes += skb->len; 1400 queue->stats.rx_bytes += skb->len; 1401 1402 gem_ptp_do_rxstamp(bp, skb, desc); 1403 1404 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 1405 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1406 skb->len, skb->csum); 1407 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1408 skb_mac_header(skb), 16, true); 1409 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1410 skb->data, 32, true); 1411 #endif 1412 1413 napi_gro_receive(napi, skb); 1414 } 1415 1416 gem_rx_refill(queue); 1417 1418 return count; 1419 } 1420 1421 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1422 unsigned int first_frag, unsigned int last_frag) 1423 { 1424 unsigned int len; 1425 unsigned int frag; 1426 unsigned int offset; 1427 struct sk_buff *skb; 1428 struct macb_dma_desc *desc; 1429 struct macb *bp = queue->bp; 1430 1431 desc = macb_rx_desc(queue, last_frag); 1432 len = desc->ctrl & bp->rx_frm_len_mask; 1433 1434 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1435 macb_rx_ring_wrap(bp, first_frag), 1436 macb_rx_ring_wrap(bp, last_frag), len); 1437 1438 /* The ethernet header starts NET_IP_ALIGN bytes into the 1439 * first buffer. Since the header is 14 bytes, this makes the 1440 * payload word-aligned. 1441 * 1442 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1443 * the two padding bytes into the skb so that we avoid hitting 1444 * the slowpath in memcpy(), and pull them off afterwards. 1445 */ 1446 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1447 if (!skb) { 1448 bp->dev->stats.rx_dropped++; 1449 for (frag = first_frag; ; frag++) { 1450 desc = macb_rx_desc(queue, frag); 1451 desc->addr &= ~MACB_BIT(RX_USED); 1452 if (frag == last_frag) 1453 break; 1454 } 1455 1456 /* Make descriptor updates visible to hardware */ 1457 wmb(); 1458 1459 return 1; 1460 } 1461 1462 offset = 0; 1463 len += NET_IP_ALIGN; 1464 skb_checksum_none_assert(skb); 1465 skb_put(skb, len); 1466 1467 for (frag = first_frag; ; frag++) { 1468 unsigned int frag_len = bp->rx_buffer_size; 1469 1470 if (offset + frag_len > len) { 1471 if (unlikely(frag != last_frag)) { 1472 dev_kfree_skb_any(skb); 1473 return -1; 1474 } 1475 frag_len = len - offset; 1476 } 1477 skb_copy_to_linear_data_offset(skb, offset, 1478 macb_rx_buffer(queue, frag), 1479 frag_len); 1480 offset += bp->rx_buffer_size; 1481 desc = macb_rx_desc(queue, frag); 1482 desc->addr &= ~MACB_BIT(RX_USED); 1483 1484 if (frag == last_frag) 1485 break; 1486 } 1487 1488 /* Make descriptor updates visible to hardware */ 1489 wmb(); 1490 1491 __skb_pull(skb, NET_IP_ALIGN); 1492 skb->protocol = eth_type_trans(skb, bp->dev); 1493 1494 bp->dev->stats.rx_packets++; 1495 bp->dev->stats.rx_bytes += skb->len; 1496 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1497 skb->len, skb->csum); 1498 napi_gro_receive(napi, skb); 1499 1500 return 0; 1501 } 1502 1503 static inline void macb_init_rx_ring(struct macb_queue *queue) 1504 { 1505 struct macb *bp = queue->bp; 1506 dma_addr_t addr; 1507 struct macb_dma_desc *desc = NULL; 1508 int i; 1509 1510 addr = queue->rx_buffers_dma; 1511 for (i = 0; i < bp->rx_ring_size; i++) { 1512 desc = macb_rx_desc(queue, i); 1513 macb_set_addr(bp, desc, addr); 1514 desc->ctrl = 0; 1515 addr += bp->rx_buffer_size; 1516 } 1517 desc->addr |= MACB_BIT(RX_WRAP); 1518 queue->rx_tail = 0; 1519 } 1520 1521 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1522 int budget) 1523 { 1524 struct macb *bp = queue->bp; 1525 bool reset_rx_queue = false; 1526 int received = 0; 1527 unsigned int tail; 1528 int first_frag = -1; 1529 1530 for (tail = queue->rx_tail; budget > 0; tail++) { 1531 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1532 u32 ctrl; 1533 1534 /* Make hw descriptor updates visible to CPU */ 1535 rmb(); 1536 1537 if (!(desc->addr & MACB_BIT(RX_USED))) 1538 break; 1539 1540 /* Ensure ctrl is at least as up-to-date as addr */ 1541 dma_rmb(); 1542 1543 ctrl = desc->ctrl; 1544 1545 if (ctrl & MACB_BIT(RX_SOF)) { 1546 if (first_frag != -1) 1547 discard_partial_frame(queue, first_frag, tail); 1548 first_frag = tail; 1549 } 1550 1551 if (ctrl & MACB_BIT(RX_EOF)) { 1552 int dropped; 1553 1554 if (unlikely(first_frag == -1)) { 1555 reset_rx_queue = true; 1556 continue; 1557 } 1558 1559 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1560 first_frag = -1; 1561 if (unlikely(dropped < 0)) { 1562 reset_rx_queue = true; 1563 continue; 1564 } 1565 if (!dropped) { 1566 received++; 1567 budget--; 1568 } 1569 } 1570 } 1571 1572 if (unlikely(reset_rx_queue)) { 1573 unsigned long flags; 1574 u32 ctrl; 1575 1576 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1577 1578 spin_lock_irqsave(&bp->lock, flags); 1579 1580 ctrl = macb_readl(bp, NCR); 1581 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1582 1583 macb_init_rx_ring(queue); 1584 queue_writel(queue, RBQP, queue->rx_ring_dma); 1585 1586 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1587 1588 spin_unlock_irqrestore(&bp->lock, flags); 1589 return received; 1590 } 1591 1592 if (first_frag != -1) 1593 queue->rx_tail = first_frag; 1594 else 1595 queue->rx_tail = tail; 1596 1597 return received; 1598 } 1599 1600 static int macb_poll(struct napi_struct *napi, int budget) 1601 { 1602 struct macb_queue *queue = container_of(napi, struct macb_queue, napi); 1603 struct macb *bp = queue->bp; 1604 int work_done; 1605 u32 status; 1606 1607 status = macb_readl(bp, RSR); 1608 macb_writel(bp, RSR, status); 1609 1610 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", 1611 (unsigned long)status, budget); 1612 1613 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 1614 if (work_done < budget) { 1615 napi_complete_done(napi, work_done); 1616 1617 /* Packets received while interrupts were disabled */ 1618 status = macb_readl(bp, RSR); 1619 if (status) { 1620 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1621 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1622 napi_reschedule(napi); 1623 } else { 1624 queue_writel(queue, IER, bp->rx_intr_mask); 1625 } 1626 } 1627 1628 /* TODO: Handle errors */ 1629 1630 return work_done; 1631 } 1632 1633 static void macb_hresp_error_task(struct tasklet_struct *t) 1634 { 1635 struct macb *bp = from_tasklet(bp, t, hresp_err_tasklet); 1636 struct net_device *dev = bp->dev; 1637 struct macb_queue *queue; 1638 unsigned int q; 1639 u32 ctrl; 1640 1641 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1642 queue_writel(queue, IDR, bp->rx_intr_mask | 1643 MACB_TX_INT_FLAGS | 1644 MACB_BIT(HRESP)); 1645 } 1646 ctrl = macb_readl(bp, NCR); 1647 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1648 macb_writel(bp, NCR, ctrl); 1649 1650 netif_tx_stop_all_queues(dev); 1651 netif_carrier_off(dev); 1652 1653 bp->macbgem_ops.mog_init_rings(bp); 1654 1655 /* Initialize TX and RX buffers */ 1656 macb_init_buffers(bp); 1657 1658 /* Enable interrupts */ 1659 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1660 queue_writel(queue, IER, 1661 bp->rx_intr_mask | 1662 MACB_TX_INT_FLAGS | 1663 MACB_BIT(HRESP)); 1664 1665 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1666 macb_writel(bp, NCR, ctrl); 1667 1668 netif_carrier_on(dev); 1669 netif_tx_start_all_queues(dev); 1670 } 1671 1672 static void macb_tx_restart(struct macb_queue *queue) 1673 { 1674 unsigned int head = queue->tx_head; 1675 unsigned int tail = queue->tx_tail; 1676 struct macb *bp = queue->bp; 1677 1678 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1679 queue_writel(queue, ISR, MACB_BIT(TXUBR)); 1680 1681 if (head == tail) 1682 return; 1683 1684 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1685 } 1686 1687 static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) 1688 { 1689 struct macb_queue *queue = dev_id; 1690 struct macb *bp = queue->bp; 1691 u32 status; 1692 1693 status = queue_readl(queue, ISR); 1694 1695 if (unlikely(!status)) 1696 return IRQ_NONE; 1697 1698 spin_lock(&bp->lock); 1699 1700 if (status & MACB_BIT(WOL)) { 1701 queue_writel(queue, IDR, MACB_BIT(WOL)); 1702 macb_writel(bp, WOL, 0); 1703 netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", 1704 (unsigned int)(queue - bp->queues), 1705 (unsigned long)status); 1706 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1707 queue_writel(queue, ISR, MACB_BIT(WOL)); 1708 pm_wakeup_event(&bp->pdev->dev, 0); 1709 } 1710 1711 spin_unlock(&bp->lock); 1712 1713 return IRQ_HANDLED; 1714 } 1715 1716 static irqreturn_t gem_wol_interrupt(int irq, void *dev_id) 1717 { 1718 struct macb_queue *queue = dev_id; 1719 struct macb *bp = queue->bp; 1720 u32 status; 1721 1722 status = queue_readl(queue, ISR); 1723 1724 if (unlikely(!status)) 1725 return IRQ_NONE; 1726 1727 spin_lock(&bp->lock); 1728 1729 if (status & GEM_BIT(WOL)) { 1730 queue_writel(queue, IDR, GEM_BIT(WOL)); 1731 gem_writel(bp, WOL, 0); 1732 netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", 1733 (unsigned int)(queue - bp->queues), 1734 (unsigned long)status); 1735 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1736 queue_writel(queue, ISR, GEM_BIT(WOL)); 1737 pm_wakeup_event(&bp->pdev->dev, 0); 1738 } 1739 1740 spin_unlock(&bp->lock); 1741 1742 return IRQ_HANDLED; 1743 } 1744 1745 static irqreturn_t macb_interrupt(int irq, void *dev_id) 1746 { 1747 struct macb_queue *queue = dev_id; 1748 struct macb *bp = queue->bp; 1749 struct net_device *dev = bp->dev; 1750 u32 status, ctrl; 1751 1752 status = queue_readl(queue, ISR); 1753 1754 if (unlikely(!status)) 1755 return IRQ_NONE; 1756 1757 spin_lock(&bp->lock); 1758 1759 while (status) { 1760 /* close possible race with dev_close */ 1761 if (unlikely(!netif_running(dev))) { 1762 queue_writel(queue, IDR, -1); 1763 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1764 queue_writel(queue, ISR, -1); 1765 break; 1766 } 1767 1768 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1769 (unsigned int)(queue - bp->queues), 1770 (unsigned long)status); 1771 1772 if (status & bp->rx_intr_mask) { 1773 /* There's no point taking any more interrupts 1774 * until we have processed the buffers. The 1775 * scheduling call may fail if the poll routine 1776 * is already scheduled, so disable interrupts 1777 * now. 1778 */ 1779 queue_writel(queue, IDR, bp->rx_intr_mask); 1780 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1781 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1782 1783 if (napi_schedule_prep(&queue->napi)) { 1784 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1785 __napi_schedule(&queue->napi); 1786 } 1787 } 1788 1789 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 1790 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 1791 schedule_work(&queue->tx_error_task); 1792 1793 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1794 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 1795 1796 break; 1797 } 1798 1799 if (status & MACB_BIT(TCOMP)) 1800 macb_tx_interrupt(queue); 1801 1802 if (status & MACB_BIT(TXUBR)) 1803 macb_tx_restart(queue); 1804 1805 /* Link change detection isn't possible with RMII, so we'll 1806 * add that if/when we get our hands on a full-blown MII PHY. 1807 */ 1808 1809 /* There is a hardware issue under heavy load where DMA can 1810 * stop, this causes endless "used buffer descriptor read" 1811 * interrupts but it can be cleared by re-enabling RX. See 1812 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 1813 * section 16.7.4 for details. RXUBR is only enabled for 1814 * these two versions. 1815 */ 1816 if (status & MACB_BIT(RXUBR)) { 1817 ctrl = macb_readl(bp, NCR); 1818 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1819 wmb(); 1820 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1821 1822 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1823 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 1824 } 1825 1826 if (status & MACB_BIT(ISR_ROVR)) { 1827 /* We missed at least one packet */ 1828 if (macb_is_gem(bp)) 1829 bp->hw_stats.gem.rx_overruns++; 1830 else 1831 bp->hw_stats.macb.rx_overruns++; 1832 1833 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1834 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 1835 } 1836 1837 if (status & MACB_BIT(HRESP)) { 1838 tasklet_schedule(&bp->hresp_err_tasklet); 1839 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 1840 1841 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1842 queue_writel(queue, ISR, MACB_BIT(HRESP)); 1843 } 1844 status = queue_readl(queue, ISR); 1845 } 1846 1847 spin_unlock(&bp->lock); 1848 1849 return IRQ_HANDLED; 1850 } 1851 1852 #ifdef CONFIG_NET_POLL_CONTROLLER 1853 /* Polling receive - used by netconsole and other diagnostic tools 1854 * to allow network i/o with interrupts disabled. 1855 */ 1856 static void macb_poll_controller(struct net_device *dev) 1857 { 1858 struct macb *bp = netdev_priv(dev); 1859 struct macb_queue *queue; 1860 unsigned long flags; 1861 unsigned int q; 1862 1863 local_irq_save(flags); 1864 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1865 macb_interrupt(dev->irq, queue); 1866 local_irq_restore(flags); 1867 } 1868 #endif 1869 1870 static unsigned int macb_tx_map(struct macb *bp, 1871 struct macb_queue *queue, 1872 struct sk_buff *skb, 1873 unsigned int hdrlen) 1874 { 1875 dma_addr_t mapping; 1876 unsigned int len, entry, i, tx_head = queue->tx_head; 1877 struct macb_tx_skb *tx_skb = NULL; 1878 struct macb_dma_desc *desc; 1879 unsigned int offset, size, count = 0; 1880 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 1881 unsigned int eof = 1, mss_mfs = 0; 1882 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 1883 1884 /* LSO */ 1885 if (skb_shinfo(skb)->gso_size != 0) { 1886 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 1887 /* UDP - UFO */ 1888 lso_ctrl = MACB_LSO_UFO_ENABLE; 1889 else 1890 /* TCP - TSO */ 1891 lso_ctrl = MACB_LSO_TSO_ENABLE; 1892 } 1893 1894 /* First, map non-paged data */ 1895 len = skb_headlen(skb); 1896 1897 /* first buffer length */ 1898 size = hdrlen; 1899 1900 offset = 0; 1901 while (len) { 1902 entry = macb_tx_ring_wrap(bp, tx_head); 1903 tx_skb = &queue->tx_skb[entry]; 1904 1905 mapping = dma_map_single(&bp->pdev->dev, 1906 skb->data + offset, 1907 size, DMA_TO_DEVICE); 1908 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1909 goto dma_error; 1910 1911 /* Save info to properly release resources */ 1912 tx_skb->skb = NULL; 1913 tx_skb->mapping = mapping; 1914 tx_skb->size = size; 1915 tx_skb->mapped_as_page = false; 1916 1917 len -= size; 1918 offset += size; 1919 count++; 1920 tx_head++; 1921 1922 size = min(len, bp->max_tx_length); 1923 } 1924 1925 /* Then, map paged data from fragments */ 1926 for (f = 0; f < nr_frags; f++) { 1927 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 1928 1929 len = skb_frag_size(frag); 1930 offset = 0; 1931 while (len) { 1932 size = min(len, bp->max_tx_length); 1933 entry = macb_tx_ring_wrap(bp, tx_head); 1934 tx_skb = &queue->tx_skb[entry]; 1935 1936 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 1937 offset, size, DMA_TO_DEVICE); 1938 if (dma_mapping_error(&bp->pdev->dev, mapping)) 1939 goto dma_error; 1940 1941 /* Save info to properly release resources */ 1942 tx_skb->skb = NULL; 1943 tx_skb->mapping = mapping; 1944 tx_skb->size = size; 1945 tx_skb->mapped_as_page = true; 1946 1947 len -= size; 1948 offset += size; 1949 count++; 1950 tx_head++; 1951 } 1952 } 1953 1954 /* Should never happen */ 1955 if (unlikely(!tx_skb)) { 1956 netdev_err(bp->dev, "BUG! empty skb!\n"); 1957 return 0; 1958 } 1959 1960 /* This is the last buffer of the frame: save socket buffer */ 1961 tx_skb->skb = skb; 1962 1963 /* Update TX ring: update buffer descriptors in reverse order 1964 * to avoid race condition 1965 */ 1966 1967 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 1968 * to set the end of TX queue 1969 */ 1970 i = tx_head; 1971 entry = macb_tx_ring_wrap(bp, i); 1972 ctrl = MACB_BIT(TX_USED); 1973 desc = macb_tx_desc(queue, entry); 1974 desc->ctrl = ctrl; 1975 1976 if (lso_ctrl) { 1977 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 1978 /* include header and FCS in value given to h/w */ 1979 mss_mfs = skb_shinfo(skb)->gso_size + 1980 skb_transport_offset(skb) + 1981 ETH_FCS_LEN; 1982 else /* TSO */ { 1983 mss_mfs = skb_shinfo(skb)->gso_size; 1984 /* TCP Sequence Number Source Select 1985 * can be set only for TSO 1986 */ 1987 seq_ctrl = 0; 1988 } 1989 } 1990 1991 do { 1992 i--; 1993 entry = macb_tx_ring_wrap(bp, i); 1994 tx_skb = &queue->tx_skb[entry]; 1995 desc = macb_tx_desc(queue, entry); 1996 1997 ctrl = (u32)tx_skb->size; 1998 if (eof) { 1999 ctrl |= MACB_BIT(TX_LAST); 2000 eof = 0; 2001 } 2002 if (unlikely(entry == (bp->tx_ring_size - 1))) 2003 ctrl |= MACB_BIT(TX_WRAP); 2004 2005 /* First descriptor is header descriptor */ 2006 if (i == queue->tx_head) { 2007 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 2008 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 2009 if ((bp->dev->features & NETIF_F_HW_CSUM) && 2010 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) 2011 ctrl |= MACB_BIT(TX_NOCRC); 2012 } else 2013 /* Only set MSS/MFS on payload descriptors 2014 * (second or later descriptor) 2015 */ 2016 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 2017 2018 /* Set TX buffer descriptor */ 2019 macb_set_addr(bp, desc, tx_skb->mapping); 2020 /* desc->addr must be visible to hardware before clearing 2021 * 'TX_USED' bit in desc->ctrl. 2022 */ 2023 wmb(); 2024 desc->ctrl = ctrl; 2025 } while (i != queue->tx_head); 2026 2027 queue->tx_head = tx_head; 2028 2029 return count; 2030 2031 dma_error: 2032 netdev_err(bp->dev, "TX DMA map failed\n"); 2033 2034 for (i = queue->tx_head; i != tx_head; i++) { 2035 tx_skb = macb_tx_skb(queue, i); 2036 2037 macb_tx_unmap(bp, tx_skb); 2038 } 2039 2040 return 0; 2041 } 2042 2043 static netdev_features_t macb_features_check(struct sk_buff *skb, 2044 struct net_device *dev, 2045 netdev_features_t features) 2046 { 2047 unsigned int nr_frags, f; 2048 unsigned int hdrlen; 2049 2050 /* Validate LSO compatibility */ 2051 2052 /* there is only one buffer or protocol is not UDP */ 2053 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) 2054 return features; 2055 2056 /* length of header */ 2057 hdrlen = skb_transport_offset(skb); 2058 2059 /* For UFO only: 2060 * When software supplies two or more payload buffers all payload buffers 2061 * apart from the last must be a multiple of 8 bytes in size. 2062 */ 2063 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 2064 return features & ~MACB_NETIF_LSO; 2065 2066 nr_frags = skb_shinfo(skb)->nr_frags; 2067 /* No need to check last fragment */ 2068 nr_frags--; 2069 for (f = 0; f < nr_frags; f++) { 2070 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2071 2072 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 2073 return features & ~MACB_NETIF_LSO; 2074 } 2075 return features; 2076 } 2077 2078 static inline int macb_clear_csum(struct sk_buff *skb) 2079 { 2080 /* no change for packets without checksum offloading */ 2081 if (skb->ip_summed != CHECKSUM_PARTIAL) 2082 return 0; 2083 2084 /* make sure we can modify the header */ 2085 if (unlikely(skb_cow_head(skb, 0))) 2086 return -1; 2087 2088 /* initialize checksum field 2089 * This is required - at least for Zynq, which otherwise calculates 2090 * wrong UDP header checksums for UDP packets with UDP data len <=2 2091 */ 2092 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 2093 return 0; 2094 } 2095 2096 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 2097 { 2098 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || 2099 skb_is_nonlinear(*skb); 2100 int padlen = ETH_ZLEN - (*skb)->len; 2101 int headroom = skb_headroom(*skb); 2102 int tailroom = skb_tailroom(*skb); 2103 struct sk_buff *nskb; 2104 u32 fcs; 2105 2106 if (!(ndev->features & NETIF_F_HW_CSUM) || 2107 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 2108 skb_shinfo(*skb)->gso_size) /* Not available for GSO */ 2109 return 0; 2110 2111 if (padlen <= 0) { 2112 /* FCS could be appeded to tailroom. */ 2113 if (tailroom >= ETH_FCS_LEN) 2114 goto add_fcs; 2115 /* FCS could be appeded by moving data to headroom. */ 2116 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN) 2117 padlen = 0; 2118 /* No room for FCS, need to reallocate skb. */ 2119 else 2120 padlen = ETH_FCS_LEN; 2121 } else { 2122 /* Add room for FCS. */ 2123 padlen += ETH_FCS_LEN; 2124 } 2125 2126 if (!cloned && headroom + tailroom >= padlen) { 2127 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len); 2128 skb_set_tail_pointer(*skb, (*skb)->len); 2129 } else { 2130 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 2131 if (!nskb) 2132 return -ENOMEM; 2133 2134 dev_consume_skb_any(*skb); 2135 *skb = nskb; 2136 } 2137 2138 if (padlen > ETH_FCS_LEN) 2139 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 2140 2141 add_fcs: 2142 /* set FCS to packet */ 2143 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 2144 fcs = ~fcs; 2145 2146 skb_put_u8(*skb, fcs & 0xff); 2147 skb_put_u8(*skb, (fcs >> 8) & 0xff); 2148 skb_put_u8(*skb, (fcs >> 16) & 0xff); 2149 skb_put_u8(*skb, (fcs >> 24) & 0xff); 2150 2151 return 0; 2152 } 2153 2154 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 2155 { 2156 u16 queue_index = skb_get_queue_mapping(skb); 2157 struct macb *bp = netdev_priv(dev); 2158 struct macb_queue *queue = &bp->queues[queue_index]; 2159 unsigned long flags; 2160 unsigned int desc_cnt, nr_frags, frag_size, f; 2161 unsigned int hdrlen; 2162 bool is_lso; 2163 netdev_tx_t ret = NETDEV_TX_OK; 2164 2165 if (macb_clear_csum(skb)) { 2166 dev_kfree_skb_any(skb); 2167 return ret; 2168 } 2169 2170 if (macb_pad_and_fcs(&skb, dev)) { 2171 dev_kfree_skb_any(skb); 2172 return ret; 2173 } 2174 2175 is_lso = (skb_shinfo(skb)->gso_size != 0); 2176 2177 if (is_lso) { 2178 /* length of headers */ 2179 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2180 /* only queue eth + ip headers separately for UDP */ 2181 hdrlen = skb_transport_offset(skb); 2182 else 2183 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); 2184 if (skb_headlen(skb) < hdrlen) { 2185 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 2186 /* if this is required, would need to copy to single buffer */ 2187 return NETDEV_TX_BUSY; 2188 } 2189 } else 2190 hdrlen = min(skb_headlen(skb), bp->max_tx_length); 2191 2192 #if defined(DEBUG) && defined(VERBOSE_DEBUG) 2193 netdev_vdbg(bp->dev, 2194 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 2195 queue_index, skb->len, skb->head, skb->data, 2196 skb_tail_pointer(skb), skb_end_pointer(skb)); 2197 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 2198 skb->data, 16, true); 2199 #endif 2200 2201 /* Count how many TX buffer descriptors are needed to send this 2202 * socket buffer: skb fragments of jumbo frames may need to be 2203 * split into many buffer descriptors. 2204 */ 2205 if (is_lso && (skb_headlen(skb) > hdrlen)) 2206 /* extra header descriptor if also payload in first buffer */ 2207 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 2208 else 2209 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 2210 nr_frags = skb_shinfo(skb)->nr_frags; 2211 for (f = 0; f < nr_frags; f++) { 2212 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 2213 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 2214 } 2215 2216 spin_lock_irqsave(&bp->lock, flags); 2217 2218 /* This is a hard error, log it. */ 2219 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 2220 bp->tx_ring_size) < desc_cnt) { 2221 netif_stop_subqueue(dev, queue_index); 2222 spin_unlock_irqrestore(&bp->lock, flags); 2223 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 2224 queue->tx_head, queue->tx_tail); 2225 return NETDEV_TX_BUSY; 2226 } 2227 2228 /* Map socket buffer for DMA transfer */ 2229 if (!macb_tx_map(bp, queue, skb, hdrlen)) { 2230 dev_kfree_skb_any(skb); 2231 goto unlock; 2232 } 2233 2234 /* Make newly initialized descriptor visible to hardware */ 2235 wmb(); 2236 skb_tx_timestamp(skb); 2237 2238 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 2239 2240 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 2241 netif_stop_subqueue(dev, queue_index); 2242 2243 unlock: 2244 spin_unlock_irqrestore(&bp->lock, flags); 2245 2246 return ret; 2247 } 2248 2249 static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 2250 { 2251 if (!macb_is_gem(bp)) { 2252 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 2253 } else { 2254 bp->rx_buffer_size = size; 2255 2256 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 2257 netdev_dbg(bp->dev, 2258 "RX buffer must be multiple of %d bytes, expanding\n", 2259 RX_BUFFER_MULTIPLE); 2260 bp->rx_buffer_size = 2261 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 2262 } 2263 } 2264 2265 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 2266 bp->dev->mtu, bp->rx_buffer_size); 2267 } 2268 2269 static void gem_free_rx_buffers(struct macb *bp) 2270 { 2271 struct sk_buff *skb; 2272 struct macb_dma_desc *desc; 2273 struct macb_queue *queue; 2274 dma_addr_t addr; 2275 unsigned int q; 2276 int i; 2277 2278 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2279 if (!queue->rx_skbuff) 2280 continue; 2281 2282 for (i = 0; i < bp->rx_ring_size; i++) { 2283 skb = queue->rx_skbuff[i]; 2284 2285 if (!skb) 2286 continue; 2287 2288 desc = macb_rx_desc(queue, i); 2289 addr = macb_get_addr(bp, desc); 2290 2291 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 2292 DMA_FROM_DEVICE); 2293 dev_kfree_skb_any(skb); 2294 skb = NULL; 2295 } 2296 2297 kfree(queue->rx_skbuff); 2298 queue->rx_skbuff = NULL; 2299 } 2300 } 2301 2302 static void macb_free_rx_buffers(struct macb *bp) 2303 { 2304 struct macb_queue *queue = &bp->queues[0]; 2305 2306 if (queue->rx_buffers) { 2307 dma_free_coherent(&bp->pdev->dev, 2308 bp->rx_ring_size * bp->rx_buffer_size, 2309 queue->rx_buffers, queue->rx_buffers_dma); 2310 queue->rx_buffers = NULL; 2311 } 2312 } 2313 2314 static void macb_free_consistent(struct macb *bp) 2315 { 2316 struct macb_queue *queue; 2317 unsigned int q; 2318 int size; 2319 2320 bp->macbgem_ops.mog_free_rx_buffers(bp); 2321 2322 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2323 kfree(queue->tx_skb); 2324 queue->tx_skb = NULL; 2325 if (queue->tx_ring) { 2326 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2327 dma_free_coherent(&bp->pdev->dev, size, 2328 queue->tx_ring, queue->tx_ring_dma); 2329 queue->tx_ring = NULL; 2330 } 2331 if (queue->rx_ring) { 2332 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2333 dma_free_coherent(&bp->pdev->dev, size, 2334 queue->rx_ring, queue->rx_ring_dma); 2335 queue->rx_ring = NULL; 2336 } 2337 } 2338 } 2339 2340 static int gem_alloc_rx_buffers(struct macb *bp) 2341 { 2342 struct macb_queue *queue; 2343 unsigned int q; 2344 int size; 2345 2346 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2347 size = bp->rx_ring_size * sizeof(struct sk_buff *); 2348 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 2349 if (!queue->rx_skbuff) 2350 return -ENOMEM; 2351 else 2352 netdev_dbg(bp->dev, 2353 "Allocated %d RX struct sk_buff entries at %p\n", 2354 bp->rx_ring_size, queue->rx_skbuff); 2355 } 2356 return 0; 2357 } 2358 2359 static int macb_alloc_rx_buffers(struct macb *bp) 2360 { 2361 struct macb_queue *queue = &bp->queues[0]; 2362 int size; 2363 2364 size = bp->rx_ring_size * bp->rx_buffer_size; 2365 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 2366 &queue->rx_buffers_dma, GFP_KERNEL); 2367 if (!queue->rx_buffers) 2368 return -ENOMEM; 2369 2370 netdev_dbg(bp->dev, 2371 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 2372 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 2373 return 0; 2374 } 2375 2376 static int macb_alloc_consistent(struct macb *bp) 2377 { 2378 struct macb_queue *queue; 2379 unsigned int q; 2380 int size; 2381 2382 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2383 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch; 2384 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2385 &queue->tx_ring_dma, 2386 GFP_KERNEL); 2387 if (!queue->tx_ring) 2388 goto out_err; 2389 netdev_dbg(bp->dev, 2390 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", 2391 q, size, (unsigned long)queue->tx_ring_dma, 2392 queue->tx_ring); 2393 2394 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2395 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2396 if (!queue->tx_skb) 2397 goto out_err; 2398 2399 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch; 2400 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size, 2401 &queue->rx_ring_dma, GFP_KERNEL); 2402 if (!queue->rx_ring) 2403 goto out_err; 2404 netdev_dbg(bp->dev, 2405 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", 2406 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); 2407 } 2408 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2409 goto out_err; 2410 2411 return 0; 2412 2413 out_err: 2414 macb_free_consistent(bp); 2415 return -ENOMEM; 2416 } 2417 2418 static void gem_init_rings(struct macb *bp) 2419 { 2420 struct macb_queue *queue; 2421 struct macb_dma_desc *desc = NULL; 2422 unsigned int q; 2423 int i; 2424 2425 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2426 for (i = 0; i < bp->tx_ring_size; i++) { 2427 desc = macb_tx_desc(queue, i); 2428 macb_set_addr(bp, desc, 0); 2429 desc->ctrl = MACB_BIT(TX_USED); 2430 } 2431 desc->ctrl |= MACB_BIT(TX_WRAP); 2432 queue->tx_head = 0; 2433 queue->tx_tail = 0; 2434 2435 queue->rx_tail = 0; 2436 queue->rx_prepared_head = 0; 2437 2438 gem_rx_refill(queue); 2439 } 2440 2441 } 2442 2443 static void macb_init_rings(struct macb *bp) 2444 { 2445 int i; 2446 struct macb_dma_desc *desc = NULL; 2447 2448 macb_init_rx_ring(&bp->queues[0]); 2449 2450 for (i = 0; i < bp->tx_ring_size; i++) { 2451 desc = macb_tx_desc(&bp->queues[0], i); 2452 macb_set_addr(bp, desc, 0); 2453 desc->ctrl = MACB_BIT(TX_USED); 2454 } 2455 bp->queues[0].tx_head = 0; 2456 bp->queues[0].tx_tail = 0; 2457 desc->ctrl |= MACB_BIT(TX_WRAP); 2458 } 2459 2460 static void macb_reset_hw(struct macb *bp) 2461 { 2462 struct macb_queue *queue; 2463 unsigned int q; 2464 u32 ctrl = macb_readl(bp, NCR); 2465 2466 /* Disable RX and TX (XXX: Should we halt the transmission 2467 * more gracefully?) 2468 */ 2469 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2470 2471 /* Clear the stats registers (XXX: Update stats first?) */ 2472 ctrl |= MACB_BIT(CLRSTAT); 2473 2474 macb_writel(bp, NCR, ctrl); 2475 2476 /* Clear all status flags */ 2477 macb_writel(bp, TSR, -1); 2478 macb_writel(bp, RSR, -1); 2479 2480 /* Disable all interrupts */ 2481 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2482 queue_writel(queue, IDR, -1); 2483 queue_readl(queue, ISR); 2484 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2485 queue_writel(queue, ISR, -1); 2486 } 2487 } 2488 2489 static u32 gem_mdc_clk_div(struct macb *bp) 2490 { 2491 u32 config; 2492 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2493 2494 if (pclk_hz <= 20000000) 2495 config = GEM_BF(CLK, GEM_CLK_DIV8); 2496 else if (pclk_hz <= 40000000) 2497 config = GEM_BF(CLK, GEM_CLK_DIV16); 2498 else if (pclk_hz <= 80000000) 2499 config = GEM_BF(CLK, GEM_CLK_DIV32); 2500 else if (pclk_hz <= 120000000) 2501 config = GEM_BF(CLK, GEM_CLK_DIV48); 2502 else if (pclk_hz <= 160000000) 2503 config = GEM_BF(CLK, GEM_CLK_DIV64); 2504 else 2505 config = GEM_BF(CLK, GEM_CLK_DIV96); 2506 2507 return config; 2508 } 2509 2510 static u32 macb_mdc_clk_div(struct macb *bp) 2511 { 2512 u32 config; 2513 unsigned long pclk_hz; 2514 2515 if (macb_is_gem(bp)) 2516 return gem_mdc_clk_div(bp); 2517 2518 pclk_hz = clk_get_rate(bp->pclk); 2519 if (pclk_hz <= 20000000) 2520 config = MACB_BF(CLK, MACB_CLK_DIV8); 2521 else if (pclk_hz <= 40000000) 2522 config = MACB_BF(CLK, MACB_CLK_DIV16); 2523 else if (pclk_hz <= 80000000) 2524 config = MACB_BF(CLK, MACB_CLK_DIV32); 2525 else 2526 config = MACB_BF(CLK, MACB_CLK_DIV64); 2527 2528 return config; 2529 } 2530 2531 /* Get the DMA bus width field of the network configuration register that we 2532 * should program. We find the width from decoding the design configuration 2533 * register to find the maximum supported data bus width. 2534 */ 2535 static u32 macb_dbw(struct macb *bp) 2536 { 2537 if (!macb_is_gem(bp)) 2538 return 0; 2539 2540 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2541 case 4: 2542 return GEM_BF(DBW, GEM_DBW128); 2543 case 2: 2544 return GEM_BF(DBW, GEM_DBW64); 2545 case 1: 2546 default: 2547 return GEM_BF(DBW, GEM_DBW32); 2548 } 2549 } 2550 2551 /* Configure the receive DMA engine 2552 * - use the correct receive buffer size 2553 * - set best burst length for DMA operations 2554 * (if not supported by FIFO, it will fallback to default) 2555 * - set both rx/tx packet buffers to full memory size 2556 * These are configurable parameters for GEM. 2557 */ 2558 static void macb_configure_dma(struct macb *bp) 2559 { 2560 struct macb_queue *queue; 2561 u32 buffer_size; 2562 unsigned int q; 2563 u32 dmacfg; 2564 2565 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2566 if (macb_is_gem(bp)) { 2567 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2568 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2569 if (q) 2570 queue_writel(queue, RBQS, buffer_size); 2571 else 2572 dmacfg |= GEM_BF(RXBS, buffer_size); 2573 } 2574 if (bp->dma_burst_length) 2575 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2576 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2577 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2578 2579 if (bp->native_io) 2580 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2581 else 2582 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2583 2584 if (bp->dev->features & NETIF_F_HW_CSUM) 2585 dmacfg |= GEM_BIT(TXCOEN); 2586 else 2587 dmacfg &= ~GEM_BIT(TXCOEN); 2588 2589 dmacfg &= ~GEM_BIT(ADDR64); 2590 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 2591 if (bp->hw_dma_cap & HW_DMA_CAP_64B) 2592 dmacfg |= GEM_BIT(ADDR64); 2593 #endif 2594 #ifdef CONFIG_MACB_USE_HWSTAMP 2595 if (bp->hw_dma_cap & HW_DMA_CAP_PTP) 2596 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2597 #endif 2598 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2599 dmacfg); 2600 gem_writel(bp, DMACFG, dmacfg); 2601 } 2602 } 2603 2604 static void macb_init_hw(struct macb *bp) 2605 { 2606 u32 config; 2607 2608 macb_reset_hw(bp); 2609 macb_set_hwaddr(bp); 2610 2611 config = macb_mdc_clk_div(bp); 2612 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ 2613 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2614 if (bp->caps & MACB_CAPS_JUMBO) 2615 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2616 else 2617 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2618 if (bp->dev->flags & IFF_PROMISC) 2619 config |= MACB_BIT(CAF); /* Copy All Frames */ 2620 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2621 config |= GEM_BIT(RXCOEN); 2622 if (!(bp->dev->flags & IFF_BROADCAST)) 2623 config |= MACB_BIT(NBC); /* No BroadCast */ 2624 config |= macb_dbw(bp); 2625 macb_writel(bp, NCFGR, config); 2626 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2627 gem_writel(bp, JML, bp->jumbo_max_len); 2628 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2629 if (bp->caps & MACB_CAPS_JUMBO) 2630 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2631 2632 macb_configure_dma(bp); 2633 } 2634 2635 /* The hash address register is 64 bits long and takes up two 2636 * locations in the memory map. The least significant bits are stored 2637 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2638 * 2639 * The unicast hash enable and the multicast hash enable bits in the 2640 * network configuration register enable the reception of hash matched 2641 * frames. The destination address is reduced to a 6 bit index into 2642 * the 64 bit hash register using the following hash function. The 2643 * hash function is an exclusive or of every sixth bit of the 2644 * destination address. 2645 * 2646 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2647 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2648 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2649 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2650 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2651 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2652 * 2653 * da[0] represents the least significant bit of the first byte 2654 * received, that is, the multicast/unicast indicator, and da[47] 2655 * represents the most significant bit of the last byte received. If 2656 * the hash index, hi[n], points to a bit that is set in the hash 2657 * register then the frame will be matched according to whether the 2658 * frame is multicast or unicast. A multicast match will be signalled 2659 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2660 * index points to a bit set in the hash register. A unicast match 2661 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2662 * and the hash index points to a bit set in the hash register. To 2663 * receive all multicast frames, the hash register should be set with 2664 * all ones and the multicast hash enable bit should be set in the 2665 * network configuration register. 2666 */ 2667 2668 static inline int hash_bit_value(int bitnr, __u8 *addr) 2669 { 2670 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2671 return 1; 2672 return 0; 2673 } 2674 2675 /* Return the hash index value for the specified address. */ 2676 static int hash_get_index(__u8 *addr) 2677 { 2678 int i, j, bitval; 2679 int hash_index = 0; 2680 2681 for (j = 0; j < 6; j++) { 2682 for (i = 0, bitval = 0; i < 8; i++) 2683 bitval ^= hash_bit_value(i * 6 + j, addr); 2684 2685 hash_index |= (bitval << j); 2686 } 2687 2688 return hash_index; 2689 } 2690 2691 /* Add multicast addresses to the internal multicast-hash table. */ 2692 static void macb_sethashtable(struct net_device *dev) 2693 { 2694 struct netdev_hw_addr *ha; 2695 unsigned long mc_filter[2]; 2696 unsigned int bitnr; 2697 struct macb *bp = netdev_priv(dev); 2698 2699 mc_filter[0] = 0; 2700 mc_filter[1] = 0; 2701 2702 netdev_for_each_mc_addr(ha, dev) { 2703 bitnr = hash_get_index(ha->addr); 2704 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2705 } 2706 2707 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2708 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2709 } 2710 2711 /* Enable/Disable promiscuous and multicast modes. */ 2712 static void macb_set_rx_mode(struct net_device *dev) 2713 { 2714 unsigned long cfg; 2715 struct macb *bp = netdev_priv(dev); 2716 2717 cfg = macb_readl(bp, NCFGR); 2718 2719 if (dev->flags & IFF_PROMISC) { 2720 /* Enable promiscuous mode */ 2721 cfg |= MACB_BIT(CAF); 2722 2723 /* Disable RX checksum offload */ 2724 if (macb_is_gem(bp)) 2725 cfg &= ~GEM_BIT(RXCOEN); 2726 } else { 2727 /* Disable promiscuous mode */ 2728 cfg &= ~MACB_BIT(CAF); 2729 2730 /* Enable RX checksum offload only if requested */ 2731 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 2732 cfg |= GEM_BIT(RXCOEN); 2733 } 2734 2735 if (dev->flags & IFF_ALLMULTI) { 2736 /* Enable all multicast mode */ 2737 macb_or_gem_writel(bp, HRB, -1); 2738 macb_or_gem_writel(bp, HRT, -1); 2739 cfg |= MACB_BIT(NCFGR_MTI); 2740 } else if (!netdev_mc_empty(dev)) { 2741 /* Enable specific multicasts */ 2742 macb_sethashtable(dev); 2743 cfg |= MACB_BIT(NCFGR_MTI); 2744 } else if (dev->flags & (~IFF_ALLMULTI)) { 2745 /* Disable all multicast mode */ 2746 macb_or_gem_writel(bp, HRB, 0); 2747 macb_or_gem_writel(bp, HRT, 0); 2748 cfg &= ~MACB_BIT(NCFGR_MTI); 2749 } 2750 2751 macb_writel(bp, NCFGR, cfg); 2752 } 2753 2754 static int macb_open(struct net_device *dev) 2755 { 2756 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 2757 struct macb *bp = netdev_priv(dev); 2758 struct macb_queue *queue; 2759 unsigned int q; 2760 int err; 2761 2762 netdev_dbg(bp->dev, "open\n"); 2763 2764 err = pm_runtime_get_sync(&bp->pdev->dev); 2765 if (err < 0) 2766 goto pm_exit; 2767 2768 /* RX buffers initialization */ 2769 macb_init_rx_buffer_size(bp, bufsz); 2770 2771 err = macb_alloc_consistent(bp); 2772 if (err) { 2773 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 2774 err); 2775 goto pm_exit; 2776 } 2777 2778 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2779 napi_enable(&queue->napi); 2780 2781 macb_init_hw(bp); 2782 2783 err = macb_phylink_connect(bp); 2784 if (err) 2785 goto reset_hw; 2786 2787 netif_tx_start_all_queues(dev); 2788 2789 if (bp->ptp_info) 2790 bp->ptp_info->ptp_init(dev); 2791 2792 return 0; 2793 2794 reset_hw: 2795 macb_reset_hw(bp); 2796 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2797 napi_disable(&queue->napi); 2798 macb_free_consistent(bp); 2799 pm_exit: 2800 pm_runtime_put_sync(&bp->pdev->dev); 2801 return err; 2802 } 2803 2804 static int macb_close(struct net_device *dev) 2805 { 2806 struct macb *bp = netdev_priv(dev); 2807 struct macb_queue *queue; 2808 unsigned long flags; 2809 unsigned int q; 2810 2811 netif_tx_stop_all_queues(dev); 2812 2813 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2814 napi_disable(&queue->napi); 2815 2816 phylink_stop(bp->phylink); 2817 phylink_disconnect_phy(bp->phylink); 2818 2819 spin_lock_irqsave(&bp->lock, flags); 2820 macb_reset_hw(bp); 2821 netif_carrier_off(dev); 2822 spin_unlock_irqrestore(&bp->lock, flags); 2823 2824 macb_free_consistent(bp); 2825 2826 if (bp->ptp_info) 2827 bp->ptp_info->ptp_remove(dev); 2828 2829 pm_runtime_put(&bp->pdev->dev); 2830 2831 return 0; 2832 } 2833 2834 static int macb_change_mtu(struct net_device *dev, int new_mtu) 2835 { 2836 if (netif_running(dev)) 2837 return -EBUSY; 2838 2839 dev->mtu = new_mtu; 2840 2841 return 0; 2842 } 2843 2844 static void gem_update_stats(struct macb *bp) 2845 { 2846 struct macb_queue *queue; 2847 unsigned int i, q, idx; 2848 unsigned long *stat; 2849 2850 u32 *p = &bp->hw_stats.gem.tx_octets_31_0; 2851 2852 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 2853 u32 offset = gem_statistics[i].offset; 2854 u64 val = bp->macb_reg_readl(bp, offset); 2855 2856 bp->ethtool_stats[i] += val; 2857 *p += val; 2858 2859 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 2860 /* Add GEM_OCTTXH, GEM_OCTRXH */ 2861 val = bp->macb_reg_readl(bp, offset + 4); 2862 bp->ethtool_stats[i] += ((u64)val) << 32; 2863 *(++p) += val; 2864 } 2865 } 2866 2867 idx = GEM_STATS_LEN; 2868 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2869 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 2870 bp->ethtool_stats[idx++] = *stat; 2871 } 2872 2873 static struct net_device_stats *gem_get_stats(struct macb *bp) 2874 { 2875 struct gem_stats *hwstat = &bp->hw_stats.gem; 2876 struct net_device_stats *nstat = &bp->dev->stats; 2877 2878 if (!netif_running(bp->dev)) 2879 return nstat; 2880 2881 gem_update_stats(bp); 2882 2883 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 2884 hwstat->rx_alignment_errors + 2885 hwstat->rx_resource_errors + 2886 hwstat->rx_overruns + 2887 hwstat->rx_oversize_frames + 2888 hwstat->rx_jabbers + 2889 hwstat->rx_undersized_frames + 2890 hwstat->rx_length_field_frame_errors); 2891 nstat->tx_errors = (hwstat->tx_late_collisions + 2892 hwstat->tx_excessive_collisions + 2893 hwstat->tx_underrun + 2894 hwstat->tx_carrier_sense_errors); 2895 nstat->multicast = hwstat->rx_multicast_frames; 2896 nstat->collisions = (hwstat->tx_single_collision_frames + 2897 hwstat->tx_multiple_collision_frames + 2898 hwstat->tx_excessive_collisions); 2899 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 2900 hwstat->rx_jabbers + 2901 hwstat->rx_undersized_frames + 2902 hwstat->rx_length_field_frame_errors); 2903 nstat->rx_over_errors = hwstat->rx_resource_errors; 2904 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 2905 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 2906 nstat->rx_fifo_errors = hwstat->rx_overruns; 2907 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 2908 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 2909 nstat->tx_fifo_errors = hwstat->tx_underrun; 2910 2911 return nstat; 2912 } 2913 2914 static void gem_get_ethtool_stats(struct net_device *dev, 2915 struct ethtool_stats *stats, u64 *data) 2916 { 2917 struct macb *bp; 2918 2919 bp = netdev_priv(dev); 2920 gem_update_stats(bp); 2921 memcpy(data, &bp->ethtool_stats, sizeof(u64) 2922 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); 2923 } 2924 2925 static int gem_get_sset_count(struct net_device *dev, int sset) 2926 { 2927 struct macb *bp = netdev_priv(dev); 2928 2929 switch (sset) { 2930 case ETH_SS_STATS: 2931 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 2932 default: 2933 return -EOPNOTSUPP; 2934 } 2935 } 2936 2937 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 2938 { 2939 char stat_string[ETH_GSTRING_LEN]; 2940 struct macb *bp = netdev_priv(dev); 2941 struct macb_queue *queue; 2942 unsigned int i; 2943 unsigned int q; 2944 2945 switch (sset) { 2946 case ETH_SS_STATS: 2947 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 2948 memcpy(p, gem_statistics[i].stat_string, 2949 ETH_GSTRING_LEN); 2950 2951 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2952 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 2953 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 2954 q, queue_statistics[i].stat_string); 2955 memcpy(p, stat_string, ETH_GSTRING_LEN); 2956 } 2957 } 2958 break; 2959 } 2960 } 2961 2962 static struct net_device_stats *macb_get_stats(struct net_device *dev) 2963 { 2964 struct macb *bp = netdev_priv(dev); 2965 struct net_device_stats *nstat = &bp->dev->stats; 2966 struct macb_stats *hwstat = &bp->hw_stats.macb; 2967 2968 if (macb_is_gem(bp)) 2969 return gem_get_stats(bp); 2970 2971 /* read stats from hardware */ 2972 macb_update_stats(bp); 2973 2974 /* Convert HW stats into netdevice stats */ 2975 nstat->rx_errors = (hwstat->rx_fcs_errors + 2976 hwstat->rx_align_errors + 2977 hwstat->rx_resource_errors + 2978 hwstat->rx_overruns + 2979 hwstat->rx_oversize_pkts + 2980 hwstat->rx_jabbers + 2981 hwstat->rx_undersize_pkts + 2982 hwstat->rx_length_mismatch); 2983 nstat->tx_errors = (hwstat->tx_late_cols + 2984 hwstat->tx_excessive_cols + 2985 hwstat->tx_underruns + 2986 hwstat->tx_carrier_errors + 2987 hwstat->sqe_test_errors); 2988 nstat->collisions = (hwstat->tx_single_cols + 2989 hwstat->tx_multiple_cols + 2990 hwstat->tx_excessive_cols); 2991 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 2992 hwstat->rx_jabbers + 2993 hwstat->rx_undersize_pkts + 2994 hwstat->rx_length_mismatch); 2995 nstat->rx_over_errors = hwstat->rx_resource_errors + 2996 hwstat->rx_overruns; 2997 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 2998 nstat->rx_frame_errors = hwstat->rx_align_errors; 2999 nstat->rx_fifo_errors = hwstat->rx_overruns; 3000 /* XXX: What does "missed" mean? */ 3001 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 3002 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 3003 nstat->tx_fifo_errors = hwstat->tx_underruns; 3004 /* Don't know about heartbeat or window errors... */ 3005 3006 return nstat; 3007 } 3008 3009 static int macb_get_regs_len(struct net_device *netdev) 3010 { 3011 return MACB_GREGS_NBR * sizeof(u32); 3012 } 3013 3014 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 3015 void *p) 3016 { 3017 struct macb *bp = netdev_priv(dev); 3018 unsigned int tail, head; 3019 u32 *regs_buff = p; 3020 3021 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 3022 | MACB_GREGS_VERSION; 3023 3024 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 3025 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 3026 3027 regs_buff[0] = macb_readl(bp, NCR); 3028 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 3029 regs_buff[2] = macb_readl(bp, NSR); 3030 regs_buff[3] = macb_readl(bp, TSR); 3031 regs_buff[4] = macb_readl(bp, RBQP); 3032 regs_buff[5] = macb_readl(bp, TBQP); 3033 regs_buff[6] = macb_readl(bp, RSR); 3034 regs_buff[7] = macb_readl(bp, IMR); 3035 3036 regs_buff[8] = tail; 3037 regs_buff[9] = head; 3038 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 3039 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 3040 3041 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 3042 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 3043 if (macb_is_gem(bp)) 3044 regs_buff[13] = gem_readl(bp, DMACFG); 3045 } 3046 3047 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 3048 { 3049 struct macb *bp = netdev_priv(netdev); 3050 3051 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { 3052 phylink_ethtool_get_wol(bp->phylink, wol); 3053 wol->supported |= WAKE_MAGIC; 3054 3055 if (bp->wol & MACB_WOL_ENABLED) 3056 wol->wolopts |= WAKE_MAGIC; 3057 } 3058 } 3059 3060 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 3061 { 3062 struct macb *bp = netdev_priv(netdev); 3063 int ret; 3064 3065 /* Pass the order to phylink layer */ 3066 ret = phylink_ethtool_set_wol(bp->phylink, wol); 3067 /* Don't manage WoL on MAC if handled by the PHY 3068 * or if there's a failure in talking to the PHY 3069 */ 3070 if (!ret || ret != -EOPNOTSUPP) 3071 return ret; 3072 3073 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || 3074 (wol->wolopts & ~WAKE_MAGIC)) 3075 return -EOPNOTSUPP; 3076 3077 if (wol->wolopts & WAKE_MAGIC) 3078 bp->wol |= MACB_WOL_ENABLED; 3079 else 3080 bp->wol &= ~MACB_WOL_ENABLED; 3081 3082 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); 3083 3084 return 0; 3085 } 3086 3087 static int macb_get_link_ksettings(struct net_device *netdev, 3088 struct ethtool_link_ksettings *kset) 3089 { 3090 struct macb *bp = netdev_priv(netdev); 3091 3092 return phylink_ethtool_ksettings_get(bp->phylink, kset); 3093 } 3094 3095 static int macb_set_link_ksettings(struct net_device *netdev, 3096 const struct ethtool_link_ksettings *kset) 3097 { 3098 struct macb *bp = netdev_priv(netdev); 3099 3100 return phylink_ethtool_ksettings_set(bp->phylink, kset); 3101 } 3102 3103 static void macb_get_ringparam(struct net_device *netdev, 3104 struct ethtool_ringparam *ring) 3105 { 3106 struct macb *bp = netdev_priv(netdev); 3107 3108 ring->rx_max_pending = MAX_RX_RING_SIZE; 3109 ring->tx_max_pending = MAX_TX_RING_SIZE; 3110 3111 ring->rx_pending = bp->rx_ring_size; 3112 ring->tx_pending = bp->tx_ring_size; 3113 } 3114 3115 static int macb_set_ringparam(struct net_device *netdev, 3116 struct ethtool_ringparam *ring) 3117 { 3118 struct macb *bp = netdev_priv(netdev); 3119 u32 new_rx_size, new_tx_size; 3120 unsigned int reset = 0; 3121 3122 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 3123 return -EINVAL; 3124 3125 new_rx_size = clamp_t(u32, ring->rx_pending, 3126 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 3127 new_rx_size = roundup_pow_of_two(new_rx_size); 3128 3129 new_tx_size = clamp_t(u32, ring->tx_pending, 3130 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 3131 new_tx_size = roundup_pow_of_two(new_tx_size); 3132 3133 if ((new_tx_size == bp->tx_ring_size) && 3134 (new_rx_size == bp->rx_ring_size)) { 3135 /* nothing to do */ 3136 return 0; 3137 } 3138 3139 if (netif_running(bp->dev)) { 3140 reset = 1; 3141 macb_close(bp->dev); 3142 } 3143 3144 bp->rx_ring_size = new_rx_size; 3145 bp->tx_ring_size = new_tx_size; 3146 3147 if (reset) 3148 macb_open(bp->dev); 3149 3150 return 0; 3151 } 3152 3153 #ifdef CONFIG_MACB_USE_HWSTAMP 3154 static unsigned int gem_get_tsu_rate(struct macb *bp) 3155 { 3156 struct clk *tsu_clk; 3157 unsigned int tsu_rate; 3158 3159 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 3160 if (!IS_ERR(tsu_clk)) 3161 tsu_rate = clk_get_rate(tsu_clk); 3162 /* try pclk instead */ 3163 else if (!IS_ERR(bp->pclk)) { 3164 tsu_clk = bp->pclk; 3165 tsu_rate = clk_get_rate(tsu_clk); 3166 } else 3167 return -ENOTSUPP; 3168 return tsu_rate; 3169 } 3170 3171 static s32 gem_get_ptp_max_adj(void) 3172 { 3173 return 64000000; 3174 } 3175 3176 static int gem_get_ts_info(struct net_device *dev, 3177 struct ethtool_ts_info *info) 3178 { 3179 struct macb *bp = netdev_priv(dev); 3180 3181 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { 3182 ethtool_op_get_ts_info(dev, info); 3183 return 0; 3184 } 3185 3186 info->so_timestamping = 3187 SOF_TIMESTAMPING_TX_SOFTWARE | 3188 SOF_TIMESTAMPING_RX_SOFTWARE | 3189 SOF_TIMESTAMPING_SOFTWARE | 3190 SOF_TIMESTAMPING_TX_HARDWARE | 3191 SOF_TIMESTAMPING_RX_HARDWARE | 3192 SOF_TIMESTAMPING_RAW_HARDWARE; 3193 info->tx_types = 3194 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 3195 (1 << HWTSTAMP_TX_OFF) | 3196 (1 << HWTSTAMP_TX_ON); 3197 info->rx_filters = 3198 (1 << HWTSTAMP_FILTER_NONE) | 3199 (1 << HWTSTAMP_FILTER_ALL); 3200 3201 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; 3202 3203 return 0; 3204 } 3205 3206 static struct macb_ptp_info gem_ptp_info = { 3207 .ptp_init = gem_ptp_init, 3208 .ptp_remove = gem_ptp_remove, 3209 .get_ptp_max_adj = gem_get_ptp_max_adj, 3210 .get_tsu_rate = gem_get_tsu_rate, 3211 .get_ts_info = gem_get_ts_info, 3212 .get_hwtst = gem_get_hwtst, 3213 .set_hwtst = gem_set_hwtst, 3214 }; 3215 #endif 3216 3217 static int macb_get_ts_info(struct net_device *netdev, 3218 struct ethtool_ts_info *info) 3219 { 3220 struct macb *bp = netdev_priv(netdev); 3221 3222 if (bp->ptp_info) 3223 return bp->ptp_info->get_ts_info(netdev, info); 3224 3225 return ethtool_op_get_ts_info(netdev, info); 3226 } 3227 3228 static void gem_enable_flow_filters(struct macb *bp, bool enable) 3229 { 3230 struct net_device *netdev = bp->dev; 3231 struct ethtool_rx_fs_item *item; 3232 u32 t2_scr; 3233 int num_t2_scr; 3234 3235 if (!(netdev->features & NETIF_F_NTUPLE)) 3236 return; 3237 3238 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 3239 3240 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3241 struct ethtool_rx_flow_spec *fs = &item->fs; 3242 struct ethtool_tcpip4_spec *tp4sp_m; 3243 3244 if (fs->location >= num_t2_scr) 3245 continue; 3246 3247 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 3248 3249 /* enable/disable screener regs for the flow entry */ 3250 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 3251 3252 /* only enable fields with no masking */ 3253 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3254 3255 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 3256 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 3257 else 3258 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 3259 3260 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 3261 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 3262 else 3263 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 3264 3265 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 3266 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 3267 else 3268 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 3269 3270 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 3271 } 3272 } 3273 3274 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 3275 { 3276 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 3277 uint16_t index = fs->location; 3278 u32 w0, w1, t2_scr; 3279 bool cmp_a = false; 3280 bool cmp_b = false; 3281 bool cmp_c = false; 3282 3283 if (!macb_is_gem(bp)) 3284 return; 3285 3286 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 3287 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3288 3289 /* ignore field if any masking set */ 3290 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 3291 /* 1st compare reg - IP source address */ 3292 w0 = 0; 3293 w1 = 0; 3294 w0 = tp4sp_v->ip4src; 3295 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3296 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3297 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 3298 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 3299 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 3300 cmp_a = true; 3301 } 3302 3303 /* ignore field if any masking set */ 3304 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 3305 /* 2nd compare reg - IP destination address */ 3306 w0 = 0; 3307 w1 = 0; 3308 w0 = tp4sp_v->ip4dst; 3309 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3310 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3311 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 3312 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 3313 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 3314 cmp_b = true; 3315 } 3316 3317 /* ignore both port fields if masking set in both */ 3318 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 3319 /* 3rd compare reg - source port, destination port */ 3320 w0 = 0; 3321 w1 = 0; 3322 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 3323 if (tp4sp_m->psrc == tp4sp_m->pdst) { 3324 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 3325 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3326 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3327 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3328 } else { 3329 /* only one port definition */ 3330 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 3331 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 3332 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 3333 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 3334 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3335 } else { /* dst port */ 3336 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3337 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 3338 } 3339 } 3340 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 3341 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 3342 cmp_c = true; 3343 } 3344 3345 t2_scr = 0; 3346 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 3347 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 3348 if (cmp_a) 3349 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 3350 if (cmp_b) 3351 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 3352 if (cmp_c) 3353 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 3354 gem_writel_n(bp, SCRT2, index, t2_scr); 3355 } 3356 3357 static int gem_add_flow_filter(struct net_device *netdev, 3358 struct ethtool_rxnfc *cmd) 3359 { 3360 struct macb *bp = netdev_priv(netdev); 3361 struct ethtool_rx_flow_spec *fs = &cmd->fs; 3362 struct ethtool_rx_fs_item *item, *newfs; 3363 unsigned long flags; 3364 int ret = -EINVAL; 3365 bool added = false; 3366 3367 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 3368 if (newfs == NULL) 3369 return -ENOMEM; 3370 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 3371 3372 netdev_dbg(netdev, 3373 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3374 fs->flow_type, (int)fs->ring_cookie, fs->location, 3375 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3376 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3377 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); 3378 3379 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3380 3381 /* find correct place to add in list */ 3382 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3383 if (item->fs.location > newfs->fs.location) { 3384 list_add_tail(&newfs->list, &item->list); 3385 added = true; 3386 break; 3387 } else if (item->fs.location == fs->location) { 3388 netdev_err(netdev, "Rule not added: location %d not free!\n", 3389 fs->location); 3390 ret = -EBUSY; 3391 goto err; 3392 } 3393 } 3394 if (!added) 3395 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3396 3397 gem_prog_cmp_regs(bp, fs); 3398 bp->rx_fs_list.count++; 3399 /* enable filtering if NTUPLE on */ 3400 gem_enable_flow_filters(bp, 1); 3401 3402 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3403 return 0; 3404 3405 err: 3406 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3407 kfree(newfs); 3408 return ret; 3409 } 3410 3411 static int gem_del_flow_filter(struct net_device *netdev, 3412 struct ethtool_rxnfc *cmd) 3413 { 3414 struct macb *bp = netdev_priv(netdev); 3415 struct ethtool_rx_fs_item *item; 3416 struct ethtool_rx_flow_spec *fs; 3417 unsigned long flags; 3418 3419 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3420 3421 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3422 if (item->fs.location == cmd->fs.location) { 3423 /* disable screener regs for the flow entry */ 3424 fs = &(item->fs); 3425 netdev_dbg(netdev, 3426 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3427 fs->flow_type, (int)fs->ring_cookie, fs->location, 3428 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3429 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3430 htons(fs->h_u.tcp_ip4_spec.psrc), 3431 htons(fs->h_u.tcp_ip4_spec.pdst)); 3432 3433 gem_writel_n(bp, SCRT2, fs->location, 0); 3434 3435 list_del(&item->list); 3436 bp->rx_fs_list.count--; 3437 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3438 kfree(item); 3439 return 0; 3440 } 3441 } 3442 3443 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3444 return -EINVAL; 3445 } 3446 3447 static int gem_get_flow_entry(struct net_device *netdev, 3448 struct ethtool_rxnfc *cmd) 3449 { 3450 struct macb *bp = netdev_priv(netdev); 3451 struct ethtool_rx_fs_item *item; 3452 3453 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3454 if (item->fs.location == cmd->fs.location) { 3455 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3456 return 0; 3457 } 3458 } 3459 return -EINVAL; 3460 } 3461 3462 static int gem_get_all_flow_entries(struct net_device *netdev, 3463 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3464 { 3465 struct macb *bp = netdev_priv(netdev); 3466 struct ethtool_rx_fs_item *item; 3467 uint32_t cnt = 0; 3468 3469 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3470 if (cnt == cmd->rule_cnt) 3471 return -EMSGSIZE; 3472 rule_locs[cnt] = item->fs.location; 3473 cnt++; 3474 } 3475 cmd->data = bp->max_tuples; 3476 cmd->rule_cnt = cnt; 3477 3478 return 0; 3479 } 3480 3481 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3482 u32 *rule_locs) 3483 { 3484 struct macb *bp = netdev_priv(netdev); 3485 int ret = 0; 3486 3487 switch (cmd->cmd) { 3488 case ETHTOOL_GRXRINGS: 3489 cmd->data = bp->num_queues; 3490 break; 3491 case ETHTOOL_GRXCLSRLCNT: 3492 cmd->rule_cnt = bp->rx_fs_list.count; 3493 break; 3494 case ETHTOOL_GRXCLSRULE: 3495 ret = gem_get_flow_entry(netdev, cmd); 3496 break; 3497 case ETHTOOL_GRXCLSRLALL: 3498 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3499 break; 3500 default: 3501 netdev_err(netdev, 3502 "Command parameter %d is not supported\n", cmd->cmd); 3503 ret = -EOPNOTSUPP; 3504 } 3505 3506 return ret; 3507 } 3508 3509 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3510 { 3511 struct macb *bp = netdev_priv(netdev); 3512 int ret; 3513 3514 switch (cmd->cmd) { 3515 case ETHTOOL_SRXCLSRLINS: 3516 if ((cmd->fs.location >= bp->max_tuples) 3517 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3518 ret = -EINVAL; 3519 break; 3520 } 3521 ret = gem_add_flow_filter(netdev, cmd); 3522 break; 3523 case ETHTOOL_SRXCLSRLDEL: 3524 ret = gem_del_flow_filter(netdev, cmd); 3525 break; 3526 default: 3527 netdev_err(netdev, 3528 "Command parameter %d is not supported\n", cmd->cmd); 3529 ret = -EOPNOTSUPP; 3530 } 3531 3532 return ret; 3533 } 3534 3535 static const struct ethtool_ops macb_ethtool_ops = { 3536 .get_regs_len = macb_get_regs_len, 3537 .get_regs = macb_get_regs, 3538 .get_link = ethtool_op_get_link, 3539 .get_ts_info = ethtool_op_get_ts_info, 3540 .get_wol = macb_get_wol, 3541 .set_wol = macb_set_wol, 3542 .get_link_ksettings = macb_get_link_ksettings, 3543 .set_link_ksettings = macb_set_link_ksettings, 3544 .get_ringparam = macb_get_ringparam, 3545 .set_ringparam = macb_set_ringparam, 3546 }; 3547 3548 static const struct ethtool_ops gem_ethtool_ops = { 3549 .get_regs_len = macb_get_regs_len, 3550 .get_regs = macb_get_regs, 3551 .get_wol = macb_get_wol, 3552 .set_wol = macb_set_wol, 3553 .get_link = ethtool_op_get_link, 3554 .get_ts_info = macb_get_ts_info, 3555 .get_ethtool_stats = gem_get_ethtool_stats, 3556 .get_strings = gem_get_ethtool_strings, 3557 .get_sset_count = gem_get_sset_count, 3558 .get_link_ksettings = macb_get_link_ksettings, 3559 .set_link_ksettings = macb_set_link_ksettings, 3560 .get_ringparam = macb_get_ringparam, 3561 .set_ringparam = macb_set_ringparam, 3562 .get_rxnfc = gem_get_rxnfc, 3563 .set_rxnfc = gem_set_rxnfc, 3564 }; 3565 3566 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3567 { 3568 struct macb *bp = netdev_priv(dev); 3569 3570 if (!netif_running(dev)) 3571 return -EINVAL; 3572 3573 if (bp->ptp_info) { 3574 switch (cmd) { 3575 case SIOCSHWTSTAMP: 3576 return bp->ptp_info->set_hwtst(dev, rq, cmd); 3577 case SIOCGHWTSTAMP: 3578 return bp->ptp_info->get_hwtst(dev, rq); 3579 } 3580 } 3581 3582 return phylink_mii_ioctl(bp->phylink, rq, cmd); 3583 } 3584 3585 static inline void macb_set_txcsum_feature(struct macb *bp, 3586 netdev_features_t features) 3587 { 3588 u32 val; 3589 3590 if (!macb_is_gem(bp)) 3591 return; 3592 3593 val = gem_readl(bp, DMACFG); 3594 if (features & NETIF_F_HW_CSUM) 3595 val |= GEM_BIT(TXCOEN); 3596 else 3597 val &= ~GEM_BIT(TXCOEN); 3598 3599 gem_writel(bp, DMACFG, val); 3600 } 3601 3602 static inline void macb_set_rxcsum_feature(struct macb *bp, 3603 netdev_features_t features) 3604 { 3605 struct net_device *netdev = bp->dev; 3606 u32 val; 3607 3608 if (!macb_is_gem(bp)) 3609 return; 3610 3611 val = gem_readl(bp, NCFGR); 3612 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 3613 val |= GEM_BIT(RXCOEN); 3614 else 3615 val &= ~GEM_BIT(RXCOEN); 3616 3617 gem_writel(bp, NCFGR, val); 3618 } 3619 3620 static inline void macb_set_rxflow_feature(struct macb *bp, 3621 netdev_features_t features) 3622 { 3623 if (!macb_is_gem(bp)) 3624 return; 3625 3626 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 3627 } 3628 3629 static int macb_set_features(struct net_device *netdev, 3630 netdev_features_t features) 3631 { 3632 struct macb *bp = netdev_priv(netdev); 3633 netdev_features_t changed = features ^ netdev->features; 3634 3635 /* TX checksum offload */ 3636 if (changed & NETIF_F_HW_CSUM) 3637 macb_set_txcsum_feature(bp, features); 3638 3639 /* RX checksum offload */ 3640 if (changed & NETIF_F_RXCSUM) 3641 macb_set_rxcsum_feature(bp, features); 3642 3643 /* RX Flow Filters */ 3644 if (changed & NETIF_F_NTUPLE) 3645 macb_set_rxflow_feature(bp, features); 3646 3647 return 0; 3648 } 3649 3650 static void macb_restore_features(struct macb *bp) 3651 { 3652 struct net_device *netdev = bp->dev; 3653 netdev_features_t features = netdev->features; 3654 struct ethtool_rx_fs_item *item; 3655 3656 /* TX checksum offload */ 3657 macb_set_txcsum_feature(bp, features); 3658 3659 /* RX checksum offload */ 3660 macb_set_rxcsum_feature(bp, features); 3661 3662 /* RX Flow Filters */ 3663 list_for_each_entry(item, &bp->rx_fs_list.list, list) 3664 gem_prog_cmp_regs(bp, &item->fs); 3665 3666 macb_set_rxflow_feature(bp, features); 3667 } 3668 3669 static const struct net_device_ops macb_netdev_ops = { 3670 .ndo_open = macb_open, 3671 .ndo_stop = macb_close, 3672 .ndo_start_xmit = macb_start_xmit, 3673 .ndo_set_rx_mode = macb_set_rx_mode, 3674 .ndo_get_stats = macb_get_stats, 3675 .ndo_eth_ioctl = macb_ioctl, 3676 .ndo_validate_addr = eth_validate_addr, 3677 .ndo_change_mtu = macb_change_mtu, 3678 .ndo_set_mac_address = eth_mac_addr, 3679 #ifdef CONFIG_NET_POLL_CONTROLLER 3680 .ndo_poll_controller = macb_poll_controller, 3681 #endif 3682 .ndo_set_features = macb_set_features, 3683 .ndo_features_check = macb_features_check, 3684 }; 3685 3686 /* Configure peripheral capabilities according to device tree 3687 * and integration options used 3688 */ 3689 static void macb_configure_caps(struct macb *bp, 3690 const struct macb_config *dt_conf) 3691 { 3692 u32 dcfg; 3693 3694 if (dt_conf) 3695 bp->caps = dt_conf->caps; 3696 3697 if (hw_is_gem(bp->regs, bp->native_io)) { 3698 bp->caps |= MACB_CAPS_MACB_IS_GEM; 3699 3700 dcfg = gem_readl(bp, DCFG1); 3701 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 3702 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 3703 if (GEM_BFEXT(NO_PCS, dcfg) == 0) 3704 bp->caps |= MACB_CAPS_PCS; 3705 dcfg = gem_readl(bp, DCFG12); 3706 if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) 3707 bp->caps |= MACB_CAPS_HIGH_SPEED; 3708 dcfg = gem_readl(bp, DCFG2); 3709 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 3710 bp->caps |= MACB_CAPS_FIFO_MODE; 3711 #ifdef CONFIG_MACB_USE_HWSTAMP 3712 if (gem_has_ptp(bp)) { 3713 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 3714 dev_err(&bp->pdev->dev, 3715 "GEM doesn't support hardware ptp.\n"); 3716 else { 3717 bp->hw_dma_cap |= HW_DMA_CAP_PTP; 3718 bp->ptp_info = &gem_ptp_info; 3719 } 3720 } 3721 #endif 3722 } 3723 3724 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 3725 } 3726 3727 static void macb_probe_queues(void __iomem *mem, 3728 bool native_io, 3729 unsigned int *queue_mask, 3730 unsigned int *num_queues) 3731 { 3732 *queue_mask = 0x1; 3733 *num_queues = 1; 3734 3735 /* is it macb or gem ? 3736 * 3737 * We need to read directly from the hardware here because 3738 * we are early in the probe process and don't have the 3739 * MACB_CAPS_MACB_IS_GEM flag positioned 3740 */ 3741 if (!hw_is_gem(mem, native_io)) 3742 return; 3743 3744 /* bit 0 is never set but queue 0 always exists */ 3745 *queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff; 3746 *num_queues = hweight32(*queue_mask); 3747 } 3748 3749 static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk, 3750 struct clk *rx_clk, struct clk *tsu_clk) 3751 { 3752 struct clk_bulk_data clks[] = { 3753 { .clk = tsu_clk, }, 3754 { .clk = rx_clk, }, 3755 { .clk = pclk, }, 3756 { .clk = hclk, }, 3757 { .clk = tx_clk }, 3758 }; 3759 3760 clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); 3761 } 3762 3763 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 3764 struct clk **hclk, struct clk **tx_clk, 3765 struct clk **rx_clk, struct clk **tsu_clk) 3766 { 3767 struct macb_platform_data *pdata; 3768 int err; 3769 3770 pdata = dev_get_platdata(&pdev->dev); 3771 if (pdata) { 3772 *pclk = pdata->pclk; 3773 *hclk = pdata->hclk; 3774 } else { 3775 *pclk = devm_clk_get(&pdev->dev, "pclk"); 3776 *hclk = devm_clk_get(&pdev->dev, "hclk"); 3777 } 3778 3779 if (IS_ERR_OR_NULL(*pclk)) 3780 return dev_err_probe(&pdev->dev, 3781 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, 3782 "failed to get pclk\n"); 3783 3784 if (IS_ERR_OR_NULL(*hclk)) 3785 return dev_err_probe(&pdev->dev, 3786 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, 3787 "failed to get hclk\n"); 3788 3789 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); 3790 if (IS_ERR(*tx_clk)) 3791 return PTR_ERR(*tx_clk); 3792 3793 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); 3794 if (IS_ERR(*rx_clk)) 3795 return PTR_ERR(*rx_clk); 3796 3797 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); 3798 if (IS_ERR(*tsu_clk)) 3799 return PTR_ERR(*tsu_clk); 3800 3801 err = clk_prepare_enable(*pclk); 3802 if (err) { 3803 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 3804 return err; 3805 } 3806 3807 err = clk_prepare_enable(*hclk); 3808 if (err) { 3809 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 3810 goto err_disable_pclk; 3811 } 3812 3813 err = clk_prepare_enable(*tx_clk); 3814 if (err) { 3815 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 3816 goto err_disable_hclk; 3817 } 3818 3819 err = clk_prepare_enable(*rx_clk); 3820 if (err) { 3821 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 3822 goto err_disable_txclk; 3823 } 3824 3825 err = clk_prepare_enable(*tsu_clk); 3826 if (err) { 3827 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 3828 goto err_disable_rxclk; 3829 } 3830 3831 return 0; 3832 3833 err_disable_rxclk: 3834 clk_disable_unprepare(*rx_clk); 3835 3836 err_disable_txclk: 3837 clk_disable_unprepare(*tx_clk); 3838 3839 err_disable_hclk: 3840 clk_disable_unprepare(*hclk); 3841 3842 err_disable_pclk: 3843 clk_disable_unprepare(*pclk); 3844 3845 return err; 3846 } 3847 3848 static int macb_init(struct platform_device *pdev) 3849 { 3850 struct net_device *dev = platform_get_drvdata(pdev); 3851 unsigned int hw_q, q; 3852 struct macb *bp = netdev_priv(dev); 3853 struct macb_queue *queue; 3854 int err; 3855 u32 val, reg; 3856 3857 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 3858 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 3859 3860 /* set the queue register mapping once for all: queue0 has a special 3861 * register mapping but we don't want to test the queue index then 3862 * compute the corresponding register offset at run time. 3863 */ 3864 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { 3865 if (!(bp->queue_mask & (1 << hw_q))) 3866 continue; 3867 3868 queue = &bp->queues[q]; 3869 queue->bp = bp; 3870 netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT); 3871 if (hw_q) { 3872 queue->ISR = GEM_ISR(hw_q - 1); 3873 queue->IER = GEM_IER(hw_q - 1); 3874 queue->IDR = GEM_IDR(hw_q - 1); 3875 queue->IMR = GEM_IMR(hw_q - 1); 3876 queue->TBQP = GEM_TBQP(hw_q - 1); 3877 queue->RBQP = GEM_RBQP(hw_q - 1); 3878 queue->RBQS = GEM_RBQS(hw_q - 1); 3879 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3880 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3881 queue->TBQPH = GEM_TBQPH(hw_q - 1); 3882 queue->RBQPH = GEM_RBQPH(hw_q - 1); 3883 } 3884 #endif 3885 } else { 3886 /* queue0 uses legacy registers */ 3887 queue->ISR = MACB_ISR; 3888 queue->IER = MACB_IER; 3889 queue->IDR = MACB_IDR; 3890 queue->IMR = MACB_IMR; 3891 queue->TBQP = MACB_TBQP; 3892 queue->RBQP = MACB_RBQP; 3893 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 3894 if (bp->hw_dma_cap & HW_DMA_CAP_64B) { 3895 queue->TBQPH = MACB_TBQPH; 3896 queue->RBQPH = MACB_RBQPH; 3897 } 3898 #endif 3899 } 3900 3901 /* get irq: here we use the linux queue index, not the hardware 3902 * queue index. the queue irq definitions in the device tree 3903 * must remove the optional gaps that could exist in the 3904 * hardware queue mask. 3905 */ 3906 queue->irq = platform_get_irq(pdev, q); 3907 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 3908 IRQF_SHARED, dev->name, queue); 3909 if (err) { 3910 dev_err(&pdev->dev, 3911 "Unable to request IRQ %d (error %d)\n", 3912 queue->irq, err); 3913 return err; 3914 } 3915 3916 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 3917 q++; 3918 } 3919 3920 dev->netdev_ops = &macb_netdev_ops; 3921 3922 /* setup appropriated routines according to adapter type */ 3923 if (macb_is_gem(bp)) { 3924 bp->max_tx_length = GEM_MAX_TX_LEN; 3925 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 3926 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 3927 bp->macbgem_ops.mog_init_rings = gem_init_rings; 3928 bp->macbgem_ops.mog_rx = gem_rx; 3929 dev->ethtool_ops = &gem_ethtool_ops; 3930 } else { 3931 bp->max_tx_length = MACB_MAX_TX_LEN; 3932 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 3933 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 3934 bp->macbgem_ops.mog_init_rings = macb_init_rings; 3935 bp->macbgem_ops.mog_rx = macb_rx; 3936 dev->ethtool_ops = &macb_ethtool_ops; 3937 } 3938 3939 /* Set features */ 3940 dev->hw_features = NETIF_F_SG; 3941 3942 /* Check LSO capability */ 3943 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 3944 dev->hw_features |= MACB_NETIF_LSO; 3945 3946 /* Checksum offload is only available on gem with packet buffer */ 3947 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 3948 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 3949 if (bp->caps & MACB_CAPS_SG_DISABLED) 3950 dev->hw_features &= ~NETIF_F_SG; 3951 dev->features = dev->hw_features; 3952 3953 /* Check RX Flow Filters support. 3954 * Max Rx flows set by availability of screeners & compare regs: 3955 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 3956 */ 3957 reg = gem_readl(bp, DCFG8); 3958 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), 3959 GEM_BFEXT(T2SCR, reg)); 3960 INIT_LIST_HEAD(&bp->rx_fs_list.list); 3961 if (bp->max_tuples > 0) { 3962 /* also needs one ethtype match to check IPv4 */ 3963 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 3964 /* program this reg now */ 3965 reg = 0; 3966 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 3967 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 3968 /* Filtering is supported in hw but don't enable it in kernel now */ 3969 dev->hw_features |= NETIF_F_NTUPLE; 3970 /* init Rx flow definitions */ 3971 bp->rx_fs_list.count = 0; 3972 spin_lock_init(&bp->rx_fs_lock); 3973 } else 3974 bp->max_tuples = 0; 3975 } 3976 3977 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 3978 val = 0; 3979 if (phy_interface_mode_is_rgmii(bp->phy_interface)) 3980 val = bp->usrio->rgmii; 3981 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 3982 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3983 val = bp->usrio->rmii; 3984 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 3985 val = bp->usrio->mii; 3986 3987 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 3988 val |= bp->usrio->refclk; 3989 3990 macb_or_gem_writel(bp, USRIO, val); 3991 } 3992 3993 /* Set MII management clock divider */ 3994 val = macb_mdc_clk_div(bp); 3995 val |= macb_dbw(bp); 3996 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 3997 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 3998 macb_writel(bp, NCFGR, val); 3999 4000 return 0; 4001 } 4002 4003 static const struct macb_usrio_config macb_default_usrio = { 4004 .mii = MACB_BIT(MII), 4005 .rmii = MACB_BIT(RMII), 4006 .rgmii = GEM_BIT(RGMII), 4007 .refclk = MACB_BIT(CLKEN), 4008 }; 4009 4010 #if defined(CONFIG_OF) 4011 /* 1518 rounded up */ 4012 #define AT91ETHER_MAX_RBUFF_SZ 0x600 4013 /* max number of receive buffers */ 4014 #define AT91ETHER_MAX_RX_DESCR 9 4015 4016 static struct sifive_fu540_macb_mgmt *mgmt; 4017 4018 static int at91ether_alloc_coherent(struct macb *lp) 4019 { 4020 struct macb_queue *q = &lp->queues[0]; 4021 4022 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 4023 (AT91ETHER_MAX_RX_DESCR * 4024 macb_dma_desc_get_size(lp)), 4025 &q->rx_ring_dma, GFP_KERNEL); 4026 if (!q->rx_ring) 4027 return -ENOMEM; 4028 4029 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 4030 AT91ETHER_MAX_RX_DESCR * 4031 AT91ETHER_MAX_RBUFF_SZ, 4032 &q->rx_buffers_dma, GFP_KERNEL); 4033 if (!q->rx_buffers) { 4034 dma_free_coherent(&lp->pdev->dev, 4035 AT91ETHER_MAX_RX_DESCR * 4036 macb_dma_desc_get_size(lp), 4037 q->rx_ring, q->rx_ring_dma); 4038 q->rx_ring = NULL; 4039 return -ENOMEM; 4040 } 4041 4042 return 0; 4043 } 4044 4045 static void at91ether_free_coherent(struct macb *lp) 4046 { 4047 struct macb_queue *q = &lp->queues[0]; 4048 4049 if (q->rx_ring) { 4050 dma_free_coherent(&lp->pdev->dev, 4051 AT91ETHER_MAX_RX_DESCR * 4052 macb_dma_desc_get_size(lp), 4053 q->rx_ring, q->rx_ring_dma); 4054 q->rx_ring = NULL; 4055 } 4056 4057 if (q->rx_buffers) { 4058 dma_free_coherent(&lp->pdev->dev, 4059 AT91ETHER_MAX_RX_DESCR * 4060 AT91ETHER_MAX_RBUFF_SZ, 4061 q->rx_buffers, q->rx_buffers_dma); 4062 q->rx_buffers = NULL; 4063 } 4064 } 4065 4066 /* Initialize and start the Receiver and Transmit subsystems */ 4067 static int at91ether_start(struct macb *lp) 4068 { 4069 struct macb_queue *q = &lp->queues[0]; 4070 struct macb_dma_desc *desc; 4071 dma_addr_t addr; 4072 u32 ctl; 4073 int i, ret; 4074 4075 ret = at91ether_alloc_coherent(lp); 4076 if (ret) 4077 return ret; 4078 4079 addr = q->rx_buffers_dma; 4080 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 4081 desc = macb_rx_desc(q, i); 4082 macb_set_addr(lp, desc, addr); 4083 desc->ctrl = 0; 4084 addr += AT91ETHER_MAX_RBUFF_SZ; 4085 } 4086 4087 /* Set the Wrap bit on the last descriptor */ 4088 desc->addr |= MACB_BIT(RX_WRAP); 4089 4090 /* Reset buffer index */ 4091 q->rx_tail = 0; 4092 4093 /* Program address of descriptor list in Rx Buffer Queue register */ 4094 macb_writel(lp, RBQP, q->rx_ring_dma); 4095 4096 /* Enable Receive and Transmit */ 4097 ctl = macb_readl(lp, NCR); 4098 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 4099 4100 /* Enable MAC interrupts */ 4101 macb_writel(lp, IER, MACB_BIT(RCOMP) | 4102 MACB_BIT(RXUBR) | 4103 MACB_BIT(ISR_TUND) | 4104 MACB_BIT(ISR_RLE) | 4105 MACB_BIT(TCOMP) | 4106 MACB_BIT(ISR_ROVR) | 4107 MACB_BIT(HRESP)); 4108 4109 return 0; 4110 } 4111 4112 static void at91ether_stop(struct macb *lp) 4113 { 4114 u32 ctl; 4115 4116 /* Disable MAC interrupts */ 4117 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 4118 MACB_BIT(RXUBR) | 4119 MACB_BIT(ISR_TUND) | 4120 MACB_BIT(ISR_RLE) | 4121 MACB_BIT(TCOMP) | 4122 MACB_BIT(ISR_ROVR) | 4123 MACB_BIT(HRESP)); 4124 4125 /* Disable Receiver and Transmitter */ 4126 ctl = macb_readl(lp, NCR); 4127 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 4128 4129 /* Free resources. */ 4130 at91ether_free_coherent(lp); 4131 } 4132 4133 /* Open the ethernet interface */ 4134 static int at91ether_open(struct net_device *dev) 4135 { 4136 struct macb *lp = netdev_priv(dev); 4137 u32 ctl; 4138 int ret; 4139 4140 ret = pm_runtime_get_sync(&lp->pdev->dev); 4141 if (ret < 0) { 4142 pm_runtime_put_noidle(&lp->pdev->dev); 4143 return ret; 4144 } 4145 4146 /* Clear internal statistics */ 4147 ctl = macb_readl(lp, NCR); 4148 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 4149 4150 macb_set_hwaddr(lp); 4151 4152 ret = at91ether_start(lp); 4153 if (ret) 4154 goto pm_exit; 4155 4156 ret = macb_phylink_connect(lp); 4157 if (ret) 4158 goto stop; 4159 4160 netif_start_queue(dev); 4161 4162 return 0; 4163 4164 stop: 4165 at91ether_stop(lp); 4166 pm_exit: 4167 pm_runtime_put_sync(&lp->pdev->dev); 4168 return ret; 4169 } 4170 4171 /* Close the interface */ 4172 static int at91ether_close(struct net_device *dev) 4173 { 4174 struct macb *lp = netdev_priv(dev); 4175 4176 netif_stop_queue(dev); 4177 4178 phylink_stop(lp->phylink); 4179 phylink_disconnect_phy(lp->phylink); 4180 4181 at91ether_stop(lp); 4182 4183 return pm_runtime_put(&lp->pdev->dev); 4184 } 4185 4186 /* Transmit packet */ 4187 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 4188 struct net_device *dev) 4189 { 4190 struct macb *lp = netdev_priv(dev); 4191 4192 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 4193 int desc = 0; 4194 4195 netif_stop_queue(dev); 4196 4197 /* Store packet information (to free when Tx completed) */ 4198 lp->rm9200_txq[desc].skb = skb; 4199 lp->rm9200_txq[desc].size = skb->len; 4200 lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data, 4201 skb->len, DMA_TO_DEVICE); 4202 if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) { 4203 dev_kfree_skb_any(skb); 4204 dev->stats.tx_dropped++; 4205 netdev_err(dev, "%s: DMA mapping error\n", __func__); 4206 return NETDEV_TX_OK; 4207 } 4208 4209 /* Set address of the data in the Transmit Address register */ 4210 macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping); 4211 /* Set length of the packet in the Transmit Control register */ 4212 macb_writel(lp, TCR, skb->len); 4213 4214 } else { 4215 netdev_err(dev, "%s called, but device is busy!\n", __func__); 4216 return NETDEV_TX_BUSY; 4217 } 4218 4219 return NETDEV_TX_OK; 4220 } 4221 4222 /* Extract received frame from buffer descriptors and sent to upper layers. 4223 * (Called from interrupt context) 4224 */ 4225 static void at91ether_rx(struct net_device *dev) 4226 { 4227 struct macb *lp = netdev_priv(dev); 4228 struct macb_queue *q = &lp->queues[0]; 4229 struct macb_dma_desc *desc; 4230 unsigned char *p_recv; 4231 struct sk_buff *skb; 4232 unsigned int pktlen; 4233 4234 desc = macb_rx_desc(q, q->rx_tail); 4235 while (desc->addr & MACB_BIT(RX_USED)) { 4236 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 4237 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 4238 skb = netdev_alloc_skb(dev, pktlen + 2); 4239 if (skb) { 4240 skb_reserve(skb, 2); 4241 skb_put_data(skb, p_recv, pktlen); 4242 4243 skb->protocol = eth_type_trans(skb, dev); 4244 dev->stats.rx_packets++; 4245 dev->stats.rx_bytes += pktlen; 4246 netif_rx(skb); 4247 } else { 4248 dev->stats.rx_dropped++; 4249 } 4250 4251 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 4252 dev->stats.multicast++; 4253 4254 /* reset ownership bit */ 4255 desc->addr &= ~MACB_BIT(RX_USED); 4256 4257 /* wrap after last buffer */ 4258 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 4259 q->rx_tail = 0; 4260 else 4261 q->rx_tail++; 4262 4263 desc = macb_rx_desc(q, q->rx_tail); 4264 } 4265 } 4266 4267 /* MAC interrupt handler */ 4268 static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 4269 { 4270 struct net_device *dev = dev_id; 4271 struct macb *lp = netdev_priv(dev); 4272 u32 intstatus, ctl; 4273 unsigned int desc; 4274 4275 /* MAC Interrupt Status register indicates what interrupts are pending. 4276 * It is automatically cleared once read. 4277 */ 4278 intstatus = macb_readl(lp, ISR); 4279 4280 /* Receive complete */ 4281 if (intstatus & MACB_BIT(RCOMP)) 4282 at91ether_rx(dev); 4283 4284 /* Transmit complete */ 4285 if (intstatus & MACB_BIT(TCOMP)) { 4286 /* The TCOM bit is set even if the transmission failed */ 4287 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 4288 dev->stats.tx_errors++; 4289 4290 desc = 0; 4291 if (lp->rm9200_txq[desc].skb) { 4292 dev_consume_skb_irq(lp->rm9200_txq[desc].skb); 4293 lp->rm9200_txq[desc].skb = NULL; 4294 dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, 4295 lp->rm9200_txq[desc].size, DMA_TO_DEVICE); 4296 dev->stats.tx_packets++; 4297 dev->stats.tx_bytes += lp->rm9200_txq[desc].size; 4298 } 4299 netif_wake_queue(dev); 4300 } 4301 4302 /* Work-around for EMAC Errata section 41.3.1 */ 4303 if (intstatus & MACB_BIT(RXUBR)) { 4304 ctl = macb_readl(lp, NCR); 4305 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 4306 wmb(); 4307 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 4308 } 4309 4310 if (intstatus & MACB_BIT(ISR_ROVR)) 4311 netdev_err(dev, "ROVR error\n"); 4312 4313 return IRQ_HANDLED; 4314 } 4315 4316 #ifdef CONFIG_NET_POLL_CONTROLLER 4317 static void at91ether_poll_controller(struct net_device *dev) 4318 { 4319 unsigned long flags; 4320 4321 local_irq_save(flags); 4322 at91ether_interrupt(dev->irq, dev); 4323 local_irq_restore(flags); 4324 } 4325 #endif 4326 4327 static const struct net_device_ops at91ether_netdev_ops = { 4328 .ndo_open = at91ether_open, 4329 .ndo_stop = at91ether_close, 4330 .ndo_start_xmit = at91ether_start_xmit, 4331 .ndo_get_stats = macb_get_stats, 4332 .ndo_set_rx_mode = macb_set_rx_mode, 4333 .ndo_set_mac_address = eth_mac_addr, 4334 .ndo_eth_ioctl = macb_ioctl, 4335 .ndo_validate_addr = eth_validate_addr, 4336 #ifdef CONFIG_NET_POLL_CONTROLLER 4337 .ndo_poll_controller = at91ether_poll_controller, 4338 #endif 4339 }; 4340 4341 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 4342 struct clk **hclk, struct clk **tx_clk, 4343 struct clk **rx_clk, struct clk **tsu_clk) 4344 { 4345 int err; 4346 4347 *hclk = NULL; 4348 *tx_clk = NULL; 4349 *rx_clk = NULL; 4350 *tsu_clk = NULL; 4351 4352 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 4353 if (IS_ERR(*pclk)) 4354 return PTR_ERR(*pclk); 4355 4356 err = clk_prepare_enable(*pclk); 4357 if (err) { 4358 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 4359 return err; 4360 } 4361 4362 return 0; 4363 } 4364 4365 static int at91ether_init(struct platform_device *pdev) 4366 { 4367 struct net_device *dev = platform_get_drvdata(pdev); 4368 struct macb *bp = netdev_priv(dev); 4369 int err; 4370 4371 bp->queues[0].bp = bp; 4372 4373 dev->netdev_ops = &at91ether_netdev_ops; 4374 dev->ethtool_ops = &macb_ethtool_ops; 4375 4376 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 4377 0, dev->name, dev); 4378 if (err) 4379 return err; 4380 4381 macb_writel(bp, NCR, 0); 4382 4383 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); 4384 4385 return 0; 4386 } 4387 4388 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 4389 unsigned long parent_rate) 4390 { 4391 return mgmt->rate; 4392 } 4393 4394 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, 4395 unsigned long *parent_rate) 4396 { 4397 if (WARN_ON(rate < 2500000)) 4398 return 2500000; 4399 else if (rate == 2500000) 4400 return 2500000; 4401 else if (WARN_ON(rate < 13750000)) 4402 return 2500000; 4403 else if (WARN_ON(rate < 25000000)) 4404 return 25000000; 4405 else if (rate == 25000000) 4406 return 25000000; 4407 else if (WARN_ON(rate < 75000000)) 4408 return 25000000; 4409 else if (WARN_ON(rate < 125000000)) 4410 return 125000000; 4411 else if (rate == 125000000) 4412 return 125000000; 4413 4414 WARN_ON(rate > 125000000); 4415 4416 return 125000000; 4417 } 4418 4419 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 4420 unsigned long parent_rate) 4421 { 4422 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); 4423 if (rate != 125000000) 4424 iowrite32(1, mgmt->reg); 4425 else 4426 iowrite32(0, mgmt->reg); 4427 mgmt->rate = rate; 4428 4429 return 0; 4430 } 4431 4432 static const struct clk_ops fu540_c000_ops = { 4433 .recalc_rate = fu540_macb_tx_recalc_rate, 4434 .round_rate = fu540_macb_tx_round_rate, 4435 .set_rate = fu540_macb_tx_set_rate, 4436 }; 4437 4438 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 4439 struct clk **hclk, struct clk **tx_clk, 4440 struct clk **rx_clk, struct clk **tsu_clk) 4441 { 4442 struct clk_init_data init; 4443 int err = 0; 4444 4445 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 4446 if (err) 4447 return err; 4448 4449 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 4450 if (!mgmt) { 4451 err = -ENOMEM; 4452 goto err_disable_clks; 4453 } 4454 4455 init.name = "sifive-gemgxl-mgmt"; 4456 init.ops = &fu540_c000_ops; 4457 init.flags = 0; 4458 init.num_parents = 0; 4459 4460 mgmt->rate = 0; 4461 mgmt->hw.init = &init; 4462 4463 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); 4464 if (IS_ERR(*tx_clk)) { 4465 err = PTR_ERR(*tx_clk); 4466 goto err_disable_clks; 4467 } 4468 4469 err = clk_prepare_enable(*tx_clk); 4470 if (err) { 4471 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 4472 *tx_clk = NULL; 4473 goto err_disable_clks; 4474 } else { 4475 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 4476 } 4477 4478 return 0; 4479 4480 err_disable_clks: 4481 macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk); 4482 4483 return err; 4484 } 4485 4486 static int fu540_c000_init(struct platform_device *pdev) 4487 { 4488 mgmt->reg = devm_platform_ioremap_resource(pdev, 1); 4489 if (IS_ERR(mgmt->reg)) 4490 return PTR_ERR(mgmt->reg); 4491 4492 return macb_init(pdev); 4493 } 4494 4495 static const struct macb_usrio_config sama7g5_usrio = { 4496 .mii = 0, 4497 .rmii = 1, 4498 .rgmii = 2, 4499 .refclk = BIT(2), 4500 .hdfctlen = BIT(6), 4501 }; 4502 4503 static const struct macb_config fu540_c000_config = { 4504 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 4505 MACB_CAPS_GEM_HAS_PTP, 4506 .dma_burst_length = 16, 4507 .clk_init = fu540_c000_clk_init, 4508 .init = fu540_c000_init, 4509 .jumbo_max_len = 10240, 4510 .usrio = &macb_default_usrio, 4511 }; 4512 4513 static const struct macb_config at91sam9260_config = { 4514 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4515 .clk_init = macb_clk_init, 4516 .init = macb_init, 4517 .usrio = &macb_default_usrio, 4518 }; 4519 4520 static const struct macb_config sama5d3macb_config = { 4521 .caps = MACB_CAPS_SG_DISABLED 4522 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4523 .clk_init = macb_clk_init, 4524 .init = macb_init, 4525 .usrio = &macb_default_usrio, 4526 }; 4527 4528 static const struct macb_config pc302gem_config = { 4529 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 4530 .dma_burst_length = 16, 4531 .clk_init = macb_clk_init, 4532 .init = macb_init, 4533 .usrio = &macb_default_usrio, 4534 }; 4535 4536 static const struct macb_config sama5d2_config = { 4537 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4538 .dma_burst_length = 16, 4539 .clk_init = macb_clk_init, 4540 .init = macb_init, 4541 .usrio = &macb_default_usrio, 4542 }; 4543 4544 static const struct macb_config sama5d29_config = { 4545 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP, 4546 .dma_burst_length = 16, 4547 .clk_init = macb_clk_init, 4548 .init = macb_init, 4549 .usrio = &macb_default_usrio, 4550 }; 4551 4552 static const struct macb_config sama5d3_config = { 4553 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE 4554 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 4555 .dma_burst_length = 16, 4556 .clk_init = macb_clk_init, 4557 .init = macb_init, 4558 .jumbo_max_len = 10240, 4559 .usrio = &macb_default_usrio, 4560 }; 4561 4562 static const struct macb_config sama5d4_config = { 4563 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 4564 .dma_burst_length = 4, 4565 .clk_init = macb_clk_init, 4566 .init = macb_init, 4567 .usrio = &macb_default_usrio, 4568 }; 4569 4570 static const struct macb_config emac_config = { 4571 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC, 4572 .clk_init = at91ether_clk_init, 4573 .init = at91ether_init, 4574 .usrio = &macb_default_usrio, 4575 }; 4576 4577 static const struct macb_config np4_config = { 4578 .caps = MACB_CAPS_USRIO_DISABLED, 4579 .clk_init = macb_clk_init, 4580 .init = macb_init, 4581 .usrio = &macb_default_usrio, 4582 }; 4583 4584 static const struct macb_config zynqmp_config = { 4585 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4586 MACB_CAPS_JUMBO | 4587 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 4588 .dma_burst_length = 16, 4589 .clk_init = macb_clk_init, 4590 .init = macb_init, 4591 .jumbo_max_len = 10240, 4592 .usrio = &macb_default_usrio, 4593 }; 4594 4595 static const struct macb_config zynq_config = { 4596 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 4597 MACB_CAPS_NEEDS_RSTONUBR, 4598 .dma_burst_length = 16, 4599 .clk_init = macb_clk_init, 4600 .init = macb_init, 4601 .usrio = &macb_default_usrio, 4602 }; 4603 4604 static const struct macb_config sama7g5_gem_config = { 4605 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | 4606 MACB_CAPS_MIIONRGMII, 4607 .dma_burst_length = 16, 4608 .clk_init = macb_clk_init, 4609 .init = macb_init, 4610 .usrio = &sama7g5_usrio, 4611 }; 4612 4613 static const struct macb_config sama7g5_emac_config = { 4614 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 4615 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII, 4616 .dma_burst_length = 16, 4617 .clk_init = macb_clk_init, 4618 .init = macb_init, 4619 .usrio = &sama7g5_usrio, 4620 }; 4621 4622 static const struct of_device_id macb_dt_ids[] = { 4623 { .compatible = "cdns,at32ap7000-macb" }, 4624 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 4625 { .compatible = "cdns,macb" }, 4626 { .compatible = "cdns,np4-macb", .data = &np4_config }, 4627 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 4628 { .compatible = "cdns,gem", .data = &pc302gem_config }, 4629 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 4630 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 4631 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config }, 4632 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 4633 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 4634 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 4635 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 4636 { .compatible = "cdns,emac", .data = &emac_config }, 4637 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, 4638 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, 4639 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 4640 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, 4641 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, 4642 { /* sentinel */ } 4643 }; 4644 MODULE_DEVICE_TABLE(of, macb_dt_ids); 4645 #endif /* CONFIG_OF */ 4646 4647 static const struct macb_config default_gem_config = { 4648 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 4649 MACB_CAPS_JUMBO | 4650 MACB_CAPS_GEM_HAS_PTP, 4651 .dma_burst_length = 16, 4652 .clk_init = macb_clk_init, 4653 .init = macb_init, 4654 .usrio = &macb_default_usrio, 4655 .jumbo_max_len = 10240, 4656 }; 4657 4658 static int macb_probe(struct platform_device *pdev) 4659 { 4660 const struct macb_config *macb_config = &default_gem_config; 4661 int (*clk_init)(struct platform_device *, struct clk **, 4662 struct clk **, struct clk **, struct clk **, 4663 struct clk **) = macb_config->clk_init; 4664 int (*init)(struct platform_device *) = macb_config->init; 4665 struct device_node *np = pdev->dev.of_node; 4666 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 4667 struct clk *tsu_clk = NULL; 4668 unsigned int queue_mask, num_queues; 4669 bool native_io; 4670 phy_interface_t interface; 4671 struct net_device *dev; 4672 struct resource *regs; 4673 void __iomem *mem; 4674 struct macb *bp; 4675 int err, val; 4676 4677 mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); 4678 if (IS_ERR(mem)) 4679 return PTR_ERR(mem); 4680 4681 if (np) { 4682 const struct of_device_id *match; 4683 4684 match = of_match_node(macb_dt_ids, np); 4685 if (match && match->data) { 4686 macb_config = match->data; 4687 clk_init = macb_config->clk_init; 4688 init = macb_config->init; 4689 } 4690 } 4691 4692 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 4693 if (err) 4694 return err; 4695 4696 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 4697 pm_runtime_use_autosuspend(&pdev->dev); 4698 pm_runtime_get_noresume(&pdev->dev); 4699 pm_runtime_set_active(&pdev->dev); 4700 pm_runtime_enable(&pdev->dev); 4701 native_io = hw_is_native_io(mem); 4702 4703 macb_probe_queues(mem, native_io, &queue_mask, &num_queues); 4704 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 4705 if (!dev) { 4706 err = -ENOMEM; 4707 goto err_disable_clocks; 4708 } 4709 4710 dev->base_addr = regs->start; 4711 4712 SET_NETDEV_DEV(dev, &pdev->dev); 4713 4714 bp = netdev_priv(dev); 4715 bp->pdev = pdev; 4716 bp->dev = dev; 4717 bp->regs = mem; 4718 bp->native_io = native_io; 4719 if (native_io) { 4720 bp->macb_reg_readl = hw_readl_native; 4721 bp->macb_reg_writel = hw_writel_native; 4722 } else { 4723 bp->macb_reg_readl = hw_readl; 4724 bp->macb_reg_writel = hw_writel; 4725 } 4726 bp->num_queues = num_queues; 4727 bp->queue_mask = queue_mask; 4728 if (macb_config) 4729 bp->dma_burst_length = macb_config->dma_burst_length; 4730 bp->pclk = pclk; 4731 bp->hclk = hclk; 4732 bp->tx_clk = tx_clk; 4733 bp->rx_clk = rx_clk; 4734 bp->tsu_clk = tsu_clk; 4735 if (macb_config) 4736 bp->jumbo_max_len = macb_config->jumbo_max_len; 4737 4738 bp->wol = 0; 4739 if (of_get_property(np, "magic-packet", NULL)) 4740 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; 4741 device_set_wakeup_capable(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); 4742 4743 bp->usrio = macb_config->usrio; 4744 4745 spin_lock_init(&bp->lock); 4746 4747 /* setup capabilities */ 4748 macb_configure_caps(bp, macb_config); 4749 4750 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 4751 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 4752 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); 4753 bp->hw_dma_cap |= HW_DMA_CAP_64B; 4754 } 4755 #endif 4756 platform_set_drvdata(pdev, dev); 4757 4758 dev->irq = platform_get_irq(pdev, 0); 4759 if (dev->irq < 0) { 4760 err = dev->irq; 4761 goto err_out_free_netdev; 4762 } 4763 4764 /* MTU range: 68 - 1500 or 10240 */ 4765 dev->min_mtu = GEM_MTU_MIN_SIZE; 4766 if (bp->caps & MACB_CAPS_JUMBO) 4767 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; 4768 else 4769 dev->max_mtu = ETH_DATA_LEN; 4770 4771 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 4772 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 4773 if (val) 4774 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 4775 macb_dma_desc_get_size(bp); 4776 4777 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 4778 if (val) 4779 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 4780 macb_dma_desc_get_size(bp); 4781 } 4782 4783 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 4784 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 4785 bp->rx_intr_mask |= MACB_BIT(RXUBR); 4786 4787 err = of_get_ethdev_address(np, bp->dev); 4788 if (err == -EPROBE_DEFER) 4789 goto err_out_free_netdev; 4790 else if (err) 4791 macb_get_hwaddr(bp); 4792 4793 err = of_get_phy_mode(np, &interface); 4794 if (err) 4795 /* not found in DT, MII by default */ 4796 bp->phy_interface = PHY_INTERFACE_MODE_MII; 4797 else 4798 bp->phy_interface = interface; 4799 4800 /* IP specific init */ 4801 err = init(pdev); 4802 if (err) 4803 goto err_out_free_netdev; 4804 4805 err = macb_mii_init(bp); 4806 if (err) 4807 goto err_out_free_netdev; 4808 4809 netif_carrier_off(dev); 4810 4811 err = register_netdev(dev); 4812 if (err) { 4813 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 4814 goto err_out_unregister_mdio; 4815 } 4816 4817 tasklet_setup(&bp->hresp_err_tasklet, macb_hresp_error_task); 4818 4819 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 4820 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 4821 dev->base_addr, dev->irq, dev->dev_addr); 4822 4823 pm_runtime_mark_last_busy(&bp->pdev->dev); 4824 pm_runtime_put_autosuspend(&bp->pdev->dev); 4825 4826 return 0; 4827 4828 err_out_unregister_mdio: 4829 mdiobus_unregister(bp->mii_bus); 4830 mdiobus_free(bp->mii_bus); 4831 4832 err_out_free_netdev: 4833 free_netdev(dev); 4834 4835 err_disable_clocks: 4836 macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); 4837 pm_runtime_disable(&pdev->dev); 4838 pm_runtime_set_suspended(&pdev->dev); 4839 pm_runtime_dont_use_autosuspend(&pdev->dev); 4840 4841 return err; 4842 } 4843 4844 static int macb_remove(struct platform_device *pdev) 4845 { 4846 struct net_device *dev; 4847 struct macb *bp; 4848 4849 dev = platform_get_drvdata(pdev); 4850 4851 if (dev) { 4852 bp = netdev_priv(dev); 4853 mdiobus_unregister(bp->mii_bus); 4854 mdiobus_free(bp->mii_bus); 4855 4856 unregister_netdev(dev); 4857 tasklet_kill(&bp->hresp_err_tasklet); 4858 pm_runtime_disable(&pdev->dev); 4859 pm_runtime_dont_use_autosuspend(&pdev->dev); 4860 if (!pm_runtime_suspended(&pdev->dev)) { 4861 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, 4862 bp->rx_clk, bp->tsu_clk); 4863 pm_runtime_set_suspended(&pdev->dev); 4864 } 4865 phylink_destroy(bp->phylink); 4866 free_netdev(dev); 4867 } 4868 4869 return 0; 4870 } 4871 4872 static int __maybe_unused macb_suspend(struct device *dev) 4873 { 4874 struct net_device *netdev = dev_get_drvdata(dev); 4875 struct macb *bp = netdev_priv(netdev); 4876 struct macb_queue *queue; 4877 unsigned long flags; 4878 unsigned int q; 4879 int err; 4880 4881 if (!netif_running(netdev)) 4882 return 0; 4883 4884 if (bp->wol & MACB_WOL_ENABLED) { 4885 spin_lock_irqsave(&bp->lock, flags); 4886 /* Flush all status bits */ 4887 macb_writel(bp, TSR, -1); 4888 macb_writel(bp, RSR, -1); 4889 for (q = 0, queue = bp->queues; q < bp->num_queues; 4890 ++q, ++queue) { 4891 /* Disable all interrupts */ 4892 queue_writel(queue, IDR, -1); 4893 queue_readl(queue, ISR); 4894 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 4895 queue_writel(queue, ISR, -1); 4896 } 4897 /* Change interrupt handler and 4898 * Enable WoL IRQ on queue 0 4899 */ 4900 devm_free_irq(dev, bp->queues[0].irq, bp->queues); 4901 if (macb_is_gem(bp)) { 4902 err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, 4903 IRQF_SHARED, netdev->name, bp->queues); 4904 if (err) { 4905 dev_err(dev, 4906 "Unable to request IRQ %d (error %d)\n", 4907 bp->queues[0].irq, err); 4908 spin_unlock_irqrestore(&bp->lock, flags); 4909 return err; 4910 } 4911 queue_writel(bp->queues, IER, GEM_BIT(WOL)); 4912 gem_writel(bp, WOL, MACB_BIT(MAG)); 4913 } else { 4914 err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, 4915 IRQF_SHARED, netdev->name, bp->queues); 4916 if (err) { 4917 dev_err(dev, 4918 "Unable to request IRQ %d (error %d)\n", 4919 bp->queues[0].irq, err); 4920 spin_unlock_irqrestore(&bp->lock, flags); 4921 return err; 4922 } 4923 queue_writel(bp->queues, IER, MACB_BIT(WOL)); 4924 macb_writel(bp, WOL, MACB_BIT(MAG)); 4925 } 4926 spin_unlock_irqrestore(&bp->lock, flags); 4927 4928 enable_irq_wake(bp->queues[0].irq); 4929 } 4930 4931 netif_device_detach(netdev); 4932 for (q = 0, queue = bp->queues; q < bp->num_queues; 4933 ++q, ++queue) 4934 napi_disable(&queue->napi); 4935 4936 if (!(bp->wol & MACB_WOL_ENABLED)) { 4937 rtnl_lock(); 4938 phylink_stop(bp->phylink); 4939 rtnl_unlock(); 4940 spin_lock_irqsave(&bp->lock, flags); 4941 macb_reset_hw(bp); 4942 spin_unlock_irqrestore(&bp->lock, flags); 4943 } 4944 4945 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 4946 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 4947 4948 if (netdev->hw_features & NETIF_F_NTUPLE) 4949 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 4950 4951 if (bp->ptp_info) 4952 bp->ptp_info->ptp_remove(netdev); 4953 if (!device_may_wakeup(dev)) 4954 pm_runtime_force_suspend(dev); 4955 4956 return 0; 4957 } 4958 4959 static int __maybe_unused macb_resume(struct device *dev) 4960 { 4961 struct net_device *netdev = dev_get_drvdata(dev); 4962 struct macb *bp = netdev_priv(netdev); 4963 struct macb_queue *queue; 4964 unsigned long flags; 4965 unsigned int q; 4966 int err; 4967 4968 if (!netif_running(netdev)) 4969 return 0; 4970 4971 if (!device_may_wakeup(dev)) 4972 pm_runtime_force_resume(dev); 4973 4974 if (bp->wol & MACB_WOL_ENABLED) { 4975 spin_lock_irqsave(&bp->lock, flags); 4976 /* Disable WoL */ 4977 if (macb_is_gem(bp)) { 4978 queue_writel(bp->queues, IDR, GEM_BIT(WOL)); 4979 gem_writel(bp, WOL, 0); 4980 } else { 4981 queue_writel(bp->queues, IDR, MACB_BIT(WOL)); 4982 macb_writel(bp, WOL, 0); 4983 } 4984 /* Clear ISR on queue 0 */ 4985 queue_readl(bp->queues, ISR); 4986 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 4987 queue_writel(bp->queues, ISR, -1); 4988 /* Replace interrupt handler on queue 0 */ 4989 devm_free_irq(dev, bp->queues[0].irq, bp->queues); 4990 err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, 4991 IRQF_SHARED, netdev->name, bp->queues); 4992 if (err) { 4993 dev_err(dev, 4994 "Unable to request IRQ %d (error %d)\n", 4995 bp->queues[0].irq, err); 4996 spin_unlock_irqrestore(&bp->lock, flags); 4997 return err; 4998 } 4999 spin_unlock_irqrestore(&bp->lock, flags); 5000 5001 disable_irq_wake(bp->queues[0].irq); 5002 5003 /* Now make sure we disable phy before moving 5004 * to common restore path 5005 */ 5006 rtnl_lock(); 5007 phylink_stop(bp->phylink); 5008 rtnl_unlock(); 5009 } 5010 5011 for (q = 0, queue = bp->queues; q < bp->num_queues; 5012 ++q, ++queue) 5013 napi_enable(&queue->napi); 5014 5015 if (netdev->hw_features & NETIF_F_NTUPLE) 5016 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 5017 5018 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 5019 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 5020 5021 macb_writel(bp, NCR, MACB_BIT(MPE)); 5022 macb_init_hw(bp); 5023 macb_set_rx_mode(netdev); 5024 macb_restore_features(bp); 5025 rtnl_lock(); 5026 phylink_start(bp->phylink); 5027 rtnl_unlock(); 5028 5029 netif_device_attach(netdev); 5030 if (bp->ptp_info) 5031 bp->ptp_info->ptp_init(netdev); 5032 5033 return 0; 5034 } 5035 5036 static int __maybe_unused macb_runtime_suspend(struct device *dev) 5037 { 5038 struct net_device *netdev = dev_get_drvdata(dev); 5039 struct macb *bp = netdev_priv(netdev); 5040 5041 if (!(device_may_wakeup(dev))) 5042 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); 5043 else 5044 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); 5045 5046 return 0; 5047 } 5048 5049 static int __maybe_unused macb_runtime_resume(struct device *dev) 5050 { 5051 struct net_device *netdev = dev_get_drvdata(dev); 5052 struct macb *bp = netdev_priv(netdev); 5053 5054 if (!(device_may_wakeup(dev))) { 5055 clk_prepare_enable(bp->pclk); 5056 clk_prepare_enable(bp->hclk); 5057 clk_prepare_enable(bp->tx_clk); 5058 clk_prepare_enable(bp->rx_clk); 5059 } 5060 clk_prepare_enable(bp->tsu_clk); 5061 5062 return 0; 5063 } 5064 5065 static const struct dev_pm_ops macb_pm_ops = { 5066 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 5067 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 5068 }; 5069 5070 static struct platform_driver macb_driver = { 5071 .probe = macb_probe, 5072 .remove = macb_remove, 5073 .driver = { 5074 .name = "macb", 5075 .of_match_table = of_match_ptr(macb_dt_ids), 5076 .pm = &macb_pm_ops, 5077 }, 5078 }; 5079 5080 module_platform_driver(macb_driver); 5081 5082 MODULE_LICENSE("GPL"); 5083 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 5084 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 5085 MODULE_ALIAS("platform:macb"); 5086