1 /******************************************************************************* 2 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 3 ST Ethernet IPs are built around a Synopsys IP Core. 4 5 Copyright(C) 2007-2011 STMicroelectronics Ltd 6 7 This program is free software; you can redistribute it and/or modify it 8 under the terms and conditions of the GNU General Public License, 9 version 2, as published by the Free Software Foundation. 10 11 This program is distributed in the hope it will be useful, but WITHOUT 12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 more details. 15 16 The full GNU General Public License is included in this distribution in 17 the file called "COPYING". 18 19 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 20 21 Documentation available at: 22 http://www.stlinux.com 23 Support available at: 24 https://bugzilla.stlinux.com/ 25 *******************************************************************************/ 26 27 #include <linux/clk.h> 28 #include <linux/kernel.h> 29 #include <linux/interrupt.h> 30 #include <linux/ip.h> 31 #include <linux/tcp.h> 32 #include <linux/skbuff.h> 33 #include <linux/ethtool.h> 34 #include <linux/if_ether.h> 35 #include <linux/crc32.h> 36 #include <linux/mii.h> 37 #include <linux/if.h> 38 #include <linux/if_vlan.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/slab.h> 41 #include <linux/prefetch.h> 42 #include <linux/pinctrl/consumer.h> 43 #ifdef CONFIG_DEBUG_FS 44 #include <linux/debugfs.h> 45 #include <linux/seq_file.h> 46 #endif /* CONFIG_DEBUG_FS */ 47 #include <linux/net_tstamp.h> 48 #include "stmmac_ptp.h" 49 #include "stmmac.h" 50 #include <linux/reset.h> 51 #include <linux/of_mdio.h> 52 #include "dwmac1000.h" 53 54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x) 55 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 56 57 /* Module parameters */ 58 #define TX_TIMEO 5000 59 static int watchdog = TX_TIMEO; 60 module_param(watchdog, int, S_IRUGO | S_IWUSR); 61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 62 63 static int debug = -1; 64 module_param(debug, int, S_IRUGO | S_IWUSR); 65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 66 67 static int phyaddr = -1; 68 module_param(phyaddr, int, S_IRUGO); 69 MODULE_PARM_DESC(phyaddr, "Physical device address"); 70 71 #define STMMAC_TX_THRESH (DMA_TX_SIZE / 4) 72 #define STMMAC_RX_THRESH (DMA_RX_SIZE / 4) 73 74 static int flow_ctrl = FLOW_OFF; 75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR); 76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 77 78 static int pause = PAUSE_TIME; 79 module_param(pause, int, S_IRUGO | S_IWUSR); 80 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 81 82 #define TC_DEFAULT 64 83 static int tc = TC_DEFAULT; 84 module_param(tc, int, S_IRUGO | S_IWUSR); 85 MODULE_PARM_DESC(tc, "DMA threshold control value"); 86 87 #define DEFAULT_BUFSIZE 1536 88 static int buf_sz = DEFAULT_BUFSIZE; 89 module_param(buf_sz, int, S_IRUGO | S_IWUSR); 90 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 91 92 #define STMMAC_RX_COPYBREAK 256 93 94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 95 NETIF_MSG_LINK | NETIF_MSG_IFUP | 96 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 97 98 #define STMMAC_DEFAULT_LPI_TIMER 1000 99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR); 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x)) 103 104 /* By default the driver will use the ring mode to manage tx and rx descriptors, 105 * but allow user to force to use the chain instead of the ring 106 */ 107 static unsigned int chain_mode; 108 module_param(chain_mode, int, S_IRUGO); 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 110 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 112 113 #ifdef CONFIG_DEBUG_FS 114 static int stmmac_init_fs(struct net_device *dev); 115 static void stmmac_exit_fs(struct net_device *dev); 116 #endif 117 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x)) 119 120 /** 121 * stmmac_verify_args - verify the driver parameters. 122 * Description: it checks the driver parameters and set a default in case of 123 * errors. 124 */ 125 static void stmmac_verify_args(void) 126 { 127 if (unlikely(watchdog < 0)) 128 watchdog = TX_TIMEO; 129 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 130 buf_sz = DEFAULT_BUFSIZE; 131 if (unlikely(flow_ctrl > 1)) 132 flow_ctrl = FLOW_AUTO; 133 else if (likely(flow_ctrl < 0)) 134 flow_ctrl = FLOW_OFF; 135 if (unlikely((pause < 0) || (pause > 0xffff))) 136 pause = PAUSE_TIME; 137 if (eee_timer < 0) 138 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 139 } 140 141 /** 142 * stmmac_disable_all_queues - Disable all queues 143 * @priv: driver private structure 144 */ 145 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 146 { 147 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 148 u32 queue; 149 150 for (queue = 0; queue < rx_queues_cnt; queue++) { 151 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 152 153 napi_disable(&rx_q->napi); 154 } 155 } 156 157 /** 158 * stmmac_enable_all_queues - Enable all queues 159 * @priv: driver private structure 160 */ 161 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 162 { 163 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 164 u32 queue; 165 166 for (queue = 0; queue < rx_queues_cnt; queue++) { 167 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 168 169 napi_enable(&rx_q->napi); 170 } 171 } 172 173 /** 174 * stmmac_stop_all_queues - Stop all queues 175 * @priv: driver private structure 176 */ 177 static void stmmac_stop_all_queues(struct stmmac_priv *priv) 178 { 179 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 180 u32 queue; 181 182 for (queue = 0; queue < tx_queues_cnt; queue++) 183 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 184 } 185 186 /** 187 * stmmac_start_all_queues - Start all queues 188 * @priv: driver private structure 189 */ 190 static void stmmac_start_all_queues(struct stmmac_priv *priv) 191 { 192 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 193 u32 queue; 194 195 for (queue = 0; queue < tx_queues_cnt; queue++) 196 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue)); 197 } 198 199 /** 200 * stmmac_clk_csr_set - dynamically set the MDC clock 201 * @priv: driver private structure 202 * Description: this is to dynamically set the MDC clock according to the csr 203 * clock input. 204 * Note: 205 * If a specific clk_csr value is passed from the platform 206 * this means that the CSR Clock Range selection cannot be 207 * changed at run-time and it is fixed (as reported in the driver 208 * documentation). Viceversa the driver will try to set the MDC 209 * clock dynamically according to the actual clock input. 210 */ 211 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 212 { 213 u32 clk_rate; 214 215 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 216 217 /* Platform provided default clk_csr would be assumed valid 218 * for all other cases except for the below mentioned ones. 219 * For values higher than the IEEE 802.3 specified frequency 220 * we can not estimate the proper divider as it is not known 221 * the frequency of clk_csr_i. So we do not change the default 222 * divider. 223 */ 224 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 225 if (clk_rate < CSR_F_35M) 226 priv->clk_csr = STMMAC_CSR_20_35M; 227 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 228 priv->clk_csr = STMMAC_CSR_35_60M; 229 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 230 priv->clk_csr = STMMAC_CSR_60_100M; 231 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 232 priv->clk_csr = STMMAC_CSR_100_150M; 233 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 234 priv->clk_csr = STMMAC_CSR_150_250M; 235 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M)) 236 priv->clk_csr = STMMAC_CSR_250_300M; 237 } 238 239 if (priv->plat->has_sun8i) { 240 if (clk_rate > 160000000) 241 priv->clk_csr = 0x03; 242 else if (clk_rate > 80000000) 243 priv->clk_csr = 0x02; 244 else if (clk_rate > 40000000) 245 priv->clk_csr = 0x01; 246 else 247 priv->clk_csr = 0; 248 } 249 } 250 251 static void print_pkt(unsigned char *buf, int len) 252 { 253 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 254 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 255 } 256 257 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 258 { 259 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 260 u32 avail; 261 262 if (tx_q->dirty_tx > tx_q->cur_tx) 263 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 264 else 265 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1; 266 267 return avail; 268 } 269 270 /** 271 * stmmac_rx_dirty - Get RX queue dirty 272 * @priv: driver private structure 273 * @queue: RX queue index 274 */ 275 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 276 { 277 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 278 u32 dirty; 279 280 if (rx_q->dirty_rx <= rx_q->cur_rx) 281 dirty = rx_q->cur_rx - rx_q->dirty_rx; 282 else 283 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx; 284 285 return dirty; 286 } 287 288 /** 289 * stmmac_hw_fix_mac_speed - callback for speed selection 290 * @priv: driver private structure 291 * Description: on some platforms (e.g. ST), some HW system configuration 292 * registers have to be set according to the link speed negotiated. 293 */ 294 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv) 295 { 296 struct net_device *ndev = priv->dev; 297 struct phy_device *phydev = ndev->phydev; 298 299 if (likely(priv->plat->fix_mac_speed)) 300 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed); 301 } 302 303 /** 304 * stmmac_enable_eee_mode - check and enter in LPI mode 305 * @priv: driver private structure 306 * Description: this function is to verify and enter in LPI mode in case of 307 * EEE. 308 */ 309 static void stmmac_enable_eee_mode(struct stmmac_priv *priv) 310 { 311 u32 tx_cnt = priv->plat->tx_queues_to_use; 312 u32 queue; 313 314 /* check if all TX queues have the work finished */ 315 for (queue = 0; queue < tx_cnt; queue++) { 316 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 317 318 if (tx_q->dirty_tx != tx_q->cur_tx) 319 return; /* still unfinished work */ 320 } 321 322 /* Check and enter in LPI mode */ 323 if (!priv->tx_path_in_lpi_mode) 324 priv->hw->mac->set_eee_mode(priv->hw, 325 priv->plat->en_tx_lpi_clockgating); 326 } 327 328 /** 329 * stmmac_disable_eee_mode - disable and exit from LPI mode 330 * @priv: driver private structure 331 * Description: this function is to exit and disable EEE in case of 332 * LPI state is true. This is called by the xmit. 333 */ 334 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 335 { 336 priv->hw->mac->reset_eee_mode(priv->hw); 337 del_timer_sync(&priv->eee_ctrl_timer); 338 priv->tx_path_in_lpi_mode = false; 339 } 340 341 /** 342 * stmmac_eee_ctrl_timer - EEE TX SW timer. 343 * @arg : data hook 344 * Description: 345 * if there is no data transfer and if we are not in LPI state, 346 * then MAC Transmitter can be moved to LPI state. 347 */ 348 static void stmmac_eee_ctrl_timer(unsigned long arg) 349 { 350 struct stmmac_priv *priv = (struct stmmac_priv *)arg; 351 352 stmmac_enable_eee_mode(priv); 353 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 354 } 355 356 /** 357 * stmmac_eee_init - init EEE 358 * @priv: driver private structure 359 * Description: 360 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 361 * can also manage EEE, this function enable the LPI state and start related 362 * timer. 363 */ 364 bool stmmac_eee_init(struct stmmac_priv *priv) 365 { 366 struct net_device *ndev = priv->dev; 367 unsigned long flags; 368 bool ret = false; 369 370 /* Using PCS we cannot dial with the phy registers at this stage 371 * so we do not support extra feature like EEE. 372 */ 373 if ((priv->hw->pcs == STMMAC_PCS_RGMII) || 374 (priv->hw->pcs == STMMAC_PCS_TBI) || 375 (priv->hw->pcs == STMMAC_PCS_RTBI)) 376 goto out; 377 378 /* MAC core supports the EEE feature. */ 379 if (priv->dma_cap.eee) { 380 int tx_lpi_timer = priv->tx_lpi_timer; 381 382 /* Check if the PHY supports EEE */ 383 if (phy_init_eee(ndev->phydev, 1)) { 384 /* To manage at run-time if the EEE cannot be supported 385 * anymore (for example because the lp caps have been 386 * changed). 387 * In that case the driver disable own timers. 388 */ 389 spin_lock_irqsave(&priv->lock, flags); 390 if (priv->eee_active) { 391 netdev_dbg(priv->dev, "disable EEE\n"); 392 del_timer_sync(&priv->eee_ctrl_timer); 393 priv->hw->mac->set_eee_timer(priv->hw, 0, 394 tx_lpi_timer); 395 } 396 priv->eee_active = 0; 397 spin_unlock_irqrestore(&priv->lock, flags); 398 goto out; 399 } 400 /* Activate the EEE and start timers */ 401 spin_lock_irqsave(&priv->lock, flags); 402 if (!priv->eee_active) { 403 priv->eee_active = 1; 404 setup_timer(&priv->eee_ctrl_timer, 405 stmmac_eee_ctrl_timer, 406 (unsigned long)priv); 407 mod_timer(&priv->eee_ctrl_timer, 408 STMMAC_LPI_T(eee_timer)); 409 410 priv->hw->mac->set_eee_timer(priv->hw, 411 STMMAC_DEFAULT_LIT_LS, 412 tx_lpi_timer); 413 } 414 /* Set HW EEE according to the speed */ 415 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link); 416 417 ret = true; 418 spin_unlock_irqrestore(&priv->lock, flags); 419 420 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 421 } 422 out: 423 return ret; 424 } 425 426 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 427 * @priv: driver private structure 428 * @p : descriptor pointer 429 * @skb : the socket buffer 430 * Description : 431 * This function will read timestamp from the descriptor & pass it to stack. 432 * and also perform some sanity checks. 433 */ 434 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 435 struct dma_desc *p, struct sk_buff *skb) 436 { 437 struct skb_shared_hwtstamps shhwtstamp; 438 u64 ns; 439 440 if (!priv->hwts_tx_en) 441 return; 442 443 /* exit if skb doesn't support hw tstamp */ 444 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 445 return; 446 447 /* check tx tstamp status */ 448 if (priv->hw->desc->get_tx_timestamp_status(p)) { 449 /* get the valid tstamp */ 450 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts); 451 452 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 453 shhwtstamp.hwtstamp = ns_to_ktime(ns); 454 455 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 456 /* pass tstamp to stack */ 457 skb_tstamp_tx(skb, &shhwtstamp); 458 } 459 460 return; 461 } 462 463 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 464 * @priv: driver private structure 465 * @p : descriptor pointer 466 * @np : next descriptor pointer 467 * @skb : the socket buffer 468 * Description : 469 * This function will read received packet's timestamp from the descriptor 470 * and pass it to stack. It also perform some sanity checks. 471 */ 472 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 473 struct dma_desc *np, struct sk_buff *skb) 474 { 475 struct skb_shared_hwtstamps *shhwtstamp = NULL; 476 struct dma_desc *desc = p; 477 u64 ns; 478 479 if (!priv->hwts_rx_en) 480 return; 481 /* For GMAC4, the valid timestamp is from CTX next desc. */ 482 if (priv->plat->has_gmac4) 483 desc = np; 484 485 /* Check if timestamp is available */ 486 if (priv->hw->desc->get_rx_timestamp_status(desc, priv->adv_ts)) { 487 ns = priv->hw->desc->get_timestamp(desc, priv->adv_ts); 488 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 489 shhwtstamp = skb_hwtstamps(skb); 490 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 491 shhwtstamp->hwtstamp = ns_to_ktime(ns); 492 } else { 493 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 494 } 495 } 496 497 /** 498 * stmmac_hwtstamp_ioctl - control hardware timestamping. 499 * @dev: device pointer. 500 * @ifr: An IOCTL specific structure, that can contain a pointer to 501 * a proprietary structure used to pass information to the driver. 502 * Description: 503 * This function configures the MAC to enable/disable both outgoing(TX) 504 * and incoming(RX) packets time stamping based on user input. 505 * Return Value: 506 * 0 on success and an appropriate -ve integer on failure. 507 */ 508 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr) 509 { 510 struct stmmac_priv *priv = netdev_priv(dev); 511 struct hwtstamp_config config; 512 struct timespec64 now; 513 u64 temp = 0; 514 u32 ptp_v2 = 0; 515 u32 tstamp_all = 0; 516 u32 ptp_over_ipv4_udp = 0; 517 u32 ptp_over_ipv6_udp = 0; 518 u32 ptp_over_ethernet = 0; 519 u32 snap_type_sel = 0; 520 u32 ts_master_en = 0; 521 u32 ts_event_en = 0; 522 u32 value = 0; 523 u32 sec_inc; 524 525 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 526 netdev_alert(priv->dev, "No support for HW time stamping\n"); 527 priv->hwts_tx_en = 0; 528 priv->hwts_rx_en = 0; 529 530 return -EOPNOTSUPP; 531 } 532 533 if (copy_from_user(&config, ifr->ifr_data, 534 sizeof(struct hwtstamp_config))) 535 return -EFAULT; 536 537 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 538 __func__, config.flags, config.tx_type, config.rx_filter); 539 540 /* reserved for future extensions */ 541 if (config.flags) 542 return -EINVAL; 543 544 if (config.tx_type != HWTSTAMP_TX_OFF && 545 config.tx_type != HWTSTAMP_TX_ON) 546 return -ERANGE; 547 548 if (priv->adv_ts) { 549 switch (config.rx_filter) { 550 case HWTSTAMP_FILTER_NONE: 551 /* time stamp no incoming packet at all */ 552 config.rx_filter = HWTSTAMP_FILTER_NONE; 553 break; 554 555 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 556 /* PTP v1, UDP, any kind of event packet */ 557 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 558 /* take time stamp for all event messages */ 559 if (priv->plat->has_gmac4) 560 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 561 else 562 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 563 564 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 565 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 566 break; 567 568 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 569 /* PTP v1, UDP, Sync packet */ 570 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 571 /* take time stamp for SYNC messages only */ 572 ts_event_en = PTP_TCR_TSEVNTENA; 573 574 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 575 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 576 break; 577 578 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 579 /* PTP v1, UDP, Delay_req packet */ 580 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 581 /* take time stamp for Delay_Req messages only */ 582 ts_master_en = PTP_TCR_TSMSTRENA; 583 ts_event_en = PTP_TCR_TSEVNTENA; 584 585 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 586 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 587 break; 588 589 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 590 /* PTP v2, UDP, any kind of event packet */ 591 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 592 ptp_v2 = PTP_TCR_TSVER2ENA; 593 /* take time stamp for all event messages */ 594 if (priv->plat->has_gmac4) 595 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 596 else 597 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 598 599 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 600 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 601 break; 602 603 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 604 /* PTP v2, UDP, Sync packet */ 605 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 606 ptp_v2 = PTP_TCR_TSVER2ENA; 607 /* take time stamp for SYNC messages only */ 608 ts_event_en = PTP_TCR_TSEVNTENA; 609 610 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 611 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 612 break; 613 614 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 615 /* PTP v2, UDP, Delay_req packet */ 616 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 617 ptp_v2 = PTP_TCR_TSVER2ENA; 618 /* take time stamp for Delay_Req messages only */ 619 ts_master_en = PTP_TCR_TSMSTRENA; 620 ts_event_en = PTP_TCR_TSEVNTENA; 621 622 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 623 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 624 break; 625 626 case HWTSTAMP_FILTER_PTP_V2_EVENT: 627 /* PTP v2/802.AS1 any layer, any kind of event packet */ 628 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 629 ptp_v2 = PTP_TCR_TSVER2ENA; 630 /* take time stamp for all event messages */ 631 if (priv->plat->has_gmac4) 632 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 633 else 634 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 635 636 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 637 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 638 ptp_over_ethernet = PTP_TCR_TSIPENA; 639 break; 640 641 case HWTSTAMP_FILTER_PTP_V2_SYNC: 642 /* PTP v2/802.AS1, any layer, Sync packet */ 643 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 644 ptp_v2 = PTP_TCR_TSVER2ENA; 645 /* take time stamp for SYNC messages only */ 646 ts_event_en = PTP_TCR_TSEVNTENA; 647 648 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 649 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 650 ptp_over_ethernet = PTP_TCR_TSIPENA; 651 break; 652 653 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 654 /* PTP v2/802.AS1, any layer, Delay_req packet */ 655 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 656 ptp_v2 = PTP_TCR_TSVER2ENA; 657 /* take time stamp for Delay_Req messages only */ 658 ts_master_en = PTP_TCR_TSMSTRENA; 659 ts_event_en = PTP_TCR_TSEVNTENA; 660 661 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 662 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 663 ptp_over_ethernet = PTP_TCR_TSIPENA; 664 break; 665 666 case HWTSTAMP_FILTER_NTP_ALL: 667 case HWTSTAMP_FILTER_ALL: 668 /* time stamp any incoming packet */ 669 config.rx_filter = HWTSTAMP_FILTER_ALL; 670 tstamp_all = PTP_TCR_TSENALL; 671 break; 672 673 default: 674 return -ERANGE; 675 } 676 } else { 677 switch (config.rx_filter) { 678 case HWTSTAMP_FILTER_NONE: 679 config.rx_filter = HWTSTAMP_FILTER_NONE; 680 break; 681 default: 682 /* PTP v1, UDP, any kind of event packet */ 683 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 684 break; 685 } 686 } 687 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 688 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 689 690 if (!priv->hwts_tx_en && !priv->hwts_rx_en) 691 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0); 692 else { 693 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR | 694 tstamp_all | ptp_v2 | ptp_over_ethernet | 695 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en | 696 ts_master_en | snap_type_sel); 697 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value); 698 699 /* program Sub Second Increment reg */ 700 sec_inc = priv->hw->ptp->config_sub_second_increment( 701 priv->ptpaddr, priv->plat->clk_ptp_rate, 702 priv->plat->has_gmac4); 703 temp = div_u64(1000000000ULL, sec_inc); 704 705 /* calculate default added value: 706 * formula is : 707 * addend = (2^32)/freq_div_ratio; 708 * where, freq_div_ratio = 1e9ns/sec_inc 709 */ 710 temp = (u64)(temp << 32); 711 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 712 priv->hw->ptp->config_addend(priv->ptpaddr, 713 priv->default_addend); 714 715 /* initialize system time */ 716 ktime_get_real_ts64(&now); 717 718 /* lower 32 bits of tv_sec are safe until y2106 */ 719 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec, 720 now.tv_nsec); 721 } 722 723 return copy_to_user(ifr->ifr_data, &config, 724 sizeof(struct hwtstamp_config)) ? -EFAULT : 0; 725 } 726 727 /** 728 * stmmac_init_ptp - init PTP 729 * @priv: driver private structure 730 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 731 * This is done by looking at the HW cap. register. 732 * This function also registers the ptp driver. 733 */ 734 static int stmmac_init_ptp(struct stmmac_priv *priv) 735 { 736 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 737 return -EOPNOTSUPP; 738 739 priv->adv_ts = 0; 740 /* Check if adv_ts can be enabled for dwmac 4.x core */ 741 if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp) 742 priv->adv_ts = 1; 743 /* Dwmac 3.x core with extend_desc can support adv_ts */ 744 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 745 priv->adv_ts = 1; 746 747 if (priv->dma_cap.time_stamp) 748 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 749 750 if (priv->adv_ts) 751 netdev_info(priv->dev, 752 "IEEE 1588-2008 Advanced Timestamp supported\n"); 753 754 priv->hw->ptp = &stmmac_ptp; 755 priv->hwts_tx_en = 0; 756 priv->hwts_rx_en = 0; 757 758 stmmac_ptp_register(priv); 759 760 return 0; 761 } 762 763 static void stmmac_release_ptp(struct stmmac_priv *priv) 764 { 765 if (priv->plat->clk_ptp_ref) 766 clk_disable_unprepare(priv->plat->clk_ptp_ref); 767 stmmac_ptp_unregister(priv); 768 } 769 770 /** 771 * stmmac_mac_flow_ctrl - Configure flow control in all queues 772 * @priv: driver private structure 773 * Description: It is used for configuring the flow control in all queues 774 */ 775 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 776 { 777 u32 tx_cnt = priv->plat->tx_queues_to_use; 778 779 priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl, 780 priv->pause, tx_cnt); 781 } 782 783 /** 784 * stmmac_adjust_link - adjusts the link parameters 785 * @dev: net device structure 786 * Description: this is the helper called by the physical abstraction layer 787 * drivers to communicate the phy link status. According the speed and duplex 788 * this driver can invoke registered glue-logic as well. 789 * It also invoke the eee initialization because it could happen when switch 790 * on different networks (that are eee capable). 791 */ 792 static void stmmac_adjust_link(struct net_device *dev) 793 { 794 struct stmmac_priv *priv = netdev_priv(dev); 795 struct phy_device *phydev = dev->phydev; 796 unsigned long flags; 797 bool new_state = false; 798 799 if (!phydev) 800 return; 801 802 spin_lock_irqsave(&priv->lock, flags); 803 804 if (phydev->link) { 805 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 806 807 /* Now we make sure that we can be in full duplex mode. 808 * If not, we operate in half-duplex mode. */ 809 if (phydev->duplex != priv->oldduplex) { 810 new_state = true; 811 if (!phydev->duplex) 812 ctrl &= ~priv->hw->link.duplex; 813 else 814 ctrl |= priv->hw->link.duplex; 815 priv->oldduplex = phydev->duplex; 816 } 817 /* Flow Control operation */ 818 if (phydev->pause) 819 stmmac_mac_flow_ctrl(priv, phydev->duplex); 820 821 if (phydev->speed != priv->speed) { 822 new_state = true; 823 ctrl &= ~priv->hw->link.speed_mask; 824 switch (phydev->speed) { 825 case SPEED_1000: 826 ctrl |= priv->hw->link.speed1000; 827 break; 828 case SPEED_100: 829 ctrl |= priv->hw->link.speed100; 830 break; 831 case SPEED_10: 832 ctrl |= priv->hw->link.speed10; 833 break; 834 default: 835 netif_warn(priv, link, priv->dev, 836 "broken speed: %d\n", phydev->speed); 837 phydev->speed = SPEED_UNKNOWN; 838 break; 839 } 840 if (phydev->speed != SPEED_UNKNOWN) 841 stmmac_hw_fix_mac_speed(priv); 842 priv->speed = phydev->speed; 843 } 844 845 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 846 847 if (!priv->oldlink) { 848 new_state = true; 849 priv->oldlink = true; 850 } 851 } else if (priv->oldlink) { 852 new_state = true; 853 priv->oldlink = false; 854 priv->speed = SPEED_UNKNOWN; 855 priv->oldduplex = DUPLEX_UNKNOWN; 856 } 857 858 if (new_state && netif_msg_link(priv)) 859 phy_print_status(phydev); 860 861 spin_unlock_irqrestore(&priv->lock, flags); 862 863 if (phydev->is_pseudo_fixed_link) 864 /* Stop PHY layer to call the hook to adjust the link in case 865 * of a switch is attached to the stmmac driver. 866 */ 867 phydev->irq = PHY_IGNORE_INTERRUPT; 868 else 869 /* At this stage, init the EEE if supported. 870 * Never called in case of fixed_link. 871 */ 872 priv->eee_enabled = stmmac_eee_init(priv); 873 } 874 875 /** 876 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 877 * @priv: driver private structure 878 * Description: this is to verify if the HW supports the PCS. 879 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 880 * configured for the TBI, RTBI, or SGMII PHY interface. 881 */ 882 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 883 { 884 int interface = priv->plat->interface; 885 886 if (priv->dma_cap.pcs) { 887 if ((interface == PHY_INTERFACE_MODE_RGMII) || 888 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 889 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 890 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 891 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 892 priv->hw->pcs = STMMAC_PCS_RGMII; 893 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 894 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 895 priv->hw->pcs = STMMAC_PCS_SGMII; 896 } 897 } 898 } 899 900 /** 901 * stmmac_init_phy - PHY initialization 902 * @dev: net device structure 903 * Description: it initializes the driver's PHY state, and attaches the PHY 904 * to the mac driver. 905 * Return value: 906 * 0 on success 907 */ 908 static int stmmac_init_phy(struct net_device *dev) 909 { 910 struct stmmac_priv *priv = netdev_priv(dev); 911 struct phy_device *phydev; 912 char phy_id_fmt[MII_BUS_ID_SIZE + 3]; 913 char bus_id[MII_BUS_ID_SIZE]; 914 int interface = priv->plat->interface; 915 int max_speed = priv->plat->max_speed; 916 priv->oldlink = false; 917 priv->speed = SPEED_UNKNOWN; 918 priv->oldduplex = DUPLEX_UNKNOWN; 919 920 if (priv->plat->phy_node) { 921 phydev = of_phy_connect(dev, priv->plat->phy_node, 922 &stmmac_adjust_link, 0, interface); 923 } else { 924 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x", 925 priv->plat->bus_id); 926 927 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id, 928 priv->plat->phy_addr); 929 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__, 930 phy_id_fmt); 931 932 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link, 933 interface); 934 } 935 936 if (IS_ERR_OR_NULL(phydev)) { 937 netdev_err(priv->dev, "Could not attach to PHY\n"); 938 if (!phydev) 939 return -ENODEV; 940 941 return PTR_ERR(phydev); 942 } 943 944 /* Stop Advertising 1000BASE Capability if interface is not GMII */ 945 if ((interface == PHY_INTERFACE_MODE_MII) || 946 (interface == PHY_INTERFACE_MODE_RMII) || 947 (max_speed < 1000 && max_speed > 0)) 948 phydev->advertising &= ~(SUPPORTED_1000baseT_Half | 949 SUPPORTED_1000baseT_Full); 950 951 /* 952 * Broken HW is sometimes missing the pull-up resistor on the 953 * MDIO line, which results in reads to non-existent devices returning 954 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent 955 * device as well. 956 * Note: phydev->phy_id is the result of reading the UID PHY registers. 957 */ 958 if (!priv->plat->phy_node && phydev->phy_id == 0) { 959 phy_disconnect(phydev); 960 return -ENODEV; 961 } 962 963 /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid 964 * subsequent PHY polling, make sure we force a link transition if 965 * we have a UP/DOWN/UP transition 966 */ 967 if (phydev->is_pseudo_fixed_link) 968 phydev->irq = PHY_POLL; 969 970 phy_attached_info(phydev); 971 return 0; 972 } 973 974 static void stmmac_display_rx_rings(struct stmmac_priv *priv) 975 { 976 u32 rx_cnt = priv->plat->rx_queues_to_use; 977 void *head_rx; 978 u32 queue; 979 980 /* Display RX rings */ 981 for (queue = 0; queue < rx_cnt; queue++) { 982 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 983 984 pr_info("\tRX Queue %u rings\n", queue); 985 986 if (priv->extend_desc) 987 head_rx = (void *)rx_q->dma_erx; 988 else 989 head_rx = (void *)rx_q->dma_rx; 990 991 /* Display RX ring */ 992 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true); 993 } 994 } 995 996 static void stmmac_display_tx_rings(struct stmmac_priv *priv) 997 { 998 u32 tx_cnt = priv->plat->tx_queues_to_use; 999 void *head_tx; 1000 u32 queue; 1001 1002 /* Display TX rings */ 1003 for (queue = 0; queue < tx_cnt; queue++) { 1004 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1005 1006 pr_info("\tTX Queue %d rings\n", queue); 1007 1008 if (priv->extend_desc) 1009 head_tx = (void *)tx_q->dma_etx; 1010 else 1011 head_tx = (void *)tx_q->dma_tx; 1012 1013 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false); 1014 } 1015 } 1016 1017 static void stmmac_display_rings(struct stmmac_priv *priv) 1018 { 1019 /* Display RX ring */ 1020 stmmac_display_rx_rings(priv); 1021 1022 /* Display TX ring */ 1023 stmmac_display_tx_rings(priv); 1024 } 1025 1026 static int stmmac_set_bfsize(int mtu, int bufsize) 1027 { 1028 int ret = bufsize; 1029 1030 if (mtu >= BUF_SIZE_4KiB) 1031 ret = BUF_SIZE_8KiB; 1032 else if (mtu >= BUF_SIZE_2KiB) 1033 ret = BUF_SIZE_4KiB; 1034 else if (mtu > DEFAULT_BUFSIZE) 1035 ret = BUF_SIZE_2KiB; 1036 else 1037 ret = DEFAULT_BUFSIZE; 1038 1039 return ret; 1040 } 1041 1042 /** 1043 * stmmac_clear_rx_descriptors - clear RX descriptors 1044 * @priv: driver private structure 1045 * @queue: RX queue index 1046 * Description: this function is called to clear the RX descriptors 1047 * in case of both basic and extended descriptors are used. 1048 */ 1049 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) 1050 { 1051 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1052 int i; 1053 1054 /* Clear the RX descriptors */ 1055 for (i = 0; i < DMA_RX_SIZE; i++) 1056 if (priv->extend_desc) 1057 priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic, 1058 priv->use_riwt, priv->mode, 1059 (i == DMA_RX_SIZE - 1)); 1060 else 1061 priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i], 1062 priv->use_riwt, priv->mode, 1063 (i == DMA_RX_SIZE - 1)); 1064 } 1065 1066 /** 1067 * stmmac_clear_tx_descriptors - clear tx descriptors 1068 * @priv: driver private structure 1069 * @queue: TX queue index. 1070 * Description: this function is called to clear the TX descriptors 1071 * in case of both basic and extended descriptors are used. 1072 */ 1073 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) 1074 { 1075 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1076 int i; 1077 1078 /* Clear the TX descriptors */ 1079 for (i = 0; i < DMA_TX_SIZE; i++) 1080 if (priv->extend_desc) 1081 priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic, 1082 priv->mode, 1083 (i == DMA_TX_SIZE - 1)); 1084 else 1085 priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i], 1086 priv->mode, 1087 (i == DMA_TX_SIZE - 1)); 1088 } 1089 1090 /** 1091 * stmmac_clear_descriptors - clear descriptors 1092 * @priv: driver private structure 1093 * Description: this function is called to clear the TX and RX descriptors 1094 * in case of both basic and extended descriptors are used. 1095 */ 1096 static void stmmac_clear_descriptors(struct stmmac_priv *priv) 1097 { 1098 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1099 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1100 u32 queue; 1101 1102 /* Clear the RX descriptors */ 1103 for (queue = 0; queue < rx_queue_cnt; queue++) 1104 stmmac_clear_rx_descriptors(priv, queue); 1105 1106 /* Clear the TX descriptors */ 1107 for (queue = 0; queue < tx_queue_cnt; queue++) 1108 stmmac_clear_tx_descriptors(priv, queue); 1109 } 1110 1111 /** 1112 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1113 * @priv: driver private structure 1114 * @p: descriptor pointer 1115 * @i: descriptor index 1116 * @flags: gfp flag 1117 * @queue: RX queue index 1118 * Description: this function is called to allocate a receive buffer, perform 1119 * the DMA mapping and init the descriptor. 1120 */ 1121 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, 1122 int i, gfp_t flags, u32 queue) 1123 { 1124 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1125 struct sk_buff *skb; 1126 1127 skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags); 1128 if (!skb) { 1129 netdev_err(priv->dev, 1130 "%s: Rx init fails; skb is NULL\n", __func__); 1131 return -ENOMEM; 1132 } 1133 rx_q->rx_skbuff[i] = skb; 1134 rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data, 1135 priv->dma_buf_sz, 1136 DMA_FROM_DEVICE); 1137 if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) { 1138 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__); 1139 dev_kfree_skb_any(skb); 1140 return -EINVAL; 1141 } 1142 1143 if (priv->synopsys_id >= DWMAC_CORE_4_00) 1144 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]); 1145 else 1146 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]); 1147 1148 if ((priv->hw->mode->init_desc3) && 1149 (priv->dma_buf_sz == BUF_SIZE_16KiB)) 1150 priv->hw->mode->init_desc3(p); 1151 1152 return 0; 1153 } 1154 1155 /** 1156 * stmmac_free_rx_buffer - free RX dma buffers 1157 * @priv: private structure 1158 * @queue: RX queue index 1159 * @i: buffer index. 1160 */ 1161 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1162 { 1163 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1164 1165 if (rx_q->rx_skbuff[i]) { 1166 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i], 1167 priv->dma_buf_sz, DMA_FROM_DEVICE); 1168 dev_kfree_skb_any(rx_q->rx_skbuff[i]); 1169 } 1170 rx_q->rx_skbuff[i] = NULL; 1171 } 1172 1173 /** 1174 * stmmac_free_tx_buffer - free RX dma buffers 1175 * @priv: private structure 1176 * @queue: RX queue index 1177 * @i: buffer index. 1178 */ 1179 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1180 { 1181 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1182 1183 if (tx_q->tx_skbuff_dma[i].buf) { 1184 if (tx_q->tx_skbuff_dma[i].map_as_page) 1185 dma_unmap_page(priv->device, 1186 tx_q->tx_skbuff_dma[i].buf, 1187 tx_q->tx_skbuff_dma[i].len, 1188 DMA_TO_DEVICE); 1189 else 1190 dma_unmap_single(priv->device, 1191 tx_q->tx_skbuff_dma[i].buf, 1192 tx_q->tx_skbuff_dma[i].len, 1193 DMA_TO_DEVICE); 1194 } 1195 1196 if (tx_q->tx_skbuff[i]) { 1197 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1198 tx_q->tx_skbuff[i] = NULL; 1199 tx_q->tx_skbuff_dma[i].buf = 0; 1200 tx_q->tx_skbuff_dma[i].map_as_page = false; 1201 } 1202 } 1203 1204 /** 1205 * init_dma_rx_desc_rings - init the RX descriptor rings 1206 * @dev: net device structure 1207 * @flags: gfp flag. 1208 * Description: this function initializes the DMA RX descriptors 1209 * and allocates the socket buffers. It supports the chained and ring 1210 * modes. 1211 */ 1212 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) 1213 { 1214 struct stmmac_priv *priv = netdev_priv(dev); 1215 u32 rx_count = priv->plat->rx_queues_to_use; 1216 unsigned int bfsize = 0; 1217 int ret = -ENOMEM; 1218 int queue; 1219 int i; 1220 1221 if (priv->hw->mode->set_16kib_bfsize) 1222 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu); 1223 1224 if (bfsize < BUF_SIZE_16KiB) 1225 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); 1226 1227 priv->dma_buf_sz = bfsize; 1228 1229 /* RX INITIALIZATION */ 1230 netif_dbg(priv, probe, priv->dev, 1231 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1232 1233 for (queue = 0; queue < rx_count; queue++) { 1234 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1235 1236 netif_dbg(priv, probe, priv->dev, 1237 "(%s) dma_rx_phy=0x%08x\n", __func__, 1238 (u32)rx_q->dma_rx_phy); 1239 1240 for (i = 0; i < DMA_RX_SIZE; i++) { 1241 struct dma_desc *p; 1242 1243 if (priv->extend_desc) 1244 p = &((rx_q->dma_erx + i)->basic); 1245 else 1246 p = rx_q->dma_rx + i; 1247 1248 ret = stmmac_init_rx_buffers(priv, p, i, flags, 1249 queue); 1250 if (ret) 1251 goto err_init_rx_buffers; 1252 1253 netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n", 1254 rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data, 1255 (unsigned int)rx_q->rx_skbuff_dma[i]); 1256 } 1257 1258 rx_q->cur_rx = 0; 1259 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE); 1260 1261 stmmac_clear_rx_descriptors(priv, queue); 1262 1263 /* Setup the chained descriptor addresses */ 1264 if (priv->mode == STMMAC_CHAIN_MODE) { 1265 if (priv->extend_desc) 1266 priv->hw->mode->init(rx_q->dma_erx, 1267 rx_q->dma_rx_phy, 1268 DMA_RX_SIZE, 1); 1269 else 1270 priv->hw->mode->init(rx_q->dma_rx, 1271 rx_q->dma_rx_phy, 1272 DMA_RX_SIZE, 0); 1273 } 1274 } 1275 1276 buf_sz = bfsize; 1277 1278 return 0; 1279 1280 err_init_rx_buffers: 1281 while (queue >= 0) { 1282 while (--i >= 0) 1283 stmmac_free_rx_buffer(priv, queue, i); 1284 1285 if (queue == 0) 1286 break; 1287 1288 i = DMA_RX_SIZE; 1289 queue--; 1290 } 1291 1292 return ret; 1293 } 1294 1295 /** 1296 * init_dma_tx_desc_rings - init the TX descriptor rings 1297 * @dev: net device structure. 1298 * Description: this function initializes the DMA TX descriptors 1299 * and allocates the socket buffers. It supports the chained and ring 1300 * modes. 1301 */ 1302 static int init_dma_tx_desc_rings(struct net_device *dev) 1303 { 1304 struct stmmac_priv *priv = netdev_priv(dev); 1305 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1306 u32 queue; 1307 int i; 1308 1309 for (queue = 0; queue < tx_queue_cnt; queue++) { 1310 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1311 1312 netif_dbg(priv, probe, priv->dev, 1313 "(%s) dma_tx_phy=0x%08x\n", __func__, 1314 (u32)tx_q->dma_tx_phy); 1315 1316 /* Setup the chained descriptor addresses */ 1317 if (priv->mode == STMMAC_CHAIN_MODE) { 1318 if (priv->extend_desc) 1319 priv->hw->mode->init(tx_q->dma_etx, 1320 tx_q->dma_tx_phy, 1321 DMA_TX_SIZE, 1); 1322 else 1323 priv->hw->mode->init(tx_q->dma_tx, 1324 tx_q->dma_tx_phy, 1325 DMA_TX_SIZE, 0); 1326 } 1327 1328 for (i = 0; i < DMA_TX_SIZE; i++) { 1329 struct dma_desc *p; 1330 if (priv->extend_desc) 1331 p = &((tx_q->dma_etx + i)->basic); 1332 else 1333 p = tx_q->dma_tx + i; 1334 1335 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1336 p->des0 = 0; 1337 p->des1 = 0; 1338 p->des2 = 0; 1339 p->des3 = 0; 1340 } else { 1341 p->des2 = 0; 1342 } 1343 1344 tx_q->tx_skbuff_dma[i].buf = 0; 1345 tx_q->tx_skbuff_dma[i].map_as_page = false; 1346 tx_q->tx_skbuff_dma[i].len = 0; 1347 tx_q->tx_skbuff_dma[i].last_segment = false; 1348 tx_q->tx_skbuff[i] = NULL; 1349 } 1350 1351 tx_q->dirty_tx = 0; 1352 tx_q->cur_tx = 0; 1353 1354 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 1355 } 1356 1357 return 0; 1358 } 1359 1360 /** 1361 * init_dma_desc_rings - init the RX/TX descriptor rings 1362 * @dev: net device structure 1363 * @flags: gfp flag. 1364 * Description: this function initializes the DMA RX/TX descriptors 1365 * and allocates the socket buffers. It supports the chained and ring 1366 * modes. 1367 */ 1368 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags) 1369 { 1370 struct stmmac_priv *priv = netdev_priv(dev); 1371 int ret; 1372 1373 ret = init_dma_rx_desc_rings(dev, flags); 1374 if (ret) 1375 return ret; 1376 1377 ret = init_dma_tx_desc_rings(dev); 1378 1379 stmmac_clear_descriptors(priv); 1380 1381 if (netif_msg_hw(priv)) 1382 stmmac_display_rings(priv); 1383 1384 return ret; 1385 } 1386 1387 /** 1388 * dma_free_rx_skbufs - free RX dma buffers 1389 * @priv: private structure 1390 * @queue: RX queue index 1391 */ 1392 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) 1393 { 1394 int i; 1395 1396 for (i = 0; i < DMA_RX_SIZE; i++) 1397 stmmac_free_rx_buffer(priv, queue, i); 1398 } 1399 1400 /** 1401 * dma_free_tx_skbufs - free TX dma buffers 1402 * @priv: private structure 1403 * @queue: TX queue index 1404 */ 1405 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) 1406 { 1407 int i; 1408 1409 for (i = 0; i < DMA_TX_SIZE; i++) 1410 stmmac_free_tx_buffer(priv, queue, i); 1411 } 1412 1413 /** 1414 * free_dma_rx_desc_resources - free RX dma desc resources 1415 * @priv: private structure 1416 */ 1417 static void free_dma_rx_desc_resources(struct stmmac_priv *priv) 1418 { 1419 u32 rx_count = priv->plat->rx_queues_to_use; 1420 u32 queue; 1421 1422 /* Free RX queue resources */ 1423 for (queue = 0; queue < rx_count; queue++) { 1424 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1425 1426 /* Release the DMA RX socket buffers */ 1427 dma_free_rx_skbufs(priv, queue); 1428 1429 /* Free DMA regions of consistent memory previously allocated */ 1430 if (!priv->extend_desc) 1431 dma_free_coherent(priv->device, 1432 DMA_RX_SIZE * sizeof(struct dma_desc), 1433 rx_q->dma_rx, rx_q->dma_rx_phy); 1434 else 1435 dma_free_coherent(priv->device, DMA_RX_SIZE * 1436 sizeof(struct dma_extended_desc), 1437 rx_q->dma_erx, rx_q->dma_rx_phy); 1438 1439 kfree(rx_q->rx_skbuff_dma); 1440 kfree(rx_q->rx_skbuff); 1441 } 1442 } 1443 1444 /** 1445 * free_dma_tx_desc_resources - free TX dma desc resources 1446 * @priv: private structure 1447 */ 1448 static void free_dma_tx_desc_resources(struct stmmac_priv *priv) 1449 { 1450 u32 tx_count = priv->plat->tx_queues_to_use; 1451 u32 queue; 1452 1453 /* Free TX queue resources */ 1454 for (queue = 0; queue < tx_count; queue++) { 1455 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1456 1457 /* Release the DMA TX socket buffers */ 1458 dma_free_tx_skbufs(priv, queue); 1459 1460 /* Free DMA regions of consistent memory previously allocated */ 1461 if (!priv->extend_desc) 1462 dma_free_coherent(priv->device, 1463 DMA_TX_SIZE * sizeof(struct dma_desc), 1464 tx_q->dma_tx, tx_q->dma_tx_phy); 1465 else 1466 dma_free_coherent(priv->device, DMA_TX_SIZE * 1467 sizeof(struct dma_extended_desc), 1468 tx_q->dma_etx, tx_q->dma_tx_phy); 1469 1470 kfree(tx_q->tx_skbuff_dma); 1471 kfree(tx_q->tx_skbuff); 1472 } 1473 } 1474 1475 /** 1476 * alloc_dma_rx_desc_resources - alloc RX resources. 1477 * @priv: private structure 1478 * Description: according to which descriptor can be used (extend or basic) 1479 * this function allocates the resources for TX and RX paths. In case of 1480 * reception, for example, it pre-allocated the RX socket buffer in order to 1481 * allow zero-copy mechanism. 1482 */ 1483 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) 1484 { 1485 u32 rx_count = priv->plat->rx_queues_to_use; 1486 int ret = -ENOMEM; 1487 u32 queue; 1488 1489 /* RX queues buffers and DMA */ 1490 for (queue = 0; queue < rx_count; queue++) { 1491 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1492 1493 rx_q->queue_index = queue; 1494 rx_q->priv_data = priv; 1495 1496 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE, 1497 sizeof(dma_addr_t), 1498 GFP_KERNEL); 1499 if (!rx_q->rx_skbuff_dma) 1500 goto err_dma; 1501 1502 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE, 1503 sizeof(struct sk_buff *), 1504 GFP_KERNEL); 1505 if (!rx_q->rx_skbuff) 1506 goto err_dma; 1507 1508 if (priv->extend_desc) { 1509 rx_q->dma_erx = dma_zalloc_coherent(priv->device, 1510 DMA_RX_SIZE * 1511 sizeof(struct 1512 dma_extended_desc), 1513 &rx_q->dma_rx_phy, 1514 GFP_KERNEL); 1515 if (!rx_q->dma_erx) 1516 goto err_dma; 1517 1518 } else { 1519 rx_q->dma_rx = dma_zalloc_coherent(priv->device, 1520 DMA_RX_SIZE * 1521 sizeof(struct 1522 dma_desc), 1523 &rx_q->dma_rx_phy, 1524 GFP_KERNEL); 1525 if (!rx_q->dma_rx) 1526 goto err_dma; 1527 } 1528 } 1529 1530 return 0; 1531 1532 err_dma: 1533 free_dma_rx_desc_resources(priv); 1534 1535 return ret; 1536 } 1537 1538 /** 1539 * alloc_dma_tx_desc_resources - alloc TX resources. 1540 * @priv: private structure 1541 * Description: according to which descriptor can be used (extend or basic) 1542 * this function allocates the resources for TX and RX paths. In case of 1543 * reception, for example, it pre-allocated the RX socket buffer in order to 1544 * allow zero-copy mechanism. 1545 */ 1546 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) 1547 { 1548 u32 tx_count = priv->plat->tx_queues_to_use; 1549 int ret = -ENOMEM; 1550 u32 queue; 1551 1552 /* TX queues buffers and DMA */ 1553 for (queue = 0; queue < tx_count; queue++) { 1554 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1555 1556 tx_q->queue_index = queue; 1557 tx_q->priv_data = priv; 1558 1559 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE, 1560 sizeof(*tx_q->tx_skbuff_dma), 1561 GFP_KERNEL); 1562 if (!tx_q->tx_skbuff_dma) 1563 goto err_dma; 1564 1565 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE, 1566 sizeof(struct sk_buff *), 1567 GFP_KERNEL); 1568 if (!tx_q->tx_skbuff) 1569 goto err_dma; 1570 1571 if (priv->extend_desc) { 1572 tx_q->dma_etx = dma_zalloc_coherent(priv->device, 1573 DMA_TX_SIZE * 1574 sizeof(struct 1575 dma_extended_desc), 1576 &tx_q->dma_tx_phy, 1577 GFP_KERNEL); 1578 if (!tx_q->dma_etx) 1579 goto err_dma; 1580 } else { 1581 tx_q->dma_tx = dma_zalloc_coherent(priv->device, 1582 DMA_TX_SIZE * 1583 sizeof(struct 1584 dma_desc), 1585 &tx_q->dma_tx_phy, 1586 GFP_KERNEL); 1587 if (!tx_q->dma_tx) 1588 goto err_dma; 1589 } 1590 } 1591 1592 return 0; 1593 1594 err_dma: 1595 free_dma_tx_desc_resources(priv); 1596 1597 return ret; 1598 } 1599 1600 /** 1601 * alloc_dma_desc_resources - alloc TX/RX resources. 1602 * @priv: private structure 1603 * Description: according to which descriptor can be used (extend or basic) 1604 * this function allocates the resources for TX and RX paths. In case of 1605 * reception, for example, it pre-allocated the RX socket buffer in order to 1606 * allow zero-copy mechanism. 1607 */ 1608 static int alloc_dma_desc_resources(struct stmmac_priv *priv) 1609 { 1610 /* RX Allocation */ 1611 int ret = alloc_dma_rx_desc_resources(priv); 1612 1613 if (ret) 1614 return ret; 1615 1616 ret = alloc_dma_tx_desc_resources(priv); 1617 1618 return ret; 1619 } 1620 1621 /** 1622 * free_dma_desc_resources - free dma desc resources 1623 * @priv: private structure 1624 */ 1625 static void free_dma_desc_resources(struct stmmac_priv *priv) 1626 { 1627 /* Release the DMA RX socket buffers */ 1628 free_dma_rx_desc_resources(priv); 1629 1630 /* Release the DMA TX socket buffers */ 1631 free_dma_tx_desc_resources(priv); 1632 } 1633 1634 /** 1635 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 1636 * @priv: driver private structure 1637 * Description: It is used for enabling the rx queues in the MAC 1638 */ 1639 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 1640 { 1641 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1642 int queue; 1643 u8 mode; 1644 1645 for (queue = 0; queue < rx_queues_count; queue++) { 1646 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 1647 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue); 1648 } 1649 } 1650 1651 /** 1652 * stmmac_start_rx_dma - start RX DMA channel 1653 * @priv: driver private structure 1654 * @chan: RX channel index 1655 * Description: 1656 * This starts a RX DMA channel 1657 */ 1658 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 1659 { 1660 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 1661 priv->hw->dma->start_rx(priv->ioaddr, chan); 1662 } 1663 1664 /** 1665 * stmmac_start_tx_dma - start TX DMA channel 1666 * @priv: driver private structure 1667 * @chan: TX channel index 1668 * Description: 1669 * This starts a TX DMA channel 1670 */ 1671 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 1672 { 1673 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 1674 priv->hw->dma->start_tx(priv->ioaddr, chan); 1675 } 1676 1677 /** 1678 * stmmac_stop_rx_dma - stop RX DMA channel 1679 * @priv: driver private structure 1680 * @chan: RX channel index 1681 * Description: 1682 * This stops a RX DMA channel 1683 */ 1684 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 1685 { 1686 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 1687 priv->hw->dma->stop_rx(priv->ioaddr, chan); 1688 } 1689 1690 /** 1691 * stmmac_stop_tx_dma - stop TX DMA channel 1692 * @priv: driver private structure 1693 * @chan: TX channel index 1694 * Description: 1695 * This stops a TX DMA channel 1696 */ 1697 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 1698 { 1699 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 1700 priv->hw->dma->stop_tx(priv->ioaddr, chan); 1701 } 1702 1703 /** 1704 * stmmac_start_all_dma - start all RX and TX DMA channels 1705 * @priv: driver private structure 1706 * Description: 1707 * This starts all the RX and TX DMA channels 1708 */ 1709 static void stmmac_start_all_dma(struct stmmac_priv *priv) 1710 { 1711 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1712 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1713 u32 chan = 0; 1714 1715 for (chan = 0; chan < rx_channels_count; chan++) 1716 stmmac_start_rx_dma(priv, chan); 1717 1718 for (chan = 0; chan < tx_channels_count; chan++) 1719 stmmac_start_tx_dma(priv, chan); 1720 } 1721 1722 /** 1723 * stmmac_stop_all_dma - stop all RX and TX DMA channels 1724 * @priv: driver private structure 1725 * Description: 1726 * This stops the RX and TX DMA channels 1727 */ 1728 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 1729 { 1730 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1731 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1732 u32 chan = 0; 1733 1734 for (chan = 0; chan < rx_channels_count; chan++) 1735 stmmac_stop_rx_dma(priv, chan); 1736 1737 for (chan = 0; chan < tx_channels_count; chan++) 1738 stmmac_stop_tx_dma(priv, chan); 1739 } 1740 1741 /** 1742 * stmmac_dma_operation_mode - HW DMA operation mode 1743 * @priv: driver private structure 1744 * Description: it is used for configuring the DMA operation mode register in 1745 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 1746 */ 1747 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 1748 { 1749 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1750 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1751 int rxfifosz = priv->plat->rx_fifo_size; 1752 u32 txmode = 0; 1753 u32 rxmode = 0; 1754 u32 chan = 0; 1755 1756 if (rxfifosz == 0) 1757 rxfifosz = priv->dma_cap.rx_fifo_size; 1758 1759 if (priv->plat->force_thresh_dma_mode) { 1760 txmode = tc; 1761 rxmode = tc; 1762 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 1763 /* 1764 * In case of GMAC, SF mode can be enabled 1765 * to perform the TX COE in HW. This depends on: 1766 * 1) TX COE if actually supported 1767 * 2) There is no bugged Jumbo frame support 1768 * that needs to not insert csum in the TDES. 1769 */ 1770 txmode = SF_DMA_MODE; 1771 rxmode = SF_DMA_MODE; 1772 priv->xstats.threshold = SF_DMA_MODE; 1773 } else { 1774 txmode = tc; 1775 rxmode = SF_DMA_MODE; 1776 } 1777 1778 /* configure all channels */ 1779 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1780 for (chan = 0; chan < rx_channels_count; chan++) 1781 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan, 1782 rxfifosz); 1783 1784 for (chan = 0; chan < tx_channels_count; chan++) 1785 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan); 1786 } else { 1787 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode, 1788 rxfifosz); 1789 } 1790 } 1791 1792 /** 1793 * stmmac_tx_clean - to manage the transmission completion 1794 * @priv: driver private structure 1795 * @queue: TX queue index 1796 * Description: it reclaims the transmit resources after transmission completes. 1797 */ 1798 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue) 1799 { 1800 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1801 unsigned int bytes_compl = 0, pkts_compl = 0; 1802 unsigned int entry; 1803 1804 netif_tx_lock(priv->dev); 1805 1806 priv->xstats.tx_clean++; 1807 1808 entry = tx_q->dirty_tx; 1809 while (entry != tx_q->cur_tx) { 1810 struct sk_buff *skb = tx_q->tx_skbuff[entry]; 1811 struct dma_desc *p; 1812 int status; 1813 1814 if (priv->extend_desc) 1815 p = (struct dma_desc *)(tx_q->dma_etx + entry); 1816 else 1817 p = tx_q->dma_tx + entry; 1818 1819 status = priv->hw->desc->tx_status(&priv->dev->stats, 1820 &priv->xstats, p, 1821 priv->ioaddr); 1822 /* Check if the descriptor is owned by the DMA */ 1823 if (unlikely(status & tx_dma_own)) 1824 break; 1825 1826 /* Just consider the last segment and ...*/ 1827 if (likely(!(status & tx_not_ls))) { 1828 /* ... verify the status error condition */ 1829 if (unlikely(status & tx_err)) { 1830 priv->dev->stats.tx_errors++; 1831 } else { 1832 priv->dev->stats.tx_packets++; 1833 priv->xstats.tx_pkt_n++; 1834 } 1835 stmmac_get_tx_hwtstamp(priv, p, skb); 1836 } 1837 1838 if (likely(tx_q->tx_skbuff_dma[entry].buf)) { 1839 if (tx_q->tx_skbuff_dma[entry].map_as_page) 1840 dma_unmap_page(priv->device, 1841 tx_q->tx_skbuff_dma[entry].buf, 1842 tx_q->tx_skbuff_dma[entry].len, 1843 DMA_TO_DEVICE); 1844 else 1845 dma_unmap_single(priv->device, 1846 tx_q->tx_skbuff_dma[entry].buf, 1847 tx_q->tx_skbuff_dma[entry].len, 1848 DMA_TO_DEVICE); 1849 tx_q->tx_skbuff_dma[entry].buf = 0; 1850 tx_q->tx_skbuff_dma[entry].len = 0; 1851 tx_q->tx_skbuff_dma[entry].map_as_page = false; 1852 } 1853 1854 if (priv->hw->mode->clean_desc3) 1855 priv->hw->mode->clean_desc3(tx_q, p); 1856 1857 tx_q->tx_skbuff_dma[entry].last_segment = false; 1858 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1859 1860 if (likely(skb != NULL)) { 1861 pkts_compl++; 1862 bytes_compl += skb->len; 1863 dev_consume_skb_any(skb); 1864 tx_q->tx_skbuff[entry] = NULL; 1865 } 1866 1867 priv->hw->desc->release_tx_desc(p, priv->mode); 1868 1869 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 1870 } 1871 tx_q->dirty_tx = entry; 1872 1873 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 1874 pkts_compl, bytes_compl); 1875 1876 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 1877 queue))) && 1878 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) { 1879 1880 netif_dbg(priv, tx_done, priv->dev, 1881 "%s: restart transmit\n", __func__); 1882 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 1883 } 1884 1885 if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) { 1886 stmmac_enable_eee_mode(priv); 1887 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 1888 } 1889 netif_tx_unlock(priv->dev); 1890 } 1891 1892 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan) 1893 { 1894 priv->hw->dma->enable_dma_irq(priv->ioaddr, chan); 1895 } 1896 1897 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan) 1898 { 1899 priv->hw->dma->disable_dma_irq(priv->ioaddr, chan); 1900 } 1901 1902 /** 1903 * stmmac_tx_err - to manage the tx error 1904 * @priv: driver private structure 1905 * @chan: channel index 1906 * Description: it cleans the descriptors and restarts the transmission 1907 * in case of transmission errors. 1908 */ 1909 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 1910 { 1911 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 1912 int i; 1913 1914 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 1915 1916 stmmac_stop_tx_dma(priv, chan); 1917 dma_free_tx_skbufs(priv, chan); 1918 for (i = 0; i < DMA_TX_SIZE; i++) 1919 if (priv->extend_desc) 1920 priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic, 1921 priv->mode, 1922 (i == DMA_TX_SIZE - 1)); 1923 else 1924 priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i], 1925 priv->mode, 1926 (i == DMA_TX_SIZE - 1)); 1927 tx_q->dirty_tx = 0; 1928 tx_q->cur_tx = 0; 1929 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); 1930 stmmac_start_tx_dma(priv, chan); 1931 1932 priv->dev->stats.tx_errors++; 1933 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 1934 } 1935 1936 /** 1937 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 1938 * @priv: driver private structure 1939 * @txmode: TX operating mode 1940 * @rxmode: RX operating mode 1941 * @chan: channel index 1942 * Description: it is used for configuring of the DMA operation mode in 1943 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 1944 * mode. 1945 */ 1946 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 1947 u32 rxmode, u32 chan) 1948 { 1949 int rxfifosz = priv->plat->rx_fifo_size; 1950 1951 if (rxfifosz == 0) 1952 rxfifosz = priv->dma_cap.rx_fifo_size; 1953 1954 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1955 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan, 1956 rxfifosz); 1957 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan); 1958 } else { 1959 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode, 1960 rxfifosz); 1961 } 1962 } 1963 1964 /** 1965 * stmmac_dma_interrupt - DMA ISR 1966 * @priv: driver private structure 1967 * Description: this is the DMA ISR. It is called by the main ISR. 1968 * It calls the dwmac dma routine and schedule poll method in case of some 1969 * work can be done. 1970 */ 1971 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 1972 { 1973 u32 tx_channel_count = priv->plat->tx_queues_to_use; 1974 int status; 1975 u32 chan; 1976 1977 for (chan = 0; chan < tx_channel_count; chan++) { 1978 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; 1979 1980 status = priv->hw->dma->dma_interrupt(priv->ioaddr, 1981 &priv->xstats, chan); 1982 if (likely((status & handle_rx)) || (status & handle_tx)) { 1983 if (likely(napi_schedule_prep(&rx_q->napi))) { 1984 stmmac_disable_dma_irq(priv, chan); 1985 __napi_schedule(&rx_q->napi); 1986 } 1987 } 1988 1989 if (unlikely(status & tx_hard_error_bump_tc)) { 1990 /* Try to bump up the dma threshold on this failure */ 1991 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && 1992 (tc <= 256)) { 1993 tc += 64; 1994 if (priv->plat->force_thresh_dma_mode) 1995 stmmac_set_dma_operation_mode(priv, 1996 tc, 1997 tc, 1998 chan); 1999 else 2000 stmmac_set_dma_operation_mode(priv, 2001 tc, 2002 SF_DMA_MODE, 2003 chan); 2004 priv->xstats.threshold = tc; 2005 } 2006 } else if (unlikely(status == tx_hard_error)) { 2007 stmmac_tx_err(priv, chan); 2008 } 2009 } 2010 } 2011 2012 /** 2013 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2014 * @priv: driver private structure 2015 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2016 */ 2017 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2018 { 2019 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2020 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2021 2022 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 2023 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET; 2024 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET; 2025 } else { 2026 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET; 2027 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET; 2028 } 2029 2030 dwmac_mmc_intr_all_mask(priv->mmcaddr); 2031 2032 if (priv->dma_cap.rmon) { 2033 dwmac_mmc_ctrl(priv->mmcaddr, mode); 2034 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2035 } else 2036 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2037 } 2038 2039 /** 2040 * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors 2041 * @priv: driver private structure 2042 * Description: select the Enhanced/Alternate or Normal descriptors. 2043 * In case of Enhanced/Alternate, it checks if the extended descriptors are 2044 * supported by the HW capability register. 2045 */ 2046 static void stmmac_selec_desc_mode(struct stmmac_priv *priv) 2047 { 2048 if (priv->plat->enh_desc) { 2049 dev_info(priv->device, "Enhanced/Alternate descriptors\n"); 2050 2051 /* GMAC older than 3.50 has no extended descriptors */ 2052 if (priv->synopsys_id >= DWMAC_CORE_3_50) { 2053 dev_info(priv->device, "Enabled extended descriptors\n"); 2054 priv->extend_desc = 1; 2055 } else 2056 dev_warn(priv->device, "Extended descriptors not supported\n"); 2057 2058 priv->hw->desc = &enh_desc_ops; 2059 } else { 2060 dev_info(priv->device, "Normal descriptors\n"); 2061 priv->hw->desc = &ndesc_ops; 2062 } 2063 } 2064 2065 /** 2066 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2067 * @priv: driver private structure 2068 * Description: 2069 * new GMAC chip generations have a new register to indicate the 2070 * presence of the optional feature/functions. 2071 * This can be also used to override the value passed through the 2072 * platform and necessary for old MAC10/100 and GMAC chips. 2073 */ 2074 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2075 { 2076 u32 ret = 0; 2077 2078 if (priv->hw->dma->get_hw_feature) { 2079 priv->hw->dma->get_hw_feature(priv->ioaddr, 2080 &priv->dma_cap); 2081 ret = 1; 2082 } 2083 2084 return ret; 2085 } 2086 2087 /** 2088 * stmmac_check_ether_addr - check if the MAC addr is valid 2089 * @priv: driver private structure 2090 * Description: 2091 * it is to verify if the MAC address is valid, in case of failures it 2092 * generates a random MAC address 2093 */ 2094 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2095 { 2096 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2097 priv->hw->mac->get_umac_addr(priv->hw, 2098 priv->dev->dev_addr, 0); 2099 if (!is_valid_ether_addr(priv->dev->dev_addr)) 2100 eth_hw_addr_random(priv->dev); 2101 netdev_info(priv->dev, "device MAC address %pM\n", 2102 priv->dev->dev_addr); 2103 } 2104 } 2105 2106 /** 2107 * stmmac_init_dma_engine - DMA init. 2108 * @priv: driver private structure 2109 * Description: 2110 * It inits the DMA invoking the specific MAC/GMAC callback. 2111 * Some DMA parameters can be passed from the platform; 2112 * in case of these are not passed a default is kept for the MAC or GMAC. 2113 */ 2114 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2115 { 2116 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2117 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2118 struct stmmac_rx_queue *rx_q; 2119 struct stmmac_tx_queue *tx_q; 2120 u32 dummy_dma_rx_phy = 0; 2121 u32 dummy_dma_tx_phy = 0; 2122 u32 chan = 0; 2123 int atds = 0; 2124 int ret = 0; 2125 2126 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2127 dev_err(priv->device, "Invalid DMA configuration\n"); 2128 return -EINVAL; 2129 } 2130 2131 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2132 atds = 1; 2133 2134 ret = priv->hw->dma->reset(priv->ioaddr); 2135 if (ret) { 2136 dev_err(priv->device, "Failed to reset the dma\n"); 2137 return ret; 2138 } 2139 2140 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 2141 /* DMA Configuration */ 2142 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg, 2143 dummy_dma_tx_phy, dummy_dma_rx_phy, atds); 2144 2145 /* DMA RX Channel Configuration */ 2146 for (chan = 0; chan < rx_channels_count; chan++) { 2147 rx_q = &priv->rx_queue[chan]; 2148 2149 priv->hw->dma->init_rx_chan(priv->ioaddr, 2150 priv->plat->dma_cfg, 2151 rx_q->dma_rx_phy, chan); 2152 2153 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2154 (DMA_RX_SIZE * sizeof(struct dma_desc)); 2155 priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, 2156 rx_q->rx_tail_addr, 2157 chan); 2158 } 2159 2160 /* DMA TX Channel Configuration */ 2161 for (chan = 0; chan < tx_channels_count; chan++) { 2162 tx_q = &priv->tx_queue[chan]; 2163 2164 priv->hw->dma->init_chan(priv->ioaddr, 2165 priv->plat->dma_cfg, 2166 chan); 2167 2168 priv->hw->dma->init_tx_chan(priv->ioaddr, 2169 priv->plat->dma_cfg, 2170 tx_q->dma_tx_phy, chan); 2171 2172 tx_q->tx_tail_addr = tx_q->dma_tx_phy + 2173 (DMA_TX_SIZE * sizeof(struct dma_desc)); 2174 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, 2175 tx_q->tx_tail_addr, 2176 chan); 2177 } 2178 } else { 2179 rx_q = &priv->rx_queue[chan]; 2180 tx_q = &priv->tx_queue[chan]; 2181 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg, 2182 tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds); 2183 } 2184 2185 if (priv->plat->axi && priv->hw->dma->axi) 2186 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi); 2187 2188 return ret; 2189 } 2190 2191 /** 2192 * stmmac_tx_timer - mitigation sw timer for tx. 2193 * @data: data pointer 2194 * Description: 2195 * This is the timer handler to directly invoke the stmmac_tx_clean. 2196 */ 2197 static void stmmac_tx_timer(unsigned long data) 2198 { 2199 struct stmmac_priv *priv = (struct stmmac_priv *)data; 2200 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2201 u32 queue; 2202 2203 /* let's scan all the tx queues */ 2204 for (queue = 0; queue < tx_queues_count; queue++) 2205 stmmac_tx_clean(priv, queue); 2206 } 2207 2208 /** 2209 * stmmac_init_tx_coalesce - init tx mitigation options. 2210 * @priv: driver private structure 2211 * Description: 2212 * This inits the transmit coalesce parameters: i.e. timer rate, 2213 * timer handler and default threshold used for enabling the 2214 * interrupt on completion bit. 2215 */ 2216 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv) 2217 { 2218 priv->tx_coal_frames = STMMAC_TX_FRAMES; 2219 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER; 2220 init_timer(&priv->txtimer); 2221 priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer); 2222 priv->txtimer.data = (unsigned long)priv; 2223 priv->txtimer.function = stmmac_tx_timer; 2224 add_timer(&priv->txtimer); 2225 } 2226 2227 static void stmmac_set_rings_length(struct stmmac_priv *priv) 2228 { 2229 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2230 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2231 u32 chan; 2232 2233 /* set TX ring length */ 2234 if (priv->hw->dma->set_tx_ring_len) { 2235 for (chan = 0; chan < tx_channels_count; chan++) 2236 priv->hw->dma->set_tx_ring_len(priv->ioaddr, 2237 (DMA_TX_SIZE - 1), chan); 2238 } 2239 2240 /* set RX ring length */ 2241 if (priv->hw->dma->set_rx_ring_len) { 2242 for (chan = 0; chan < rx_channels_count; chan++) 2243 priv->hw->dma->set_rx_ring_len(priv->ioaddr, 2244 (DMA_RX_SIZE - 1), chan); 2245 } 2246 } 2247 2248 /** 2249 * stmmac_set_tx_queue_weight - Set TX queue weight 2250 * @priv: driver private structure 2251 * Description: It is used for setting TX queues weight 2252 */ 2253 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 2254 { 2255 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2256 u32 weight; 2257 u32 queue; 2258 2259 for (queue = 0; queue < tx_queues_count; queue++) { 2260 weight = priv->plat->tx_queues_cfg[queue].weight; 2261 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue); 2262 } 2263 } 2264 2265 /** 2266 * stmmac_configure_cbs - Configure CBS in TX queue 2267 * @priv: driver private structure 2268 * Description: It is used for configuring CBS in AVB TX queues 2269 */ 2270 static void stmmac_configure_cbs(struct stmmac_priv *priv) 2271 { 2272 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2273 u32 mode_to_use; 2274 u32 queue; 2275 2276 /* queue 0 is reserved for legacy traffic */ 2277 for (queue = 1; queue < tx_queues_count; queue++) { 2278 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 2279 if (mode_to_use == MTL_QUEUE_DCB) 2280 continue; 2281 2282 priv->hw->mac->config_cbs(priv->hw, 2283 priv->plat->tx_queues_cfg[queue].send_slope, 2284 priv->plat->tx_queues_cfg[queue].idle_slope, 2285 priv->plat->tx_queues_cfg[queue].high_credit, 2286 priv->plat->tx_queues_cfg[queue].low_credit, 2287 queue); 2288 } 2289 } 2290 2291 /** 2292 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 2293 * @priv: driver private structure 2294 * Description: It is used for mapping RX queues to RX dma channels 2295 */ 2296 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 2297 { 2298 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2299 u32 queue; 2300 u32 chan; 2301 2302 for (queue = 0; queue < rx_queues_count; queue++) { 2303 chan = priv->plat->rx_queues_cfg[queue].chan; 2304 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan); 2305 } 2306 } 2307 2308 /** 2309 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 2310 * @priv: driver private structure 2311 * Description: It is used for configuring the RX Queue Priority 2312 */ 2313 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 2314 { 2315 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2316 u32 queue; 2317 u32 prio; 2318 2319 for (queue = 0; queue < rx_queues_count; queue++) { 2320 if (!priv->plat->rx_queues_cfg[queue].use_prio) 2321 continue; 2322 2323 prio = priv->plat->rx_queues_cfg[queue].prio; 2324 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue); 2325 } 2326 } 2327 2328 /** 2329 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 2330 * @priv: driver private structure 2331 * Description: It is used for configuring the TX Queue Priority 2332 */ 2333 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 2334 { 2335 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2336 u32 queue; 2337 u32 prio; 2338 2339 for (queue = 0; queue < tx_queues_count; queue++) { 2340 if (!priv->plat->tx_queues_cfg[queue].use_prio) 2341 continue; 2342 2343 prio = priv->plat->tx_queues_cfg[queue].prio; 2344 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue); 2345 } 2346 } 2347 2348 /** 2349 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 2350 * @priv: driver private structure 2351 * Description: It is used for configuring the RX queue routing 2352 */ 2353 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 2354 { 2355 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2356 u32 queue; 2357 u8 packet; 2358 2359 for (queue = 0; queue < rx_queues_count; queue++) { 2360 /* no specific packet type routing specified for the queue */ 2361 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 2362 continue; 2363 2364 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 2365 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue); 2366 } 2367 } 2368 2369 /** 2370 * stmmac_mtl_configuration - Configure MTL 2371 * @priv: driver private structure 2372 * Description: It is used for configurring MTL 2373 */ 2374 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 2375 { 2376 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2377 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2378 2379 if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight) 2380 stmmac_set_tx_queue_weight(priv); 2381 2382 /* Configure MTL RX algorithms */ 2383 if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms) 2384 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw, 2385 priv->plat->rx_sched_algorithm); 2386 2387 /* Configure MTL TX algorithms */ 2388 if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms) 2389 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw, 2390 priv->plat->tx_sched_algorithm); 2391 2392 /* Configure CBS in AVB TX queues */ 2393 if (tx_queues_count > 1 && priv->hw->mac->config_cbs) 2394 stmmac_configure_cbs(priv); 2395 2396 /* Map RX MTL to DMA channels */ 2397 if (priv->hw->mac->map_mtl_to_dma) 2398 stmmac_rx_queue_dma_chan_map(priv); 2399 2400 /* Enable MAC RX Queues */ 2401 if (priv->hw->mac->rx_queue_enable) 2402 stmmac_mac_enable_rx_queues(priv); 2403 2404 /* Set RX priorities */ 2405 if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio) 2406 stmmac_mac_config_rx_queues_prio(priv); 2407 2408 /* Set TX priorities */ 2409 if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio) 2410 stmmac_mac_config_tx_queues_prio(priv); 2411 2412 /* Set RX routing */ 2413 if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing) 2414 stmmac_mac_config_rx_queues_routing(priv); 2415 } 2416 2417 /** 2418 * stmmac_hw_setup - setup mac in a usable state. 2419 * @dev : pointer to the device structure. 2420 * Description: 2421 * this is the main function to setup the HW in a usable state because the 2422 * dma engine is reset, the core registers are configured (e.g. AXI, 2423 * Checksum features, timers). The DMA is ready to start receiving and 2424 * transmitting. 2425 * Return value: 2426 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2427 * file on failure. 2428 */ 2429 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) 2430 { 2431 struct stmmac_priv *priv = netdev_priv(dev); 2432 u32 rx_cnt = priv->plat->rx_queues_to_use; 2433 u32 tx_cnt = priv->plat->tx_queues_to_use; 2434 u32 chan; 2435 int ret; 2436 2437 /* DMA initialization and SW reset */ 2438 ret = stmmac_init_dma_engine(priv); 2439 if (ret < 0) { 2440 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 2441 __func__); 2442 return ret; 2443 } 2444 2445 /* Copy the MAC addr into the HW */ 2446 priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0); 2447 2448 /* PS and related bits will be programmed according to the speed */ 2449 if (priv->hw->pcs) { 2450 int speed = priv->plat->mac_port_sel_speed; 2451 2452 if ((speed == SPEED_10) || (speed == SPEED_100) || 2453 (speed == SPEED_1000)) { 2454 priv->hw->ps = speed; 2455 } else { 2456 dev_warn(priv->device, "invalid port speed\n"); 2457 priv->hw->ps = 0; 2458 } 2459 } 2460 2461 /* Initialize the MAC Core */ 2462 priv->hw->mac->core_init(priv->hw, dev->mtu); 2463 2464 /* Initialize MTL*/ 2465 if (priv->synopsys_id >= DWMAC_CORE_4_00) 2466 stmmac_mtl_configuration(priv); 2467 2468 ret = priv->hw->mac->rx_ipc(priv->hw); 2469 if (!ret) { 2470 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 2471 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 2472 priv->hw->rx_csum = 0; 2473 } 2474 2475 /* Enable the MAC Rx/Tx */ 2476 priv->hw->mac->set_mac(priv->ioaddr, true); 2477 2478 /* Set the HW DMA mode and the COE */ 2479 stmmac_dma_operation_mode(priv); 2480 2481 stmmac_mmc_setup(priv); 2482 2483 if (init_ptp) { 2484 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 2485 if (ret < 0) 2486 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret); 2487 2488 ret = stmmac_init_ptp(priv); 2489 if (ret == -EOPNOTSUPP) 2490 netdev_warn(priv->dev, "PTP not supported by HW\n"); 2491 else if (ret) 2492 netdev_warn(priv->dev, "PTP init failed\n"); 2493 } 2494 2495 #ifdef CONFIG_DEBUG_FS 2496 ret = stmmac_init_fs(dev); 2497 if (ret < 0) 2498 netdev_warn(priv->dev, "%s: failed debugFS registration\n", 2499 __func__); 2500 #endif 2501 /* Start the ball rolling... */ 2502 stmmac_start_all_dma(priv); 2503 2504 priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS; 2505 2506 if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) { 2507 priv->rx_riwt = MAX_DMA_RIWT; 2508 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt); 2509 } 2510 2511 if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane) 2512 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0); 2513 2514 /* set TX and RX rings length */ 2515 stmmac_set_rings_length(priv); 2516 2517 /* Enable TSO */ 2518 if (priv->tso) { 2519 for (chan = 0; chan < tx_cnt; chan++) 2520 priv->hw->dma->enable_tso(priv->ioaddr, 1, chan); 2521 } 2522 2523 return 0; 2524 } 2525 2526 static void stmmac_hw_teardown(struct net_device *dev) 2527 { 2528 struct stmmac_priv *priv = netdev_priv(dev); 2529 2530 clk_disable_unprepare(priv->plat->clk_ptp_ref); 2531 } 2532 2533 /** 2534 * stmmac_open - open entry point of the driver 2535 * @dev : pointer to the device structure. 2536 * Description: 2537 * This function is the open entry point of the driver. 2538 * Return value: 2539 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2540 * file on failure. 2541 */ 2542 static int stmmac_open(struct net_device *dev) 2543 { 2544 struct stmmac_priv *priv = netdev_priv(dev); 2545 int ret; 2546 2547 stmmac_check_ether_addr(priv); 2548 2549 if (priv->hw->pcs != STMMAC_PCS_RGMII && 2550 priv->hw->pcs != STMMAC_PCS_TBI && 2551 priv->hw->pcs != STMMAC_PCS_RTBI) { 2552 ret = stmmac_init_phy(dev); 2553 if (ret) { 2554 netdev_err(priv->dev, 2555 "%s: Cannot attach to PHY (error: %d)\n", 2556 __func__, ret); 2557 return ret; 2558 } 2559 } 2560 2561 /* Extra statistics */ 2562 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); 2563 priv->xstats.threshold = tc; 2564 2565 priv->dma_buf_sz = STMMAC_ALIGN(buf_sz); 2566 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 2567 2568 ret = alloc_dma_desc_resources(priv); 2569 if (ret < 0) { 2570 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 2571 __func__); 2572 goto dma_desc_error; 2573 } 2574 2575 ret = init_dma_desc_rings(dev, GFP_KERNEL); 2576 if (ret < 0) { 2577 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 2578 __func__); 2579 goto init_error; 2580 } 2581 2582 ret = stmmac_hw_setup(dev, true); 2583 if (ret < 0) { 2584 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 2585 goto init_error; 2586 } 2587 2588 stmmac_init_tx_coalesce(priv); 2589 2590 if (dev->phydev) 2591 phy_start(dev->phydev); 2592 2593 /* Request the IRQ lines */ 2594 ret = request_irq(dev->irq, stmmac_interrupt, 2595 IRQF_SHARED, dev->name, dev); 2596 if (unlikely(ret < 0)) { 2597 netdev_err(priv->dev, 2598 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 2599 __func__, dev->irq, ret); 2600 goto irq_error; 2601 } 2602 2603 /* Request the Wake IRQ in case of another line is used for WoL */ 2604 if (priv->wol_irq != dev->irq) { 2605 ret = request_irq(priv->wol_irq, stmmac_interrupt, 2606 IRQF_SHARED, dev->name, dev); 2607 if (unlikely(ret < 0)) { 2608 netdev_err(priv->dev, 2609 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 2610 __func__, priv->wol_irq, ret); 2611 goto wolirq_error; 2612 } 2613 } 2614 2615 /* Request the IRQ lines */ 2616 if (priv->lpi_irq > 0) { 2617 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED, 2618 dev->name, dev); 2619 if (unlikely(ret < 0)) { 2620 netdev_err(priv->dev, 2621 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 2622 __func__, priv->lpi_irq, ret); 2623 goto lpiirq_error; 2624 } 2625 } 2626 2627 stmmac_enable_all_queues(priv); 2628 stmmac_start_all_queues(priv); 2629 2630 return 0; 2631 2632 lpiirq_error: 2633 if (priv->wol_irq != dev->irq) 2634 free_irq(priv->wol_irq, dev); 2635 wolirq_error: 2636 free_irq(dev->irq, dev); 2637 irq_error: 2638 if (dev->phydev) 2639 phy_stop(dev->phydev); 2640 2641 del_timer_sync(&priv->txtimer); 2642 stmmac_hw_teardown(dev); 2643 init_error: 2644 free_dma_desc_resources(priv); 2645 dma_desc_error: 2646 if (dev->phydev) 2647 phy_disconnect(dev->phydev); 2648 2649 return ret; 2650 } 2651 2652 /** 2653 * stmmac_release - close entry point of the driver 2654 * @dev : device pointer. 2655 * Description: 2656 * This is the stop entry point of the driver. 2657 */ 2658 static int stmmac_release(struct net_device *dev) 2659 { 2660 struct stmmac_priv *priv = netdev_priv(dev); 2661 2662 if (priv->eee_enabled) 2663 del_timer_sync(&priv->eee_ctrl_timer); 2664 2665 /* Stop and disconnect the PHY */ 2666 if (dev->phydev) { 2667 phy_stop(dev->phydev); 2668 phy_disconnect(dev->phydev); 2669 } 2670 2671 stmmac_stop_all_queues(priv); 2672 2673 stmmac_disable_all_queues(priv); 2674 2675 del_timer_sync(&priv->txtimer); 2676 2677 /* Free the IRQ lines */ 2678 free_irq(dev->irq, dev); 2679 if (priv->wol_irq != dev->irq) 2680 free_irq(priv->wol_irq, dev); 2681 if (priv->lpi_irq > 0) 2682 free_irq(priv->lpi_irq, dev); 2683 2684 /* Stop TX/RX DMA and clear the descriptors */ 2685 stmmac_stop_all_dma(priv); 2686 2687 /* Release and free the Rx/Tx resources */ 2688 free_dma_desc_resources(priv); 2689 2690 /* Disable the MAC Rx/Tx */ 2691 priv->hw->mac->set_mac(priv->ioaddr, false); 2692 2693 netif_carrier_off(dev); 2694 2695 #ifdef CONFIG_DEBUG_FS 2696 stmmac_exit_fs(dev); 2697 #endif 2698 2699 stmmac_release_ptp(priv); 2700 2701 return 0; 2702 } 2703 2704 /** 2705 * stmmac_tso_allocator - close entry point of the driver 2706 * @priv: driver private structure 2707 * @des: buffer start address 2708 * @total_len: total length to fill in descriptors 2709 * @last_segmant: condition for the last descriptor 2710 * @queue: TX queue index 2711 * Description: 2712 * This function fills descriptor and request new descriptors according to 2713 * buffer length to fill 2714 */ 2715 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des, 2716 int total_len, bool last_segment, u32 queue) 2717 { 2718 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2719 struct dma_desc *desc; 2720 u32 buff_size; 2721 int tmp_len; 2722 2723 tmp_len = total_len; 2724 2725 while (tmp_len > 0) { 2726 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2727 desc = tx_q->dma_tx + tx_q->cur_tx; 2728 2729 desc->des0 = cpu_to_le32(des + (total_len - tmp_len)); 2730 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 2731 TSO_MAX_BUFF_SIZE : tmp_len; 2732 2733 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size, 2734 0, 1, 2735 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 2736 0, 0); 2737 2738 tmp_len -= TSO_MAX_BUFF_SIZE; 2739 } 2740 } 2741 2742 /** 2743 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 2744 * @skb : the socket buffer 2745 * @dev : device pointer 2746 * Description: this is the transmit function that is called on TSO frames 2747 * (support available on GMAC4 and newer chips). 2748 * Diagram below show the ring programming in case of TSO frames: 2749 * 2750 * First Descriptor 2751 * -------- 2752 * | DES0 |---> buffer1 = L2/L3/L4 header 2753 * | DES1 |---> TCP Payload (can continue on next descr...) 2754 * | DES2 |---> buffer 1 and 2 len 2755 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 2756 * -------- 2757 * | 2758 * ... 2759 * | 2760 * -------- 2761 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 2762 * | DES1 | --| 2763 * | DES2 | --> buffer 1 and 2 len 2764 * | DES3 | 2765 * -------- 2766 * 2767 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 2768 */ 2769 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 2770 { 2771 struct dma_desc *desc, *first, *mss_desc = NULL; 2772 struct stmmac_priv *priv = netdev_priv(dev); 2773 int nfrags = skb_shinfo(skb)->nr_frags; 2774 u32 queue = skb_get_queue_mapping(skb); 2775 unsigned int first_entry, des; 2776 struct stmmac_tx_queue *tx_q; 2777 int tmp_pay_len = 0; 2778 u32 pay_len, mss; 2779 u8 proto_hdr_len; 2780 int i; 2781 2782 tx_q = &priv->tx_queue[queue]; 2783 2784 /* Compute header lengths */ 2785 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2786 2787 /* Desc availability based on threshold should be enough safe */ 2788 if (unlikely(stmmac_tx_avail(priv, queue) < 2789 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 2790 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 2791 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 2792 queue)); 2793 /* This is a hard error, log it. */ 2794 netdev_err(priv->dev, 2795 "%s: Tx Ring full when queue awake\n", 2796 __func__); 2797 } 2798 return NETDEV_TX_BUSY; 2799 } 2800 2801 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 2802 2803 mss = skb_shinfo(skb)->gso_size; 2804 2805 /* set new MSS value if needed */ 2806 if (mss != priv->mss) { 2807 mss_desc = tx_q->dma_tx + tx_q->cur_tx; 2808 priv->hw->desc->set_mss(mss_desc, mss); 2809 priv->mss = mss; 2810 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2811 } 2812 2813 if (netif_msg_tx_queued(priv)) { 2814 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 2815 __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss); 2816 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 2817 skb->data_len); 2818 } 2819 2820 first_entry = tx_q->cur_tx; 2821 2822 desc = tx_q->dma_tx + first_entry; 2823 first = desc; 2824 2825 /* first descriptor: fill Headers on Buf1 */ 2826 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 2827 DMA_TO_DEVICE); 2828 if (dma_mapping_error(priv->device, des)) 2829 goto dma_map_err; 2830 2831 tx_q->tx_skbuff_dma[first_entry].buf = des; 2832 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 2833 2834 first->des0 = cpu_to_le32(des); 2835 2836 /* Fill start of payload in buff2 of first descriptor */ 2837 if (pay_len) 2838 first->des1 = cpu_to_le32(des + proto_hdr_len); 2839 2840 /* If needed take extra descriptors to fill the remaining payload */ 2841 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 2842 2843 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 2844 2845 /* Prepare fragments */ 2846 for (i = 0; i < nfrags; i++) { 2847 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2848 2849 des = skb_frag_dma_map(priv->device, frag, 0, 2850 skb_frag_size(frag), 2851 DMA_TO_DEVICE); 2852 if (dma_mapping_error(priv->device, des)) 2853 goto dma_map_err; 2854 2855 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 2856 (i == nfrags - 1), queue); 2857 2858 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 2859 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 2860 tx_q->tx_skbuff[tx_q->cur_tx] = NULL; 2861 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 2862 } 2863 2864 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 2865 2866 /* Only the last descriptor gets to point to the skb. */ 2867 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 2868 2869 /* We've used all descriptors we need for this skb, however, 2870 * advance cur_tx so that it references a fresh descriptor. 2871 * ndo_start_xmit will fill this descriptor the next time it's 2872 * called and stmmac_tx_clean may clean up to this descriptor. 2873 */ 2874 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2875 2876 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 2877 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 2878 __func__); 2879 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 2880 } 2881 2882 dev->stats.tx_bytes += skb->len; 2883 priv->xstats.tx_tso_frames++; 2884 priv->xstats.tx_tso_nfrags += nfrags; 2885 2886 /* Manage tx mitigation */ 2887 priv->tx_count_frames += nfrags + 1; 2888 if (likely(priv->tx_coal_frames > priv->tx_count_frames)) { 2889 mod_timer(&priv->txtimer, 2890 STMMAC_COAL_TIMER(priv->tx_coal_timer)); 2891 } else { 2892 priv->tx_count_frames = 0; 2893 priv->hw->desc->set_tx_ic(desc); 2894 priv->xstats.tx_set_ic_bit++; 2895 } 2896 2897 skb_tx_timestamp(skb); 2898 2899 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 2900 priv->hwts_tx_en)) { 2901 /* declare that device is doing timestamping */ 2902 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2903 priv->hw->desc->enable_tx_timestamp(first); 2904 } 2905 2906 /* Complete the first descriptor before granting the DMA */ 2907 priv->hw->desc->prepare_tso_tx_desc(first, 1, 2908 proto_hdr_len, 2909 pay_len, 2910 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 2911 tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len)); 2912 2913 /* If context desc is used to change MSS */ 2914 if (mss_desc) 2915 priv->hw->desc->set_tx_owner(mss_desc); 2916 2917 /* The own bit must be the latest setting done when prepare the 2918 * descriptor and then barrier is needed to make sure that 2919 * all is coherent before granting the DMA engine. 2920 */ 2921 dma_wmb(); 2922 2923 if (netif_msg_pktdata(priv)) { 2924 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 2925 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 2926 tx_q->cur_tx, first, nfrags); 2927 2928 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE, 2929 0); 2930 2931 pr_info(">>> frame to be transmitted: "); 2932 print_pkt(skb->data, skb_headlen(skb)); 2933 } 2934 2935 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 2936 2937 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr, 2938 queue); 2939 2940 return NETDEV_TX_OK; 2941 2942 dma_map_err: 2943 dev_err(priv->device, "Tx dma map failed\n"); 2944 dev_kfree_skb(skb); 2945 priv->dev->stats.tx_dropped++; 2946 return NETDEV_TX_OK; 2947 } 2948 2949 /** 2950 * stmmac_xmit - Tx entry point of the driver 2951 * @skb : the socket buffer 2952 * @dev : device pointer 2953 * Description : this is the tx entry point of the driver. 2954 * It programs the chain or the ring and supports oversized frames 2955 * and SG feature. 2956 */ 2957 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 2958 { 2959 struct stmmac_priv *priv = netdev_priv(dev); 2960 unsigned int nopaged_len = skb_headlen(skb); 2961 int i, csum_insertion = 0, is_jumbo = 0; 2962 u32 queue = skb_get_queue_mapping(skb); 2963 int nfrags = skb_shinfo(skb)->nr_frags; 2964 int entry; 2965 unsigned int first_entry; 2966 struct dma_desc *desc, *first; 2967 struct stmmac_tx_queue *tx_q; 2968 unsigned int enh_desc; 2969 unsigned int des; 2970 2971 tx_q = &priv->tx_queue[queue]; 2972 2973 /* Manage oversized TCP frames for GMAC4 device */ 2974 if (skb_is_gso(skb) && priv->tso) { 2975 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 2976 return stmmac_tso_xmit(skb, dev); 2977 } 2978 2979 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 2980 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 2981 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 2982 queue)); 2983 /* This is a hard error, log it. */ 2984 netdev_err(priv->dev, 2985 "%s: Tx Ring full when queue awake\n", 2986 __func__); 2987 } 2988 return NETDEV_TX_BUSY; 2989 } 2990 2991 if (priv->tx_path_in_lpi_mode) 2992 stmmac_disable_eee_mode(priv); 2993 2994 entry = tx_q->cur_tx; 2995 first_entry = entry; 2996 2997 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 2998 2999 if (likely(priv->extend_desc)) 3000 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3001 else 3002 desc = tx_q->dma_tx + entry; 3003 3004 first = desc; 3005 3006 enh_desc = priv->plat->enh_desc; 3007 /* To program the descriptors according to the size of the frame */ 3008 if (enh_desc) 3009 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc); 3010 3011 if (unlikely(is_jumbo) && likely(priv->synopsys_id < 3012 DWMAC_CORE_4_00)) { 3013 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion); 3014 if (unlikely(entry < 0)) 3015 goto dma_map_err; 3016 } 3017 3018 for (i = 0; i < nfrags; i++) { 3019 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3020 int len = skb_frag_size(frag); 3021 bool last_segment = (i == (nfrags - 1)); 3022 3023 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3024 3025 if (likely(priv->extend_desc)) 3026 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3027 else 3028 desc = tx_q->dma_tx + entry; 3029 3030 des = skb_frag_dma_map(priv->device, frag, 0, len, 3031 DMA_TO_DEVICE); 3032 if (dma_mapping_error(priv->device, des)) 3033 goto dma_map_err; /* should reuse desc w/o issues */ 3034 3035 tx_q->tx_skbuff[entry] = NULL; 3036 3037 tx_q->tx_skbuff_dma[entry].buf = des; 3038 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3039 desc->des0 = cpu_to_le32(des); 3040 else 3041 desc->des2 = cpu_to_le32(des); 3042 3043 tx_q->tx_skbuff_dma[entry].map_as_page = true; 3044 tx_q->tx_skbuff_dma[entry].len = len; 3045 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 3046 3047 /* Prepare the descriptor and set the own bit too */ 3048 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion, 3049 priv->mode, 1, last_segment, 3050 skb->len); 3051 } 3052 3053 /* Only the last descriptor gets to point to the skb. */ 3054 tx_q->tx_skbuff[entry] = skb; 3055 3056 /* We've used all descriptors we need for this skb, however, 3057 * advance cur_tx so that it references a fresh descriptor. 3058 * ndo_start_xmit will fill this descriptor the next time it's 3059 * called and stmmac_tx_clean may clean up to this descriptor. 3060 */ 3061 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3062 tx_q->cur_tx = entry; 3063 3064 if (netif_msg_pktdata(priv)) { 3065 void *tx_head; 3066 3067 netdev_dbg(priv->dev, 3068 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 3069 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 3070 entry, first, nfrags); 3071 3072 if (priv->extend_desc) 3073 tx_head = (void *)tx_q->dma_etx; 3074 else 3075 tx_head = (void *)tx_q->dma_tx; 3076 3077 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false); 3078 3079 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 3080 print_pkt(skb->data, skb->len); 3081 } 3082 3083 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 3084 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 3085 __func__); 3086 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 3087 } 3088 3089 dev->stats.tx_bytes += skb->len; 3090 3091 /* According to the coalesce parameter the IC bit for the latest 3092 * segment is reset and the timer re-started to clean the tx status. 3093 * This approach takes care about the fragments: desc is the first 3094 * element in case of no SG. 3095 */ 3096 priv->tx_count_frames += nfrags + 1; 3097 if (likely(priv->tx_coal_frames > priv->tx_count_frames)) { 3098 mod_timer(&priv->txtimer, 3099 STMMAC_COAL_TIMER(priv->tx_coal_timer)); 3100 } else { 3101 priv->tx_count_frames = 0; 3102 priv->hw->desc->set_tx_ic(desc); 3103 priv->xstats.tx_set_ic_bit++; 3104 } 3105 3106 skb_tx_timestamp(skb); 3107 3108 /* Ready to fill the first descriptor and set the OWN bit w/o any 3109 * problems because all the descriptors are actually ready to be 3110 * passed to the DMA engine. 3111 */ 3112 if (likely(!is_jumbo)) { 3113 bool last_segment = (nfrags == 0); 3114 3115 des = dma_map_single(priv->device, skb->data, 3116 nopaged_len, DMA_TO_DEVICE); 3117 if (dma_mapping_error(priv->device, des)) 3118 goto dma_map_err; 3119 3120 tx_q->tx_skbuff_dma[first_entry].buf = des; 3121 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3122 first->des0 = cpu_to_le32(des); 3123 else 3124 first->des2 = cpu_to_le32(des); 3125 3126 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 3127 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 3128 3129 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 3130 priv->hwts_tx_en)) { 3131 /* declare that device is doing timestamping */ 3132 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 3133 priv->hw->desc->enable_tx_timestamp(first); 3134 } 3135 3136 /* Prepare the first descriptor setting the OWN bit too */ 3137 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len, 3138 csum_insertion, priv->mode, 1, 3139 last_segment, skb->len); 3140 3141 /* The own bit must be the latest setting done when prepare the 3142 * descriptor and then barrier is needed to make sure that 3143 * all is coherent before granting the DMA engine. 3144 */ 3145 dma_wmb(); 3146 } 3147 3148 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 3149 3150 if (priv->synopsys_id < DWMAC_CORE_4_00) 3151 priv->hw->dma->enable_dma_transmission(priv->ioaddr); 3152 else 3153 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr, 3154 queue); 3155 3156 return NETDEV_TX_OK; 3157 3158 dma_map_err: 3159 netdev_err(priv->dev, "Tx DMA map failed\n"); 3160 dev_kfree_skb(skb); 3161 priv->dev->stats.tx_dropped++; 3162 return NETDEV_TX_OK; 3163 } 3164 3165 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 3166 { 3167 struct ethhdr *ehdr; 3168 u16 vlanid; 3169 3170 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 3171 NETIF_F_HW_VLAN_CTAG_RX && 3172 !__vlan_get_tag(skb, &vlanid)) { 3173 /* pop the vlan tag */ 3174 ehdr = (struct ethhdr *)skb->data; 3175 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2); 3176 skb_pull(skb, VLAN_HLEN); 3177 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid); 3178 } 3179 } 3180 3181 3182 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q) 3183 { 3184 if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH) 3185 return 0; 3186 3187 return 1; 3188 } 3189 3190 /** 3191 * stmmac_rx_refill - refill used skb preallocated buffers 3192 * @priv: driver private structure 3193 * @queue: RX queue index 3194 * Description : this is to reallocate the skb for the reception process 3195 * that is based on zero-copy. 3196 */ 3197 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 3198 { 3199 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3200 int dirty = stmmac_rx_dirty(priv, queue); 3201 unsigned int entry = rx_q->dirty_rx; 3202 3203 int bfsize = priv->dma_buf_sz; 3204 3205 while (dirty-- > 0) { 3206 struct dma_desc *p; 3207 3208 if (priv->extend_desc) 3209 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3210 else 3211 p = rx_q->dma_rx + entry; 3212 3213 if (likely(!rx_q->rx_skbuff[entry])) { 3214 struct sk_buff *skb; 3215 3216 skb = netdev_alloc_skb_ip_align(priv->dev, bfsize); 3217 if (unlikely(!skb)) { 3218 /* so for a while no zero-copy! */ 3219 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH; 3220 if (unlikely(net_ratelimit())) 3221 dev_err(priv->device, 3222 "fail to alloc skb entry %d\n", 3223 entry); 3224 break; 3225 } 3226 3227 rx_q->rx_skbuff[entry] = skb; 3228 rx_q->rx_skbuff_dma[entry] = 3229 dma_map_single(priv->device, skb->data, bfsize, 3230 DMA_FROM_DEVICE); 3231 if (dma_mapping_error(priv->device, 3232 rx_q->rx_skbuff_dma[entry])) { 3233 netdev_err(priv->dev, "Rx DMA map failed\n"); 3234 dev_kfree_skb(skb); 3235 break; 3236 } 3237 3238 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) { 3239 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]); 3240 p->des1 = 0; 3241 } else { 3242 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]); 3243 } 3244 if (priv->hw->mode->refill_desc3) 3245 priv->hw->mode->refill_desc3(rx_q, p); 3246 3247 if (rx_q->rx_zeroc_thresh > 0) 3248 rx_q->rx_zeroc_thresh--; 3249 3250 netif_dbg(priv, rx_status, priv->dev, 3251 "refill entry #%d\n", entry); 3252 } 3253 dma_wmb(); 3254 3255 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3256 priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0); 3257 else 3258 priv->hw->desc->set_rx_owner(p); 3259 3260 dma_wmb(); 3261 3262 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE); 3263 } 3264 rx_q->dirty_rx = entry; 3265 } 3266 3267 /** 3268 * stmmac_rx - manage the receive process 3269 * @priv: driver private structure 3270 * @limit: napi bugget 3271 * @queue: RX queue index. 3272 * Description : this the function called by the napi poll method. 3273 * It gets all the frames inside the ring. 3274 */ 3275 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 3276 { 3277 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3278 unsigned int entry = rx_q->cur_rx; 3279 int coe = priv->hw->rx_csum; 3280 unsigned int next_entry; 3281 unsigned int count = 0; 3282 3283 if (netif_msg_rx_status(priv)) { 3284 void *rx_head; 3285 3286 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 3287 if (priv->extend_desc) 3288 rx_head = (void *)rx_q->dma_erx; 3289 else 3290 rx_head = (void *)rx_q->dma_rx; 3291 3292 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true); 3293 } 3294 while (count < limit) { 3295 int status; 3296 struct dma_desc *p; 3297 struct dma_desc *np; 3298 3299 if (priv->extend_desc) 3300 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3301 else 3302 p = rx_q->dma_rx + entry; 3303 3304 /* read the status of the incoming frame */ 3305 status = priv->hw->desc->rx_status(&priv->dev->stats, 3306 &priv->xstats, p); 3307 /* check if managed by the DMA otherwise go ahead */ 3308 if (unlikely(status & dma_own)) 3309 break; 3310 3311 count++; 3312 3313 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE); 3314 next_entry = rx_q->cur_rx; 3315 3316 if (priv->extend_desc) 3317 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 3318 else 3319 np = rx_q->dma_rx + next_entry; 3320 3321 prefetch(np); 3322 3323 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status)) 3324 priv->hw->desc->rx_extended_status(&priv->dev->stats, 3325 &priv->xstats, 3326 rx_q->dma_erx + 3327 entry); 3328 if (unlikely(status == discard_frame)) { 3329 priv->dev->stats.rx_errors++; 3330 if (priv->hwts_rx_en && !priv->extend_desc) { 3331 /* DESC2 & DESC3 will be overwritten by device 3332 * with timestamp value, hence reinitialize 3333 * them in stmmac_rx_refill() function so that 3334 * device can reuse it. 3335 */ 3336 dev_kfree_skb_any(rx_q->rx_skbuff[entry]); 3337 rx_q->rx_skbuff[entry] = NULL; 3338 dma_unmap_single(priv->device, 3339 rx_q->rx_skbuff_dma[entry], 3340 priv->dma_buf_sz, 3341 DMA_FROM_DEVICE); 3342 } 3343 } else { 3344 struct sk_buff *skb; 3345 int frame_len; 3346 unsigned int des; 3347 3348 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3349 des = le32_to_cpu(p->des0); 3350 else 3351 des = le32_to_cpu(p->des2); 3352 3353 frame_len = priv->hw->desc->get_rx_frame_len(p, coe); 3354 3355 /* If frame length is greater than skb buffer size 3356 * (preallocated during init) then the packet is 3357 * ignored 3358 */ 3359 if (frame_len > priv->dma_buf_sz) { 3360 netdev_err(priv->dev, 3361 "len %d larger than size (%d)\n", 3362 frame_len, priv->dma_buf_sz); 3363 priv->dev->stats.rx_length_errors++; 3364 break; 3365 } 3366 3367 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 3368 * Type frames (LLC/LLC-SNAP) 3369 */ 3370 if (unlikely(status != llc_snap)) 3371 frame_len -= ETH_FCS_LEN; 3372 3373 if (netif_msg_rx_status(priv)) { 3374 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n", 3375 p, entry, des); 3376 if (frame_len > ETH_FRAME_LEN) 3377 netdev_dbg(priv->dev, "frame size %d, COE: %d\n", 3378 frame_len, status); 3379 } 3380 3381 /* The zero-copy is always used for all the sizes 3382 * in case of GMAC4 because it needs 3383 * to refill the used descriptors, always. 3384 */ 3385 if (unlikely(!priv->plat->has_gmac4 && 3386 ((frame_len < priv->rx_copybreak) || 3387 stmmac_rx_threshold_count(rx_q)))) { 3388 skb = netdev_alloc_skb_ip_align(priv->dev, 3389 frame_len); 3390 if (unlikely(!skb)) { 3391 if (net_ratelimit()) 3392 dev_warn(priv->device, 3393 "packet dropped\n"); 3394 priv->dev->stats.rx_dropped++; 3395 break; 3396 } 3397 3398 dma_sync_single_for_cpu(priv->device, 3399 rx_q->rx_skbuff_dma 3400 [entry], frame_len, 3401 DMA_FROM_DEVICE); 3402 skb_copy_to_linear_data(skb, 3403 rx_q-> 3404 rx_skbuff[entry]->data, 3405 frame_len); 3406 3407 skb_put(skb, frame_len); 3408 dma_sync_single_for_device(priv->device, 3409 rx_q->rx_skbuff_dma 3410 [entry], frame_len, 3411 DMA_FROM_DEVICE); 3412 } else { 3413 skb = rx_q->rx_skbuff[entry]; 3414 if (unlikely(!skb)) { 3415 netdev_err(priv->dev, 3416 "%s: Inconsistent Rx chain\n", 3417 priv->dev->name); 3418 priv->dev->stats.rx_dropped++; 3419 break; 3420 } 3421 prefetch(skb->data - NET_IP_ALIGN); 3422 rx_q->rx_skbuff[entry] = NULL; 3423 rx_q->rx_zeroc_thresh++; 3424 3425 skb_put(skb, frame_len); 3426 dma_unmap_single(priv->device, 3427 rx_q->rx_skbuff_dma[entry], 3428 priv->dma_buf_sz, 3429 DMA_FROM_DEVICE); 3430 } 3431 3432 if (netif_msg_pktdata(priv)) { 3433 netdev_dbg(priv->dev, "frame received (%dbytes)", 3434 frame_len); 3435 print_pkt(skb->data, frame_len); 3436 } 3437 3438 stmmac_get_rx_hwtstamp(priv, p, np, skb); 3439 3440 stmmac_rx_vlan(priv->dev, skb); 3441 3442 skb->protocol = eth_type_trans(skb, priv->dev); 3443 3444 if (unlikely(!coe)) 3445 skb_checksum_none_assert(skb); 3446 else 3447 skb->ip_summed = CHECKSUM_UNNECESSARY; 3448 3449 napi_gro_receive(&rx_q->napi, skb); 3450 3451 priv->dev->stats.rx_packets++; 3452 priv->dev->stats.rx_bytes += frame_len; 3453 } 3454 entry = next_entry; 3455 } 3456 3457 stmmac_rx_refill(priv, queue); 3458 3459 priv->xstats.rx_pkt_n += count; 3460 3461 return count; 3462 } 3463 3464 /** 3465 * stmmac_poll - stmmac poll method (NAPI) 3466 * @napi : pointer to the napi structure. 3467 * @budget : maximum number of packets that the current CPU can receive from 3468 * all interfaces. 3469 * Description : 3470 * To look at the incoming frames and clear the tx resources. 3471 */ 3472 static int stmmac_poll(struct napi_struct *napi, int budget) 3473 { 3474 struct stmmac_rx_queue *rx_q = 3475 container_of(napi, struct stmmac_rx_queue, napi); 3476 struct stmmac_priv *priv = rx_q->priv_data; 3477 u32 tx_count = priv->plat->tx_queues_to_use; 3478 u32 chan = rx_q->queue_index; 3479 int work_done = 0; 3480 u32 queue; 3481 3482 priv->xstats.napi_poll++; 3483 3484 /* check all the queues */ 3485 for (queue = 0; queue < tx_count; queue++) 3486 stmmac_tx_clean(priv, queue); 3487 3488 work_done = stmmac_rx(priv, budget, rx_q->queue_index); 3489 if (work_done < budget) { 3490 napi_complete_done(napi, work_done); 3491 stmmac_enable_dma_irq(priv, chan); 3492 } 3493 return work_done; 3494 } 3495 3496 /** 3497 * stmmac_tx_timeout 3498 * @dev : Pointer to net device structure 3499 * Description: this function is called when a packet transmission fails to 3500 * complete within a reasonable time. The driver will mark the error in the 3501 * netdev structure and arrange for the device to be reset to a sane state 3502 * in order to transmit a new packet. 3503 */ 3504 static void stmmac_tx_timeout(struct net_device *dev) 3505 { 3506 struct stmmac_priv *priv = netdev_priv(dev); 3507 u32 tx_count = priv->plat->tx_queues_to_use; 3508 u32 chan; 3509 3510 /* Clear Tx resources and restart transmitting again */ 3511 for (chan = 0; chan < tx_count; chan++) 3512 stmmac_tx_err(priv, chan); 3513 } 3514 3515 /** 3516 * stmmac_set_rx_mode - entry point for multicast addressing 3517 * @dev : pointer to the device structure 3518 * Description: 3519 * This function is a driver entry point which gets called by the kernel 3520 * whenever multicast addresses must be enabled/disabled. 3521 * Return value: 3522 * void. 3523 */ 3524 static void stmmac_set_rx_mode(struct net_device *dev) 3525 { 3526 struct stmmac_priv *priv = netdev_priv(dev); 3527 3528 priv->hw->mac->set_filter(priv->hw, dev); 3529 } 3530 3531 /** 3532 * stmmac_change_mtu - entry point to change MTU size for the device. 3533 * @dev : device pointer. 3534 * @new_mtu : the new MTU size for the device. 3535 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 3536 * to drive packet transmission. Ethernet has an MTU of 1500 octets 3537 * (ETH_DATA_LEN). This value can be changed with ifconfig. 3538 * Return value: 3539 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3540 * file on failure. 3541 */ 3542 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 3543 { 3544 struct stmmac_priv *priv = netdev_priv(dev); 3545 3546 if (netif_running(dev)) { 3547 netdev_err(priv->dev, "must be stopped to change its MTU\n"); 3548 return -EBUSY; 3549 } 3550 3551 dev->mtu = new_mtu; 3552 3553 netdev_update_features(dev); 3554 3555 return 0; 3556 } 3557 3558 static netdev_features_t stmmac_fix_features(struct net_device *dev, 3559 netdev_features_t features) 3560 { 3561 struct stmmac_priv *priv = netdev_priv(dev); 3562 3563 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 3564 features &= ~NETIF_F_RXCSUM; 3565 3566 if (!priv->plat->tx_coe) 3567 features &= ~NETIF_F_CSUM_MASK; 3568 3569 /* Some GMAC devices have a bugged Jumbo frame support that 3570 * needs to have the Tx COE disabled for oversized frames 3571 * (due to limited buffer sizes). In this case we disable 3572 * the TX csum insertion in the TDES and not use SF. 3573 */ 3574 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 3575 features &= ~NETIF_F_CSUM_MASK; 3576 3577 /* Disable tso if asked by ethtool */ 3578 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 3579 if (features & NETIF_F_TSO) 3580 priv->tso = true; 3581 else 3582 priv->tso = false; 3583 } 3584 3585 return features; 3586 } 3587 3588 static int stmmac_set_features(struct net_device *netdev, 3589 netdev_features_t features) 3590 { 3591 struct stmmac_priv *priv = netdev_priv(netdev); 3592 3593 /* Keep the COE Type in case of csum is supporting */ 3594 if (features & NETIF_F_RXCSUM) 3595 priv->hw->rx_csum = priv->plat->rx_coe; 3596 else 3597 priv->hw->rx_csum = 0; 3598 /* No check needed because rx_coe has been set before and it will be 3599 * fixed in case of issue. 3600 */ 3601 priv->hw->mac->rx_ipc(priv->hw); 3602 3603 return 0; 3604 } 3605 3606 /** 3607 * stmmac_interrupt - main ISR 3608 * @irq: interrupt number. 3609 * @dev_id: to pass the net device pointer. 3610 * Description: this is the main driver interrupt service routine. 3611 * It can call: 3612 * o DMA service routine (to manage incoming frame reception and transmission 3613 * status) 3614 * o Core interrupts to manage: remote wake-up, management counter, LPI 3615 * interrupts. 3616 */ 3617 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 3618 { 3619 struct net_device *dev = (struct net_device *)dev_id; 3620 struct stmmac_priv *priv = netdev_priv(dev); 3621 u32 rx_cnt = priv->plat->rx_queues_to_use; 3622 u32 tx_cnt = priv->plat->tx_queues_to_use; 3623 u32 queues_count; 3624 u32 queue; 3625 3626 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 3627 3628 if (priv->irq_wake) 3629 pm_wakeup_event(priv->device, 0); 3630 3631 if (unlikely(!dev)) { 3632 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 3633 return IRQ_NONE; 3634 } 3635 3636 /* To handle GMAC own interrupts */ 3637 if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) { 3638 int status = priv->hw->mac->host_irq_status(priv->hw, 3639 &priv->xstats); 3640 3641 if (unlikely(status)) { 3642 /* For LPI we need to save the tx status */ 3643 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 3644 priv->tx_path_in_lpi_mode = true; 3645 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 3646 priv->tx_path_in_lpi_mode = false; 3647 } 3648 3649 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3650 for (queue = 0; queue < queues_count; queue++) { 3651 struct stmmac_rx_queue *rx_q = 3652 &priv->rx_queue[queue]; 3653 3654 status |= 3655 priv->hw->mac->host_mtl_irq_status(priv->hw, 3656 queue); 3657 3658 if (status & CORE_IRQ_MTL_RX_OVERFLOW && 3659 priv->hw->dma->set_rx_tail_ptr) 3660 priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, 3661 rx_q->rx_tail_addr, 3662 queue); 3663 } 3664 } 3665 3666 /* PCS link status */ 3667 if (priv->hw->pcs) { 3668 if (priv->xstats.pcs_link) 3669 netif_carrier_on(dev); 3670 else 3671 netif_carrier_off(dev); 3672 } 3673 } 3674 3675 /* To handle DMA interrupts */ 3676 stmmac_dma_interrupt(priv); 3677 3678 return IRQ_HANDLED; 3679 } 3680 3681 #ifdef CONFIG_NET_POLL_CONTROLLER 3682 /* Polling receive - used by NETCONSOLE and other diagnostic tools 3683 * to allow network I/O with interrupts disabled. 3684 */ 3685 static void stmmac_poll_controller(struct net_device *dev) 3686 { 3687 disable_irq(dev->irq); 3688 stmmac_interrupt(dev->irq, dev); 3689 enable_irq(dev->irq); 3690 } 3691 #endif 3692 3693 /** 3694 * stmmac_ioctl - Entry point for the Ioctl 3695 * @dev: Device pointer. 3696 * @rq: An IOCTL specefic structure, that can contain a pointer to 3697 * a proprietary structure used to pass information to the driver. 3698 * @cmd: IOCTL command 3699 * Description: 3700 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 3701 */ 3702 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3703 { 3704 int ret = -EOPNOTSUPP; 3705 3706 if (!netif_running(dev)) 3707 return -EINVAL; 3708 3709 switch (cmd) { 3710 case SIOCGMIIPHY: 3711 case SIOCGMIIREG: 3712 case SIOCSMIIREG: 3713 if (!dev->phydev) 3714 return -EINVAL; 3715 ret = phy_mii_ioctl(dev->phydev, rq, cmd); 3716 break; 3717 case SIOCSHWTSTAMP: 3718 ret = stmmac_hwtstamp_ioctl(dev, rq); 3719 break; 3720 default: 3721 break; 3722 } 3723 3724 return ret; 3725 } 3726 3727 #ifdef CONFIG_DEBUG_FS 3728 static struct dentry *stmmac_fs_dir; 3729 3730 static void sysfs_display_ring(void *head, int size, int extend_desc, 3731 struct seq_file *seq) 3732 { 3733 int i; 3734 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 3735 struct dma_desc *p = (struct dma_desc *)head; 3736 3737 for (i = 0; i < size; i++) { 3738 if (extend_desc) { 3739 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3740 i, (unsigned int)virt_to_phys(ep), 3741 le32_to_cpu(ep->basic.des0), 3742 le32_to_cpu(ep->basic.des1), 3743 le32_to_cpu(ep->basic.des2), 3744 le32_to_cpu(ep->basic.des3)); 3745 ep++; 3746 } else { 3747 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3748 i, (unsigned int)virt_to_phys(p), 3749 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 3750 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 3751 p++; 3752 } 3753 seq_printf(seq, "\n"); 3754 } 3755 } 3756 3757 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v) 3758 { 3759 struct net_device *dev = seq->private; 3760 struct stmmac_priv *priv = netdev_priv(dev); 3761 u32 rx_count = priv->plat->rx_queues_to_use; 3762 u32 tx_count = priv->plat->tx_queues_to_use; 3763 u32 queue; 3764 3765 for (queue = 0; queue < rx_count; queue++) { 3766 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3767 3768 seq_printf(seq, "RX Queue %d:\n", queue); 3769 3770 if (priv->extend_desc) { 3771 seq_printf(seq, "Extended descriptor ring:\n"); 3772 sysfs_display_ring((void *)rx_q->dma_erx, 3773 DMA_RX_SIZE, 1, seq); 3774 } else { 3775 seq_printf(seq, "Descriptor ring:\n"); 3776 sysfs_display_ring((void *)rx_q->dma_rx, 3777 DMA_RX_SIZE, 0, seq); 3778 } 3779 } 3780 3781 for (queue = 0; queue < tx_count; queue++) { 3782 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 3783 3784 seq_printf(seq, "TX Queue %d:\n", queue); 3785 3786 if (priv->extend_desc) { 3787 seq_printf(seq, "Extended descriptor ring:\n"); 3788 sysfs_display_ring((void *)tx_q->dma_etx, 3789 DMA_TX_SIZE, 1, seq); 3790 } else { 3791 seq_printf(seq, "Descriptor ring:\n"); 3792 sysfs_display_ring((void *)tx_q->dma_tx, 3793 DMA_TX_SIZE, 0, seq); 3794 } 3795 } 3796 3797 return 0; 3798 } 3799 3800 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file) 3801 { 3802 return single_open(file, stmmac_sysfs_ring_read, inode->i_private); 3803 } 3804 3805 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */ 3806 3807 static const struct file_operations stmmac_rings_status_fops = { 3808 .owner = THIS_MODULE, 3809 .open = stmmac_sysfs_ring_open, 3810 .read = seq_read, 3811 .llseek = seq_lseek, 3812 .release = single_release, 3813 }; 3814 3815 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v) 3816 { 3817 struct net_device *dev = seq->private; 3818 struct stmmac_priv *priv = netdev_priv(dev); 3819 3820 if (!priv->hw_cap_support) { 3821 seq_printf(seq, "DMA HW features not supported\n"); 3822 return 0; 3823 } 3824 3825 seq_printf(seq, "==============================\n"); 3826 seq_printf(seq, "\tDMA HW features\n"); 3827 seq_printf(seq, "==============================\n"); 3828 3829 seq_printf(seq, "\t10/100 Mbps: %s\n", 3830 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 3831 seq_printf(seq, "\t1000 Mbps: %s\n", 3832 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 3833 seq_printf(seq, "\tHalf duplex: %s\n", 3834 (priv->dma_cap.half_duplex) ? "Y" : "N"); 3835 seq_printf(seq, "\tHash Filter: %s\n", 3836 (priv->dma_cap.hash_filter) ? "Y" : "N"); 3837 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 3838 (priv->dma_cap.multi_addr) ? "Y" : "N"); 3839 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 3840 (priv->dma_cap.pcs) ? "Y" : "N"); 3841 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 3842 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 3843 seq_printf(seq, "\tPMT Remote wake up: %s\n", 3844 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 3845 seq_printf(seq, "\tPMT Magic Frame: %s\n", 3846 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 3847 seq_printf(seq, "\tRMON module: %s\n", 3848 (priv->dma_cap.rmon) ? "Y" : "N"); 3849 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 3850 (priv->dma_cap.time_stamp) ? "Y" : "N"); 3851 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 3852 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 3853 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 3854 (priv->dma_cap.eee) ? "Y" : "N"); 3855 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 3856 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 3857 (priv->dma_cap.tx_coe) ? "Y" : "N"); 3858 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3859 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 3860 (priv->dma_cap.rx_coe) ? "Y" : "N"); 3861 } else { 3862 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 3863 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 3864 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 3865 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 3866 } 3867 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 3868 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 3869 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 3870 priv->dma_cap.number_rx_channel); 3871 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 3872 priv->dma_cap.number_tx_channel); 3873 seq_printf(seq, "\tEnhanced descriptors: %s\n", 3874 (priv->dma_cap.enh_desc) ? "Y" : "N"); 3875 3876 return 0; 3877 } 3878 3879 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file) 3880 { 3881 return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private); 3882 } 3883 3884 static const struct file_operations stmmac_dma_cap_fops = { 3885 .owner = THIS_MODULE, 3886 .open = stmmac_sysfs_dma_cap_open, 3887 .read = seq_read, 3888 .llseek = seq_lseek, 3889 .release = single_release, 3890 }; 3891 3892 static int stmmac_init_fs(struct net_device *dev) 3893 { 3894 struct stmmac_priv *priv = netdev_priv(dev); 3895 3896 /* Create per netdev entries */ 3897 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 3898 3899 if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) { 3900 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n"); 3901 3902 return -ENOMEM; 3903 } 3904 3905 /* Entry to report DMA RX/TX rings */ 3906 priv->dbgfs_rings_status = 3907 debugfs_create_file("descriptors_status", S_IRUGO, 3908 priv->dbgfs_dir, dev, 3909 &stmmac_rings_status_fops); 3910 3911 if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) { 3912 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n"); 3913 debugfs_remove_recursive(priv->dbgfs_dir); 3914 3915 return -ENOMEM; 3916 } 3917 3918 /* Entry to report the DMA HW features */ 3919 priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO, 3920 priv->dbgfs_dir, 3921 dev, &stmmac_dma_cap_fops); 3922 3923 if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) { 3924 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n"); 3925 debugfs_remove_recursive(priv->dbgfs_dir); 3926 3927 return -ENOMEM; 3928 } 3929 3930 return 0; 3931 } 3932 3933 static void stmmac_exit_fs(struct net_device *dev) 3934 { 3935 struct stmmac_priv *priv = netdev_priv(dev); 3936 3937 debugfs_remove_recursive(priv->dbgfs_dir); 3938 } 3939 #endif /* CONFIG_DEBUG_FS */ 3940 3941 static const struct net_device_ops stmmac_netdev_ops = { 3942 .ndo_open = stmmac_open, 3943 .ndo_start_xmit = stmmac_xmit, 3944 .ndo_stop = stmmac_release, 3945 .ndo_change_mtu = stmmac_change_mtu, 3946 .ndo_fix_features = stmmac_fix_features, 3947 .ndo_set_features = stmmac_set_features, 3948 .ndo_set_rx_mode = stmmac_set_rx_mode, 3949 .ndo_tx_timeout = stmmac_tx_timeout, 3950 .ndo_do_ioctl = stmmac_ioctl, 3951 #ifdef CONFIG_NET_POLL_CONTROLLER 3952 .ndo_poll_controller = stmmac_poll_controller, 3953 #endif 3954 .ndo_set_mac_address = eth_mac_addr, 3955 }; 3956 3957 /** 3958 * stmmac_hw_init - Init the MAC device 3959 * @priv: driver private structure 3960 * Description: this function is to configure the MAC device according to 3961 * some platform parameters or the HW capability register. It prepares the 3962 * driver to use either ring or chain modes and to setup either enhanced or 3963 * normal descriptors. 3964 */ 3965 static int stmmac_hw_init(struct stmmac_priv *priv) 3966 { 3967 struct mac_device_info *mac; 3968 3969 /* Identify the MAC HW device */ 3970 if (priv->plat->setup) { 3971 mac = priv->plat->setup(priv); 3972 } else if (priv->plat->has_gmac) { 3973 priv->dev->priv_flags |= IFF_UNICAST_FLT; 3974 mac = dwmac1000_setup(priv->ioaddr, 3975 priv->plat->multicast_filter_bins, 3976 priv->plat->unicast_filter_entries, 3977 &priv->synopsys_id); 3978 } else if (priv->plat->has_gmac4) { 3979 priv->dev->priv_flags |= IFF_UNICAST_FLT; 3980 mac = dwmac4_setup(priv->ioaddr, 3981 priv->plat->multicast_filter_bins, 3982 priv->plat->unicast_filter_entries, 3983 &priv->synopsys_id); 3984 } else { 3985 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id); 3986 } 3987 if (!mac) 3988 return -ENOMEM; 3989 3990 priv->hw = mac; 3991 3992 /* dwmac-sun8i only work in chain mode */ 3993 if (priv->plat->has_sun8i) 3994 chain_mode = 1; 3995 3996 /* To use the chained or ring mode */ 3997 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3998 priv->hw->mode = &dwmac4_ring_mode_ops; 3999 } else { 4000 if (chain_mode) { 4001 priv->hw->mode = &chain_mode_ops; 4002 dev_info(priv->device, "Chain mode enabled\n"); 4003 priv->mode = STMMAC_CHAIN_MODE; 4004 } else { 4005 priv->hw->mode = &ring_mode_ops; 4006 dev_info(priv->device, "Ring mode enabled\n"); 4007 priv->mode = STMMAC_RING_MODE; 4008 } 4009 } 4010 4011 /* Get the HW capability (new GMAC newer than 3.50a) */ 4012 priv->hw_cap_support = stmmac_get_hw_features(priv); 4013 if (priv->hw_cap_support) { 4014 dev_info(priv->device, "DMA HW capability register supported\n"); 4015 4016 /* We can override some gmac/dma configuration fields: e.g. 4017 * enh_desc, tx_coe (e.g. that are passed through the 4018 * platform) with the values from the HW capability 4019 * register (if supported). 4020 */ 4021 priv->plat->enh_desc = priv->dma_cap.enh_desc; 4022 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up; 4023 priv->hw->pmt = priv->plat->pmt; 4024 4025 /* TXCOE doesn't work in thresh DMA mode */ 4026 if (priv->plat->force_thresh_dma_mode) 4027 priv->plat->tx_coe = 0; 4028 else 4029 priv->plat->tx_coe = priv->dma_cap.tx_coe; 4030 4031 /* In case of GMAC4 rx_coe is from HW cap register. */ 4032 priv->plat->rx_coe = priv->dma_cap.rx_coe; 4033 4034 if (priv->dma_cap.rx_coe_type2) 4035 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 4036 else if (priv->dma_cap.rx_coe_type1) 4037 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 4038 4039 } else { 4040 dev_info(priv->device, "No HW DMA feature register supported\n"); 4041 } 4042 4043 /* To use alternate (extended), normal or GMAC4 descriptor structures */ 4044 if (priv->synopsys_id >= DWMAC_CORE_4_00) 4045 priv->hw->desc = &dwmac4_desc_ops; 4046 else 4047 stmmac_selec_desc_mode(priv); 4048 4049 if (priv->plat->rx_coe) { 4050 priv->hw->rx_csum = priv->plat->rx_coe; 4051 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 4052 if (priv->synopsys_id < DWMAC_CORE_4_00) 4053 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 4054 } 4055 if (priv->plat->tx_coe) 4056 dev_info(priv->device, "TX Checksum insertion supported\n"); 4057 4058 if (priv->plat->pmt) { 4059 dev_info(priv->device, "Wake-Up On Lan supported\n"); 4060 device_set_wakeup_capable(priv->device, 1); 4061 } 4062 4063 if (priv->dma_cap.tsoen) 4064 dev_info(priv->device, "TSO supported\n"); 4065 4066 return 0; 4067 } 4068 4069 /** 4070 * stmmac_dvr_probe 4071 * @device: device pointer 4072 * @plat_dat: platform data pointer 4073 * @res: stmmac resource pointer 4074 * Description: this is the main probe function used to 4075 * call the alloc_etherdev, allocate the priv structure. 4076 * Return: 4077 * returns 0 on success, otherwise errno. 4078 */ 4079 int stmmac_dvr_probe(struct device *device, 4080 struct plat_stmmacenet_data *plat_dat, 4081 struct stmmac_resources *res) 4082 { 4083 struct net_device *ndev = NULL; 4084 struct stmmac_priv *priv; 4085 int ret = 0; 4086 u32 queue; 4087 4088 ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv), 4089 MTL_MAX_TX_QUEUES, 4090 MTL_MAX_RX_QUEUES); 4091 if (!ndev) 4092 return -ENOMEM; 4093 4094 SET_NETDEV_DEV(ndev, device); 4095 4096 priv = netdev_priv(ndev); 4097 priv->device = device; 4098 priv->dev = ndev; 4099 4100 stmmac_set_ethtool_ops(ndev); 4101 priv->pause = pause; 4102 priv->plat = plat_dat; 4103 priv->ioaddr = res->addr; 4104 priv->dev->base_addr = (unsigned long)res->addr; 4105 4106 priv->dev->irq = res->irq; 4107 priv->wol_irq = res->wol_irq; 4108 priv->lpi_irq = res->lpi_irq; 4109 4110 if (res->mac) 4111 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); 4112 4113 dev_set_drvdata(device, priv->dev); 4114 4115 /* Verify driver arguments */ 4116 stmmac_verify_args(); 4117 4118 /* Override with kernel parameters if supplied XXX CRS XXX 4119 * this needs to have multiple instances 4120 */ 4121 if ((phyaddr >= 0) && (phyaddr <= 31)) 4122 priv->plat->phy_addr = phyaddr; 4123 4124 if (priv->plat->stmmac_rst) { 4125 ret = reset_control_assert(priv->plat->stmmac_rst); 4126 reset_control_deassert(priv->plat->stmmac_rst); 4127 /* Some reset controllers have only reset callback instead of 4128 * assert + deassert callbacks pair. 4129 */ 4130 if (ret == -ENOTSUPP) 4131 reset_control_reset(priv->plat->stmmac_rst); 4132 } 4133 4134 /* Init MAC and get the capabilities */ 4135 ret = stmmac_hw_init(priv); 4136 if (ret) 4137 goto error_hw_init; 4138 4139 /* Configure real RX and TX queues */ 4140 netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use); 4141 netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use); 4142 4143 ndev->netdev_ops = &stmmac_netdev_ops; 4144 4145 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4146 NETIF_F_RXCSUM; 4147 4148 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 4149 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 4150 priv->tso = true; 4151 dev_info(priv->device, "TSO feature enabled\n"); 4152 } 4153 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 4154 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 4155 #ifdef STMMAC_VLAN_TAG_USED 4156 /* Both mac100 and gmac support receive VLAN tag detection */ 4157 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; 4158 #endif 4159 priv->msg_enable = netif_msg_init(debug, default_msg_level); 4160 4161 /* MTU range: 46 - hw-specific max */ 4162 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 4163 if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 4164 ndev->max_mtu = JUMBO_LEN; 4165 else 4166 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 4167 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 4168 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 4169 */ 4170 if ((priv->plat->maxmtu < ndev->max_mtu) && 4171 (priv->plat->maxmtu >= ndev->min_mtu)) 4172 ndev->max_mtu = priv->plat->maxmtu; 4173 else if (priv->plat->maxmtu < ndev->min_mtu) 4174 dev_warn(priv->device, 4175 "%s: warning: maxmtu having invalid value (%d)\n", 4176 __func__, priv->plat->maxmtu); 4177 4178 if (flow_ctrl) 4179 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 4180 4181 /* Rx Watchdog is available in the COREs newer than the 3.40. 4182 * In some case, for example on bugged HW this feature 4183 * has to be disable and this can be done by passing the 4184 * riwt_off field from the platform. 4185 */ 4186 if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) { 4187 priv->use_riwt = 1; 4188 dev_info(priv->device, 4189 "Enable RX Mitigation via HW Watchdog Timer\n"); 4190 } 4191 4192 for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) { 4193 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4194 4195 netif_napi_add(ndev, &rx_q->napi, stmmac_poll, 4196 (8 * priv->plat->rx_queues_to_use)); 4197 } 4198 4199 spin_lock_init(&priv->lock); 4200 4201 /* If a specific clk_csr value is passed from the platform 4202 * this means that the CSR Clock Range selection cannot be 4203 * changed at run-time and it is fixed. Viceversa the driver'll try to 4204 * set the MDC clock dynamically according to the csr actual 4205 * clock input. 4206 */ 4207 if (!priv->plat->clk_csr) 4208 stmmac_clk_csr_set(priv); 4209 else 4210 priv->clk_csr = priv->plat->clk_csr; 4211 4212 stmmac_check_pcs_mode(priv); 4213 4214 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4215 priv->hw->pcs != STMMAC_PCS_TBI && 4216 priv->hw->pcs != STMMAC_PCS_RTBI) { 4217 /* MDIO bus Registration */ 4218 ret = stmmac_mdio_register(ndev); 4219 if (ret < 0) { 4220 dev_err(priv->device, 4221 "%s: MDIO bus (id: %d) registration failed", 4222 __func__, priv->plat->bus_id); 4223 goto error_mdio_register; 4224 } 4225 } 4226 4227 ret = register_netdev(ndev); 4228 if (ret) { 4229 dev_err(priv->device, "%s: ERROR %i registering the device\n", 4230 __func__, ret); 4231 goto error_netdev_register; 4232 } 4233 4234 return ret; 4235 4236 error_netdev_register: 4237 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4238 priv->hw->pcs != STMMAC_PCS_TBI && 4239 priv->hw->pcs != STMMAC_PCS_RTBI) 4240 stmmac_mdio_unregister(ndev); 4241 error_mdio_register: 4242 for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) { 4243 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4244 4245 netif_napi_del(&rx_q->napi); 4246 } 4247 error_hw_init: 4248 free_netdev(ndev); 4249 4250 return ret; 4251 } 4252 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 4253 4254 /** 4255 * stmmac_dvr_remove 4256 * @dev: device pointer 4257 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 4258 * changes the link status, releases the DMA descriptor rings. 4259 */ 4260 int stmmac_dvr_remove(struct device *dev) 4261 { 4262 struct net_device *ndev = dev_get_drvdata(dev); 4263 struct stmmac_priv *priv = netdev_priv(ndev); 4264 4265 netdev_info(priv->dev, "%s: removing driver", __func__); 4266 4267 stmmac_stop_all_dma(priv); 4268 4269 priv->hw->mac->set_mac(priv->ioaddr, false); 4270 netif_carrier_off(ndev); 4271 unregister_netdev(ndev); 4272 if (priv->plat->stmmac_rst) 4273 reset_control_assert(priv->plat->stmmac_rst); 4274 clk_disable_unprepare(priv->plat->pclk); 4275 clk_disable_unprepare(priv->plat->stmmac_clk); 4276 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4277 priv->hw->pcs != STMMAC_PCS_TBI && 4278 priv->hw->pcs != STMMAC_PCS_RTBI) 4279 stmmac_mdio_unregister(ndev); 4280 free_netdev(ndev); 4281 4282 return 0; 4283 } 4284 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 4285 4286 /** 4287 * stmmac_suspend - suspend callback 4288 * @dev: device pointer 4289 * Description: this is the function to suspend the device and it is called 4290 * by the platform driver to stop the network queue, release the resources, 4291 * program the PMT register (for WoL), clean and release driver resources. 4292 */ 4293 int stmmac_suspend(struct device *dev) 4294 { 4295 struct net_device *ndev = dev_get_drvdata(dev); 4296 struct stmmac_priv *priv = netdev_priv(ndev); 4297 unsigned long flags; 4298 4299 if (!ndev || !netif_running(ndev)) 4300 return 0; 4301 4302 if (ndev->phydev) 4303 phy_stop(ndev->phydev); 4304 4305 spin_lock_irqsave(&priv->lock, flags); 4306 4307 netif_device_detach(ndev); 4308 stmmac_stop_all_queues(priv); 4309 4310 stmmac_disable_all_queues(priv); 4311 4312 /* Stop TX/RX DMA */ 4313 stmmac_stop_all_dma(priv); 4314 4315 /* Enable Power down mode by programming the PMT regs */ 4316 if (device_may_wakeup(priv->device)) { 4317 priv->hw->mac->pmt(priv->hw, priv->wolopts); 4318 priv->irq_wake = 1; 4319 } else { 4320 priv->hw->mac->set_mac(priv->ioaddr, false); 4321 pinctrl_pm_select_sleep_state(priv->device); 4322 /* Disable clock in case of PWM is off */ 4323 clk_disable(priv->plat->pclk); 4324 clk_disable(priv->plat->stmmac_clk); 4325 } 4326 spin_unlock_irqrestore(&priv->lock, flags); 4327 4328 priv->oldlink = false; 4329 priv->speed = SPEED_UNKNOWN; 4330 priv->oldduplex = DUPLEX_UNKNOWN; 4331 return 0; 4332 } 4333 EXPORT_SYMBOL_GPL(stmmac_suspend); 4334 4335 /** 4336 * stmmac_reset_queues_param - reset queue parameters 4337 * @dev: device pointer 4338 */ 4339 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 4340 { 4341 u32 rx_cnt = priv->plat->rx_queues_to_use; 4342 u32 tx_cnt = priv->plat->tx_queues_to_use; 4343 u32 queue; 4344 4345 for (queue = 0; queue < rx_cnt; queue++) { 4346 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4347 4348 rx_q->cur_rx = 0; 4349 rx_q->dirty_rx = 0; 4350 } 4351 4352 for (queue = 0; queue < tx_cnt; queue++) { 4353 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 4354 4355 tx_q->cur_tx = 0; 4356 tx_q->dirty_tx = 0; 4357 } 4358 } 4359 4360 /** 4361 * stmmac_resume - resume callback 4362 * @dev: device pointer 4363 * Description: when resume this function is invoked to setup the DMA and CORE 4364 * in a usable state. 4365 */ 4366 int stmmac_resume(struct device *dev) 4367 { 4368 struct net_device *ndev = dev_get_drvdata(dev); 4369 struct stmmac_priv *priv = netdev_priv(ndev); 4370 unsigned long flags; 4371 4372 if (!netif_running(ndev)) 4373 return 0; 4374 4375 /* Power Down bit, into the PM register, is cleared 4376 * automatically as soon as a magic packet or a Wake-up frame 4377 * is received. Anyway, it's better to manually clear 4378 * this bit because it can generate problems while resuming 4379 * from another devices (e.g. serial console). 4380 */ 4381 if (device_may_wakeup(priv->device)) { 4382 spin_lock_irqsave(&priv->lock, flags); 4383 priv->hw->mac->pmt(priv->hw, 0); 4384 spin_unlock_irqrestore(&priv->lock, flags); 4385 priv->irq_wake = 0; 4386 } else { 4387 pinctrl_pm_select_default_state(priv->device); 4388 /* enable the clk previously disabled */ 4389 clk_enable(priv->plat->stmmac_clk); 4390 clk_enable(priv->plat->pclk); 4391 /* reset the phy so that it's ready */ 4392 if (priv->mii) 4393 stmmac_mdio_reset(priv->mii); 4394 } 4395 4396 netif_device_attach(ndev); 4397 4398 spin_lock_irqsave(&priv->lock, flags); 4399 4400 stmmac_reset_queues_param(priv); 4401 4402 /* reset private mss value to force mss context settings at 4403 * next tso xmit (only used for gmac4). 4404 */ 4405 priv->mss = 0; 4406 4407 stmmac_clear_descriptors(priv); 4408 4409 stmmac_hw_setup(ndev, false); 4410 stmmac_init_tx_coalesce(priv); 4411 stmmac_set_rx_mode(ndev); 4412 4413 stmmac_enable_all_queues(priv); 4414 4415 stmmac_start_all_queues(priv); 4416 4417 spin_unlock_irqrestore(&priv->lock, flags); 4418 4419 if (ndev->phydev) 4420 phy_start(ndev->phydev); 4421 4422 return 0; 4423 } 4424 EXPORT_SYMBOL_GPL(stmmac_resume); 4425 4426 #ifndef MODULE 4427 static int __init stmmac_cmdline_opt(char *str) 4428 { 4429 char *opt; 4430 4431 if (!str || !*str) 4432 return -EINVAL; 4433 while ((opt = strsep(&str, ",")) != NULL) { 4434 if (!strncmp(opt, "debug:", 6)) { 4435 if (kstrtoint(opt + 6, 0, &debug)) 4436 goto err; 4437 } else if (!strncmp(opt, "phyaddr:", 8)) { 4438 if (kstrtoint(opt + 8, 0, &phyaddr)) 4439 goto err; 4440 } else if (!strncmp(opt, "buf_sz:", 7)) { 4441 if (kstrtoint(opt + 7, 0, &buf_sz)) 4442 goto err; 4443 } else if (!strncmp(opt, "tc:", 3)) { 4444 if (kstrtoint(opt + 3, 0, &tc)) 4445 goto err; 4446 } else if (!strncmp(opt, "watchdog:", 9)) { 4447 if (kstrtoint(opt + 9, 0, &watchdog)) 4448 goto err; 4449 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 4450 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 4451 goto err; 4452 } else if (!strncmp(opt, "pause:", 6)) { 4453 if (kstrtoint(opt + 6, 0, &pause)) 4454 goto err; 4455 } else if (!strncmp(opt, "eee_timer:", 10)) { 4456 if (kstrtoint(opt + 10, 0, &eee_timer)) 4457 goto err; 4458 } else if (!strncmp(opt, "chain_mode:", 11)) { 4459 if (kstrtoint(opt + 11, 0, &chain_mode)) 4460 goto err; 4461 } 4462 } 4463 return 0; 4464 4465 err: 4466 pr_err("%s: ERROR broken module parameter conversion", __func__); 4467 return -EINVAL; 4468 } 4469 4470 __setup("stmmaceth=", stmmac_cmdline_opt); 4471 #endif /* MODULE */ 4472 4473 static int __init stmmac_init(void) 4474 { 4475 #ifdef CONFIG_DEBUG_FS 4476 /* Create debugfs main directory if it doesn't exist yet */ 4477 if (!stmmac_fs_dir) { 4478 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 4479 4480 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) { 4481 pr_err("ERROR %s, debugfs create directory failed\n", 4482 STMMAC_RESOURCE_NAME); 4483 4484 return -ENOMEM; 4485 } 4486 } 4487 #endif 4488 4489 return 0; 4490 } 4491 4492 static void __exit stmmac_exit(void) 4493 { 4494 #ifdef CONFIG_DEBUG_FS 4495 debugfs_remove_recursive(stmmac_fs_dir); 4496 #endif 4497 } 4498 4499 module_init(stmmac_init) 4500 module_exit(stmmac_exit) 4501 4502 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 4503 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 4504 MODULE_LICENSE("GPL"); 4505