1 /******************************************************************************* 2 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 3 ST Ethernet IPs are built around a Synopsys IP Core. 4 5 Copyright(C) 2007-2011 STMicroelectronics Ltd 6 7 This program is free software; you can redistribute it and/or modify it 8 under the terms and conditions of the GNU General Public License, 9 version 2, as published by the Free Software Foundation. 10 11 This program is distributed in the hope it will be useful, but WITHOUT 12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 more details. 15 16 The full GNU General Public License is included in this distribution in 17 the file called "COPYING". 18 19 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 20 21 Documentation available at: 22 http://www.stlinux.com 23 Support available at: 24 https://bugzilla.stlinux.com/ 25 *******************************************************************************/ 26 27 #include <linux/clk.h> 28 #include <linux/kernel.h> 29 #include <linux/interrupt.h> 30 #include <linux/ip.h> 31 #include <linux/tcp.h> 32 #include <linux/skbuff.h> 33 #include <linux/ethtool.h> 34 #include <linux/if_ether.h> 35 #include <linux/crc32.h> 36 #include <linux/mii.h> 37 #include <linux/if.h> 38 #include <linux/if_vlan.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/slab.h> 41 #include <linux/prefetch.h> 42 #include <linux/pinctrl/consumer.h> 43 #ifdef CONFIG_DEBUG_FS 44 #include <linux/debugfs.h> 45 #include <linux/seq_file.h> 46 #endif /* CONFIG_DEBUG_FS */ 47 #include <linux/net_tstamp.h> 48 #include "stmmac_ptp.h" 49 #include "stmmac.h" 50 #include <linux/reset.h> 51 #include <linux/of_mdio.h> 52 #include "dwmac1000.h" 53 54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x) 55 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 56 57 /* Module parameters */ 58 #define TX_TIMEO 5000 59 static int watchdog = TX_TIMEO; 60 module_param(watchdog, int, S_IRUGO | S_IWUSR); 61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 62 63 static int debug = -1; 64 module_param(debug, int, S_IRUGO | S_IWUSR); 65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 66 67 static int phyaddr = -1; 68 module_param(phyaddr, int, S_IRUGO); 69 MODULE_PARM_DESC(phyaddr, "Physical device address"); 70 71 #define STMMAC_TX_THRESH (DMA_TX_SIZE / 4) 72 #define STMMAC_RX_THRESH (DMA_RX_SIZE / 4) 73 74 static int flow_ctrl = FLOW_OFF; 75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR); 76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 77 78 static int pause = PAUSE_TIME; 79 module_param(pause, int, S_IRUGO | S_IWUSR); 80 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 81 82 #define TC_DEFAULT 64 83 static int tc = TC_DEFAULT; 84 module_param(tc, int, S_IRUGO | S_IWUSR); 85 MODULE_PARM_DESC(tc, "DMA threshold control value"); 86 87 #define DEFAULT_BUFSIZE 1536 88 static int buf_sz = DEFAULT_BUFSIZE; 89 module_param(buf_sz, int, S_IRUGO | S_IWUSR); 90 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 91 92 #define STMMAC_RX_COPYBREAK 256 93 94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 95 NETIF_MSG_LINK | NETIF_MSG_IFUP | 96 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 97 98 #define STMMAC_DEFAULT_LPI_TIMER 1000 99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR); 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x)) 103 104 /* By default the driver will use the ring mode to manage tx and rx descriptors, 105 * but allow user to force to use the chain instead of the ring 106 */ 107 static unsigned int chain_mode; 108 module_param(chain_mode, int, S_IRUGO); 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 110 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 112 113 #ifdef CONFIG_DEBUG_FS 114 static int stmmac_init_fs(struct net_device *dev); 115 static void stmmac_exit_fs(struct net_device *dev); 116 #endif 117 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x)) 119 120 /** 121 * stmmac_verify_args - verify the driver parameters. 122 * Description: it checks the driver parameters and set a default in case of 123 * errors. 124 */ 125 static void stmmac_verify_args(void) 126 { 127 if (unlikely(watchdog < 0)) 128 watchdog = TX_TIMEO; 129 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 130 buf_sz = DEFAULT_BUFSIZE; 131 if (unlikely(flow_ctrl > 1)) 132 flow_ctrl = FLOW_AUTO; 133 else if (likely(flow_ctrl < 0)) 134 flow_ctrl = FLOW_OFF; 135 if (unlikely((pause < 0) || (pause > 0xffff))) 136 pause = PAUSE_TIME; 137 if (eee_timer < 0) 138 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 139 } 140 141 /** 142 * stmmac_clk_csr_set - dynamically set the MDC clock 143 * @priv: driver private structure 144 * Description: this is to dynamically set the MDC clock according to the csr 145 * clock input. 146 * Note: 147 * If a specific clk_csr value is passed from the platform 148 * this means that the CSR Clock Range selection cannot be 149 * changed at run-time and it is fixed (as reported in the driver 150 * documentation). Viceversa the driver will try to set the MDC 151 * clock dynamically according to the actual clock input. 152 */ 153 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 154 { 155 u32 clk_rate; 156 157 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 158 159 /* Platform provided default clk_csr would be assumed valid 160 * for all other cases except for the below mentioned ones. 161 * For values higher than the IEEE 802.3 specified frequency 162 * we can not estimate the proper divider as it is not known 163 * the frequency of clk_csr_i. So we do not change the default 164 * divider. 165 */ 166 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 167 if (clk_rate < CSR_F_35M) 168 priv->clk_csr = STMMAC_CSR_20_35M; 169 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 170 priv->clk_csr = STMMAC_CSR_35_60M; 171 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 172 priv->clk_csr = STMMAC_CSR_60_100M; 173 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 174 priv->clk_csr = STMMAC_CSR_100_150M; 175 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 176 priv->clk_csr = STMMAC_CSR_150_250M; 177 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M)) 178 priv->clk_csr = STMMAC_CSR_250_300M; 179 } 180 } 181 182 static void print_pkt(unsigned char *buf, int len) 183 { 184 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 185 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 186 } 187 188 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv) 189 { 190 u32 avail; 191 192 if (priv->dirty_tx > priv->cur_tx) 193 avail = priv->dirty_tx - priv->cur_tx - 1; 194 else 195 avail = DMA_TX_SIZE - priv->cur_tx + priv->dirty_tx - 1; 196 197 return avail; 198 } 199 200 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv) 201 { 202 u32 dirty; 203 204 if (priv->dirty_rx <= priv->cur_rx) 205 dirty = priv->cur_rx - priv->dirty_rx; 206 else 207 dirty = DMA_RX_SIZE - priv->dirty_rx + priv->cur_rx; 208 209 return dirty; 210 } 211 212 /** 213 * stmmac_hw_fix_mac_speed - callback for speed selection 214 * @priv: driver private structure 215 * Description: on some platforms (e.g. ST), some HW system configuration 216 * registers have to be set according to the link speed negotiated. 217 */ 218 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv) 219 { 220 struct net_device *ndev = priv->dev; 221 struct phy_device *phydev = ndev->phydev; 222 223 if (likely(priv->plat->fix_mac_speed)) 224 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed); 225 } 226 227 /** 228 * stmmac_enable_eee_mode - check and enter in LPI mode 229 * @priv: driver private structure 230 * Description: this function is to verify and enter in LPI mode in case of 231 * EEE. 232 */ 233 static void stmmac_enable_eee_mode(struct stmmac_priv *priv) 234 { 235 /* Check and enter in LPI mode */ 236 if ((priv->dirty_tx == priv->cur_tx) && 237 (priv->tx_path_in_lpi_mode == false)) 238 priv->hw->mac->set_eee_mode(priv->hw, 239 priv->plat->en_tx_lpi_clockgating); 240 } 241 242 /** 243 * stmmac_disable_eee_mode - disable and exit from LPI mode 244 * @priv: driver private structure 245 * Description: this function is to exit and disable EEE in case of 246 * LPI state is true. This is called by the xmit. 247 */ 248 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 249 { 250 priv->hw->mac->reset_eee_mode(priv->hw); 251 del_timer_sync(&priv->eee_ctrl_timer); 252 priv->tx_path_in_lpi_mode = false; 253 } 254 255 /** 256 * stmmac_eee_ctrl_timer - EEE TX SW timer. 257 * @arg : data hook 258 * Description: 259 * if there is no data transfer and if we are not in LPI state, 260 * then MAC Transmitter can be moved to LPI state. 261 */ 262 static void stmmac_eee_ctrl_timer(unsigned long arg) 263 { 264 struct stmmac_priv *priv = (struct stmmac_priv *)arg; 265 266 stmmac_enable_eee_mode(priv); 267 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 268 } 269 270 /** 271 * stmmac_eee_init - init EEE 272 * @priv: driver private structure 273 * Description: 274 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 275 * can also manage EEE, this function enable the LPI state and start related 276 * timer. 277 */ 278 bool stmmac_eee_init(struct stmmac_priv *priv) 279 { 280 struct net_device *ndev = priv->dev; 281 unsigned long flags; 282 bool ret = false; 283 284 /* Using PCS we cannot dial with the phy registers at this stage 285 * so we do not support extra feature like EEE. 286 */ 287 if ((priv->hw->pcs == STMMAC_PCS_RGMII) || 288 (priv->hw->pcs == STMMAC_PCS_TBI) || 289 (priv->hw->pcs == STMMAC_PCS_RTBI)) 290 goto out; 291 292 /* MAC core supports the EEE feature. */ 293 if (priv->dma_cap.eee) { 294 int tx_lpi_timer = priv->tx_lpi_timer; 295 296 /* Check if the PHY supports EEE */ 297 if (phy_init_eee(ndev->phydev, 1)) { 298 /* To manage at run-time if the EEE cannot be supported 299 * anymore (for example because the lp caps have been 300 * changed). 301 * In that case the driver disable own timers. 302 */ 303 spin_lock_irqsave(&priv->lock, flags); 304 if (priv->eee_active) { 305 netdev_dbg(priv->dev, "disable EEE\n"); 306 del_timer_sync(&priv->eee_ctrl_timer); 307 priv->hw->mac->set_eee_timer(priv->hw, 0, 308 tx_lpi_timer); 309 } 310 priv->eee_active = 0; 311 spin_unlock_irqrestore(&priv->lock, flags); 312 goto out; 313 } 314 /* Activate the EEE and start timers */ 315 spin_lock_irqsave(&priv->lock, flags); 316 if (!priv->eee_active) { 317 priv->eee_active = 1; 318 setup_timer(&priv->eee_ctrl_timer, 319 stmmac_eee_ctrl_timer, 320 (unsigned long)priv); 321 mod_timer(&priv->eee_ctrl_timer, 322 STMMAC_LPI_T(eee_timer)); 323 324 priv->hw->mac->set_eee_timer(priv->hw, 325 STMMAC_DEFAULT_LIT_LS, 326 tx_lpi_timer); 327 } 328 /* Set HW EEE according to the speed */ 329 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link); 330 331 ret = true; 332 spin_unlock_irqrestore(&priv->lock, flags); 333 334 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 335 } 336 out: 337 return ret; 338 } 339 340 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 341 * @priv: driver private structure 342 * @p : descriptor pointer 343 * @skb : the socket buffer 344 * Description : 345 * This function will read timestamp from the descriptor & pass it to stack. 346 * and also perform some sanity checks. 347 */ 348 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 349 struct dma_desc *p, struct sk_buff *skb) 350 { 351 struct skb_shared_hwtstamps shhwtstamp; 352 u64 ns; 353 354 if (!priv->hwts_tx_en) 355 return; 356 357 /* exit if skb doesn't support hw tstamp */ 358 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 359 return; 360 361 /* check tx tstamp status */ 362 if (!priv->hw->desc->get_tx_timestamp_status(p)) { 363 /* get the valid tstamp */ 364 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts); 365 366 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 367 shhwtstamp.hwtstamp = ns_to_ktime(ns); 368 369 netdev_info(priv->dev, "get valid TX hw timestamp %llu\n", ns); 370 /* pass tstamp to stack */ 371 skb_tstamp_tx(skb, &shhwtstamp); 372 } 373 374 return; 375 } 376 377 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 378 * @priv: driver private structure 379 * @p : descriptor pointer 380 * @np : next descriptor pointer 381 * @skb : the socket buffer 382 * Description : 383 * This function will read received packet's timestamp from the descriptor 384 * and pass it to stack. It also perform some sanity checks. 385 */ 386 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 387 struct dma_desc *np, struct sk_buff *skb) 388 { 389 struct skb_shared_hwtstamps *shhwtstamp = NULL; 390 u64 ns; 391 392 if (!priv->hwts_rx_en) 393 return; 394 395 /* Check if timestamp is available */ 396 if (!priv->hw->desc->get_rx_timestamp_status(p, priv->adv_ts)) { 397 /* For GMAC4, the valid timestamp is from CTX next desc. */ 398 if (priv->plat->has_gmac4) 399 ns = priv->hw->desc->get_timestamp(np, priv->adv_ts); 400 else 401 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts); 402 403 netdev_info(priv->dev, "get valid RX hw timestamp %llu\n", ns); 404 shhwtstamp = skb_hwtstamps(skb); 405 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 406 shhwtstamp->hwtstamp = ns_to_ktime(ns); 407 } else { 408 netdev_err(priv->dev, "cannot get RX hw timestamp\n"); 409 } 410 } 411 412 /** 413 * stmmac_hwtstamp_ioctl - control hardware timestamping. 414 * @dev: device pointer. 415 * @ifr: An IOCTL specific structure, that can contain a pointer to 416 * a proprietary structure used to pass information to the driver. 417 * Description: 418 * This function configures the MAC to enable/disable both outgoing(TX) 419 * and incoming(RX) packets time stamping based on user input. 420 * Return Value: 421 * 0 on success and an appropriate -ve integer on failure. 422 */ 423 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr) 424 { 425 struct stmmac_priv *priv = netdev_priv(dev); 426 struct hwtstamp_config config; 427 struct timespec64 now; 428 u64 temp = 0; 429 u32 ptp_v2 = 0; 430 u32 tstamp_all = 0; 431 u32 ptp_over_ipv4_udp = 0; 432 u32 ptp_over_ipv6_udp = 0; 433 u32 ptp_over_ethernet = 0; 434 u32 snap_type_sel = 0; 435 u32 ts_master_en = 0; 436 u32 ts_event_en = 0; 437 u32 value = 0; 438 u32 sec_inc; 439 440 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 441 netdev_alert(priv->dev, "No support for HW time stamping\n"); 442 priv->hwts_tx_en = 0; 443 priv->hwts_rx_en = 0; 444 445 return -EOPNOTSUPP; 446 } 447 448 if (copy_from_user(&config, ifr->ifr_data, 449 sizeof(struct hwtstamp_config))) 450 return -EFAULT; 451 452 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 453 __func__, config.flags, config.tx_type, config.rx_filter); 454 455 /* reserved for future extensions */ 456 if (config.flags) 457 return -EINVAL; 458 459 if (config.tx_type != HWTSTAMP_TX_OFF && 460 config.tx_type != HWTSTAMP_TX_ON) 461 return -ERANGE; 462 463 if (priv->adv_ts) { 464 switch (config.rx_filter) { 465 case HWTSTAMP_FILTER_NONE: 466 /* time stamp no incoming packet at all */ 467 config.rx_filter = HWTSTAMP_FILTER_NONE; 468 break; 469 470 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 471 /* PTP v1, UDP, any kind of event packet */ 472 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 473 /* take time stamp for all event messages */ 474 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 475 476 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 477 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 478 break; 479 480 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 481 /* PTP v1, UDP, Sync packet */ 482 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 483 /* take time stamp for SYNC messages only */ 484 ts_event_en = PTP_TCR_TSEVNTENA; 485 486 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 487 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 488 break; 489 490 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 491 /* PTP v1, UDP, Delay_req packet */ 492 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 493 /* take time stamp for Delay_Req messages only */ 494 ts_master_en = PTP_TCR_TSMSTRENA; 495 ts_event_en = PTP_TCR_TSEVNTENA; 496 497 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 498 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 499 break; 500 501 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 502 /* PTP v2, UDP, any kind of event packet */ 503 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 504 ptp_v2 = PTP_TCR_TSVER2ENA; 505 /* take time stamp for all event messages */ 506 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 507 508 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 509 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 510 break; 511 512 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 513 /* PTP v2, UDP, Sync packet */ 514 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 515 ptp_v2 = PTP_TCR_TSVER2ENA; 516 /* take time stamp for SYNC messages only */ 517 ts_event_en = PTP_TCR_TSEVNTENA; 518 519 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 520 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 521 break; 522 523 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 524 /* PTP v2, UDP, Delay_req packet */ 525 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 526 ptp_v2 = PTP_TCR_TSVER2ENA; 527 /* take time stamp for Delay_Req messages only */ 528 ts_master_en = PTP_TCR_TSMSTRENA; 529 ts_event_en = PTP_TCR_TSEVNTENA; 530 531 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 532 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 533 break; 534 535 case HWTSTAMP_FILTER_PTP_V2_EVENT: 536 /* PTP v2/802.AS1 any layer, any kind of event packet */ 537 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 538 ptp_v2 = PTP_TCR_TSVER2ENA; 539 /* take time stamp for all event messages */ 540 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 541 542 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 543 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 544 ptp_over_ethernet = PTP_TCR_TSIPENA; 545 break; 546 547 case HWTSTAMP_FILTER_PTP_V2_SYNC: 548 /* PTP v2/802.AS1, any layer, Sync packet */ 549 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 550 ptp_v2 = PTP_TCR_TSVER2ENA; 551 /* take time stamp for SYNC messages only */ 552 ts_event_en = PTP_TCR_TSEVNTENA; 553 554 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 555 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 556 ptp_over_ethernet = PTP_TCR_TSIPENA; 557 break; 558 559 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 560 /* PTP v2/802.AS1, any layer, Delay_req packet */ 561 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 562 ptp_v2 = PTP_TCR_TSVER2ENA; 563 /* take time stamp for Delay_Req messages only */ 564 ts_master_en = PTP_TCR_TSMSTRENA; 565 ts_event_en = PTP_TCR_TSEVNTENA; 566 567 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 568 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 569 ptp_over_ethernet = PTP_TCR_TSIPENA; 570 break; 571 572 case HWTSTAMP_FILTER_ALL: 573 /* time stamp any incoming packet */ 574 config.rx_filter = HWTSTAMP_FILTER_ALL; 575 tstamp_all = PTP_TCR_TSENALL; 576 break; 577 578 default: 579 return -ERANGE; 580 } 581 } else { 582 switch (config.rx_filter) { 583 case HWTSTAMP_FILTER_NONE: 584 config.rx_filter = HWTSTAMP_FILTER_NONE; 585 break; 586 default: 587 /* PTP v1, UDP, any kind of event packet */ 588 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 589 break; 590 } 591 } 592 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 593 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 594 595 if (!priv->hwts_tx_en && !priv->hwts_rx_en) 596 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0); 597 else { 598 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR | 599 tstamp_all | ptp_v2 | ptp_over_ethernet | 600 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en | 601 ts_master_en | snap_type_sel); 602 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value); 603 604 /* program Sub Second Increment reg */ 605 sec_inc = priv->hw->ptp->config_sub_second_increment( 606 priv->ptpaddr, priv->plat->clk_ptp_rate, 607 priv->plat->has_gmac4); 608 temp = div_u64(1000000000ULL, sec_inc); 609 610 /* calculate default added value: 611 * formula is : 612 * addend = (2^32)/freq_div_ratio; 613 * where, freq_div_ratio = 1e9ns/sec_inc 614 */ 615 temp = (u64)(temp << 32); 616 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 617 priv->hw->ptp->config_addend(priv->ptpaddr, 618 priv->default_addend); 619 620 /* initialize system time */ 621 ktime_get_real_ts64(&now); 622 623 /* lower 32 bits of tv_sec are safe until y2106 */ 624 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec, 625 now.tv_nsec); 626 } 627 628 return copy_to_user(ifr->ifr_data, &config, 629 sizeof(struct hwtstamp_config)) ? -EFAULT : 0; 630 } 631 632 /** 633 * stmmac_init_ptp - init PTP 634 * @priv: driver private structure 635 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 636 * This is done by looking at the HW cap. register. 637 * This function also registers the ptp driver. 638 */ 639 static int stmmac_init_ptp(struct stmmac_priv *priv) 640 { 641 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 642 return -EOPNOTSUPP; 643 644 priv->adv_ts = 0; 645 /* Check if adv_ts can be enabled for dwmac 4.x core */ 646 if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp) 647 priv->adv_ts = 1; 648 /* Dwmac 3.x core with extend_desc can support adv_ts */ 649 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 650 priv->adv_ts = 1; 651 652 if (priv->dma_cap.time_stamp) 653 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 654 655 if (priv->adv_ts) 656 netdev_info(priv->dev, 657 "IEEE 1588-2008 Advanced Timestamp supported\n"); 658 659 priv->hw->ptp = &stmmac_ptp; 660 priv->hwts_tx_en = 0; 661 priv->hwts_rx_en = 0; 662 663 stmmac_ptp_register(priv); 664 665 return 0; 666 } 667 668 static void stmmac_release_ptp(struct stmmac_priv *priv) 669 { 670 if (priv->plat->clk_ptp_ref) 671 clk_disable_unprepare(priv->plat->clk_ptp_ref); 672 stmmac_ptp_unregister(priv); 673 } 674 675 /** 676 * stmmac_mac_flow_ctrl - Configure flow control in all queues 677 * @priv: driver private structure 678 * Description: It is used for configuring the flow control in all queues 679 */ 680 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 681 { 682 u32 tx_cnt = priv->plat->tx_queues_to_use; 683 684 priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl, 685 priv->pause, tx_cnt); 686 } 687 688 /** 689 * stmmac_adjust_link - adjusts the link parameters 690 * @dev: net device structure 691 * Description: this is the helper called by the physical abstraction layer 692 * drivers to communicate the phy link status. According the speed and duplex 693 * this driver can invoke registered glue-logic as well. 694 * It also invoke the eee initialization because it could happen when switch 695 * on different networks (that are eee capable). 696 */ 697 static void stmmac_adjust_link(struct net_device *dev) 698 { 699 struct stmmac_priv *priv = netdev_priv(dev); 700 struct phy_device *phydev = dev->phydev; 701 unsigned long flags; 702 int new_state = 0; 703 704 if (!phydev) 705 return; 706 707 spin_lock_irqsave(&priv->lock, flags); 708 709 if (phydev->link) { 710 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 711 712 /* Now we make sure that we can be in full duplex mode. 713 * If not, we operate in half-duplex mode. */ 714 if (phydev->duplex != priv->oldduplex) { 715 new_state = 1; 716 if (!(phydev->duplex)) 717 ctrl &= ~priv->hw->link.duplex; 718 else 719 ctrl |= priv->hw->link.duplex; 720 priv->oldduplex = phydev->duplex; 721 } 722 /* Flow Control operation */ 723 if (phydev->pause) 724 stmmac_mac_flow_ctrl(priv, phydev->duplex); 725 726 if (phydev->speed != priv->speed) { 727 new_state = 1; 728 switch (phydev->speed) { 729 case 1000: 730 if (priv->plat->has_gmac || 731 priv->plat->has_gmac4) 732 ctrl &= ~priv->hw->link.port; 733 break; 734 case 100: 735 if (priv->plat->has_gmac || 736 priv->plat->has_gmac4) { 737 ctrl |= priv->hw->link.port; 738 ctrl |= priv->hw->link.speed; 739 } else { 740 ctrl &= ~priv->hw->link.port; 741 } 742 break; 743 case 10: 744 if (priv->plat->has_gmac || 745 priv->plat->has_gmac4) { 746 ctrl |= priv->hw->link.port; 747 ctrl &= ~(priv->hw->link.speed); 748 } else { 749 ctrl &= ~priv->hw->link.port; 750 } 751 break; 752 default: 753 netif_warn(priv, link, priv->dev, 754 "broken speed: %d\n", phydev->speed); 755 phydev->speed = SPEED_UNKNOWN; 756 break; 757 } 758 if (phydev->speed != SPEED_UNKNOWN) 759 stmmac_hw_fix_mac_speed(priv); 760 priv->speed = phydev->speed; 761 } 762 763 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 764 765 if (!priv->oldlink) { 766 new_state = 1; 767 priv->oldlink = 1; 768 } 769 } else if (priv->oldlink) { 770 new_state = 1; 771 priv->oldlink = 0; 772 priv->speed = SPEED_UNKNOWN; 773 priv->oldduplex = DUPLEX_UNKNOWN; 774 } 775 776 if (new_state && netif_msg_link(priv)) 777 phy_print_status(phydev); 778 779 spin_unlock_irqrestore(&priv->lock, flags); 780 781 if (phydev->is_pseudo_fixed_link) 782 /* Stop PHY layer to call the hook to adjust the link in case 783 * of a switch is attached to the stmmac driver. 784 */ 785 phydev->irq = PHY_IGNORE_INTERRUPT; 786 else 787 /* At this stage, init the EEE if supported. 788 * Never called in case of fixed_link. 789 */ 790 priv->eee_enabled = stmmac_eee_init(priv); 791 } 792 793 /** 794 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 795 * @priv: driver private structure 796 * Description: this is to verify if the HW supports the PCS. 797 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 798 * configured for the TBI, RTBI, or SGMII PHY interface. 799 */ 800 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 801 { 802 int interface = priv->plat->interface; 803 804 if (priv->dma_cap.pcs) { 805 if ((interface == PHY_INTERFACE_MODE_RGMII) || 806 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 807 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 808 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 809 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 810 priv->hw->pcs = STMMAC_PCS_RGMII; 811 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 812 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 813 priv->hw->pcs = STMMAC_PCS_SGMII; 814 } 815 } 816 } 817 818 /** 819 * stmmac_init_phy - PHY initialization 820 * @dev: net device structure 821 * Description: it initializes the driver's PHY state, and attaches the PHY 822 * to the mac driver. 823 * Return value: 824 * 0 on success 825 */ 826 static int stmmac_init_phy(struct net_device *dev) 827 { 828 struct stmmac_priv *priv = netdev_priv(dev); 829 struct phy_device *phydev; 830 char phy_id_fmt[MII_BUS_ID_SIZE + 3]; 831 char bus_id[MII_BUS_ID_SIZE]; 832 int interface = priv->plat->interface; 833 int max_speed = priv->plat->max_speed; 834 priv->oldlink = 0; 835 priv->speed = SPEED_UNKNOWN; 836 priv->oldduplex = DUPLEX_UNKNOWN; 837 838 if (priv->plat->phy_node) { 839 phydev = of_phy_connect(dev, priv->plat->phy_node, 840 &stmmac_adjust_link, 0, interface); 841 } else { 842 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x", 843 priv->plat->bus_id); 844 845 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id, 846 priv->plat->phy_addr); 847 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__, 848 phy_id_fmt); 849 850 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link, 851 interface); 852 } 853 854 if (IS_ERR_OR_NULL(phydev)) { 855 netdev_err(priv->dev, "Could not attach to PHY\n"); 856 if (!phydev) 857 return -ENODEV; 858 859 return PTR_ERR(phydev); 860 } 861 862 /* Stop Advertising 1000BASE Capability if interface is not GMII */ 863 if ((interface == PHY_INTERFACE_MODE_MII) || 864 (interface == PHY_INTERFACE_MODE_RMII) || 865 (max_speed < 1000 && max_speed > 0)) 866 phydev->advertising &= ~(SUPPORTED_1000baseT_Half | 867 SUPPORTED_1000baseT_Full); 868 869 /* 870 * Broken HW is sometimes missing the pull-up resistor on the 871 * MDIO line, which results in reads to non-existent devices returning 872 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent 873 * device as well. 874 * Note: phydev->phy_id is the result of reading the UID PHY registers. 875 */ 876 if (!priv->plat->phy_node && phydev->phy_id == 0) { 877 phy_disconnect(phydev); 878 return -ENODEV; 879 } 880 881 /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid 882 * subsequent PHY polling, make sure we force a link transition if 883 * we have a UP/DOWN/UP transition 884 */ 885 if (phydev->is_pseudo_fixed_link) 886 phydev->irq = PHY_POLL; 887 888 phy_attached_info(phydev); 889 return 0; 890 } 891 892 static void stmmac_display_rings(struct stmmac_priv *priv) 893 { 894 void *head_rx, *head_tx; 895 896 if (priv->extend_desc) { 897 head_rx = (void *)priv->dma_erx; 898 head_tx = (void *)priv->dma_etx; 899 } else { 900 head_rx = (void *)priv->dma_rx; 901 head_tx = (void *)priv->dma_tx; 902 } 903 904 /* Display Rx ring */ 905 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true); 906 /* Display Tx ring */ 907 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false); 908 } 909 910 static int stmmac_set_bfsize(int mtu, int bufsize) 911 { 912 int ret = bufsize; 913 914 if (mtu >= BUF_SIZE_4KiB) 915 ret = BUF_SIZE_8KiB; 916 else if (mtu >= BUF_SIZE_2KiB) 917 ret = BUF_SIZE_4KiB; 918 else if (mtu > DEFAULT_BUFSIZE) 919 ret = BUF_SIZE_2KiB; 920 else 921 ret = DEFAULT_BUFSIZE; 922 923 return ret; 924 } 925 926 /** 927 * stmmac_clear_descriptors - clear descriptors 928 * @priv: driver private structure 929 * Description: this function is called to clear the tx and rx descriptors 930 * in case of both basic and extended descriptors are used. 931 */ 932 static void stmmac_clear_descriptors(struct stmmac_priv *priv) 933 { 934 int i; 935 936 /* Clear the Rx/Tx descriptors */ 937 for (i = 0; i < DMA_RX_SIZE; i++) 938 if (priv->extend_desc) 939 priv->hw->desc->init_rx_desc(&priv->dma_erx[i].basic, 940 priv->use_riwt, priv->mode, 941 (i == DMA_RX_SIZE - 1)); 942 else 943 priv->hw->desc->init_rx_desc(&priv->dma_rx[i], 944 priv->use_riwt, priv->mode, 945 (i == DMA_RX_SIZE - 1)); 946 for (i = 0; i < DMA_TX_SIZE; i++) 947 if (priv->extend_desc) 948 priv->hw->desc->init_tx_desc(&priv->dma_etx[i].basic, 949 priv->mode, 950 (i == DMA_TX_SIZE - 1)); 951 else 952 priv->hw->desc->init_tx_desc(&priv->dma_tx[i], 953 priv->mode, 954 (i == DMA_TX_SIZE - 1)); 955 } 956 957 /** 958 * stmmac_init_rx_buffers - init the RX descriptor buffer. 959 * @priv: driver private structure 960 * @p: descriptor pointer 961 * @i: descriptor index 962 * @flags: gfp flag. 963 * Description: this function is called to allocate a receive buffer, perform 964 * the DMA mapping and init the descriptor. 965 */ 966 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, 967 int i, gfp_t flags) 968 { 969 struct sk_buff *skb; 970 971 skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags); 972 if (!skb) { 973 netdev_err(priv->dev, 974 "%s: Rx init fails; skb is NULL\n", __func__); 975 return -ENOMEM; 976 } 977 priv->rx_skbuff[i] = skb; 978 priv->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data, 979 priv->dma_buf_sz, 980 DMA_FROM_DEVICE); 981 if (dma_mapping_error(priv->device, priv->rx_skbuff_dma[i])) { 982 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__); 983 dev_kfree_skb_any(skb); 984 return -EINVAL; 985 } 986 987 if (priv->synopsys_id >= DWMAC_CORE_4_00) 988 p->des0 = cpu_to_le32(priv->rx_skbuff_dma[i]); 989 else 990 p->des2 = cpu_to_le32(priv->rx_skbuff_dma[i]); 991 992 if ((priv->hw->mode->init_desc3) && 993 (priv->dma_buf_sz == BUF_SIZE_16KiB)) 994 priv->hw->mode->init_desc3(p); 995 996 return 0; 997 } 998 999 static void stmmac_free_rx_buffers(struct stmmac_priv *priv, int i) 1000 { 1001 if (priv->rx_skbuff[i]) { 1002 dma_unmap_single(priv->device, priv->rx_skbuff_dma[i], 1003 priv->dma_buf_sz, DMA_FROM_DEVICE); 1004 dev_kfree_skb_any(priv->rx_skbuff[i]); 1005 } 1006 priv->rx_skbuff[i] = NULL; 1007 } 1008 1009 /** 1010 * init_dma_desc_rings - init the RX/TX descriptor rings 1011 * @dev: net device structure 1012 * @flags: gfp flag. 1013 * Description: this function initializes the DMA RX/TX descriptors 1014 * and allocates the socket buffers. It supports the chained and ring 1015 * modes. 1016 */ 1017 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags) 1018 { 1019 int i; 1020 struct stmmac_priv *priv = netdev_priv(dev); 1021 unsigned int bfsize = 0; 1022 int ret = -ENOMEM; 1023 1024 if (priv->hw->mode->set_16kib_bfsize) 1025 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu); 1026 1027 if (bfsize < BUF_SIZE_16KiB) 1028 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); 1029 1030 priv->dma_buf_sz = bfsize; 1031 1032 netif_dbg(priv, probe, priv->dev, 1033 "(%s) dma_rx_phy=0x%08x dma_tx_phy=0x%08x\n", 1034 __func__, (u32)priv->dma_rx_phy, (u32)priv->dma_tx_phy); 1035 1036 /* RX INITIALIZATION */ 1037 netif_dbg(priv, probe, priv->dev, 1038 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1039 1040 for (i = 0; i < DMA_RX_SIZE; i++) { 1041 struct dma_desc *p; 1042 if (priv->extend_desc) 1043 p = &((priv->dma_erx + i)->basic); 1044 else 1045 p = priv->dma_rx + i; 1046 1047 ret = stmmac_init_rx_buffers(priv, p, i, flags); 1048 if (ret) 1049 goto err_init_rx_buffers; 1050 1051 netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n", 1052 priv->rx_skbuff[i], priv->rx_skbuff[i]->data, 1053 (unsigned int)priv->rx_skbuff_dma[i]); 1054 } 1055 priv->cur_rx = 0; 1056 priv->dirty_rx = (unsigned int)(i - DMA_RX_SIZE); 1057 buf_sz = bfsize; 1058 1059 /* Setup the chained descriptor addresses */ 1060 if (priv->mode == STMMAC_CHAIN_MODE) { 1061 if (priv->extend_desc) { 1062 priv->hw->mode->init(priv->dma_erx, priv->dma_rx_phy, 1063 DMA_RX_SIZE, 1); 1064 priv->hw->mode->init(priv->dma_etx, priv->dma_tx_phy, 1065 DMA_TX_SIZE, 1); 1066 } else { 1067 priv->hw->mode->init(priv->dma_rx, priv->dma_rx_phy, 1068 DMA_RX_SIZE, 0); 1069 priv->hw->mode->init(priv->dma_tx, priv->dma_tx_phy, 1070 DMA_TX_SIZE, 0); 1071 } 1072 } 1073 1074 /* TX INITIALIZATION */ 1075 for (i = 0; i < DMA_TX_SIZE; i++) { 1076 struct dma_desc *p; 1077 if (priv->extend_desc) 1078 p = &((priv->dma_etx + i)->basic); 1079 else 1080 p = priv->dma_tx + i; 1081 1082 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1083 p->des0 = 0; 1084 p->des1 = 0; 1085 p->des2 = 0; 1086 p->des3 = 0; 1087 } else { 1088 p->des2 = 0; 1089 } 1090 1091 priv->tx_skbuff_dma[i].buf = 0; 1092 priv->tx_skbuff_dma[i].map_as_page = false; 1093 priv->tx_skbuff_dma[i].len = 0; 1094 priv->tx_skbuff_dma[i].last_segment = false; 1095 priv->tx_skbuff[i] = NULL; 1096 } 1097 1098 priv->dirty_tx = 0; 1099 priv->cur_tx = 0; 1100 netdev_reset_queue(priv->dev); 1101 1102 stmmac_clear_descriptors(priv); 1103 1104 if (netif_msg_hw(priv)) 1105 stmmac_display_rings(priv); 1106 1107 return 0; 1108 err_init_rx_buffers: 1109 while (--i >= 0) 1110 stmmac_free_rx_buffers(priv, i); 1111 return ret; 1112 } 1113 1114 static void dma_free_rx_skbufs(struct stmmac_priv *priv) 1115 { 1116 int i; 1117 1118 for (i = 0; i < DMA_RX_SIZE; i++) 1119 stmmac_free_rx_buffers(priv, i); 1120 } 1121 1122 static void dma_free_tx_skbufs(struct stmmac_priv *priv) 1123 { 1124 int i; 1125 1126 for (i = 0; i < DMA_TX_SIZE; i++) { 1127 if (priv->tx_skbuff_dma[i].buf) { 1128 if (priv->tx_skbuff_dma[i].map_as_page) 1129 dma_unmap_page(priv->device, 1130 priv->tx_skbuff_dma[i].buf, 1131 priv->tx_skbuff_dma[i].len, 1132 DMA_TO_DEVICE); 1133 else 1134 dma_unmap_single(priv->device, 1135 priv->tx_skbuff_dma[i].buf, 1136 priv->tx_skbuff_dma[i].len, 1137 DMA_TO_DEVICE); 1138 } 1139 1140 if (priv->tx_skbuff[i]) { 1141 dev_kfree_skb_any(priv->tx_skbuff[i]); 1142 priv->tx_skbuff[i] = NULL; 1143 priv->tx_skbuff_dma[i].buf = 0; 1144 priv->tx_skbuff_dma[i].map_as_page = false; 1145 } 1146 } 1147 } 1148 1149 /** 1150 * alloc_dma_desc_resources - alloc TX/RX resources. 1151 * @priv: private structure 1152 * Description: according to which descriptor can be used (extend or basic) 1153 * this function allocates the resources for TX and RX paths. In case of 1154 * reception, for example, it pre-allocated the RX socket buffer in order to 1155 * allow zero-copy mechanism. 1156 */ 1157 static int alloc_dma_desc_resources(struct stmmac_priv *priv) 1158 { 1159 int ret = -ENOMEM; 1160 1161 priv->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE, sizeof(dma_addr_t), 1162 GFP_KERNEL); 1163 if (!priv->rx_skbuff_dma) 1164 return -ENOMEM; 1165 1166 priv->rx_skbuff = kmalloc_array(DMA_RX_SIZE, sizeof(struct sk_buff *), 1167 GFP_KERNEL); 1168 if (!priv->rx_skbuff) 1169 goto err_rx_skbuff; 1170 1171 priv->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE, 1172 sizeof(*priv->tx_skbuff_dma), 1173 GFP_KERNEL); 1174 if (!priv->tx_skbuff_dma) 1175 goto err_tx_skbuff_dma; 1176 1177 priv->tx_skbuff = kmalloc_array(DMA_TX_SIZE, sizeof(struct sk_buff *), 1178 GFP_KERNEL); 1179 if (!priv->tx_skbuff) 1180 goto err_tx_skbuff; 1181 1182 if (priv->extend_desc) { 1183 priv->dma_erx = dma_zalloc_coherent(priv->device, DMA_RX_SIZE * 1184 sizeof(struct 1185 dma_extended_desc), 1186 &priv->dma_rx_phy, 1187 GFP_KERNEL); 1188 if (!priv->dma_erx) 1189 goto err_dma; 1190 1191 priv->dma_etx = dma_zalloc_coherent(priv->device, DMA_TX_SIZE * 1192 sizeof(struct 1193 dma_extended_desc), 1194 &priv->dma_tx_phy, 1195 GFP_KERNEL); 1196 if (!priv->dma_etx) { 1197 dma_free_coherent(priv->device, DMA_RX_SIZE * 1198 sizeof(struct dma_extended_desc), 1199 priv->dma_erx, priv->dma_rx_phy); 1200 goto err_dma; 1201 } 1202 } else { 1203 priv->dma_rx = dma_zalloc_coherent(priv->device, DMA_RX_SIZE * 1204 sizeof(struct dma_desc), 1205 &priv->dma_rx_phy, 1206 GFP_KERNEL); 1207 if (!priv->dma_rx) 1208 goto err_dma; 1209 1210 priv->dma_tx = dma_zalloc_coherent(priv->device, DMA_TX_SIZE * 1211 sizeof(struct dma_desc), 1212 &priv->dma_tx_phy, 1213 GFP_KERNEL); 1214 if (!priv->dma_tx) { 1215 dma_free_coherent(priv->device, DMA_RX_SIZE * 1216 sizeof(struct dma_desc), 1217 priv->dma_rx, priv->dma_rx_phy); 1218 goto err_dma; 1219 } 1220 } 1221 1222 return 0; 1223 1224 err_dma: 1225 kfree(priv->tx_skbuff); 1226 err_tx_skbuff: 1227 kfree(priv->tx_skbuff_dma); 1228 err_tx_skbuff_dma: 1229 kfree(priv->rx_skbuff); 1230 err_rx_skbuff: 1231 kfree(priv->rx_skbuff_dma); 1232 return ret; 1233 } 1234 1235 static void free_dma_desc_resources(struct stmmac_priv *priv) 1236 { 1237 /* Release the DMA TX/RX socket buffers */ 1238 dma_free_rx_skbufs(priv); 1239 dma_free_tx_skbufs(priv); 1240 1241 /* Free DMA regions of consistent memory previously allocated */ 1242 if (!priv->extend_desc) { 1243 dma_free_coherent(priv->device, 1244 DMA_TX_SIZE * sizeof(struct dma_desc), 1245 priv->dma_tx, priv->dma_tx_phy); 1246 dma_free_coherent(priv->device, 1247 DMA_RX_SIZE * sizeof(struct dma_desc), 1248 priv->dma_rx, priv->dma_rx_phy); 1249 } else { 1250 dma_free_coherent(priv->device, DMA_TX_SIZE * 1251 sizeof(struct dma_extended_desc), 1252 priv->dma_etx, priv->dma_tx_phy); 1253 dma_free_coherent(priv->device, DMA_RX_SIZE * 1254 sizeof(struct dma_extended_desc), 1255 priv->dma_erx, priv->dma_rx_phy); 1256 } 1257 kfree(priv->rx_skbuff_dma); 1258 kfree(priv->rx_skbuff); 1259 kfree(priv->tx_skbuff_dma); 1260 kfree(priv->tx_skbuff); 1261 } 1262 1263 /** 1264 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 1265 * @priv: driver private structure 1266 * Description: It is used for enabling the rx queues in the MAC 1267 */ 1268 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 1269 { 1270 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1271 int queue; 1272 u8 mode; 1273 1274 for (queue = 0; queue < rx_queues_count; queue++) { 1275 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 1276 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue); 1277 } 1278 } 1279 1280 /** 1281 * stmmac_start_rx_dma - start RX DMA channel 1282 * @priv: driver private structure 1283 * @chan: RX channel index 1284 * Description: 1285 * This starts a RX DMA channel 1286 */ 1287 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 1288 { 1289 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 1290 priv->hw->dma->start_rx(priv->ioaddr, chan); 1291 } 1292 1293 /** 1294 * stmmac_start_tx_dma - start TX DMA channel 1295 * @priv: driver private structure 1296 * @chan: TX channel index 1297 * Description: 1298 * This starts a TX DMA channel 1299 */ 1300 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 1301 { 1302 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 1303 priv->hw->dma->start_tx(priv->ioaddr, chan); 1304 } 1305 1306 /** 1307 * stmmac_stop_rx_dma - stop RX DMA channel 1308 * @priv: driver private structure 1309 * @chan: RX channel index 1310 * Description: 1311 * This stops a RX DMA channel 1312 */ 1313 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 1314 { 1315 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 1316 priv->hw->dma->stop_rx(priv->ioaddr, chan); 1317 } 1318 1319 /** 1320 * stmmac_stop_tx_dma - stop TX DMA channel 1321 * @priv: driver private structure 1322 * @chan: TX channel index 1323 * Description: 1324 * This stops a TX DMA channel 1325 */ 1326 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 1327 { 1328 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 1329 priv->hw->dma->stop_tx(priv->ioaddr, chan); 1330 } 1331 1332 /** 1333 * stmmac_start_all_dma - start all RX and TX DMA channels 1334 * @priv: driver private structure 1335 * Description: 1336 * This starts all the RX and TX DMA channels 1337 */ 1338 static void stmmac_start_all_dma(struct stmmac_priv *priv) 1339 { 1340 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1341 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1342 u32 chan = 0; 1343 1344 for (chan = 0; chan < rx_channels_count; chan++) 1345 stmmac_start_rx_dma(priv, chan); 1346 1347 for (chan = 0; chan < tx_channels_count; chan++) 1348 stmmac_start_tx_dma(priv, chan); 1349 } 1350 1351 /** 1352 * stmmac_stop_all_dma - stop all RX and TX DMA channels 1353 * @priv: driver private structure 1354 * Description: 1355 * This stops the RX and TX DMA channels 1356 */ 1357 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 1358 { 1359 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1360 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1361 u32 chan = 0; 1362 1363 for (chan = 0; chan < rx_channels_count; chan++) 1364 stmmac_stop_rx_dma(priv, chan); 1365 1366 for (chan = 0; chan < tx_channels_count; chan++) 1367 stmmac_stop_tx_dma(priv, chan); 1368 } 1369 1370 /** 1371 * stmmac_dma_operation_mode - HW DMA operation mode 1372 * @priv: driver private structure 1373 * Description: it is used for configuring the DMA operation mode register in 1374 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 1375 */ 1376 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 1377 { 1378 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1379 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1380 int rxfifosz = priv->plat->rx_fifo_size; 1381 u32 txmode = 0; 1382 u32 rxmode = 0; 1383 u32 chan = 0; 1384 1385 if (rxfifosz == 0) 1386 rxfifosz = priv->dma_cap.rx_fifo_size; 1387 1388 if (priv->plat->force_thresh_dma_mode) { 1389 txmode = tc; 1390 rxmode = tc; 1391 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 1392 /* 1393 * In case of GMAC, SF mode can be enabled 1394 * to perform the TX COE in HW. This depends on: 1395 * 1) TX COE if actually supported 1396 * 2) There is no bugged Jumbo frame support 1397 * that needs to not insert csum in the TDES. 1398 */ 1399 txmode = SF_DMA_MODE; 1400 rxmode = SF_DMA_MODE; 1401 priv->xstats.threshold = SF_DMA_MODE; 1402 } else { 1403 txmode = tc; 1404 rxmode = SF_DMA_MODE; 1405 } 1406 1407 /* configure all channels */ 1408 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1409 for (chan = 0; chan < rx_channels_count; chan++) 1410 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan, 1411 rxfifosz); 1412 1413 for (chan = 0; chan < tx_channels_count; chan++) 1414 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan); 1415 } else { 1416 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode, 1417 rxfifosz); 1418 } 1419 } 1420 1421 /** 1422 * stmmac_tx_clean - to manage the transmission completion 1423 * @priv: driver private structure 1424 * Description: it reclaims the transmit resources after transmission completes. 1425 */ 1426 static void stmmac_tx_clean(struct stmmac_priv *priv) 1427 { 1428 unsigned int bytes_compl = 0, pkts_compl = 0; 1429 unsigned int entry = priv->dirty_tx; 1430 1431 netif_tx_lock(priv->dev); 1432 1433 priv->xstats.tx_clean++; 1434 1435 while (entry != priv->cur_tx) { 1436 struct sk_buff *skb = priv->tx_skbuff[entry]; 1437 struct dma_desc *p; 1438 int status; 1439 1440 if (priv->extend_desc) 1441 p = (struct dma_desc *)(priv->dma_etx + entry); 1442 else 1443 p = priv->dma_tx + entry; 1444 1445 status = priv->hw->desc->tx_status(&priv->dev->stats, 1446 &priv->xstats, p, 1447 priv->ioaddr); 1448 /* Check if the descriptor is owned by the DMA */ 1449 if (unlikely(status & tx_dma_own)) 1450 break; 1451 1452 /* Just consider the last segment and ...*/ 1453 if (likely(!(status & tx_not_ls))) { 1454 /* ... verify the status error condition */ 1455 if (unlikely(status & tx_err)) { 1456 priv->dev->stats.tx_errors++; 1457 } else { 1458 priv->dev->stats.tx_packets++; 1459 priv->xstats.tx_pkt_n++; 1460 } 1461 stmmac_get_tx_hwtstamp(priv, p, skb); 1462 } 1463 1464 if (likely(priv->tx_skbuff_dma[entry].buf)) { 1465 if (priv->tx_skbuff_dma[entry].map_as_page) 1466 dma_unmap_page(priv->device, 1467 priv->tx_skbuff_dma[entry].buf, 1468 priv->tx_skbuff_dma[entry].len, 1469 DMA_TO_DEVICE); 1470 else 1471 dma_unmap_single(priv->device, 1472 priv->tx_skbuff_dma[entry].buf, 1473 priv->tx_skbuff_dma[entry].len, 1474 DMA_TO_DEVICE); 1475 priv->tx_skbuff_dma[entry].buf = 0; 1476 priv->tx_skbuff_dma[entry].len = 0; 1477 priv->tx_skbuff_dma[entry].map_as_page = false; 1478 } 1479 1480 if (priv->hw->mode->clean_desc3) 1481 priv->hw->mode->clean_desc3(priv, p); 1482 1483 priv->tx_skbuff_dma[entry].last_segment = false; 1484 priv->tx_skbuff_dma[entry].is_jumbo = false; 1485 1486 if (likely(skb != NULL)) { 1487 pkts_compl++; 1488 bytes_compl += skb->len; 1489 dev_consume_skb_any(skb); 1490 priv->tx_skbuff[entry] = NULL; 1491 } 1492 1493 priv->hw->desc->release_tx_desc(p, priv->mode); 1494 1495 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 1496 } 1497 priv->dirty_tx = entry; 1498 1499 netdev_completed_queue(priv->dev, pkts_compl, bytes_compl); 1500 1501 if (unlikely(netif_queue_stopped(priv->dev) && 1502 stmmac_tx_avail(priv) > STMMAC_TX_THRESH)) { 1503 netif_dbg(priv, tx_done, priv->dev, 1504 "%s: restart transmit\n", __func__); 1505 netif_wake_queue(priv->dev); 1506 } 1507 1508 if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) { 1509 stmmac_enable_eee_mode(priv); 1510 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 1511 } 1512 netif_tx_unlock(priv->dev); 1513 } 1514 1515 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan) 1516 { 1517 priv->hw->dma->enable_dma_irq(priv->ioaddr, chan); 1518 } 1519 1520 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan) 1521 { 1522 priv->hw->dma->disable_dma_irq(priv->ioaddr, chan); 1523 } 1524 1525 /** 1526 * stmmac_tx_err - to manage the tx error 1527 * @priv: driver private structure 1528 * @chan: channel index 1529 * Description: it cleans the descriptors and restarts the transmission 1530 * in case of transmission errors. 1531 */ 1532 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 1533 { 1534 int i; 1535 netif_stop_queue(priv->dev); 1536 1537 stmmac_stop_tx_dma(priv, chan); 1538 dma_free_tx_skbufs(priv); 1539 for (i = 0; i < DMA_TX_SIZE; i++) 1540 if (priv->extend_desc) 1541 priv->hw->desc->init_tx_desc(&priv->dma_etx[i].basic, 1542 priv->mode, 1543 (i == DMA_TX_SIZE - 1)); 1544 else 1545 priv->hw->desc->init_tx_desc(&priv->dma_tx[i], 1546 priv->mode, 1547 (i == DMA_TX_SIZE - 1)); 1548 priv->dirty_tx = 0; 1549 priv->cur_tx = 0; 1550 netdev_reset_queue(priv->dev); 1551 stmmac_start_tx_dma(priv, chan); 1552 1553 priv->dev->stats.tx_errors++; 1554 netif_wake_queue(priv->dev); 1555 } 1556 1557 /** 1558 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 1559 * @priv: driver private structure 1560 * @txmode: TX operating mode 1561 * @rxmode: RX operating mode 1562 * @chan: channel index 1563 * Description: it is used for configuring of the DMA operation mode in 1564 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 1565 * mode. 1566 */ 1567 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 1568 u32 rxmode, u32 chan) 1569 { 1570 int rxfifosz = priv->plat->rx_fifo_size; 1571 1572 if (rxfifosz == 0) 1573 rxfifosz = priv->dma_cap.rx_fifo_size; 1574 1575 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1576 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan, 1577 rxfifosz); 1578 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan); 1579 } else { 1580 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode, 1581 rxfifosz); 1582 } 1583 } 1584 1585 /** 1586 * stmmac_dma_interrupt - DMA ISR 1587 * @priv: driver private structure 1588 * Description: this is the DMA ISR. It is called by the main ISR. 1589 * It calls the dwmac dma routine and schedule poll method in case of some 1590 * work can be done. 1591 */ 1592 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 1593 { 1594 u32 tx_channel_count = priv->plat->tx_queues_to_use; 1595 int status; 1596 u32 chan; 1597 1598 for (chan = 0; chan < tx_channel_count; chan++) { 1599 status = priv->hw->dma->dma_interrupt(priv->ioaddr, 1600 &priv->xstats, chan); 1601 if (likely((status & handle_rx)) || (status & handle_tx)) { 1602 if (likely(napi_schedule_prep(&priv->napi))) { 1603 stmmac_disable_dma_irq(priv, chan); 1604 __napi_schedule(&priv->napi); 1605 } 1606 } 1607 1608 if (unlikely(status & tx_hard_error_bump_tc)) { 1609 /* Try to bump up the dma threshold on this failure */ 1610 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && 1611 (tc <= 256)) { 1612 tc += 64; 1613 if (priv->plat->force_thresh_dma_mode) 1614 stmmac_set_dma_operation_mode(priv, 1615 tc, 1616 tc, 1617 chan); 1618 else 1619 stmmac_set_dma_operation_mode(priv, 1620 tc, 1621 SF_DMA_MODE, 1622 chan); 1623 priv->xstats.threshold = tc; 1624 } 1625 } else if (unlikely(status == tx_hard_error)) { 1626 stmmac_tx_err(priv, chan); 1627 } 1628 } 1629 } 1630 1631 /** 1632 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 1633 * @priv: driver private structure 1634 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 1635 */ 1636 static void stmmac_mmc_setup(struct stmmac_priv *priv) 1637 { 1638 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 1639 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 1640 1641 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1642 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET; 1643 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET; 1644 } else { 1645 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET; 1646 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET; 1647 } 1648 1649 dwmac_mmc_intr_all_mask(priv->mmcaddr); 1650 1651 if (priv->dma_cap.rmon) { 1652 dwmac_mmc_ctrl(priv->mmcaddr, mode); 1653 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 1654 } else 1655 netdev_info(priv->dev, "No MAC Management Counters available\n"); 1656 } 1657 1658 /** 1659 * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors 1660 * @priv: driver private structure 1661 * Description: select the Enhanced/Alternate or Normal descriptors. 1662 * In case of Enhanced/Alternate, it checks if the extended descriptors are 1663 * supported by the HW capability register. 1664 */ 1665 static void stmmac_selec_desc_mode(struct stmmac_priv *priv) 1666 { 1667 if (priv->plat->enh_desc) { 1668 dev_info(priv->device, "Enhanced/Alternate descriptors\n"); 1669 1670 /* GMAC older than 3.50 has no extended descriptors */ 1671 if (priv->synopsys_id >= DWMAC_CORE_3_50) { 1672 dev_info(priv->device, "Enabled extended descriptors\n"); 1673 priv->extend_desc = 1; 1674 } else 1675 dev_warn(priv->device, "Extended descriptors not supported\n"); 1676 1677 priv->hw->desc = &enh_desc_ops; 1678 } else { 1679 dev_info(priv->device, "Normal descriptors\n"); 1680 priv->hw->desc = &ndesc_ops; 1681 } 1682 } 1683 1684 /** 1685 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 1686 * @priv: driver private structure 1687 * Description: 1688 * new GMAC chip generations have a new register to indicate the 1689 * presence of the optional feature/functions. 1690 * This can be also used to override the value passed through the 1691 * platform and necessary for old MAC10/100 and GMAC chips. 1692 */ 1693 static int stmmac_get_hw_features(struct stmmac_priv *priv) 1694 { 1695 u32 ret = 0; 1696 1697 if (priv->hw->dma->get_hw_feature) { 1698 priv->hw->dma->get_hw_feature(priv->ioaddr, 1699 &priv->dma_cap); 1700 ret = 1; 1701 } 1702 1703 return ret; 1704 } 1705 1706 /** 1707 * stmmac_check_ether_addr - check if the MAC addr is valid 1708 * @priv: driver private structure 1709 * Description: 1710 * it is to verify if the MAC address is valid, in case of failures it 1711 * generates a random MAC address 1712 */ 1713 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 1714 { 1715 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 1716 priv->hw->mac->get_umac_addr(priv->hw, 1717 priv->dev->dev_addr, 0); 1718 if (!is_valid_ether_addr(priv->dev->dev_addr)) 1719 eth_hw_addr_random(priv->dev); 1720 netdev_info(priv->dev, "device MAC address %pM\n", 1721 priv->dev->dev_addr); 1722 } 1723 } 1724 1725 /** 1726 * stmmac_init_dma_engine - DMA init. 1727 * @priv: driver private structure 1728 * Description: 1729 * It inits the DMA invoking the specific MAC/GMAC callback. 1730 * Some DMA parameters can be passed from the platform; 1731 * in case of these are not passed a default is kept for the MAC or GMAC. 1732 */ 1733 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 1734 { 1735 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1736 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1737 u32 dummy_dma_rx_phy = 0; 1738 u32 dummy_dma_tx_phy = 0; 1739 u32 chan = 0; 1740 int atds = 0; 1741 int ret = 0; 1742 1743 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 1744 dev_err(priv->device, "Invalid DMA configuration\n"); 1745 return -EINVAL; 1746 } 1747 1748 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 1749 atds = 1; 1750 1751 ret = priv->hw->dma->reset(priv->ioaddr); 1752 if (ret) { 1753 dev_err(priv->device, "Failed to reset the dma\n"); 1754 return ret; 1755 } 1756 1757 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1758 /* DMA Configuration */ 1759 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg, 1760 dummy_dma_tx_phy, dummy_dma_rx_phy, atds); 1761 1762 /* DMA RX Channel Configuration */ 1763 for (chan = 0; chan < rx_channels_count; chan++) { 1764 priv->hw->dma->init_rx_chan(priv->ioaddr, 1765 priv->plat->dma_cfg, 1766 priv->dma_rx_phy, chan); 1767 1768 priv->rx_tail_addr = priv->dma_rx_phy + 1769 (DMA_RX_SIZE * sizeof(struct dma_desc)); 1770 priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, 1771 priv->rx_tail_addr, 1772 chan); 1773 } 1774 1775 /* DMA TX Channel Configuration */ 1776 for (chan = 0; chan < tx_channels_count; chan++) { 1777 priv->hw->dma->init_chan(priv->ioaddr, 1778 priv->plat->dma_cfg, 1779 chan); 1780 1781 priv->hw->dma->init_tx_chan(priv->ioaddr, 1782 priv->plat->dma_cfg, 1783 priv->dma_tx_phy, chan); 1784 1785 priv->tx_tail_addr = priv->dma_tx_phy + 1786 (DMA_TX_SIZE * sizeof(struct dma_desc)); 1787 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, 1788 priv->tx_tail_addr, 1789 chan); 1790 } 1791 } else { 1792 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg, 1793 priv->dma_tx_phy, priv->dma_rx_phy, atds); 1794 } 1795 1796 if (priv->plat->axi && priv->hw->dma->axi) 1797 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi); 1798 1799 return ret; 1800 } 1801 1802 /** 1803 * stmmac_tx_timer - mitigation sw timer for tx. 1804 * @data: data pointer 1805 * Description: 1806 * This is the timer handler to directly invoke the stmmac_tx_clean. 1807 */ 1808 static void stmmac_tx_timer(unsigned long data) 1809 { 1810 struct stmmac_priv *priv = (struct stmmac_priv *)data; 1811 1812 stmmac_tx_clean(priv); 1813 } 1814 1815 /** 1816 * stmmac_init_tx_coalesce - init tx mitigation options. 1817 * @priv: driver private structure 1818 * Description: 1819 * This inits the transmit coalesce parameters: i.e. timer rate, 1820 * timer handler and default threshold used for enabling the 1821 * interrupt on completion bit. 1822 */ 1823 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv) 1824 { 1825 priv->tx_coal_frames = STMMAC_TX_FRAMES; 1826 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER; 1827 init_timer(&priv->txtimer); 1828 priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer); 1829 priv->txtimer.data = (unsigned long)priv; 1830 priv->txtimer.function = stmmac_tx_timer; 1831 add_timer(&priv->txtimer); 1832 } 1833 1834 static void stmmac_set_rings_length(struct stmmac_priv *priv) 1835 { 1836 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1837 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1838 u32 chan; 1839 1840 /* set TX ring length */ 1841 if (priv->hw->dma->set_tx_ring_len) { 1842 for (chan = 0; chan < tx_channels_count; chan++) 1843 priv->hw->dma->set_tx_ring_len(priv->ioaddr, 1844 (DMA_TX_SIZE - 1), chan); 1845 } 1846 1847 /* set RX ring length */ 1848 if (priv->hw->dma->set_rx_ring_len) { 1849 for (chan = 0; chan < rx_channels_count; chan++) 1850 priv->hw->dma->set_rx_ring_len(priv->ioaddr, 1851 (DMA_RX_SIZE - 1), chan); 1852 } 1853 } 1854 1855 /** 1856 * stmmac_set_tx_queue_weight - Set TX queue weight 1857 * @priv: driver private structure 1858 * Description: It is used for setting TX queues weight 1859 */ 1860 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 1861 { 1862 u32 tx_queues_count = priv->plat->tx_queues_to_use; 1863 u32 weight; 1864 u32 queue; 1865 1866 for (queue = 0; queue < tx_queues_count; queue++) { 1867 weight = priv->plat->tx_queues_cfg[queue].weight; 1868 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue); 1869 } 1870 } 1871 1872 /** 1873 * stmmac_configure_cbs - Configure CBS in TX queue 1874 * @priv: driver private structure 1875 * Description: It is used for configuring CBS in AVB TX queues 1876 */ 1877 static void stmmac_configure_cbs(struct stmmac_priv *priv) 1878 { 1879 u32 tx_queues_count = priv->plat->tx_queues_to_use; 1880 u32 mode_to_use; 1881 u32 queue; 1882 1883 /* queue 0 is reserved for legacy traffic */ 1884 for (queue = 1; queue < tx_queues_count; queue++) { 1885 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 1886 if (mode_to_use == MTL_QUEUE_DCB) 1887 continue; 1888 1889 priv->hw->mac->config_cbs(priv->hw, 1890 priv->plat->tx_queues_cfg[queue].send_slope, 1891 priv->plat->tx_queues_cfg[queue].idle_slope, 1892 priv->plat->tx_queues_cfg[queue].high_credit, 1893 priv->plat->tx_queues_cfg[queue].low_credit, 1894 queue); 1895 } 1896 } 1897 1898 /** 1899 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 1900 * @priv: driver private structure 1901 * Description: It is used for mapping RX queues to RX dma channels 1902 */ 1903 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 1904 { 1905 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1906 u32 queue; 1907 u32 chan; 1908 1909 for (queue = 0; queue < rx_queues_count; queue++) { 1910 chan = priv->plat->rx_queues_cfg[queue].chan; 1911 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan); 1912 } 1913 } 1914 1915 /** 1916 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 1917 * @priv: driver private structure 1918 * Description: It is used for configuring the RX Queue Priority 1919 */ 1920 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 1921 { 1922 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1923 u32 queue; 1924 u32 prio; 1925 1926 for (queue = 0; queue < rx_queues_count; queue++) { 1927 if (!priv->plat->rx_queues_cfg[queue].use_prio) 1928 continue; 1929 1930 prio = priv->plat->rx_queues_cfg[queue].prio; 1931 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue); 1932 } 1933 } 1934 1935 /** 1936 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 1937 * @priv: driver private structure 1938 * Description: It is used for configuring the TX Queue Priority 1939 */ 1940 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 1941 { 1942 u32 tx_queues_count = priv->plat->tx_queues_to_use; 1943 u32 queue; 1944 u32 prio; 1945 1946 for (queue = 0; queue < tx_queues_count; queue++) { 1947 if (!priv->plat->tx_queues_cfg[queue].use_prio) 1948 continue; 1949 1950 prio = priv->plat->tx_queues_cfg[queue].prio; 1951 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue); 1952 } 1953 } 1954 1955 /** 1956 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 1957 * @priv: driver private structure 1958 * Description: It is used for configuring the RX queue routing 1959 */ 1960 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 1961 { 1962 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1963 u32 queue; 1964 u8 packet; 1965 1966 for (queue = 0; queue < rx_queues_count; queue++) { 1967 /* no specific packet type routing specified for the queue */ 1968 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 1969 continue; 1970 1971 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 1972 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue); 1973 } 1974 } 1975 1976 /** 1977 * stmmac_mtl_configuration - Configure MTL 1978 * @priv: driver private structure 1979 * Description: It is used for configurring MTL 1980 */ 1981 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 1982 { 1983 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1984 u32 tx_queues_count = priv->plat->tx_queues_to_use; 1985 1986 if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight) 1987 stmmac_set_tx_queue_weight(priv); 1988 1989 /* Configure MTL RX algorithms */ 1990 if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms) 1991 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw, 1992 priv->plat->rx_sched_algorithm); 1993 1994 /* Configure MTL TX algorithms */ 1995 if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms) 1996 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw, 1997 priv->plat->tx_sched_algorithm); 1998 1999 /* Configure CBS in AVB TX queues */ 2000 if (tx_queues_count > 1 && priv->hw->mac->config_cbs) 2001 stmmac_configure_cbs(priv); 2002 2003 /* Map RX MTL to DMA channels */ 2004 if (priv->hw->mac->map_mtl_to_dma) 2005 stmmac_rx_queue_dma_chan_map(priv); 2006 2007 /* Enable MAC RX Queues */ 2008 if (priv->hw->mac->rx_queue_enable) 2009 stmmac_mac_enable_rx_queues(priv); 2010 2011 /* Set RX priorities */ 2012 if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio) 2013 stmmac_mac_config_rx_queues_prio(priv); 2014 2015 /* Set TX priorities */ 2016 if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio) 2017 stmmac_mac_config_tx_queues_prio(priv); 2018 2019 /* Set RX routing */ 2020 if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing) 2021 stmmac_mac_config_rx_queues_routing(priv); 2022 } 2023 2024 /** 2025 * stmmac_hw_setup - setup mac in a usable state. 2026 * @dev : pointer to the device structure. 2027 * Description: 2028 * this is the main function to setup the HW in a usable state because the 2029 * dma engine is reset, the core registers are configured (e.g. AXI, 2030 * Checksum features, timers). The DMA is ready to start receiving and 2031 * transmitting. 2032 * Return value: 2033 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2034 * file on failure. 2035 */ 2036 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) 2037 { 2038 struct stmmac_priv *priv = netdev_priv(dev); 2039 u32 rx_cnt = priv->plat->rx_queues_to_use; 2040 u32 tx_cnt = priv->plat->tx_queues_to_use; 2041 u32 chan; 2042 int ret; 2043 2044 /* DMA initialization and SW reset */ 2045 ret = stmmac_init_dma_engine(priv); 2046 if (ret < 0) { 2047 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 2048 __func__); 2049 return ret; 2050 } 2051 2052 /* Copy the MAC addr into the HW */ 2053 priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0); 2054 2055 /* PS and related bits will be programmed according to the speed */ 2056 if (priv->hw->pcs) { 2057 int speed = priv->plat->mac_port_sel_speed; 2058 2059 if ((speed == SPEED_10) || (speed == SPEED_100) || 2060 (speed == SPEED_1000)) { 2061 priv->hw->ps = speed; 2062 } else { 2063 dev_warn(priv->device, "invalid port speed\n"); 2064 priv->hw->ps = 0; 2065 } 2066 } 2067 2068 /* Initialize the MAC Core */ 2069 priv->hw->mac->core_init(priv->hw, dev->mtu); 2070 2071 /* Initialize MTL*/ 2072 if (priv->synopsys_id >= DWMAC_CORE_4_00) 2073 stmmac_mtl_configuration(priv); 2074 2075 ret = priv->hw->mac->rx_ipc(priv->hw); 2076 if (!ret) { 2077 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 2078 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 2079 priv->hw->rx_csum = 0; 2080 } 2081 2082 /* Enable the MAC Rx/Tx */ 2083 priv->hw->mac->set_mac(priv->ioaddr, true); 2084 2085 /* Set the HW DMA mode and the COE */ 2086 stmmac_dma_operation_mode(priv); 2087 2088 stmmac_mmc_setup(priv); 2089 2090 if (init_ptp) { 2091 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 2092 if (ret < 0) 2093 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret); 2094 2095 ret = stmmac_init_ptp(priv); 2096 if (ret == -EOPNOTSUPP) 2097 netdev_warn(priv->dev, "PTP not supported by HW\n"); 2098 else if (ret) 2099 netdev_warn(priv->dev, "PTP init failed\n"); 2100 } 2101 2102 #ifdef CONFIG_DEBUG_FS 2103 ret = stmmac_init_fs(dev); 2104 if (ret < 0) 2105 netdev_warn(priv->dev, "%s: failed debugFS registration\n", 2106 __func__); 2107 #endif 2108 /* Start the ball rolling... */ 2109 stmmac_start_all_dma(priv); 2110 2111 priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS; 2112 2113 if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) { 2114 priv->rx_riwt = MAX_DMA_RIWT; 2115 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt); 2116 } 2117 2118 if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane) 2119 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0); 2120 2121 /* set TX and RX rings length */ 2122 stmmac_set_rings_length(priv); 2123 2124 /* Enable TSO */ 2125 if (priv->tso) { 2126 for (chan = 0; chan < tx_cnt; chan++) 2127 priv->hw->dma->enable_tso(priv->ioaddr, 1, chan); 2128 } 2129 2130 return 0; 2131 } 2132 2133 static void stmmac_hw_teardown(struct net_device *dev) 2134 { 2135 struct stmmac_priv *priv = netdev_priv(dev); 2136 2137 clk_disable_unprepare(priv->plat->clk_ptp_ref); 2138 } 2139 2140 /** 2141 * stmmac_open - open entry point of the driver 2142 * @dev : pointer to the device structure. 2143 * Description: 2144 * This function is the open entry point of the driver. 2145 * Return value: 2146 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2147 * file on failure. 2148 */ 2149 static int stmmac_open(struct net_device *dev) 2150 { 2151 struct stmmac_priv *priv = netdev_priv(dev); 2152 int ret; 2153 2154 stmmac_check_ether_addr(priv); 2155 2156 if (priv->hw->pcs != STMMAC_PCS_RGMII && 2157 priv->hw->pcs != STMMAC_PCS_TBI && 2158 priv->hw->pcs != STMMAC_PCS_RTBI) { 2159 ret = stmmac_init_phy(dev); 2160 if (ret) { 2161 netdev_err(priv->dev, 2162 "%s: Cannot attach to PHY (error: %d)\n", 2163 __func__, ret); 2164 return ret; 2165 } 2166 } 2167 2168 /* Extra statistics */ 2169 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); 2170 priv->xstats.threshold = tc; 2171 2172 priv->dma_buf_sz = STMMAC_ALIGN(buf_sz); 2173 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 2174 2175 ret = alloc_dma_desc_resources(priv); 2176 if (ret < 0) { 2177 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 2178 __func__); 2179 goto dma_desc_error; 2180 } 2181 2182 ret = init_dma_desc_rings(dev, GFP_KERNEL); 2183 if (ret < 0) { 2184 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 2185 __func__); 2186 goto init_error; 2187 } 2188 2189 ret = stmmac_hw_setup(dev, true); 2190 if (ret < 0) { 2191 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 2192 goto init_error; 2193 } 2194 2195 stmmac_init_tx_coalesce(priv); 2196 2197 if (dev->phydev) 2198 phy_start(dev->phydev); 2199 2200 /* Request the IRQ lines */ 2201 ret = request_irq(dev->irq, stmmac_interrupt, 2202 IRQF_SHARED, dev->name, dev); 2203 if (unlikely(ret < 0)) { 2204 netdev_err(priv->dev, 2205 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 2206 __func__, dev->irq, ret); 2207 goto irq_error; 2208 } 2209 2210 /* Request the Wake IRQ in case of another line is used for WoL */ 2211 if (priv->wol_irq != dev->irq) { 2212 ret = request_irq(priv->wol_irq, stmmac_interrupt, 2213 IRQF_SHARED, dev->name, dev); 2214 if (unlikely(ret < 0)) { 2215 netdev_err(priv->dev, 2216 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 2217 __func__, priv->wol_irq, ret); 2218 goto wolirq_error; 2219 } 2220 } 2221 2222 /* Request the IRQ lines */ 2223 if (priv->lpi_irq > 0) { 2224 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED, 2225 dev->name, dev); 2226 if (unlikely(ret < 0)) { 2227 netdev_err(priv->dev, 2228 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 2229 __func__, priv->lpi_irq, ret); 2230 goto lpiirq_error; 2231 } 2232 } 2233 2234 napi_enable(&priv->napi); 2235 netif_start_queue(dev); 2236 2237 return 0; 2238 2239 lpiirq_error: 2240 if (priv->wol_irq != dev->irq) 2241 free_irq(priv->wol_irq, dev); 2242 wolirq_error: 2243 free_irq(dev->irq, dev); 2244 irq_error: 2245 if (dev->phydev) 2246 phy_stop(dev->phydev); 2247 2248 del_timer_sync(&priv->txtimer); 2249 stmmac_hw_teardown(dev); 2250 init_error: 2251 free_dma_desc_resources(priv); 2252 dma_desc_error: 2253 if (dev->phydev) 2254 phy_disconnect(dev->phydev); 2255 2256 return ret; 2257 } 2258 2259 /** 2260 * stmmac_release - close entry point of the driver 2261 * @dev : device pointer. 2262 * Description: 2263 * This is the stop entry point of the driver. 2264 */ 2265 static int stmmac_release(struct net_device *dev) 2266 { 2267 struct stmmac_priv *priv = netdev_priv(dev); 2268 2269 if (priv->eee_enabled) 2270 del_timer_sync(&priv->eee_ctrl_timer); 2271 2272 /* Stop and disconnect the PHY */ 2273 if (dev->phydev) { 2274 phy_stop(dev->phydev); 2275 phy_disconnect(dev->phydev); 2276 } 2277 2278 netif_stop_queue(dev); 2279 2280 napi_disable(&priv->napi); 2281 2282 del_timer_sync(&priv->txtimer); 2283 2284 /* Free the IRQ lines */ 2285 free_irq(dev->irq, dev); 2286 if (priv->wol_irq != dev->irq) 2287 free_irq(priv->wol_irq, dev); 2288 if (priv->lpi_irq > 0) 2289 free_irq(priv->lpi_irq, dev); 2290 2291 /* Stop TX/RX DMA and clear the descriptors */ 2292 stmmac_stop_all_dma(priv); 2293 2294 /* Release and free the Rx/Tx resources */ 2295 free_dma_desc_resources(priv); 2296 2297 /* Disable the MAC Rx/Tx */ 2298 priv->hw->mac->set_mac(priv->ioaddr, false); 2299 2300 netif_carrier_off(dev); 2301 2302 #ifdef CONFIG_DEBUG_FS 2303 stmmac_exit_fs(dev); 2304 #endif 2305 2306 stmmac_release_ptp(priv); 2307 2308 return 0; 2309 } 2310 2311 /** 2312 * stmmac_tso_allocator - close entry point of the driver 2313 * @priv: driver private structure 2314 * @des: buffer start address 2315 * @total_len: total length to fill in descriptors 2316 * @last_segmant: condition for the last descriptor 2317 * Description: 2318 * This function fills descriptor and request new descriptors according to 2319 * buffer length to fill 2320 */ 2321 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des, 2322 int total_len, bool last_segment) 2323 { 2324 struct dma_desc *desc; 2325 int tmp_len; 2326 u32 buff_size; 2327 2328 tmp_len = total_len; 2329 2330 while (tmp_len > 0) { 2331 priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE); 2332 desc = priv->dma_tx + priv->cur_tx; 2333 2334 desc->des0 = cpu_to_le32(des + (total_len - tmp_len)); 2335 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 2336 TSO_MAX_BUFF_SIZE : tmp_len; 2337 2338 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size, 2339 0, 1, 2340 (last_segment) && (buff_size < TSO_MAX_BUFF_SIZE), 2341 0, 0); 2342 2343 tmp_len -= TSO_MAX_BUFF_SIZE; 2344 } 2345 } 2346 2347 /** 2348 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 2349 * @skb : the socket buffer 2350 * @dev : device pointer 2351 * Description: this is the transmit function that is called on TSO frames 2352 * (support available on GMAC4 and newer chips). 2353 * Diagram below show the ring programming in case of TSO frames: 2354 * 2355 * First Descriptor 2356 * -------- 2357 * | DES0 |---> buffer1 = L2/L3/L4 header 2358 * | DES1 |---> TCP Payload (can continue on next descr...) 2359 * | DES2 |---> buffer 1 and 2 len 2360 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 2361 * -------- 2362 * | 2363 * ... 2364 * | 2365 * -------- 2366 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 2367 * | DES1 | --| 2368 * | DES2 | --> buffer 1 and 2 len 2369 * | DES3 | 2370 * -------- 2371 * 2372 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 2373 */ 2374 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 2375 { 2376 u32 pay_len, mss; 2377 int tmp_pay_len = 0; 2378 struct stmmac_priv *priv = netdev_priv(dev); 2379 int nfrags = skb_shinfo(skb)->nr_frags; 2380 unsigned int first_entry, des; 2381 struct dma_desc *desc, *first, *mss_desc = NULL; 2382 u8 proto_hdr_len; 2383 int i; 2384 2385 /* Compute header lengths */ 2386 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2387 2388 /* Desc availability based on threshold should be enough safe */ 2389 if (unlikely(stmmac_tx_avail(priv) < 2390 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 2391 if (!netif_queue_stopped(dev)) { 2392 netif_stop_queue(dev); 2393 /* This is a hard error, log it. */ 2394 netdev_err(priv->dev, 2395 "%s: Tx Ring full when queue awake\n", 2396 __func__); 2397 } 2398 return NETDEV_TX_BUSY; 2399 } 2400 2401 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 2402 2403 mss = skb_shinfo(skb)->gso_size; 2404 2405 /* set new MSS value if needed */ 2406 if (mss != priv->mss) { 2407 mss_desc = priv->dma_tx + priv->cur_tx; 2408 priv->hw->desc->set_mss(mss_desc, mss); 2409 priv->mss = mss; 2410 priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE); 2411 } 2412 2413 if (netif_msg_tx_queued(priv)) { 2414 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 2415 __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss); 2416 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 2417 skb->data_len); 2418 } 2419 2420 first_entry = priv->cur_tx; 2421 2422 desc = priv->dma_tx + first_entry; 2423 first = desc; 2424 2425 /* first descriptor: fill Headers on Buf1 */ 2426 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 2427 DMA_TO_DEVICE); 2428 if (dma_mapping_error(priv->device, des)) 2429 goto dma_map_err; 2430 2431 priv->tx_skbuff_dma[first_entry].buf = des; 2432 priv->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 2433 priv->tx_skbuff[first_entry] = skb; 2434 2435 first->des0 = cpu_to_le32(des); 2436 2437 /* Fill start of payload in buff2 of first descriptor */ 2438 if (pay_len) 2439 first->des1 = cpu_to_le32(des + proto_hdr_len); 2440 2441 /* If needed take extra descriptors to fill the remaining payload */ 2442 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 2443 2444 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0)); 2445 2446 /* Prepare fragments */ 2447 for (i = 0; i < nfrags; i++) { 2448 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2449 2450 des = skb_frag_dma_map(priv->device, frag, 0, 2451 skb_frag_size(frag), 2452 DMA_TO_DEVICE); 2453 if (dma_mapping_error(priv->device, des)) 2454 goto dma_map_err; 2455 2456 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 2457 (i == nfrags - 1)); 2458 2459 priv->tx_skbuff_dma[priv->cur_tx].buf = des; 2460 priv->tx_skbuff_dma[priv->cur_tx].len = skb_frag_size(frag); 2461 priv->tx_skbuff[priv->cur_tx] = NULL; 2462 priv->tx_skbuff_dma[priv->cur_tx].map_as_page = true; 2463 } 2464 2465 priv->tx_skbuff_dma[priv->cur_tx].last_segment = true; 2466 2467 priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE); 2468 2469 if (unlikely(stmmac_tx_avail(priv) <= (MAX_SKB_FRAGS + 1))) { 2470 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 2471 __func__); 2472 netif_stop_queue(dev); 2473 } 2474 2475 dev->stats.tx_bytes += skb->len; 2476 priv->xstats.tx_tso_frames++; 2477 priv->xstats.tx_tso_nfrags += nfrags; 2478 2479 /* Manage tx mitigation */ 2480 priv->tx_count_frames += nfrags + 1; 2481 if (likely(priv->tx_coal_frames > priv->tx_count_frames)) { 2482 mod_timer(&priv->txtimer, 2483 STMMAC_COAL_TIMER(priv->tx_coal_timer)); 2484 } else { 2485 priv->tx_count_frames = 0; 2486 priv->hw->desc->set_tx_ic(desc); 2487 priv->xstats.tx_set_ic_bit++; 2488 } 2489 2490 if (!priv->hwts_tx_en) 2491 skb_tx_timestamp(skb); 2492 2493 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 2494 priv->hwts_tx_en)) { 2495 /* declare that device is doing timestamping */ 2496 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2497 priv->hw->desc->enable_tx_timestamp(first); 2498 } 2499 2500 /* Complete the first descriptor before granting the DMA */ 2501 priv->hw->desc->prepare_tso_tx_desc(first, 1, 2502 proto_hdr_len, 2503 pay_len, 2504 1, priv->tx_skbuff_dma[first_entry].last_segment, 2505 tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len)); 2506 2507 /* If context desc is used to change MSS */ 2508 if (mss_desc) 2509 priv->hw->desc->set_tx_owner(mss_desc); 2510 2511 /* The own bit must be the latest setting done when prepare the 2512 * descriptor and then barrier is needed to make sure that 2513 * all is coherent before granting the DMA engine. 2514 */ 2515 dma_wmb(); 2516 2517 if (netif_msg_pktdata(priv)) { 2518 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 2519 __func__, priv->cur_tx, priv->dirty_tx, first_entry, 2520 priv->cur_tx, first, nfrags); 2521 2522 priv->hw->desc->display_ring((void *)priv->dma_tx, DMA_TX_SIZE, 2523 0); 2524 2525 pr_info(">>> frame to be transmitted: "); 2526 print_pkt(skb->data, skb_headlen(skb)); 2527 } 2528 2529 netdev_sent_queue(dev, skb->len); 2530 2531 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, priv->tx_tail_addr, 2532 STMMAC_CHAN0); 2533 2534 return NETDEV_TX_OK; 2535 2536 dma_map_err: 2537 dev_err(priv->device, "Tx dma map failed\n"); 2538 dev_kfree_skb(skb); 2539 priv->dev->stats.tx_dropped++; 2540 return NETDEV_TX_OK; 2541 } 2542 2543 /** 2544 * stmmac_xmit - Tx entry point of the driver 2545 * @skb : the socket buffer 2546 * @dev : device pointer 2547 * Description : this is the tx entry point of the driver. 2548 * It programs the chain or the ring and supports oversized frames 2549 * and SG feature. 2550 */ 2551 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 2552 { 2553 struct stmmac_priv *priv = netdev_priv(dev); 2554 unsigned int nopaged_len = skb_headlen(skb); 2555 int i, csum_insertion = 0, is_jumbo = 0; 2556 int nfrags = skb_shinfo(skb)->nr_frags; 2557 unsigned int entry, first_entry; 2558 struct dma_desc *desc, *first; 2559 unsigned int enh_desc; 2560 unsigned int des; 2561 2562 /* Manage oversized TCP frames for GMAC4 device */ 2563 if (skb_is_gso(skb) && priv->tso) { 2564 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 2565 return stmmac_tso_xmit(skb, dev); 2566 } 2567 2568 if (unlikely(stmmac_tx_avail(priv) < nfrags + 1)) { 2569 if (!netif_queue_stopped(dev)) { 2570 netif_stop_queue(dev); 2571 /* This is a hard error, log it. */ 2572 netdev_err(priv->dev, 2573 "%s: Tx Ring full when queue awake\n", 2574 __func__); 2575 } 2576 return NETDEV_TX_BUSY; 2577 } 2578 2579 if (priv->tx_path_in_lpi_mode) 2580 stmmac_disable_eee_mode(priv); 2581 2582 entry = priv->cur_tx; 2583 first_entry = entry; 2584 2585 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 2586 2587 if (likely(priv->extend_desc)) 2588 desc = (struct dma_desc *)(priv->dma_etx + entry); 2589 else 2590 desc = priv->dma_tx + entry; 2591 2592 first = desc; 2593 2594 priv->tx_skbuff[first_entry] = skb; 2595 2596 enh_desc = priv->plat->enh_desc; 2597 /* To program the descriptors according to the size of the frame */ 2598 if (enh_desc) 2599 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc); 2600 2601 if (unlikely(is_jumbo) && likely(priv->synopsys_id < 2602 DWMAC_CORE_4_00)) { 2603 entry = priv->hw->mode->jumbo_frm(priv, skb, csum_insertion); 2604 if (unlikely(entry < 0)) 2605 goto dma_map_err; 2606 } 2607 2608 for (i = 0; i < nfrags; i++) { 2609 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2610 int len = skb_frag_size(frag); 2611 bool last_segment = (i == (nfrags - 1)); 2612 2613 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 2614 2615 if (likely(priv->extend_desc)) 2616 desc = (struct dma_desc *)(priv->dma_etx + entry); 2617 else 2618 desc = priv->dma_tx + entry; 2619 2620 des = skb_frag_dma_map(priv->device, frag, 0, len, 2621 DMA_TO_DEVICE); 2622 if (dma_mapping_error(priv->device, des)) 2623 goto dma_map_err; /* should reuse desc w/o issues */ 2624 2625 priv->tx_skbuff[entry] = NULL; 2626 2627 priv->tx_skbuff_dma[entry].buf = des; 2628 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 2629 desc->des0 = cpu_to_le32(des); 2630 else 2631 desc->des2 = cpu_to_le32(des); 2632 2633 priv->tx_skbuff_dma[entry].map_as_page = true; 2634 priv->tx_skbuff_dma[entry].len = len; 2635 priv->tx_skbuff_dma[entry].last_segment = last_segment; 2636 2637 /* Prepare the descriptor and set the own bit too */ 2638 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion, 2639 priv->mode, 1, last_segment); 2640 } 2641 2642 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 2643 2644 priv->cur_tx = entry; 2645 2646 if (netif_msg_pktdata(priv)) { 2647 void *tx_head; 2648 2649 netdev_dbg(priv->dev, 2650 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 2651 __func__, priv->cur_tx, priv->dirty_tx, first_entry, 2652 entry, first, nfrags); 2653 2654 if (priv->extend_desc) 2655 tx_head = (void *)priv->dma_etx; 2656 else 2657 tx_head = (void *)priv->dma_tx; 2658 2659 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false); 2660 2661 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 2662 print_pkt(skb->data, skb->len); 2663 } 2664 2665 if (unlikely(stmmac_tx_avail(priv) <= (MAX_SKB_FRAGS + 1))) { 2666 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 2667 __func__); 2668 netif_stop_queue(dev); 2669 } 2670 2671 dev->stats.tx_bytes += skb->len; 2672 2673 /* According to the coalesce parameter the IC bit for the latest 2674 * segment is reset and the timer re-started to clean the tx status. 2675 * This approach takes care about the fragments: desc is the first 2676 * element in case of no SG. 2677 */ 2678 priv->tx_count_frames += nfrags + 1; 2679 if (likely(priv->tx_coal_frames > priv->tx_count_frames)) { 2680 mod_timer(&priv->txtimer, 2681 STMMAC_COAL_TIMER(priv->tx_coal_timer)); 2682 } else { 2683 priv->tx_count_frames = 0; 2684 priv->hw->desc->set_tx_ic(desc); 2685 priv->xstats.tx_set_ic_bit++; 2686 } 2687 2688 if (!priv->hwts_tx_en) 2689 skb_tx_timestamp(skb); 2690 2691 /* Ready to fill the first descriptor and set the OWN bit w/o any 2692 * problems because all the descriptors are actually ready to be 2693 * passed to the DMA engine. 2694 */ 2695 if (likely(!is_jumbo)) { 2696 bool last_segment = (nfrags == 0); 2697 2698 des = dma_map_single(priv->device, skb->data, 2699 nopaged_len, DMA_TO_DEVICE); 2700 if (dma_mapping_error(priv->device, des)) 2701 goto dma_map_err; 2702 2703 priv->tx_skbuff_dma[first_entry].buf = des; 2704 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 2705 first->des0 = cpu_to_le32(des); 2706 else 2707 first->des2 = cpu_to_le32(des); 2708 2709 priv->tx_skbuff_dma[first_entry].len = nopaged_len; 2710 priv->tx_skbuff_dma[first_entry].last_segment = last_segment; 2711 2712 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 2713 priv->hwts_tx_en)) { 2714 /* declare that device is doing timestamping */ 2715 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2716 priv->hw->desc->enable_tx_timestamp(first); 2717 } 2718 2719 /* Prepare the first descriptor setting the OWN bit too */ 2720 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len, 2721 csum_insertion, priv->mode, 1, 2722 last_segment); 2723 2724 /* The own bit must be the latest setting done when prepare the 2725 * descriptor and then barrier is needed to make sure that 2726 * all is coherent before granting the DMA engine. 2727 */ 2728 dma_wmb(); 2729 } 2730 2731 netdev_sent_queue(dev, skb->len); 2732 2733 if (priv->synopsys_id < DWMAC_CORE_4_00) 2734 priv->hw->dma->enable_dma_transmission(priv->ioaddr); 2735 else 2736 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, priv->tx_tail_addr, 2737 STMMAC_CHAN0); 2738 2739 return NETDEV_TX_OK; 2740 2741 dma_map_err: 2742 netdev_err(priv->dev, "Tx DMA map failed\n"); 2743 dev_kfree_skb(skb); 2744 priv->dev->stats.tx_dropped++; 2745 return NETDEV_TX_OK; 2746 } 2747 2748 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 2749 { 2750 struct ethhdr *ehdr; 2751 u16 vlanid; 2752 2753 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 2754 NETIF_F_HW_VLAN_CTAG_RX && 2755 !__vlan_get_tag(skb, &vlanid)) { 2756 /* pop the vlan tag */ 2757 ehdr = (struct ethhdr *)skb->data; 2758 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2); 2759 skb_pull(skb, VLAN_HLEN); 2760 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid); 2761 } 2762 } 2763 2764 2765 static inline int stmmac_rx_threshold_count(struct stmmac_priv *priv) 2766 { 2767 if (priv->rx_zeroc_thresh < STMMAC_RX_THRESH) 2768 return 0; 2769 2770 return 1; 2771 } 2772 2773 /** 2774 * stmmac_rx_refill - refill used skb preallocated buffers 2775 * @priv: driver private structure 2776 * Description : this is to reallocate the skb for the reception process 2777 * that is based on zero-copy. 2778 */ 2779 static inline void stmmac_rx_refill(struct stmmac_priv *priv) 2780 { 2781 int bfsize = priv->dma_buf_sz; 2782 unsigned int entry = priv->dirty_rx; 2783 int dirty = stmmac_rx_dirty(priv); 2784 2785 while (dirty-- > 0) { 2786 struct dma_desc *p; 2787 2788 if (priv->extend_desc) 2789 p = (struct dma_desc *)(priv->dma_erx + entry); 2790 else 2791 p = priv->dma_rx + entry; 2792 2793 if (likely(priv->rx_skbuff[entry] == NULL)) { 2794 struct sk_buff *skb; 2795 2796 skb = netdev_alloc_skb_ip_align(priv->dev, bfsize); 2797 if (unlikely(!skb)) { 2798 /* so for a while no zero-copy! */ 2799 priv->rx_zeroc_thresh = STMMAC_RX_THRESH; 2800 if (unlikely(net_ratelimit())) 2801 dev_err(priv->device, 2802 "fail to alloc skb entry %d\n", 2803 entry); 2804 break; 2805 } 2806 2807 priv->rx_skbuff[entry] = skb; 2808 priv->rx_skbuff_dma[entry] = 2809 dma_map_single(priv->device, skb->data, bfsize, 2810 DMA_FROM_DEVICE); 2811 if (dma_mapping_error(priv->device, 2812 priv->rx_skbuff_dma[entry])) { 2813 netdev_err(priv->dev, "Rx DMA map failed\n"); 2814 dev_kfree_skb(skb); 2815 break; 2816 } 2817 2818 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) { 2819 p->des0 = cpu_to_le32(priv->rx_skbuff_dma[entry]); 2820 p->des1 = 0; 2821 } else { 2822 p->des2 = cpu_to_le32(priv->rx_skbuff_dma[entry]); 2823 } 2824 if (priv->hw->mode->refill_desc3) 2825 priv->hw->mode->refill_desc3(priv, p); 2826 2827 if (priv->rx_zeroc_thresh > 0) 2828 priv->rx_zeroc_thresh--; 2829 2830 netif_dbg(priv, rx_status, priv->dev, 2831 "refill entry #%d\n", entry); 2832 } 2833 dma_wmb(); 2834 2835 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 2836 priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0); 2837 else 2838 priv->hw->desc->set_rx_owner(p); 2839 2840 dma_wmb(); 2841 2842 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE); 2843 } 2844 priv->dirty_rx = entry; 2845 } 2846 2847 /** 2848 * stmmac_rx - manage the receive process 2849 * @priv: driver private structure 2850 * @limit: napi bugget. 2851 * Description : this the function called by the napi poll method. 2852 * It gets all the frames inside the ring. 2853 */ 2854 static int stmmac_rx(struct stmmac_priv *priv, int limit) 2855 { 2856 unsigned int entry = priv->cur_rx; 2857 unsigned int next_entry; 2858 unsigned int count = 0; 2859 int coe = priv->hw->rx_csum; 2860 2861 if (netif_msg_rx_status(priv)) { 2862 void *rx_head; 2863 2864 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 2865 if (priv->extend_desc) 2866 rx_head = (void *)priv->dma_erx; 2867 else 2868 rx_head = (void *)priv->dma_rx; 2869 2870 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true); 2871 } 2872 while (count < limit) { 2873 int status; 2874 struct dma_desc *p; 2875 struct dma_desc *np; 2876 2877 if (priv->extend_desc) 2878 p = (struct dma_desc *)(priv->dma_erx + entry); 2879 else 2880 p = priv->dma_rx + entry; 2881 2882 /* read the status of the incoming frame */ 2883 status = priv->hw->desc->rx_status(&priv->dev->stats, 2884 &priv->xstats, p); 2885 /* check if managed by the DMA otherwise go ahead */ 2886 if (unlikely(status & dma_own)) 2887 break; 2888 2889 count++; 2890 2891 priv->cur_rx = STMMAC_GET_ENTRY(priv->cur_rx, DMA_RX_SIZE); 2892 next_entry = priv->cur_rx; 2893 2894 if (priv->extend_desc) 2895 np = (struct dma_desc *)(priv->dma_erx + next_entry); 2896 else 2897 np = priv->dma_rx + next_entry; 2898 2899 prefetch(np); 2900 2901 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status)) 2902 priv->hw->desc->rx_extended_status(&priv->dev->stats, 2903 &priv->xstats, 2904 priv->dma_erx + 2905 entry); 2906 if (unlikely(status == discard_frame)) { 2907 priv->dev->stats.rx_errors++; 2908 if (priv->hwts_rx_en && !priv->extend_desc) { 2909 /* DESC2 & DESC3 will be overwritten by device 2910 * with timestamp value, hence reinitialize 2911 * them in stmmac_rx_refill() function so that 2912 * device can reuse it. 2913 */ 2914 priv->rx_skbuff[entry] = NULL; 2915 dma_unmap_single(priv->device, 2916 priv->rx_skbuff_dma[entry], 2917 priv->dma_buf_sz, 2918 DMA_FROM_DEVICE); 2919 } 2920 } else { 2921 struct sk_buff *skb; 2922 int frame_len; 2923 unsigned int des; 2924 2925 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 2926 des = le32_to_cpu(p->des0); 2927 else 2928 des = le32_to_cpu(p->des2); 2929 2930 frame_len = priv->hw->desc->get_rx_frame_len(p, coe); 2931 2932 /* If frame length is greater than skb buffer size 2933 * (preallocated during init) then the packet is 2934 * ignored 2935 */ 2936 if (frame_len > priv->dma_buf_sz) { 2937 netdev_err(priv->dev, 2938 "len %d larger than size (%d)\n", 2939 frame_len, priv->dma_buf_sz); 2940 priv->dev->stats.rx_length_errors++; 2941 break; 2942 } 2943 2944 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 2945 * Type frames (LLC/LLC-SNAP) 2946 */ 2947 if (unlikely(status != llc_snap)) 2948 frame_len -= ETH_FCS_LEN; 2949 2950 if (netif_msg_rx_status(priv)) { 2951 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n", 2952 p, entry, des); 2953 if (frame_len > ETH_FRAME_LEN) 2954 netdev_dbg(priv->dev, "frame size %d, COE: %d\n", 2955 frame_len, status); 2956 } 2957 2958 /* The zero-copy is always used for all the sizes 2959 * in case of GMAC4 because it needs 2960 * to refill the used descriptors, always. 2961 */ 2962 if (unlikely(!priv->plat->has_gmac4 && 2963 ((frame_len < priv->rx_copybreak) || 2964 stmmac_rx_threshold_count(priv)))) { 2965 skb = netdev_alloc_skb_ip_align(priv->dev, 2966 frame_len); 2967 if (unlikely(!skb)) { 2968 if (net_ratelimit()) 2969 dev_warn(priv->device, 2970 "packet dropped\n"); 2971 priv->dev->stats.rx_dropped++; 2972 break; 2973 } 2974 2975 dma_sync_single_for_cpu(priv->device, 2976 priv->rx_skbuff_dma 2977 [entry], frame_len, 2978 DMA_FROM_DEVICE); 2979 skb_copy_to_linear_data(skb, 2980 priv-> 2981 rx_skbuff[entry]->data, 2982 frame_len); 2983 2984 skb_put(skb, frame_len); 2985 dma_sync_single_for_device(priv->device, 2986 priv->rx_skbuff_dma 2987 [entry], frame_len, 2988 DMA_FROM_DEVICE); 2989 } else { 2990 skb = priv->rx_skbuff[entry]; 2991 if (unlikely(!skb)) { 2992 netdev_err(priv->dev, 2993 "%s: Inconsistent Rx chain\n", 2994 priv->dev->name); 2995 priv->dev->stats.rx_dropped++; 2996 break; 2997 } 2998 prefetch(skb->data - NET_IP_ALIGN); 2999 priv->rx_skbuff[entry] = NULL; 3000 priv->rx_zeroc_thresh++; 3001 3002 skb_put(skb, frame_len); 3003 dma_unmap_single(priv->device, 3004 priv->rx_skbuff_dma[entry], 3005 priv->dma_buf_sz, 3006 DMA_FROM_DEVICE); 3007 } 3008 3009 if (netif_msg_pktdata(priv)) { 3010 netdev_dbg(priv->dev, "frame received (%dbytes)", 3011 frame_len); 3012 print_pkt(skb->data, frame_len); 3013 } 3014 3015 stmmac_get_rx_hwtstamp(priv, p, np, skb); 3016 3017 stmmac_rx_vlan(priv->dev, skb); 3018 3019 skb->protocol = eth_type_trans(skb, priv->dev); 3020 3021 if (unlikely(!coe)) 3022 skb_checksum_none_assert(skb); 3023 else 3024 skb->ip_summed = CHECKSUM_UNNECESSARY; 3025 3026 napi_gro_receive(&priv->napi, skb); 3027 3028 priv->dev->stats.rx_packets++; 3029 priv->dev->stats.rx_bytes += frame_len; 3030 } 3031 entry = next_entry; 3032 } 3033 3034 stmmac_rx_refill(priv); 3035 3036 priv->xstats.rx_pkt_n += count; 3037 3038 return count; 3039 } 3040 3041 /** 3042 * stmmac_poll - stmmac poll method (NAPI) 3043 * @napi : pointer to the napi structure. 3044 * @budget : maximum number of packets that the current CPU can receive from 3045 * all interfaces. 3046 * Description : 3047 * To look at the incoming frames and clear the tx resources. 3048 */ 3049 static int stmmac_poll(struct napi_struct *napi, int budget) 3050 { 3051 struct stmmac_priv *priv = container_of(napi, struct stmmac_priv, napi); 3052 int work_done = 0; 3053 u32 chan = STMMAC_CHAN0; 3054 3055 priv->xstats.napi_poll++; 3056 stmmac_tx_clean(priv); 3057 3058 work_done = stmmac_rx(priv, budget); 3059 if (work_done < budget) { 3060 napi_complete_done(napi, work_done); 3061 stmmac_enable_dma_irq(priv, chan); 3062 } 3063 return work_done; 3064 } 3065 3066 /** 3067 * stmmac_tx_timeout 3068 * @dev : Pointer to net device structure 3069 * Description: this function is called when a packet transmission fails to 3070 * complete within a reasonable time. The driver will mark the error in the 3071 * netdev structure and arrange for the device to be reset to a sane state 3072 * in order to transmit a new packet. 3073 */ 3074 static void stmmac_tx_timeout(struct net_device *dev) 3075 { 3076 struct stmmac_priv *priv = netdev_priv(dev); 3077 u32 chan = STMMAC_CHAN0; 3078 3079 /* Clear Tx resources and restart transmitting again */ 3080 stmmac_tx_err(priv, chan); 3081 } 3082 3083 /** 3084 * stmmac_set_rx_mode - entry point for multicast addressing 3085 * @dev : pointer to the device structure 3086 * Description: 3087 * This function is a driver entry point which gets called by the kernel 3088 * whenever multicast addresses must be enabled/disabled. 3089 * Return value: 3090 * void. 3091 */ 3092 static void stmmac_set_rx_mode(struct net_device *dev) 3093 { 3094 struct stmmac_priv *priv = netdev_priv(dev); 3095 3096 priv->hw->mac->set_filter(priv->hw, dev); 3097 } 3098 3099 /** 3100 * stmmac_change_mtu - entry point to change MTU size for the device. 3101 * @dev : device pointer. 3102 * @new_mtu : the new MTU size for the device. 3103 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 3104 * to drive packet transmission. Ethernet has an MTU of 1500 octets 3105 * (ETH_DATA_LEN). This value can be changed with ifconfig. 3106 * Return value: 3107 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3108 * file on failure. 3109 */ 3110 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 3111 { 3112 struct stmmac_priv *priv = netdev_priv(dev); 3113 3114 if (netif_running(dev)) { 3115 netdev_err(priv->dev, "must be stopped to change its MTU\n"); 3116 return -EBUSY; 3117 } 3118 3119 dev->mtu = new_mtu; 3120 3121 netdev_update_features(dev); 3122 3123 return 0; 3124 } 3125 3126 static netdev_features_t stmmac_fix_features(struct net_device *dev, 3127 netdev_features_t features) 3128 { 3129 struct stmmac_priv *priv = netdev_priv(dev); 3130 3131 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 3132 features &= ~NETIF_F_RXCSUM; 3133 3134 if (!priv->plat->tx_coe) 3135 features &= ~NETIF_F_CSUM_MASK; 3136 3137 /* Some GMAC devices have a bugged Jumbo frame support that 3138 * needs to have the Tx COE disabled for oversized frames 3139 * (due to limited buffer sizes). In this case we disable 3140 * the TX csum insertion in the TDES and not use SF. 3141 */ 3142 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 3143 features &= ~NETIF_F_CSUM_MASK; 3144 3145 /* Disable tso if asked by ethtool */ 3146 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 3147 if (features & NETIF_F_TSO) 3148 priv->tso = true; 3149 else 3150 priv->tso = false; 3151 } 3152 3153 return features; 3154 } 3155 3156 static int stmmac_set_features(struct net_device *netdev, 3157 netdev_features_t features) 3158 { 3159 struct stmmac_priv *priv = netdev_priv(netdev); 3160 3161 /* Keep the COE Type in case of csum is supporting */ 3162 if (features & NETIF_F_RXCSUM) 3163 priv->hw->rx_csum = priv->plat->rx_coe; 3164 else 3165 priv->hw->rx_csum = 0; 3166 /* No check needed because rx_coe has been set before and it will be 3167 * fixed in case of issue. 3168 */ 3169 priv->hw->mac->rx_ipc(priv->hw); 3170 3171 return 0; 3172 } 3173 3174 /** 3175 * stmmac_interrupt - main ISR 3176 * @irq: interrupt number. 3177 * @dev_id: to pass the net device pointer. 3178 * Description: this is the main driver interrupt service routine. 3179 * It can call: 3180 * o DMA service routine (to manage incoming frame reception and transmission 3181 * status) 3182 * o Core interrupts to manage: remote wake-up, management counter, LPI 3183 * interrupts. 3184 */ 3185 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 3186 { 3187 struct net_device *dev = (struct net_device *)dev_id; 3188 struct stmmac_priv *priv = netdev_priv(dev); 3189 u32 rx_cnt = priv->plat->rx_queues_to_use; 3190 u32 tx_cnt = priv->plat->tx_queues_to_use; 3191 u32 queues_count; 3192 u32 queue; 3193 3194 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 3195 3196 if (priv->irq_wake) 3197 pm_wakeup_event(priv->device, 0); 3198 3199 if (unlikely(!dev)) { 3200 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 3201 return IRQ_NONE; 3202 } 3203 3204 /* To handle GMAC own interrupts */ 3205 if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) { 3206 int status = priv->hw->mac->host_irq_status(priv->hw, 3207 &priv->xstats); 3208 3209 if (unlikely(status)) { 3210 /* For LPI we need to save the tx status */ 3211 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 3212 priv->tx_path_in_lpi_mode = true; 3213 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 3214 priv->tx_path_in_lpi_mode = false; 3215 } 3216 3217 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3218 for (queue = 0; queue < queues_count; queue++) { 3219 status |= 3220 priv->hw->mac->host_mtl_irq_status(priv->hw, 3221 queue); 3222 3223 if (status & CORE_IRQ_MTL_RX_OVERFLOW && 3224 priv->hw->dma->set_rx_tail_ptr) 3225 priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, 3226 priv->rx_tail_addr, 3227 queue); 3228 } 3229 } 3230 3231 /* PCS link status */ 3232 if (priv->hw->pcs) { 3233 if (priv->xstats.pcs_link) 3234 netif_carrier_on(dev); 3235 else 3236 netif_carrier_off(dev); 3237 } 3238 } 3239 3240 /* To handle DMA interrupts */ 3241 stmmac_dma_interrupt(priv); 3242 3243 return IRQ_HANDLED; 3244 } 3245 3246 #ifdef CONFIG_NET_POLL_CONTROLLER 3247 /* Polling receive - used by NETCONSOLE and other diagnostic tools 3248 * to allow network I/O with interrupts disabled. 3249 */ 3250 static void stmmac_poll_controller(struct net_device *dev) 3251 { 3252 disable_irq(dev->irq); 3253 stmmac_interrupt(dev->irq, dev); 3254 enable_irq(dev->irq); 3255 } 3256 #endif 3257 3258 /** 3259 * stmmac_ioctl - Entry point for the Ioctl 3260 * @dev: Device pointer. 3261 * @rq: An IOCTL specefic structure, that can contain a pointer to 3262 * a proprietary structure used to pass information to the driver. 3263 * @cmd: IOCTL command 3264 * Description: 3265 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 3266 */ 3267 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3268 { 3269 int ret = -EOPNOTSUPP; 3270 3271 if (!netif_running(dev)) 3272 return -EINVAL; 3273 3274 switch (cmd) { 3275 case SIOCGMIIPHY: 3276 case SIOCGMIIREG: 3277 case SIOCSMIIREG: 3278 if (!dev->phydev) 3279 return -EINVAL; 3280 ret = phy_mii_ioctl(dev->phydev, rq, cmd); 3281 break; 3282 case SIOCSHWTSTAMP: 3283 ret = stmmac_hwtstamp_ioctl(dev, rq); 3284 break; 3285 default: 3286 break; 3287 } 3288 3289 return ret; 3290 } 3291 3292 #ifdef CONFIG_DEBUG_FS 3293 static struct dentry *stmmac_fs_dir; 3294 3295 static void sysfs_display_ring(void *head, int size, int extend_desc, 3296 struct seq_file *seq) 3297 { 3298 int i; 3299 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 3300 struct dma_desc *p = (struct dma_desc *)head; 3301 3302 for (i = 0; i < size; i++) { 3303 if (extend_desc) { 3304 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3305 i, (unsigned int)virt_to_phys(ep), 3306 le32_to_cpu(ep->basic.des0), 3307 le32_to_cpu(ep->basic.des1), 3308 le32_to_cpu(ep->basic.des2), 3309 le32_to_cpu(ep->basic.des3)); 3310 ep++; 3311 } else { 3312 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3313 i, (unsigned int)virt_to_phys(ep), 3314 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 3315 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 3316 p++; 3317 } 3318 seq_printf(seq, "\n"); 3319 } 3320 } 3321 3322 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v) 3323 { 3324 struct net_device *dev = seq->private; 3325 struct stmmac_priv *priv = netdev_priv(dev); 3326 3327 if (priv->extend_desc) { 3328 seq_printf(seq, "Extended RX descriptor ring:\n"); 3329 sysfs_display_ring((void *)priv->dma_erx, DMA_RX_SIZE, 1, seq); 3330 seq_printf(seq, "Extended TX descriptor ring:\n"); 3331 sysfs_display_ring((void *)priv->dma_etx, DMA_TX_SIZE, 1, seq); 3332 } else { 3333 seq_printf(seq, "RX descriptor ring:\n"); 3334 sysfs_display_ring((void *)priv->dma_rx, DMA_RX_SIZE, 0, seq); 3335 seq_printf(seq, "TX descriptor ring:\n"); 3336 sysfs_display_ring((void *)priv->dma_tx, DMA_TX_SIZE, 0, seq); 3337 } 3338 3339 return 0; 3340 } 3341 3342 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file) 3343 { 3344 return single_open(file, stmmac_sysfs_ring_read, inode->i_private); 3345 } 3346 3347 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */ 3348 3349 static const struct file_operations stmmac_rings_status_fops = { 3350 .owner = THIS_MODULE, 3351 .open = stmmac_sysfs_ring_open, 3352 .read = seq_read, 3353 .llseek = seq_lseek, 3354 .release = single_release, 3355 }; 3356 3357 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v) 3358 { 3359 struct net_device *dev = seq->private; 3360 struct stmmac_priv *priv = netdev_priv(dev); 3361 3362 if (!priv->hw_cap_support) { 3363 seq_printf(seq, "DMA HW features not supported\n"); 3364 return 0; 3365 } 3366 3367 seq_printf(seq, "==============================\n"); 3368 seq_printf(seq, "\tDMA HW features\n"); 3369 seq_printf(seq, "==============================\n"); 3370 3371 seq_printf(seq, "\t10/100 Mbps: %s\n", 3372 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 3373 seq_printf(seq, "\t1000 Mbps: %s\n", 3374 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 3375 seq_printf(seq, "\tHalf duplex: %s\n", 3376 (priv->dma_cap.half_duplex) ? "Y" : "N"); 3377 seq_printf(seq, "\tHash Filter: %s\n", 3378 (priv->dma_cap.hash_filter) ? "Y" : "N"); 3379 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 3380 (priv->dma_cap.multi_addr) ? "Y" : "N"); 3381 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 3382 (priv->dma_cap.pcs) ? "Y" : "N"); 3383 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 3384 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 3385 seq_printf(seq, "\tPMT Remote wake up: %s\n", 3386 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 3387 seq_printf(seq, "\tPMT Magic Frame: %s\n", 3388 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 3389 seq_printf(seq, "\tRMON module: %s\n", 3390 (priv->dma_cap.rmon) ? "Y" : "N"); 3391 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 3392 (priv->dma_cap.time_stamp) ? "Y" : "N"); 3393 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 3394 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 3395 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 3396 (priv->dma_cap.eee) ? "Y" : "N"); 3397 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 3398 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 3399 (priv->dma_cap.tx_coe) ? "Y" : "N"); 3400 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3401 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 3402 (priv->dma_cap.rx_coe) ? "Y" : "N"); 3403 } else { 3404 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 3405 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 3406 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 3407 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 3408 } 3409 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 3410 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 3411 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 3412 priv->dma_cap.number_rx_channel); 3413 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 3414 priv->dma_cap.number_tx_channel); 3415 seq_printf(seq, "\tEnhanced descriptors: %s\n", 3416 (priv->dma_cap.enh_desc) ? "Y" : "N"); 3417 3418 return 0; 3419 } 3420 3421 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file) 3422 { 3423 return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private); 3424 } 3425 3426 static const struct file_operations stmmac_dma_cap_fops = { 3427 .owner = THIS_MODULE, 3428 .open = stmmac_sysfs_dma_cap_open, 3429 .read = seq_read, 3430 .llseek = seq_lseek, 3431 .release = single_release, 3432 }; 3433 3434 static int stmmac_init_fs(struct net_device *dev) 3435 { 3436 struct stmmac_priv *priv = netdev_priv(dev); 3437 3438 /* Create per netdev entries */ 3439 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 3440 3441 if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) { 3442 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n"); 3443 3444 return -ENOMEM; 3445 } 3446 3447 /* Entry to report DMA RX/TX rings */ 3448 priv->dbgfs_rings_status = 3449 debugfs_create_file("descriptors_status", S_IRUGO, 3450 priv->dbgfs_dir, dev, 3451 &stmmac_rings_status_fops); 3452 3453 if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) { 3454 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n"); 3455 debugfs_remove_recursive(priv->dbgfs_dir); 3456 3457 return -ENOMEM; 3458 } 3459 3460 /* Entry to report the DMA HW features */ 3461 priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO, 3462 priv->dbgfs_dir, 3463 dev, &stmmac_dma_cap_fops); 3464 3465 if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) { 3466 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n"); 3467 debugfs_remove_recursive(priv->dbgfs_dir); 3468 3469 return -ENOMEM; 3470 } 3471 3472 return 0; 3473 } 3474 3475 static void stmmac_exit_fs(struct net_device *dev) 3476 { 3477 struct stmmac_priv *priv = netdev_priv(dev); 3478 3479 debugfs_remove_recursive(priv->dbgfs_dir); 3480 } 3481 #endif /* CONFIG_DEBUG_FS */ 3482 3483 static const struct net_device_ops stmmac_netdev_ops = { 3484 .ndo_open = stmmac_open, 3485 .ndo_start_xmit = stmmac_xmit, 3486 .ndo_stop = stmmac_release, 3487 .ndo_change_mtu = stmmac_change_mtu, 3488 .ndo_fix_features = stmmac_fix_features, 3489 .ndo_set_features = stmmac_set_features, 3490 .ndo_set_rx_mode = stmmac_set_rx_mode, 3491 .ndo_tx_timeout = stmmac_tx_timeout, 3492 .ndo_do_ioctl = stmmac_ioctl, 3493 #ifdef CONFIG_NET_POLL_CONTROLLER 3494 .ndo_poll_controller = stmmac_poll_controller, 3495 #endif 3496 .ndo_set_mac_address = eth_mac_addr, 3497 }; 3498 3499 /** 3500 * stmmac_hw_init - Init the MAC device 3501 * @priv: driver private structure 3502 * Description: this function is to configure the MAC device according to 3503 * some platform parameters or the HW capability register. It prepares the 3504 * driver to use either ring or chain modes and to setup either enhanced or 3505 * normal descriptors. 3506 */ 3507 static int stmmac_hw_init(struct stmmac_priv *priv) 3508 { 3509 struct mac_device_info *mac; 3510 3511 /* Identify the MAC HW device */ 3512 if (priv->plat->has_gmac) { 3513 priv->dev->priv_flags |= IFF_UNICAST_FLT; 3514 mac = dwmac1000_setup(priv->ioaddr, 3515 priv->plat->multicast_filter_bins, 3516 priv->plat->unicast_filter_entries, 3517 &priv->synopsys_id); 3518 } else if (priv->plat->has_gmac4) { 3519 priv->dev->priv_flags |= IFF_UNICAST_FLT; 3520 mac = dwmac4_setup(priv->ioaddr, 3521 priv->plat->multicast_filter_bins, 3522 priv->plat->unicast_filter_entries, 3523 &priv->synopsys_id); 3524 } else { 3525 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id); 3526 } 3527 if (!mac) 3528 return -ENOMEM; 3529 3530 priv->hw = mac; 3531 3532 /* To use the chained or ring mode */ 3533 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3534 priv->hw->mode = &dwmac4_ring_mode_ops; 3535 } else { 3536 if (chain_mode) { 3537 priv->hw->mode = &chain_mode_ops; 3538 dev_info(priv->device, "Chain mode enabled\n"); 3539 priv->mode = STMMAC_CHAIN_MODE; 3540 } else { 3541 priv->hw->mode = &ring_mode_ops; 3542 dev_info(priv->device, "Ring mode enabled\n"); 3543 priv->mode = STMMAC_RING_MODE; 3544 } 3545 } 3546 3547 /* Get the HW capability (new GMAC newer than 3.50a) */ 3548 priv->hw_cap_support = stmmac_get_hw_features(priv); 3549 if (priv->hw_cap_support) { 3550 dev_info(priv->device, "DMA HW capability register supported\n"); 3551 3552 /* We can override some gmac/dma configuration fields: e.g. 3553 * enh_desc, tx_coe (e.g. that are passed through the 3554 * platform) with the values from the HW capability 3555 * register (if supported). 3556 */ 3557 priv->plat->enh_desc = priv->dma_cap.enh_desc; 3558 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up; 3559 priv->hw->pmt = priv->plat->pmt; 3560 3561 /* TXCOE doesn't work in thresh DMA mode */ 3562 if (priv->plat->force_thresh_dma_mode) 3563 priv->plat->tx_coe = 0; 3564 else 3565 priv->plat->tx_coe = priv->dma_cap.tx_coe; 3566 3567 /* In case of GMAC4 rx_coe is from HW cap register. */ 3568 priv->plat->rx_coe = priv->dma_cap.rx_coe; 3569 3570 if (priv->dma_cap.rx_coe_type2) 3571 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 3572 else if (priv->dma_cap.rx_coe_type1) 3573 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 3574 3575 } else { 3576 dev_info(priv->device, "No HW DMA feature register supported\n"); 3577 } 3578 3579 /* To use alternate (extended), normal or GMAC4 descriptor structures */ 3580 if (priv->synopsys_id >= DWMAC_CORE_4_00) 3581 priv->hw->desc = &dwmac4_desc_ops; 3582 else 3583 stmmac_selec_desc_mode(priv); 3584 3585 if (priv->plat->rx_coe) { 3586 priv->hw->rx_csum = priv->plat->rx_coe; 3587 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 3588 if (priv->synopsys_id < DWMAC_CORE_4_00) 3589 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 3590 } 3591 if (priv->plat->tx_coe) 3592 dev_info(priv->device, "TX Checksum insertion supported\n"); 3593 3594 if (priv->plat->pmt) { 3595 dev_info(priv->device, "Wake-Up On Lan supported\n"); 3596 device_set_wakeup_capable(priv->device, 1); 3597 } 3598 3599 if (priv->dma_cap.tsoen) 3600 dev_info(priv->device, "TSO supported\n"); 3601 3602 return 0; 3603 } 3604 3605 /** 3606 * stmmac_dvr_probe 3607 * @device: device pointer 3608 * @plat_dat: platform data pointer 3609 * @res: stmmac resource pointer 3610 * Description: this is the main probe function used to 3611 * call the alloc_etherdev, allocate the priv structure. 3612 * Return: 3613 * returns 0 on success, otherwise errno. 3614 */ 3615 int stmmac_dvr_probe(struct device *device, 3616 struct plat_stmmacenet_data *plat_dat, 3617 struct stmmac_resources *res) 3618 { 3619 int ret = 0; 3620 struct net_device *ndev = NULL; 3621 struct stmmac_priv *priv; 3622 3623 ndev = alloc_etherdev(sizeof(struct stmmac_priv)); 3624 if (!ndev) 3625 return -ENOMEM; 3626 3627 SET_NETDEV_DEV(ndev, device); 3628 3629 priv = netdev_priv(ndev); 3630 priv->device = device; 3631 priv->dev = ndev; 3632 3633 stmmac_set_ethtool_ops(ndev); 3634 priv->pause = pause; 3635 priv->plat = plat_dat; 3636 priv->ioaddr = res->addr; 3637 priv->dev->base_addr = (unsigned long)res->addr; 3638 3639 priv->dev->irq = res->irq; 3640 priv->wol_irq = res->wol_irq; 3641 priv->lpi_irq = res->lpi_irq; 3642 3643 if (res->mac) 3644 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); 3645 3646 dev_set_drvdata(device, priv->dev); 3647 3648 /* Verify driver arguments */ 3649 stmmac_verify_args(); 3650 3651 /* Override with kernel parameters if supplied XXX CRS XXX 3652 * this needs to have multiple instances 3653 */ 3654 if ((phyaddr >= 0) && (phyaddr <= 31)) 3655 priv->plat->phy_addr = phyaddr; 3656 3657 if (priv->plat->stmmac_rst) 3658 reset_control_deassert(priv->plat->stmmac_rst); 3659 3660 /* Init MAC and get the capabilities */ 3661 ret = stmmac_hw_init(priv); 3662 if (ret) 3663 goto error_hw_init; 3664 3665 ndev->netdev_ops = &stmmac_netdev_ops; 3666 3667 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3668 NETIF_F_RXCSUM; 3669 3670 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 3671 ndev->hw_features |= NETIF_F_TSO; 3672 priv->tso = true; 3673 dev_info(priv->device, "TSO feature enabled\n"); 3674 } 3675 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 3676 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 3677 #ifdef STMMAC_VLAN_TAG_USED 3678 /* Both mac100 and gmac support receive VLAN tag detection */ 3679 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; 3680 #endif 3681 priv->msg_enable = netif_msg_init(debug, default_msg_level); 3682 3683 /* MTU range: 46 - hw-specific max */ 3684 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 3685 if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 3686 ndev->max_mtu = JUMBO_LEN; 3687 else 3688 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 3689 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 3690 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 3691 */ 3692 if ((priv->plat->maxmtu < ndev->max_mtu) && 3693 (priv->plat->maxmtu >= ndev->min_mtu)) 3694 ndev->max_mtu = priv->plat->maxmtu; 3695 else if (priv->plat->maxmtu < ndev->min_mtu) 3696 dev_warn(priv->device, 3697 "%s: warning: maxmtu having invalid value (%d)\n", 3698 __func__, priv->plat->maxmtu); 3699 3700 if (flow_ctrl) 3701 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 3702 3703 /* Rx Watchdog is available in the COREs newer than the 3.40. 3704 * In some case, for example on bugged HW this feature 3705 * has to be disable and this can be done by passing the 3706 * riwt_off field from the platform. 3707 */ 3708 if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) { 3709 priv->use_riwt = 1; 3710 dev_info(priv->device, 3711 "Enable RX Mitigation via HW Watchdog Timer\n"); 3712 } 3713 3714 netif_napi_add(ndev, &priv->napi, stmmac_poll, 64); 3715 3716 spin_lock_init(&priv->lock); 3717 3718 /* If a specific clk_csr value is passed from the platform 3719 * this means that the CSR Clock Range selection cannot be 3720 * changed at run-time and it is fixed. Viceversa the driver'll try to 3721 * set the MDC clock dynamically according to the csr actual 3722 * clock input. 3723 */ 3724 if (!priv->plat->clk_csr) 3725 stmmac_clk_csr_set(priv); 3726 else 3727 priv->clk_csr = priv->plat->clk_csr; 3728 3729 stmmac_check_pcs_mode(priv); 3730 3731 if (priv->hw->pcs != STMMAC_PCS_RGMII && 3732 priv->hw->pcs != STMMAC_PCS_TBI && 3733 priv->hw->pcs != STMMAC_PCS_RTBI) { 3734 /* MDIO bus Registration */ 3735 ret = stmmac_mdio_register(ndev); 3736 if (ret < 0) { 3737 dev_err(priv->device, 3738 "%s: MDIO bus (id: %d) registration failed", 3739 __func__, priv->plat->bus_id); 3740 goto error_mdio_register; 3741 } 3742 } 3743 3744 ret = register_netdev(ndev); 3745 if (ret) { 3746 dev_err(priv->device, "%s: ERROR %i registering the device\n", 3747 __func__, ret); 3748 goto error_netdev_register; 3749 } 3750 3751 return ret; 3752 3753 error_netdev_register: 3754 if (priv->hw->pcs != STMMAC_PCS_RGMII && 3755 priv->hw->pcs != STMMAC_PCS_TBI && 3756 priv->hw->pcs != STMMAC_PCS_RTBI) 3757 stmmac_mdio_unregister(ndev); 3758 error_mdio_register: 3759 netif_napi_del(&priv->napi); 3760 error_hw_init: 3761 free_netdev(ndev); 3762 3763 return ret; 3764 } 3765 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 3766 3767 /** 3768 * stmmac_dvr_remove 3769 * @dev: device pointer 3770 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 3771 * changes the link status, releases the DMA descriptor rings. 3772 */ 3773 int stmmac_dvr_remove(struct device *dev) 3774 { 3775 struct net_device *ndev = dev_get_drvdata(dev); 3776 struct stmmac_priv *priv = netdev_priv(ndev); 3777 3778 netdev_info(priv->dev, "%s: removing driver", __func__); 3779 3780 stmmac_stop_all_dma(priv); 3781 3782 priv->hw->mac->set_mac(priv->ioaddr, false); 3783 netif_carrier_off(ndev); 3784 unregister_netdev(ndev); 3785 if (priv->plat->stmmac_rst) 3786 reset_control_assert(priv->plat->stmmac_rst); 3787 clk_disable_unprepare(priv->plat->pclk); 3788 clk_disable_unprepare(priv->plat->stmmac_clk); 3789 if (priv->hw->pcs != STMMAC_PCS_RGMII && 3790 priv->hw->pcs != STMMAC_PCS_TBI && 3791 priv->hw->pcs != STMMAC_PCS_RTBI) 3792 stmmac_mdio_unregister(ndev); 3793 free_netdev(ndev); 3794 3795 return 0; 3796 } 3797 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 3798 3799 /** 3800 * stmmac_suspend - suspend callback 3801 * @dev: device pointer 3802 * Description: this is the function to suspend the device and it is called 3803 * by the platform driver to stop the network queue, release the resources, 3804 * program the PMT register (for WoL), clean and release driver resources. 3805 */ 3806 int stmmac_suspend(struct device *dev) 3807 { 3808 struct net_device *ndev = dev_get_drvdata(dev); 3809 struct stmmac_priv *priv = netdev_priv(ndev); 3810 unsigned long flags; 3811 3812 if (!ndev || !netif_running(ndev)) 3813 return 0; 3814 3815 if (ndev->phydev) 3816 phy_stop(ndev->phydev); 3817 3818 spin_lock_irqsave(&priv->lock, flags); 3819 3820 netif_device_detach(ndev); 3821 netif_stop_queue(ndev); 3822 3823 napi_disable(&priv->napi); 3824 3825 /* Stop TX/RX DMA */ 3826 stmmac_stop_all_dma(priv); 3827 3828 /* Enable Power down mode by programming the PMT regs */ 3829 if (device_may_wakeup(priv->device)) { 3830 priv->hw->mac->pmt(priv->hw, priv->wolopts); 3831 priv->irq_wake = 1; 3832 } else { 3833 priv->hw->mac->set_mac(priv->ioaddr, false); 3834 pinctrl_pm_select_sleep_state(priv->device); 3835 /* Disable clock in case of PWM is off */ 3836 clk_disable(priv->plat->pclk); 3837 clk_disable(priv->plat->stmmac_clk); 3838 } 3839 spin_unlock_irqrestore(&priv->lock, flags); 3840 3841 priv->oldlink = 0; 3842 priv->speed = SPEED_UNKNOWN; 3843 priv->oldduplex = DUPLEX_UNKNOWN; 3844 return 0; 3845 } 3846 EXPORT_SYMBOL_GPL(stmmac_suspend); 3847 3848 /** 3849 * stmmac_resume - resume callback 3850 * @dev: device pointer 3851 * Description: when resume this function is invoked to setup the DMA and CORE 3852 * in a usable state. 3853 */ 3854 int stmmac_resume(struct device *dev) 3855 { 3856 struct net_device *ndev = dev_get_drvdata(dev); 3857 struct stmmac_priv *priv = netdev_priv(ndev); 3858 unsigned long flags; 3859 3860 if (!netif_running(ndev)) 3861 return 0; 3862 3863 /* Power Down bit, into the PM register, is cleared 3864 * automatically as soon as a magic packet or a Wake-up frame 3865 * is received. Anyway, it's better to manually clear 3866 * this bit because it can generate problems while resuming 3867 * from another devices (e.g. serial console). 3868 */ 3869 if (device_may_wakeup(priv->device)) { 3870 spin_lock_irqsave(&priv->lock, flags); 3871 priv->hw->mac->pmt(priv->hw, 0); 3872 spin_unlock_irqrestore(&priv->lock, flags); 3873 priv->irq_wake = 0; 3874 } else { 3875 pinctrl_pm_select_default_state(priv->device); 3876 /* enable the clk previously disabled */ 3877 clk_enable(priv->plat->stmmac_clk); 3878 clk_enable(priv->plat->pclk); 3879 /* reset the phy so that it's ready */ 3880 if (priv->mii) 3881 stmmac_mdio_reset(priv->mii); 3882 } 3883 3884 netif_device_attach(ndev); 3885 3886 spin_lock_irqsave(&priv->lock, flags); 3887 3888 priv->cur_rx = 0; 3889 priv->dirty_rx = 0; 3890 priv->dirty_tx = 0; 3891 priv->cur_tx = 0; 3892 /* reset private mss value to force mss context settings at 3893 * next tso xmit (only used for gmac4). 3894 */ 3895 priv->mss = 0; 3896 3897 stmmac_clear_descriptors(priv); 3898 3899 stmmac_hw_setup(ndev, false); 3900 stmmac_init_tx_coalesce(priv); 3901 stmmac_set_rx_mode(ndev); 3902 3903 napi_enable(&priv->napi); 3904 3905 netif_start_queue(ndev); 3906 3907 spin_unlock_irqrestore(&priv->lock, flags); 3908 3909 if (ndev->phydev) 3910 phy_start(ndev->phydev); 3911 3912 return 0; 3913 } 3914 EXPORT_SYMBOL_GPL(stmmac_resume); 3915 3916 #ifndef MODULE 3917 static int __init stmmac_cmdline_opt(char *str) 3918 { 3919 char *opt; 3920 3921 if (!str || !*str) 3922 return -EINVAL; 3923 while ((opt = strsep(&str, ",")) != NULL) { 3924 if (!strncmp(opt, "debug:", 6)) { 3925 if (kstrtoint(opt + 6, 0, &debug)) 3926 goto err; 3927 } else if (!strncmp(opt, "phyaddr:", 8)) { 3928 if (kstrtoint(opt + 8, 0, &phyaddr)) 3929 goto err; 3930 } else if (!strncmp(opt, "buf_sz:", 7)) { 3931 if (kstrtoint(opt + 7, 0, &buf_sz)) 3932 goto err; 3933 } else if (!strncmp(opt, "tc:", 3)) { 3934 if (kstrtoint(opt + 3, 0, &tc)) 3935 goto err; 3936 } else if (!strncmp(opt, "watchdog:", 9)) { 3937 if (kstrtoint(opt + 9, 0, &watchdog)) 3938 goto err; 3939 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 3940 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 3941 goto err; 3942 } else if (!strncmp(opt, "pause:", 6)) { 3943 if (kstrtoint(opt + 6, 0, &pause)) 3944 goto err; 3945 } else if (!strncmp(opt, "eee_timer:", 10)) { 3946 if (kstrtoint(opt + 10, 0, &eee_timer)) 3947 goto err; 3948 } else if (!strncmp(opt, "chain_mode:", 11)) { 3949 if (kstrtoint(opt + 11, 0, &chain_mode)) 3950 goto err; 3951 } 3952 } 3953 return 0; 3954 3955 err: 3956 pr_err("%s: ERROR broken module parameter conversion", __func__); 3957 return -EINVAL; 3958 } 3959 3960 __setup("stmmaceth=", stmmac_cmdline_opt); 3961 #endif /* MODULE */ 3962 3963 static int __init stmmac_init(void) 3964 { 3965 #ifdef CONFIG_DEBUG_FS 3966 /* Create debugfs main directory if it doesn't exist yet */ 3967 if (!stmmac_fs_dir) { 3968 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 3969 3970 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) { 3971 pr_err("ERROR %s, debugfs create directory failed\n", 3972 STMMAC_RESOURCE_NAME); 3973 3974 return -ENOMEM; 3975 } 3976 } 3977 #endif 3978 3979 return 0; 3980 } 3981 3982 static void __exit stmmac_exit(void) 3983 { 3984 #ifdef CONFIG_DEBUG_FS 3985 debugfs_remove_recursive(stmmac_fs_dir); 3986 #endif 3987 } 3988 3989 module_init(stmmac_init) 3990 module_exit(stmmac_exit) 3991 3992 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 3993 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 3994 MODULE_LICENSE("GPL"); 3995