1 /******************************************************************************* 2 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 3 ST Ethernet IPs are built around a Synopsys IP Core. 4 5 Copyright(C) 2007-2011 STMicroelectronics Ltd 6 7 This program is free software; you can redistribute it and/or modify it 8 under the terms and conditions of the GNU General Public License, 9 version 2, as published by the Free Software Foundation. 10 11 This program is distributed in the hope it will be useful, but WITHOUT 12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 more details. 15 16 The full GNU General Public License is included in this distribution in 17 the file called "COPYING". 18 19 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 20 21 Documentation available at: 22 http://www.stlinux.com 23 Support available at: 24 https://bugzilla.stlinux.com/ 25 *******************************************************************************/ 26 27 #include <linux/clk.h> 28 #include <linux/kernel.h> 29 #include <linux/interrupt.h> 30 #include <linux/ip.h> 31 #include <linux/tcp.h> 32 #include <linux/skbuff.h> 33 #include <linux/ethtool.h> 34 #include <linux/if_ether.h> 35 #include <linux/crc32.h> 36 #include <linux/mii.h> 37 #include <linux/if.h> 38 #include <linux/if_vlan.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/slab.h> 41 #include <linux/prefetch.h> 42 #include <linux/pinctrl/consumer.h> 43 #ifdef CONFIG_DEBUG_FS 44 #include <linux/debugfs.h> 45 #include <linux/seq_file.h> 46 #endif /* CONFIG_DEBUG_FS */ 47 #include <linux/net_tstamp.h> 48 #include "stmmac_ptp.h" 49 #include "stmmac.h" 50 #include <linux/reset.h> 51 #include <linux/of_mdio.h> 52 #include "dwmac1000.h" 53 54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x) 55 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 56 57 /* Module parameters */ 58 #define TX_TIMEO 5000 59 static int watchdog = TX_TIMEO; 60 module_param(watchdog, int, S_IRUGO | S_IWUSR); 61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 62 63 static int debug = -1; 64 module_param(debug, int, S_IRUGO | S_IWUSR); 65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 66 67 static int phyaddr = -1; 68 module_param(phyaddr, int, S_IRUGO); 69 MODULE_PARM_DESC(phyaddr, "Physical device address"); 70 71 #define STMMAC_TX_THRESH (DMA_TX_SIZE / 4) 72 #define STMMAC_RX_THRESH (DMA_RX_SIZE / 4) 73 74 static int flow_ctrl = FLOW_OFF; 75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR); 76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 77 78 static int pause = PAUSE_TIME; 79 module_param(pause, int, S_IRUGO | S_IWUSR); 80 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 81 82 #define TC_DEFAULT 64 83 static int tc = TC_DEFAULT; 84 module_param(tc, int, S_IRUGO | S_IWUSR); 85 MODULE_PARM_DESC(tc, "DMA threshold control value"); 86 87 #define DEFAULT_BUFSIZE 1536 88 static int buf_sz = DEFAULT_BUFSIZE; 89 module_param(buf_sz, int, S_IRUGO | S_IWUSR); 90 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 91 92 #define STMMAC_RX_COPYBREAK 256 93 94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 95 NETIF_MSG_LINK | NETIF_MSG_IFUP | 96 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 97 98 #define STMMAC_DEFAULT_LPI_TIMER 1000 99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR); 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x)) 103 104 /* By default the driver will use the ring mode to manage tx and rx descriptors, 105 * but allow user to force to use the chain instead of the ring 106 */ 107 static unsigned int chain_mode; 108 module_param(chain_mode, int, S_IRUGO); 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 110 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 112 113 #ifdef CONFIG_DEBUG_FS 114 static int stmmac_init_fs(struct net_device *dev); 115 static void stmmac_exit_fs(struct net_device *dev); 116 #endif 117 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x)) 119 120 /** 121 * stmmac_verify_args - verify the driver parameters. 122 * Description: it checks the driver parameters and set a default in case of 123 * errors. 124 */ 125 static void stmmac_verify_args(void) 126 { 127 if (unlikely(watchdog < 0)) 128 watchdog = TX_TIMEO; 129 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 130 buf_sz = DEFAULT_BUFSIZE; 131 if (unlikely(flow_ctrl > 1)) 132 flow_ctrl = FLOW_AUTO; 133 else if (likely(flow_ctrl < 0)) 134 flow_ctrl = FLOW_OFF; 135 if (unlikely((pause < 0) || (pause > 0xffff))) 136 pause = PAUSE_TIME; 137 if (eee_timer < 0) 138 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 139 } 140 141 /** 142 * stmmac_disable_all_queues - Disable all queues 143 * @priv: driver private structure 144 */ 145 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 146 { 147 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 148 u32 queue; 149 150 for (queue = 0; queue < rx_queues_cnt; queue++) { 151 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 152 153 napi_disable(&rx_q->napi); 154 } 155 } 156 157 /** 158 * stmmac_enable_all_queues - Enable all queues 159 * @priv: driver private structure 160 */ 161 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 162 { 163 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 164 u32 queue; 165 166 for (queue = 0; queue < rx_queues_cnt; queue++) { 167 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 168 169 napi_enable(&rx_q->napi); 170 } 171 } 172 173 /** 174 * stmmac_stop_all_queues - Stop all queues 175 * @priv: driver private structure 176 */ 177 static void stmmac_stop_all_queues(struct stmmac_priv *priv) 178 { 179 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 180 u32 queue; 181 182 for (queue = 0; queue < tx_queues_cnt; queue++) 183 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 184 } 185 186 /** 187 * stmmac_start_all_queues - Start all queues 188 * @priv: driver private structure 189 */ 190 static void stmmac_start_all_queues(struct stmmac_priv *priv) 191 { 192 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 193 u32 queue; 194 195 for (queue = 0; queue < tx_queues_cnt; queue++) 196 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue)); 197 } 198 199 /** 200 * stmmac_clk_csr_set - dynamically set the MDC clock 201 * @priv: driver private structure 202 * Description: this is to dynamically set the MDC clock according to the csr 203 * clock input. 204 * Note: 205 * If a specific clk_csr value is passed from the platform 206 * this means that the CSR Clock Range selection cannot be 207 * changed at run-time and it is fixed (as reported in the driver 208 * documentation). Viceversa the driver will try to set the MDC 209 * clock dynamically according to the actual clock input. 210 */ 211 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 212 { 213 u32 clk_rate; 214 215 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 216 217 /* Platform provided default clk_csr would be assumed valid 218 * for all other cases except for the below mentioned ones. 219 * For values higher than the IEEE 802.3 specified frequency 220 * we can not estimate the proper divider as it is not known 221 * the frequency of clk_csr_i. So we do not change the default 222 * divider. 223 */ 224 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 225 if (clk_rate < CSR_F_35M) 226 priv->clk_csr = STMMAC_CSR_20_35M; 227 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 228 priv->clk_csr = STMMAC_CSR_35_60M; 229 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 230 priv->clk_csr = STMMAC_CSR_60_100M; 231 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 232 priv->clk_csr = STMMAC_CSR_100_150M; 233 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 234 priv->clk_csr = STMMAC_CSR_150_250M; 235 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M)) 236 priv->clk_csr = STMMAC_CSR_250_300M; 237 } 238 } 239 240 static void print_pkt(unsigned char *buf, int len) 241 { 242 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 243 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 244 } 245 246 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 247 { 248 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 249 u32 avail; 250 251 if (tx_q->dirty_tx > tx_q->cur_tx) 252 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 253 else 254 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1; 255 256 return avail; 257 } 258 259 /** 260 * stmmac_rx_dirty - Get RX queue dirty 261 * @priv: driver private structure 262 * @queue: RX queue index 263 */ 264 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 265 { 266 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 267 u32 dirty; 268 269 if (rx_q->dirty_rx <= rx_q->cur_rx) 270 dirty = rx_q->cur_rx - rx_q->dirty_rx; 271 else 272 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx; 273 274 return dirty; 275 } 276 277 /** 278 * stmmac_hw_fix_mac_speed - callback for speed selection 279 * @priv: driver private structure 280 * Description: on some platforms (e.g. ST), some HW system configuration 281 * registers have to be set according to the link speed negotiated. 282 */ 283 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv) 284 { 285 struct net_device *ndev = priv->dev; 286 struct phy_device *phydev = ndev->phydev; 287 288 if (likely(priv->plat->fix_mac_speed)) 289 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed); 290 } 291 292 /** 293 * stmmac_enable_eee_mode - check and enter in LPI mode 294 * @priv: driver private structure 295 * Description: this function is to verify and enter in LPI mode in case of 296 * EEE. 297 */ 298 static void stmmac_enable_eee_mode(struct stmmac_priv *priv) 299 { 300 u32 tx_cnt = priv->plat->tx_queues_to_use; 301 u32 queue; 302 303 /* check if all TX queues have the work finished */ 304 for (queue = 0; queue < tx_cnt; queue++) { 305 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 306 307 if (tx_q->dirty_tx != tx_q->cur_tx) 308 return; /* still unfinished work */ 309 } 310 311 /* Check and enter in LPI mode */ 312 if (!priv->tx_path_in_lpi_mode) 313 priv->hw->mac->set_eee_mode(priv->hw, 314 priv->plat->en_tx_lpi_clockgating); 315 } 316 317 /** 318 * stmmac_disable_eee_mode - disable and exit from LPI mode 319 * @priv: driver private structure 320 * Description: this function is to exit and disable EEE in case of 321 * LPI state is true. This is called by the xmit. 322 */ 323 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 324 { 325 priv->hw->mac->reset_eee_mode(priv->hw); 326 del_timer_sync(&priv->eee_ctrl_timer); 327 priv->tx_path_in_lpi_mode = false; 328 } 329 330 /** 331 * stmmac_eee_ctrl_timer - EEE TX SW timer. 332 * @arg : data hook 333 * Description: 334 * if there is no data transfer and if we are not in LPI state, 335 * then MAC Transmitter can be moved to LPI state. 336 */ 337 static void stmmac_eee_ctrl_timer(unsigned long arg) 338 { 339 struct stmmac_priv *priv = (struct stmmac_priv *)arg; 340 341 stmmac_enable_eee_mode(priv); 342 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 343 } 344 345 /** 346 * stmmac_eee_init - init EEE 347 * @priv: driver private structure 348 * Description: 349 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 350 * can also manage EEE, this function enable the LPI state and start related 351 * timer. 352 */ 353 bool stmmac_eee_init(struct stmmac_priv *priv) 354 { 355 struct net_device *ndev = priv->dev; 356 unsigned long flags; 357 bool ret = false; 358 359 /* Using PCS we cannot dial with the phy registers at this stage 360 * so we do not support extra feature like EEE. 361 */ 362 if ((priv->hw->pcs == STMMAC_PCS_RGMII) || 363 (priv->hw->pcs == STMMAC_PCS_TBI) || 364 (priv->hw->pcs == STMMAC_PCS_RTBI)) 365 goto out; 366 367 /* MAC core supports the EEE feature. */ 368 if (priv->dma_cap.eee) { 369 int tx_lpi_timer = priv->tx_lpi_timer; 370 371 /* Check if the PHY supports EEE */ 372 if (phy_init_eee(ndev->phydev, 1)) { 373 /* To manage at run-time if the EEE cannot be supported 374 * anymore (for example because the lp caps have been 375 * changed). 376 * In that case the driver disable own timers. 377 */ 378 spin_lock_irqsave(&priv->lock, flags); 379 if (priv->eee_active) { 380 netdev_dbg(priv->dev, "disable EEE\n"); 381 del_timer_sync(&priv->eee_ctrl_timer); 382 priv->hw->mac->set_eee_timer(priv->hw, 0, 383 tx_lpi_timer); 384 } 385 priv->eee_active = 0; 386 spin_unlock_irqrestore(&priv->lock, flags); 387 goto out; 388 } 389 /* Activate the EEE and start timers */ 390 spin_lock_irqsave(&priv->lock, flags); 391 if (!priv->eee_active) { 392 priv->eee_active = 1; 393 setup_timer(&priv->eee_ctrl_timer, 394 stmmac_eee_ctrl_timer, 395 (unsigned long)priv); 396 mod_timer(&priv->eee_ctrl_timer, 397 STMMAC_LPI_T(eee_timer)); 398 399 priv->hw->mac->set_eee_timer(priv->hw, 400 STMMAC_DEFAULT_LIT_LS, 401 tx_lpi_timer); 402 } 403 /* Set HW EEE according to the speed */ 404 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link); 405 406 ret = true; 407 spin_unlock_irqrestore(&priv->lock, flags); 408 409 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 410 } 411 out: 412 return ret; 413 } 414 415 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 416 * @priv: driver private structure 417 * @p : descriptor pointer 418 * @skb : the socket buffer 419 * Description : 420 * This function will read timestamp from the descriptor & pass it to stack. 421 * and also perform some sanity checks. 422 */ 423 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 424 struct dma_desc *p, struct sk_buff *skb) 425 { 426 struct skb_shared_hwtstamps shhwtstamp; 427 u64 ns; 428 429 if (!priv->hwts_tx_en) 430 return; 431 432 /* exit if skb doesn't support hw tstamp */ 433 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 434 return; 435 436 /* check tx tstamp status */ 437 if (!priv->hw->desc->get_tx_timestamp_status(p)) { 438 /* get the valid tstamp */ 439 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts); 440 441 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 442 shhwtstamp.hwtstamp = ns_to_ktime(ns); 443 444 netdev_info(priv->dev, "get valid TX hw timestamp %llu\n", ns); 445 /* pass tstamp to stack */ 446 skb_tstamp_tx(skb, &shhwtstamp); 447 } 448 449 return; 450 } 451 452 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 453 * @priv: driver private structure 454 * @p : descriptor pointer 455 * @np : next descriptor pointer 456 * @skb : the socket buffer 457 * Description : 458 * This function will read received packet's timestamp from the descriptor 459 * and pass it to stack. It also perform some sanity checks. 460 */ 461 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 462 struct dma_desc *np, struct sk_buff *skb) 463 { 464 struct skb_shared_hwtstamps *shhwtstamp = NULL; 465 u64 ns; 466 467 if (!priv->hwts_rx_en) 468 return; 469 470 /* Check if timestamp is available */ 471 if (!priv->hw->desc->get_rx_timestamp_status(p, priv->adv_ts)) { 472 /* For GMAC4, the valid timestamp is from CTX next desc. */ 473 if (priv->plat->has_gmac4) 474 ns = priv->hw->desc->get_timestamp(np, priv->adv_ts); 475 else 476 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts); 477 478 netdev_info(priv->dev, "get valid RX hw timestamp %llu\n", ns); 479 shhwtstamp = skb_hwtstamps(skb); 480 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 481 shhwtstamp->hwtstamp = ns_to_ktime(ns); 482 } else { 483 netdev_err(priv->dev, "cannot get RX hw timestamp\n"); 484 } 485 } 486 487 /** 488 * stmmac_hwtstamp_ioctl - control hardware timestamping. 489 * @dev: device pointer. 490 * @ifr: An IOCTL specific structure, that can contain a pointer to 491 * a proprietary structure used to pass information to the driver. 492 * Description: 493 * This function configures the MAC to enable/disable both outgoing(TX) 494 * and incoming(RX) packets time stamping based on user input. 495 * Return Value: 496 * 0 on success and an appropriate -ve integer on failure. 497 */ 498 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr) 499 { 500 struct stmmac_priv *priv = netdev_priv(dev); 501 struct hwtstamp_config config; 502 struct timespec64 now; 503 u64 temp = 0; 504 u32 ptp_v2 = 0; 505 u32 tstamp_all = 0; 506 u32 ptp_over_ipv4_udp = 0; 507 u32 ptp_over_ipv6_udp = 0; 508 u32 ptp_over_ethernet = 0; 509 u32 snap_type_sel = 0; 510 u32 ts_master_en = 0; 511 u32 ts_event_en = 0; 512 u32 value = 0; 513 u32 sec_inc; 514 515 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 516 netdev_alert(priv->dev, "No support for HW time stamping\n"); 517 priv->hwts_tx_en = 0; 518 priv->hwts_rx_en = 0; 519 520 return -EOPNOTSUPP; 521 } 522 523 if (copy_from_user(&config, ifr->ifr_data, 524 sizeof(struct hwtstamp_config))) 525 return -EFAULT; 526 527 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 528 __func__, config.flags, config.tx_type, config.rx_filter); 529 530 /* reserved for future extensions */ 531 if (config.flags) 532 return -EINVAL; 533 534 if (config.tx_type != HWTSTAMP_TX_OFF && 535 config.tx_type != HWTSTAMP_TX_ON) 536 return -ERANGE; 537 538 if (priv->adv_ts) { 539 switch (config.rx_filter) { 540 case HWTSTAMP_FILTER_NONE: 541 /* time stamp no incoming packet at all */ 542 config.rx_filter = HWTSTAMP_FILTER_NONE; 543 break; 544 545 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 546 /* PTP v1, UDP, any kind of event packet */ 547 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 548 /* take time stamp for all event messages */ 549 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 550 551 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 552 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 553 break; 554 555 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 556 /* PTP v1, UDP, Sync packet */ 557 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 558 /* take time stamp for SYNC messages only */ 559 ts_event_en = PTP_TCR_TSEVNTENA; 560 561 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 562 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 563 break; 564 565 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 566 /* PTP v1, UDP, Delay_req packet */ 567 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 568 /* take time stamp for Delay_Req messages only */ 569 ts_master_en = PTP_TCR_TSMSTRENA; 570 ts_event_en = PTP_TCR_TSEVNTENA; 571 572 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 573 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 574 break; 575 576 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 577 /* PTP v2, UDP, any kind of event packet */ 578 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 579 ptp_v2 = PTP_TCR_TSVER2ENA; 580 /* take time stamp for all event messages */ 581 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 582 583 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 584 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 585 break; 586 587 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 588 /* PTP v2, UDP, Sync packet */ 589 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 590 ptp_v2 = PTP_TCR_TSVER2ENA; 591 /* take time stamp for SYNC messages only */ 592 ts_event_en = PTP_TCR_TSEVNTENA; 593 594 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 595 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 596 break; 597 598 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 599 /* PTP v2, UDP, Delay_req packet */ 600 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 601 ptp_v2 = PTP_TCR_TSVER2ENA; 602 /* take time stamp for Delay_Req messages only */ 603 ts_master_en = PTP_TCR_TSMSTRENA; 604 ts_event_en = PTP_TCR_TSEVNTENA; 605 606 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 607 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 608 break; 609 610 case HWTSTAMP_FILTER_PTP_V2_EVENT: 611 /* PTP v2/802.AS1 any layer, any kind of event packet */ 612 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 613 ptp_v2 = PTP_TCR_TSVER2ENA; 614 /* take time stamp for all event messages */ 615 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 616 617 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 618 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 619 ptp_over_ethernet = PTP_TCR_TSIPENA; 620 break; 621 622 case HWTSTAMP_FILTER_PTP_V2_SYNC: 623 /* PTP v2/802.AS1, any layer, Sync packet */ 624 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 625 ptp_v2 = PTP_TCR_TSVER2ENA; 626 /* take time stamp for SYNC messages only */ 627 ts_event_en = PTP_TCR_TSEVNTENA; 628 629 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 630 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 631 ptp_over_ethernet = PTP_TCR_TSIPENA; 632 break; 633 634 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 635 /* PTP v2/802.AS1, any layer, Delay_req packet */ 636 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 637 ptp_v2 = PTP_TCR_TSVER2ENA; 638 /* take time stamp for Delay_Req messages only */ 639 ts_master_en = PTP_TCR_TSMSTRENA; 640 ts_event_en = PTP_TCR_TSEVNTENA; 641 642 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 643 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 644 ptp_over_ethernet = PTP_TCR_TSIPENA; 645 break; 646 647 case HWTSTAMP_FILTER_ALL: 648 /* time stamp any incoming packet */ 649 config.rx_filter = HWTSTAMP_FILTER_ALL; 650 tstamp_all = PTP_TCR_TSENALL; 651 break; 652 653 default: 654 return -ERANGE; 655 } 656 } else { 657 switch (config.rx_filter) { 658 case HWTSTAMP_FILTER_NONE: 659 config.rx_filter = HWTSTAMP_FILTER_NONE; 660 break; 661 default: 662 /* PTP v1, UDP, any kind of event packet */ 663 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 664 break; 665 } 666 } 667 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 668 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 669 670 if (!priv->hwts_tx_en && !priv->hwts_rx_en) 671 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0); 672 else { 673 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR | 674 tstamp_all | ptp_v2 | ptp_over_ethernet | 675 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en | 676 ts_master_en | snap_type_sel); 677 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value); 678 679 /* program Sub Second Increment reg */ 680 sec_inc = priv->hw->ptp->config_sub_second_increment( 681 priv->ptpaddr, priv->plat->clk_ptp_rate, 682 priv->plat->has_gmac4); 683 temp = div_u64(1000000000ULL, sec_inc); 684 685 /* calculate default added value: 686 * formula is : 687 * addend = (2^32)/freq_div_ratio; 688 * where, freq_div_ratio = 1e9ns/sec_inc 689 */ 690 temp = (u64)(temp << 32); 691 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 692 priv->hw->ptp->config_addend(priv->ptpaddr, 693 priv->default_addend); 694 695 /* initialize system time */ 696 ktime_get_real_ts64(&now); 697 698 /* lower 32 bits of tv_sec are safe until y2106 */ 699 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec, 700 now.tv_nsec); 701 } 702 703 return copy_to_user(ifr->ifr_data, &config, 704 sizeof(struct hwtstamp_config)) ? -EFAULT : 0; 705 } 706 707 /** 708 * stmmac_init_ptp - init PTP 709 * @priv: driver private structure 710 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 711 * This is done by looking at the HW cap. register. 712 * This function also registers the ptp driver. 713 */ 714 static int stmmac_init_ptp(struct stmmac_priv *priv) 715 { 716 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 717 return -EOPNOTSUPP; 718 719 priv->adv_ts = 0; 720 /* Check if adv_ts can be enabled for dwmac 4.x core */ 721 if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp) 722 priv->adv_ts = 1; 723 /* Dwmac 3.x core with extend_desc can support adv_ts */ 724 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 725 priv->adv_ts = 1; 726 727 if (priv->dma_cap.time_stamp) 728 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 729 730 if (priv->adv_ts) 731 netdev_info(priv->dev, 732 "IEEE 1588-2008 Advanced Timestamp supported\n"); 733 734 priv->hw->ptp = &stmmac_ptp; 735 priv->hwts_tx_en = 0; 736 priv->hwts_rx_en = 0; 737 738 stmmac_ptp_register(priv); 739 740 return 0; 741 } 742 743 static void stmmac_release_ptp(struct stmmac_priv *priv) 744 { 745 if (priv->plat->clk_ptp_ref) 746 clk_disable_unprepare(priv->plat->clk_ptp_ref); 747 stmmac_ptp_unregister(priv); 748 } 749 750 /** 751 * stmmac_mac_flow_ctrl - Configure flow control in all queues 752 * @priv: driver private structure 753 * Description: It is used for configuring the flow control in all queues 754 */ 755 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 756 { 757 u32 tx_cnt = priv->plat->tx_queues_to_use; 758 759 priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl, 760 priv->pause, tx_cnt); 761 } 762 763 /** 764 * stmmac_adjust_link - adjusts the link parameters 765 * @dev: net device structure 766 * Description: this is the helper called by the physical abstraction layer 767 * drivers to communicate the phy link status. According the speed and duplex 768 * this driver can invoke registered glue-logic as well. 769 * It also invoke the eee initialization because it could happen when switch 770 * on different networks (that are eee capable). 771 */ 772 static void stmmac_adjust_link(struct net_device *dev) 773 { 774 struct stmmac_priv *priv = netdev_priv(dev); 775 struct phy_device *phydev = dev->phydev; 776 unsigned long flags; 777 int new_state = 0; 778 779 if (!phydev) 780 return; 781 782 spin_lock_irqsave(&priv->lock, flags); 783 784 if (phydev->link) { 785 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 786 787 /* Now we make sure that we can be in full duplex mode. 788 * If not, we operate in half-duplex mode. */ 789 if (phydev->duplex != priv->oldduplex) { 790 new_state = 1; 791 if (!(phydev->duplex)) 792 ctrl &= ~priv->hw->link.duplex; 793 else 794 ctrl |= priv->hw->link.duplex; 795 priv->oldduplex = phydev->duplex; 796 } 797 /* Flow Control operation */ 798 if (phydev->pause) 799 stmmac_mac_flow_ctrl(priv, phydev->duplex); 800 801 if (phydev->speed != priv->speed) { 802 new_state = 1; 803 switch (phydev->speed) { 804 case 1000: 805 if (priv->plat->has_gmac || 806 priv->plat->has_gmac4) 807 ctrl &= ~priv->hw->link.port; 808 break; 809 case 100: 810 if (priv->plat->has_gmac || 811 priv->plat->has_gmac4) { 812 ctrl |= priv->hw->link.port; 813 ctrl |= priv->hw->link.speed; 814 } else { 815 ctrl &= ~priv->hw->link.port; 816 } 817 break; 818 case 10: 819 if (priv->plat->has_gmac || 820 priv->plat->has_gmac4) { 821 ctrl |= priv->hw->link.port; 822 ctrl &= ~(priv->hw->link.speed); 823 } else { 824 ctrl &= ~priv->hw->link.port; 825 } 826 break; 827 default: 828 netif_warn(priv, link, priv->dev, 829 "broken speed: %d\n", phydev->speed); 830 phydev->speed = SPEED_UNKNOWN; 831 break; 832 } 833 if (phydev->speed != SPEED_UNKNOWN) 834 stmmac_hw_fix_mac_speed(priv); 835 priv->speed = phydev->speed; 836 } 837 838 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 839 840 if (!priv->oldlink) { 841 new_state = 1; 842 priv->oldlink = 1; 843 } 844 } else if (priv->oldlink) { 845 new_state = 1; 846 priv->oldlink = 0; 847 priv->speed = SPEED_UNKNOWN; 848 priv->oldduplex = DUPLEX_UNKNOWN; 849 } 850 851 if (new_state && netif_msg_link(priv)) 852 phy_print_status(phydev); 853 854 spin_unlock_irqrestore(&priv->lock, flags); 855 856 if (phydev->is_pseudo_fixed_link) 857 /* Stop PHY layer to call the hook to adjust the link in case 858 * of a switch is attached to the stmmac driver. 859 */ 860 phydev->irq = PHY_IGNORE_INTERRUPT; 861 else 862 /* At this stage, init the EEE if supported. 863 * Never called in case of fixed_link. 864 */ 865 priv->eee_enabled = stmmac_eee_init(priv); 866 } 867 868 /** 869 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 870 * @priv: driver private structure 871 * Description: this is to verify if the HW supports the PCS. 872 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 873 * configured for the TBI, RTBI, or SGMII PHY interface. 874 */ 875 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 876 { 877 int interface = priv->plat->interface; 878 879 if (priv->dma_cap.pcs) { 880 if ((interface == PHY_INTERFACE_MODE_RGMII) || 881 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 882 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 883 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 884 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 885 priv->hw->pcs = STMMAC_PCS_RGMII; 886 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 887 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 888 priv->hw->pcs = STMMAC_PCS_SGMII; 889 } 890 } 891 } 892 893 /** 894 * stmmac_init_phy - PHY initialization 895 * @dev: net device structure 896 * Description: it initializes the driver's PHY state, and attaches the PHY 897 * to the mac driver. 898 * Return value: 899 * 0 on success 900 */ 901 static int stmmac_init_phy(struct net_device *dev) 902 { 903 struct stmmac_priv *priv = netdev_priv(dev); 904 struct phy_device *phydev; 905 char phy_id_fmt[MII_BUS_ID_SIZE + 3]; 906 char bus_id[MII_BUS_ID_SIZE]; 907 int interface = priv->plat->interface; 908 int max_speed = priv->plat->max_speed; 909 priv->oldlink = 0; 910 priv->speed = SPEED_UNKNOWN; 911 priv->oldduplex = DUPLEX_UNKNOWN; 912 913 if (priv->plat->phy_node) { 914 phydev = of_phy_connect(dev, priv->plat->phy_node, 915 &stmmac_adjust_link, 0, interface); 916 } else { 917 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x", 918 priv->plat->bus_id); 919 920 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id, 921 priv->plat->phy_addr); 922 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__, 923 phy_id_fmt); 924 925 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link, 926 interface); 927 } 928 929 if (IS_ERR_OR_NULL(phydev)) { 930 netdev_err(priv->dev, "Could not attach to PHY\n"); 931 if (!phydev) 932 return -ENODEV; 933 934 return PTR_ERR(phydev); 935 } 936 937 /* Stop Advertising 1000BASE Capability if interface is not GMII */ 938 if ((interface == PHY_INTERFACE_MODE_MII) || 939 (interface == PHY_INTERFACE_MODE_RMII) || 940 (max_speed < 1000 && max_speed > 0)) 941 phydev->advertising &= ~(SUPPORTED_1000baseT_Half | 942 SUPPORTED_1000baseT_Full); 943 944 /* 945 * Broken HW is sometimes missing the pull-up resistor on the 946 * MDIO line, which results in reads to non-existent devices returning 947 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent 948 * device as well. 949 * Note: phydev->phy_id is the result of reading the UID PHY registers. 950 */ 951 if (!priv->plat->phy_node && phydev->phy_id == 0) { 952 phy_disconnect(phydev); 953 return -ENODEV; 954 } 955 956 /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid 957 * subsequent PHY polling, make sure we force a link transition if 958 * we have a UP/DOWN/UP transition 959 */ 960 if (phydev->is_pseudo_fixed_link) 961 phydev->irq = PHY_POLL; 962 963 phy_attached_info(phydev); 964 return 0; 965 } 966 967 static void stmmac_display_rx_rings(struct stmmac_priv *priv) 968 { 969 u32 rx_cnt = priv->plat->rx_queues_to_use; 970 void *head_rx; 971 u32 queue; 972 973 /* Display RX rings */ 974 for (queue = 0; queue < rx_cnt; queue++) { 975 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 976 977 pr_info("\tRX Queue %u rings\n", queue); 978 979 if (priv->extend_desc) 980 head_rx = (void *)rx_q->dma_erx; 981 else 982 head_rx = (void *)rx_q->dma_rx; 983 984 /* Display RX ring */ 985 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true); 986 } 987 } 988 989 static void stmmac_display_tx_rings(struct stmmac_priv *priv) 990 { 991 u32 tx_cnt = priv->plat->tx_queues_to_use; 992 void *head_tx; 993 u32 queue; 994 995 /* Display TX rings */ 996 for (queue = 0; queue < tx_cnt; queue++) { 997 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 998 999 pr_info("\tTX Queue %d rings\n", queue); 1000 1001 if (priv->extend_desc) 1002 head_tx = (void *)tx_q->dma_etx; 1003 else 1004 head_tx = (void *)tx_q->dma_tx; 1005 1006 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false); 1007 } 1008 } 1009 1010 static void stmmac_display_rings(struct stmmac_priv *priv) 1011 { 1012 /* Display RX ring */ 1013 stmmac_display_rx_rings(priv); 1014 1015 /* Display TX ring */ 1016 stmmac_display_tx_rings(priv); 1017 } 1018 1019 static int stmmac_set_bfsize(int mtu, int bufsize) 1020 { 1021 int ret = bufsize; 1022 1023 if (mtu >= BUF_SIZE_4KiB) 1024 ret = BUF_SIZE_8KiB; 1025 else if (mtu >= BUF_SIZE_2KiB) 1026 ret = BUF_SIZE_4KiB; 1027 else if (mtu > DEFAULT_BUFSIZE) 1028 ret = BUF_SIZE_2KiB; 1029 else 1030 ret = DEFAULT_BUFSIZE; 1031 1032 return ret; 1033 } 1034 1035 /** 1036 * stmmac_clear_rx_descriptors - clear RX descriptors 1037 * @priv: driver private structure 1038 * @queue: RX queue index 1039 * Description: this function is called to clear the RX descriptors 1040 * in case of both basic and extended descriptors are used. 1041 */ 1042 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) 1043 { 1044 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1045 int i; 1046 1047 /* Clear the RX descriptors */ 1048 for (i = 0; i < DMA_RX_SIZE; i++) 1049 if (priv->extend_desc) 1050 priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic, 1051 priv->use_riwt, priv->mode, 1052 (i == DMA_RX_SIZE - 1)); 1053 else 1054 priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i], 1055 priv->use_riwt, priv->mode, 1056 (i == DMA_RX_SIZE - 1)); 1057 } 1058 1059 /** 1060 * stmmac_clear_tx_descriptors - clear tx descriptors 1061 * @priv: driver private structure 1062 * @queue: TX queue index. 1063 * Description: this function is called to clear the TX descriptors 1064 * in case of both basic and extended descriptors are used. 1065 */ 1066 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) 1067 { 1068 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1069 int i; 1070 1071 /* Clear the TX descriptors */ 1072 for (i = 0; i < DMA_TX_SIZE; i++) 1073 if (priv->extend_desc) 1074 priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic, 1075 priv->mode, 1076 (i == DMA_TX_SIZE - 1)); 1077 else 1078 priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i], 1079 priv->mode, 1080 (i == DMA_TX_SIZE - 1)); 1081 } 1082 1083 /** 1084 * stmmac_clear_descriptors - clear descriptors 1085 * @priv: driver private structure 1086 * Description: this function is called to clear the TX and RX descriptors 1087 * in case of both basic and extended descriptors are used. 1088 */ 1089 static void stmmac_clear_descriptors(struct stmmac_priv *priv) 1090 { 1091 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1092 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1093 u32 queue; 1094 1095 /* Clear the RX descriptors */ 1096 for (queue = 0; queue < rx_queue_cnt; queue++) 1097 stmmac_clear_rx_descriptors(priv, queue); 1098 1099 /* Clear the TX descriptors */ 1100 for (queue = 0; queue < tx_queue_cnt; queue++) 1101 stmmac_clear_tx_descriptors(priv, queue); 1102 } 1103 1104 /** 1105 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1106 * @priv: driver private structure 1107 * @p: descriptor pointer 1108 * @i: descriptor index 1109 * @flags: gfp flag 1110 * @queue: RX queue index 1111 * Description: this function is called to allocate a receive buffer, perform 1112 * the DMA mapping and init the descriptor. 1113 */ 1114 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, 1115 int i, gfp_t flags, u32 queue) 1116 { 1117 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1118 struct sk_buff *skb; 1119 1120 skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags); 1121 if (!skb) { 1122 netdev_err(priv->dev, 1123 "%s: Rx init fails; skb is NULL\n", __func__); 1124 return -ENOMEM; 1125 } 1126 rx_q->rx_skbuff[i] = skb; 1127 rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data, 1128 priv->dma_buf_sz, 1129 DMA_FROM_DEVICE); 1130 if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) { 1131 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__); 1132 dev_kfree_skb_any(skb); 1133 return -EINVAL; 1134 } 1135 1136 if (priv->synopsys_id >= DWMAC_CORE_4_00) 1137 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]); 1138 else 1139 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]); 1140 1141 if ((priv->hw->mode->init_desc3) && 1142 (priv->dma_buf_sz == BUF_SIZE_16KiB)) 1143 priv->hw->mode->init_desc3(p); 1144 1145 return 0; 1146 } 1147 1148 /** 1149 * stmmac_free_rx_buffer - free RX dma buffers 1150 * @priv: private structure 1151 * @queue: RX queue index 1152 * @i: buffer index. 1153 */ 1154 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1155 { 1156 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1157 1158 if (rx_q->rx_skbuff[i]) { 1159 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i], 1160 priv->dma_buf_sz, DMA_FROM_DEVICE); 1161 dev_kfree_skb_any(rx_q->rx_skbuff[i]); 1162 } 1163 rx_q->rx_skbuff[i] = NULL; 1164 } 1165 1166 /** 1167 * stmmac_free_tx_buffer - free RX dma buffers 1168 * @priv: private structure 1169 * @queue: RX queue index 1170 * @i: buffer index. 1171 */ 1172 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1173 { 1174 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1175 1176 if (tx_q->tx_skbuff_dma[i].buf) { 1177 if (tx_q->tx_skbuff_dma[i].map_as_page) 1178 dma_unmap_page(priv->device, 1179 tx_q->tx_skbuff_dma[i].buf, 1180 tx_q->tx_skbuff_dma[i].len, 1181 DMA_TO_DEVICE); 1182 else 1183 dma_unmap_single(priv->device, 1184 tx_q->tx_skbuff_dma[i].buf, 1185 tx_q->tx_skbuff_dma[i].len, 1186 DMA_TO_DEVICE); 1187 } 1188 1189 if (tx_q->tx_skbuff[i]) { 1190 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1191 tx_q->tx_skbuff[i] = NULL; 1192 tx_q->tx_skbuff_dma[i].buf = 0; 1193 tx_q->tx_skbuff_dma[i].map_as_page = false; 1194 } 1195 } 1196 1197 /** 1198 * init_dma_rx_desc_rings - init the RX descriptor rings 1199 * @dev: net device structure 1200 * @flags: gfp flag. 1201 * Description: this function initializes the DMA RX descriptors 1202 * and allocates the socket buffers. It supports the chained and ring 1203 * modes. 1204 */ 1205 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) 1206 { 1207 struct stmmac_priv *priv = netdev_priv(dev); 1208 u32 rx_count = priv->plat->rx_queues_to_use; 1209 unsigned int bfsize = 0; 1210 int ret = -ENOMEM; 1211 u32 queue; 1212 int i; 1213 1214 if (priv->hw->mode->set_16kib_bfsize) 1215 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu); 1216 1217 if (bfsize < BUF_SIZE_16KiB) 1218 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); 1219 1220 priv->dma_buf_sz = bfsize; 1221 1222 /* RX INITIALIZATION */ 1223 netif_dbg(priv, probe, priv->dev, 1224 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1225 1226 for (queue = 0; queue < rx_count; queue++) { 1227 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1228 1229 netif_dbg(priv, probe, priv->dev, 1230 "(%s) dma_rx_phy=0x%08x\n", __func__, 1231 (u32)rx_q->dma_rx_phy); 1232 1233 for (i = 0; i < DMA_RX_SIZE; i++) { 1234 struct dma_desc *p; 1235 1236 if (priv->extend_desc) 1237 p = &((rx_q->dma_erx + i)->basic); 1238 else 1239 p = rx_q->dma_rx + i; 1240 1241 ret = stmmac_init_rx_buffers(priv, p, i, flags, 1242 queue); 1243 if (ret) 1244 goto err_init_rx_buffers; 1245 1246 netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n", 1247 rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data, 1248 (unsigned int)rx_q->rx_skbuff_dma[i]); 1249 } 1250 1251 rx_q->cur_rx = 0; 1252 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE); 1253 1254 stmmac_clear_rx_descriptors(priv, queue); 1255 1256 /* Setup the chained descriptor addresses */ 1257 if (priv->mode == STMMAC_CHAIN_MODE) { 1258 if (priv->extend_desc) 1259 priv->hw->mode->init(rx_q->dma_erx, 1260 rx_q->dma_rx_phy, 1261 DMA_RX_SIZE, 1); 1262 else 1263 priv->hw->mode->init(rx_q->dma_rx, 1264 rx_q->dma_rx_phy, 1265 DMA_RX_SIZE, 0); 1266 } 1267 } 1268 1269 buf_sz = bfsize; 1270 1271 return 0; 1272 1273 err_init_rx_buffers: 1274 while (queue >= 0) { 1275 while (--i >= 0) 1276 stmmac_free_rx_buffer(priv, queue, i); 1277 1278 if (queue == 0) 1279 break; 1280 1281 i = DMA_RX_SIZE; 1282 queue--; 1283 } 1284 1285 return ret; 1286 } 1287 1288 /** 1289 * init_dma_tx_desc_rings - init the TX descriptor rings 1290 * @dev: net device structure. 1291 * Description: this function initializes the DMA TX descriptors 1292 * and allocates the socket buffers. It supports the chained and ring 1293 * modes. 1294 */ 1295 static int init_dma_tx_desc_rings(struct net_device *dev) 1296 { 1297 struct stmmac_priv *priv = netdev_priv(dev); 1298 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1299 u32 queue; 1300 int i; 1301 1302 for (queue = 0; queue < tx_queue_cnt; queue++) { 1303 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1304 1305 netif_dbg(priv, probe, priv->dev, 1306 "(%s) dma_tx_phy=0x%08x\n", __func__, 1307 (u32)tx_q->dma_tx_phy); 1308 1309 /* Setup the chained descriptor addresses */ 1310 if (priv->mode == STMMAC_CHAIN_MODE) { 1311 if (priv->extend_desc) 1312 priv->hw->mode->init(tx_q->dma_etx, 1313 tx_q->dma_tx_phy, 1314 DMA_TX_SIZE, 1); 1315 else 1316 priv->hw->mode->init(tx_q->dma_tx, 1317 tx_q->dma_tx_phy, 1318 DMA_TX_SIZE, 0); 1319 } 1320 1321 for (i = 0; i < DMA_TX_SIZE; i++) { 1322 struct dma_desc *p; 1323 if (priv->extend_desc) 1324 p = &((tx_q->dma_etx + i)->basic); 1325 else 1326 p = tx_q->dma_tx + i; 1327 1328 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1329 p->des0 = 0; 1330 p->des1 = 0; 1331 p->des2 = 0; 1332 p->des3 = 0; 1333 } else { 1334 p->des2 = 0; 1335 } 1336 1337 tx_q->tx_skbuff_dma[i].buf = 0; 1338 tx_q->tx_skbuff_dma[i].map_as_page = false; 1339 tx_q->tx_skbuff_dma[i].len = 0; 1340 tx_q->tx_skbuff_dma[i].last_segment = false; 1341 tx_q->tx_skbuff[i] = NULL; 1342 } 1343 1344 tx_q->dirty_tx = 0; 1345 tx_q->cur_tx = 0; 1346 1347 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 1348 } 1349 1350 return 0; 1351 } 1352 1353 /** 1354 * init_dma_desc_rings - init the RX/TX descriptor rings 1355 * @dev: net device structure 1356 * @flags: gfp flag. 1357 * Description: this function initializes the DMA RX/TX descriptors 1358 * and allocates the socket buffers. It supports the chained and ring 1359 * modes. 1360 */ 1361 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags) 1362 { 1363 struct stmmac_priv *priv = netdev_priv(dev); 1364 int ret; 1365 1366 ret = init_dma_rx_desc_rings(dev, flags); 1367 if (ret) 1368 return ret; 1369 1370 ret = init_dma_tx_desc_rings(dev); 1371 1372 stmmac_clear_descriptors(priv); 1373 1374 if (netif_msg_hw(priv)) 1375 stmmac_display_rings(priv); 1376 1377 return ret; 1378 } 1379 1380 /** 1381 * dma_free_rx_skbufs - free RX dma buffers 1382 * @priv: private structure 1383 * @queue: RX queue index 1384 */ 1385 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) 1386 { 1387 int i; 1388 1389 for (i = 0; i < DMA_RX_SIZE; i++) 1390 stmmac_free_rx_buffer(priv, queue, i); 1391 } 1392 1393 /** 1394 * dma_free_tx_skbufs - free TX dma buffers 1395 * @priv: private structure 1396 * @queue: TX queue index 1397 */ 1398 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) 1399 { 1400 int i; 1401 1402 for (i = 0; i < DMA_TX_SIZE; i++) 1403 stmmac_free_tx_buffer(priv, queue, i); 1404 } 1405 1406 /** 1407 * free_dma_rx_desc_resources - free RX dma desc resources 1408 * @priv: private structure 1409 */ 1410 static void free_dma_rx_desc_resources(struct stmmac_priv *priv) 1411 { 1412 u32 rx_count = priv->plat->rx_queues_to_use; 1413 u32 queue; 1414 1415 /* Free RX queue resources */ 1416 for (queue = 0; queue < rx_count; queue++) { 1417 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1418 1419 /* Release the DMA RX socket buffers */ 1420 dma_free_rx_skbufs(priv, queue); 1421 1422 /* Free DMA regions of consistent memory previously allocated */ 1423 if (!priv->extend_desc) 1424 dma_free_coherent(priv->device, 1425 DMA_RX_SIZE * sizeof(struct dma_desc), 1426 rx_q->dma_rx, rx_q->dma_rx_phy); 1427 else 1428 dma_free_coherent(priv->device, DMA_RX_SIZE * 1429 sizeof(struct dma_extended_desc), 1430 rx_q->dma_erx, rx_q->dma_rx_phy); 1431 1432 kfree(rx_q->rx_skbuff_dma); 1433 kfree(rx_q->rx_skbuff); 1434 } 1435 } 1436 1437 /** 1438 * free_dma_tx_desc_resources - free TX dma desc resources 1439 * @priv: private structure 1440 */ 1441 static void free_dma_tx_desc_resources(struct stmmac_priv *priv) 1442 { 1443 u32 tx_count = priv->plat->tx_queues_to_use; 1444 u32 queue = 0; 1445 1446 /* Free TX queue resources */ 1447 for (queue = 0; queue < tx_count; queue++) { 1448 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1449 1450 /* Release the DMA TX socket buffers */ 1451 dma_free_tx_skbufs(priv, queue); 1452 1453 /* Free DMA regions of consistent memory previously allocated */ 1454 if (!priv->extend_desc) 1455 dma_free_coherent(priv->device, 1456 DMA_TX_SIZE * sizeof(struct dma_desc), 1457 tx_q->dma_tx, tx_q->dma_tx_phy); 1458 else 1459 dma_free_coherent(priv->device, DMA_TX_SIZE * 1460 sizeof(struct dma_extended_desc), 1461 tx_q->dma_etx, tx_q->dma_tx_phy); 1462 1463 kfree(tx_q->tx_skbuff_dma); 1464 kfree(tx_q->tx_skbuff); 1465 } 1466 } 1467 1468 /** 1469 * alloc_dma_rx_desc_resources - alloc RX resources. 1470 * @priv: private structure 1471 * Description: according to which descriptor can be used (extend or basic) 1472 * this function allocates the resources for TX and RX paths. In case of 1473 * reception, for example, it pre-allocated the RX socket buffer in order to 1474 * allow zero-copy mechanism. 1475 */ 1476 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) 1477 { 1478 u32 rx_count = priv->plat->rx_queues_to_use; 1479 int ret = -ENOMEM; 1480 u32 queue; 1481 1482 /* RX queues buffers and DMA */ 1483 for (queue = 0; queue < rx_count; queue++) { 1484 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1485 1486 rx_q->queue_index = queue; 1487 rx_q->priv_data = priv; 1488 1489 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE, 1490 sizeof(dma_addr_t), 1491 GFP_KERNEL); 1492 if (!rx_q->rx_skbuff_dma) 1493 return -ENOMEM; 1494 1495 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE, 1496 sizeof(struct sk_buff *), 1497 GFP_KERNEL); 1498 if (!rx_q->rx_skbuff) 1499 goto err_dma; 1500 1501 if (priv->extend_desc) { 1502 rx_q->dma_erx = dma_zalloc_coherent(priv->device, 1503 DMA_RX_SIZE * 1504 sizeof(struct 1505 dma_extended_desc), 1506 &rx_q->dma_rx_phy, 1507 GFP_KERNEL); 1508 if (!rx_q->dma_erx) 1509 goto err_dma; 1510 1511 } else { 1512 rx_q->dma_rx = dma_zalloc_coherent(priv->device, 1513 DMA_RX_SIZE * 1514 sizeof(struct 1515 dma_desc), 1516 &rx_q->dma_rx_phy, 1517 GFP_KERNEL); 1518 if (!rx_q->dma_rx) 1519 goto err_dma; 1520 } 1521 } 1522 1523 return 0; 1524 1525 err_dma: 1526 free_dma_rx_desc_resources(priv); 1527 1528 return ret; 1529 } 1530 1531 /** 1532 * alloc_dma_tx_desc_resources - alloc TX resources. 1533 * @priv: private structure 1534 * Description: according to which descriptor can be used (extend or basic) 1535 * this function allocates the resources for TX and RX paths. In case of 1536 * reception, for example, it pre-allocated the RX socket buffer in order to 1537 * allow zero-copy mechanism. 1538 */ 1539 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) 1540 { 1541 u32 tx_count = priv->plat->tx_queues_to_use; 1542 int ret = -ENOMEM; 1543 u32 queue; 1544 1545 /* TX queues buffers and DMA */ 1546 for (queue = 0; queue < tx_count; queue++) { 1547 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1548 1549 tx_q->queue_index = queue; 1550 tx_q->priv_data = priv; 1551 1552 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE, 1553 sizeof(*tx_q->tx_skbuff_dma), 1554 GFP_KERNEL); 1555 if (!tx_q->tx_skbuff_dma) 1556 return -ENOMEM; 1557 1558 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE, 1559 sizeof(struct sk_buff *), 1560 GFP_KERNEL); 1561 if (!tx_q->tx_skbuff) 1562 goto err_dma_buffers; 1563 1564 if (priv->extend_desc) { 1565 tx_q->dma_etx = dma_zalloc_coherent(priv->device, 1566 DMA_TX_SIZE * 1567 sizeof(struct 1568 dma_extended_desc), 1569 &tx_q->dma_tx_phy, 1570 GFP_KERNEL); 1571 if (!tx_q->dma_etx) 1572 goto err_dma_buffers; 1573 } else { 1574 tx_q->dma_tx = dma_zalloc_coherent(priv->device, 1575 DMA_TX_SIZE * 1576 sizeof(struct 1577 dma_desc), 1578 &tx_q->dma_tx_phy, 1579 GFP_KERNEL); 1580 if (!tx_q->dma_tx) 1581 goto err_dma_buffers; 1582 } 1583 } 1584 1585 return 0; 1586 1587 err_dma_buffers: 1588 free_dma_tx_desc_resources(priv); 1589 1590 return ret; 1591 } 1592 1593 /** 1594 * alloc_dma_desc_resources - alloc TX/RX resources. 1595 * @priv: private structure 1596 * Description: according to which descriptor can be used (extend or basic) 1597 * this function allocates the resources for TX and RX paths. In case of 1598 * reception, for example, it pre-allocated the RX socket buffer in order to 1599 * allow zero-copy mechanism. 1600 */ 1601 static int alloc_dma_desc_resources(struct stmmac_priv *priv) 1602 { 1603 /* RX Allocation */ 1604 int ret = alloc_dma_rx_desc_resources(priv); 1605 1606 if (ret) 1607 return ret; 1608 1609 ret = alloc_dma_tx_desc_resources(priv); 1610 1611 return ret; 1612 } 1613 1614 /** 1615 * free_dma_desc_resources - free dma desc resources 1616 * @priv: private structure 1617 */ 1618 static void free_dma_desc_resources(struct stmmac_priv *priv) 1619 { 1620 /* Release the DMA RX socket buffers */ 1621 free_dma_rx_desc_resources(priv); 1622 1623 /* Release the DMA TX socket buffers */ 1624 free_dma_tx_desc_resources(priv); 1625 } 1626 1627 /** 1628 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 1629 * @priv: driver private structure 1630 * Description: It is used for enabling the rx queues in the MAC 1631 */ 1632 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 1633 { 1634 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1635 int queue; 1636 u8 mode; 1637 1638 for (queue = 0; queue < rx_queues_count; queue++) { 1639 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 1640 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue); 1641 } 1642 } 1643 1644 /** 1645 * stmmac_start_rx_dma - start RX DMA channel 1646 * @priv: driver private structure 1647 * @chan: RX channel index 1648 * Description: 1649 * This starts a RX DMA channel 1650 */ 1651 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 1652 { 1653 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 1654 priv->hw->dma->start_rx(priv->ioaddr, chan); 1655 } 1656 1657 /** 1658 * stmmac_start_tx_dma - start TX DMA channel 1659 * @priv: driver private structure 1660 * @chan: TX channel index 1661 * Description: 1662 * This starts a TX DMA channel 1663 */ 1664 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 1665 { 1666 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 1667 priv->hw->dma->start_tx(priv->ioaddr, chan); 1668 } 1669 1670 /** 1671 * stmmac_stop_rx_dma - stop RX DMA channel 1672 * @priv: driver private structure 1673 * @chan: RX channel index 1674 * Description: 1675 * This stops a RX DMA channel 1676 */ 1677 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 1678 { 1679 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 1680 priv->hw->dma->stop_rx(priv->ioaddr, chan); 1681 } 1682 1683 /** 1684 * stmmac_stop_tx_dma - stop TX DMA channel 1685 * @priv: driver private structure 1686 * @chan: TX channel index 1687 * Description: 1688 * This stops a TX DMA channel 1689 */ 1690 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 1691 { 1692 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 1693 priv->hw->dma->stop_tx(priv->ioaddr, chan); 1694 } 1695 1696 /** 1697 * stmmac_start_all_dma - start all RX and TX DMA channels 1698 * @priv: driver private structure 1699 * Description: 1700 * This starts all the RX and TX DMA channels 1701 */ 1702 static void stmmac_start_all_dma(struct stmmac_priv *priv) 1703 { 1704 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1705 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1706 u32 chan = 0; 1707 1708 for (chan = 0; chan < rx_channels_count; chan++) 1709 stmmac_start_rx_dma(priv, chan); 1710 1711 for (chan = 0; chan < tx_channels_count; chan++) 1712 stmmac_start_tx_dma(priv, chan); 1713 } 1714 1715 /** 1716 * stmmac_stop_all_dma - stop all RX and TX DMA channels 1717 * @priv: driver private structure 1718 * Description: 1719 * This stops the RX and TX DMA channels 1720 */ 1721 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 1722 { 1723 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1724 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1725 u32 chan = 0; 1726 1727 for (chan = 0; chan < rx_channels_count; chan++) 1728 stmmac_stop_rx_dma(priv, chan); 1729 1730 for (chan = 0; chan < tx_channels_count; chan++) 1731 stmmac_stop_tx_dma(priv, chan); 1732 } 1733 1734 /** 1735 * stmmac_dma_operation_mode - HW DMA operation mode 1736 * @priv: driver private structure 1737 * Description: it is used for configuring the DMA operation mode register in 1738 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 1739 */ 1740 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 1741 { 1742 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1743 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1744 int rxfifosz = priv->plat->rx_fifo_size; 1745 u32 txmode = 0; 1746 u32 rxmode = 0; 1747 u32 chan = 0; 1748 1749 if (rxfifosz == 0) 1750 rxfifosz = priv->dma_cap.rx_fifo_size; 1751 1752 if (priv->plat->force_thresh_dma_mode) { 1753 txmode = tc; 1754 rxmode = tc; 1755 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 1756 /* 1757 * In case of GMAC, SF mode can be enabled 1758 * to perform the TX COE in HW. This depends on: 1759 * 1) TX COE if actually supported 1760 * 2) There is no bugged Jumbo frame support 1761 * that needs to not insert csum in the TDES. 1762 */ 1763 txmode = SF_DMA_MODE; 1764 rxmode = SF_DMA_MODE; 1765 priv->xstats.threshold = SF_DMA_MODE; 1766 } else { 1767 txmode = tc; 1768 rxmode = SF_DMA_MODE; 1769 } 1770 1771 /* configure all channels */ 1772 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1773 for (chan = 0; chan < rx_channels_count; chan++) 1774 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan, 1775 rxfifosz); 1776 1777 for (chan = 0; chan < tx_channels_count; chan++) 1778 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan); 1779 } else { 1780 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode, 1781 rxfifosz); 1782 } 1783 } 1784 1785 /** 1786 * stmmac_tx_clean - to manage the transmission completion 1787 * @priv: driver private structure 1788 * @queue: TX queue index 1789 * Description: it reclaims the transmit resources after transmission completes. 1790 */ 1791 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue) 1792 { 1793 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1794 unsigned int bytes_compl = 0, pkts_compl = 0; 1795 unsigned int entry = tx_q->dirty_tx; 1796 1797 netif_tx_lock(priv->dev); 1798 1799 priv->xstats.tx_clean++; 1800 1801 while (entry != tx_q->cur_tx) { 1802 struct sk_buff *skb = tx_q->tx_skbuff[entry]; 1803 struct dma_desc *p; 1804 int status; 1805 1806 if (priv->extend_desc) 1807 p = (struct dma_desc *)(tx_q->dma_etx + entry); 1808 else 1809 p = tx_q->dma_tx + entry; 1810 1811 status = priv->hw->desc->tx_status(&priv->dev->stats, 1812 &priv->xstats, p, 1813 priv->ioaddr); 1814 /* Check if the descriptor is owned by the DMA */ 1815 if (unlikely(status & tx_dma_own)) 1816 break; 1817 1818 /* Just consider the last segment and ...*/ 1819 if (likely(!(status & tx_not_ls))) { 1820 /* ... verify the status error condition */ 1821 if (unlikely(status & tx_err)) { 1822 priv->dev->stats.tx_errors++; 1823 } else { 1824 priv->dev->stats.tx_packets++; 1825 priv->xstats.tx_pkt_n++; 1826 } 1827 stmmac_get_tx_hwtstamp(priv, p, skb); 1828 } 1829 1830 if (likely(tx_q->tx_skbuff_dma[entry].buf)) { 1831 if (tx_q->tx_skbuff_dma[entry].map_as_page) 1832 dma_unmap_page(priv->device, 1833 tx_q->tx_skbuff_dma[entry].buf, 1834 tx_q->tx_skbuff_dma[entry].len, 1835 DMA_TO_DEVICE); 1836 else 1837 dma_unmap_single(priv->device, 1838 tx_q->tx_skbuff_dma[entry].buf, 1839 tx_q->tx_skbuff_dma[entry].len, 1840 DMA_TO_DEVICE); 1841 tx_q->tx_skbuff_dma[entry].buf = 0; 1842 tx_q->tx_skbuff_dma[entry].len = 0; 1843 tx_q->tx_skbuff_dma[entry].map_as_page = false; 1844 } 1845 1846 if (priv->hw->mode->clean_desc3) 1847 priv->hw->mode->clean_desc3(tx_q, p); 1848 1849 tx_q->tx_skbuff_dma[entry].last_segment = false; 1850 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1851 1852 if (likely(skb != NULL)) { 1853 pkts_compl++; 1854 bytes_compl += skb->len; 1855 dev_consume_skb_any(skb); 1856 tx_q->tx_skbuff[entry] = NULL; 1857 } 1858 1859 priv->hw->desc->release_tx_desc(p, priv->mode); 1860 1861 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 1862 } 1863 tx_q->dirty_tx = entry; 1864 1865 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 1866 pkts_compl, bytes_compl); 1867 1868 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 1869 queue))) && 1870 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) { 1871 1872 netif_dbg(priv, tx_done, priv->dev, 1873 "%s: restart transmit\n", __func__); 1874 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 1875 } 1876 1877 if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) { 1878 stmmac_enable_eee_mode(priv); 1879 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 1880 } 1881 netif_tx_unlock(priv->dev); 1882 } 1883 1884 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan) 1885 { 1886 priv->hw->dma->enable_dma_irq(priv->ioaddr, chan); 1887 } 1888 1889 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan) 1890 { 1891 priv->hw->dma->disable_dma_irq(priv->ioaddr, chan); 1892 } 1893 1894 /** 1895 * stmmac_tx_err - to manage the tx error 1896 * @priv: driver private structure 1897 * @chan: channel index 1898 * Description: it cleans the descriptors and restarts the transmission 1899 * in case of transmission errors. 1900 */ 1901 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 1902 { 1903 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 1904 int i; 1905 1906 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 1907 1908 stmmac_stop_tx_dma(priv, chan); 1909 dma_free_tx_skbufs(priv, chan); 1910 for (i = 0; i < DMA_TX_SIZE; i++) 1911 if (priv->extend_desc) 1912 priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic, 1913 priv->mode, 1914 (i == DMA_TX_SIZE - 1)); 1915 else 1916 priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i], 1917 priv->mode, 1918 (i == DMA_TX_SIZE - 1)); 1919 tx_q->dirty_tx = 0; 1920 tx_q->cur_tx = 0; 1921 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); 1922 stmmac_start_tx_dma(priv, chan); 1923 1924 priv->dev->stats.tx_errors++; 1925 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 1926 } 1927 1928 /** 1929 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 1930 * @priv: driver private structure 1931 * @txmode: TX operating mode 1932 * @rxmode: RX operating mode 1933 * @chan: channel index 1934 * Description: it is used for configuring of the DMA operation mode in 1935 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 1936 * mode. 1937 */ 1938 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 1939 u32 rxmode, u32 chan) 1940 { 1941 int rxfifosz = priv->plat->rx_fifo_size; 1942 1943 if (rxfifosz == 0) 1944 rxfifosz = priv->dma_cap.rx_fifo_size; 1945 1946 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1947 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan, 1948 rxfifosz); 1949 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan); 1950 } else { 1951 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode, 1952 rxfifosz); 1953 } 1954 } 1955 1956 /** 1957 * stmmac_dma_interrupt - DMA ISR 1958 * @priv: driver private structure 1959 * Description: this is the DMA ISR. It is called by the main ISR. 1960 * It calls the dwmac dma routine and schedule poll method in case of some 1961 * work can be done. 1962 */ 1963 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 1964 { 1965 u32 tx_channel_count = priv->plat->tx_queues_to_use; 1966 int status; 1967 u32 chan; 1968 1969 for (chan = 0; chan < tx_channel_count; chan++) { 1970 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; 1971 1972 status = priv->hw->dma->dma_interrupt(priv->ioaddr, 1973 &priv->xstats, chan); 1974 if (likely((status & handle_rx)) || (status & handle_tx)) { 1975 if (likely(napi_schedule_prep(&rx_q->napi))) { 1976 stmmac_disable_dma_irq(priv, chan); 1977 __napi_schedule(&rx_q->napi); 1978 } 1979 } 1980 1981 if (unlikely(status & tx_hard_error_bump_tc)) { 1982 /* Try to bump up the dma threshold on this failure */ 1983 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && 1984 (tc <= 256)) { 1985 tc += 64; 1986 if (priv->plat->force_thresh_dma_mode) 1987 stmmac_set_dma_operation_mode(priv, 1988 tc, 1989 tc, 1990 chan); 1991 else 1992 stmmac_set_dma_operation_mode(priv, 1993 tc, 1994 SF_DMA_MODE, 1995 chan); 1996 priv->xstats.threshold = tc; 1997 } 1998 } else if (unlikely(status == tx_hard_error)) { 1999 stmmac_tx_err(priv, chan); 2000 } 2001 } 2002 } 2003 2004 /** 2005 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2006 * @priv: driver private structure 2007 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2008 */ 2009 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2010 { 2011 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2012 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2013 2014 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 2015 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET; 2016 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET; 2017 } else { 2018 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET; 2019 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET; 2020 } 2021 2022 dwmac_mmc_intr_all_mask(priv->mmcaddr); 2023 2024 if (priv->dma_cap.rmon) { 2025 dwmac_mmc_ctrl(priv->mmcaddr, mode); 2026 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2027 } else 2028 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2029 } 2030 2031 /** 2032 * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors 2033 * @priv: driver private structure 2034 * Description: select the Enhanced/Alternate or Normal descriptors. 2035 * In case of Enhanced/Alternate, it checks if the extended descriptors are 2036 * supported by the HW capability register. 2037 */ 2038 static void stmmac_selec_desc_mode(struct stmmac_priv *priv) 2039 { 2040 if (priv->plat->enh_desc) { 2041 dev_info(priv->device, "Enhanced/Alternate descriptors\n"); 2042 2043 /* GMAC older than 3.50 has no extended descriptors */ 2044 if (priv->synopsys_id >= DWMAC_CORE_3_50) { 2045 dev_info(priv->device, "Enabled extended descriptors\n"); 2046 priv->extend_desc = 1; 2047 } else 2048 dev_warn(priv->device, "Extended descriptors not supported\n"); 2049 2050 priv->hw->desc = &enh_desc_ops; 2051 } else { 2052 dev_info(priv->device, "Normal descriptors\n"); 2053 priv->hw->desc = &ndesc_ops; 2054 } 2055 } 2056 2057 /** 2058 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2059 * @priv: driver private structure 2060 * Description: 2061 * new GMAC chip generations have a new register to indicate the 2062 * presence of the optional feature/functions. 2063 * This can be also used to override the value passed through the 2064 * platform and necessary for old MAC10/100 and GMAC chips. 2065 */ 2066 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2067 { 2068 u32 ret = 0; 2069 2070 if (priv->hw->dma->get_hw_feature) { 2071 priv->hw->dma->get_hw_feature(priv->ioaddr, 2072 &priv->dma_cap); 2073 ret = 1; 2074 } 2075 2076 return ret; 2077 } 2078 2079 /** 2080 * stmmac_check_ether_addr - check if the MAC addr is valid 2081 * @priv: driver private structure 2082 * Description: 2083 * it is to verify if the MAC address is valid, in case of failures it 2084 * generates a random MAC address 2085 */ 2086 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2087 { 2088 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2089 priv->hw->mac->get_umac_addr(priv->hw, 2090 priv->dev->dev_addr, 0); 2091 if (!is_valid_ether_addr(priv->dev->dev_addr)) 2092 eth_hw_addr_random(priv->dev); 2093 netdev_info(priv->dev, "device MAC address %pM\n", 2094 priv->dev->dev_addr); 2095 } 2096 } 2097 2098 /** 2099 * stmmac_init_dma_engine - DMA init. 2100 * @priv: driver private structure 2101 * Description: 2102 * It inits the DMA invoking the specific MAC/GMAC callback. 2103 * Some DMA parameters can be passed from the platform; 2104 * in case of these are not passed a default is kept for the MAC or GMAC. 2105 */ 2106 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2107 { 2108 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2109 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2110 struct stmmac_rx_queue *rx_q; 2111 struct stmmac_tx_queue *tx_q; 2112 u32 dummy_dma_rx_phy = 0; 2113 u32 dummy_dma_tx_phy = 0; 2114 u32 chan = 0; 2115 int atds = 0; 2116 int ret = 0; 2117 2118 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2119 dev_err(priv->device, "Invalid DMA configuration\n"); 2120 return -EINVAL; 2121 } 2122 2123 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2124 atds = 1; 2125 2126 ret = priv->hw->dma->reset(priv->ioaddr); 2127 if (ret) { 2128 dev_err(priv->device, "Failed to reset the dma\n"); 2129 return ret; 2130 } 2131 2132 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 2133 /* DMA Configuration */ 2134 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg, 2135 dummy_dma_tx_phy, dummy_dma_rx_phy, atds); 2136 2137 /* DMA RX Channel Configuration */ 2138 for (chan = 0; chan < rx_channels_count; chan++) { 2139 rx_q = &priv->rx_queue[chan]; 2140 2141 priv->hw->dma->init_rx_chan(priv->ioaddr, 2142 priv->plat->dma_cfg, 2143 rx_q->dma_rx_phy, chan); 2144 2145 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2146 (DMA_RX_SIZE * sizeof(struct dma_desc)); 2147 priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, 2148 rx_q->rx_tail_addr, 2149 chan); 2150 } 2151 2152 /* DMA TX Channel Configuration */ 2153 for (chan = 0; chan < tx_channels_count; chan++) { 2154 tx_q = &priv->tx_queue[chan]; 2155 2156 priv->hw->dma->init_chan(priv->ioaddr, 2157 priv->plat->dma_cfg, 2158 chan); 2159 2160 priv->hw->dma->init_tx_chan(priv->ioaddr, 2161 priv->plat->dma_cfg, 2162 tx_q->dma_tx_phy, chan); 2163 2164 tx_q->tx_tail_addr = tx_q->dma_tx_phy + 2165 (DMA_TX_SIZE * sizeof(struct dma_desc)); 2166 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, 2167 tx_q->tx_tail_addr, 2168 chan); 2169 } 2170 } else { 2171 rx_q = &priv->rx_queue[chan]; 2172 tx_q = &priv->tx_queue[chan]; 2173 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg, 2174 tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds); 2175 } 2176 2177 if (priv->plat->axi && priv->hw->dma->axi) 2178 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi); 2179 2180 return ret; 2181 } 2182 2183 /** 2184 * stmmac_tx_timer - mitigation sw timer for tx. 2185 * @data: data pointer 2186 * Description: 2187 * This is the timer handler to directly invoke the stmmac_tx_clean. 2188 */ 2189 static void stmmac_tx_timer(unsigned long data) 2190 { 2191 struct stmmac_priv *priv = (struct stmmac_priv *)data; 2192 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2193 u32 queue; 2194 2195 /* let's scan all the tx queues */ 2196 for (queue = 0; queue < tx_queues_count; queue++) 2197 stmmac_tx_clean(priv, queue); 2198 } 2199 2200 /** 2201 * stmmac_init_tx_coalesce - init tx mitigation options. 2202 * @priv: driver private structure 2203 * Description: 2204 * This inits the transmit coalesce parameters: i.e. timer rate, 2205 * timer handler and default threshold used for enabling the 2206 * interrupt on completion bit. 2207 */ 2208 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv) 2209 { 2210 priv->tx_coal_frames = STMMAC_TX_FRAMES; 2211 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER; 2212 init_timer(&priv->txtimer); 2213 priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer); 2214 priv->txtimer.data = (unsigned long)priv; 2215 priv->txtimer.function = stmmac_tx_timer; 2216 add_timer(&priv->txtimer); 2217 } 2218 2219 static void stmmac_set_rings_length(struct stmmac_priv *priv) 2220 { 2221 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2222 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2223 u32 chan; 2224 2225 /* set TX ring length */ 2226 if (priv->hw->dma->set_tx_ring_len) { 2227 for (chan = 0; chan < tx_channels_count; chan++) 2228 priv->hw->dma->set_tx_ring_len(priv->ioaddr, 2229 (DMA_TX_SIZE - 1), chan); 2230 } 2231 2232 /* set RX ring length */ 2233 if (priv->hw->dma->set_rx_ring_len) { 2234 for (chan = 0; chan < rx_channels_count; chan++) 2235 priv->hw->dma->set_rx_ring_len(priv->ioaddr, 2236 (DMA_RX_SIZE - 1), chan); 2237 } 2238 } 2239 2240 /** 2241 * stmmac_set_tx_queue_weight - Set TX queue weight 2242 * @priv: driver private structure 2243 * Description: It is used for setting TX queues weight 2244 */ 2245 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 2246 { 2247 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2248 u32 weight; 2249 u32 queue; 2250 2251 for (queue = 0; queue < tx_queues_count; queue++) { 2252 weight = priv->plat->tx_queues_cfg[queue].weight; 2253 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue); 2254 } 2255 } 2256 2257 /** 2258 * stmmac_configure_cbs - Configure CBS in TX queue 2259 * @priv: driver private structure 2260 * Description: It is used for configuring CBS in AVB TX queues 2261 */ 2262 static void stmmac_configure_cbs(struct stmmac_priv *priv) 2263 { 2264 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2265 u32 mode_to_use; 2266 u32 queue; 2267 2268 /* queue 0 is reserved for legacy traffic */ 2269 for (queue = 1; queue < tx_queues_count; queue++) { 2270 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 2271 if (mode_to_use == MTL_QUEUE_DCB) 2272 continue; 2273 2274 priv->hw->mac->config_cbs(priv->hw, 2275 priv->plat->tx_queues_cfg[queue].send_slope, 2276 priv->plat->tx_queues_cfg[queue].idle_slope, 2277 priv->plat->tx_queues_cfg[queue].high_credit, 2278 priv->plat->tx_queues_cfg[queue].low_credit, 2279 queue); 2280 } 2281 } 2282 2283 /** 2284 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 2285 * @priv: driver private structure 2286 * Description: It is used for mapping RX queues to RX dma channels 2287 */ 2288 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 2289 { 2290 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2291 u32 queue; 2292 u32 chan; 2293 2294 for (queue = 0; queue < rx_queues_count; queue++) { 2295 chan = priv->plat->rx_queues_cfg[queue].chan; 2296 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan); 2297 } 2298 } 2299 2300 /** 2301 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 2302 * @priv: driver private structure 2303 * Description: It is used for configuring the RX Queue Priority 2304 */ 2305 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 2306 { 2307 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2308 u32 queue; 2309 u32 prio; 2310 2311 for (queue = 0; queue < rx_queues_count; queue++) { 2312 if (!priv->plat->rx_queues_cfg[queue].use_prio) 2313 continue; 2314 2315 prio = priv->plat->rx_queues_cfg[queue].prio; 2316 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue); 2317 } 2318 } 2319 2320 /** 2321 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 2322 * @priv: driver private structure 2323 * Description: It is used for configuring the TX Queue Priority 2324 */ 2325 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 2326 { 2327 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2328 u32 queue; 2329 u32 prio; 2330 2331 for (queue = 0; queue < tx_queues_count; queue++) { 2332 if (!priv->plat->tx_queues_cfg[queue].use_prio) 2333 continue; 2334 2335 prio = priv->plat->tx_queues_cfg[queue].prio; 2336 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue); 2337 } 2338 } 2339 2340 /** 2341 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 2342 * @priv: driver private structure 2343 * Description: It is used for configuring the RX queue routing 2344 */ 2345 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 2346 { 2347 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2348 u32 queue; 2349 u8 packet; 2350 2351 for (queue = 0; queue < rx_queues_count; queue++) { 2352 /* no specific packet type routing specified for the queue */ 2353 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 2354 continue; 2355 2356 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 2357 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue); 2358 } 2359 } 2360 2361 /** 2362 * stmmac_mtl_configuration - Configure MTL 2363 * @priv: driver private structure 2364 * Description: It is used for configurring MTL 2365 */ 2366 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 2367 { 2368 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2369 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2370 2371 if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight) 2372 stmmac_set_tx_queue_weight(priv); 2373 2374 /* Configure MTL RX algorithms */ 2375 if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms) 2376 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw, 2377 priv->plat->rx_sched_algorithm); 2378 2379 /* Configure MTL TX algorithms */ 2380 if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms) 2381 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw, 2382 priv->plat->tx_sched_algorithm); 2383 2384 /* Configure CBS in AVB TX queues */ 2385 if (tx_queues_count > 1 && priv->hw->mac->config_cbs) 2386 stmmac_configure_cbs(priv); 2387 2388 /* Map RX MTL to DMA channels */ 2389 if (priv->hw->mac->map_mtl_to_dma) 2390 stmmac_rx_queue_dma_chan_map(priv); 2391 2392 /* Enable MAC RX Queues */ 2393 if (priv->hw->mac->rx_queue_enable) 2394 stmmac_mac_enable_rx_queues(priv); 2395 2396 /* Set RX priorities */ 2397 if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio) 2398 stmmac_mac_config_rx_queues_prio(priv); 2399 2400 /* Set TX priorities */ 2401 if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio) 2402 stmmac_mac_config_tx_queues_prio(priv); 2403 2404 /* Set RX routing */ 2405 if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing) 2406 stmmac_mac_config_rx_queues_routing(priv); 2407 } 2408 2409 /** 2410 * stmmac_hw_setup - setup mac in a usable state. 2411 * @dev : pointer to the device structure. 2412 * Description: 2413 * this is the main function to setup the HW in a usable state because the 2414 * dma engine is reset, the core registers are configured (e.g. AXI, 2415 * Checksum features, timers). The DMA is ready to start receiving and 2416 * transmitting. 2417 * Return value: 2418 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2419 * file on failure. 2420 */ 2421 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) 2422 { 2423 struct stmmac_priv *priv = netdev_priv(dev); 2424 u32 rx_cnt = priv->plat->rx_queues_to_use; 2425 u32 tx_cnt = priv->plat->tx_queues_to_use; 2426 u32 chan; 2427 int ret; 2428 2429 /* DMA initialization and SW reset */ 2430 ret = stmmac_init_dma_engine(priv); 2431 if (ret < 0) { 2432 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 2433 __func__); 2434 return ret; 2435 } 2436 2437 /* Copy the MAC addr into the HW */ 2438 priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0); 2439 2440 /* PS and related bits will be programmed according to the speed */ 2441 if (priv->hw->pcs) { 2442 int speed = priv->plat->mac_port_sel_speed; 2443 2444 if ((speed == SPEED_10) || (speed == SPEED_100) || 2445 (speed == SPEED_1000)) { 2446 priv->hw->ps = speed; 2447 } else { 2448 dev_warn(priv->device, "invalid port speed\n"); 2449 priv->hw->ps = 0; 2450 } 2451 } 2452 2453 /* Initialize the MAC Core */ 2454 priv->hw->mac->core_init(priv->hw, dev->mtu); 2455 2456 /* Initialize MTL*/ 2457 if (priv->synopsys_id >= DWMAC_CORE_4_00) 2458 stmmac_mtl_configuration(priv); 2459 2460 ret = priv->hw->mac->rx_ipc(priv->hw); 2461 if (!ret) { 2462 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 2463 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 2464 priv->hw->rx_csum = 0; 2465 } 2466 2467 /* Enable the MAC Rx/Tx */ 2468 priv->hw->mac->set_mac(priv->ioaddr, true); 2469 2470 /* Set the HW DMA mode and the COE */ 2471 stmmac_dma_operation_mode(priv); 2472 2473 stmmac_mmc_setup(priv); 2474 2475 if (init_ptp) { 2476 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 2477 if (ret < 0) 2478 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret); 2479 2480 ret = stmmac_init_ptp(priv); 2481 if (ret == -EOPNOTSUPP) 2482 netdev_warn(priv->dev, "PTP not supported by HW\n"); 2483 else if (ret) 2484 netdev_warn(priv->dev, "PTP init failed\n"); 2485 } 2486 2487 #ifdef CONFIG_DEBUG_FS 2488 ret = stmmac_init_fs(dev); 2489 if (ret < 0) 2490 netdev_warn(priv->dev, "%s: failed debugFS registration\n", 2491 __func__); 2492 #endif 2493 /* Start the ball rolling... */ 2494 stmmac_start_all_dma(priv); 2495 2496 priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS; 2497 2498 if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) { 2499 priv->rx_riwt = MAX_DMA_RIWT; 2500 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt); 2501 } 2502 2503 if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane) 2504 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0); 2505 2506 /* set TX and RX rings length */ 2507 stmmac_set_rings_length(priv); 2508 2509 /* Enable TSO */ 2510 if (priv->tso) { 2511 for (chan = 0; chan < tx_cnt; chan++) 2512 priv->hw->dma->enable_tso(priv->ioaddr, 1, chan); 2513 } 2514 2515 return 0; 2516 } 2517 2518 static void stmmac_hw_teardown(struct net_device *dev) 2519 { 2520 struct stmmac_priv *priv = netdev_priv(dev); 2521 2522 clk_disable_unprepare(priv->plat->clk_ptp_ref); 2523 } 2524 2525 /** 2526 * stmmac_open - open entry point of the driver 2527 * @dev : pointer to the device structure. 2528 * Description: 2529 * This function is the open entry point of the driver. 2530 * Return value: 2531 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2532 * file on failure. 2533 */ 2534 static int stmmac_open(struct net_device *dev) 2535 { 2536 struct stmmac_priv *priv = netdev_priv(dev); 2537 int ret; 2538 2539 stmmac_check_ether_addr(priv); 2540 2541 if (priv->hw->pcs != STMMAC_PCS_RGMII && 2542 priv->hw->pcs != STMMAC_PCS_TBI && 2543 priv->hw->pcs != STMMAC_PCS_RTBI) { 2544 ret = stmmac_init_phy(dev); 2545 if (ret) { 2546 netdev_err(priv->dev, 2547 "%s: Cannot attach to PHY (error: %d)\n", 2548 __func__, ret); 2549 return ret; 2550 } 2551 } 2552 2553 /* Extra statistics */ 2554 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); 2555 priv->xstats.threshold = tc; 2556 2557 priv->dma_buf_sz = STMMAC_ALIGN(buf_sz); 2558 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 2559 2560 ret = alloc_dma_desc_resources(priv); 2561 if (ret < 0) { 2562 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 2563 __func__); 2564 goto dma_desc_error; 2565 } 2566 2567 ret = init_dma_desc_rings(dev, GFP_KERNEL); 2568 if (ret < 0) { 2569 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 2570 __func__); 2571 goto init_error; 2572 } 2573 2574 ret = stmmac_hw_setup(dev, true); 2575 if (ret < 0) { 2576 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 2577 goto init_error; 2578 } 2579 2580 stmmac_init_tx_coalesce(priv); 2581 2582 if (dev->phydev) 2583 phy_start(dev->phydev); 2584 2585 /* Request the IRQ lines */ 2586 ret = request_irq(dev->irq, stmmac_interrupt, 2587 IRQF_SHARED, dev->name, dev); 2588 if (unlikely(ret < 0)) { 2589 netdev_err(priv->dev, 2590 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 2591 __func__, dev->irq, ret); 2592 goto irq_error; 2593 } 2594 2595 /* Request the Wake IRQ in case of another line is used for WoL */ 2596 if (priv->wol_irq != dev->irq) { 2597 ret = request_irq(priv->wol_irq, stmmac_interrupt, 2598 IRQF_SHARED, dev->name, dev); 2599 if (unlikely(ret < 0)) { 2600 netdev_err(priv->dev, 2601 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 2602 __func__, priv->wol_irq, ret); 2603 goto wolirq_error; 2604 } 2605 } 2606 2607 /* Request the IRQ lines */ 2608 if (priv->lpi_irq > 0) { 2609 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED, 2610 dev->name, dev); 2611 if (unlikely(ret < 0)) { 2612 netdev_err(priv->dev, 2613 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 2614 __func__, priv->lpi_irq, ret); 2615 goto lpiirq_error; 2616 } 2617 } 2618 2619 stmmac_enable_all_queues(priv); 2620 stmmac_start_all_queues(priv); 2621 2622 return 0; 2623 2624 lpiirq_error: 2625 if (priv->wol_irq != dev->irq) 2626 free_irq(priv->wol_irq, dev); 2627 wolirq_error: 2628 free_irq(dev->irq, dev); 2629 irq_error: 2630 if (dev->phydev) 2631 phy_stop(dev->phydev); 2632 2633 del_timer_sync(&priv->txtimer); 2634 stmmac_hw_teardown(dev); 2635 init_error: 2636 free_dma_desc_resources(priv); 2637 dma_desc_error: 2638 if (dev->phydev) 2639 phy_disconnect(dev->phydev); 2640 2641 return ret; 2642 } 2643 2644 /** 2645 * stmmac_release - close entry point of the driver 2646 * @dev : device pointer. 2647 * Description: 2648 * This is the stop entry point of the driver. 2649 */ 2650 static int stmmac_release(struct net_device *dev) 2651 { 2652 struct stmmac_priv *priv = netdev_priv(dev); 2653 2654 if (priv->eee_enabled) 2655 del_timer_sync(&priv->eee_ctrl_timer); 2656 2657 /* Stop and disconnect the PHY */ 2658 if (dev->phydev) { 2659 phy_stop(dev->phydev); 2660 phy_disconnect(dev->phydev); 2661 } 2662 2663 stmmac_stop_all_queues(priv); 2664 2665 stmmac_disable_all_queues(priv); 2666 2667 del_timer_sync(&priv->txtimer); 2668 2669 /* Free the IRQ lines */ 2670 free_irq(dev->irq, dev); 2671 if (priv->wol_irq != dev->irq) 2672 free_irq(priv->wol_irq, dev); 2673 if (priv->lpi_irq > 0) 2674 free_irq(priv->lpi_irq, dev); 2675 2676 /* Stop TX/RX DMA and clear the descriptors */ 2677 stmmac_stop_all_dma(priv); 2678 2679 /* Release and free the Rx/Tx resources */ 2680 free_dma_desc_resources(priv); 2681 2682 /* Disable the MAC Rx/Tx */ 2683 priv->hw->mac->set_mac(priv->ioaddr, false); 2684 2685 netif_carrier_off(dev); 2686 2687 #ifdef CONFIG_DEBUG_FS 2688 stmmac_exit_fs(dev); 2689 #endif 2690 2691 stmmac_release_ptp(priv); 2692 2693 return 0; 2694 } 2695 2696 /** 2697 * stmmac_tso_allocator - close entry point of the driver 2698 * @priv: driver private structure 2699 * @des: buffer start address 2700 * @total_len: total length to fill in descriptors 2701 * @last_segmant: condition for the last descriptor 2702 * @queue: TX queue index 2703 * Description: 2704 * This function fills descriptor and request new descriptors according to 2705 * buffer length to fill 2706 */ 2707 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des, 2708 int total_len, bool last_segment, u32 queue) 2709 { 2710 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2711 struct dma_desc *desc; 2712 u32 buff_size; 2713 int tmp_len; 2714 2715 tmp_len = total_len; 2716 2717 while (tmp_len > 0) { 2718 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2719 desc = tx_q->dma_tx + tx_q->cur_tx; 2720 2721 desc->des0 = cpu_to_le32(des + (total_len - tmp_len)); 2722 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 2723 TSO_MAX_BUFF_SIZE : tmp_len; 2724 2725 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size, 2726 0, 1, 2727 (last_segment) && (buff_size < TSO_MAX_BUFF_SIZE), 2728 0, 0); 2729 2730 tmp_len -= TSO_MAX_BUFF_SIZE; 2731 } 2732 } 2733 2734 /** 2735 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 2736 * @skb : the socket buffer 2737 * @dev : device pointer 2738 * Description: this is the transmit function that is called on TSO frames 2739 * (support available on GMAC4 and newer chips). 2740 * Diagram below show the ring programming in case of TSO frames: 2741 * 2742 * First Descriptor 2743 * -------- 2744 * | DES0 |---> buffer1 = L2/L3/L4 header 2745 * | DES1 |---> TCP Payload (can continue on next descr...) 2746 * | DES2 |---> buffer 1 and 2 len 2747 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 2748 * -------- 2749 * | 2750 * ... 2751 * | 2752 * -------- 2753 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 2754 * | DES1 | --| 2755 * | DES2 | --> buffer 1 and 2 len 2756 * | DES3 | 2757 * -------- 2758 * 2759 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 2760 */ 2761 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 2762 { 2763 struct dma_desc *desc, *first, *mss_desc = NULL; 2764 struct stmmac_priv *priv = netdev_priv(dev); 2765 int nfrags = skb_shinfo(skb)->nr_frags; 2766 u32 queue = skb_get_queue_mapping(skb); 2767 unsigned int first_entry, des; 2768 struct stmmac_tx_queue *tx_q; 2769 int tmp_pay_len = 0; 2770 u32 pay_len, mss; 2771 u8 proto_hdr_len; 2772 int i; 2773 2774 tx_q = &priv->tx_queue[queue]; 2775 2776 /* Compute header lengths */ 2777 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2778 2779 /* Desc availability based on threshold should be enough safe */ 2780 if (unlikely(stmmac_tx_avail(priv, queue) < 2781 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 2782 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 2783 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 2784 queue)); 2785 /* This is a hard error, log it. */ 2786 netdev_err(priv->dev, 2787 "%s: Tx Ring full when queue awake\n", 2788 __func__); 2789 } 2790 return NETDEV_TX_BUSY; 2791 } 2792 2793 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 2794 2795 mss = skb_shinfo(skb)->gso_size; 2796 2797 /* set new MSS value if needed */ 2798 if (mss != priv->mss) { 2799 mss_desc = tx_q->dma_tx + tx_q->cur_tx; 2800 priv->hw->desc->set_mss(mss_desc, mss); 2801 priv->mss = mss; 2802 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2803 } 2804 2805 if (netif_msg_tx_queued(priv)) { 2806 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 2807 __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss); 2808 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 2809 skb->data_len); 2810 } 2811 2812 first_entry = tx_q->cur_tx; 2813 2814 desc = tx_q->dma_tx + first_entry; 2815 first = desc; 2816 2817 /* first descriptor: fill Headers on Buf1 */ 2818 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 2819 DMA_TO_DEVICE); 2820 if (dma_mapping_error(priv->device, des)) 2821 goto dma_map_err; 2822 2823 tx_q->tx_skbuff_dma[first_entry].buf = des; 2824 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 2825 tx_q->tx_skbuff[first_entry] = skb; 2826 2827 first->des0 = cpu_to_le32(des); 2828 2829 /* Fill start of payload in buff2 of first descriptor */ 2830 if (pay_len) 2831 first->des1 = cpu_to_le32(des + proto_hdr_len); 2832 2833 /* If needed take extra descriptors to fill the remaining payload */ 2834 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 2835 2836 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 2837 2838 /* Prepare fragments */ 2839 for (i = 0; i < nfrags; i++) { 2840 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2841 2842 des = skb_frag_dma_map(priv->device, frag, 0, 2843 skb_frag_size(frag), 2844 DMA_TO_DEVICE); 2845 if (dma_mapping_error(priv->device, des)) 2846 goto dma_map_err; 2847 2848 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 2849 (i == nfrags - 1), queue); 2850 2851 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 2852 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 2853 tx_q->tx_skbuff[tx_q->cur_tx] = NULL; 2854 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 2855 } 2856 2857 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 2858 2859 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2860 2861 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 2862 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 2863 __func__); 2864 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 2865 } 2866 2867 dev->stats.tx_bytes += skb->len; 2868 priv->xstats.tx_tso_frames++; 2869 priv->xstats.tx_tso_nfrags += nfrags; 2870 2871 /* Manage tx mitigation */ 2872 priv->tx_count_frames += nfrags + 1; 2873 if (likely(priv->tx_coal_frames > priv->tx_count_frames)) { 2874 mod_timer(&priv->txtimer, 2875 STMMAC_COAL_TIMER(priv->tx_coal_timer)); 2876 } else { 2877 priv->tx_count_frames = 0; 2878 priv->hw->desc->set_tx_ic(desc); 2879 priv->xstats.tx_set_ic_bit++; 2880 } 2881 2882 if (!priv->hwts_tx_en) 2883 skb_tx_timestamp(skb); 2884 2885 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 2886 priv->hwts_tx_en)) { 2887 /* declare that device is doing timestamping */ 2888 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2889 priv->hw->desc->enable_tx_timestamp(first); 2890 } 2891 2892 /* Complete the first descriptor before granting the DMA */ 2893 priv->hw->desc->prepare_tso_tx_desc(first, 1, 2894 proto_hdr_len, 2895 pay_len, 2896 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 2897 tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len)); 2898 2899 /* If context desc is used to change MSS */ 2900 if (mss_desc) 2901 priv->hw->desc->set_tx_owner(mss_desc); 2902 2903 /* The own bit must be the latest setting done when prepare the 2904 * descriptor and then barrier is needed to make sure that 2905 * all is coherent before granting the DMA engine. 2906 */ 2907 dma_wmb(); 2908 2909 if (netif_msg_pktdata(priv)) { 2910 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 2911 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 2912 tx_q->cur_tx, first, nfrags); 2913 2914 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE, 2915 0); 2916 2917 pr_info(">>> frame to be transmitted: "); 2918 print_pkt(skb->data, skb_headlen(skb)); 2919 } 2920 2921 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 2922 2923 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr, 2924 queue); 2925 2926 return NETDEV_TX_OK; 2927 2928 dma_map_err: 2929 dev_err(priv->device, "Tx dma map failed\n"); 2930 dev_kfree_skb(skb); 2931 priv->dev->stats.tx_dropped++; 2932 return NETDEV_TX_OK; 2933 } 2934 2935 /** 2936 * stmmac_xmit - Tx entry point of the driver 2937 * @skb : the socket buffer 2938 * @dev : device pointer 2939 * Description : this is the tx entry point of the driver. 2940 * It programs the chain or the ring and supports oversized frames 2941 * and SG feature. 2942 */ 2943 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 2944 { 2945 struct stmmac_priv *priv = netdev_priv(dev); 2946 unsigned int nopaged_len = skb_headlen(skb); 2947 int i, csum_insertion = 0, is_jumbo = 0; 2948 u32 queue = skb_get_queue_mapping(skb); 2949 int nfrags = skb_shinfo(skb)->nr_frags; 2950 unsigned int entry, first_entry; 2951 struct dma_desc *desc, *first; 2952 struct stmmac_tx_queue *tx_q; 2953 unsigned int enh_desc; 2954 unsigned int des; 2955 2956 tx_q = &priv->tx_queue[queue]; 2957 2958 /* Manage oversized TCP frames for GMAC4 device */ 2959 if (skb_is_gso(skb) && priv->tso) { 2960 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 2961 return stmmac_tso_xmit(skb, dev); 2962 } 2963 2964 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 2965 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 2966 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 2967 queue)); 2968 /* This is a hard error, log it. */ 2969 netdev_err(priv->dev, 2970 "%s: Tx Ring full when queue awake\n", 2971 __func__); 2972 } 2973 return NETDEV_TX_BUSY; 2974 } 2975 2976 if (priv->tx_path_in_lpi_mode) 2977 stmmac_disable_eee_mode(priv); 2978 2979 entry = tx_q->cur_tx; 2980 first_entry = entry; 2981 2982 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 2983 2984 if (likely(priv->extend_desc)) 2985 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2986 else 2987 desc = tx_q->dma_tx + entry; 2988 2989 first = desc; 2990 2991 tx_q->tx_skbuff[first_entry] = skb; 2992 2993 enh_desc = priv->plat->enh_desc; 2994 /* To program the descriptors according to the size of the frame */ 2995 if (enh_desc) 2996 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc); 2997 2998 if (unlikely(is_jumbo) && likely(priv->synopsys_id < 2999 DWMAC_CORE_4_00)) { 3000 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion); 3001 if (unlikely(entry < 0)) 3002 goto dma_map_err; 3003 } 3004 3005 for (i = 0; i < nfrags; i++) { 3006 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3007 int len = skb_frag_size(frag); 3008 bool last_segment = (i == (nfrags - 1)); 3009 3010 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3011 3012 if (likely(priv->extend_desc)) 3013 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3014 else 3015 desc = tx_q->dma_tx + entry; 3016 3017 des = skb_frag_dma_map(priv->device, frag, 0, len, 3018 DMA_TO_DEVICE); 3019 if (dma_mapping_error(priv->device, des)) 3020 goto dma_map_err; /* should reuse desc w/o issues */ 3021 3022 tx_q->tx_skbuff[entry] = NULL; 3023 3024 tx_q->tx_skbuff_dma[entry].buf = des; 3025 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3026 desc->des0 = cpu_to_le32(des); 3027 else 3028 desc->des2 = cpu_to_le32(des); 3029 3030 tx_q->tx_skbuff_dma[entry].map_as_page = true; 3031 tx_q->tx_skbuff_dma[entry].len = len; 3032 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 3033 3034 /* Prepare the descriptor and set the own bit too */ 3035 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion, 3036 priv->mode, 1, last_segment, 3037 skb->len); 3038 } 3039 3040 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3041 3042 tx_q->cur_tx = entry; 3043 3044 if (netif_msg_pktdata(priv)) { 3045 void *tx_head; 3046 3047 netdev_dbg(priv->dev, 3048 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 3049 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 3050 entry, first, nfrags); 3051 3052 if (priv->extend_desc) 3053 tx_head = (void *)tx_q->dma_etx; 3054 else 3055 tx_head = (void *)tx_q->dma_tx; 3056 3057 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false); 3058 3059 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 3060 print_pkt(skb->data, skb->len); 3061 } 3062 3063 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 3064 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 3065 __func__); 3066 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 3067 } 3068 3069 dev->stats.tx_bytes += skb->len; 3070 3071 /* According to the coalesce parameter the IC bit for the latest 3072 * segment is reset and the timer re-started to clean the tx status. 3073 * This approach takes care about the fragments: desc is the first 3074 * element in case of no SG. 3075 */ 3076 priv->tx_count_frames += nfrags + 1; 3077 if (likely(priv->tx_coal_frames > priv->tx_count_frames)) { 3078 mod_timer(&priv->txtimer, 3079 STMMAC_COAL_TIMER(priv->tx_coal_timer)); 3080 } else { 3081 priv->tx_count_frames = 0; 3082 priv->hw->desc->set_tx_ic(desc); 3083 priv->xstats.tx_set_ic_bit++; 3084 } 3085 3086 if (!priv->hwts_tx_en) 3087 skb_tx_timestamp(skb); 3088 3089 /* Ready to fill the first descriptor and set the OWN bit w/o any 3090 * problems because all the descriptors are actually ready to be 3091 * passed to the DMA engine. 3092 */ 3093 if (likely(!is_jumbo)) { 3094 bool last_segment = (nfrags == 0); 3095 3096 des = dma_map_single(priv->device, skb->data, 3097 nopaged_len, DMA_TO_DEVICE); 3098 if (dma_mapping_error(priv->device, des)) 3099 goto dma_map_err; 3100 3101 tx_q->tx_skbuff_dma[first_entry].buf = des; 3102 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3103 first->des0 = cpu_to_le32(des); 3104 else 3105 first->des2 = cpu_to_le32(des); 3106 3107 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 3108 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 3109 3110 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 3111 priv->hwts_tx_en)) { 3112 /* declare that device is doing timestamping */ 3113 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 3114 priv->hw->desc->enable_tx_timestamp(first); 3115 } 3116 3117 /* Prepare the first descriptor setting the OWN bit too */ 3118 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len, 3119 csum_insertion, priv->mode, 1, 3120 last_segment, skb->len); 3121 3122 /* The own bit must be the latest setting done when prepare the 3123 * descriptor and then barrier is needed to make sure that 3124 * all is coherent before granting the DMA engine. 3125 */ 3126 dma_wmb(); 3127 } 3128 3129 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 3130 3131 if (priv->synopsys_id < DWMAC_CORE_4_00) 3132 priv->hw->dma->enable_dma_transmission(priv->ioaddr); 3133 else 3134 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr, 3135 queue); 3136 3137 return NETDEV_TX_OK; 3138 3139 dma_map_err: 3140 netdev_err(priv->dev, "Tx DMA map failed\n"); 3141 dev_kfree_skb(skb); 3142 priv->dev->stats.tx_dropped++; 3143 return NETDEV_TX_OK; 3144 } 3145 3146 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 3147 { 3148 struct ethhdr *ehdr; 3149 u16 vlanid; 3150 3151 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 3152 NETIF_F_HW_VLAN_CTAG_RX && 3153 !__vlan_get_tag(skb, &vlanid)) { 3154 /* pop the vlan tag */ 3155 ehdr = (struct ethhdr *)skb->data; 3156 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2); 3157 skb_pull(skb, VLAN_HLEN); 3158 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid); 3159 } 3160 } 3161 3162 3163 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q) 3164 { 3165 if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH) 3166 return 0; 3167 3168 return 1; 3169 } 3170 3171 /** 3172 * stmmac_rx_refill - refill used skb preallocated buffers 3173 * @priv: driver private structure 3174 * @queue: RX queue index 3175 * Description : this is to reallocate the skb for the reception process 3176 * that is based on zero-copy. 3177 */ 3178 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 3179 { 3180 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3181 int dirty = stmmac_rx_dirty(priv, queue); 3182 unsigned int entry = rx_q->dirty_rx; 3183 3184 int bfsize = priv->dma_buf_sz; 3185 3186 while (dirty-- > 0) { 3187 struct dma_desc *p; 3188 3189 if (priv->extend_desc) 3190 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3191 else 3192 p = rx_q->dma_rx + entry; 3193 3194 if (likely(!rx_q->rx_skbuff[entry])) { 3195 struct sk_buff *skb; 3196 3197 skb = netdev_alloc_skb_ip_align(priv->dev, bfsize); 3198 if (unlikely(!skb)) { 3199 /* so for a while no zero-copy! */ 3200 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH; 3201 if (unlikely(net_ratelimit())) 3202 dev_err(priv->device, 3203 "fail to alloc skb entry %d\n", 3204 entry); 3205 break; 3206 } 3207 3208 rx_q->rx_skbuff[entry] = skb; 3209 rx_q->rx_skbuff_dma[entry] = 3210 dma_map_single(priv->device, skb->data, bfsize, 3211 DMA_FROM_DEVICE); 3212 if (dma_mapping_error(priv->device, 3213 rx_q->rx_skbuff_dma[entry])) { 3214 netdev_err(priv->dev, "Rx DMA map failed\n"); 3215 dev_kfree_skb(skb); 3216 break; 3217 } 3218 3219 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) { 3220 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]); 3221 p->des1 = 0; 3222 } else { 3223 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]); 3224 } 3225 if (priv->hw->mode->refill_desc3) 3226 priv->hw->mode->refill_desc3(rx_q, p); 3227 3228 if (rx_q->rx_zeroc_thresh > 0) 3229 rx_q->rx_zeroc_thresh--; 3230 3231 netif_dbg(priv, rx_status, priv->dev, 3232 "refill entry #%d\n", entry); 3233 } 3234 dma_wmb(); 3235 3236 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3237 priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0); 3238 else 3239 priv->hw->desc->set_rx_owner(p); 3240 3241 dma_wmb(); 3242 3243 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE); 3244 } 3245 rx_q->dirty_rx = entry; 3246 } 3247 3248 /** 3249 * stmmac_rx - manage the receive process 3250 * @priv: driver private structure 3251 * @limit: napi bugget 3252 * @queue: RX queue index. 3253 * Description : this the function called by the napi poll method. 3254 * It gets all the frames inside the ring. 3255 */ 3256 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 3257 { 3258 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3259 unsigned int entry = rx_q->cur_rx; 3260 int coe = priv->hw->rx_csum; 3261 unsigned int next_entry; 3262 unsigned int count = 0; 3263 3264 if (netif_msg_rx_status(priv)) { 3265 void *rx_head; 3266 3267 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 3268 if (priv->extend_desc) 3269 rx_head = (void *)rx_q->dma_erx; 3270 else 3271 rx_head = (void *)rx_q->dma_rx; 3272 3273 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true); 3274 } 3275 while (count < limit) { 3276 int status; 3277 struct dma_desc *p; 3278 struct dma_desc *np; 3279 3280 if (priv->extend_desc) 3281 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3282 else 3283 p = rx_q->dma_rx + entry; 3284 3285 /* read the status of the incoming frame */ 3286 status = priv->hw->desc->rx_status(&priv->dev->stats, 3287 &priv->xstats, p); 3288 /* check if managed by the DMA otherwise go ahead */ 3289 if (unlikely(status & dma_own)) 3290 break; 3291 3292 count++; 3293 3294 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE); 3295 next_entry = rx_q->cur_rx; 3296 3297 if (priv->extend_desc) 3298 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 3299 else 3300 np = rx_q->dma_rx + next_entry; 3301 3302 prefetch(np); 3303 3304 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status)) 3305 priv->hw->desc->rx_extended_status(&priv->dev->stats, 3306 &priv->xstats, 3307 rx_q->dma_erx + 3308 entry); 3309 if (unlikely(status == discard_frame)) { 3310 priv->dev->stats.rx_errors++; 3311 if (priv->hwts_rx_en && !priv->extend_desc) { 3312 /* DESC2 & DESC3 will be overwritten by device 3313 * with timestamp value, hence reinitialize 3314 * them in stmmac_rx_refill() function so that 3315 * device can reuse it. 3316 */ 3317 rx_q->rx_skbuff[entry] = NULL; 3318 dma_unmap_single(priv->device, 3319 rx_q->rx_skbuff_dma[entry], 3320 priv->dma_buf_sz, 3321 DMA_FROM_DEVICE); 3322 } 3323 } else { 3324 struct sk_buff *skb; 3325 int frame_len; 3326 unsigned int des; 3327 3328 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3329 des = le32_to_cpu(p->des0); 3330 else 3331 des = le32_to_cpu(p->des2); 3332 3333 frame_len = priv->hw->desc->get_rx_frame_len(p, coe); 3334 3335 /* If frame length is greater than skb buffer size 3336 * (preallocated during init) then the packet is 3337 * ignored 3338 */ 3339 if (frame_len > priv->dma_buf_sz) { 3340 netdev_err(priv->dev, 3341 "len %d larger than size (%d)\n", 3342 frame_len, priv->dma_buf_sz); 3343 priv->dev->stats.rx_length_errors++; 3344 break; 3345 } 3346 3347 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 3348 * Type frames (LLC/LLC-SNAP) 3349 */ 3350 if (unlikely(status != llc_snap)) 3351 frame_len -= ETH_FCS_LEN; 3352 3353 if (netif_msg_rx_status(priv)) { 3354 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n", 3355 p, entry, des); 3356 if (frame_len > ETH_FRAME_LEN) 3357 netdev_dbg(priv->dev, "frame size %d, COE: %d\n", 3358 frame_len, status); 3359 } 3360 3361 /* The zero-copy is always used for all the sizes 3362 * in case of GMAC4 because it needs 3363 * to refill the used descriptors, always. 3364 */ 3365 if (unlikely(!priv->plat->has_gmac4 && 3366 ((frame_len < priv->rx_copybreak) || 3367 stmmac_rx_threshold_count(rx_q)))) { 3368 skb = netdev_alloc_skb_ip_align(priv->dev, 3369 frame_len); 3370 if (unlikely(!skb)) { 3371 if (net_ratelimit()) 3372 dev_warn(priv->device, 3373 "packet dropped\n"); 3374 priv->dev->stats.rx_dropped++; 3375 break; 3376 } 3377 3378 dma_sync_single_for_cpu(priv->device, 3379 rx_q->rx_skbuff_dma 3380 [entry], frame_len, 3381 DMA_FROM_DEVICE); 3382 skb_copy_to_linear_data(skb, 3383 rx_q-> 3384 rx_skbuff[entry]->data, 3385 frame_len); 3386 3387 skb_put(skb, frame_len); 3388 dma_sync_single_for_device(priv->device, 3389 rx_q->rx_skbuff_dma 3390 [entry], frame_len, 3391 DMA_FROM_DEVICE); 3392 } else { 3393 skb = rx_q->rx_skbuff[entry]; 3394 if (unlikely(!skb)) { 3395 netdev_err(priv->dev, 3396 "%s: Inconsistent Rx chain\n", 3397 priv->dev->name); 3398 priv->dev->stats.rx_dropped++; 3399 break; 3400 } 3401 prefetch(skb->data - NET_IP_ALIGN); 3402 rx_q->rx_skbuff[entry] = NULL; 3403 rx_q->rx_zeroc_thresh++; 3404 3405 skb_put(skb, frame_len); 3406 dma_unmap_single(priv->device, 3407 rx_q->rx_skbuff_dma[entry], 3408 priv->dma_buf_sz, 3409 DMA_FROM_DEVICE); 3410 } 3411 3412 if (netif_msg_pktdata(priv)) { 3413 netdev_dbg(priv->dev, "frame received (%dbytes)", 3414 frame_len); 3415 print_pkt(skb->data, frame_len); 3416 } 3417 3418 stmmac_get_rx_hwtstamp(priv, p, np, skb); 3419 3420 stmmac_rx_vlan(priv->dev, skb); 3421 3422 skb->protocol = eth_type_trans(skb, priv->dev); 3423 3424 if (unlikely(!coe)) 3425 skb_checksum_none_assert(skb); 3426 else 3427 skb->ip_summed = CHECKSUM_UNNECESSARY; 3428 3429 napi_gro_receive(&rx_q->napi, skb); 3430 3431 priv->dev->stats.rx_packets++; 3432 priv->dev->stats.rx_bytes += frame_len; 3433 } 3434 entry = next_entry; 3435 } 3436 3437 stmmac_rx_refill(priv, queue); 3438 3439 priv->xstats.rx_pkt_n += count; 3440 3441 return count; 3442 } 3443 3444 /** 3445 * stmmac_poll - stmmac poll method (NAPI) 3446 * @napi : pointer to the napi structure. 3447 * @budget : maximum number of packets that the current CPU can receive from 3448 * all interfaces. 3449 * Description : 3450 * To look at the incoming frames and clear the tx resources. 3451 */ 3452 static int stmmac_poll(struct napi_struct *napi, int budget) 3453 { 3454 struct stmmac_rx_queue *rx_q = 3455 container_of(napi, struct stmmac_rx_queue, napi); 3456 struct stmmac_priv *priv = rx_q->priv_data; 3457 u32 tx_count = priv->plat->tx_queues_to_use; 3458 u32 chan = rx_q->queue_index; 3459 int work_done = 0; 3460 u32 queue; 3461 3462 priv->xstats.napi_poll++; 3463 3464 /* check all the queues */ 3465 for (queue = 0; queue < tx_count; queue++) 3466 stmmac_tx_clean(priv, queue); 3467 3468 work_done = stmmac_rx(priv, budget, rx_q->queue_index); 3469 if (work_done < budget) { 3470 napi_complete_done(napi, work_done); 3471 stmmac_enable_dma_irq(priv, chan); 3472 } 3473 return work_done; 3474 } 3475 3476 /** 3477 * stmmac_tx_timeout 3478 * @dev : Pointer to net device structure 3479 * Description: this function is called when a packet transmission fails to 3480 * complete within a reasonable time. The driver will mark the error in the 3481 * netdev structure and arrange for the device to be reset to a sane state 3482 * in order to transmit a new packet. 3483 */ 3484 static void stmmac_tx_timeout(struct net_device *dev) 3485 { 3486 struct stmmac_priv *priv = netdev_priv(dev); 3487 u32 tx_count = priv->plat->tx_queues_to_use; 3488 u32 chan; 3489 3490 /* Clear Tx resources and restart transmitting again */ 3491 for (chan = 0; chan < tx_count; chan++) 3492 stmmac_tx_err(priv, chan); 3493 } 3494 3495 /** 3496 * stmmac_set_rx_mode - entry point for multicast addressing 3497 * @dev : pointer to the device structure 3498 * Description: 3499 * This function is a driver entry point which gets called by the kernel 3500 * whenever multicast addresses must be enabled/disabled. 3501 * Return value: 3502 * void. 3503 */ 3504 static void stmmac_set_rx_mode(struct net_device *dev) 3505 { 3506 struct stmmac_priv *priv = netdev_priv(dev); 3507 3508 priv->hw->mac->set_filter(priv->hw, dev); 3509 } 3510 3511 /** 3512 * stmmac_change_mtu - entry point to change MTU size for the device. 3513 * @dev : device pointer. 3514 * @new_mtu : the new MTU size for the device. 3515 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 3516 * to drive packet transmission. Ethernet has an MTU of 1500 octets 3517 * (ETH_DATA_LEN). This value can be changed with ifconfig. 3518 * Return value: 3519 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3520 * file on failure. 3521 */ 3522 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 3523 { 3524 struct stmmac_priv *priv = netdev_priv(dev); 3525 3526 if (netif_running(dev)) { 3527 netdev_err(priv->dev, "must be stopped to change its MTU\n"); 3528 return -EBUSY; 3529 } 3530 3531 dev->mtu = new_mtu; 3532 3533 netdev_update_features(dev); 3534 3535 return 0; 3536 } 3537 3538 static netdev_features_t stmmac_fix_features(struct net_device *dev, 3539 netdev_features_t features) 3540 { 3541 struct stmmac_priv *priv = netdev_priv(dev); 3542 3543 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 3544 features &= ~NETIF_F_RXCSUM; 3545 3546 if (!priv->plat->tx_coe) 3547 features &= ~NETIF_F_CSUM_MASK; 3548 3549 /* Some GMAC devices have a bugged Jumbo frame support that 3550 * needs to have the Tx COE disabled for oversized frames 3551 * (due to limited buffer sizes). In this case we disable 3552 * the TX csum insertion in the TDES and not use SF. 3553 */ 3554 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 3555 features &= ~NETIF_F_CSUM_MASK; 3556 3557 /* Disable tso if asked by ethtool */ 3558 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 3559 if (features & NETIF_F_TSO) 3560 priv->tso = true; 3561 else 3562 priv->tso = false; 3563 } 3564 3565 return features; 3566 } 3567 3568 static int stmmac_set_features(struct net_device *netdev, 3569 netdev_features_t features) 3570 { 3571 struct stmmac_priv *priv = netdev_priv(netdev); 3572 3573 /* Keep the COE Type in case of csum is supporting */ 3574 if (features & NETIF_F_RXCSUM) 3575 priv->hw->rx_csum = priv->plat->rx_coe; 3576 else 3577 priv->hw->rx_csum = 0; 3578 /* No check needed because rx_coe has been set before and it will be 3579 * fixed in case of issue. 3580 */ 3581 priv->hw->mac->rx_ipc(priv->hw); 3582 3583 return 0; 3584 } 3585 3586 /** 3587 * stmmac_interrupt - main ISR 3588 * @irq: interrupt number. 3589 * @dev_id: to pass the net device pointer. 3590 * Description: this is the main driver interrupt service routine. 3591 * It can call: 3592 * o DMA service routine (to manage incoming frame reception and transmission 3593 * status) 3594 * o Core interrupts to manage: remote wake-up, management counter, LPI 3595 * interrupts. 3596 */ 3597 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 3598 { 3599 struct net_device *dev = (struct net_device *)dev_id; 3600 struct stmmac_priv *priv = netdev_priv(dev); 3601 u32 rx_cnt = priv->plat->rx_queues_to_use; 3602 u32 tx_cnt = priv->plat->tx_queues_to_use; 3603 u32 queues_count; 3604 u32 queue; 3605 3606 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 3607 3608 if (priv->irq_wake) 3609 pm_wakeup_event(priv->device, 0); 3610 3611 if (unlikely(!dev)) { 3612 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 3613 return IRQ_NONE; 3614 } 3615 3616 /* To handle GMAC own interrupts */ 3617 if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) { 3618 int status = priv->hw->mac->host_irq_status(priv->hw, 3619 &priv->xstats); 3620 3621 if (unlikely(status)) { 3622 /* For LPI we need to save the tx status */ 3623 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 3624 priv->tx_path_in_lpi_mode = true; 3625 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 3626 priv->tx_path_in_lpi_mode = false; 3627 } 3628 3629 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3630 for (queue = 0; queue < queues_count; queue++) { 3631 struct stmmac_rx_queue *rx_q = 3632 &priv->rx_queue[queue]; 3633 3634 status |= 3635 priv->hw->mac->host_mtl_irq_status(priv->hw, 3636 queue); 3637 3638 if (status & CORE_IRQ_MTL_RX_OVERFLOW && 3639 priv->hw->dma->set_rx_tail_ptr) 3640 priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, 3641 rx_q->rx_tail_addr, 3642 queue); 3643 } 3644 } 3645 3646 /* PCS link status */ 3647 if (priv->hw->pcs) { 3648 if (priv->xstats.pcs_link) 3649 netif_carrier_on(dev); 3650 else 3651 netif_carrier_off(dev); 3652 } 3653 } 3654 3655 /* To handle DMA interrupts */ 3656 stmmac_dma_interrupt(priv); 3657 3658 return IRQ_HANDLED; 3659 } 3660 3661 #ifdef CONFIG_NET_POLL_CONTROLLER 3662 /* Polling receive - used by NETCONSOLE and other diagnostic tools 3663 * to allow network I/O with interrupts disabled. 3664 */ 3665 static void stmmac_poll_controller(struct net_device *dev) 3666 { 3667 disable_irq(dev->irq); 3668 stmmac_interrupt(dev->irq, dev); 3669 enable_irq(dev->irq); 3670 } 3671 #endif 3672 3673 /** 3674 * stmmac_ioctl - Entry point for the Ioctl 3675 * @dev: Device pointer. 3676 * @rq: An IOCTL specefic structure, that can contain a pointer to 3677 * a proprietary structure used to pass information to the driver. 3678 * @cmd: IOCTL command 3679 * Description: 3680 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 3681 */ 3682 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3683 { 3684 int ret = -EOPNOTSUPP; 3685 3686 if (!netif_running(dev)) 3687 return -EINVAL; 3688 3689 switch (cmd) { 3690 case SIOCGMIIPHY: 3691 case SIOCGMIIREG: 3692 case SIOCSMIIREG: 3693 if (!dev->phydev) 3694 return -EINVAL; 3695 ret = phy_mii_ioctl(dev->phydev, rq, cmd); 3696 break; 3697 case SIOCSHWTSTAMP: 3698 ret = stmmac_hwtstamp_ioctl(dev, rq); 3699 break; 3700 default: 3701 break; 3702 } 3703 3704 return ret; 3705 } 3706 3707 #ifdef CONFIG_DEBUG_FS 3708 static struct dentry *stmmac_fs_dir; 3709 3710 static void sysfs_display_ring(void *head, int size, int extend_desc, 3711 struct seq_file *seq) 3712 { 3713 int i; 3714 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 3715 struct dma_desc *p = (struct dma_desc *)head; 3716 3717 for (i = 0; i < size; i++) { 3718 if (extend_desc) { 3719 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3720 i, (unsigned int)virt_to_phys(ep), 3721 le32_to_cpu(ep->basic.des0), 3722 le32_to_cpu(ep->basic.des1), 3723 le32_to_cpu(ep->basic.des2), 3724 le32_to_cpu(ep->basic.des3)); 3725 ep++; 3726 } else { 3727 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3728 i, (unsigned int)virt_to_phys(p), 3729 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 3730 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 3731 p++; 3732 } 3733 seq_printf(seq, "\n"); 3734 } 3735 } 3736 3737 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v) 3738 { 3739 struct net_device *dev = seq->private; 3740 struct stmmac_priv *priv = netdev_priv(dev); 3741 u32 rx_count = priv->plat->rx_queues_to_use; 3742 u32 tx_count = priv->plat->tx_queues_to_use; 3743 u32 queue; 3744 3745 for (queue = 0; queue < rx_count; queue++) { 3746 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3747 3748 seq_printf(seq, "RX Queue %d:\n", queue); 3749 3750 if (priv->extend_desc) { 3751 seq_printf(seq, "Extended descriptor ring:\n"); 3752 sysfs_display_ring((void *)rx_q->dma_erx, 3753 DMA_RX_SIZE, 1, seq); 3754 } else { 3755 seq_printf(seq, "Descriptor ring:\n"); 3756 sysfs_display_ring((void *)rx_q->dma_rx, 3757 DMA_RX_SIZE, 0, seq); 3758 } 3759 } 3760 3761 for (queue = 0; queue < tx_count; queue++) { 3762 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 3763 3764 seq_printf(seq, "TX Queue %d:\n", queue); 3765 3766 if (priv->extend_desc) { 3767 seq_printf(seq, "Extended descriptor ring:\n"); 3768 sysfs_display_ring((void *)tx_q->dma_etx, 3769 DMA_TX_SIZE, 1, seq); 3770 } else { 3771 seq_printf(seq, "Descriptor ring:\n"); 3772 sysfs_display_ring((void *)tx_q->dma_tx, 3773 DMA_TX_SIZE, 0, seq); 3774 } 3775 } 3776 3777 return 0; 3778 } 3779 3780 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file) 3781 { 3782 return single_open(file, stmmac_sysfs_ring_read, inode->i_private); 3783 } 3784 3785 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */ 3786 3787 static const struct file_operations stmmac_rings_status_fops = { 3788 .owner = THIS_MODULE, 3789 .open = stmmac_sysfs_ring_open, 3790 .read = seq_read, 3791 .llseek = seq_lseek, 3792 .release = single_release, 3793 }; 3794 3795 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v) 3796 { 3797 struct net_device *dev = seq->private; 3798 struct stmmac_priv *priv = netdev_priv(dev); 3799 3800 if (!priv->hw_cap_support) { 3801 seq_printf(seq, "DMA HW features not supported\n"); 3802 return 0; 3803 } 3804 3805 seq_printf(seq, "==============================\n"); 3806 seq_printf(seq, "\tDMA HW features\n"); 3807 seq_printf(seq, "==============================\n"); 3808 3809 seq_printf(seq, "\t10/100 Mbps: %s\n", 3810 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 3811 seq_printf(seq, "\t1000 Mbps: %s\n", 3812 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 3813 seq_printf(seq, "\tHalf duplex: %s\n", 3814 (priv->dma_cap.half_duplex) ? "Y" : "N"); 3815 seq_printf(seq, "\tHash Filter: %s\n", 3816 (priv->dma_cap.hash_filter) ? "Y" : "N"); 3817 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 3818 (priv->dma_cap.multi_addr) ? "Y" : "N"); 3819 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 3820 (priv->dma_cap.pcs) ? "Y" : "N"); 3821 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 3822 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 3823 seq_printf(seq, "\tPMT Remote wake up: %s\n", 3824 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 3825 seq_printf(seq, "\tPMT Magic Frame: %s\n", 3826 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 3827 seq_printf(seq, "\tRMON module: %s\n", 3828 (priv->dma_cap.rmon) ? "Y" : "N"); 3829 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 3830 (priv->dma_cap.time_stamp) ? "Y" : "N"); 3831 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 3832 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 3833 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 3834 (priv->dma_cap.eee) ? "Y" : "N"); 3835 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 3836 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 3837 (priv->dma_cap.tx_coe) ? "Y" : "N"); 3838 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3839 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 3840 (priv->dma_cap.rx_coe) ? "Y" : "N"); 3841 } else { 3842 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 3843 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 3844 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 3845 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 3846 } 3847 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 3848 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 3849 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 3850 priv->dma_cap.number_rx_channel); 3851 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 3852 priv->dma_cap.number_tx_channel); 3853 seq_printf(seq, "\tEnhanced descriptors: %s\n", 3854 (priv->dma_cap.enh_desc) ? "Y" : "N"); 3855 3856 return 0; 3857 } 3858 3859 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file) 3860 { 3861 return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private); 3862 } 3863 3864 static const struct file_operations stmmac_dma_cap_fops = { 3865 .owner = THIS_MODULE, 3866 .open = stmmac_sysfs_dma_cap_open, 3867 .read = seq_read, 3868 .llseek = seq_lseek, 3869 .release = single_release, 3870 }; 3871 3872 static int stmmac_init_fs(struct net_device *dev) 3873 { 3874 struct stmmac_priv *priv = netdev_priv(dev); 3875 3876 /* Create per netdev entries */ 3877 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 3878 3879 if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) { 3880 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n"); 3881 3882 return -ENOMEM; 3883 } 3884 3885 /* Entry to report DMA RX/TX rings */ 3886 priv->dbgfs_rings_status = 3887 debugfs_create_file("descriptors_status", S_IRUGO, 3888 priv->dbgfs_dir, dev, 3889 &stmmac_rings_status_fops); 3890 3891 if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) { 3892 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n"); 3893 debugfs_remove_recursive(priv->dbgfs_dir); 3894 3895 return -ENOMEM; 3896 } 3897 3898 /* Entry to report the DMA HW features */ 3899 priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO, 3900 priv->dbgfs_dir, 3901 dev, &stmmac_dma_cap_fops); 3902 3903 if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) { 3904 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n"); 3905 debugfs_remove_recursive(priv->dbgfs_dir); 3906 3907 return -ENOMEM; 3908 } 3909 3910 return 0; 3911 } 3912 3913 static void stmmac_exit_fs(struct net_device *dev) 3914 { 3915 struct stmmac_priv *priv = netdev_priv(dev); 3916 3917 debugfs_remove_recursive(priv->dbgfs_dir); 3918 } 3919 #endif /* CONFIG_DEBUG_FS */ 3920 3921 static const struct net_device_ops stmmac_netdev_ops = { 3922 .ndo_open = stmmac_open, 3923 .ndo_start_xmit = stmmac_xmit, 3924 .ndo_stop = stmmac_release, 3925 .ndo_change_mtu = stmmac_change_mtu, 3926 .ndo_fix_features = stmmac_fix_features, 3927 .ndo_set_features = stmmac_set_features, 3928 .ndo_set_rx_mode = stmmac_set_rx_mode, 3929 .ndo_tx_timeout = stmmac_tx_timeout, 3930 .ndo_do_ioctl = stmmac_ioctl, 3931 #ifdef CONFIG_NET_POLL_CONTROLLER 3932 .ndo_poll_controller = stmmac_poll_controller, 3933 #endif 3934 .ndo_set_mac_address = eth_mac_addr, 3935 }; 3936 3937 /** 3938 * stmmac_hw_init - Init the MAC device 3939 * @priv: driver private structure 3940 * Description: this function is to configure the MAC device according to 3941 * some platform parameters or the HW capability register. It prepares the 3942 * driver to use either ring or chain modes and to setup either enhanced or 3943 * normal descriptors. 3944 */ 3945 static int stmmac_hw_init(struct stmmac_priv *priv) 3946 { 3947 struct mac_device_info *mac; 3948 3949 /* Identify the MAC HW device */ 3950 if (priv->plat->has_gmac) { 3951 priv->dev->priv_flags |= IFF_UNICAST_FLT; 3952 mac = dwmac1000_setup(priv->ioaddr, 3953 priv->plat->multicast_filter_bins, 3954 priv->plat->unicast_filter_entries, 3955 &priv->synopsys_id); 3956 } else if (priv->plat->has_gmac4) { 3957 priv->dev->priv_flags |= IFF_UNICAST_FLT; 3958 mac = dwmac4_setup(priv->ioaddr, 3959 priv->plat->multicast_filter_bins, 3960 priv->plat->unicast_filter_entries, 3961 &priv->synopsys_id); 3962 } else { 3963 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id); 3964 } 3965 if (!mac) 3966 return -ENOMEM; 3967 3968 priv->hw = mac; 3969 3970 /* To use the chained or ring mode */ 3971 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3972 priv->hw->mode = &dwmac4_ring_mode_ops; 3973 } else { 3974 if (chain_mode) { 3975 priv->hw->mode = &chain_mode_ops; 3976 dev_info(priv->device, "Chain mode enabled\n"); 3977 priv->mode = STMMAC_CHAIN_MODE; 3978 } else { 3979 priv->hw->mode = &ring_mode_ops; 3980 dev_info(priv->device, "Ring mode enabled\n"); 3981 priv->mode = STMMAC_RING_MODE; 3982 } 3983 } 3984 3985 /* Get the HW capability (new GMAC newer than 3.50a) */ 3986 priv->hw_cap_support = stmmac_get_hw_features(priv); 3987 if (priv->hw_cap_support) { 3988 dev_info(priv->device, "DMA HW capability register supported\n"); 3989 3990 /* We can override some gmac/dma configuration fields: e.g. 3991 * enh_desc, tx_coe (e.g. that are passed through the 3992 * platform) with the values from the HW capability 3993 * register (if supported). 3994 */ 3995 priv->plat->enh_desc = priv->dma_cap.enh_desc; 3996 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up; 3997 priv->hw->pmt = priv->plat->pmt; 3998 3999 /* TXCOE doesn't work in thresh DMA mode */ 4000 if (priv->plat->force_thresh_dma_mode) 4001 priv->plat->tx_coe = 0; 4002 else 4003 priv->plat->tx_coe = priv->dma_cap.tx_coe; 4004 4005 /* In case of GMAC4 rx_coe is from HW cap register. */ 4006 priv->plat->rx_coe = priv->dma_cap.rx_coe; 4007 4008 if (priv->dma_cap.rx_coe_type2) 4009 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 4010 else if (priv->dma_cap.rx_coe_type1) 4011 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 4012 4013 } else { 4014 dev_info(priv->device, "No HW DMA feature register supported\n"); 4015 } 4016 4017 /* To use alternate (extended), normal or GMAC4 descriptor structures */ 4018 if (priv->synopsys_id >= DWMAC_CORE_4_00) 4019 priv->hw->desc = &dwmac4_desc_ops; 4020 else 4021 stmmac_selec_desc_mode(priv); 4022 4023 if (priv->plat->rx_coe) { 4024 priv->hw->rx_csum = priv->plat->rx_coe; 4025 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 4026 if (priv->synopsys_id < DWMAC_CORE_4_00) 4027 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 4028 } 4029 if (priv->plat->tx_coe) 4030 dev_info(priv->device, "TX Checksum insertion supported\n"); 4031 4032 if (priv->plat->pmt) { 4033 dev_info(priv->device, "Wake-Up On Lan supported\n"); 4034 device_set_wakeup_capable(priv->device, 1); 4035 } 4036 4037 if (priv->dma_cap.tsoen) 4038 dev_info(priv->device, "TSO supported\n"); 4039 4040 return 0; 4041 } 4042 4043 /** 4044 * stmmac_dvr_probe 4045 * @device: device pointer 4046 * @plat_dat: platform data pointer 4047 * @res: stmmac resource pointer 4048 * Description: this is the main probe function used to 4049 * call the alloc_etherdev, allocate the priv structure. 4050 * Return: 4051 * returns 0 on success, otherwise errno. 4052 */ 4053 int stmmac_dvr_probe(struct device *device, 4054 struct plat_stmmacenet_data *plat_dat, 4055 struct stmmac_resources *res) 4056 { 4057 struct net_device *ndev = NULL; 4058 struct stmmac_priv *priv; 4059 int ret = 0; 4060 u32 queue; 4061 4062 ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv), 4063 MTL_MAX_TX_QUEUES, 4064 MTL_MAX_RX_QUEUES); 4065 if (!ndev) 4066 return -ENOMEM; 4067 4068 SET_NETDEV_DEV(ndev, device); 4069 4070 priv = netdev_priv(ndev); 4071 priv->device = device; 4072 priv->dev = ndev; 4073 4074 stmmac_set_ethtool_ops(ndev); 4075 priv->pause = pause; 4076 priv->plat = plat_dat; 4077 priv->ioaddr = res->addr; 4078 priv->dev->base_addr = (unsigned long)res->addr; 4079 4080 priv->dev->irq = res->irq; 4081 priv->wol_irq = res->wol_irq; 4082 priv->lpi_irq = res->lpi_irq; 4083 4084 if (res->mac) 4085 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); 4086 4087 dev_set_drvdata(device, priv->dev); 4088 4089 /* Verify driver arguments */ 4090 stmmac_verify_args(); 4091 4092 /* Override with kernel parameters if supplied XXX CRS XXX 4093 * this needs to have multiple instances 4094 */ 4095 if ((phyaddr >= 0) && (phyaddr <= 31)) 4096 priv->plat->phy_addr = phyaddr; 4097 4098 if (priv->plat->stmmac_rst) 4099 reset_control_deassert(priv->plat->stmmac_rst); 4100 4101 /* Init MAC and get the capabilities */ 4102 ret = stmmac_hw_init(priv); 4103 if (ret) 4104 goto error_hw_init; 4105 4106 /* Configure real RX and TX queues */ 4107 netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use); 4108 netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use); 4109 4110 ndev->netdev_ops = &stmmac_netdev_ops; 4111 4112 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4113 NETIF_F_RXCSUM; 4114 4115 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 4116 ndev->hw_features |= NETIF_F_TSO; 4117 priv->tso = true; 4118 dev_info(priv->device, "TSO feature enabled\n"); 4119 } 4120 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 4121 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 4122 #ifdef STMMAC_VLAN_TAG_USED 4123 /* Both mac100 and gmac support receive VLAN tag detection */ 4124 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; 4125 #endif 4126 priv->msg_enable = netif_msg_init(debug, default_msg_level); 4127 4128 /* MTU range: 46 - hw-specific max */ 4129 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 4130 if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 4131 ndev->max_mtu = JUMBO_LEN; 4132 else 4133 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 4134 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 4135 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 4136 */ 4137 if ((priv->plat->maxmtu < ndev->max_mtu) && 4138 (priv->plat->maxmtu >= ndev->min_mtu)) 4139 ndev->max_mtu = priv->plat->maxmtu; 4140 else if (priv->plat->maxmtu < ndev->min_mtu) 4141 dev_warn(priv->device, 4142 "%s: warning: maxmtu having invalid value (%d)\n", 4143 __func__, priv->plat->maxmtu); 4144 4145 if (flow_ctrl) 4146 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 4147 4148 /* Rx Watchdog is available in the COREs newer than the 3.40. 4149 * In some case, for example on bugged HW this feature 4150 * has to be disable and this can be done by passing the 4151 * riwt_off field from the platform. 4152 */ 4153 if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) { 4154 priv->use_riwt = 1; 4155 dev_info(priv->device, 4156 "Enable RX Mitigation via HW Watchdog Timer\n"); 4157 } 4158 4159 for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) { 4160 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4161 4162 netif_napi_add(ndev, &rx_q->napi, stmmac_poll, 4163 (8 * priv->plat->rx_queues_to_use)); 4164 } 4165 4166 spin_lock_init(&priv->lock); 4167 4168 /* If a specific clk_csr value is passed from the platform 4169 * this means that the CSR Clock Range selection cannot be 4170 * changed at run-time and it is fixed. Viceversa the driver'll try to 4171 * set the MDC clock dynamically according to the csr actual 4172 * clock input. 4173 */ 4174 if (!priv->plat->clk_csr) 4175 stmmac_clk_csr_set(priv); 4176 else 4177 priv->clk_csr = priv->plat->clk_csr; 4178 4179 stmmac_check_pcs_mode(priv); 4180 4181 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4182 priv->hw->pcs != STMMAC_PCS_TBI && 4183 priv->hw->pcs != STMMAC_PCS_RTBI) { 4184 /* MDIO bus Registration */ 4185 ret = stmmac_mdio_register(ndev); 4186 if (ret < 0) { 4187 dev_err(priv->device, 4188 "%s: MDIO bus (id: %d) registration failed", 4189 __func__, priv->plat->bus_id); 4190 goto error_mdio_register; 4191 } 4192 } 4193 4194 ret = register_netdev(ndev); 4195 if (ret) { 4196 dev_err(priv->device, "%s: ERROR %i registering the device\n", 4197 __func__, ret); 4198 goto error_netdev_register; 4199 } 4200 4201 return ret; 4202 4203 error_netdev_register: 4204 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4205 priv->hw->pcs != STMMAC_PCS_TBI && 4206 priv->hw->pcs != STMMAC_PCS_RTBI) 4207 stmmac_mdio_unregister(ndev); 4208 error_mdio_register: 4209 for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) { 4210 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4211 4212 netif_napi_del(&rx_q->napi); 4213 } 4214 error_hw_init: 4215 free_netdev(ndev); 4216 4217 return ret; 4218 } 4219 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 4220 4221 /** 4222 * stmmac_dvr_remove 4223 * @dev: device pointer 4224 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 4225 * changes the link status, releases the DMA descriptor rings. 4226 */ 4227 int stmmac_dvr_remove(struct device *dev) 4228 { 4229 struct net_device *ndev = dev_get_drvdata(dev); 4230 struct stmmac_priv *priv = netdev_priv(ndev); 4231 4232 netdev_info(priv->dev, "%s: removing driver", __func__); 4233 4234 stmmac_stop_all_dma(priv); 4235 4236 priv->hw->mac->set_mac(priv->ioaddr, false); 4237 netif_carrier_off(ndev); 4238 unregister_netdev(ndev); 4239 if (priv->plat->stmmac_rst) 4240 reset_control_assert(priv->plat->stmmac_rst); 4241 clk_disable_unprepare(priv->plat->pclk); 4242 clk_disable_unprepare(priv->plat->stmmac_clk); 4243 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4244 priv->hw->pcs != STMMAC_PCS_TBI && 4245 priv->hw->pcs != STMMAC_PCS_RTBI) 4246 stmmac_mdio_unregister(ndev); 4247 free_netdev(ndev); 4248 4249 return 0; 4250 } 4251 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 4252 4253 /** 4254 * stmmac_suspend - suspend callback 4255 * @dev: device pointer 4256 * Description: this is the function to suspend the device and it is called 4257 * by the platform driver to stop the network queue, release the resources, 4258 * program the PMT register (for WoL), clean and release driver resources. 4259 */ 4260 int stmmac_suspend(struct device *dev) 4261 { 4262 struct net_device *ndev = dev_get_drvdata(dev); 4263 struct stmmac_priv *priv = netdev_priv(ndev); 4264 unsigned long flags; 4265 4266 if (!ndev || !netif_running(ndev)) 4267 return 0; 4268 4269 if (ndev->phydev) 4270 phy_stop(ndev->phydev); 4271 4272 spin_lock_irqsave(&priv->lock, flags); 4273 4274 netif_device_detach(ndev); 4275 stmmac_stop_all_queues(priv); 4276 4277 stmmac_disable_all_queues(priv); 4278 4279 /* Stop TX/RX DMA */ 4280 stmmac_stop_all_dma(priv); 4281 4282 /* Enable Power down mode by programming the PMT regs */ 4283 if (device_may_wakeup(priv->device)) { 4284 priv->hw->mac->pmt(priv->hw, priv->wolopts); 4285 priv->irq_wake = 1; 4286 } else { 4287 priv->hw->mac->set_mac(priv->ioaddr, false); 4288 pinctrl_pm_select_sleep_state(priv->device); 4289 /* Disable clock in case of PWM is off */ 4290 clk_disable(priv->plat->pclk); 4291 clk_disable(priv->plat->stmmac_clk); 4292 } 4293 spin_unlock_irqrestore(&priv->lock, flags); 4294 4295 priv->oldlink = 0; 4296 priv->speed = SPEED_UNKNOWN; 4297 priv->oldduplex = DUPLEX_UNKNOWN; 4298 return 0; 4299 } 4300 EXPORT_SYMBOL_GPL(stmmac_suspend); 4301 4302 /** 4303 * stmmac_reset_queues_param - reset queue parameters 4304 * @dev: device pointer 4305 */ 4306 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 4307 { 4308 u32 rx_cnt = priv->plat->rx_queues_to_use; 4309 u32 tx_cnt = priv->plat->tx_queues_to_use; 4310 u32 queue; 4311 4312 for (queue = 0; queue < rx_cnt; queue++) { 4313 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4314 4315 rx_q->cur_rx = 0; 4316 rx_q->dirty_rx = 0; 4317 } 4318 4319 for (queue = 0; queue < tx_cnt; queue++) { 4320 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 4321 4322 tx_q->cur_tx = 0; 4323 tx_q->dirty_tx = 0; 4324 } 4325 } 4326 4327 /** 4328 * stmmac_resume - resume callback 4329 * @dev: device pointer 4330 * Description: when resume this function is invoked to setup the DMA and CORE 4331 * in a usable state. 4332 */ 4333 int stmmac_resume(struct device *dev) 4334 { 4335 struct net_device *ndev = dev_get_drvdata(dev); 4336 struct stmmac_priv *priv = netdev_priv(ndev); 4337 unsigned long flags; 4338 4339 if (!netif_running(ndev)) 4340 return 0; 4341 4342 /* Power Down bit, into the PM register, is cleared 4343 * automatically as soon as a magic packet or a Wake-up frame 4344 * is received. Anyway, it's better to manually clear 4345 * this bit because it can generate problems while resuming 4346 * from another devices (e.g. serial console). 4347 */ 4348 if (device_may_wakeup(priv->device)) { 4349 spin_lock_irqsave(&priv->lock, flags); 4350 priv->hw->mac->pmt(priv->hw, 0); 4351 spin_unlock_irqrestore(&priv->lock, flags); 4352 priv->irq_wake = 0; 4353 } else { 4354 pinctrl_pm_select_default_state(priv->device); 4355 /* enable the clk previously disabled */ 4356 clk_enable(priv->plat->stmmac_clk); 4357 clk_enable(priv->plat->pclk); 4358 /* reset the phy so that it's ready */ 4359 if (priv->mii) 4360 stmmac_mdio_reset(priv->mii); 4361 } 4362 4363 netif_device_attach(ndev); 4364 4365 spin_lock_irqsave(&priv->lock, flags); 4366 4367 stmmac_reset_queues_param(priv); 4368 4369 /* reset private mss value to force mss context settings at 4370 * next tso xmit (only used for gmac4). 4371 */ 4372 priv->mss = 0; 4373 4374 stmmac_clear_descriptors(priv); 4375 4376 stmmac_hw_setup(ndev, false); 4377 stmmac_init_tx_coalesce(priv); 4378 stmmac_set_rx_mode(ndev); 4379 4380 stmmac_enable_all_queues(priv); 4381 4382 stmmac_start_all_queues(priv); 4383 4384 spin_unlock_irqrestore(&priv->lock, flags); 4385 4386 if (ndev->phydev) 4387 phy_start(ndev->phydev); 4388 4389 return 0; 4390 } 4391 EXPORT_SYMBOL_GPL(stmmac_resume); 4392 4393 #ifndef MODULE 4394 static int __init stmmac_cmdline_opt(char *str) 4395 { 4396 char *opt; 4397 4398 if (!str || !*str) 4399 return -EINVAL; 4400 while ((opt = strsep(&str, ",")) != NULL) { 4401 if (!strncmp(opt, "debug:", 6)) { 4402 if (kstrtoint(opt + 6, 0, &debug)) 4403 goto err; 4404 } else if (!strncmp(opt, "phyaddr:", 8)) { 4405 if (kstrtoint(opt + 8, 0, &phyaddr)) 4406 goto err; 4407 } else if (!strncmp(opt, "buf_sz:", 7)) { 4408 if (kstrtoint(opt + 7, 0, &buf_sz)) 4409 goto err; 4410 } else if (!strncmp(opt, "tc:", 3)) { 4411 if (kstrtoint(opt + 3, 0, &tc)) 4412 goto err; 4413 } else if (!strncmp(opt, "watchdog:", 9)) { 4414 if (kstrtoint(opt + 9, 0, &watchdog)) 4415 goto err; 4416 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 4417 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 4418 goto err; 4419 } else if (!strncmp(opt, "pause:", 6)) { 4420 if (kstrtoint(opt + 6, 0, &pause)) 4421 goto err; 4422 } else if (!strncmp(opt, "eee_timer:", 10)) { 4423 if (kstrtoint(opt + 10, 0, &eee_timer)) 4424 goto err; 4425 } else if (!strncmp(opt, "chain_mode:", 11)) { 4426 if (kstrtoint(opt + 11, 0, &chain_mode)) 4427 goto err; 4428 } 4429 } 4430 return 0; 4431 4432 err: 4433 pr_err("%s: ERROR broken module parameter conversion", __func__); 4434 return -EINVAL; 4435 } 4436 4437 __setup("stmmaceth=", stmmac_cmdline_opt); 4438 #endif /* MODULE */ 4439 4440 static int __init stmmac_init(void) 4441 { 4442 #ifdef CONFIG_DEBUG_FS 4443 /* Create debugfs main directory if it doesn't exist yet */ 4444 if (!stmmac_fs_dir) { 4445 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 4446 4447 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) { 4448 pr_err("ERROR %s, debugfs create directory failed\n", 4449 STMMAC_RESOURCE_NAME); 4450 4451 return -ENOMEM; 4452 } 4453 } 4454 #endif 4455 4456 return 0; 4457 } 4458 4459 static void __exit stmmac_exit(void) 4460 { 4461 #ifdef CONFIG_DEBUG_FS 4462 debugfs_remove_recursive(stmmac_fs_dir); 4463 #endif 4464 } 4465 4466 module_init(stmmac_init) 4467 module_exit(stmmac_exit) 4468 4469 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 4470 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 4471 MODULE_LICENSE("GPL"); 4472