1 /******************************************************************************* 2 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 3 ST Ethernet IPs are built around a Synopsys IP Core. 4 5 Copyright(C) 2007-2011 STMicroelectronics Ltd 6 7 This program is free software; you can redistribute it and/or modify it 8 under the terms and conditions of the GNU General Public License, 9 version 2, as published by the Free Software Foundation. 10 11 This program is distributed in the hope it will be useful, but WITHOUT 12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 more details. 15 16 The full GNU General Public License is included in this distribution in 17 the file called "COPYING". 18 19 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 20 21 Documentation available at: 22 http://www.stlinux.com 23 Support available at: 24 https://bugzilla.stlinux.com/ 25 *******************************************************************************/ 26 27 #include <linux/clk.h> 28 #include <linux/kernel.h> 29 #include <linux/interrupt.h> 30 #include <linux/ip.h> 31 #include <linux/tcp.h> 32 #include <linux/skbuff.h> 33 #include <linux/ethtool.h> 34 #include <linux/if_ether.h> 35 #include <linux/crc32.h> 36 #include <linux/mii.h> 37 #include <linux/if.h> 38 #include <linux/if_vlan.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/slab.h> 41 #include <linux/prefetch.h> 42 #include <linux/pinctrl/consumer.h> 43 #ifdef CONFIG_DEBUG_FS 44 #include <linux/debugfs.h> 45 #include <linux/seq_file.h> 46 #endif /* CONFIG_DEBUG_FS */ 47 #include <linux/net_tstamp.h> 48 #include "stmmac_ptp.h" 49 #include "stmmac.h" 50 #include <linux/reset.h> 51 #include <linux/of_mdio.h> 52 #include "dwmac1000.h" 53 54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x) 55 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 56 57 /* Module parameters */ 58 #define TX_TIMEO 5000 59 static int watchdog = TX_TIMEO; 60 module_param(watchdog, int, 0644); 61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 62 63 static int debug = -1; 64 module_param(debug, int, 0644); 65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 66 67 static int phyaddr = -1; 68 module_param(phyaddr, int, 0444); 69 MODULE_PARM_DESC(phyaddr, "Physical device address"); 70 71 #define STMMAC_TX_THRESH (DMA_TX_SIZE / 4) 72 #define STMMAC_RX_THRESH (DMA_RX_SIZE / 4) 73 74 static int flow_ctrl = FLOW_OFF; 75 module_param(flow_ctrl, int, 0644); 76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 77 78 static int pause = PAUSE_TIME; 79 module_param(pause, int, 0644); 80 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 81 82 #define TC_DEFAULT 64 83 static int tc = TC_DEFAULT; 84 module_param(tc, int, 0644); 85 MODULE_PARM_DESC(tc, "DMA threshold control value"); 86 87 #define DEFAULT_BUFSIZE 1536 88 static int buf_sz = DEFAULT_BUFSIZE; 89 module_param(buf_sz, int, 0644); 90 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 91 92 #define STMMAC_RX_COPYBREAK 256 93 94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 95 NETIF_MSG_LINK | NETIF_MSG_IFUP | 96 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 97 98 #define STMMAC_DEFAULT_LPI_TIMER 1000 99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 100 module_param(eee_timer, int, 0644); 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x)) 103 104 /* By default the driver will use the ring mode to manage tx and rx descriptors, 105 * but allow user to force to use the chain instead of the ring 106 */ 107 static unsigned int chain_mode; 108 module_param(chain_mode, int, 0444); 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 110 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 112 113 #ifdef CONFIG_DEBUG_FS 114 static int stmmac_init_fs(struct net_device *dev); 115 static void stmmac_exit_fs(struct net_device *dev); 116 #endif 117 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x)) 119 120 /** 121 * stmmac_verify_args - verify the driver parameters. 122 * Description: it checks the driver parameters and set a default in case of 123 * errors. 124 */ 125 static void stmmac_verify_args(void) 126 { 127 if (unlikely(watchdog < 0)) 128 watchdog = TX_TIMEO; 129 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 130 buf_sz = DEFAULT_BUFSIZE; 131 if (unlikely(flow_ctrl > 1)) 132 flow_ctrl = FLOW_AUTO; 133 else if (likely(flow_ctrl < 0)) 134 flow_ctrl = FLOW_OFF; 135 if (unlikely((pause < 0) || (pause > 0xffff))) 136 pause = PAUSE_TIME; 137 if (eee_timer < 0) 138 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 139 } 140 141 /** 142 * stmmac_disable_all_queues - Disable all queues 143 * @priv: driver private structure 144 */ 145 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 146 { 147 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 148 u32 queue; 149 150 for (queue = 0; queue < rx_queues_cnt; queue++) { 151 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 152 153 napi_disable(&rx_q->napi); 154 } 155 } 156 157 /** 158 * stmmac_enable_all_queues - Enable all queues 159 * @priv: driver private structure 160 */ 161 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 162 { 163 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 164 u32 queue; 165 166 for (queue = 0; queue < rx_queues_cnt; queue++) { 167 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 168 169 napi_enable(&rx_q->napi); 170 } 171 } 172 173 /** 174 * stmmac_stop_all_queues - Stop all queues 175 * @priv: driver private structure 176 */ 177 static void stmmac_stop_all_queues(struct stmmac_priv *priv) 178 { 179 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 180 u32 queue; 181 182 for (queue = 0; queue < tx_queues_cnt; queue++) 183 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 184 } 185 186 /** 187 * stmmac_start_all_queues - Start all queues 188 * @priv: driver private structure 189 */ 190 static void stmmac_start_all_queues(struct stmmac_priv *priv) 191 { 192 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 193 u32 queue; 194 195 for (queue = 0; queue < tx_queues_cnt; queue++) 196 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue)); 197 } 198 199 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 200 { 201 if (!test_bit(STMMAC_DOWN, &priv->state) && 202 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 203 queue_work(priv->wq, &priv->service_task); 204 } 205 206 static void stmmac_global_err(struct stmmac_priv *priv) 207 { 208 netif_carrier_off(priv->dev); 209 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 210 stmmac_service_event_schedule(priv); 211 } 212 213 /** 214 * stmmac_clk_csr_set - dynamically set the MDC clock 215 * @priv: driver private structure 216 * Description: this is to dynamically set the MDC clock according to the csr 217 * clock input. 218 * Note: 219 * If a specific clk_csr value is passed from the platform 220 * this means that the CSR Clock Range selection cannot be 221 * changed at run-time and it is fixed (as reported in the driver 222 * documentation). Viceversa the driver will try to set the MDC 223 * clock dynamically according to the actual clock input. 224 */ 225 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 226 { 227 u32 clk_rate; 228 229 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 230 231 /* Platform provided default clk_csr would be assumed valid 232 * for all other cases except for the below mentioned ones. 233 * For values higher than the IEEE 802.3 specified frequency 234 * we can not estimate the proper divider as it is not known 235 * the frequency of clk_csr_i. So we do not change the default 236 * divider. 237 */ 238 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 239 if (clk_rate < CSR_F_35M) 240 priv->clk_csr = STMMAC_CSR_20_35M; 241 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 242 priv->clk_csr = STMMAC_CSR_35_60M; 243 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 244 priv->clk_csr = STMMAC_CSR_60_100M; 245 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 246 priv->clk_csr = STMMAC_CSR_100_150M; 247 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 248 priv->clk_csr = STMMAC_CSR_150_250M; 249 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M)) 250 priv->clk_csr = STMMAC_CSR_250_300M; 251 } 252 253 if (priv->plat->has_sun8i) { 254 if (clk_rate > 160000000) 255 priv->clk_csr = 0x03; 256 else if (clk_rate > 80000000) 257 priv->clk_csr = 0x02; 258 else if (clk_rate > 40000000) 259 priv->clk_csr = 0x01; 260 else 261 priv->clk_csr = 0; 262 } 263 } 264 265 static void print_pkt(unsigned char *buf, int len) 266 { 267 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 268 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 269 } 270 271 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 272 { 273 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 274 u32 avail; 275 276 if (tx_q->dirty_tx > tx_q->cur_tx) 277 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 278 else 279 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1; 280 281 return avail; 282 } 283 284 /** 285 * stmmac_rx_dirty - Get RX queue dirty 286 * @priv: driver private structure 287 * @queue: RX queue index 288 */ 289 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 290 { 291 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 292 u32 dirty; 293 294 if (rx_q->dirty_rx <= rx_q->cur_rx) 295 dirty = rx_q->cur_rx - rx_q->dirty_rx; 296 else 297 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx; 298 299 return dirty; 300 } 301 302 /** 303 * stmmac_hw_fix_mac_speed - callback for speed selection 304 * @priv: driver private structure 305 * Description: on some platforms (e.g. ST), some HW system configuration 306 * registers have to be set according to the link speed negotiated. 307 */ 308 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv) 309 { 310 struct net_device *ndev = priv->dev; 311 struct phy_device *phydev = ndev->phydev; 312 313 if (likely(priv->plat->fix_mac_speed)) 314 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed); 315 } 316 317 /** 318 * stmmac_enable_eee_mode - check and enter in LPI mode 319 * @priv: driver private structure 320 * Description: this function is to verify and enter in LPI mode in case of 321 * EEE. 322 */ 323 static void stmmac_enable_eee_mode(struct stmmac_priv *priv) 324 { 325 u32 tx_cnt = priv->plat->tx_queues_to_use; 326 u32 queue; 327 328 /* check if all TX queues have the work finished */ 329 for (queue = 0; queue < tx_cnt; queue++) { 330 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 331 332 if (tx_q->dirty_tx != tx_q->cur_tx) 333 return; /* still unfinished work */ 334 } 335 336 /* Check and enter in LPI mode */ 337 if (!priv->tx_path_in_lpi_mode) 338 priv->hw->mac->set_eee_mode(priv->hw, 339 priv->plat->en_tx_lpi_clockgating); 340 } 341 342 /** 343 * stmmac_disable_eee_mode - disable and exit from LPI mode 344 * @priv: driver private structure 345 * Description: this function is to exit and disable EEE in case of 346 * LPI state is true. This is called by the xmit. 347 */ 348 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 349 { 350 priv->hw->mac->reset_eee_mode(priv->hw); 351 del_timer_sync(&priv->eee_ctrl_timer); 352 priv->tx_path_in_lpi_mode = false; 353 } 354 355 /** 356 * stmmac_eee_ctrl_timer - EEE TX SW timer. 357 * @arg : data hook 358 * Description: 359 * if there is no data transfer and if we are not in LPI state, 360 * then MAC Transmitter can be moved to LPI state. 361 */ 362 static void stmmac_eee_ctrl_timer(struct timer_list *t) 363 { 364 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 365 366 stmmac_enable_eee_mode(priv); 367 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 368 } 369 370 /** 371 * stmmac_eee_init - init EEE 372 * @priv: driver private structure 373 * Description: 374 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 375 * can also manage EEE, this function enable the LPI state and start related 376 * timer. 377 */ 378 bool stmmac_eee_init(struct stmmac_priv *priv) 379 { 380 struct net_device *ndev = priv->dev; 381 int interface = priv->plat->interface; 382 unsigned long flags; 383 bool ret = false; 384 385 if ((interface != PHY_INTERFACE_MODE_MII) && 386 (interface != PHY_INTERFACE_MODE_GMII) && 387 !phy_interface_mode_is_rgmii(interface)) 388 goto out; 389 390 /* Using PCS we cannot dial with the phy registers at this stage 391 * so we do not support extra feature like EEE. 392 */ 393 if ((priv->hw->pcs == STMMAC_PCS_RGMII) || 394 (priv->hw->pcs == STMMAC_PCS_TBI) || 395 (priv->hw->pcs == STMMAC_PCS_RTBI)) 396 goto out; 397 398 /* MAC core supports the EEE feature. */ 399 if (priv->dma_cap.eee) { 400 int tx_lpi_timer = priv->tx_lpi_timer; 401 402 /* Check if the PHY supports EEE */ 403 if (phy_init_eee(ndev->phydev, 1)) { 404 /* To manage at run-time if the EEE cannot be supported 405 * anymore (for example because the lp caps have been 406 * changed). 407 * In that case the driver disable own timers. 408 */ 409 spin_lock_irqsave(&priv->lock, flags); 410 if (priv->eee_active) { 411 netdev_dbg(priv->dev, "disable EEE\n"); 412 del_timer_sync(&priv->eee_ctrl_timer); 413 priv->hw->mac->set_eee_timer(priv->hw, 0, 414 tx_lpi_timer); 415 } 416 priv->eee_active = 0; 417 spin_unlock_irqrestore(&priv->lock, flags); 418 goto out; 419 } 420 /* Activate the EEE and start timers */ 421 spin_lock_irqsave(&priv->lock, flags); 422 if (!priv->eee_active) { 423 priv->eee_active = 1; 424 timer_setup(&priv->eee_ctrl_timer, 425 stmmac_eee_ctrl_timer, 0); 426 mod_timer(&priv->eee_ctrl_timer, 427 STMMAC_LPI_T(eee_timer)); 428 429 priv->hw->mac->set_eee_timer(priv->hw, 430 STMMAC_DEFAULT_LIT_LS, 431 tx_lpi_timer); 432 } 433 /* Set HW EEE according to the speed */ 434 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link); 435 436 ret = true; 437 spin_unlock_irqrestore(&priv->lock, flags); 438 439 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 440 } 441 out: 442 return ret; 443 } 444 445 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 446 * @priv: driver private structure 447 * @p : descriptor pointer 448 * @skb : the socket buffer 449 * Description : 450 * This function will read timestamp from the descriptor & pass it to stack. 451 * and also perform some sanity checks. 452 */ 453 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 454 struct dma_desc *p, struct sk_buff *skb) 455 { 456 struct skb_shared_hwtstamps shhwtstamp; 457 u64 ns; 458 459 if (!priv->hwts_tx_en) 460 return; 461 462 /* exit if skb doesn't support hw tstamp */ 463 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 464 return; 465 466 /* check tx tstamp status */ 467 if (priv->hw->desc->get_tx_timestamp_status(p)) { 468 /* get the valid tstamp */ 469 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts); 470 471 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 472 shhwtstamp.hwtstamp = ns_to_ktime(ns); 473 474 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 475 /* pass tstamp to stack */ 476 skb_tstamp_tx(skb, &shhwtstamp); 477 } 478 479 return; 480 } 481 482 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 483 * @priv: driver private structure 484 * @p : descriptor pointer 485 * @np : next descriptor pointer 486 * @skb : the socket buffer 487 * Description : 488 * This function will read received packet's timestamp from the descriptor 489 * and pass it to stack. It also perform some sanity checks. 490 */ 491 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 492 struct dma_desc *np, struct sk_buff *skb) 493 { 494 struct skb_shared_hwtstamps *shhwtstamp = NULL; 495 struct dma_desc *desc = p; 496 u64 ns; 497 498 if (!priv->hwts_rx_en) 499 return; 500 /* For GMAC4, the valid timestamp is from CTX next desc. */ 501 if (priv->plat->has_gmac4) 502 desc = np; 503 504 /* Check if timestamp is available */ 505 if (priv->hw->desc->get_rx_timestamp_status(p, np, priv->adv_ts)) { 506 ns = priv->hw->desc->get_timestamp(desc, priv->adv_ts); 507 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 508 shhwtstamp = skb_hwtstamps(skb); 509 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 510 shhwtstamp->hwtstamp = ns_to_ktime(ns); 511 } else { 512 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 513 } 514 } 515 516 /** 517 * stmmac_hwtstamp_ioctl - control hardware timestamping. 518 * @dev: device pointer. 519 * @ifr: An IOCTL specific structure, that can contain a pointer to 520 * a proprietary structure used to pass information to the driver. 521 * Description: 522 * This function configures the MAC to enable/disable both outgoing(TX) 523 * and incoming(RX) packets time stamping based on user input. 524 * Return Value: 525 * 0 on success and an appropriate -ve integer on failure. 526 */ 527 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr) 528 { 529 struct stmmac_priv *priv = netdev_priv(dev); 530 struct hwtstamp_config config; 531 struct timespec64 now; 532 u64 temp = 0; 533 u32 ptp_v2 = 0; 534 u32 tstamp_all = 0; 535 u32 ptp_over_ipv4_udp = 0; 536 u32 ptp_over_ipv6_udp = 0; 537 u32 ptp_over_ethernet = 0; 538 u32 snap_type_sel = 0; 539 u32 ts_master_en = 0; 540 u32 ts_event_en = 0; 541 u32 value = 0; 542 u32 sec_inc; 543 544 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 545 netdev_alert(priv->dev, "No support for HW time stamping\n"); 546 priv->hwts_tx_en = 0; 547 priv->hwts_rx_en = 0; 548 549 return -EOPNOTSUPP; 550 } 551 552 if (copy_from_user(&config, ifr->ifr_data, 553 sizeof(struct hwtstamp_config))) 554 return -EFAULT; 555 556 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 557 __func__, config.flags, config.tx_type, config.rx_filter); 558 559 /* reserved for future extensions */ 560 if (config.flags) 561 return -EINVAL; 562 563 if (config.tx_type != HWTSTAMP_TX_OFF && 564 config.tx_type != HWTSTAMP_TX_ON) 565 return -ERANGE; 566 567 if (priv->adv_ts) { 568 switch (config.rx_filter) { 569 case HWTSTAMP_FILTER_NONE: 570 /* time stamp no incoming packet at all */ 571 config.rx_filter = HWTSTAMP_FILTER_NONE; 572 break; 573 574 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 575 /* PTP v1, UDP, any kind of event packet */ 576 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 577 /* take time stamp for all event messages */ 578 if (priv->plat->has_gmac4) 579 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 580 else 581 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 582 583 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 584 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 585 break; 586 587 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 588 /* PTP v1, UDP, Sync packet */ 589 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 590 /* take time stamp for SYNC messages only */ 591 ts_event_en = PTP_TCR_TSEVNTENA; 592 593 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 594 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 595 break; 596 597 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 598 /* PTP v1, UDP, Delay_req packet */ 599 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 600 /* take time stamp for Delay_Req messages only */ 601 ts_master_en = PTP_TCR_TSMSTRENA; 602 ts_event_en = PTP_TCR_TSEVNTENA; 603 604 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 605 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 606 break; 607 608 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 609 /* PTP v2, UDP, any kind of event packet */ 610 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 611 ptp_v2 = PTP_TCR_TSVER2ENA; 612 /* take time stamp for all event messages */ 613 if (priv->plat->has_gmac4) 614 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 615 else 616 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 617 618 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 619 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 620 break; 621 622 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 623 /* PTP v2, UDP, Sync packet */ 624 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 625 ptp_v2 = PTP_TCR_TSVER2ENA; 626 /* take time stamp for SYNC messages only */ 627 ts_event_en = PTP_TCR_TSEVNTENA; 628 629 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 630 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 631 break; 632 633 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 634 /* PTP v2, UDP, Delay_req packet */ 635 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 636 ptp_v2 = PTP_TCR_TSVER2ENA; 637 /* take time stamp for Delay_Req messages only */ 638 ts_master_en = PTP_TCR_TSMSTRENA; 639 ts_event_en = PTP_TCR_TSEVNTENA; 640 641 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 642 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 643 break; 644 645 case HWTSTAMP_FILTER_PTP_V2_EVENT: 646 /* PTP v2/802.AS1 any layer, any kind of event packet */ 647 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 648 ptp_v2 = PTP_TCR_TSVER2ENA; 649 /* take time stamp for all event messages */ 650 if (priv->plat->has_gmac4) 651 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 652 else 653 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 654 655 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 656 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 657 ptp_over_ethernet = PTP_TCR_TSIPENA; 658 break; 659 660 case HWTSTAMP_FILTER_PTP_V2_SYNC: 661 /* PTP v2/802.AS1, any layer, Sync packet */ 662 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 663 ptp_v2 = PTP_TCR_TSVER2ENA; 664 /* take time stamp for SYNC messages only */ 665 ts_event_en = PTP_TCR_TSEVNTENA; 666 667 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 668 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 669 ptp_over_ethernet = PTP_TCR_TSIPENA; 670 break; 671 672 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 673 /* PTP v2/802.AS1, any layer, Delay_req packet */ 674 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 675 ptp_v2 = PTP_TCR_TSVER2ENA; 676 /* take time stamp for Delay_Req messages only */ 677 ts_master_en = PTP_TCR_TSMSTRENA; 678 ts_event_en = PTP_TCR_TSEVNTENA; 679 680 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 681 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 682 ptp_over_ethernet = PTP_TCR_TSIPENA; 683 break; 684 685 case HWTSTAMP_FILTER_NTP_ALL: 686 case HWTSTAMP_FILTER_ALL: 687 /* time stamp any incoming packet */ 688 config.rx_filter = HWTSTAMP_FILTER_ALL; 689 tstamp_all = PTP_TCR_TSENALL; 690 break; 691 692 default: 693 return -ERANGE; 694 } 695 } else { 696 switch (config.rx_filter) { 697 case HWTSTAMP_FILTER_NONE: 698 config.rx_filter = HWTSTAMP_FILTER_NONE; 699 break; 700 default: 701 /* PTP v1, UDP, any kind of event packet */ 702 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 703 break; 704 } 705 } 706 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 707 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 708 709 if (!priv->hwts_tx_en && !priv->hwts_rx_en) 710 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0); 711 else { 712 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR | 713 tstamp_all | ptp_v2 | ptp_over_ethernet | 714 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en | 715 ts_master_en | snap_type_sel); 716 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value); 717 718 /* program Sub Second Increment reg */ 719 sec_inc = priv->hw->ptp->config_sub_second_increment( 720 priv->ptpaddr, priv->plat->clk_ptp_rate, 721 priv->plat->has_gmac4); 722 temp = div_u64(1000000000ULL, sec_inc); 723 724 /* calculate default added value: 725 * formula is : 726 * addend = (2^32)/freq_div_ratio; 727 * where, freq_div_ratio = 1e9ns/sec_inc 728 */ 729 temp = (u64)(temp << 32); 730 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 731 priv->hw->ptp->config_addend(priv->ptpaddr, 732 priv->default_addend); 733 734 /* initialize system time */ 735 ktime_get_real_ts64(&now); 736 737 /* lower 32 bits of tv_sec are safe until y2106 */ 738 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec, 739 now.tv_nsec); 740 } 741 742 return copy_to_user(ifr->ifr_data, &config, 743 sizeof(struct hwtstamp_config)) ? -EFAULT : 0; 744 } 745 746 /** 747 * stmmac_init_ptp - init PTP 748 * @priv: driver private structure 749 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 750 * This is done by looking at the HW cap. register. 751 * This function also registers the ptp driver. 752 */ 753 static int stmmac_init_ptp(struct stmmac_priv *priv) 754 { 755 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 756 return -EOPNOTSUPP; 757 758 priv->adv_ts = 0; 759 /* Check if adv_ts can be enabled for dwmac 4.x core */ 760 if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp) 761 priv->adv_ts = 1; 762 /* Dwmac 3.x core with extend_desc can support adv_ts */ 763 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 764 priv->adv_ts = 1; 765 766 if (priv->dma_cap.time_stamp) 767 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 768 769 if (priv->adv_ts) 770 netdev_info(priv->dev, 771 "IEEE 1588-2008 Advanced Timestamp supported\n"); 772 773 priv->hw->ptp = &stmmac_ptp; 774 priv->hwts_tx_en = 0; 775 priv->hwts_rx_en = 0; 776 777 stmmac_ptp_register(priv); 778 779 return 0; 780 } 781 782 static void stmmac_release_ptp(struct stmmac_priv *priv) 783 { 784 if (priv->plat->clk_ptp_ref) 785 clk_disable_unprepare(priv->plat->clk_ptp_ref); 786 stmmac_ptp_unregister(priv); 787 } 788 789 /** 790 * stmmac_mac_flow_ctrl - Configure flow control in all queues 791 * @priv: driver private structure 792 * Description: It is used for configuring the flow control in all queues 793 */ 794 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 795 { 796 u32 tx_cnt = priv->plat->tx_queues_to_use; 797 798 priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl, 799 priv->pause, tx_cnt); 800 } 801 802 /** 803 * stmmac_adjust_link - adjusts the link parameters 804 * @dev: net device structure 805 * Description: this is the helper called by the physical abstraction layer 806 * drivers to communicate the phy link status. According the speed and duplex 807 * this driver can invoke registered glue-logic as well. 808 * It also invoke the eee initialization because it could happen when switch 809 * on different networks (that are eee capable). 810 */ 811 static void stmmac_adjust_link(struct net_device *dev) 812 { 813 struct stmmac_priv *priv = netdev_priv(dev); 814 struct phy_device *phydev = dev->phydev; 815 unsigned long flags; 816 bool new_state = false; 817 818 if (!phydev) 819 return; 820 821 spin_lock_irqsave(&priv->lock, flags); 822 823 if (phydev->link) { 824 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 825 826 /* Now we make sure that we can be in full duplex mode. 827 * If not, we operate in half-duplex mode. */ 828 if (phydev->duplex != priv->oldduplex) { 829 new_state = true; 830 if (!phydev->duplex) 831 ctrl &= ~priv->hw->link.duplex; 832 else 833 ctrl |= priv->hw->link.duplex; 834 priv->oldduplex = phydev->duplex; 835 } 836 /* Flow Control operation */ 837 if (phydev->pause) 838 stmmac_mac_flow_ctrl(priv, phydev->duplex); 839 840 if (phydev->speed != priv->speed) { 841 new_state = true; 842 ctrl &= ~priv->hw->link.speed_mask; 843 switch (phydev->speed) { 844 case SPEED_1000: 845 ctrl |= priv->hw->link.speed1000; 846 break; 847 case SPEED_100: 848 ctrl |= priv->hw->link.speed100; 849 break; 850 case SPEED_10: 851 ctrl |= priv->hw->link.speed10; 852 break; 853 default: 854 netif_warn(priv, link, priv->dev, 855 "broken speed: %d\n", phydev->speed); 856 phydev->speed = SPEED_UNKNOWN; 857 break; 858 } 859 if (phydev->speed != SPEED_UNKNOWN) 860 stmmac_hw_fix_mac_speed(priv); 861 priv->speed = phydev->speed; 862 } 863 864 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 865 866 if (!priv->oldlink) { 867 new_state = true; 868 priv->oldlink = true; 869 } 870 } else if (priv->oldlink) { 871 new_state = true; 872 priv->oldlink = false; 873 priv->speed = SPEED_UNKNOWN; 874 priv->oldduplex = DUPLEX_UNKNOWN; 875 } 876 877 if (new_state && netif_msg_link(priv)) 878 phy_print_status(phydev); 879 880 spin_unlock_irqrestore(&priv->lock, flags); 881 882 if (phydev->is_pseudo_fixed_link) 883 /* Stop PHY layer to call the hook to adjust the link in case 884 * of a switch is attached to the stmmac driver. 885 */ 886 phydev->irq = PHY_IGNORE_INTERRUPT; 887 else 888 /* At this stage, init the EEE if supported. 889 * Never called in case of fixed_link. 890 */ 891 priv->eee_enabled = stmmac_eee_init(priv); 892 } 893 894 /** 895 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 896 * @priv: driver private structure 897 * Description: this is to verify if the HW supports the PCS. 898 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 899 * configured for the TBI, RTBI, or SGMII PHY interface. 900 */ 901 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 902 { 903 int interface = priv->plat->interface; 904 905 if (priv->dma_cap.pcs) { 906 if ((interface == PHY_INTERFACE_MODE_RGMII) || 907 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 908 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 909 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 910 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 911 priv->hw->pcs = STMMAC_PCS_RGMII; 912 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 913 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 914 priv->hw->pcs = STMMAC_PCS_SGMII; 915 } 916 } 917 } 918 919 /** 920 * stmmac_init_phy - PHY initialization 921 * @dev: net device structure 922 * Description: it initializes the driver's PHY state, and attaches the PHY 923 * to the mac driver. 924 * Return value: 925 * 0 on success 926 */ 927 static int stmmac_init_phy(struct net_device *dev) 928 { 929 struct stmmac_priv *priv = netdev_priv(dev); 930 struct phy_device *phydev; 931 char phy_id_fmt[MII_BUS_ID_SIZE + 3]; 932 char bus_id[MII_BUS_ID_SIZE]; 933 int interface = priv->plat->interface; 934 int max_speed = priv->plat->max_speed; 935 priv->oldlink = false; 936 priv->speed = SPEED_UNKNOWN; 937 priv->oldduplex = DUPLEX_UNKNOWN; 938 939 if (priv->plat->phy_node) { 940 phydev = of_phy_connect(dev, priv->plat->phy_node, 941 &stmmac_adjust_link, 0, interface); 942 } else { 943 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x", 944 priv->plat->bus_id); 945 946 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id, 947 priv->plat->phy_addr); 948 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__, 949 phy_id_fmt); 950 951 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link, 952 interface); 953 } 954 955 if (IS_ERR_OR_NULL(phydev)) { 956 netdev_err(priv->dev, "Could not attach to PHY\n"); 957 if (!phydev) 958 return -ENODEV; 959 960 return PTR_ERR(phydev); 961 } 962 963 /* Stop Advertising 1000BASE Capability if interface is not GMII */ 964 if ((interface == PHY_INTERFACE_MODE_MII) || 965 (interface == PHY_INTERFACE_MODE_RMII) || 966 (max_speed < 1000 && max_speed > 0)) 967 phydev->advertising &= ~(SUPPORTED_1000baseT_Half | 968 SUPPORTED_1000baseT_Full); 969 970 /* 971 * Broken HW is sometimes missing the pull-up resistor on the 972 * MDIO line, which results in reads to non-existent devices returning 973 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent 974 * device as well. 975 * Note: phydev->phy_id is the result of reading the UID PHY registers. 976 */ 977 if (!priv->plat->phy_node && phydev->phy_id == 0) { 978 phy_disconnect(phydev); 979 return -ENODEV; 980 } 981 982 /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid 983 * subsequent PHY polling, make sure we force a link transition if 984 * we have a UP/DOWN/UP transition 985 */ 986 if (phydev->is_pseudo_fixed_link) 987 phydev->irq = PHY_POLL; 988 989 phy_attached_info(phydev); 990 return 0; 991 } 992 993 static void stmmac_display_rx_rings(struct stmmac_priv *priv) 994 { 995 u32 rx_cnt = priv->plat->rx_queues_to_use; 996 void *head_rx; 997 u32 queue; 998 999 /* Display RX rings */ 1000 for (queue = 0; queue < rx_cnt; queue++) { 1001 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1002 1003 pr_info("\tRX Queue %u rings\n", queue); 1004 1005 if (priv->extend_desc) 1006 head_rx = (void *)rx_q->dma_erx; 1007 else 1008 head_rx = (void *)rx_q->dma_rx; 1009 1010 /* Display RX ring */ 1011 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true); 1012 } 1013 } 1014 1015 static void stmmac_display_tx_rings(struct stmmac_priv *priv) 1016 { 1017 u32 tx_cnt = priv->plat->tx_queues_to_use; 1018 void *head_tx; 1019 u32 queue; 1020 1021 /* Display TX rings */ 1022 for (queue = 0; queue < tx_cnt; queue++) { 1023 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1024 1025 pr_info("\tTX Queue %d rings\n", queue); 1026 1027 if (priv->extend_desc) 1028 head_tx = (void *)tx_q->dma_etx; 1029 else 1030 head_tx = (void *)tx_q->dma_tx; 1031 1032 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false); 1033 } 1034 } 1035 1036 static void stmmac_display_rings(struct stmmac_priv *priv) 1037 { 1038 /* Display RX ring */ 1039 stmmac_display_rx_rings(priv); 1040 1041 /* Display TX ring */ 1042 stmmac_display_tx_rings(priv); 1043 } 1044 1045 static int stmmac_set_bfsize(int mtu, int bufsize) 1046 { 1047 int ret = bufsize; 1048 1049 if (mtu >= BUF_SIZE_4KiB) 1050 ret = BUF_SIZE_8KiB; 1051 else if (mtu >= BUF_SIZE_2KiB) 1052 ret = BUF_SIZE_4KiB; 1053 else if (mtu > DEFAULT_BUFSIZE) 1054 ret = BUF_SIZE_2KiB; 1055 else 1056 ret = DEFAULT_BUFSIZE; 1057 1058 return ret; 1059 } 1060 1061 /** 1062 * stmmac_clear_rx_descriptors - clear RX descriptors 1063 * @priv: driver private structure 1064 * @queue: RX queue index 1065 * Description: this function is called to clear the RX descriptors 1066 * in case of both basic and extended descriptors are used. 1067 */ 1068 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) 1069 { 1070 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1071 int i; 1072 1073 /* Clear the RX descriptors */ 1074 for (i = 0; i < DMA_RX_SIZE; i++) 1075 if (priv->extend_desc) 1076 priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic, 1077 priv->use_riwt, priv->mode, 1078 (i == DMA_RX_SIZE - 1)); 1079 else 1080 priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i], 1081 priv->use_riwt, priv->mode, 1082 (i == DMA_RX_SIZE - 1)); 1083 } 1084 1085 /** 1086 * stmmac_clear_tx_descriptors - clear tx descriptors 1087 * @priv: driver private structure 1088 * @queue: TX queue index. 1089 * Description: this function is called to clear the TX descriptors 1090 * in case of both basic and extended descriptors are used. 1091 */ 1092 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) 1093 { 1094 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1095 int i; 1096 1097 /* Clear the TX descriptors */ 1098 for (i = 0; i < DMA_TX_SIZE; i++) 1099 if (priv->extend_desc) 1100 priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic, 1101 priv->mode, 1102 (i == DMA_TX_SIZE - 1)); 1103 else 1104 priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i], 1105 priv->mode, 1106 (i == DMA_TX_SIZE - 1)); 1107 } 1108 1109 /** 1110 * stmmac_clear_descriptors - clear descriptors 1111 * @priv: driver private structure 1112 * Description: this function is called to clear the TX and RX descriptors 1113 * in case of both basic and extended descriptors are used. 1114 */ 1115 static void stmmac_clear_descriptors(struct stmmac_priv *priv) 1116 { 1117 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1118 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1119 u32 queue; 1120 1121 /* Clear the RX descriptors */ 1122 for (queue = 0; queue < rx_queue_cnt; queue++) 1123 stmmac_clear_rx_descriptors(priv, queue); 1124 1125 /* Clear the TX descriptors */ 1126 for (queue = 0; queue < tx_queue_cnt; queue++) 1127 stmmac_clear_tx_descriptors(priv, queue); 1128 } 1129 1130 /** 1131 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1132 * @priv: driver private structure 1133 * @p: descriptor pointer 1134 * @i: descriptor index 1135 * @flags: gfp flag 1136 * @queue: RX queue index 1137 * Description: this function is called to allocate a receive buffer, perform 1138 * the DMA mapping and init the descriptor. 1139 */ 1140 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, 1141 int i, gfp_t flags, u32 queue) 1142 { 1143 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1144 struct sk_buff *skb; 1145 1146 skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags); 1147 if (!skb) { 1148 netdev_err(priv->dev, 1149 "%s: Rx init fails; skb is NULL\n", __func__); 1150 return -ENOMEM; 1151 } 1152 rx_q->rx_skbuff[i] = skb; 1153 rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data, 1154 priv->dma_buf_sz, 1155 DMA_FROM_DEVICE); 1156 if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) { 1157 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__); 1158 dev_kfree_skb_any(skb); 1159 return -EINVAL; 1160 } 1161 1162 if (priv->synopsys_id >= DWMAC_CORE_4_00) 1163 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]); 1164 else 1165 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]); 1166 1167 if ((priv->hw->mode->init_desc3) && 1168 (priv->dma_buf_sz == BUF_SIZE_16KiB)) 1169 priv->hw->mode->init_desc3(p); 1170 1171 return 0; 1172 } 1173 1174 /** 1175 * stmmac_free_rx_buffer - free RX dma buffers 1176 * @priv: private structure 1177 * @queue: RX queue index 1178 * @i: buffer index. 1179 */ 1180 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1181 { 1182 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1183 1184 if (rx_q->rx_skbuff[i]) { 1185 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i], 1186 priv->dma_buf_sz, DMA_FROM_DEVICE); 1187 dev_kfree_skb_any(rx_q->rx_skbuff[i]); 1188 } 1189 rx_q->rx_skbuff[i] = NULL; 1190 } 1191 1192 /** 1193 * stmmac_free_tx_buffer - free RX dma buffers 1194 * @priv: private structure 1195 * @queue: RX queue index 1196 * @i: buffer index. 1197 */ 1198 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1199 { 1200 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1201 1202 if (tx_q->tx_skbuff_dma[i].buf) { 1203 if (tx_q->tx_skbuff_dma[i].map_as_page) 1204 dma_unmap_page(priv->device, 1205 tx_q->tx_skbuff_dma[i].buf, 1206 tx_q->tx_skbuff_dma[i].len, 1207 DMA_TO_DEVICE); 1208 else 1209 dma_unmap_single(priv->device, 1210 tx_q->tx_skbuff_dma[i].buf, 1211 tx_q->tx_skbuff_dma[i].len, 1212 DMA_TO_DEVICE); 1213 } 1214 1215 if (tx_q->tx_skbuff[i]) { 1216 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1217 tx_q->tx_skbuff[i] = NULL; 1218 tx_q->tx_skbuff_dma[i].buf = 0; 1219 tx_q->tx_skbuff_dma[i].map_as_page = false; 1220 } 1221 } 1222 1223 /** 1224 * init_dma_rx_desc_rings - init the RX descriptor rings 1225 * @dev: net device structure 1226 * @flags: gfp flag. 1227 * Description: this function initializes the DMA RX descriptors 1228 * and allocates the socket buffers. It supports the chained and ring 1229 * modes. 1230 */ 1231 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) 1232 { 1233 struct stmmac_priv *priv = netdev_priv(dev); 1234 u32 rx_count = priv->plat->rx_queues_to_use; 1235 unsigned int bfsize = 0; 1236 int ret = -ENOMEM; 1237 int queue; 1238 int i; 1239 1240 if (priv->hw->mode->set_16kib_bfsize) 1241 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu); 1242 1243 if (bfsize < BUF_SIZE_16KiB) 1244 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); 1245 1246 priv->dma_buf_sz = bfsize; 1247 1248 /* RX INITIALIZATION */ 1249 netif_dbg(priv, probe, priv->dev, 1250 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1251 1252 for (queue = 0; queue < rx_count; queue++) { 1253 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1254 1255 netif_dbg(priv, probe, priv->dev, 1256 "(%s) dma_rx_phy=0x%08x\n", __func__, 1257 (u32)rx_q->dma_rx_phy); 1258 1259 for (i = 0; i < DMA_RX_SIZE; i++) { 1260 struct dma_desc *p; 1261 1262 if (priv->extend_desc) 1263 p = &((rx_q->dma_erx + i)->basic); 1264 else 1265 p = rx_q->dma_rx + i; 1266 1267 ret = stmmac_init_rx_buffers(priv, p, i, flags, 1268 queue); 1269 if (ret) 1270 goto err_init_rx_buffers; 1271 1272 netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n", 1273 rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data, 1274 (unsigned int)rx_q->rx_skbuff_dma[i]); 1275 } 1276 1277 rx_q->cur_rx = 0; 1278 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE); 1279 1280 stmmac_clear_rx_descriptors(priv, queue); 1281 1282 /* Setup the chained descriptor addresses */ 1283 if (priv->mode == STMMAC_CHAIN_MODE) { 1284 if (priv->extend_desc) 1285 priv->hw->mode->init(rx_q->dma_erx, 1286 rx_q->dma_rx_phy, 1287 DMA_RX_SIZE, 1); 1288 else 1289 priv->hw->mode->init(rx_q->dma_rx, 1290 rx_q->dma_rx_phy, 1291 DMA_RX_SIZE, 0); 1292 } 1293 } 1294 1295 buf_sz = bfsize; 1296 1297 return 0; 1298 1299 err_init_rx_buffers: 1300 while (queue >= 0) { 1301 while (--i >= 0) 1302 stmmac_free_rx_buffer(priv, queue, i); 1303 1304 if (queue == 0) 1305 break; 1306 1307 i = DMA_RX_SIZE; 1308 queue--; 1309 } 1310 1311 return ret; 1312 } 1313 1314 /** 1315 * init_dma_tx_desc_rings - init the TX descriptor rings 1316 * @dev: net device structure. 1317 * Description: this function initializes the DMA TX descriptors 1318 * and allocates the socket buffers. It supports the chained and ring 1319 * modes. 1320 */ 1321 static int init_dma_tx_desc_rings(struct net_device *dev) 1322 { 1323 struct stmmac_priv *priv = netdev_priv(dev); 1324 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1325 u32 queue; 1326 int i; 1327 1328 for (queue = 0; queue < tx_queue_cnt; queue++) { 1329 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1330 1331 netif_dbg(priv, probe, priv->dev, 1332 "(%s) dma_tx_phy=0x%08x\n", __func__, 1333 (u32)tx_q->dma_tx_phy); 1334 1335 /* Setup the chained descriptor addresses */ 1336 if (priv->mode == STMMAC_CHAIN_MODE) { 1337 if (priv->extend_desc) 1338 priv->hw->mode->init(tx_q->dma_etx, 1339 tx_q->dma_tx_phy, 1340 DMA_TX_SIZE, 1); 1341 else 1342 priv->hw->mode->init(tx_q->dma_tx, 1343 tx_q->dma_tx_phy, 1344 DMA_TX_SIZE, 0); 1345 } 1346 1347 for (i = 0; i < DMA_TX_SIZE; i++) { 1348 struct dma_desc *p; 1349 if (priv->extend_desc) 1350 p = &((tx_q->dma_etx + i)->basic); 1351 else 1352 p = tx_q->dma_tx + i; 1353 1354 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1355 p->des0 = 0; 1356 p->des1 = 0; 1357 p->des2 = 0; 1358 p->des3 = 0; 1359 } else { 1360 p->des2 = 0; 1361 } 1362 1363 tx_q->tx_skbuff_dma[i].buf = 0; 1364 tx_q->tx_skbuff_dma[i].map_as_page = false; 1365 tx_q->tx_skbuff_dma[i].len = 0; 1366 tx_q->tx_skbuff_dma[i].last_segment = false; 1367 tx_q->tx_skbuff[i] = NULL; 1368 } 1369 1370 tx_q->dirty_tx = 0; 1371 tx_q->cur_tx = 0; 1372 tx_q->mss = 0; 1373 1374 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 1375 } 1376 1377 return 0; 1378 } 1379 1380 /** 1381 * init_dma_desc_rings - init the RX/TX descriptor rings 1382 * @dev: net device structure 1383 * @flags: gfp flag. 1384 * Description: this function initializes the DMA RX/TX descriptors 1385 * and allocates the socket buffers. It supports the chained and ring 1386 * modes. 1387 */ 1388 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags) 1389 { 1390 struct stmmac_priv *priv = netdev_priv(dev); 1391 int ret; 1392 1393 ret = init_dma_rx_desc_rings(dev, flags); 1394 if (ret) 1395 return ret; 1396 1397 ret = init_dma_tx_desc_rings(dev); 1398 1399 stmmac_clear_descriptors(priv); 1400 1401 if (netif_msg_hw(priv)) 1402 stmmac_display_rings(priv); 1403 1404 return ret; 1405 } 1406 1407 /** 1408 * dma_free_rx_skbufs - free RX dma buffers 1409 * @priv: private structure 1410 * @queue: RX queue index 1411 */ 1412 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) 1413 { 1414 int i; 1415 1416 for (i = 0; i < DMA_RX_SIZE; i++) 1417 stmmac_free_rx_buffer(priv, queue, i); 1418 } 1419 1420 /** 1421 * dma_free_tx_skbufs - free TX dma buffers 1422 * @priv: private structure 1423 * @queue: TX queue index 1424 */ 1425 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) 1426 { 1427 int i; 1428 1429 for (i = 0; i < DMA_TX_SIZE; i++) 1430 stmmac_free_tx_buffer(priv, queue, i); 1431 } 1432 1433 /** 1434 * free_dma_rx_desc_resources - free RX dma desc resources 1435 * @priv: private structure 1436 */ 1437 static void free_dma_rx_desc_resources(struct stmmac_priv *priv) 1438 { 1439 u32 rx_count = priv->plat->rx_queues_to_use; 1440 u32 queue; 1441 1442 /* Free RX queue resources */ 1443 for (queue = 0; queue < rx_count; queue++) { 1444 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1445 1446 /* Release the DMA RX socket buffers */ 1447 dma_free_rx_skbufs(priv, queue); 1448 1449 /* Free DMA regions of consistent memory previously allocated */ 1450 if (!priv->extend_desc) 1451 dma_free_coherent(priv->device, 1452 DMA_RX_SIZE * sizeof(struct dma_desc), 1453 rx_q->dma_rx, rx_q->dma_rx_phy); 1454 else 1455 dma_free_coherent(priv->device, DMA_RX_SIZE * 1456 sizeof(struct dma_extended_desc), 1457 rx_q->dma_erx, rx_q->dma_rx_phy); 1458 1459 kfree(rx_q->rx_skbuff_dma); 1460 kfree(rx_q->rx_skbuff); 1461 } 1462 } 1463 1464 /** 1465 * free_dma_tx_desc_resources - free TX dma desc resources 1466 * @priv: private structure 1467 */ 1468 static void free_dma_tx_desc_resources(struct stmmac_priv *priv) 1469 { 1470 u32 tx_count = priv->plat->tx_queues_to_use; 1471 u32 queue; 1472 1473 /* Free TX queue resources */ 1474 for (queue = 0; queue < tx_count; queue++) { 1475 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1476 1477 /* Release the DMA TX socket buffers */ 1478 dma_free_tx_skbufs(priv, queue); 1479 1480 /* Free DMA regions of consistent memory previously allocated */ 1481 if (!priv->extend_desc) 1482 dma_free_coherent(priv->device, 1483 DMA_TX_SIZE * sizeof(struct dma_desc), 1484 tx_q->dma_tx, tx_q->dma_tx_phy); 1485 else 1486 dma_free_coherent(priv->device, DMA_TX_SIZE * 1487 sizeof(struct dma_extended_desc), 1488 tx_q->dma_etx, tx_q->dma_tx_phy); 1489 1490 kfree(tx_q->tx_skbuff_dma); 1491 kfree(tx_q->tx_skbuff); 1492 } 1493 } 1494 1495 /** 1496 * alloc_dma_rx_desc_resources - alloc RX resources. 1497 * @priv: private structure 1498 * Description: according to which descriptor can be used (extend or basic) 1499 * this function allocates the resources for TX and RX paths. In case of 1500 * reception, for example, it pre-allocated the RX socket buffer in order to 1501 * allow zero-copy mechanism. 1502 */ 1503 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) 1504 { 1505 u32 rx_count = priv->plat->rx_queues_to_use; 1506 int ret = -ENOMEM; 1507 u32 queue; 1508 1509 /* RX queues buffers and DMA */ 1510 for (queue = 0; queue < rx_count; queue++) { 1511 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1512 1513 rx_q->queue_index = queue; 1514 rx_q->priv_data = priv; 1515 1516 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE, 1517 sizeof(dma_addr_t), 1518 GFP_KERNEL); 1519 if (!rx_q->rx_skbuff_dma) 1520 goto err_dma; 1521 1522 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE, 1523 sizeof(struct sk_buff *), 1524 GFP_KERNEL); 1525 if (!rx_q->rx_skbuff) 1526 goto err_dma; 1527 1528 if (priv->extend_desc) { 1529 rx_q->dma_erx = dma_zalloc_coherent(priv->device, 1530 DMA_RX_SIZE * 1531 sizeof(struct 1532 dma_extended_desc), 1533 &rx_q->dma_rx_phy, 1534 GFP_KERNEL); 1535 if (!rx_q->dma_erx) 1536 goto err_dma; 1537 1538 } else { 1539 rx_q->dma_rx = dma_zalloc_coherent(priv->device, 1540 DMA_RX_SIZE * 1541 sizeof(struct 1542 dma_desc), 1543 &rx_q->dma_rx_phy, 1544 GFP_KERNEL); 1545 if (!rx_q->dma_rx) 1546 goto err_dma; 1547 } 1548 } 1549 1550 return 0; 1551 1552 err_dma: 1553 free_dma_rx_desc_resources(priv); 1554 1555 return ret; 1556 } 1557 1558 /** 1559 * alloc_dma_tx_desc_resources - alloc TX resources. 1560 * @priv: private structure 1561 * Description: according to which descriptor can be used (extend or basic) 1562 * this function allocates the resources for TX and RX paths. In case of 1563 * reception, for example, it pre-allocated the RX socket buffer in order to 1564 * allow zero-copy mechanism. 1565 */ 1566 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) 1567 { 1568 u32 tx_count = priv->plat->tx_queues_to_use; 1569 int ret = -ENOMEM; 1570 u32 queue; 1571 1572 /* TX queues buffers and DMA */ 1573 for (queue = 0; queue < tx_count; queue++) { 1574 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1575 1576 tx_q->queue_index = queue; 1577 tx_q->priv_data = priv; 1578 1579 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE, 1580 sizeof(*tx_q->tx_skbuff_dma), 1581 GFP_KERNEL); 1582 if (!tx_q->tx_skbuff_dma) 1583 goto err_dma; 1584 1585 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE, 1586 sizeof(struct sk_buff *), 1587 GFP_KERNEL); 1588 if (!tx_q->tx_skbuff) 1589 goto err_dma; 1590 1591 if (priv->extend_desc) { 1592 tx_q->dma_etx = dma_zalloc_coherent(priv->device, 1593 DMA_TX_SIZE * 1594 sizeof(struct 1595 dma_extended_desc), 1596 &tx_q->dma_tx_phy, 1597 GFP_KERNEL); 1598 if (!tx_q->dma_etx) 1599 goto err_dma; 1600 } else { 1601 tx_q->dma_tx = dma_zalloc_coherent(priv->device, 1602 DMA_TX_SIZE * 1603 sizeof(struct 1604 dma_desc), 1605 &tx_q->dma_tx_phy, 1606 GFP_KERNEL); 1607 if (!tx_q->dma_tx) 1608 goto err_dma; 1609 } 1610 } 1611 1612 return 0; 1613 1614 err_dma: 1615 free_dma_tx_desc_resources(priv); 1616 1617 return ret; 1618 } 1619 1620 /** 1621 * alloc_dma_desc_resources - alloc TX/RX resources. 1622 * @priv: private structure 1623 * Description: according to which descriptor can be used (extend or basic) 1624 * this function allocates the resources for TX and RX paths. In case of 1625 * reception, for example, it pre-allocated the RX socket buffer in order to 1626 * allow zero-copy mechanism. 1627 */ 1628 static int alloc_dma_desc_resources(struct stmmac_priv *priv) 1629 { 1630 /* RX Allocation */ 1631 int ret = alloc_dma_rx_desc_resources(priv); 1632 1633 if (ret) 1634 return ret; 1635 1636 ret = alloc_dma_tx_desc_resources(priv); 1637 1638 return ret; 1639 } 1640 1641 /** 1642 * free_dma_desc_resources - free dma desc resources 1643 * @priv: private structure 1644 */ 1645 static void free_dma_desc_resources(struct stmmac_priv *priv) 1646 { 1647 /* Release the DMA RX socket buffers */ 1648 free_dma_rx_desc_resources(priv); 1649 1650 /* Release the DMA TX socket buffers */ 1651 free_dma_tx_desc_resources(priv); 1652 } 1653 1654 /** 1655 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 1656 * @priv: driver private structure 1657 * Description: It is used for enabling the rx queues in the MAC 1658 */ 1659 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 1660 { 1661 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1662 int queue; 1663 u8 mode; 1664 1665 for (queue = 0; queue < rx_queues_count; queue++) { 1666 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 1667 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue); 1668 } 1669 } 1670 1671 /** 1672 * stmmac_start_rx_dma - start RX DMA channel 1673 * @priv: driver private structure 1674 * @chan: RX channel index 1675 * Description: 1676 * This starts a RX DMA channel 1677 */ 1678 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 1679 { 1680 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 1681 priv->hw->dma->start_rx(priv->ioaddr, chan); 1682 } 1683 1684 /** 1685 * stmmac_start_tx_dma - start TX DMA channel 1686 * @priv: driver private structure 1687 * @chan: TX channel index 1688 * Description: 1689 * This starts a TX DMA channel 1690 */ 1691 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 1692 { 1693 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 1694 priv->hw->dma->start_tx(priv->ioaddr, chan); 1695 } 1696 1697 /** 1698 * stmmac_stop_rx_dma - stop RX DMA channel 1699 * @priv: driver private structure 1700 * @chan: RX channel index 1701 * Description: 1702 * This stops a RX DMA channel 1703 */ 1704 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 1705 { 1706 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 1707 priv->hw->dma->stop_rx(priv->ioaddr, chan); 1708 } 1709 1710 /** 1711 * stmmac_stop_tx_dma - stop TX DMA channel 1712 * @priv: driver private structure 1713 * @chan: TX channel index 1714 * Description: 1715 * This stops a TX DMA channel 1716 */ 1717 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 1718 { 1719 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 1720 priv->hw->dma->stop_tx(priv->ioaddr, chan); 1721 } 1722 1723 /** 1724 * stmmac_start_all_dma - start all RX and TX DMA channels 1725 * @priv: driver private structure 1726 * Description: 1727 * This starts all the RX and TX DMA channels 1728 */ 1729 static void stmmac_start_all_dma(struct stmmac_priv *priv) 1730 { 1731 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1732 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1733 u32 chan = 0; 1734 1735 for (chan = 0; chan < rx_channels_count; chan++) 1736 stmmac_start_rx_dma(priv, chan); 1737 1738 for (chan = 0; chan < tx_channels_count; chan++) 1739 stmmac_start_tx_dma(priv, chan); 1740 } 1741 1742 /** 1743 * stmmac_stop_all_dma - stop all RX and TX DMA channels 1744 * @priv: driver private structure 1745 * Description: 1746 * This stops the RX and TX DMA channels 1747 */ 1748 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 1749 { 1750 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1751 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1752 u32 chan = 0; 1753 1754 for (chan = 0; chan < rx_channels_count; chan++) 1755 stmmac_stop_rx_dma(priv, chan); 1756 1757 for (chan = 0; chan < tx_channels_count; chan++) 1758 stmmac_stop_tx_dma(priv, chan); 1759 } 1760 1761 /** 1762 * stmmac_dma_operation_mode - HW DMA operation mode 1763 * @priv: driver private structure 1764 * Description: it is used for configuring the DMA operation mode register in 1765 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 1766 */ 1767 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 1768 { 1769 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1770 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1771 int rxfifosz = priv->plat->rx_fifo_size; 1772 int txfifosz = priv->plat->tx_fifo_size; 1773 u32 txmode = 0; 1774 u32 rxmode = 0; 1775 u32 chan = 0; 1776 u8 qmode = 0; 1777 1778 if (rxfifosz == 0) 1779 rxfifosz = priv->dma_cap.rx_fifo_size; 1780 if (txfifosz == 0) 1781 txfifosz = priv->dma_cap.tx_fifo_size; 1782 1783 /* Adjust for real per queue fifo size */ 1784 rxfifosz /= rx_channels_count; 1785 txfifosz /= tx_channels_count; 1786 1787 if (priv->plat->force_thresh_dma_mode) { 1788 txmode = tc; 1789 rxmode = tc; 1790 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 1791 /* 1792 * In case of GMAC, SF mode can be enabled 1793 * to perform the TX COE in HW. This depends on: 1794 * 1) TX COE if actually supported 1795 * 2) There is no bugged Jumbo frame support 1796 * that needs to not insert csum in the TDES. 1797 */ 1798 txmode = SF_DMA_MODE; 1799 rxmode = SF_DMA_MODE; 1800 priv->xstats.threshold = SF_DMA_MODE; 1801 } else { 1802 txmode = tc; 1803 rxmode = SF_DMA_MODE; 1804 } 1805 1806 /* configure all channels */ 1807 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 1808 for (chan = 0; chan < rx_channels_count; chan++) { 1809 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 1810 1811 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan, 1812 rxfifosz, qmode); 1813 } 1814 1815 for (chan = 0; chan < tx_channels_count; chan++) { 1816 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 1817 1818 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan, 1819 txfifosz, qmode); 1820 } 1821 } else { 1822 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode, 1823 rxfifosz); 1824 } 1825 } 1826 1827 /** 1828 * stmmac_tx_clean - to manage the transmission completion 1829 * @priv: driver private structure 1830 * @queue: TX queue index 1831 * Description: it reclaims the transmit resources after transmission completes. 1832 */ 1833 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue) 1834 { 1835 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1836 unsigned int bytes_compl = 0, pkts_compl = 0; 1837 unsigned int entry; 1838 1839 netif_tx_lock(priv->dev); 1840 1841 priv->xstats.tx_clean++; 1842 1843 entry = tx_q->dirty_tx; 1844 while (entry != tx_q->cur_tx) { 1845 struct sk_buff *skb = tx_q->tx_skbuff[entry]; 1846 struct dma_desc *p; 1847 int status; 1848 1849 if (priv->extend_desc) 1850 p = (struct dma_desc *)(tx_q->dma_etx + entry); 1851 else 1852 p = tx_q->dma_tx + entry; 1853 1854 status = priv->hw->desc->tx_status(&priv->dev->stats, 1855 &priv->xstats, p, 1856 priv->ioaddr); 1857 /* Check if the descriptor is owned by the DMA */ 1858 if (unlikely(status & tx_dma_own)) 1859 break; 1860 1861 /* Make sure descriptor fields are read after reading 1862 * the own bit. 1863 */ 1864 dma_rmb(); 1865 1866 /* Just consider the last segment and ...*/ 1867 if (likely(!(status & tx_not_ls))) { 1868 /* ... verify the status error condition */ 1869 if (unlikely(status & tx_err)) { 1870 priv->dev->stats.tx_errors++; 1871 } else { 1872 priv->dev->stats.tx_packets++; 1873 priv->xstats.tx_pkt_n++; 1874 } 1875 stmmac_get_tx_hwtstamp(priv, p, skb); 1876 } 1877 1878 if (likely(tx_q->tx_skbuff_dma[entry].buf)) { 1879 if (tx_q->tx_skbuff_dma[entry].map_as_page) 1880 dma_unmap_page(priv->device, 1881 tx_q->tx_skbuff_dma[entry].buf, 1882 tx_q->tx_skbuff_dma[entry].len, 1883 DMA_TO_DEVICE); 1884 else 1885 dma_unmap_single(priv->device, 1886 tx_q->tx_skbuff_dma[entry].buf, 1887 tx_q->tx_skbuff_dma[entry].len, 1888 DMA_TO_DEVICE); 1889 tx_q->tx_skbuff_dma[entry].buf = 0; 1890 tx_q->tx_skbuff_dma[entry].len = 0; 1891 tx_q->tx_skbuff_dma[entry].map_as_page = false; 1892 } 1893 1894 if (priv->hw->mode->clean_desc3) 1895 priv->hw->mode->clean_desc3(tx_q, p); 1896 1897 tx_q->tx_skbuff_dma[entry].last_segment = false; 1898 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1899 1900 if (likely(skb != NULL)) { 1901 pkts_compl++; 1902 bytes_compl += skb->len; 1903 dev_consume_skb_any(skb); 1904 tx_q->tx_skbuff[entry] = NULL; 1905 } 1906 1907 priv->hw->desc->release_tx_desc(p, priv->mode); 1908 1909 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 1910 } 1911 tx_q->dirty_tx = entry; 1912 1913 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 1914 pkts_compl, bytes_compl); 1915 1916 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 1917 queue))) && 1918 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) { 1919 1920 netif_dbg(priv, tx_done, priv->dev, 1921 "%s: restart transmit\n", __func__); 1922 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 1923 } 1924 1925 if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) { 1926 stmmac_enable_eee_mode(priv); 1927 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 1928 } 1929 netif_tx_unlock(priv->dev); 1930 } 1931 1932 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan) 1933 { 1934 priv->hw->dma->enable_dma_irq(priv->ioaddr, chan); 1935 } 1936 1937 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan) 1938 { 1939 priv->hw->dma->disable_dma_irq(priv->ioaddr, chan); 1940 } 1941 1942 /** 1943 * stmmac_tx_err - to manage the tx error 1944 * @priv: driver private structure 1945 * @chan: channel index 1946 * Description: it cleans the descriptors and restarts the transmission 1947 * in case of transmission errors. 1948 */ 1949 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 1950 { 1951 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 1952 int i; 1953 1954 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 1955 1956 stmmac_stop_tx_dma(priv, chan); 1957 dma_free_tx_skbufs(priv, chan); 1958 for (i = 0; i < DMA_TX_SIZE; i++) 1959 if (priv->extend_desc) 1960 priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic, 1961 priv->mode, 1962 (i == DMA_TX_SIZE - 1)); 1963 else 1964 priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i], 1965 priv->mode, 1966 (i == DMA_TX_SIZE - 1)); 1967 tx_q->dirty_tx = 0; 1968 tx_q->cur_tx = 0; 1969 tx_q->mss = 0; 1970 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); 1971 stmmac_start_tx_dma(priv, chan); 1972 1973 priv->dev->stats.tx_errors++; 1974 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 1975 } 1976 1977 /** 1978 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 1979 * @priv: driver private structure 1980 * @txmode: TX operating mode 1981 * @rxmode: RX operating mode 1982 * @chan: channel index 1983 * Description: it is used for configuring of the DMA operation mode in 1984 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 1985 * mode. 1986 */ 1987 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 1988 u32 rxmode, u32 chan) 1989 { 1990 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 1991 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 1992 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1993 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1994 int rxfifosz = priv->plat->rx_fifo_size; 1995 int txfifosz = priv->plat->tx_fifo_size; 1996 1997 if (rxfifosz == 0) 1998 rxfifosz = priv->dma_cap.rx_fifo_size; 1999 if (txfifosz == 0) 2000 txfifosz = priv->dma_cap.tx_fifo_size; 2001 2002 /* Adjust for real per queue fifo size */ 2003 rxfifosz /= rx_channels_count; 2004 txfifosz /= tx_channels_count; 2005 2006 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 2007 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan, 2008 rxfifosz, rxqmode); 2009 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan, 2010 txfifosz, txqmode); 2011 } else { 2012 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode, 2013 rxfifosz); 2014 } 2015 } 2016 2017 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2018 { 2019 bool ret = false; 2020 2021 /* Safety features are only available in cores >= 5.10 */ 2022 if (priv->synopsys_id < DWMAC_CORE_5_10) 2023 return ret; 2024 if (priv->hw->mac->safety_feat_irq_status) 2025 ret = priv->hw->mac->safety_feat_irq_status(priv->dev, 2026 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2027 2028 if (ret) 2029 stmmac_global_err(priv); 2030 return ret; 2031 } 2032 2033 /** 2034 * stmmac_dma_interrupt - DMA ISR 2035 * @priv: driver private structure 2036 * Description: this is the DMA ISR. It is called by the main ISR. 2037 * It calls the dwmac dma routine and schedule poll method in case of some 2038 * work can be done. 2039 */ 2040 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2041 { 2042 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2043 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2044 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2045 tx_channel_count : rx_channel_count; 2046 u32 chan; 2047 bool poll_scheduled = false; 2048 int status[channels_to_check]; 2049 2050 /* Each DMA channel can be used for rx and tx simultaneously, yet 2051 * napi_struct is embedded in struct stmmac_rx_queue rather than in a 2052 * stmmac_channel struct. 2053 * Because of this, stmmac_poll currently checks (and possibly wakes) 2054 * all tx queues rather than just a single tx queue. 2055 */ 2056 for (chan = 0; chan < channels_to_check; chan++) 2057 status[chan] = priv->hw->dma->dma_interrupt(priv->ioaddr, 2058 &priv->xstats, 2059 chan); 2060 2061 for (chan = 0; chan < rx_channel_count; chan++) { 2062 if (likely(status[chan] & handle_rx)) { 2063 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; 2064 2065 if (likely(napi_schedule_prep(&rx_q->napi))) { 2066 stmmac_disable_dma_irq(priv, chan); 2067 __napi_schedule(&rx_q->napi); 2068 poll_scheduled = true; 2069 } 2070 } 2071 } 2072 2073 /* If we scheduled poll, we already know that tx queues will be checked. 2074 * If we didn't schedule poll, see if any DMA channel (used by tx) has a 2075 * completed transmission, if so, call stmmac_poll (once). 2076 */ 2077 if (!poll_scheduled) { 2078 for (chan = 0; chan < tx_channel_count; chan++) { 2079 if (status[chan] & handle_tx) { 2080 /* It doesn't matter what rx queue we choose 2081 * here. We use 0 since it always exists. 2082 */ 2083 struct stmmac_rx_queue *rx_q = 2084 &priv->rx_queue[0]; 2085 2086 if (likely(napi_schedule_prep(&rx_q->napi))) { 2087 stmmac_disable_dma_irq(priv, chan); 2088 __napi_schedule(&rx_q->napi); 2089 } 2090 break; 2091 } 2092 } 2093 } 2094 2095 for (chan = 0; chan < tx_channel_count; chan++) { 2096 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2097 /* Try to bump up the dma threshold on this failure */ 2098 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && 2099 (tc <= 256)) { 2100 tc += 64; 2101 if (priv->plat->force_thresh_dma_mode) 2102 stmmac_set_dma_operation_mode(priv, 2103 tc, 2104 tc, 2105 chan); 2106 else 2107 stmmac_set_dma_operation_mode(priv, 2108 tc, 2109 SF_DMA_MODE, 2110 chan); 2111 priv->xstats.threshold = tc; 2112 } 2113 } else if (unlikely(status[chan] == tx_hard_error)) { 2114 stmmac_tx_err(priv, chan); 2115 } 2116 } 2117 } 2118 2119 /** 2120 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2121 * @priv: driver private structure 2122 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2123 */ 2124 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2125 { 2126 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2127 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2128 2129 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 2130 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET; 2131 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET; 2132 } else { 2133 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET; 2134 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET; 2135 } 2136 2137 dwmac_mmc_intr_all_mask(priv->mmcaddr); 2138 2139 if (priv->dma_cap.rmon) { 2140 dwmac_mmc_ctrl(priv->mmcaddr, mode); 2141 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2142 } else 2143 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2144 } 2145 2146 /** 2147 * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors 2148 * @priv: driver private structure 2149 * Description: select the Enhanced/Alternate or Normal descriptors. 2150 * In case of Enhanced/Alternate, it checks if the extended descriptors are 2151 * supported by the HW capability register. 2152 */ 2153 static void stmmac_selec_desc_mode(struct stmmac_priv *priv) 2154 { 2155 if (priv->plat->enh_desc) { 2156 dev_info(priv->device, "Enhanced/Alternate descriptors\n"); 2157 2158 /* GMAC older than 3.50 has no extended descriptors */ 2159 if (priv->synopsys_id >= DWMAC_CORE_3_50) { 2160 dev_info(priv->device, "Enabled extended descriptors\n"); 2161 priv->extend_desc = 1; 2162 } else 2163 dev_warn(priv->device, "Extended descriptors not supported\n"); 2164 2165 priv->hw->desc = &enh_desc_ops; 2166 } else { 2167 dev_info(priv->device, "Normal descriptors\n"); 2168 priv->hw->desc = &ndesc_ops; 2169 } 2170 } 2171 2172 /** 2173 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2174 * @priv: driver private structure 2175 * Description: 2176 * new GMAC chip generations have a new register to indicate the 2177 * presence of the optional feature/functions. 2178 * This can be also used to override the value passed through the 2179 * platform and necessary for old MAC10/100 and GMAC chips. 2180 */ 2181 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2182 { 2183 u32 ret = 0; 2184 2185 if (priv->hw->dma->get_hw_feature) { 2186 priv->hw->dma->get_hw_feature(priv->ioaddr, 2187 &priv->dma_cap); 2188 ret = 1; 2189 } 2190 2191 return ret; 2192 } 2193 2194 /** 2195 * stmmac_check_ether_addr - check if the MAC addr is valid 2196 * @priv: driver private structure 2197 * Description: 2198 * it is to verify if the MAC address is valid, in case of failures it 2199 * generates a random MAC address 2200 */ 2201 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2202 { 2203 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2204 priv->hw->mac->get_umac_addr(priv->hw, 2205 priv->dev->dev_addr, 0); 2206 if (!is_valid_ether_addr(priv->dev->dev_addr)) 2207 eth_hw_addr_random(priv->dev); 2208 netdev_info(priv->dev, "device MAC address %pM\n", 2209 priv->dev->dev_addr); 2210 } 2211 } 2212 2213 /** 2214 * stmmac_init_dma_engine - DMA init. 2215 * @priv: driver private structure 2216 * Description: 2217 * It inits the DMA invoking the specific MAC/GMAC callback. 2218 * Some DMA parameters can be passed from the platform; 2219 * in case of these are not passed a default is kept for the MAC or GMAC. 2220 */ 2221 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2222 { 2223 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2224 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2225 struct stmmac_rx_queue *rx_q; 2226 struct stmmac_tx_queue *tx_q; 2227 u32 dummy_dma_rx_phy = 0; 2228 u32 dummy_dma_tx_phy = 0; 2229 u32 chan = 0; 2230 int atds = 0; 2231 int ret = 0; 2232 2233 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2234 dev_err(priv->device, "Invalid DMA configuration\n"); 2235 return -EINVAL; 2236 } 2237 2238 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2239 atds = 1; 2240 2241 ret = priv->hw->dma->reset(priv->ioaddr); 2242 if (ret) { 2243 dev_err(priv->device, "Failed to reset the dma\n"); 2244 return ret; 2245 } 2246 2247 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 2248 /* DMA Configuration */ 2249 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg, 2250 dummy_dma_tx_phy, dummy_dma_rx_phy, atds); 2251 2252 /* DMA RX Channel Configuration */ 2253 for (chan = 0; chan < rx_channels_count; chan++) { 2254 rx_q = &priv->rx_queue[chan]; 2255 2256 priv->hw->dma->init_rx_chan(priv->ioaddr, 2257 priv->plat->dma_cfg, 2258 rx_q->dma_rx_phy, chan); 2259 2260 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2261 (DMA_RX_SIZE * sizeof(struct dma_desc)); 2262 priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, 2263 rx_q->rx_tail_addr, 2264 chan); 2265 } 2266 2267 /* DMA TX Channel Configuration */ 2268 for (chan = 0; chan < tx_channels_count; chan++) { 2269 tx_q = &priv->tx_queue[chan]; 2270 2271 priv->hw->dma->init_chan(priv->ioaddr, 2272 priv->plat->dma_cfg, 2273 chan); 2274 2275 priv->hw->dma->init_tx_chan(priv->ioaddr, 2276 priv->plat->dma_cfg, 2277 tx_q->dma_tx_phy, chan); 2278 2279 tx_q->tx_tail_addr = tx_q->dma_tx_phy + 2280 (DMA_TX_SIZE * sizeof(struct dma_desc)); 2281 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, 2282 tx_q->tx_tail_addr, 2283 chan); 2284 } 2285 } else { 2286 rx_q = &priv->rx_queue[chan]; 2287 tx_q = &priv->tx_queue[chan]; 2288 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg, 2289 tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds); 2290 } 2291 2292 if (priv->plat->axi && priv->hw->dma->axi) 2293 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi); 2294 2295 return ret; 2296 } 2297 2298 /** 2299 * stmmac_tx_timer - mitigation sw timer for tx. 2300 * @data: data pointer 2301 * Description: 2302 * This is the timer handler to directly invoke the stmmac_tx_clean. 2303 */ 2304 static void stmmac_tx_timer(struct timer_list *t) 2305 { 2306 struct stmmac_priv *priv = from_timer(priv, t, txtimer); 2307 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2308 u32 queue; 2309 2310 /* let's scan all the tx queues */ 2311 for (queue = 0; queue < tx_queues_count; queue++) 2312 stmmac_tx_clean(priv, queue); 2313 } 2314 2315 /** 2316 * stmmac_init_tx_coalesce - init tx mitigation options. 2317 * @priv: driver private structure 2318 * Description: 2319 * This inits the transmit coalesce parameters: i.e. timer rate, 2320 * timer handler and default threshold used for enabling the 2321 * interrupt on completion bit. 2322 */ 2323 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv) 2324 { 2325 priv->tx_coal_frames = STMMAC_TX_FRAMES; 2326 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER; 2327 timer_setup(&priv->txtimer, stmmac_tx_timer, 0); 2328 priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer); 2329 add_timer(&priv->txtimer); 2330 } 2331 2332 static void stmmac_set_rings_length(struct stmmac_priv *priv) 2333 { 2334 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2335 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2336 u32 chan; 2337 2338 /* set TX ring length */ 2339 if (priv->hw->dma->set_tx_ring_len) { 2340 for (chan = 0; chan < tx_channels_count; chan++) 2341 priv->hw->dma->set_tx_ring_len(priv->ioaddr, 2342 (DMA_TX_SIZE - 1), chan); 2343 } 2344 2345 /* set RX ring length */ 2346 if (priv->hw->dma->set_rx_ring_len) { 2347 for (chan = 0; chan < rx_channels_count; chan++) 2348 priv->hw->dma->set_rx_ring_len(priv->ioaddr, 2349 (DMA_RX_SIZE - 1), chan); 2350 } 2351 } 2352 2353 /** 2354 * stmmac_set_tx_queue_weight - Set TX queue weight 2355 * @priv: driver private structure 2356 * Description: It is used for setting TX queues weight 2357 */ 2358 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 2359 { 2360 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2361 u32 weight; 2362 u32 queue; 2363 2364 for (queue = 0; queue < tx_queues_count; queue++) { 2365 weight = priv->plat->tx_queues_cfg[queue].weight; 2366 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue); 2367 } 2368 } 2369 2370 /** 2371 * stmmac_configure_cbs - Configure CBS in TX queue 2372 * @priv: driver private structure 2373 * Description: It is used for configuring CBS in AVB TX queues 2374 */ 2375 static void stmmac_configure_cbs(struct stmmac_priv *priv) 2376 { 2377 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2378 u32 mode_to_use; 2379 u32 queue; 2380 2381 /* queue 0 is reserved for legacy traffic */ 2382 for (queue = 1; queue < tx_queues_count; queue++) { 2383 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 2384 if (mode_to_use == MTL_QUEUE_DCB) 2385 continue; 2386 2387 priv->hw->mac->config_cbs(priv->hw, 2388 priv->plat->tx_queues_cfg[queue].send_slope, 2389 priv->plat->tx_queues_cfg[queue].idle_slope, 2390 priv->plat->tx_queues_cfg[queue].high_credit, 2391 priv->plat->tx_queues_cfg[queue].low_credit, 2392 queue); 2393 } 2394 } 2395 2396 /** 2397 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 2398 * @priv: driver private structure 2399 * Description: It is used for mapping RX queues to RX dma channels 2400 */ 2401 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 2402 { 2403 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2404 u32 queue; 2405 u32 chan; 2406 2407 for (queue = 0; queue < rx_queues_count; queue++) { 2408 chan = priv->plat->rx_queues_cfg[queue].chan; 2409 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan); 2410 } 2411 } 2412 2413 /** 2414 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 2415 * @priv: driver private structure 2416 * Description: It is used for configuring the RX Queue Priority 2417 */ 2418 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 2419 { 2420 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2421 u32 queue; 2422 u32 prio; 2423 2424 for (queue = 0; queue < rx_queues_count; queue++) { 2425 if (!priv->plat->rx_queues_cfg[queue].use_prio) 2426 continue; 2427 2428 prio = priv->plat->rx_queues_cfg[queue].prio; 2429 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue); 2430 } 2431 } 2432 2433 /** 2434 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 2435 * @priv: driver private structure 2436 * Description: It is used for configuring the TX Queue Priority 2437 */ 2438 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 2439 { 2440 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2441 u32 queue; 2442 u32 prio; 2443 2444 for (queue = 0; queue < tx_queues_count; queue++) { 2445 if (!priv->plat->tx_queues_cfg[queue].use_prio) 2446 continue; 2447 2448 prio = priv->plat->tx_queues_cfg[queue].prio; 2449 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue); 2450 } 2451 } 2452 2453 /** 2454 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 2455 * @priv: driver private structure 2456 * Description: It is used for configuring the RX queue routing 2457 */ 2458 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 2459 { 2460 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2461 u32 queue; 2462 u8 packet; 2463 2464 for (queue = 0; queue < rx_queues_count; queue++) { 2465 /* no specific packet type routing specified for the queue */ 2466 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 2467 continue; 2468 2469 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 2470 priv->hw->mac->rx_queue_routing(priv->hw, packet, queue); 2471 } 2472 } 2473 2474 /** 2475 * stmmac_mtl_configuration - Configure MTL 2476 * @priv: driver private structure 2477 * Description: It is used for configurring MTL 2478 */ 2479 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 2480 { 2481 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2482 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2483 2484 if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight) 2485 stmmac_set_tx_queue_weight(priv); 2486 2487 /* Configure MTL RX algorithms */ 2488 if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms) 2489 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw, 2490 priv->plat->rx_sched_algorithm); 2491 2492 /* Configure MTL TX algorithms */ 2493 if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms) 2494 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw, 2495 priv->plat->tx_sched_algorithm); 2496 2497 /* Configure CBS in AVB TX queues */ 2498 if (tx_queues_count > 1 && priv->hw->mac->config_cbs) 2499 stmmac_configure_cbs(priv); 2500 2501 /* Map RX MTL to DMA channels */ 2502 if (priv->hw->mac->map_mtl_to_dma) 2503 stmmac_rx_queue_dma_chan_map(priv); 2504 2505 /* Enable MAC RX Queues */ 2506 if (priv->hw->mac->rx_queue_enable) 2507 stmmac_mac_enable_rx_queues(priv); 2508 2509 /* Set RX priorities */ 2510 if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio) 2511 stmmac_mac_config_rx_queues_prio(priv); 2512 2513 /* Set TX priorities */ 2514 if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio) 2515 stmmac_mac_config_tx_queues_prio(priv); 2516 2517 /* Set RX routing */ 2518 if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing) 2519 stmmac_mac_config_rx_queues_routing(priv); 2520 } 2521 2522 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 2523 { 2524 if (priv->hw->mac->safety_feat_config && priv->dma_cap.asp) { 2525 netdev_info(priv->dev, "Enabling Safety Features\n"); 2526 priv->hw->mac->safety_feat_config(priv->ioaddr, 2527 priv->dma_cap.asp); 2528 } else { 2529 netdev_info(priv->dev, "No Safety Features support found\n"); 2530 } 2531 } 2532 2533 /** 2534 * stmmac_hw_setup - setup mac in a usable state. 2535 * @dev : pointer to the device structure. 2536 * Description: 2537 * this is the main function to setup the HW in a usable state because the 2538 * dma engine is reset, the core registers are configured (e.g. AXI, 2539 * Checksum features, timers). The DMA is ready to start receiving and 2540 * transmitting. 2541 * Return value: 2542 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2543 * file on failure. 2544 */ 2545 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) 2546 { 2547 struct stmmac_priv *priv = netdev_priv(dev); 2548 u32 rx_cnt = priv->plat->rx_queues_to_use; 2549 u32 tx_cnt = priv->plat->tx_queues_to_use; 2550 u32 chan; 2551 int ret; 2552 2553 /* DMA initialization and SW reset */ 2554 ret = stmmac_init_dma_engine(priv); 2555 if (ret < 0) { 2556 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 2557 __func__); 2558 return ret; 2559 } 2560 2561 /* Copy the MAC addr into the HW */ 2562 priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0); 2563 2564 /* PS and related bits will be programmed according to the speed */ 2565 if (priv->hw->pcs) { 2566 int speed = priv->plat->mac_port_sel_speed; 2567 2568 if ((speed == SPEED_10) || (speed == SPEED_100) || 2569 (speed == SPEED_1000)) { 2570 priv->hw->ps = speed; 2571 } else { 2572 dev_warn(priv->device, "invalid port speed\n"); 2573 priv->hw->ps = 0; 2574 } 2575 } 2576 2577 /* Initialize the MAC Core */ 2578 priv->hw->mac->core_init(priv->hw, dev); 2579 2580 /* Initialize MTL*/ 2581 if (priv->synopsys_id >= DWMAC_CORE_4_00) 2582 stmmac_mtl_configuration(priv); 2583 2584 /* Initialize Safety Features */ 2585 if (priv->synopsys_id >= DWMAC_CORE_5_10) 2586 stmmac_safety_feat_configuration(priv); 2587 2588 ret = priv->hw->mac->rx_ipc(priv->hw); 2589 if (!ret) { 2590 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 2591 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 2592 priv->hw->rx_csum = 0; 2593 } 2594 2595 /* Enable the MAC Rx/Tx */ 2596 priv->hw->mac->set_mac(priv->ioaddr, true); 2597 2598 /* Set the HW DMA mode and the COE */ 2599 stmmac_dma_operation_mode(priv); 2600 2601 stmmac_mmc_setup(priv); 2602 2603 if (init_ptp) { 2604 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 2605 if (ret < 0) 2606 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret); 2607 2608 ret = stmmac_init_ptp(priv); 2609 if (ret == -EOPNOTSUPP) 2610 netdev_warn(priv->dev, "PTP not supported by HW\n"); 2611 else if (ret) 2612 netdev_warn(priv->dev, "PTP init failed\n"); 2613 } 2614 2615 #ifdef CONFIG_DEBUG_FS 2616 ret = stmmac_init_fs(dev); 2617 if (ret < 0) 2618 netdev_warn(priv->dev, "%s: failed debugFS registration\n", 2619 __func__); 2620 #endif 2621 /* Start the ball rolling... */ 2622 stmmac_start_all_dma(priv); 2623 2624 priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS; 2625 2626 if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) { 2627 priv->rx_riwt = MAX_DMA_RIWT; 2628 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt); 2629 } 2630 2631 if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane) 2632 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0); 2633 2634 /* set TX and RX rings length */ 2635 stmmac_set_rings_length(priv); 2636 2637 /* Enable TSO */ 2638 if (priv->tso) { 2639 for (chan = 0; chan < tx_cnt; chan++) 2640 priv->hw->dma->enable_tso(priv->ioaddr, 1, chan); 2641 } 2642 2643 return 0; 2644 } 2645 2646 static void stmmac_hw_teardown(struct net_device *dev) 2647 { 2648 struct stmmac_priv *priv = netdev_priv(dev); 2649 2650 clk_disable_unprepare(priv->plat->clk_ptp_ref); 2651 } 2652 2653 /** 2654 * stmmac_open - open entry point of the driver 2655 * @dev : pointer to the device structure. 2656 * Description: 2657 * This function is the open entry point of the driver. 2658 * Return value: 2659 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2660 * file on failure. 2661 */ 2662 static int stmmac_open(struct net_device *dev) 2663 { 2664 struct stmmac_priv *priv = netdev_priv(dev); 2665 int ret; 2666 2667 stmmac_check_ether_addr(priv); 2668 2669 if (priv->hw->pcs != STMMAC_PCS_RGMII && 2670 priv->hw->pcs != STMMAC_PCS_TBI && 2671 priv->hw->pcs != STMMAC_PCS_RTBI) { 2672 ret = stmmac_init_phy(dev); 2673 if (ret) { 2674 netdev_err(priv->dev, 2675 "%s: Cannot attach to PHY (error: %d)\n", 2676 __func__, ret); 2677 return ret; 2678 } 2679 } 2680 2681 /* Extra statistics */ 2682 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); 2683 priv->xstats.threshold = tc; 2684 2685 priv->dma_buf_sz = STMMAC_ALIGN(buf_sz); 2686 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 2687 2688 ret = alloc_dma_desc_resources(priv); 2689 if (ret < 0) { 2690 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 2691 __func__); 2692 goto dma_desc_error; 2693 } 2694 2695 ret = init_dma_desc_rings(dev, GFP_KERNEL); 2696 if (ret < 0) { 2697 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 2698 __func__); 2699 goto init_error; 2700 } 2701 2702 ret = stmmac_hw_setup(dev, true); 2703 if (ret < 0) { 2704 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 2705 goto init_error; 2706 } 2707 2708 stmmac_init_tx_coalesce(priv); 2709 2710 if (dev->phydev) 2711 phy_start(dev->phydev); 2712 2713 /* Request the IRQ lines */ 2714 ret = request_irq(dev->irq, stmmac_interrupt, 2715 IRQF_SHARED, dev->name, dev); 2716 if (unlikely(ret < 0)) { 2717 netdev_err(priv->dev, 2718 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 2719 __func__, dev->irq, ret); 2720 goto irq_error; 2721 } 2722 2723 /* Request the Wake IRQ in case of another line is used for WoL */ 2724 if (priv->wol_irq != dev->irq) { 2725 ret = request_irq(priv->wol_irq, stmmac_interrupt, 2726 IRQF_SHARED, dev->name, dev); 2727 if (unlikely(ret < 0)) { 2728 netdev_err(priv->dev, 2729 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 2730 __func__, priv->wol_irq, ret); 2731 goto wolirq_error; 2732 } 2733 } 2734 2735 /* Request the IRQ lines */ 2736 if (priv->lpi_irq > 0) { 2737 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED, 2738 dev->name, dev); 2739 if (unlikely(ret < 0)) { 2740 netdev_err(priv->dev, 2741 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 2742 __func__, priv->lpi_irq, ret); 2743 goto lpiirq_error; 2744 } 2745 } 2746 2747 stmmac_enable_all_queues(priv); 2748 stmmac_start_all_queues(priv); 2749 2750 return 0; 2751 2752 lpiirq_error: 2753 if (priv->wol_irq != dev->irq) 2754 free_irq(priv->wol_irq, dev); 2755 wolirq_error: 2756 free_irq(dev->irq, dev); 2757 irq_error: 2758 if (dev->phydev) 2759 phy_stop(dev->phydev); 2760 2761 del_timer_sync(&priv->txtimer); 2762 stmmac_hw_teardown(dev); 2763 init_error: 2764 free_dma_desc_resources(priv); 2765 dma_desc_error: 2766 if (dev->phydev) 2767 phy_disconnect(dev->phydev); 2768 2769 return ret; 2770 } 2771 2772 /** 2773 * stmmac_release - close entry point of the driver 2774 * @dev : device pointer. 2775 * Description: 2776 * This is the stop entry point of the driver. 2777 */ 2778 static int stmmac_release(struct net_device *dev) 2779 { 2780 struct stmmac_priv *priv = netdev_priv(dev); 2781 2782 if (priv->eee_enabled) 2783 del_timer_sync(&priv->eee_ctrl_timer); 2784 2785 /* Stop and disconnect the PHY */ 2786 if (dev->phydev) { 2787 phy_stop(dev->phydev); 2788 phy_disconnect(dev->phydev); 2789 } 2790 2791 stmmac_stop_all_queues(priv); 2792 2793 stmmac_disable_all_queues(priv); 2794 2795 del_timer_sync(&priv->txtimer); 2796 2797 /* Free the IRQ lines */ 2798 free_irq(dev->irq, dev); 2799 if (priv->wol_irq != dev->irq) 2800 free_irq(priv->wol_irq, dev); 2801 if (priv->lpi_irq > 0) 2802 free_irq(priv->lpi_irq, dev); 2803 2804 /* Stop TX/RX DMA and clear the descriptors */ 2805 stmmac_stop_all_dma(priv); 2806 2807 /* Release and free the Rx/Tx resources */ 2808 free_dma_desc_resources(priv); 2809 2810 /* Disable the MAC Rx/Tx */ 2811 priv->hw->mac->set_mac(priv->ioaddr, false); 2812 2813 netif_carrier_off(dev); 2814 2815 #ifdef CONFIG_DEBUG_FS 2816 stmmac_exit_fs(dev); 2817 #endif 2818 2819 stmmac_release_ptp(priv); 2820 2821 return 0; 2822 } 2823 2824 /** 2825 * stmmac_tso_allocator - close entry point of the driver 2826 * @priv: driver private structure 2827 * @des: buffer start address 2828 * @total_len: total length to fill in descriptors 2829 * @last_segmant: condition for the last descriptor 2830 * @queue: TX queue index 2831 * Description: 2832 * This function fills descriptor and request new descriptors according to 2833 * buffer length to fill 2834 */ 2835 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des, 2836 int total_len, bool last_segment, u32 queue) 2837 { 2838 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2839 struct dma_desc *desc; 2840 u32 buff_size; 2841 int tmp_len; 2842 2843 tmp_len = total_len; 2844 2845 while (tmp_len > 0) { 2846 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2847 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 2848 desc = tx_q->dma_tx + tx_q->cur_tx; 2849 2850 desc->des0 = cpu_to_le32(des + (total_len - tmp_len)); 2851 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 2852 TSO_MAX_BUFF_SIZE : tmp_len; 2853 2854 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size, 2855 0, 1, 2856 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 2857 0, 0); 2858 2859 tmp_len -= TSO_MAX_BUFF_SIZE; 2860 } 2861 } 2862 2863 /** 2864 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 2865 * @skb : the socket buffer 2866 * @dev : device pointer 2867 * Description: this is the transmit function that is called on TSO frames 2868 * (support available on GMAC4 and newer chips). 2869 * Diagram below show the ring programming in case of TSO frames: 2870 * 2871 * First Descriptor 2872 * -------- 2873 * | DES0 |---> buffer1 = L2/L3/L4 header 2874 * | DES1 |---> TCP Payload (can continue on next descr...) 2875 * | DES2 |---> buffer 1 and 2 len 2876 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 2877 * -------- 2878 * | 2879 * ... 2880 * | 2881 * -------- 2882 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 2883 * | DES1 | --| 2884 * | DES2 | --> buffer 1 and 2 len 2885 * | DES3 | 2886 * -------- 2887 * 2888 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 2889 */ 2890 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 2891 { 2892 struct dma_desc *desc, *first, *mss_desc = NULL; 2893 struct stmmac_priv *priv = netdev_priv(dev); 2894 int nfrags = skb_shinfo(skb)->nr_frags; 2895 u32 queue = skb_get_queue_mapping(skb); 2896 unsigned int first_entry, des; 2897 struct stmmac_tx_queue *tx_q; 2898 int tmp_pay_len = 0; 2899 u32 pay_len, mss; 2900 u8 proto_hdr_len; 2901 int i; 2902 2903 tx_q = &priv->tx_queue[queue]; 2904 2905 /* Compute header lengths */ 2906 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2907 2908 /* Desc availability based on threshold should be enough safe */ 2909 if (unlikely(stmmac_tx_avail(priv, queue) < 2910 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 2911 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 2912 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 2913 queue)); 2914 /* This is a hard error, log it. */ 2915 netdev_err(priv->dev, 2916 "%s: Tx Ring full when queue awake\n", 2917 __func__); 2918 } 2919 return NETDEV_TX_BUSY; 2920 } 2921 2922 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 2923 2924 mss = skb_shinfo(skb)->gso_size; 2925 2926 /* set new MSS value if needed */ 2927 if (mss != tx_q->mss) { 2928 mss_desc = tx_q->dma_tx + tx_q->cur_tx; 2929 priv->hw->desc->set_mss(mss_desc, mss); 2930 tx_q->mss = mss; 2931 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2932 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 2933 } 2934 2935 if (netif_msg_tx_queued(priv)) { 2936 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 2937 __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss); 2938 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 2939 skb->data_len); 2940 } 2941 2942 first_entry = tx_q->cur_tx; 2943 WARN_ON(tx_q->tx_skbuff[first_entry]); 2944 2945 desc = tx_q->dma_tx + first_entry; 2946 first = desc; 2947 2948 /* first descriptor: fill Headers on Buf1 */ 2949 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 2950 DMA_TO_DEVICE); 2951 if (dma_mapping_error(priv->device, des)) 2952 goto dma_map_err; 2953 2954 tx_q->tx_skbuff_dma[first_entry].buf = des; 2955 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 2956 2957 first->des0 = cpu_to_le32(des); 2958 2959 /* Fill start of payload in buff2 of first descriptor */ 2960 if (pay_len) 2961 first->des1 = cpu_to_le32(des + proto_hdr_len); 2962 2963 /* If needed take extra descriptors to fill the remaining payload */ 2964 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 2965 2966 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 2967 2968 /* Prepare fragments */ 2969 for (i = 0; i < nfrags; i++) { 2970 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2971 2972 des = skb_frag_dma_map(priv->device, frag, 0, 2973 skb_frag_size(frag), 2974 DMA_TO_DEVICE); 2975 if (dma_mapping_error(priv->device, des)) 2976 goto dma_map_err; 2977 2978 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 2979 (i == nfrags - 1), queue); 2980 2981 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 2982 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 2983 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 2984 } 2985 2986 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 2987 2988 /* Only the last descriptor gets to point to the skb. */ 2989 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 2990 2991 /* We've used all descriptors we need for this skb, however, 2992 * advance cur_tx so that it references a fresh descriptor. 2993 * ndo_start_xmit will fill this descriptor the next time it's 2994 * called and stmmac_tx_clean may clean up to this descriptor. 2995 */ 2996 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2997 2998 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 2999 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 3000 __func__); 3001 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 3002 } 3003 3004 dev->stats.tx_bytes += skb->len; 3005 priv->xstats.tx_tso_frames++; 3006 priv->xstats.tx_tso_nfrags += nfrags; 3007 3008 /* Manage tx mitigation */ 3009 priv->tx_count_frames += nfrags + 1; 3010 if (likely(priv->tx_coal_frames > priv->tx_count_frames)) { 3011 mod_timer(&priv->txtimer, 3012 STMMAC_COAL_TIMER(priv->tx_coal_timer)); 3013 } else { 3014 priv->tx_count_frames = 0; 3015 priv->hw->desc->set_tx_ic(desc); 3016 priv->xstats.tx_set_ic_bit++; 3017 } 3018 3019 skb_tx_timestamp(skb); 3020 3021 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 3022 priv->hwts_tx_en)) { 3023 /* declare that device is doing timestamping */ 3024 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 3025 priv->hw->desc->enable_tx_timestamp(first); 3026 } 3027 3028 /* Complete the first descriptor before granting the DMA */ 3029 priv->hw->desc->prepare_tso_tx_desc(first, 1, 3030 proto_hdr_len, 3031 pay_len, 3032 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 3033 tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len)); 3034 3035 /* If context desc is used to change MSS */ 3036 if (mss_desc) { 3037 /* Make sure that first descriptor has been completely 3038 * written, including its own bit. This is because MSS is 3039 * actually before first descriptor, so we need to make 3040 * sure that MSS's own bit is the last thing written. 3041 */ 3042 dma_wmb(); 3043 priv->hw->desc->set_tx_owner(mss_desc); 3044 } 3045 3046 /* The own bit must be the latest setting done when prepare the 3047 * descriptor and then barrier is needed to make sure that 3048 * all is coherent before granting the DMA engine. 3049 */ 3050 wmb(); 3051 3052 if (netif_msg_pktdata(priv)) { 3053 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 3054 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 3055 tx_q->cur_tx, first, nfrags); 3056 3057 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE, 3058 0); 3059 3060 pr_info(">>> frame to be transmitted: "); 3061 print_pkt(skb->data, skb_headlen(skb)); 3062 } 3063 3064 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 3065 3066 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr, 3067 queue); 3068 3069 return NETDEV_TX_OK; 3070 3071 dma_map_err: 3072 dev_err(priv->device, "Tx dma map failed\n"); 3073 dev_kfree_skb(skb); 3074 priv->dev->stats.tx_dropped++; 3075 return NETDEV_TX_OK; 3076 } 3077 3078 /** 3079 * stmmac_xmit - Tx entry point of the driver 3080 * @skb : the socket buffer 3081 * @dev : device pointer 3082 * Description : this is the tx entry point of the driver. 3083 * It programs the chain or the ring and supports oversized frames 3084 * and SG feature. 3085 */ 3086 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 3087 { 3088 struct stmmac_priv *priv = netdev_priv(dev); 3089 unsigned int nopaged_len = skb_headlen(skb); 3090 int i, csum_insertion = 0, is_jumbo = 0; 3091 u32 queue = skb_get_queue_mapping(skb); 3092 int nfrags = skb_shinfo(skb)->nr_frags; 3093 int entry; 3094 unsigned int first_entry; 3095 struct dma_desc *desc, *first; 3096 struct stmmac_tx_queue *tx_q; 3097 unsigned int enh_desc; 3098 unsigned int des; 3099 3100 tx_q = &priv->tx_queue[queue]; 3101 3102 /* Manage oversized TCP frames for GMAC4 device */ 3103 if (skb_is_gso(skb) && priv->tso) { 3104 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 3105 return stmmac_tso_xmit(skb, dev); 3106 } 3107 3108 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 3109 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 3110 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 3111 queue)); 3112 /* This is a hard error, log it. */ 3113 netdev_err(priv->dev, 3114 "%s: Tx Ring full when queue awake\n", 3115 __func__); 3116 } 3117 return NETDEV_TX_BUSY; 3118 } 3119 3120 if (priv->tx_path_in_lpi_mode) 3121 stmmac_disable_eee_mode(priv); 3122 3123 entry = tx_q->cur_tx; 3124 first_entry = entry; 3125 WARN_ON(tx_q->tx_skbuff[first_entry]); 3126 3127 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 3128 3129 if (likely(priv->extend_desc)) 3130 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3131 else 3132 desc = tx_q->dma_tx + entry; 3133 3134 first = desc; 3135 3136 enh_desc = priv->plat->enh_desc; 3137 /* To program the descriptors according to the size of the frame */ 3138 if (enh_desc) 3139 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc); 3140 3141 if (unlikely(is_jumbo) && likely(priv->synopsys_id < 3142 DWMAC_CORE_4_00)) { 3143 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion); 3144 if (unlikely(entry < 0)) 3145 goto dma_map_err; 3146 } 3147 3148 for (i = 0; i < nfrags; i++) { 3149 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3150 int len = skb_frag_size(frag); 3151 bool last_segment = (i == (nfrags - 1)); 3152 3153 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3154 WARN_ON(tx_q->tx_skbuff[entry]); 3155 3156 if (likely(priv->extend_desc)) 3157 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3158 else 3159 desc = tx_q->dma_tx + entry; 3160 3161 des = skb_frag_dma_map(priv->device, frag, 0, len, 3162 DMA_TO_DEVICE); 3163 if (dma_mapping_error(priv->device, des)) 3164 goto dma_map_err; /* should reuse desc w/o issues */ 3165 3166 tx_q->tx_skbuff_dma[entry].buf = des; 3167 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3168 desc->des0 = cpu_to_le32(des); 3169 else 3170 desc->des2 = cpu_to_le32(des); 3171 3172 tx_q->tx_skbuff_dma[entry].map_as_page = true; 3173 tx_q->tx_skbuff_dma[entry].len = len; 3174 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 3175 3176 /* Prepare the descriptor and set the own bit too */ 3177 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion, 3178 priv->mode, 1, last_segment, 3179 skb->len); 3180 } 3181 3182 /* Only the last descriptor gets to point to the skb. */ 3183 tx_q->tx_skbuff[entry] = skb; 3184 3185 /* We've used all descriptors we need for this skb, however, 3186 * advance cur_tx so that it references a fresh descriptor. 3187 * ndo_start_xmit will fill this descriptor the next time it's 3188 * called and stmmac_tx_clean may clean up to this descriptor. 3189 */ 3190 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3191 tx_q->cur_tx = entry; 3192 3193 if (netif_msg_pktdata(priv)) { 3194 void *tx_head; 3195 3196 netdev_dbg(priv->dev, 3197 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 3198 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 3199 entry, first, nfrags); 3200 3201 if (priv->extend_desc) 3202 tx_head = (void *)tx_q->dma_etx; 3203 else 3204 tx_head = (void *)tx_q->dma_tx; 3205 3206 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false); 3207 3208 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 3209 print_pkt(skb->data, skb->len); 3210 } 3211 3212 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 3213 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 3214 __func__); 3215 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 3216 } 3217 3218 dev->stats.tx_bytes += skb->len; 3219 3220 /* According to the coalesce parameter the IC bit for the latest 3221 * segment is reset and the timer re-started to clean the tx status. 3222 * This approach takes care about the fragments: desc is the first 3223 * element in case of no SG. 3224 */ 3225 priv->tx_count_frames += nfrags + 1; 3226 if (likely(priv->tx_coal_frames > priv->tx_count_frames)) { 3227 mod_timer(&priv->txtimer, 3228 STMMAC_COAL_TIMER(priv->tx_coal_timer)); 3229 } else { 3230 priv->tx_count_frames = 0; 3231 priv->hw->desc->set_tx_ic(desc); 3232 priv->xstats.tx_set_ic_bit++; 3233 } 3234 3235 skb_tx_timestamp(skb); 3236 3237 /* Ready to fill the first descriptor and set the OWN bit w/o any 3238 * problems because all the descriptors are actually ready to be 3239 * passed to the DMA engine. 3240 */ 3241 if (likely(!is_jumbo)) { 3242 bool last_segment = (nfrags == 0); 3243 3244 des = dma_map_single(priv->device, skb->data, 3245 nopaged_len, DMA_TO_DEVICE); 3246 if (dma_mapping_error(priv->device, des)) 3247 goto dma_map_err; 3248 3249 tx_q->tx_skbuff_dma[first_entry].buf = des; 3250 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3251 first->des0 = cpu_to_le32(des); 3252 else 3253 first->des2 = cpu_to_le32(des); 3254 3255 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 3256 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 3257 3258 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 3259 priv->hwts_tx_en)) { 3260 /* declare that device is doing timestamping */ 3261 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 3262 priv->hw->desc->enable_tx_timestamp(first); 3263 } 3264 3265 /* Prepare the first descriptor setting the OWN bit too */ 3266 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len, 3267 csum_insertion, priv->mode, 1, 3268 last_segment, skb->len); 3269 3270 /* The own bit must be the latest setting done when prepare the 3271 * descriptor and then barrier is needed to make sure that 3272 * all is coherent before granting the DMA engine. 3273 */ 3274 wmb(); 3275 } 3276 3277 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 3278 3279 if (priv->synopsys_id < DWMAC_CORE_4_00) 3280 priv->hw->dma->enable_dma_transmission(priv->ioaddr); 3281 else 3282 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr, 3283 queue); 3284 3285 return NETDEV_TX_OK; 3286 3287 dma_map_err: 3288 netdev_err(priv->dev, "Tx DMA map failed\n"); 3289 dev_kfree_skb(skb); 3290 priv->dev->stats.tx_dropped++; 3291 return NETDEV_TX_OK; 3292 } 3293 3294 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 3295 { 3296 struct ethhdr *ehdr; 3297 u16 vlanid; 3298 3299 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 3300 NETIF_F_HW_VLAN_CTAG_RX && 3301 !__vlan_get_tag(skb, &vlanid)) { 3302 /* pop the vlan tag */ 3303 ehdr = (struct ethhdr *)skb->data; 3304 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2); 3305 skb_pull(skb, VLAN_HLEN); 3306 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid); 3307 } 3308 } 3309 3310 3311 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q) 3312 { 3313 if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH) 3314 return 0; 3315 3316 return 1; 3317 } 3318 3319 /** 3320 * stmmac_rx_refill - refill used skb preallocated buffers 3321 * @priv: driver private structure 3322 * @queue: RX queue index 3323 * Description : this is to reallocate the skb for the reception process 3324 * that is based on zero-copy. 3325 */ 3326 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 3327 { 3328 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3329 int dirty = stmmac_rx_dirty(priv, queue); 3330 unsigned int entry = rx_q->dirty_rx; 3331 3332 int bfsize = priv->dma_buf_sz; 3333 3334 while (dirty-- > 0) { 3335 struct dma_desc *p; 3336 3337 if (priv->extend_desc) 3338 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3339 else 3340 p = rx_q->dma_rx + entry; 3341 3342 if (likely(!rx_q->rx_skbuff[entry])) { 3343 struct sk_buff *skb; 3344 3345 skb = netdev_alloc_skb_ip_align(priv->dev, bfsize); 3346 if (unlikely(!skb)) { 3347 /* so for a while no zero-copy! */ 3348 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH; 3349 if (unlikely(net_ratelimit())) 3350 dev_err(priv->device, 3351 "fail to alloc skb entry %d\n", 3352 entry); 3353 break; 3354 } 3355 3356 rx_q->rx_skbuff[entry] = skb; 3357 rx_q->rx_skbuff_dma[entry] = 3358 dma_map_single(priv->device, skb->data, bfsize, 3359 DMA_FROM_DEVICE); 3360 if (dma_mapping_error(priv->device, 3361 rx_q->rx_skbuff_dma[entry])) { 3362 netdev_err(priv->dev, "Rx DMA map failed\n"); 3363 dev_kfree_skb(skb); 3364 break; 3365 } 3366 3367 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) { 3368 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]); 3369 p->des1 = 0; 3370 } else { 3371 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]); 3372 } 3373 if (priv->hw->mode->refill_desc3) 3374 priv->hw->mode->refill_desc3(rx_q, p); 3375 3376 if (rx_q->rx_zeroc_thresh > 0) 3377 rx_q->rx_zeroc_thresh--; 3378 3379 netif_dbg(priv, rx_status, priv->dev, 3380 "refill entry #%d\n", entry); 3381 } 3382 dma_wmb(); 3383 3384 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3385 priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0); 3386 else 3387 priv->hw->desc->set_rx_owner(p); 3388 3389 dma_wmb(); 3390 3391 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE); 3392 } 3393 rx_q->dirty_rx = entry; 3394 } 3395 3396 /** 3397 * stmmac_rx - manage the receive process 3398 * @priv: driver private structure 3399 * @limit: napi bugget 3400 * @queue: RX queue index. 3401 * Description : this the function called by the napi poll method. 3402 * It gets all the frames inside the ring. 3403 */ 3404 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 3405 { 3406 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3407 unsigned int entry = rx_q->cur_rx; 3408 int coe = priv->hw->rx_csum; 3409 unsigned int next_entry; 3410 unsigned int count = 0; 3411 3412 if (netif_msg_rx_status(priv)) { 3413 void *rx_head; 3414 3415 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 3416 if (priv->extend_desc) 3417 rx_head = (void *)rx_q->dma_erx; 3418 else 3419 rx_head = (void *)rx_q->dma_rx; 3420 3421 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true); 3422 } 3423 while (count < limit) { 3424 int status; 3425 struct dma_desc *p; 3426 struct dma_desc *np; 3427 3428 if (priv->extend_desc) 3429 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3430 else 3431 p = rx_q->dma_rx + entry; 3432 3433 /* read the status of the incoming frame */ 3434 status = priv->hw->desc->rx_status(&priv->dev->stats, 3435 &priv->xstats, p); 3436 /* check if managed by the DMA otherwise go ahead */ 3437 if (unlikely(status & dma_own)) 3438 break; 3439 3440 count++; 3441 3442 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE); 3443 next_entry = rx_q->cur_rx; 3444 3445 if (priv->extend_desc) 3446 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 3447 else 3448 np = rx_q->dma_rx + next_entry; 3449 3450 prefetch(np); 3451 3452 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status)) 3453 priv->hw->desc->rx_extended_status(&priv->dev->stats, 3454 &priv->xstats, 3455 rx_q->dma_erx + 3456 entry); 3457 if (unlikely(status == discard_frame)) { 3458 priv->dev->stats.rx_errors++; 3459 if (priv->hwts_rx_en && !priv->extend_desc) { 3460 /* DESC2 & DESC3 will be overwritten by device 3461 * with timestamp value, hence reinitialize 3462 * them in stmmac_rx_refill() function so that 3463 * device can reuse it. 3464 */ 3465 dev_kfree_skb_any(rx_q->rx_skbuff[entry]); 3466 rx_q->rx_skbuff[entry] = NULL; 3467 dma_unmap_single(priv->device, 3468 rx_q->rx_skbuff_dma[entry], 3469 priv->dma_buf_sz, 3470 DMA_FROM_DEVICE); 3471 } 3472 } else { 3473 struct sk_buff *skb; 3474 int frame_len; 3475 unsigned int des; 3476 3477 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) 3478 des = le32_to_cpu(p->des0); 3479 else 3480 des = le32_to_cpu(p->des2); 3481 3482 frame_len = priv->hw->desc->get_rx_frame_len(p, coe); 3483 3484 /* If frame length is greater than skb buffer size 3485 * (preallocated during init) then the packet is 3486 * ignored 3487 */ 3488 if (frame_len > priv->dma_buf_sz) { 3489 netdev_err(priv->dev, 3490 "len %d larger than size (%d)\n", 3491 frame_len, priv->dma_buf_sz); 3492 priv->dev->stats.rx_length_errors++; 3493 break; 3494 } 3495 3496 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 3497 * Type frames (LLC/LLC-SNAP) 3498 */ 3499 if (unlikely(status != llc_snap)) 3500 frame_len -= ETH_FCS_LEN; 3501 3502 if (netif_msg_rx_status(priv)) { 3503 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n", 3504 p, entry, des); 3505 netdev_dbg(priv->dev, "frame size %d, COE: %d\n", 3506 frame_len, status); 3507 } 3508 3509 /* The zero-copy is always used for all the sizes 3510 * in case of GMAC4 because it needs 3511 * to refill the used descriptors, always. 3512 */ 3513 if (unlikely(!priv->plat->has_gmac4 && 3514 ((frame_len < priv->rx_copybreak) || 3515 stmmac_rx_threshold_count(rx_q)))) { 3516 skb = netdev_alloc_skb_ip_align(priv->dev, 3517 frame_len); 3518 if (unlikely(!skb)) { 3519 if (net_ratelimit()) 3520 dev_warn(priv->device, 3521 "packet dropped\n"); 3522 priv->dev->stats.rx_dropped++; 3523 break; 3524 } 3525 3526 dma_sync_single_for_cpu(priv->device, 3527 rx_q->rx_skbuff_dma 3528 [entry], frame_len, 3529 DMA_FROM_DEVICE); 3530 skb_copy_to_linear_data(skb, 3531 rx_q-> 3532 rx_skbuff[entry]->data, 3533 frame_len); 3534 3535 skb_put(skb, frame_len); 3536 dma_sync_single_for_device(priv->device, 3537 rx_q->rx_skbuff_dma 3538 [entry], frame_len, 3539 DMA_FROM_DEVICE); 3540 } else { 3541 skb = rx_q->rx_skbuff[entry]; 3542 if (unlikely(!skb)) { 3543 netdev_err(priv->dev, 3544 "%s: Inconsistent Rx chain\n", 3545 priv->dev->name); 3546 priv->dev->stats.rx_dropped++; 3547 break; 3548 } 3549 prefetch(skb->data - NET_IP_ALIGN); 3550 rx_q->rx_skbuff[entry] = NULL; 3551 rx_q->rx_zeroc_thresh++; 3552 3553 skb_put(skb, frame_len); 3554 dma_unmap_single(priv->device, 3555 rx_q->rx_skbuff_dma[entry], 3556 priv->dma_buf_sz, 3557 DMA_FROM_DEVICE); 3558 } 3559 3560 if (netif_msg_pktdata(priv)) { 3561 netdev_dbg(priv->dev, "frame received (%dbytes)", 3562 frame_len); 3563 print_pkt(skb->data, frame_len); 3564 } 3565 3566 stmmac_get_rx_hwtstamp(priv, p, np, skb); 3567 3568 stmmac_rx_vlan(priv->dev, skb); 3569 3570 skb->protocol = eth_type_trans(skb, priv->dev); 3571 3572 if (unlikely(!coe)) 3573 skb_checksum_none_assert(skb); 3574 else 3575 skb->ip_summed = CHECKSUM_UNNECESSARY; 3576 3577 napi_gro_receive(&rx_q->napi, skb); 3578 3579 priv->dev->stats.rx_packets++; 3580 priv->dev->stats.rx_bytes += frame_len; 3581 } 3582 entry = next_entry; 3583 } 3584 3585 stmmac_rx_refill(priv, queue); 3586 3587 priv->xstats.rx_pkt_n += count; 3588 3589 return count; 3590 } 3591 3592 /** 3593 * stmmac_poll - stmmac poll method (NAPI) 3594 * @napi : pointer to the napi structure. 3595 * @budget : maximum number of packets that the current CPU can receive from 3596 * all interfaces. 3597 * Description : 3598 * To look at the incoming frames and clear the tx resources. 3599 */ 3600 static int stmmac_poll(struct napi_struct *napi, int budget) 3601 { 3602 struct stmmac_rx_queue *rx_q = 3603 container_of(napi, struct stmmac_rx_queue, napi); 3604 struct stmmac_priv *priv = rx_q->priv_data; 3605 u32 tx_count = priv->plat->tx_queues_to_use; 3606 u32 chan = rx_q->queue_index; 3607 int work_done = 0; 3608 u32 queue; 3609 3610 priv->xstats.napi_poll++; 3611 3612 /* check all the queues */ 3613 for (queue = 0; queue < tx_count; queue++) 3614 stmmac_tx_clean(priv, queue); 3615 3616 work_done = stmmac_rx(priv, budget, rx_q->queue_index); 3617 if (work_done < budget) { 3618 napi_complete_done(napi, work_done); 3619 stmmac_enable_dma_irq(priv, chan); 3620 } 3621 return work_done; 3622 } 3623 3624 /** 3625 * stmmac_tx_timeout 3626 * @dev : Pointer to net device structure 3627 * Description: this function is called when a packet transmission fails to 3628 * complete within a reasonable time. The driver will mark the error in the 3629 * netdev structure and arrange for the device to be reset to a sane state 3630 * in order to transmit a new packet. 3631 */ 3632 static void stmmac_tx_timeout(struct net_device *dev) 3633 { 3634 struct stmmac_priv *priv = netdev_priv(dev); 3635 3636 stmmac_global_err(priv); 3637 } 3638 3639 /** 3640 * stmmac_set_rx_mode - entry point for multicast addressing 3641 * @dev : pointer to the device structure 3642 * Description: 3643 * This function is a driver entry point which gets called by the kernel 3644 * whenever multicast addresses must be enabled/disabled. 3645 * Return value: 3646 * void. 3647 */ 3648 static void stmmac_set_rx_mode(struct net_device *dev) 3649 { 3650 struct stmmac_priv *priv = netdev_priv(dev); 3651 3652 priv->hw->mac->set_filter(priv->hw, dev); 3653 } 3654 3655 /** 3656 * stmmac_change_mtu - entry point to change MTU size for the device. 3657 * @dev : device pointer. 3658 * @new_mtu : the new MTU size for the device. 3659 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 3660 * to drive packet transmission. Ethernet has an MTU of 1500 octets 3661 * (ETH_DATA_LEN). This value can be changed with ifconfig. 3662 * Return value: 3663 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3664 * file on failure. 3665 */ 3666 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 3667 { 3668 struct stmmac_priv *priv = netdev_priv(dev); 3669 3670 if (netif_running(dev)) { 3671 netdev_err(priv->dev, "must be stopped to change its MTU\n"); 3672 return -EBUSY; 3673 } 3674 3675 dev->mtu = new_mtu; 3676 3677 netdev_update_features(dev); 3678 3679 return 0; 3680 } 3681 3682 static netdev_features_t stmmac_fix_features(struct net_device *dev, 3683 netdev_features_t features) 3684 { 3685 struct stmmac_priv *priv = netdev_priv(dev); 3686 3687 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 3688 features &= ~NETIF_F_RXCSUM; 3689 3690 if (!priv->plat->tx_coe) 3691 features &= ~NETIF_F_CSUM_MASK; 3692 3693 /* Some GMAC devices have a bugged Jumbo frame support that 3694 * needs to have the Tx COE disabled for oversized frames 3695 * (due to limited buffer sizes). In this case we disable 3696 * the TX csum insertion in the TDES and not use SF. 3697 */ 3698 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 3699 features &= ~NETIF_F_CSUM_MASK; 3700 3701 /* Disable tso if asked by ethtool */ 3702 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 3703 if (features & NETIF_F_TSO) 3704 priv->tso = true; 3705 else 3706 priv->tso = false; 3707 } 3708 3709 return features; 3710 } 3711 3712 static int stmmac_set_features(struct net_device *netdev, 3713 netdev_features_t features) 3714 { 3715 struct stmmac_priv *priv = netdev_priv(netdev); 3716 3717 /* Keep the COE Type in case of csum is supporting */ 3718 if (features & NETIF_F_RXCSUM) 3719 priv->hw->rx_csum = priv->plat->rx_coe; 3720 else 3721 priv->hw->rx_csum = 0; 3722 /* No check needed because rx_coe has been set before and it will be 3723 * fixed in case of issue. 3724 */ 3725 priv->hw->mac->rx_ipc(priv->hw); 3726 3727 return 0; 3728 } 3729 3730 /** 3731 * stmmac_interrupt - main ISR 3732 * @irq: interrupt number. 3733 * @dev_id: to pass the net device pointer. 3734 * Description: this is the main driver interrupt service routine. 3735 * It can call: 3736 * o DMA service routine (to manage incoming frame reception and transmission 3737 * status) 3738 * o Core interrupts to manage: remote wake-up, management counter, LPI 3739 * interrupts. 3740 */ 3741 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 3742 { 3743 struct net_device *dev = (struct net_device *)dev_id; 3744 struct stmmac_priv *priv = netdev_priv(dev); 3745 u32 rx_cnt = priv->plat->rx_queues_to_use; 3746 u32 tx_cnt = priv->plat->tx_queues_to_use; 3747 u32 queues_count; 3748 u32 queue; 3749 3750 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 3751 3752 if (priv->irq_wake) 3753 pm_wakeup_event(priv->device, 0); 3754 3755 if (unlikely(!dev)) { 3756 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 3757 return IRQ_NONE; 3758 } 3759 3760 /* Check if adapter is up */ 3761 if (test_bit(STMMAC_DOWN, &priv->state)) 3762 return IRQ_HANDLED; 3763 /* Check if a fatal error happened */ 3764 if (stmmac_safety_feat_interrupt(priv)) 3765 return IRQ_HANDLED; 3766 3767 /* To handle GMAC own interrupts */ 3768 if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) { 3769 int status = priv->hw->mac->host_irq_status(priv->hw, 3770 &priv->xstats); 3771 3772 if (unlikely(status)) { 3773 /* For LPI we need to save the tx status */ 3774 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 3775 priv->tx_path_in_lpi_mode = true; 3776 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 3777 priv->tx_path_in_lpi_mode = false; 3778 } 3779 3780 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3781 for (queue = 0; queue < queues_count; queue++) { 3782 struct stmmac_rx_queue *rx_q = 3783 &priv->rx_queue[queue]; 3784 3785 status |= 3786 priv->hw->mac->host_mtl_irq_status(priv->hw, 3787 queue); 3788 3789 if (status & CORE_IRQ_MTL_RX_OVERFLOW && 3790 priv->hw->dma->set_rx_tail_ptr) 3791 priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, 3792 rx_q->rx_tail_addr, 3793 queue); 3794 } 3795 } 3796 3797 /* PCS link status */ 3798 if (priv->hw->pcs) { 3799 if (priv->xstats.pcs_link) 3800 netif_carrier_on(dev); 3801 else 3802 netif_carrier_off(dev); 3803 } 3804 } 3805 3806 /* To handle DMA interrupts */ 3807 stmmac_dma_interrupt(priv); 3808 3809 return IRQ_HANDLED; 3810 } 3811 3812 #ifdef CONFIG_NET_POLL_CONTROLLER 3813 /* Polling receive - used by NETCONSOLE and other diagnostic tools 3814 * to allow network I/O with interrupts disabled. 3815 */ 3816 static void stmmac_poll_controller(struct net_device *dev) 3817 { 3818 disable_irq(dev->irq); 3819 stmmac_interrupt(dev->irq, dev); 3820 enable_irq(dev->irq); 3821 } 3822 #endif 3823 3824 /** 3825 * stmmac_ioctl - Entry point for the Ioctl 3826 * @dev: Device pointer. 3827 * @rq: An IOCTL specefic structure, that can contain a pointer to 3828 * a proprietary structure used to pass information to the driver. 3829 * @cmd: IOCTL command 3830 * Description: 3831 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 3832 */ 3833 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3834 { 3835 int ret = -EOPNOTSUPP; 3836 3837 if (!netif_running(dev)) 3838 return -EINVAL; 3839 3840 switch (cmd) { 3841 case SIOCGMIIPHY: 3842 case SIOCGMIIREG: 3843 case SIOCSMIIREG: 3844 if (!dev->phydev) 3845 return -EINVAL; 3846 ret = phy_mii_ioctl(dev->phydev, rq, cmd); 3847 break; 3848 case SIOCSHWTSTAMP: 3849 ret = stmmac_hwtstamp_ioctl(dev, rq); 3850 break; 3851 default: 3852 break; 3853 } 3854 3855 return ret; 3856 } 3857 3858 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 3859 { 3860 struct stmmac_priv *priv = netdev_priv(ndev); 3861 int ret = 0; 3862 3863 ret = eth_mac_addr(ndev, addr); 3864 if (ret) 3865 return ret; 3866 3867 priv->hw->mac->set_umac_addr(priv->hw, ndev->dev_addr, 0); 3868 3869 return ret; 3870 } 3871 3872 #ifdef CONFIG_DEBUG_FS 3873 static struct dentry *stmmac_fs_dir; 3874 3875 static void sysfs_display_ring(void *head, int size, int extend_desc, 3876 struct seq_file *seq) 3877 { 3878 int i; 3879 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 3880 struct dma_desc *p = (struct dma_desc *)head; 3881 3882 for (i = 0; i < size; i++) { 3883 if (extend_desc) { 3884 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3885 i, (unsigned int)virt_to_phys(ep), 3886 le32_to_cpu(ep->basic.des0), 3887 le32_to_cpu(ep->basic.des1), 3888 le32_to_cpu(ep->basic.des2), 3889 le32_to_cpu(ep->basic.des3)); 3890 ep++; 3891 } else { 3892 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3893 i, (unsigned int)virt_to_phys(p), 3894 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 3895 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 3896 p++; 3897 } 3898 seq_printf(seq, "\n"); 3899 } 3900 } 3901 3902 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v) 3903 { 3904 struct net_device *dev = seq->private; 3905 struct stmmac_priv *priv = netdev_priv(dev); 3906 u32 rx_count = priv->plat->rx_queues_to_use; 3907 u32 tx_count = priv->plat->tx_queues_to_use; 3908 u32 queue; 3909 3910 for (queue = 0; queue < rx_count; queue++) { 3911 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3912 3913 seq_printf(seq, "RX Queue %d:\n", queue); 3914 3915 if (priv->extend_desc) { 3916 seq_printf(seq, "Extended descriptor ring:\n"); 3917 sysfs_display_ring((void *)rx_q->dma_erx, 3918 DMA_RX_SIZE, 1, seq); 3919 } else { 3920 seq_printf(seq, "Descriptor ring:\n"); 3921 sysfs_display_ring((void *)rx_q->dma_rx, 3922 DMA_RX_SIZE, 0, seq); 3923 } 3924 } 3925 3926 for (queue = 0; queue < tx_count; queue++) { 3927 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 3928 3929 seq_printf(seq, "TX Queue %d:\n", queue); 3930 3931 if (priv->extend_desc) { 3932 seq_printf(seq, "Extended descriptor ring:\n"); 3933 sysfs_display_ring((void *)tx_q->dma_etx, 3934 DMA_TX_SIZE, 1, seq); 3935 } else { 3936 seq_printf(seq, "Descriptor ring:\n"); 3937 sysfs_display_ring((void *)tx_q->dma_tx, 3938 DMA_TX_SIZE, 0, seq); 3939 } 3940 } 3941 3942 return 0; 3943 } 3944 3945 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file) 3946 { 3947 return single_open(file, stmmac_sysfs_ring_read, inode->i_private); 3948 } 3949 3950 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */ 3951 3952 static const struct file_operations stmmac_rings_status_fops = { 3953 .owner = THIS_MODULE, 3954 .open = stmmac_sysfs_ring_open, 3955 .read = seq_read, 3956 .llseek = seq_lseek, 3957 .release = single_release, 3958 }; 3959 3960 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v) 3961 { 3962 struct net_device *dev = seq->private; 3963 struct stmmac_priv *priv = netdev_priv(dev); 3964 3965 if (!priv->hw_cap_support) { 3966 seq_printf(seq, "DMA HW features not supported\n"); 3967 return 0; 3968 } 3969 3970 seq_printf(seq, "==============================\n"); 3971 seq_printf(seq, "\tDMA HW features\n"); 3972 seq_printf(seq, "==============================\n"); 3973 3974 seq_printf(seq, "\t10/100 Mbps: %s\n", 3975 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 3976 seq_printf(seq, "\t1000 Mbps: %s\n", 3977 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 3978 seq_printf(seq, "\tHalf duplex: %s\n", 3979 (priv->dma_cap.half_duplex) ? "Y" : "N"); 3980 seq_printf(seq, "\tHash Filter: %s\n", 3981 (priv->dma_cap.hash_filter) ? "Y" : "N"); 3982 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 3983 (priv->dma_cap.multi_addr) ? "Y" : "N"); 3984 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 3985 (priv->dma_cap.pcs) ? "Y" : "N"); 3986 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 3987 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 3988 seq_printf(seq, "\tPMT Remote wake up: %s\n", 3989 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 3990 seq_printf(seq, "\tPMT Magic Frame: %s\n", 3991 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 3992 seq_printf(seq, "\tRMON module: %s\n", 3993 (priv->dma_cap.rmon) ? "Y" : "N"); 3994 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 3995 (priv->dma_cap.time_stamp) ? "Y" : "N"); 3996 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 3997 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 3998 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 3999 (priv->dma_cap.eee) ? "Y" : "N"); 4000 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 4001 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 4002 (priv->dma_cap.tx_coe) ? "Y" : "N"); 4003 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 4004 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 4005 (priv->dma_cap.rx_coe) ? "Y" : "N"); 4006 } else { 4007 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 4008 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 4009 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 4010 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 4011 } 4012 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 4013 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 4014 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 4015 priv->dma_cap.number_rx_channel); 4016 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 4017 priv->dma_cap.number_tx_channel); 4018 seq_printf(seq, "\tEnhanced descriptors: %s\n", 4019 (priv->dma_cap.enh_desc) ? "Y" : "N"); 4020 4021 return 0; 4022 } 4023 4024 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file) 4025 { 4026 return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private); 4027 } 4028 4029 static const struct file_operations stmmac_dma_cap_fops = { 4030 .owner = THIS_MODULE, 4031 .open = stmmac_sysfs_dma_cap_open, 4032 .read = seq_read, 4033 .llseek = seq_lseek, 4034 .release = single_release, 4035 }; 4036 4037 static int stmmac_init_fs(struct net_device *dev) 4038 { 4039 struct stmmac_priv *priv = netdev_priv(dev); 4040 4041 /* Create per netdev entries */ 4042 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 4043 4044 if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) { 4045 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n"); 4046 4047 return -ENOMEM; 4048 } 4049 4050 /* Entry to report DMA RX/TX rings */ 4051 priv->dbgfs_rings_status = 4052 debugfs_create_file("descriptors_status", 0444, 4053 priv->dbgfs_dir, dev, 4054 &stmmac_rings_status_fops); 4055 4056 if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) { 4057 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n"); 4058 debugfs_remove_recursive(priv->dbgfs_dir); 4059 4060 return -ENOMEM; 4061 } 4062 4063 /* Entry to report the DMA HW features */ 4064 priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", 0444, 4065 priv->dbgfs_dir, 4066 dev, &stmmac_dma_cap_fops); 4067 4068 if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) { 4069 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n"); 4070 debugfs_remove_recursive(priv->dbgfs_dir); 4071 4072 return -ENOMEM; 4073 } 4074 4075 return 0; 4076 } 4077 4078 static void stmmac_exit_fs(struct net_device *dev) 4079 { 4080 struct stmmac_priv *priv = netdev_priv(dev); 4081 4082 debugfs_remove_recursive(priv->dbgfs_dir); 4083 } 4084 #endif /* CONFIG_DEBUG_FS */ 4085 4086 static const struct net_device_ops stmmac_netdev_ops = { 4087 .ndo_open = stmmac_open, 4088 .ndo_start_xmit = stmmac_xmit, 4089 .ndo_stop = stmmac_release, 4090 .ndo_change_mtu = stmmac_change_mtu, 4091 .ndo_fix_features = stmmac_fix_features, 4092 .ndo_set_features = stmmac_set_features, 4093 .ndo_set_rx_mode = stmmac_set_rx_mode, 4094 .ndo_tx_timeout = stmmac_tx_timeout, 4095 .ndo_do_ioctl = stmmac_ioctl, 4096 #ifdef CONFIG_NET_POLL_CONTROLLER 4097 .ndo_poll_controller = stmmac_poll_controller, 4098 #endif 4099 .ndo_set_mac_address = stmmac_set_mac_address, 4100 }; 4101 4102 static void stmmac_reset_subtask(struct stmmac_priv *priv) 4103 { 4104 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 4105 return; 4106 if (test_bit(STMMAC_DOWN, &priv->state)) 4107 return; 4108 4109 netdev_err(priv->dev, "Reset adapter.\n"); 4110 4111 rtnl_lock(); 4112 netif_trans_update(priv->dev); 4113 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 4114 usleep_range(1000, 2000); 4115 4116 set_bit(STMMAC_DOWN, &priv->state); 4117 dev_close(priv->dev); 4118 dev_open(priv->dev); 4119 clear_bit(STMMAC_DOWN, &priv->state); 4120 clear_bit(STMMAC_RESETING, &priv->state); 4121 rtnl_unlock(); 4122 } 4123 4124 static void stmmac_service_task(struct work_struct *work) 4125 { 4126 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 4127 service_task); 4128 4129 stmmac_reset_subtask(priv); 4130 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 4131 } 4132 4133 /** 4134 * stmmac_hw_init - Init the MAC device 4135 * @priv: driver private structure 4136 * Description: this function is to configure the MAC device according to 4137 * some platform parameters or the HW capability register. It prepares the 4138 * driver to use either ring or chain modes and to setup either enhanced or 4139 * normal descriptors. 4140 */ 4141 static int stmmac_hw_init(struct stmmac_priv *priv) 4142 { 4143 struct mac_device_info *mac; 4144 4145 /* Identify the MAC HW device */ 4146 if (priv->plat->setup) { 4147 mac = priv->plat->setup(priv); 4148 } else if (priv->plat->has_gmac) { 4149 priv->dev->priv_flags |= IFF_UNICAST_FLT; 4150 mac = dwmac1000_setup(priv->ioaddr, 4151 priv->plat->multicast_filter_bins, 4152 priv->plat->unicast_filter_entries, 4153 &priv->synopsys_id); 4154 } else if (priv->plat->has_gmac4) { 4155 priv->dev->priv_flags |= IFF_UNICAST_FLT; 4156 mac = dwmac4_setup(priv->ioaddr, 4157 priv->plat->multicast_filter_bins, 4158 priv->plat->unicast_filter_entries, 4159 &priv->synopsys_id); 4160 } else { 4161 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id); 4162 } 4163 if (!mac) 4164 return -ENOMEM; 4165 4166 priv->hw = mac; 4167 4168 /* dwmac-sun8i only work in chain mode */ 4169 if (priv->plat->has_sun8i) 4170 chain_mode = 1; 4171 4172 /* To use the chained or ring mode */ 4173 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 4174 priv->hw->mode = &dwmac4_ring_mode_ops; 4175 } else { 4176 if (chain_mode) { 4177 priv->hw->mode = &chain_mode_ops; 4178 dev_info(priv->device, "Chain mode enabled\n"); 4179 priv->mode = STMMAC_CHAIN_MODE; 4180 } else { 4181 priv->hw->mode = &ring_mode_ops; 4182 dev_info(priv->device, "Ring mode enabled\n"); 4183 priv->mode = STMMAC_RING_MODE; 4184 } 4185 } 4186 4187 /* Get the HW capability (new GMAC newer than 3.50a) */ 4188 priv->hw_cap_support = stmmac_get_hw_features(priv); 4189 if (priv->hw_cap_support) { 4190 dev_info(priv->device, "DMA HW capability register supported\n"); 4191 4192 /* We can override some gmac/dma configuration fields: e.g. 4193 * enh_desc, tx_coe (e.g. that are passed through the 4194 * platform) with the values from the HW capability 4195 * register (if supported). 4196 */ 4197 priv->plat->enh_desc = priv->dma_cap.enh_desc; 4198 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up; 4199 priv->hw->pmt = priv->plat->pmt; 4200 4201 /* TXCOE doesn't work in thresh DMA mode */ 4202 if (priv->plat->force_thresh_dma_mode) 4203 priv->plat->tx_coe = 0; 4204 else 4205 priv->plat->tx_coe = priv->dma_cap.tx_coe; 4206 4207 /* In case of GMAC4 rx_coe is from HW cap register. */ 4208 priv->plat->rx_coe = priv->dma_cap.rx_coe; 4209 4210 if (priv->dma_cap.rx_coe_type2) 4211 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 4212 else if (priv->dma_cap.rx_coe_type1) 4213 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 4214 4215 } else { 4216 dev_info(priv->device, "No HW DMA feature register supported\n"); 4217 } 4218 4219 /* To use alternate (extended), normal or GMAC4 descriptor structures */ 4220 if (priv->synopsys_id >= DWMAC_CORE_4_00) 4221 priv->hw->desc = &dwmac4_desc_ops; 4222 else 4223 stmmac_selec_desc_mode(priv); 4224 4225 if (priv->plat->rx_coe) { 4226 priv->hw->rx_csum = priv->plat->rx_coe; 4227 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 4228 if (priv->synopsys_id < DWMAC_CORE_4_00) 4229 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 4230 } 4231 if (priv->plat->tx_coe) 4232 dev_info(priv->device, "TX Checksum insertion supported\n"); 4233 4234 if (priv->plat->pmt) { 4235 dev_info(priv->device, "Wake-Up On Lan supported\n"); 4236 device_set_wakeup_capable(priv->device, 1); 4237 } 4238 4239 if (priv->dma_cap.tsoen) 4240 dev_info(priv->device, "TSO supported\n"); 4241 4242 return 0; 4243 } 4244 4245 /** 4246 * stmmac_dvr_probe 4247 * @device: device pointer 4248 * @plat_dat: platform data pointer 4249 * @res: stmmac resource pointer 4250 * Description: this is the main probe function used to 4251 * call the alloc_etherdev, allocate the priv structure. 4252 * Return: 4253 * returns 0 on success, otherwise errno. 4254 */ 4255 int stmmac_dvr_probe(struct device *device, 4256 struct plat_stmmacenet_data *plat_dat, 4257 struct stmmac_resources *res) 4258 { 4259 struct net_device *ndev = NULL; 4260 struct stmmac_priv *priv; 4261 int ret = 0; 4262 u32 queue; 4263 4264 ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv), 4265 MTL_MAX_TX_QUEUES, 4266 MTL_MAX_RX_QUEUES); 4267 if (!ndev) 4268 return -ENOMEM; 4269 4270 SET_NETDEV_DEV(ndev, device); 4271 4272 priv = netdev_priv(ndev); 4273 priv->device = device; 4274 priv->dev = ndev; 4275 4276 stmmac_set_ethtool_ops(ndev); 4277 priv->pause = pause; 4278 priv->plat = plat_dat; 4279 priv->ioaddr = res->addr; 4280 priv->dev->base_addr = (unsigned long)res->addr; 4281 4282 priv->dev->irq = res->irq; 4283 priv->wol_irq = res->wol_irq; 4284 priv->lpi_irq = res->lpi_irq; 4285 4286 if (res->mac) 4287 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); 4288 4289 dev_set_drvdata(device, priv->dev); 4290 4291 /* Verify driver arguments */ 4292 stmmac_verify_args(); 4293 4294 /* Allocate workqueue */ 4295 priv->wq = create_singlethread_workqueue("stmmac_wq"); 4296 if (!priv->wq) { 4297 dev_err(priv->device, "failed to create workqueue\n"); 4298 goto error_wq; 4299 } 4300 4301 INIT_WORK(&priv->service_task, stmmac_service_task); 4302 4303 /* Override with kernel parameters if supplied XXX CRS XXX 4304 * this needs to have multiple instances 4305 */ 4306 if ((phyaddr >= 0) && (phyaddr <= 31)) 4307 priv->plat->phy_addr = phyaddr; 4308 4309 if (priv->plat->stmmac_rst) { 4310 ret = reset_control_assert(priv->plat->stmmac_rst); 4311 reset_control_deassert(priv->plat->stmmac_rst); 4312 /* Some reset controllers have only reset callback instead of 4313 * assert + deassert callbacks pair. 4314 */ 4315 if (ret == -ENOTSUPP) 4316 reset_control_reset(priv->plat->stmmac_rst); 4317 } 4318 4319 /* Init MAC and get the capabilities */ 4320 ret = stmmac_hw_init(priv); 4321 if (ret) 4322 goto error_hw_init; 4323 4324 /* Configure real RX and TX queues */ 4325 netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use); 4326 netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use); 4327 4328 ndev->netdev_ops = &stmmac_netdev_ops; 4329 4330 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4331 NETIF_F_RXCSUM; 4332 4333 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 4334 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 4335 priv->tso = true; 4336 dev_info(priv->device, "TSO feature enabled\n"); 4337 } 4338 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 4339 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 4340 #ifdef STMMAC_VLAN_TAG_USED 4341 /* Both mac100 and gmac support receive VLAN tag detection */ 4342 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; 4343 #endif 4344 priv->msg_enable = netif_msg_init(debug, default_msg_level); 4345 4346 /* MTU range: 46 - hw-specific max */ 4347 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 4348 if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 4349 ndev->max_mtu = JUMBO_LEN; 4350 else 4351 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 4352 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 4353 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 4354 */ 4355 if ((priv->plat->maxmtu < ndev->max_mtu) && 4356 (priv->plat->maxmtu >= ndev->min_mtu)) 4357 ndev->max_mtu = priv->plat->maxmtu; 4358 else if (priv->plat->maxmtu < ndev->min_mtu) 4359 dev_warn(priv->device, 4360 "%s: warning: maxmtu having invalid value (%d)\n", 4361 __func__, priv->plat->maxmtu); 4362 4363 if (flow_ctrl) 4364 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 4365 4366 /* Rx Watchdog is available in the COREs newer than the 3.40. 4367 * In some case, for example on bugged HW this feature 4368 * has to be disable and this can be done by passing the 4369 * riwt_off field from the platform. 4370 */ 4371 if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) { 4372 priv->use_riwt = 1; 4373 dev_info(priv->device, 4374 "Enable RX Mitigation via HW Watchdog Timer\n"); 4375 } 4376 4377 for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) { 4378 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4379 4380 netif_napi_add(ndev, &rx_q->napi, stmmac_poll, 4381 (8 * priv->plat->rx_queues_to_use)); 4382 } 4383 4384 spin_lock_init(&priv->lock); 4385 4386 /* If a specific clk_csr value is passed from the platform 4387 * this means that the CSR Clock Range selection cannot be 4388 * changed at run-time and it is fixed. Viceversa the driver'll try to 4389 * set the MDC clock dynamically according to the csr actual 4390 * clock input. 4391 */ 4392 if (!priv->plat->clk_csr) 4393 stmmac_clk_csr_set(priv); 4394 else 4395 priv->clk_csr = priv->plat->clk_csr; 4396 4397 stmmac_check_pcs_mode(priv); 4398 4399 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4400 priv->hw->pcs != STMMAC_PCS_TBI && 4401 priv->hw->pcs != STMMAC_PCS_RTBI) { 4402 /* MDIO bus Registration */ 4403 ret = stmmac_mdio_register(ndev); 4404 if (ret < 0) { 4405 dev_err(priv->device, 4406 "%s: MDIO bus (id: %d) registration failed", 4407 __func__, priv->plat->bus_id); 4408 goto error_mdio_register; 4409 } 4410 } 4411 4412 ret = register_netdev(ndev); 4413 if (ret) { 4414 dev_err(priv->device, "%s: ERROR %i registering the device\n", 4415 __func__, ret); 4416 goto error_netdev_register; 4417 } 4418 4419 return ret; 4420 4421 error_netdev_register: 4422 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4423 priv->hw->pcs != STMMAC_PCS_TBI && 4424 priv->hw->pcs != STMMAC_PCS_RTBI) 4425 stmmac_mdio_unregister(ndev); 4426 error_mdio_register: 4427 for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) { 4428 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4429 4430 netif_napi_del(&rx_q->napi); 4431 } 4432 error_hw_init: 4433 destroy_workqueue(priv->wq); 4434 error_wq: 4435 free_netdev(ndev); 4436 4437 return ret; 4438 } 4439 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 4440 4441 /** 4442 * stmmac_dvr_remove 4443 * @dev: device pointer 4444 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 4445 * changes the link status, releases the DMA descriptor rings. 4446 */ 4447 int stmmac_dvr_remove(struct device *dev) 4448 { 4449 struct net_device *ndev = dev_get_drvdata(dev); 4450 struct stmmac_priv *priv = netdev_priv(ndev); 4451 4452 netdev_info(priv->dev, "%s: removing driver", __func__); 4453 4454 stmmac_stop_all_dma(priv); 4455 4456 priv->hw->mac->set_mac(priv->ioaddr, false); 4457 netif_carrier_off(ndev); 4458 unregister_netdev(ndev); 4459 if (priv->plat->stmmac_rst) 4460 reset_control_assert(priv->plat->stmmac_rst); 4461 clk_disable_unprepare(priv->plat->pclk); 4462 clk_disable_unprepare(priv->plat->stmmac_clk); 4463 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4464 priv->hw->pcs != STMMAC_PCS_TBI && 4465 priv->hw->pcs != STMMAC_PCS_RTBI) 4466 stmmac_mdio_unregister(ndev); 4467 destroy_workqueue(priv->wq); 4468 free_netdev(ndev); 4469 4470 return 0; 4471 } 4472 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 4473 4474 /** 4475 * stmmac_suspend - suspend callback 4476 * @dev: device pointer 4477 * Description: this is the function to suspend the device and it is called 4478 * by the platform driver to stop the network queue, release the resources, 4479 * program the PMT register (for WoL), clean and release driver resources. 4480 */ 4481 int stmmac_suspend(struct device *dev) 4482 { 4483 struct net_device *ndev = dev_get_drvdata(dev); 4484 struct stmmac_priv *priv = netdev_priv(ndev); 4485 unsigned long flags; 4486 4487 if (!ndev || !netif_running(ndev)) 4488 return 0; 4489 4490 if (ndev->phydev) 4491 phy_stop(ndev->phydev); 4492 4493 spin_lock_irqsave(&priv->lock, flags); 4494 4495 netif_device_detach(ndev); 4496 stmmac_stop_all_queues(priv); 4497 4498 stmmac_disable_all_queues(priv); 4499 4500 /* Stop TX/RX DMA */ 4501 stmmac_stop_all_dma(priv); 4502 4503 /* Enable Power down mode by programming the PMT regs */ 4504 if (device_may_wakeup(priv->device)) { 4505 priv->hw->mac->pmt(priv->hw, priv->wolopts); 4506 priv->irq_wake = 1; 4507 } else { 4508 priv->hw->mac->set_mac(priv->ioaddr, false); 4509 pinctrl_pm_select_sleep_state(priv->device); 4510 /* Disable clock in case of PWM is off */ 4511 clk_disable(priv->plat->pclk); 4512 clk_disable(priv->plat->stmmac_clk); 4513 } 4514 spin_unlock_irqrestore(&priv->lock, flags); 4515 4516 priv->oldlink = false; 4517 priv->speed = SPEED_UNKNOWN; 4518 priv->oldduplex = DUPLEX_UNKNOWN; 4519 return 0; 4520 } 4521 EXPORT_SYMBOL_GPL(stmmac_suspend); 4522 4523 /** 4524 * stmmac_reset_queues_param - reset queue parameters 4525 * @dev: device pointer 4526 */ 4527 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 4528 { 4529 u32 rx_cnt = priv->plat->rx_queues_to_use; 4530 u32 tx_cnt = priv->plat->tx_queues_to_use; 4531 u32 queue; 4532 4533 for (queue = 0; queue < rx_cnt; queue++) { 4534 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4535 4536 rx_q->cur_rx = 0; 4537 rx_q->dirty_rx = 0; 4538 } 4539 4540 for (queue = 0; queue < tx_cnt; queue++) { 4541 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 4542 4543 tx_q->cur_tx = 0; 4544 tx_q->dirty_tx = 0; 4545 tx_q->mss = 0; 4546 } 4547 } 4548 4549 /** 4550 * stmmac_resume - resume callback 4551 * @dev: device pointer 4552 * Description: when resume this function is invoked to setup the DMA and CORE 4553 * in a usable state. 4554 */ 4555 int stmmac_resume(struct device *dev) 4556 { 4557 struct net_device *ndev = dev_get_drvdata(dev); 4558 struct stmmac_priv *priv = netdev_priv(ndev); 4559 unsigned long flags; 4560 4561 if (!netif_running(ndev)) 4562 return 0; 4563 4564 /* Power Down bit, into the PM register, is cleared 4565 * automatically as soon as a magic packet or a Wake-up frame 4566 * is received. Anyway, it's better to manually clear 4567 * this bit because it can generate problems while resuming 4568 * from another devices (e.g. serial console). 4569 */ 4570 if (device_may_wakeup(priv->device)) { 4571 spin_lock_irqsave(&priv->lock, flags); 4572 priv->hw->mac->pmt(priv->hw, 0); 4573 spin_unlock_irqrestore(&priv->lock, flags); 4574 priv->irq_wake = 0; 4575 } else { 4576 pinctrl_pm_select_default_state(priv->device); 4577 /* enable the clk previously disabled */ 4578 clk_enable(priv->plat->stmmac_clk); 4579 clk_enable(priv->plat->pclk); 4580 /* reset the phy so that it's ready */ 4581 if (priv->mii) 4582 stmmac_mdio_reset(priv->mii); 4583 } 4584 4585 netif_device_attach(ndev); 4586 4587 spin_lock_irqsave(&priv->lock, flags); 4588 4589 stmmac_reset_queues_param(priv); 4590 4591 stmmac_clear_descriptors(priv); 4592 4593 stmmac_hw_setup(ndev, false); 4594 stmmac_init_tx_coalesce(priv); 4595 stmmac_set_rx_mode(ndev); 4596 4597 stmmac_enable_all_queues(priv); 4598 4599 stmmac_start_all_queues(priv); 4600 4601 spin_unlock_irqrestore(&priv->lock, flags); 4602 4603 if (ndev->phydev) 4604 phy_start(ndev->phydev); 4605 4606 return 0; 4607 } 4608 EXPORT_SYMBOL_GPL(stmmac_resume); 4609 4610 #ifndef MODULE 4611 static int __init stmmac_cmdline_opt(char *str) 4612 { 4613 char *opt; 4614 4615 if (!str || !*str) 4616 return -EINVAL; 4617 while ((opt = strsep(&str, ",")) != NULL) { 4618 if (!strncmp(opt, "debug:", 6)) { 4619 if (kstrtoint(opt + 6, 0, &debug)) 4620 goto err; 4621 } else if (!strncmp(opt, "phyaddr:", 8)) { 4622 if (kstrtoint(opt + 8, 0, &phyaddr)) 4623 goto err; 4624 } else if (!strncmp(opt, "buf_sz:", 7)) { 4625 if (kstrtoint(opt + 7, 0, &buf_sz)) 4626 goto err; 4627 } else if (!strncmp(opt, "tc:", 3)) { 4628 if (kstrtoint(opt + 3, 0, &tc)) 4629 goto err; 4630 } else if (!strncmp(opt, "watchdog:", 9)) { 4631 if (kstrtoint(opt + 9, 0, &watchdog)) 4632 goto err; 4633 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 4634 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 4635 goto err; 4636 } else if (!strncmp(opt, "pause:", 6)) { 4637 if (kstrtoint(opt + 6, 0, &pause)) 4638 goto err; 4639 } else if (!strncmp(opt, "eee_timer:", 10)) { 4640 if (kstrtoint(opt + 10, 0, &eee_timer)) 4641 goto err; 4642 } else if (!strncmp(opt, "chain_mode:", 11)) { 4643 if (kstrtoint(opt + 11, 0, &chain_mode)) 4644 goto err; 4645 } 4646 } 4647 return 0; 4648 4649 err: 4650 pr_err("%s: ERROR broken module parameter conversion", __func__); 4651 return -EINVAL; 4652 } 4653 4654 __setup("stmmaceth=", stmmac_cmdline_opt); 4655 #endif /* MODULE */ 4656 4657 static int __init stmmac_init(void) 4658 { 4659 #ifdef CONFIG_DEBUG_FS 4660 /* Create debugfs main directory if it doesn't exist yet */ 4661 if (!stmmac_fs_dir) { 4662 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 4663 4664 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) { 4665 pr_err("ERROR %s, debugfs create directory failed\n", 4666 STMMAC_RESOURCE_NAME); 4667 4668 return -ENOMEM; 4669 } 4670 } 4671 #endif 4672 4673 return 0; 4674 } 4675 4676 static void __exit stmmac_exit(void) 4677 { 4678 #ifdef CONFIG_DEBUG_FS 4679 debugfs_remove_recursive(stmmac_fs_dir); 4680 #endif 4681 } 4682 4683 module_init(stmmac_init) 4684 module_exit(stmmac_exit) 4685 4686 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 4687 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 4688 MODULE_LICENSE("GPL"); 4689