1 /******************************************************************************* 2 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 3 ST Ethernet IPs are built around a Synopsys IP Core. 4 5 Copyright(C) 2007-2011 STMicroelectronics Ltd 6 7 This program is free software; you can redistribute it and/or modify it 8 under the terms and conditions of the GNU General Public License, 9 version 2, as published by the Free Software Foundation. 10 11 This program is distributed in the hope it will be useful, but WITHOUT 12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 more details. 15 16 The full GNU General Public License is included in this distribution in 17 the file called "COPYING". 18 19 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 20 21 Documentation available at: 22 http://www.stlinux.com 23 Support available at: 24 https://bugzilla.stlinux.com/ 25 *******************************************************************************/ 26 27 #include <linux/clk.h> 28 #include <linux/kernel.h> 29 #include <linux/interrupt.h> 30 #include <linux/ip.h> 31 #include <linux/tcp.h> 32 #include <linux/skbuff.h> 33 #include <linux/ethtool.h> 34 #include <linux/if_ether.h> 35 #include <linux/crc32.h> 36 #include <linux/mii.h> 37 #include <linux/if.h> 38 #include <linux/if_vlan.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/slab.h> 41 #include <linux/prefetch.h> 42 #include <linux/pinctrl/consumer.h> 43 #ifdef CONFIG_DEBUG_FS 44 #include <linux/debugfs.h> 45 #include <linux/seq_file.h> 46 #endif /* CONFIG_DEBUG_FS */ 47 #include <linux/net_tstamp.h> 48 #include <net/pkt_cls.h> 49 #include "stmmac_ptp.h" 50 #include "stmmac.h" 51 #include <linux/reset.h> 52 #include <linux/of_mdio.h> 53 #include "dwmac1000.h" 54 #include "dwxgmac2.h" 55 #include "hwif.h" 56 57 #define STMMAC_ALIGN(x) __ALIGN_KERNEL(x, SMP_CACHE_BYTES) 58 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 59 60 /* Module parameters */ 61 #define TX_TIMEO 5000 62 static int watchdog = TX_TIMEO; 63 module_param(watchdog, int, 0644); 64 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 65 66 static int debug = -1; 67 module_param(debug, int, 0644); 68 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 69 70 static int phyaddr = -1; 71 module_param(phyaddr, int, 0444); 72 MODULE_PARM_DESC(phyaddr, "Physical device address"); 73 74 #define STMMAC_TX_THRESH (DMA_TX_SIZE / 4) 75 #define STMMAC_RX_THRESH (DMA_RX_SIZE / 4) 76 77 static int flow_ctrl = FLOW_OFF; 78 module_param(flow_ctrl, int, 0644); 79 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 80 81 static int pause = PAUSE_TIME; 82 module_param(pause, int, 0644); 83 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 84 85 #define TC_DEFAULT 64 86 static int tc = TC_DEFAULT; 87 module_param(tc, int, 0644); 88 MODULE_PARM_DESC(tc, "DMA threshold control value"); 89 90 #define DEFAULT_BUFSIZE 1536 91 static int buf_sz = DEFAULT_BUFSIZE; 92 module_param(buf_sz, int, 0644); 93 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 94 95 #define STMMAC_RX_COPYBREAK 256 96 97 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 98 NETIF_MSG_LINK | NETIF_MSG_IFUP | 99 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 100 101 #define STMMAC_DEFAULT_LPI_TIMER 1000 102 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 103 module_param(eee_timer, int, 0644); 104 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 105 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x)) 106 107 /* By default the driver will use the ring mode to manage tx and rx descriptors, 108 * but allow user to force to use the chain instead of the ring 109 */ 110 static unsigned int chain_mode; 111 module_param(chain_mode, int, 0444); 112 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 113 114 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 115 116 #ifdef CONFIG_DEBUG_FS 117 static int stmmac_init_fs(struct net_device *dev); 118 static void stmmac_exit_fs(struct net_device *dev); 119 #endif 120 121 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x)) 122 123 /** 124 * stmmac_verify_args - verify the driver parameters. 125 * Description: it checks the driver parameters and set a default in case of 126 * errors. 127 */ 128 static void stmmac_verify_args(void) 129 { 130 if (unlikely(watchdog < 0)) 131 watchdog = TX_TIMEO; 132 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 133 buf_sz = DEFAULT_BUFSIZE; 134 if (unlikely(flow_ctrl > 1)) 135 flow_ctrl = FLOW_AUTO; 136 else if (likely(flow_ctrl < 0)) 137 flow_ctrl = FLOW_OFF; 138 if (unlikely((pause < 0) || (pause > 0xffff))) 139 pause = PAUSE_TIME; 140 if (eee_timer < 0) 141 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 142 } 143 144 /** 145 * stmmac_disable_all_queues - Disable all queues 146 * @priv: driver private structure 147 */ 148 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 149 { 150 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 151 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 152 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 153 u32 queue; 154 155 for (queue = 0; queue < maxq; queue++) { 156 struct stmmac_channel *ch = &priv->channel[queue]; 157 158 napi_disable(&ch->napi); 159 } 160 } 161 162 /** 163 * stmmac_enable_all_queues - Enable all queues 164 * @priv: driver private structure 165 */ 166 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 167 { 168 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 169 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 170 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 171 u32 queue; 172 173 for (queue = 0; queue < maxq; queue++) { 174 struct stmmac_channel *ch = &priv->channel[queue]; 175 176 napi_enable(&ch->napi); 177 } 178 } 179 180 /** 181 * stmmac_stop_all_queues - Stop all queues 182 * @priv: driver private structure 183 */ 184 static void stmmac_stop_all_queues(struct stmmac_priv *priv) 185 { 186 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 187 u32 queue; 188 189 for (queue = 0; queue < tx_queues_cnt; queue++) 190 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 191 } 192 193 /** 194 * stmmac_start_all_queues - Start all queues 195 * @priv: driver private structure 196 */ 197 static void stmmac_start_all_queues(struct stmmac_priv *priv) 198 { 199 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 200 u32 queue; 201 202 for (queue = 0; queue < tx_queues_cnt; queue++) 203 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue)); 204 } 205 206 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 207 { 208 if (!test_bit(STMMAC_DOWN, &priv->state) && 209 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 210 queue_work(priv->wq, &priv->service_task); 211 } 212 213 static void stmmac_global_err(struct stmmac_priv *priv) 214 { 215 netif_carrier_off(priv->dev); 216 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 217 stmmac_service_event_schedule(priv); 218 } 219 220 /** 221 * stmmac_clk_csr_set - dynamically set the MDC clock 222 * @priv: driver private structure 223 * Description: this is to dynamically set the MDC clock according to the csr 224 * clock input. 225 * Note: 226 * If a specific clk_csr value is passed from the platform 227 * this means that the CSR Clock Range selection cannot be 228 * changed at run-time and it is fixed (as reported in the driver 229 * documentation). Viceversa the driver will try to set the MDC 230 * clock dynamically according to the actual clock input. 231 */ 232 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 233 { 234 u32 clk_rate; 235 236 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 237 238 /* Platform provided default clk_csr would be assumed valid 239 * for all other cases except for the below mentioned ones. 240 * For values higher than the IEEE 802.3 specified frequency 241 * we can not estimate the proper divider as it is not known 242 * the frequency of clk_csr_i. So we do not change the default 243 * divider. 244 */ 245 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 246 if (clk_rate < CSR_F_35M) 247 priv->clk_csr = STMMAC_CSR_20_35M; 248 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 249 priv->clk_csr = STMMAC_CSR_35_60M; 250 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 251 priv->clk_csr = STMMAC_CSR_60_100M; 252 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 253 priv->clk_csr = STMMAC_CSR_100_150M; 254 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 255 priv->clk_csr = STMMAC_CSR_150_250M; 256 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M)) 257 priv->clk_csr = STMMAC_CSR_250_300M; 258 } 259 260 if (priv->plat->has_sun8i) { 261 if (clk_rate > 160000000) 262 priv->clk_csr = 0x03; 263 else if (clk_rate > 80000000) 264 priv->clk_csr = 0x02; 265 else if (clk_rate > 40000000) 266 priv->clk_csr = 0x01; 267 else 268 priv->clk_csr = 0; 269 } 270 271 if (priv->plat->has_xgmac) { 272 if (clk_rate > 400000000) 273 priv->clk_csr = 0x5; 274 else if (clk_rate > 350000000) 275 priv->clk_csr = 0x4; 276 else if (clk_rate > 300000000) 277 priv->clk_csr = 0x3; 278 else if (clk_rate > 250000000) 279 priv->clk_csr = 0x2; 280 else if (clk_rate > 150000000) 281 priv->clk_csr = 0x1; 282 else 283 priv->clk_csr = 0x0; 284 } 285 } 286 287 static void print_pkt(unsigned char *buf, int len) 288 { 289 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 290 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 291 } 292 293 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 294 { 295 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 296 u32 avail; 297 298 if (tx_q->dirty_tx > tx_q->cur_tx) 299 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 300 else 301 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1; 302 303 return avail; 304 } 305 306 /** 307 * stmmac_rx_dirty - Get RX queue dirty 308 * @priv: driver private structure 309 * @queue: RX queue index 310 */ 311 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 312 { 313 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 314 u32 dirty; 315 316 if (rx_q->dirty_rx <= rx_q->cur_rx) 317 dirty = rx_q->cur_rx - rx_q->dirty_rx; 318 else 319 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx; 320 321 return dirty; 322 } 323 324 /** 325 * stmmac_hw_fix_mac_speed - callback for speed selection 326 * @priv: driver private structure 327 * Description: on some platforms (e.g. ST), some HW system configuration 328 * registers have to be set according to the link speed negotiated. 329 */ 330 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv) 331 { 332 struct net_device *ndev = priv->dev; 333 struct phy_device *phydev = ndev->phydev; 334 335 if (likely(priv->plat->fix_mac_speed)) 336 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed); 337 } 338 339 /** 340 * stmmac_enable_eee_mode - check and enter in LPI mode 341 * @priv: driver private structure 342 * Description: this function is to verify and enter in LPI mode in case of 343 * EEE. 344 */ 345 static void stmmac_enable_eee_mode(struct stmmac_priv *priv) 346 { 347 u32 tx_cnt = priv->plat->tx_queues_to_use; 348 u32 queue; 349 350 /* check if all TX queues have the work finished */ 351 for (queue = 0; queue < tx_cnt; queue++) { 352 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 353 354 if (tx_q->dirty_tx != tx_q->cur_tx) 355 return; /* still unfinished work */ 356 } 357 358 /* Check and enter in LPI mode */ 359 if (!priv->tx_path_in_lpi_mode) 360 stmmac_set_eee_mode(priv, priv->hw, 361 priv->plat->en_tx_lpi_clockgating); 362 } 363 364 /** 365 * stmmac_disable_eee_mode - disable and exit from LPI mode 366 * @priv: driver private structure 367 * Description: this function is to exit and disable EEE in case of 368 * LPI state is true. This is called by the xmit. 369 */ 370 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 371 { 372 stmmac_reset_eee_mode(priv, priv->hw); 373 del_timer_sync(&priv->eee_ctrl_timer); 374 priv->tx_path_in_lpi_mode = false; 375 } 376 377 /** 378 * stmmac_eee_ctrl_timer - EEE TX SW timer. 379 * @arg : data hook 380 * Description: 381 * if there is no data transfer and if we are not in LPI state, 382 * then MAC Transmitter can be moved to LPI state. 383 */ 384 static void stmmac_eee_ctrl_timer(struct timer_list *t) 385 { 386 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 387 388 stmmac_enable_eee_mode(priv); 389 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 390 } 391 392 /** 393 * stmmac_eee_init - init EEE 394 * @priv: driver private structure 395 * Description: 396 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 397 * can also manage EEE, this function enable the LPI state and start related 398 * timer. 399 */ 400 bool stmmac_eee_init(struct stmmac_priv *priv) 401 { 402 struct net_device *ndev = priv->dev; 403 int interface = priv->plat->interface; 404 bool ret = false; 405 406 if ((interface != PHY_INTERFACE_MODE_MII) && 407 (interface != PHY_INTERFACE_MODE_GMII) && 408 !phy_interface_mode_is_rgmii(interface)) 409 goto out; 410 411 /* Using PCS we cannot dial with the phy registers at this stage 412 * so we do not support extra feature like EEE. 413 */ 414 if ((priv->hw->pcs == STMMAC_PCS_RGMII) || 415 (priv->hw->pcs == STMMAC_PCS_TBI) || 416 (priv->hw->pcs == STMMAC_PCS_RTBI)) 417 goto out; 418 419 /* MAC core supports the EEE feature. */ 420 if (priv->dma_cap.eee) { 421 int tx_lpi_timer = priv->tx_lpi_timer; 422 423 /* Check if the PHY supports EEE */ 424 if (phy_init_eee(ndev->phydev, 1)) { 425 /* To manage at run-time if the EEE cannot be supported 426 * anymore (for example because the lp caps have been 427 * changed). 428 * In that case the driver disable own timers. 429 */ 430 mutex_lock(&priv->lock); 431 if (priv->eee_active) { 432 netdev_dbg(priv->dev, "disable EEE\n"); 433 del_timer_sync(&priv->eee_ctrl_timer); 434 stmmac_set_eee_timer(priv, priv->hw, 0, 435 tx_lpi_timer); 436 } 437 priv->eee_active = 0; 438 mutex_unlock(&priv->lock); 439 goto out; 440 } 441 /* Activate the EEE and start timers */ 442 mutex_lock(&priv->lock); 443 if (!priv->eee_active) { 444 priv->eee_active = 1; 445 timer_setup(&priv->eee_ctrl_timer, 446 stmmac_eee_ctrl_timer, 0); 447 mod_timer(&priv->eee_ctrl_timer, 448 STMMAC_LPI_T(eee_timer)); 449 450 stmmac_set_eee_timer(priv, priv->hw, 451 STMMAC_DEFAULT_LIT_LS, tx_lpi_timer); 452 } 453 /* Set HW EEE according to the speed */ 454 stmmac_set_eee_pls(priv, priv->hw, ndev->phydev->link); 455 456 ret = true; 457 mutex_unlock(&priv->lock); 458 459 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 460 } 461 out: 462 return ret; 463 } 464 465 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 466 * @priv: driver private structure 467 * @p : descriptor pointer 468 * @skb : the socket buffer 469 * Description : 470 * This function will read timestamp from the descriptor & pass it to stack. 471 * and also perform some sanity checks. 472 */ 473 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 474 struct dma_desc *p, struct sk_buff *skb) 475 { 476 struct skb_shared_hwtstamps shhwtstamp; 477 u64 ns; 478 479 if (!priv->hwts_tx_en) 480 return; 481 482 /* exit if skb doesn't support hw tstamp */ 483 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 484 return; 485 486 /* check tx tstamp status */ 487 if (stmmac_get_tx_timestamp_status(priv, p)) { 488 /* get the valid tstamp */ 489 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 490 491 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 492 shhwtstamp.hwtstamp = ns_to_ktime(ns); 493 494 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 495 /* pass tstamp to stack */ 496 skb_tstamp_tx(skb, &shhwtstamp); 497 } 498 499 return; 500 } 501 502 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 503 * @priv: driver private structure 504 * @p : descriptor pointer 505 * @np : next descriptor pointer 506 * @skb : the socket buffer 507 * Description : 508 * This function will read received packet's timestamp from the descriptor 509 * and pass it to stack. It also perform some sanity checks. 510 */ 511 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 512 struct dma_desc *np, struct sk_buff *skb) 513 { 514 struct skb_shared_hwtstamps *shhwtstamp = NULL; 515 struct dma_desc *desc = p; 516 u64 ns; 517 518 if (!priv->hwts_rx_en) 519 return; 520 /* For GMAC4, the valid timestamp is from CTX next desc. */ 521 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 522 desc = np; 523 524 /* Check if timestamp is available */ 525 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 526 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 527 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 528 shhwtstamp = skb_hwtstamps(skb); 529 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 530 shhwtstamp->hwtstamp = ns_to_ktime(ns); 531 } else { 532 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 533 } 534 } 535 536 /** 537 * stmmac_hwtstamp_set - control hardware timestamping. 538 * @dev: device pointer. 539 * @ifr: An IOCTL specific structure, that can contain a pointer to 540 * a proprietary structure used to pass information to the driver. 541 * Description: 542 * This function configures the MAC to enable/disable both outgoing(TX) 543 * and incoming(RX) packets time stamping based on user input. 544 * Return Value: 545 * 0 on success and an appropriate -ve integer on failure. 546 */ 547 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 548 { 549 struct stmmac_priv *priv = netdev_priv(dev); 550 struct hwtstamp_config config; 551 struct timespec64 now; 552 u64 temp = 0; 553 u32 ptp_v2 = 0; 554 u32 tstamp_all = 0; 555 u32 ptp_over_ipv4_udp = 0; 556 u32 ptp_over_ipv6_udp = 0; 557 u32 ptp_over_ethernet = 0; 558 u32 snap_type_sel = 0; 559 u32 ts_master_en = 0; 560 u32 ts_event_en = 0; 561 u32 value = 0; 562 u32 sec_inc; 563 bool xmac; 564 565 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 566 567 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 568 netdev_alert(priv->dev, "No support for HW time stamping\n"); 569 priv->hwts_tx_en = 0; 570 priv->hwts_rx_en = 0; 571 572 return -EOPNOTSUPP; 573 } 574 575 if (copy_from_user(&config, ifr->ifr_data, 576 sizeof(config))) 577 return -EFAULT; 578 579 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 580 __func__, config.flags, config.tx_type, config.rx_filter); 581 582 /* reserved for future extensions */ 583 if (config.flags) 584 return -EINVAL; 585 586 if (config.tx_type != HWTSTAMP_TX_OFF && 587 config.tx_type != HWTSTAMP_TX_ON) 588 return -ERANGE; 589 590 if (priv->adv_ts) { 591 switch (config.rx_filter) { 592 case HWTSTAMP_FILTER_NONE: 593 /* time stamp no incoming packet at all */ 594 config.rx_filter = HWTSTAMP_FILTER_NONE; 595 break; 596 597 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 598 /* PTP v1, UDP, any kind of event packet */ 599 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 600 /* take time stamp for all event messages */ 601 if (xmac) 602 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 603 else 604 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 605 606 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 607 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 608 break; 609 610 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 611 /* PTP v1, UDP, Sync packet */ 612 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 613 /* take time stamp for SYNC messages only */ 614 ts_event_en = PTP_TCR_TSEVNTENA; 615 616 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 617 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 618 break; 619 620 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 621 /* PTP v1, UDP, Delay_req packet */ 622 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 623 /* take time stamp for Delay_Req messages only */ 624 ts_master_en = PTP_TCR_TSMSTRENA; 625 ts_event_en = PTP_TCR_TSEVNTENA; 626 627 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 628 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 629 break; 630 631 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 632 /* PTP v2, UDP, any kind of event packet */ 633 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 634 ptp_v2 = PTP_TCR_TSVER2ENA; 635 /* take time stamp for all event messages */ 636 if (xmac) 637 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 638 else 639 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 640 641 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 642 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 643 break; 644 645 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 646 /* PTP v2, UDP, Sync packet */ 647 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 648 ptp_v2 = PTP_TCR_TSVER2ENA; 649 /* take time stamp for SYNC messages only */ 650 ts_event_en = PTP_TCR_TSEVNTENA; 651 652 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 653 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 654 break; 655 656 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 657 /* PTP v2, UDP, Delay_req packet */ 658 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 659 ptp_v2 = PTP_TCR_TSVER2ENA; 660 /* take time stamp for Delay_Req messages only */ 661 ts_master_en = PTP_TCR_TSMSTRENA; 662 ts_event_en = PTP_TCR_TSEVNTENA; 663 664 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 665 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 666 break; 667 668 case HWTSTAMP_FILTER_PTP_V2_EVENT: 669 /* PTP v2/802.AS1 any layer, any kind of event packet */ 670 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 671 ptp_v2 = PTP_TCR_TSVER2ENA; 672 /* take time stamp for all event messages */ 673 if (xmac) 674 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1; 675 else 676 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 677 678 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 679 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 680 ptp_over_ethernet = PTP_TCR_TSIPENA; 681 break; 682 683 case HWTSTAMP_FILTER_PTP_V2_SYNC: 684 /* PTP v2/802.AS1, any layer, Sync packet */ 685 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 686 ptp_v2 = PTP_TCR_TSVER2ENA; 687 /* take time stamp for SYNC messages only */ 688 ts_event_en = PTP_TCR_TSEVNTENA; 689 690 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 691 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 692 ptp_over_ethernet = PTP_TCR_TSIPENA; 693 break; 694 695 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 696 /* PTP v2/802.AS1, any layer, Delay_req packet */ 697 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 698 ptp_v2 = PTP_TCR_TSVER2ENA; 699 /* take time stamp for Delay_Req messages only */ 700 ts_master_en = PTP_TCR_TSMSTRENA; 701 ts_event_en = PTP_TCR_TSEVNTENA; 702 703 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 704 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 705 ptp_over_ethernet = PTP_TCR_TSIPENA; 706 break; 707 708 case HWTSTAMP_FILTER_NTP_ALL: 709 case HWTSTAMP_FILTER_ALL: 710 /* time stamp any incoming packet */ 711 config.rx_filter = HWTSTAMP_FILTER_ALL; 712 tstamp_all = PTP_TCR_TSENALL; 713 break; 714 715 default: 716 return -ERANGE; 717 } 718 } else { 719 switch (config.rx_filter) { 720 case HWTSTAMP_FILTER_NONE: 721 config.rx_filter = HWTSTAMP_FILTER_NONE; 722 break; 723 default: 724 /* PTP v1, UDP, any kind of event packet */ 725 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 726 break; 727 } 728 } 729 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 730 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 731 732 if (!priv->hwts_tx_en && !priv->hwts_rx_en) 733 stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0); 734 else { 735 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR | 736 tstamp_all | ptp_v2 | ptp_over_ethernet | 737 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en | 738 ts_master_en | snap_type_sel); 739 stmmac_config_hw_tstamping(priv, priv->ptpaddr, value); 740 741 /* program Sub Second Increment reg */ 742 stmmac_config_sub_second_increment(priv, 743 priv->ptpaddr, priv->plat->clk_ptp_rate, 744 xmac, &sec_inc); 745 temp = div_u64(1000000000ULL, sec_inc); 746 747 /* Store sub second increment and flags for later use */ 748 priv->sub_second_inc = sec_inc; 749 priv->systime_flags = value; 750 751 /* calculate default added value: 752 * formula is : 753 * addend = (2^32)/freq_div_ratio; 754 * where, freq_div_ratio = 1e9ns/sec_inc 755 */ 756 temp = (u64)(temp << 32); 757 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 758 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 759 760 /* initialize system time */ 761 ktime_get_real_ts64(&now); 762 763 /* lower 32 bits of tv_sec are safe until y2106 */ 764 stmmac_init_systime(priv, priv->ptpaddr, 765 (u32)now.tv_sec, now.tv_nsec); 766 } 767 768 memcpy(&priv->tstamp_config, &config, sizeof(config)); 769 770 return copy_to_user(ifr->ifr_data, &config, 771 sizeof(config)) ? -EFAULT : 0; 772 } 773 774 /** 775 * stmmac_hwtstamp_get - read hardware timestamping. 776 * @dev: device pointer. 777 * @ifr: An IOCTL specific structure, that can contain a pointer to 778 * a proprietary structure used to pass information to the driver. 779 * Description: 780 * This function obtain the current hardware timestamping settings 781 as requested. 782 */ 783 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 784 { 785 struct stmmac_priv *priv = netdev_priv(dev); 786 struct hwtstamp_config *config = &priv->tstamp_config; 787 788 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 789 return -EOPNOTSUPP; 790 791 return copy_to_user(ifr->ifr_data, config, 792 sizeof(*config)) ? -EFAULT : 0; 793 } 794 795 /** 796 * stmmac_init_ptp - init PTP 797 * @priv: driver private structure 798 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 799 * This is done by looking at the HW cap. register. 800 * This function also registers the ptp driver. 801 */ 802 static int stmmac_init_ptp(struct stmmac_priv *priv) 803 { 804 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 805 806 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 807 return -EOPNOTSUPP; 808 809 priv->adv_ts = 0; 810 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 811 if (xmac && priv->dma_cap.atime_stamp) 812 priv->adv_ts = 1; 813 /* Dwmac 3.x core with extend_desc can support adv_ts */ 814 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 815 priv->adv_ts = 1; 816 817 if (priv->dma_cap.time_stamp) 818 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 819 820 if (priv->adv_ts) 821 netdev_info(priv->dev, 822 "IEEE 1588-2008 Advanced Timestamp supported\n"); 823 824 priv->hwts_tx_en = 0; 825 priv->hwts_rx_en = 0; 826 827 stmmac_ptp_register(priv); 828 829 return 0; 830 } 831 832 static void stmmac_release_ptp(struct stmmac_priv *priv) 833 { 834 if (priv->plat->clk_ptp_ref) 835 clk_disable_unprepare(priv->plat->clk_ptp_ref); 836 stmmac_ptp_unregister(priv); 837 } 838 839 /** 840 * stmmac_mac_flow_ctrl - Configure flow control in all queues 841 * @priv: driver private structure 842 * Description: It is used for configuring the flow control in all queues 843 */ 844 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 845 { 846 u32 tx_cnt = priv->plat->tx_queues_to_use; 847 848 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 849 priv->pause, tx_cnt); 850 } 851 852 /** 853 * stmmac_adjust_link - adjusts the link parameters 854 * @dev: net device structure 855 * Description: this is the helper called by the physical abstraction layer 856 * drivers to communicate the phy link status. According the speed and duplex 857 * this driver can invoke registered glue-logic as well. 858 * It also invoke the eee initialization because it could happen when switch 859 * on different networks (that are eee capable). 860 */ 861 static void stmmac_adjust_link(struct net_device *dev) 862 { 863 struct stmmac_priv *priv = netdev_priv(dev); 864 struct phy_device *phydev = dev->phydev; 865 bool new_state = false; 866 867 if (!phydev) 868 return; 869 870 mutex_lock(&priv->lock); 871 872 if (phydev->link) { 873 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 874 875 /* Now we make sure that we can be in full duplex mode. 876 * If not, we operate in half-duplex mode. */ 877 if (phydev->duplex != priv->oldduplex) { 878 new_state = true; 879 if (!phydev->duplex) 880 ctrl &= ~priv->hw->link.duplex; 881 else 882 ctrl |= priv->hw->link.duplex; 883 priv->oldduplex = phydev->duplex; 884 } 885 /* Flow Control operation */ 886 if (phydev->pause) 887 stmmac_mac_flow_ctrl(priv, phydev->duplex); 888 889 if (phydev->speed != priv->speed) { 890 new_state = true; 891 ctrl &= ~priv->hw->link.speed_mask; 892 switch (phydev->speed) { 893 case SPEED_1000: 894 ctrl |= priv->hw->link.speed1000; 895 break; 896 case SPEED_100: 897 ctrl |= priv->hw->link.speed100; 898 break; 899 case SPEED_10: 900 ctrl |= priv->hw->link.speed10; 901 break; 902 default: 903 netif_warn(priv, link, priv->dev, 904 "broken speed: %d\n", phydev->speed); 905 phydev->speed = SPEED_UNKNOWN; 906 break; 907 } 908 if (phydev->speed != SPEED_UNKNOWN) 909 stmmac_hw_fix_mac_speed(priv); 910 priv->speed = phydev->speed; 911 } 912 913 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 914 915 if (!priv->oldlink) { 916 new_state = true; 917 priv->oldlink = true; 918 } 919 } else if (priv->oldlink) { 920 new_state = true; 921 priv->oldlink = false; 922 priv->speed = SPEED_UNKNOWN; 923 priv->oldduplex = DUPLEX_UNKNOWN; 924 } 925 926 if (new_state && netif_msg_link(priv)) 927 phy_print_status(phydev); 928 929 mutex_unlock(&priv->lock); 930 931 if (phydev->is_pseudo_fixed_link) 932 /* Stop PHY layer to call the hook to adjust the link in case 933 * of a switch is attached to the stmmac driver. 934 */ 935 phydev->irq = PHY_IGNORE_INTERRUPT; 936 else 937 /* At this stage, init the EEE if supported. 938 * Never called in case of fixed_link. 939 */ 940 priv->eee_enabled = stmmac_eee_init(priv); 941 } 942 943 /** 944 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 945 * @priv: driver private structure 946 * Description: this is to verify if the HW supports the PCS. 947 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 948 * configured for the TBI, RTBI, or SGMII PHY interface. 949 */ 950 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 951 { 952 int interface = priv->plat->interface; 953 954 if (priv->dma_cap.pcs) { 955 if ((interface == PHY_INTERFACE_MODE_RGMII) || 956 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 957 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 958 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 959 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 960 priv->hw->pcs = STMMAC_PCS_RGMII; 961 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 962 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 963 priv->hw->pcs = STMMAC_PCS_SGMII; 964 } 965 } 966 } 967 968 /** 969 * stmmac_init_phy - PHY initialization 970 * @dev: net device structure 971 * Description: it initializes the driver's PHY state, and attaches the PHY 972 * to the mac driver. 973 * Return value: 974 * 0 on success 975 */ 976 static int stmmac_init_phy(struct net_device *dev) 977 { 978 struct stmmac_priv *priv = netdev_priv(dev); 979 u32 tx_cnt = priv->plat->tx_queues_to_use; 980 struct phy_device *phydev; 981 char phy_id_fmt[MII_BUS_ID_SIZE + 3]; 982 char bus_id[MII_BUS_ID_SIZE]; 983 int interface = priv->plat->interface; 984 int max_speed = priv->plat->max_speed; 985 priv->oldlink = false; 986 priv->speed = SPEED_UNKNOWN; 987 priv->oldduplex = DUPLEX_UNKNOWN; 988 989 if (priv->plat->phy_node) { 990 phydev = of_phy_connect(dev, priv->plat->phy_node, 991 &stmmac_adjust_link, 0, interface); 992 } else { 993 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x", 994 priv->plat->bus_id); 995 996 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id, 997 priv->plat->phy_addr); 998 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__, 999 phy_id_fmt); 1000 1001 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link, 1002 interface); 1003 } 1004 1005 if (IS_ERR_OR_NULL(phydev)) { 1006 netdev_err(priv->dev, "Could not attach to PHY\n"); 1007 if (!phydev) 1008 return -ENODEV; 1009 1010 return PTR_ERR(phydev); 1011 } 1012 1013 /* Stop Advertising 1000BASE Capability if interface is not GMII */ 1014 if ((interface == PHY_INTERFACE_MODE_MII) || 1015 (interface == PHY_INTERFACE_MODE_RMII) || 1016 (max_speed < 1000 && max_speed > 0)) 1017 phy_set_max_speed(phydev, SPEED_100); 1018 1019 /* 1020 * Half-duplex mode not supported with multiqueue 1021 * half-duplex can only works with single queue 1022 */ 1023 if (tx_cnt > 1) { 1024 phy_remove_link_mode(phydev, 1025 ETHTOOL_LINK_MODE_10baseT_Half_BIT); 1026 phy_remove_link_mode(phydev, 1027 ETHTOOL_LINK_MODE_100baseT_Half_BIT); 1028 phy_remove_link_mode(phydev, 1029 ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 1030 } 1031 1032 /* 1033 * Broken HW is sometimes missing the pull-up resistor on the 1034 * MDIO line, which results in reads to non-existent devices returning 1035 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent 1036 * device as well. 1037 * Note: phydev->phy_id is the result of reading the UID PHY registers. 1038 */ 1039 if (!priv->plat->phy_node && phydev->phy_id == 0) { 1040 phy_disconnect(phydev); 1041 return -ENODEV; 1042 } 1043 1044 /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid 1045 * subsequent PHY polling, make sure we force a link transition if 1046 * we have a UP/DOWN/UP transition 1047 */ 1048 if (phydev->is_pseudo_fixed_link) 1049 phydev->irq = PHY_POLL; 1050 1051 phy_attached_info(phydev); 1052 return 0; 1053 } 1054 1055 static void stmmac_display_rx_rings(struct stmmac_priv *priv) 1056 { 1057 u32 rx_cnt = priv->plat->rx_queues_to_use; 1058 void *head_rx; 1059 u32 queue; 1060 1061 /* Display RX rings */ 1062 for (queue = 0; queue < rx_cnt; queue++) { 1063 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1064 1065 pr_info("\tRX Queue %u rings\n", queue); 1066 1067 if (priv->extend_desc) 1068 head_rx = (void *)rx_q->dma_erx; 1069 else 1070 head_rx = (void *)rx_q->dma_rx; 1071 1072 /* Display RX ring */ 1073 stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true); 1074 } 1075 } 1076 1077 static void stmmac_display_tx_rings(struct stmmac_priv *priv) 1078 { 1079 u32 tx_cnt = priv->plat->tx_queues_to_use; 1080 void *head_tx; 1081 u32 queue; 1082 1083 /* Display TX rings */ 1084 for (queue = 0; queue < tx_cnt; queue++) { 1085 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1086 1087 pr_info("\tTX Queue %d rings\n", queue); 1088 1089 if (priv->extend_desc) 1090 head_tx = (void *)tx_q->dma_etx; 1091 else 1092 head_tx = (void *)tx_q->dma_tx; 1093 1094 stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false); 1095 } 1096 } 1097 1098 static void stmmac_display_rings(struct stmmac_priv *priv) 1099 { 1100 /* Display RX ring */ 1101 stmmac_display_rx_rings(priv); 1102 1103 /* Display TX ring */ 1104 stmmac_display_tx_rings(priv); 1105 } 1106 1107 static int stmmac_set_bfsize(int mtu, int bufsize) 1108 { 1109 int ret = bufsize; 1110 1111 if (mtu >= BUF_SIZE_4KiB) 1112 ret = BUF_SIZE_8KiB; 1113 else if (mtu >= BUF_SIZE_2KiB) 1114 ret = BUF_SIZE_4KiB; 1115 else if (mtu > DEFAULT_BUFSIZE) 1116 ret = BUF_SIZE_2KiB; 1117 else 1118 ret = DEFAULT_BUFSIZE; 1119 1120 return ret; 1121 } 1122 1123 /** 1124 * stmmac_clear_rx_descriptors - clear RX descriptors 1125 * @priv: driver private structure 1126 * @queue: RX queue index 1127 * Description: this function is called to clear the RX descriptors 1128 * in case of both basic and extended descriptors are used. 1129 */ 1130 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) 1131 { 1132 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1133 int i; 1134 1135 /* Clear the RX descriptors */ 1136 for (i = 0; i < DMA_RX_SIZE; i++) 1137 if (priv->extend_desc) 1138 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1139 priv->use_riwt, priv->mode, 1140 (i == DMA_RX_SIZE - 1)); 1141 else 1142 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1143 priv->use_riwt, priv->mode, 1144 (i == DMA_RX_SIZE - 1)); 1145 } 1146 1147 /** 1148 * stmmac_clear_tx_descriptors - clear tx descriptors 1149 * @priv: driver private structure 1150 * @queue: TX queue index. 1151 * Description: this function is called to clear the TX descriptors 1152 * in case of both basic and extended descriptors are used. 1153 */ 1154 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) 1155 { 1156 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1157 int i; 1158 1159 /* Clear the TX descriptors */ 1160 for (i = 0; i < DMA_TX_SIZE; i++) 1161 if (priv->extend_desc) 1162 stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic, 1163 priv->mode, (i == DMA_TX_SIZE - 1)); 1164 else 1165 stmmac_init_tx_desc(priv, &tx_q->dma_tx[i], 1166 priv->mode, (i == DMA_TX_SIZE - 1)); 1167 } 1168 1169 /** 1170 * stmmac_clear_descriptors - clear descriptors 1171 * @priv: driver private structure 1172 * Description: this function is called to clear the TX and RX descriptors 1173 * in case of both basic and extended descriptors are used. 1174 */ 1175 static void stmmac_clear_descriptors(struct stmmac_priv *priv) 1176 { 1177 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1178 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1179 u32 queue; 1180 1181 /* Clear the RX descriptors */ 1182 for (queue = 0; queue < rx_queue_cnt; queue++) 1183 stmmac_clear_rx_descriptors(priv, queue); 1184 1185 /* Clear the TX descriptors */ 1186 for (queue = 0; queue < tx_queue_cnt; queue++) 1187 stmmac_clear_tx_descriptors(priv, queue); 1188 } 1189 1190 /** 1191 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1192 * @priv: driver private structure 1193 * @p: descriptor pointer 1194 * @i: descriptor index 1195 * @flags: gfp flag 1196 * @queue: RX queue index 1197 * Description: this function is called to allocate a receive buffer, perform 1198 * the DMA mapping and init the descriptor. 1199 */ 1200 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, 1201 int i, gfp_t flags, u32 queue) 1202 { 1203 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1204 struct sk_buff *skb; 1205 1206 skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags); 1207 if (!skb) { 1208 netdev_err(priv->dev, 1209 "%s: Rx init fails; skb is NULL\n", __func__); 1210 return -ENOMEM; 1211 } 1212 rx_q->rx_skbuff[i] = skb; 1213 rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data, 1214 priv->dma_buf_sz, 1215 DMA_FROM_DEVICE); 1216 if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) { 1217 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__); 1218 dev_kfree_skb_any(skb); 1219 return -EINVAL; 1220 } 1221 1222 stmmac_set_desc_addr(priv, p, rx_q->rx_skbuff_dma[i]); 1223 1224 if (priv->dma_buf_sz == BUF_SIZE_16KiB) 1225 stmmac_init_desc3(priv, p); 1226 1227 return 0; 1228 } 1229 1230 /** 1231 * stmmac_free_rx_buffer - free RX dma buffers 1232 * @priv: private structure 1233 * @queue: RX queue index 1234 * @i: buffer index. 1235 */ 1236 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1237 { 1238 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1239 1240 if (rx_q->rx_skbuff[i]) { 1241 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i], 1242 priv->dma_buf_sz, DMA_FROM_DEVICE); 1243 dev_kfree_skb_any(rx_q->rx_skbuff[i]); 1244 } 1245 rx_q->rx_skbuff[i] = NULL; 1246 } 1247 1248 /** 1249 * stmmac_free_tx_buffer - free RX dma buffers 1250 * @priv: private structure 1251 * @queue: RX queue index 1252 * @i: buffer index. 1253 */ 1254 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1255 { 1256 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1257 1258 if (tx_q->tx_skbuff_dma[i].buf) { 1259 if (tx_q->tx_skbuff_dma[i].map_as_page) 1260 dma_unmap_page(priv->device, 1261 tx_q->tx_skbuff_dma[i].buf, 1262 tx_q->tx_skbuff_dma[i].len, 1263 DMA_TO_DEVICE); 1264 else 1265 dma_unmap_single(priv->device, 1266 tx_q->tx_skbuff_dma[i].buf, 1267 tx_q->tx_skbuff_dma[i].len, 1268 DMA_TO_DEVICE); 1269 } 1270 1271 if (tx_q->tx_skbuff[i]) { 1272 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1273 tx_q->tx_skbuff[i] = NULL; 1274 tx_q->tx_skbuff_dma[i].buf = 0; 1275 tx_q->tx_skbuff_dma[i].map_as_page = false; 1276 } 1277 } 1278 1279 /** 1280 * init_dma_rx_desc_rings - init the RX descriptor rings 1281 * @dev: net device structure 1282 * @flags: gfp flag. 1283 * Description: this function initializes the DMA RX descriptors 1284 * and allocates the socket buffers. It supports the chained and ring 1285 * modes. 1286 */ 1287 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) 1288 { 1289 struct stmmac_priv *priv = netdev_priv(dev); 1290 u32 rx_count = priv->plat->rx_queues_to_use; 1291 int ret = -ENOMEM; 1292 int bfsize = 0; 1293 int queue; 1294 int i; 1295 1296 bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu); 1297 if (bfsize < 0) 1298 bfsize = 0; 1299 1300 if (bfsize < BUF_SIZE_16KiB) 1301 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); 1302 1303 priv->dma_buf_sz = bfsize; 1304 1305 /* RX INITIALIZATION */ 1306 netif_dbg(priv, probe, priv->dev, 1307 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1308 1309 for (queue = 0; queue < rx_count; queue++) { 1310 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1311 1312 netif_dbg(priv, probe, priv->dev, 1313 "(%s) dma_rx_phy=0x%08x\n", __func__, 1314 (u32)rx_q->dma_rx_phy); 1315 1316 for (i = 0; i < DMA_RX_SIZE; i++) { 1317 struct dma_desc *p; 1318 1319 if (priv->extend_desc) 1320 p = &((rx_q->dma_erx + i)->basic); 1321 else 1322 p = rx_q->dma_rx + i; 1323 1324 ret = stmmac_init_rx_buffers(priv, p, i, flags, 1325 queue); 1326 if (ret) 1327 goto err_init_rx_buffers; 1328 1329 netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n", 1330 rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data, 1331 (unsigned int)rx_q->rx_skbuff_dma[i]); 1332 } 1333 1334 rx_q->cur_rx = 0; 1335 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE); 1336 1337 stmmac_clear_rx_descriptors(priv, queue); 1338 1339 /* Setup the chained descriptor addresses */ 1340 if (priv->mode == STMMAC_CHAIN_MODE) { 1341 if (priv->extend_desc) 1342 stmmac_mode_init(priv, rx_q->dma_erx, 1343 rx_q->dma_rx_phy, DMA_RX_SIZE, 1); 1344 else 1345 stmmac_mode_init(priv, rx_q->dma_rx, 1346 rx_q->dma_rx_phy, DMA_RX_SIZE, 0); 1347 } 1348 } 1349 1350 buf_sz = bfsize; 1351 1352 return 0; 1353 1354 err_init_rx_buffers: 1355 while (queue >= 0) { 1356 while (--i >= 0) 1357 stmmac_free_rx_buffer(priv, queue, i); 1358 1359 if (queue == 0) 1360 break; 1361 1362 i = DMA_RX_SIZE; 1363 queue--; 1364 } 1365 1366 return ret; 1367 } 1368 1369 /** 1370 * init_dma_tx_desc_rings - init the TX descriptor rings 1371 * @dev: net device structure. 1372 * Description: this function initializes the DMA TX descriptors 1373 * and allocates the socket buffers. It supports the chained and ring 1374 * modes. 1375 */ 1376 static int init_dma_tx_desc_rings(struct net_device *dev) 1377 { 1378 struct stmmac_priv *priv = netdev_priv(dev); 1379 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1380 u32 queue; 1381 int i; 1382 1383 for (queue = 0; queue < tx_queue_cnt; queue++) { 1384 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1385 1386 netif_dbg(priv, probe, priv->dev, 1387 "(%s) dma_tx_phy=0x%08x\n", __func__, 1388 (u32)tx_q->dma_tx_phy); 1389 1390 /* Setup the chained descriptor addresses */ 1391 if (priv->mode == STMMAC_CHAIN_MODE) { 1392 if (priv->extend_desc) 1393 stmmac_mode_init(priv, tx_q->dma_etx, 1394 tx_q->dma_tx_phy, DMA_TX_SIZE, 1); 1395 else 1396 stmmac_mode_init(priv, tx_q->dma_tx, 1397 tx_q->dma_tx_phy, DMA_TX_SIZE, 0); 1398 } 1399 1400 for (i = 0; i < DMA_TX_SIZE; i++) { 1401 struct dma_desc *p; 1402 if (priv->extend_desc) 1403 p = &((tx_q->dma_etx + i)->basic); 1404 else 1405 p = tx_q->dma_tx + i; 1406 1407 stmmac_clear_desc(priv, p); 1408 1409 tx_q->tx_skbuff_dma[i].buf = 0; 1410 tx_q->tx_skbuff_dma[i].map_as_page = false; 1411 tx_q->tx_skbuff_dma[i].len = 0; 1412 tx_q->tx_skbuff_dma[i].last_segment = false; 1413 tx_q->tx_skbuff[i] = NULL; 1414 } 1415 1416 tx_q->dirty_tx = 0; 1417 tx_q->cur_tx = 0; 1418 tx_q->mss = 0; 1419 1420 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 1421 } 1422 1423 return 0; 1424 } 1425 1426 /** 1427 * init_dma_desc_rings - init the RX/TX descriptor rings 1428 * @dev: net device structure 1429 * @flags: gfp flag. 1430 * Description: this function initializes the DMA RX/TX descriptors 1431 * and allocates the socket buffers. It supports the chained and ring 1432 * modes. 1433 */ 1434 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags) 1435 { 1436 struct stmmac_priv *priv = netdev_priv(dev); 1437 int ret; 1438 1439 ret = init_dma_rx_desc_rings(dev, flags); 1440 if (ret) 1441 return ret; 1442 1443 ret = init_dma_tx_desc_rings(dev); 1444 1445 stmmac_clear_descriptors(priv); 1446 1447 if (netif_msg_hw(priv)) 1448 stmmac_display_rings(priv); 1449 1450 return ret; 1451 } 1452 1453 /** 1454 * dma_free_rx_skbufs - free RX dma buffers 1455 * @priv: private structure 1456 * @queue: RX queue index 1457 */ 1458 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) 1459 { 1460 int i; 1461 1462 for (i = 0; i < DMA_RX_SIZE; i++) 1463 stmmac_free_rx_buffer(priv, queue, i); 1464 } 1465 1466 /** 1467 * dma_free_tx_skbufs - free TX dma buffers 1468 * @priv: private structure 1469 * @queue: TX queue index 1470 */ 1471 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) 1472 { 1473 int i; 1474 1475 for (i = 0; i < DMA_TX_SIZE; i++) 1476 stmmac_free_tx_buffer(priv, queue, i); 1477 } 1478 1479 /** 1480 * free_dma_rx_desc_resources - free RX dma desc resources 1481 * @priv: private structure 1482 */ 1483 static void free_dma_rx_desc_resources(struct stmmac_priv *priv) 1484 { 1485 u32 rx_count = priv->plat->rx_queues_to_use; 1486 u32 queue; 1487 1488 /* Free RX queue resources */ 1489 for (queue = 0; queue < rx_count; queue++) { 1490 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1491 1492 /* Release the DMA RX socket buffers */ 1493 dma_free_rx_skbufs(priv, queue); 1494 1495 /* Free DMA regions of consistent memory previously allocated */ 1496 if (!priv->extend_desc) 1497 dma_free_coherent(priv->device, 1498 DMA_RX_SIZE * sizeof(struct dma_desc), 1499 rx_q->dma_rx, rx_q->dma_rx_phy); 1500 else 1501 dma_free_coherent(priv->device, DMA_RX_SIZE * 1502 sizeof(struct dma_extended_desc), 1503 rx_q->dma_erx, rx_q->dma_rx_phy); 1504 1505 kfree(rx_q->rx_skbuff_dma); 1506 kfree(rx_q->rx_skbuff); 1507 } 1508 } 1509 1510 /** 1511 * free_dma_tx_desc_resources - free TX dma desc resources 1512 * @priv: private structure 1513 */ 1514 static void free_dma_tx_desc_resources(struct stmmac_priv *priv) 1515 { 1516 u32 tx_count = priv->plat->tx_queues_to_use; 1517 u32 queue; 1518 1519 /* Free TX queue resources */ 1520 for (queue = 0; queue < tx_count; queue++) { 1521 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1522 1523 /* Release the DMA TX socket buffers */ 1524 dma_free_tx_skbufs(priv, queue); 1525 1526 /* Free DMA regions of consistent memory previously allocated */ 1527 if (!priv->extend_desc) 1528 dma_free_coherent(priv->device, 1529 DMA_TX_SIZE * sizeof(struct dma_desc), 1530 tx_q->dma_tx, tx_q->dma_tx_phy); 1531 else 1532 dma_free_coherent(priv->device, DMA_TX_SIZE * 1533 sizeof(struct dma_extended_desc), 1534 tx_q->dma_etx, tx_q->dma_tx_phy); 1535 1536 kfree(tx_q->tx_skbuff_dma); 1537 kfree(tx_q->tx_skbuff); 1538 } 1539 } 1540 1541 /** 1542 * alloc_dma_rx_desc_resources - alloc RX resources. 1543 * @priv: private structure 1544 * Description: according to which descriptor can be used (extend or basic) 1545 * this function allocates the resources for TX and RX paths. In case of 1546 * reception, for example, it pre-allocated the RX socket buffer in order to 1547 * allow zero-copy mechanism. 1548 */ 1549 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) 1550 { 1551 u32 rx_count = priv->plat->rx_queues_to_use; 1552 int ret = -ENOMEM; 1553 u32 queue; 1554 1555 /* RX queues buffers and DMA */ 1556 for (queue = 0; queue < rx_count; queue++) { 1557 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1558 1559 rx_q->queue_index = queue; 1560 rx_q->priv_data = priv; 1561 1562 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE, 1563 sizeof(dma_addr_t), 1564 GFP_KERNEL); 1565 if (!rx_q->rx_skbuff_dma) 1566 goto err_dma; 1567 1568 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE, 1569 sizeof(struct sk_buff *), 1570 GFP_KERNEL); 1571 if (!rx_q->rx_skbuff) 1572 goto err_dma; 1573 1574 if (priv->extend_desc) { 1575 rx_q->dma_erx = dma_alloc_coherent(priv->device, 1576 DMA_RX_SIZE * sizeof(struct dma_extended_desc), 1577 &rx_q->dma_rx_phy, 1578 GFP_KERNEL); 1579 if (!rx_q->dma_erx) 1580 goto err_dma; 1581 1582 } else { 1583 rx_q->dma_rx = dma_alloc_coherent(priv->device, 1584 DMA_RX_SIZE * sizeof(struct dma_desc), 1585 &rx_q->dma_rx_phy, 1586 GFP_KERNEL); 1587 if (!rx_q->dma_rx) 1588 goto err_dma; 1589 } 1590 } 1591 1592 return 0; 1593 1594 err_dma: 1595 free_dma_rx_desc_resources(priv); 1596 1597 return ret; 1598 } 1599 1600 /** 1601 * alloc_dma_tx_desc_resources - alloc TX resources. 1602 * @priv: private structure 1603 * Description: according to which descriptor can be used (extend or basic) 1604 * this function allocates the resources for TX and RX paths. In case of 1605 * reception, for example, it pre-allocated the RX socket buffer in order to 1606 * allow zero-copy mechanism. 1607 */ 1608 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) 1609 { 1610 u32 tx_count = priv->plat->tx_queues_to_use; 1611 int ret = -ENOMEM; 1612 u32 queue; 1613 1614 /* TX queues buffers and DMA */ 1615 for (queue = 0; queue < tx_count; queue++) { 1616 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1617 1618 tx_q->queue_index = queue; 1619 tx_q->priv_data = priv; 1620 1621 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE, 1622 sizeof(*tx_q->tx_skbuff_dma), 1623 GFP_KERNEL); 1624 if (!tx_q->tx_skbuff_dma) 1625 goto err_dma; 1626 1627 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE, 1628 sizeof(struct sk_buff *), 1629 GFP_KERNEL); 1630 if (!tx_q->tx_skbuff) 1631 goto err_dma; 1632 1633 if (priv->extend_desc) { 1634 tx_q->dma_etx = dma_alloc_coherent(priv->device, 1635 DMA_TX_SIZE * sizeof(struct dma_extended_desc), 1636 &tx_q->dma_tx_phy, 1637 GFP_KERNEL); 1638 if (!tx_q->dma_etx) 1639 goto err_dma; 1640 } else { 1641 tx_q->dma_tx = dma_alloc_coherent(priv->device, 1642 DMA_TX_SIZE * sizeof(struct dma_desc), 1643 &tx_q->dma_tx_phy, 1644 GFP_KERNEL); 1645 if (!tx_q->dma_tx) 1646 goto err_dma; 1647 } 1648 } 1649 1650 return 0; 1651 1652 err_dma: 1653 free_dma_tx_desc_resources(priv); 1654 1655 return ret; 1656 } 1657 1658 /** 1659 * alloc_dma_desc_resources - alloc TX/RX resources. 1660 * @priv: private structure 1661 * Description: according to which descriptor can be used (extend or basic) 1662 * this function allocates the resources for TX and RX paths. In case of 1663 * reception, for example, it pre-allocated the RX socket buffer in order to 1664 * allow zero-copy mechanism. 1665 */ 1666 static int alloc_dma_desc_resources(struct stmmac_priv *priv) 1667 { 1668 /* RX Allocation */ 1669 int ret = alloc_dma_rx_desc_resources(priv); 1670 1671 if (ret) 1672 return ret; 1673 1674 ret = alloc_dma_tx_desc_resources(priv); 1675 1676 return ret; 1677 } 1678 1679 /** 1680 * free_dma_desc_resources - free dma desc resources 1681 * @priv: private structure 1682 */ 1683 static void free_dma_desc_resources(struct stmmac_priv *priv) 1684 { 1685 /* Release the DMA RX socket buffers */ 1686 free_dma_rx_desc_resources(priv); 1687 1688 /* Release the DMA TX socket buffers */ 1689 free_dma_tx_desc_resources(priv); 1690 } 1691 1692 /** 1693 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 1694 * @priv: driver private structure 1695 * Description: It is used for enabling the rx queues in the MAC 1696 */ 1697 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 1698 { 1699 u32 rx_queues_count = priv->plat->rx_queues_to_use; 1700 int queue; 1701 u8 mode; 1702 1703 for (queue = 0; queue < rx_queues_count; queue++) { 1704 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 1705 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 1706 } 1707 } 1708 1709 /** 1710 * stmmac_start_rx_dma - start RX DMA channel 1711 * @priv: driver private structure 1712 * @chan: RX channel index 1713 * Description: 1714 * This starts a RX DMA channel 1715 */ 1716 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 1717 { 1718 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 1719 stmmac_start_rx(priv, priv->ioaddr, chan); 1720 } 1721 1722 /** 1723 * stmmac_start_tx_dma - start TX DMA channel 1724 * @priv: driver private structure 1725 * @chan: TX channel index 1726 * Description: 1727 * This starts a TX DMA channel 1728 */ 1729 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 1730 { 1731 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 1732 stmmac_start_tx(priv, priv->ioaddr, chan); 1733 } 1734 1735 /** 1736 * stmmac_stop_rx_dma - stop RX DMA channel 1737 * @priv: driver private structure 1738 * @chan: RX channel index 1739 * Description: 1740 * This stops a RX DMA channel 1741 */ 1742 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 1743 { 1744 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 1745 stmmac_stop_rx(priv, priv->ioaddr, chan); 1746 } 1747 1748 /** 1749 * stmmac_stop_tx_dma - stop TX DMA channel 1750 * @priv: driver private structure 1751 * @chan: TX channel index 1752 * Description: 1753 * This stops a TX DMA channel 1754 */ 1755 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 1756 { 1757 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 1758 stmmac_stop_tx(priv, priv->ioaddr, chan); 1759 } 1760 1761 /** 1762 * stmmac_start_all_dma - start all RX and TX DMA channels 1763 * @priv: driver private structure 1764 * Description: 1765 * This starts all the RX and TX DMA channels 1766 */ 1767 static void stmmac_start_all_dma(struct stmmac_priv *priv) 1768 { 1769 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1770 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1771 u32 chan = 0; 1772 1773 for (chan = 0; chan < rx_channels_count; chan++) 1774 stmmac_start_rx_dma(priv, chan); 1775 1776 for (chan = 0; chan < tx_channels_count; chan++) 1777 stmmac_start_tx_dma(priv, chan); 1778 } 1779 1780 /** 1781 * stmmac_stop_all_dma - stop all RX and TX DMA channels 1782 * @priv: driver private structure 1783 * Description: 1784 * This stops the RX and TX DMA channels 1785 */ 1786 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 1787 { 1788 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1789 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1790 u32 chan = 0; 1791 1792 for (chan = 0; chan < rx_channels_count; chan++) 1793 stmmac_stop_rx_dma(priv, chan); 1794 1795 for (chan = 0; chan < tx_channels_count; chan++) 1796 stmmac_stop_tx_dma(priv, chan); 1797 } 1798 1799 /** 1800 * stmmac_dma_operation_mode - HW DMA operation mode 1801 * @priv: driver private structure 1802 * Description: it is used for configuring the DMA operation mode register in 1803 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 1804 */ 1805 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 1806 { 1807 u32 rx_channels_count = priv->plat->rx_queues_to_use; 1808 u32 tx_channels_count = priv->plat->tx_queues_to_use; 1809 int rxfifosz = priv->plat->rx_fifo_size; 1810 int txfifosz = priv->plat->tx_fifo_size; 1811 u32 txmode = 0; 1812 u32 rxmode = 0; 1813 u32 chan = 0; 1814 u8 qmode = 0; 1815 1816 if (rxfifosz == 0) 1817 rxfifosz = priv->dma_cap.rx_fifo_size; 1818 if (txfifosz == 0) 1819 txfifosz = priv->dma_cap.tx_fifo_size; 1820 1821 /* Adjust for real per queue fifo size */ 1822 rxfifosz /= rx_channels_count; 1823 txfifosz /= tx_channels_count; 1824 1825 if (priv->plat->force_thresh_dma_mode) { 1826 txmode = tc; 1827 rxmode = tc; 1828 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 1829 /* 1830 * In case of GMAC, SF mode can be enabled 1831 * to perform the TX COE in HW. This depends on: 1832 * 1) TX COE if actually supported 1833 * 2) There is no bugged Jumbo frame support 1834 * that needs to not insert csum in the TDES. 1835 */ 1836 txmode = SF_DMA_MODE; 1837 rxmode = SF_DMA_MODE; 1838 priv->xstats.threshold = SF_DMA_MODE; 1839 } else { 1840 txmode = tc; 1841 rxmode = SF_DMA_MODE; 1842 } 1843 1844 /* configure all channels */ 1845 for (chan = 0; chan < rx_channels_count; chan++) { 1846 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 1847 1848 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 1849 rxfifosz, qmode); 1850 stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz, 1851 chan); 1852 } 1853 1854 for (chan = 0; chan < tx_channels_count; chan++) { 1855 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 1856 1857 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 1858 txfifosz, qmode); 1859 } 1860 } 1861 1862 /** 1863 * stmmac_tx_clean - to manage the transmission completion 1864 * @priv: driver private structure 1865 * @queue: TX queue index 1866 * Description: it reclaims the transmit resources after transmission completes. 1867 */ 1868 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) 1869 { 1870 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1871 unsigned int bytes_compl = 0, pkts_compl = 0; 1872 unsigned int entry, count = 0; 1873 1874 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 1875 1876 priv->xstats.tx_clean++; 1877 1878 entry = tx_q->dirty_tx; 1879 while ((entry != tx_q->cur_tx) && (count < budget)) { 1880 struct sk_buff *skb = tx_q->tx_skbuff[entry]; 1881 struct dma_desc *p; 1882 int status; 1883 1884 if (priv->extend_desc) 1885 p = (struct dma_desc *)(tx_q->dma_etx + entry); 1886 else 1887 p = tx_q->dma_tx + entry; 1888 1889 status = stmmac_tx_status(priv, &priv->dev->stats, 1890 &priv->xstats, p, priv->ioaddr); 1891 /* Check if the descriptor is owned by the DMA */ 1892 if (unlikely(status & tx_dma_own)) 1893 break; 1894 1895 count++; 1896 1897 /* Make sure descriptor fields are read after reading 1898 * the own bit. 1899 */ 1900 dma_rmb(); 1901 1902 /* Just consider the last segment and ...*/ 1903 if (likely(!(status & tx_not_ls))) { 1904 /* ... verify the status error condition */ 1905 if (unlikely(status & tx_err)) { 1906 priv->dev->stats.tx_errors++; 1907 } else { 1908 priv->dev->stats.tx_packets++; 1909 priv->xstats.tx_pkt_n++; 1910 } 1911 stmmac_get_tx_hwtstamp(priv, p, skb); 1912 } 1913 1914 if (likely(tx_q->tx_skbuff_dma[entry].buf)) { 1915 if (tx_q->tx_skbuff_dma[entry].map_as_page) 1916 dma_unmap_page(priv->device, 1917 tx_q->tx_skbuff_dma[entry].buf, 1918 tx_q->tx_skbuff_dma[entry].len, 1919 DMA_TO_DEVICE); 1920 else 1921 dma_unmap_single(priv->device, 1922 tx_q->tx_skbuff_dma[entry].buf, 1923 tx_q->tx_skbuff_dma[entry].len, 1924 DMA_TO_DEVICE); 1925 tx_q->tx_skbuff_dma[entry].buf = 0; 1926 tx_q->tx_skbuff_dma[entry].len = 0; 1927 tx_q->tx_skbuff_dma[entry].map_as_page = false; 1928 } 1929 1930 stmmac_clean_desc3(priv, tx_q, p); 1931 1932 tx_q->tx_skbuff_dma[entry].last_segment = false; 1933 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 1934 1935 if (likely(skb != NULL)) { 1936 pkts_compl++; 1937 bytes_compl += skb->len; 1938 dev_consume_skb_any(skb); 1939 tx_q->tx_skbuff[entry] = NULL; 1940 } 1941 1942 stmmac_release_tx_desc(priv, p, priv->mode); 1943 1944 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 1945 } 1946 tx_q->dirty_tx = entry; 1947 1948 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 1949 pkts_compl, bytes_compl); 1950 1951 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 1952 queue))) && 1953 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) { 1954 1955 netif_dbg(priv, tx_done, priv->dev, 1956 "%s: restart transmit\n", __func__); 1957 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 1958 } 1959 1960 if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) { 1961 stmmac_enable_eee_mode(priv); 1962 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer)); 1963 } 1964 1965 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 1966 1967 return count; 1968 } 1969 1970 /** 1971 * stmmac_tx_err - to manage the tx error 1972 * @priv: driver private structure 1973 * @chan: channel index 1974 * Description: it cleans the descriptors and restarts the transmission 1975 * in case of transmission errors. 1976 */ 1977 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 1978 { 1979 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 1980 int i; 1981 1982 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 1983 1984 stmmac_stop_tx_dma(priv, chan); 1985 dma_free_tx_skbufs(priv, chan); 1986 for (i = 0; i < DMA_TX_SIZE; i++) 1987 if (priv->extend_desc) 1988 stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic, 1989 priv->mode, (i == DMA_TX_SIZE - 1)); 1990 else 1991 stmmac_init_tx_desc(priv, &tx_q->dma_tx[i], 1992 priv->mode, (i == DMA_TX_SIZE - 1)); 1993 tx_q->dirty_tx = 0; 1994 tx_q->cur_tx = 0; 1995 tx_q->mss = 0; 1996 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); 1997 stmmac_start_tx_dma(priv, chan); 1998 1999 priv->dev->stats.tx_errors++; 2000 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2001 } 2002 2003 /** 2004 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2005 * @priv: driver private structure 2006 * @txmode: TX operating mode 2007 * @rxmode: RX operating mode 2008 * @chan: channel index 2009 * Description: it is used for configuring of the DMA operation mode in 2010 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2011 * mode. 2012 */ 2013 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2014 u32 rxmode, u32 chan) 2015 { 2016 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2017 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2018 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2019 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2020 int rxfifosz = priv->plat->rx_fifo_size; 2021 int txfifosz = priv->plat->tx_fifo_size; 2022 2023 if (rxfifosz == 0) 2024 rxfifosz = priv->dma_cap.rx_fifo_size; 2025 if (txfifosz == 0) 2026 txfifosz = priv->dma_cap.tx_fifo_size; 2027 2028 /* Adjust for real per queue fifo size */ 2029 rxfifosz /= rx_channels_count; 2030 txfifosz /= tx_channels_count; 2031 2032 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2033 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2034 } 2035 2036 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2037 { 2038 int ret; 2039 2040 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2041 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2042 if (ret && (ret != -EINVAL)) { 2043 stmmac_global_err(priv); 2044 return true; 2045 } 2046 2047 return false; 2048 } 2049 2050 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan) 2051 { 2052 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2053 &priv->xstats, chan); 2054 struct stmmac_channel *ch = &priv->channel[chan]; 2055 bool needs_work = false; 2056 2057 if ((status & handle_rx) && ch->has_rx) { 2058 needs_work = true; 2059 } else { 2060 status &= ~handle_rx; 2061 } 2062 2063 if ((status & handle_tx) && ch->has_tx) { 2064 needs_work = true; 2065 } else { 2066 status &= ~handle_tx; 2067 } 2068 2069 if (needs_work && napi_schedule_prep(&ch->napi)) { 2070 stmmac_disable_dma_irq(priv, priv->ioaddr, chan); 2071 __napi_schedule(&ch->napi); 2072 } 2073 2074 return status; 2075 } 2076 2077 /** 2078 * stmmac_dma_interrupt - DMA ISR 2079 * @priv: driver private structure 2080 * Description: this is the DMA ISR. It is called by the main ISR. 2081 * It calls the dwmac dma routine and schedule poll method in case of some 2082 * work can be done. 2083 */ 2084 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2085 { 2086 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2087 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2088 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2089 tx_channel_count : rx_channel_count; 2090 u32 chan; 2091 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2092 2093 /* Make sure we never check beyond our status buffer. */ 2094 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2095 channels_to_check = ARRAY_SIZE(status); 2096 2097 for (chan = 0; chan < channels_to_check; chan++) 2098 status[chan] = stmmac_napi_check(priv, chan); 2099 2100 for (chan = 0; chan < tx_channel_count; chan++) { 2101 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2102 /* Try to bump up the dma threshold on this failure */ 2103 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && 2104 (tc <= 256)) { 2105 tc += 64; 2106 if (priv->plat->force_thresh_dma_mode) 2107 stmmac_set_dma_operation_mode(priv, 2108 tc, 2109 tc, 2110 chan); 2111 else 2112 stmmac_set_dma_operation_mode(priv, 2113 tc, 2114 SF_DMA_MODE, 2115 chan); 2116 priv->xstats.threshold = tc; 2117 } 2118 } else if (unlikely(status[chan] == tx_hard_error)) { 2119 stmmac_tx_err(priv, chan); 2120 } 2121 } 2122 } 2123 2124 /** 2125 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2126 * @priv: driver private structure 2127 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2128 */ 2129 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2130 { 2131 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2132 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2133 2134 dwmac_mmc_intr_all_mask(priv->mmcaddr); 2135 2136 if (priv->dma_cap.rmon) { 2137 dwmac_mmc_ctrl(priv->mmcaddr, mode); 2138 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2139 } else 2140 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2141 } 2142 2143 /** 2144 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2145 * @priv: driver private structure 2146 * Description: 2147 * new GMAC chip generations have a new register to indicate the 2148 * presence of the optional feature/functions. 2149 * This can be also used to override the value passed through the 2150 * platform and necessary for old MAC10/100 and GMAC chips. 2151 */ 2152 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2153 { 2154 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 2155 } 2156 2157 /** 2158 * stmmac_check_ether_addr - check if the MAC addr is valid 2159 * @priv: driver private structure 2160 * Description: 2161 * it is to verify if the MAC address is valid, in case of failures it 2162 * generates a random MAC address 2163 */ 2164 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2165 { 2166 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2167 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0); 2168 if (!is_valid_ether_addr(priv->dev->dev_addr)) 2169 eth_hw_addr_random(priv->dev); 2170 netdev_info(priv->dev, "device MAC address %pM\n", 2171 priv->dev->dev_addr); 2172 } 2173 } 2174 2175 /** 2176 * stmmac_init_dma_engine - DMA init. 2177 * @priv: driver private structure 2178 * Description: 2179 * It inits the DMA invoking the specific MAC/GMAC callback. 2180 * Some DMA parameters can be passed from the platform; 2181 * in case of these are not passed a default is kept for the MAC or GMAC. 2182 */ 2183 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2184 { 2185 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2186 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2187 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2188 struct stmmac_rx_queue *rx_q; 2189 struct stmmac_tx_queue *tx_q; 2190 u32 chan = 0; 2191 int atds = 0; 2192 int ret = 0; 2193 2194 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2195 dev_err(priv->device, "Invalid DMA configuration\n"); 2196 return -EINVAL; 2197 } 2198 2199 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2200 atds = 1; 2201 2202 ret = stmmac_reset(priv, priv->ioaddr); 2203 if (ret) { 2204 dev_err(priv->device, "Failed to reset the dma\n"); 2205 return ret; 2206 } 2207 2208 /* DMA Configuration */ 2209 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); 2210 2211 if (priv->plat->axi) 2212 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 2213 2214 /* DMA RX Channel Configuration */ 2215 for (chan = 0; chan < rx_channels_count; chan++) { 2216 rx_q = &priv->rx_queue[chan]; 2217 2218 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2219 rx_q->dma_rx_phy, chan); 2220 2221 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2222 (DMA_RX_SIZE * sizeof(struct dma_desc)); 2223 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 2224 rx_q->rx_tail_addr, chan); 2225 } 2226 2227 /* DMA TX Channel Configuration */ 2228 for (chan = 0; chan < tx_channels_count; chan++) { 2229 tx_q = &priv->tx_queue[chan]; 2230 2231 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2232 tx_q->dma_tx_phy, chan); 2233 2234 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 2235 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 2236 tx_q->tx_tail_addr, chan); 2237 } 2238 2239 /* DMA CSR Channel configuration */ 2240 for (chan = 0; chan < dma_csr_ch; chan++) 2241 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 2242 2243 return ret; 2244 } 2245 2246 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 2247 { 2248 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2249 2250 mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer)); 2251 } 2252 2253 /** 2254 * stmmac_tx_timer - mitigation sw timer for tx. 2255 * @data: data pointer 2256 * Description: 2257 * This is the timer handler to directly invoke the stmmac_tx_clean. 2258 */ 2259 static void stmmac_tx_timer(struct timer_list *t) 2260 { 2261 struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer); 2262 struct stmmac_priv *priv = tx_q->priv_data; 2263 struct stmmac_channel *ch; 2264 2265 ch = &priv->channel[tx_q->queue_index]; 2266 2267 if (likely(napi_schedule_prep(&ch->napi))) 2268 __napi_schedule(&ch->napi); 2269 } 2270 2271 /** 2272 * stmmac_init_tx_coalesce - init tx mitigation options. 2273 * @priv: driver private structure 2274 * Description: 2275 * This inits the transmit coalesce parameters: i.e. timer rate, 2276 * timer handler and default threshold used for enabling the 2277 * interrupt on completion bit. 2278 */ 2279 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv) 2280 { 2281 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2282 u32 chan; 2283 2284 priv->tx_coal_frames = STMMAC_TX_FRAMES; 2285 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER; 2286 2287 for (chan = 0; chan < tx_channel_count; chan++) { 2288 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 2289 2290 timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0); 2291 } 2292 } 2293 2294 static void stmmac_set_rings_length(struct stmmac_priv *priv) 2295 { 2296 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2297 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2298 u32 chan; 2299 2300 /* set TX ring length */ 2301 for (chan = 0; chan < tx_channels_count; chan++) 2302 stmmac_set_tx_ring_len(priv, priv->ioaddr, 2303 (DMA_TX_SIZE - 1), chan); 2304 2305 /* set RX ring length */ 2306 for (chan = 0; chan < rx_channels_count; chan++) 2307 stmmac_set_rx_ring_len(priv, priv->ioaddr, 2308 (DMA_RX_SIZE - 1), chan); 2309 } 2310 2311 /** 2312 * stmmac_set_tx_queue_weight - Set TX queue weight 2313 * @priv: driver private structure 2314 * Description: It is used for setting TX queues weight 2315 */ 2316 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 2317 { 2318 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2319 u32 weight; 2320 u32 queue; 2321 2322 for (queue = 0; queue < tx_queues_count; queue++) { 2323 weight = priv->plat->tx_queues_cfg[queue].weight; 2324 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 2325 } 2326 } 2327 2328 /** 2329 * stmmac_configure_cbs - Configure CBS in TX queue 2330 * @priv: driver private structure 2331 * Description: It is used for configuring CBS in AVB TX queues 2332 */ 2333 static void stmmac_configure_cbs(struct stmmac_priv *priv) 2334 { 2335 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2336 u32 mode_to_use; 2337 u32 queue; 2338 2339 /* queue 0 is reserved for legacy traffic */ 2340 for (queue = 1; queue < tx_queues_count; queue++) { 2341 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 2342 if (mode_to_use == MTL_QUEUE_DCB) 2343 continue; 2344 2345 stmmac_config_cbs(priv, priv->hw, 2346 priv->plat->tx_queues_cfg[queue].send_slope, 2347 priv->plat->tx_queues_cfg[queue].idle_slope, 2348 priv->plat->tx_queues_cfg[queue].high_credit, 2349 priv->plat->tx_queues_cfg[queue].low_credit, 2350 queue); 2351 } 2352 } 2353 2354 /** 2355 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 2356 * @priv: driver private structure 2357 * Description: It is used for mapping RX queues to RX dma channels 2358 */ 2359 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 2360 { 2361 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2362 u32 queue; 2363 u32 chan; 2364 2365 for (queue = 0; queue < rx_queues_count; queue++) { 2366 chan = priv->plat->rx_queues_cfg[queue].chan; 2367 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 2368 } 2369 } 2370 2371 /** 2372 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 2373 * @priv: driver private structure 2374 * Description: It is used for configuring the RX Queue Priority 2375 */ 2376 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 2377 { 2378 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2379 u32 queue; 2380 u32 prio; 2381 2382 for (queue = 0; queue < rx_queues_count; queue++) { 2383 if (!priv->plat->rx_queues_cfg[queue].use_prio) 2384 continue; 2385 2386 prio = priv->plat->rx_queues_cfg[queue].prio; 2387 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 2388 } 2389 } 2390 2391 /** 2392 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 2393 * @priv: driver private structure 2394 * Description: It is used for configuring the TX Queue Priority 2395 */ 2396 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 2397 { 2398 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2399 u32 queue; 2400 u32 prio; 2401 2402 for (queue = 0; queue < tx_queues_count; queue++) { 2403 if (!priv->plat->tx_queues_cfg[queue].use_prio) 2404 continue; 2405 2406 prio = priv->plat->tx_queues_cfg[queue].prio; 2407 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 2408 } 2409 } 2410 2411 /** 2412 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 2413 * @priv: driver private structure 2414 * Description: It is used for configuring the RX queue routing 2415 */ 2416 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 2417 { 2418 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2419 u32 queue; 2420 u8 packet; 2421 2422 for (queue = 0; queue < rx_queues_count; queue++) { 2423 /* no specific packet type routing specified for the queue */ 2424 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 2425 continue; 2426 2427 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 2428 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 2429 } 2430 } 2431 2432 /** 2433 * stmmac_mtl_configuration - Configure MTL 2434 * @priv: driver private structure 2435 * Description: It is used for configurring MTL 2436 */ 2437 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 2438 { 2439 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2440 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2441 2442 if (tx_queues_count > 1) 2443 stmmac_set_tx_queue_weight(priv); 2444 2445 /* Configure MTL RX algorithms */ 2446 if (rx_queues_count > 1) 2447 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 2448 priv->plat->rx_sched_algorithm); 2449 2450 /* Configure MTL TX algorithms */ 2451 if (tx_queues_count > 1) 2452 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 2453 priv->plat->tx_sched_algorithm); 2454 2455 /* Configure CBS in AVB TX queues */ 2456 if (tx_queues_count > 1) 2457 stmmac_configure_cbs(priv); 2458 2459 /* Map RX MTL to DMA channels */ 2460 stmmac_rx_queue_dma_chan_map(priv); 2461 2462 /* Enable MAC RX Queues */ 2463 stmmac_mac_enable_rx_queues(priv); 2464 2465 /* Set RX priorities */ 2466 if (rx_queues_count > 1) 2467 stmmac_mac_config_rx_queues_prio(priv); 2468 2469 /* Set TX priorities */ 2470 if (tx_queues_count > 1) 2471 stmmac_mac_config_tx_queues_prio(priv); 2472 2473 /* Set RX routing */ 2474 if (rx_queues_count > 1) 2475 stmmac_mac_config_rx_queues_routing(priv); 2476 } 2477 2478 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 2479 { 2480 if (priv->dma_cap.asp) { 2481 netdev_info(priv->dev, "Enabling Safety Features\n"); 2482 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp); 2483 } else { 2484 netdev_info(priv->dev, "No Safety Features support found\n"); 2485 } 2486 } 2487 2488 /** 2489 * stmmac_hw_setup - setup mac in a usable state. 2490 * @dev : pointer to the device structure. 2491 * Description: 2492 * this is the main function to setup the HW in a usable state because the 2493 * dma engine is reset, the core registers are configured (e.g. AXI, 2494 * Checksum features, timers). The DMA is ready to start receiving and 2495 * transmitting. 2496 * Return value: 2497 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2498 * file on failure. 2499 */ 2500 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) 2501 { 2502 struct stmmac_priv *priv = netdev_priv(dev); 2503 u32 rx_cnt = priv->plat->rx_queues_to_use; 2504 u32 tx_cnt = priv->plat->tx_queues_to_use; 2505 u32 chan; 2506 int ret; 2507 2508 /* DMA initialization and SW reset */ 2509 ret = stmmac_init_dma_engine(priv); 2510 if (ret < 0) { 2511 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 2512 __func__); 2513 return ret; 2514 } 2515 2516 /* Copy the MAC addr into the HW */ 2517 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 2518 2519 /* PS and related bits will be programmed according to the speed */ 2520 if (priv->hw->pcs) { 2521 int speed = priv->plat->mac_port_sel_speed; 2522 2523 if ((speed == SPEED_10) || (speed == SPEED_100) || 2524 (speed == SPEED_1000)) { 2525 priv->hw->ps = speed; 2526 } else { 2527 dev_warn(priv->device, "invalid port speed\n"); 2528 priv->hw->ps = 0; 2529 } 2530 } 2531 2532 /* Initialize the MAC Core */ 2533 stmmac_core_init(priv, priv->hw, dev); 2534 2535 /* Initialize MTL*/ 2536 stmmac_mtl_configuration(priv); 2537 2538 /* Initialize Safety Features */ 2539 stmmac_safety_feat_configuration(priv); 2540 2541 ret = stmmac_rx_ipc(priv, priv->hw); 2542 if (!ret) { 2543 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 2544 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 2545 priv->hw->rx_csum = 0; 2546 } 2547 2548 /* Enable the MAC Rx/Tx */ 2549 stmmac_mac_set(priv, priv->ioaddr, true); 2550 2551 /* Set the HW DMA mode and the COE */ 2552 stmmac_dma_operation_mode(priv); 2553 2554 stmmac_mmc_setup(priv); 2555 2556 if (init_ptp) { 2557 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 2558 if (ret < 0) 2559 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret); 2560 2561 ret = stmmac_init_ptp(priv); 2562 if (ret == -EOPNOTSUPP) 2563 netdev_warn(priv->dev, "PTP not supported by HW\n"); 2564 else if (ret) 2565 netdev_warn(priv->dev, "PTP init failed\n"); 2566 } 2567 2568 priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS; 2569 2570 if (priv->use_riwt) { 2571 ret = stmmac_rx_watchdog(priv, priv->ioaddr, MAX_DMA_RIWT, rx_cnt); 2572 if (!ret) 2573 priv->rx_riwt = MAX_DMA_RIWT; 2574 } 2575 2576 if (priv->hw->pcs) 2577 stmmac_pcs_ctrl_ane(priv, priv->hw, 1, priv->hw->ps, 0); 2578 2579 /* set TX and RX rings length */ 2580 stmmac_set_rings_length(priv); 2581 2582 /* Enable TSO */ 2583 if (priv->tso) { 2584 for (chan = 0; chan < tx_cnt; chan++) 2585 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 2586 } 2587 2588 /* Start the ball rolling... */ 2589 stmmac_start_all_dma(priv); 2590 2591 return 0; 2592 } 2593 2594 static void stmmac_hw_teardown(struct net_device *dev) 2595 { 2596 struct stmmac_priv *priv = netdev_priv(dev); 2597 2598 clk_disable_unprepare(priv->plat->clk_ptp_ref); 2599 } 2600 2601 /** 2602 * stmmac_open - open entry point of the driver 2603 * @dev : pointer to the device structure. 2604 * Description: 2605 * This function is the open entry point of the driver. 2606 * Return value: 2607 * 0 on success and an appropriate (-)ve integer as defined in errno.h 2608 * file on failure. 2609 */ 2610 static int stmmac_open(struct net_device *dev) 2611 { 2612 struct stmmac_priv *priv = netdev_priv(dev); 2613 u32 chan; 2614 int ret; 2615 2616 stmmac_check_ether_addr(priv); 2617 2618 if (priv->hw->pcs != STMMAC_PCS_RGMII && 2619 priv->hw->pcs != STMMAC_PCS_TBI && 2620 priv->hw->pcs != STMMAC_PCS_RTBI) { 2621 ret = stmmac_init_phy(dev); 2622 if (ret) { 2623 netdev_err(priv->dev, 2624 "%s: Cannot attach to PHY (error: %d)\n", 2625 __func__, ret); 2626 return ret; 2627 } 2628 } 2629 2630 /* Extra statistics */ 2631 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); 2632 priv->xstats.threshold = tc; 2633 2634 priv->dma_buf_sz = STMMAC_ALIGN(buf_sz); 2635 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 2636 2637 ret = alloc_dma_desc_resources(priv); 2638 if (ret < 0) { 2639 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 2640 __func__); 2641 goto dma_desc_error; 2642 } 2643 2644 ret = init_dma_desc_rings(dev, GFP_KERNEL); 2645 if (ret < 0) { 2646 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 2647 __func__); 2648 goto init_error; 2649 } 2650 2651 ret = stmmac_hw_setup(dev, true); 2652 if (ret < 0) { 2653 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 2654 goto init_error; 2655 } 2656 2657 stmmac_init_tx_coalesce(priv); 2658 2659 if (dev->phydev) 2660 phy_start(dev->phydev); 2661 2662 /* Request the IRQ lines */ 2663 ret = request_irq(dev->irq, stmmac_interrupt, 2664 IRQF_SHARED, dev->name, dev); 2665 if (unlikely(ret < 0)) { 2666 netdev_err(priv->dev, 2667 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 2668 __func__, dev->irq, ret); 2669 goto irq_error; 2670 } 2671 2672 /* Request the Wake IRQ in case of another line is used for WoL */ 2673 if (priv->wol_irq != dev->irq) { 2674 ret = request_irq(priv->wol_irq, stmmac_interrupt, 2675 IRQF_SHARED, dev->name, dev); 2676 if (unlikely(ret < 0)) { 2677 netdev_err(priv->dev, 2678 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 2679 __func__, priv->wol_irq, ret); 2680 goto wolirq_error; 2681 } 2682 } 2683 2684 /* Request the IRQ lines */ 2685 if (priv->lpi_irq > 0) { 2686 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED, 2687 dev->name, dev); 2688 if (unlikely(ret < 0)) { 2689 netdev_err(priv->dev, 2690 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 2691 __func__, priv->lpi_irq, ret); 2692 goto lpiirq_error; 2693 } 2694 } 2695 2696 stmmac_enable_all_queues(priv); 2697 stmmac_start_all_queues(priv); 2698 2699 return 0; 2700 2701 lpiirq_error: 2702 if (priv->wol_irq != dev->irq) 2703 free_irq(priv->wol_irq, dev); 2704 wolirq_error: 2705 free_irq(dev->irq, dev); 2706 irq_error: 2707 if (dev->phydev) 2708 phy_stop(dev->phydev); 2709 2710 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 2711 del_timer_sync(&priv->tx_queue[chan].txtimer); 2712 2713 stmmac_hw_teardown(dev); 2714 init_error: 2715 free_dma_desc_resources(priv); 2716 dma_desc_error: 2717 if (dev->phydev) 2718 phy_disconnect(dev->phydev); 2719 2720 return ret; 2721 } 2722 2723 /** 2724 * stmmac_release - close entry point of the driver 2725 * @dev : device pointer. 2726 * Description: 2727 * This is the stop entry point of the driver. 2728 */ 2729 static int stmmac_release(struct net_device *dev) 2730 { 2731 struct stmmac_priv *priv = netdev_priv(dev); 2732 u32 chan; 2733 2734 if (priv->eee_enabled) 2735 del_timer_sync(&priv->eee_ctrl_timer); 2736 2737 /* Stop and disconnect the PHY */ 2738 if (dev->phydev) { 2739 phy_stop(dev->phydev); 2740 phy_disconnect(dev->phydev); 2741 } 2742 2743 stmmac_stop_all_queues(priv); 2744 2745 stmmac_disable_all_queues(priv); 2746 2747 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 2748 del_timer_sync(&priv->tx_queue[chan].txtimer); 2749 2750 /* Free the IRQ lines */ 2751 free_irq(dev->irq, dev); 2752 if (priv->wol_irq != dev->irq) 2753 free_irq(priv->wol_irq, dev); 2754 if (priv->lpi_irq > 0) 2755 free_irq(priv->lpi_irq, dev); 2756 2757 /* Stop TX/RX DMA and clear the descriptors */ 2758 stmmac_stop_all_dma(priv); 2759 2760 /* Release and free the Rx/Tx resources */ 2761 free_dma_desc_resources(priv); 2762 2763 /* Disable the MAC Rx/Tx */ 2764 stmmac_mac_set(priv, priv->ioaddr, false); 2765 2766 netif_carrier_off(dev); 2767 2768 stmmac_release_ptp(priv); 2769 2770 return 0; 2771 } 2772 2773 /** 2774 * stmmac_tso_allocator - close entry point of the driver 2775 * @priv: driver private structure 2776 * @des: buffer start address 2777 * @total_len: total length to fill in descriptors 2778 * @last_segmant: condition for the last descriptor 2779 * @queue: TX queue index 2780 * Description: 2781 * This function fills descriptor and request new descriptors according to 2782 * buffer length to fill 2783 */ 2784 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des, 2785 int total_len, bool last_segment, u32 queue) 2786 { 2787 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2788 struct dma_desc *desc; 2789 u32 buff_size; 2790 int tmp_len; 2791 2792 tmp_len = total_len; 2793 2794 while (tmp_len > 0) { 2795 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2796 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 2797 desc = tx_q->dma_tx + tx_q->cur_tx; 2798 2799 desc->des0 = cpu_to_le32(des + (total_len - tmp_len)); 2800 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 2801 TSO_MAX_BUFF_SIZE : tmp_len; 2802 2803 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 2804 0, 1, 2805 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 2806 0, 0); 2807 2808 tmp_len -= TSO_MAX_BUFF_SIZE; 2809 } 2810 } 2811 2812 /** 2813 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 2814 * @skb : the socket buffer 2815 * @dev : device pointer 2816 * Description: this is the transmit function that is called on TSO frames 2817 * (support available on GMAC4 and newer chips). 2818 * Diagram below show the ring programming in case of TSO frames: 2819 * 2820 * First Descriptor 2821 * -------- 2822 * | DES0 |---> buffer1 = L2/L3/L4 header 2823 * | DES1 |---> TCP Payload (can continue on next descr...) 2824 * | DES2 |---> buffer 1 and 2 len 2825 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 2826 * -------- 2827 * | 2828 * ... 2829 * | 2830 * -------- 2831 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 2832 * | DES1 | --| 2833 * | DES2 | --> buffer 1 and 2 len 2834 * | DES3 | 2835 * -------- 2836 * 2837 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 2838 */ 2839 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 2840 { 2841 struct dma_desc *desc, *first, *mss_desc = NULL; 2842 struct stmmac_priv *priv = netdev_priv(dev); 2843 int nfrags = skb_shinfo(skb)->nr_frags; 2844 u32 queue = skb_get_queue_mapping(skb); 2845 unsigned int first_entry, des; 2846 struct stmmac_tx_queue *tx_q; 2847 int tmp_pay_len = 0; 2848 u32 pay_len, mss; 2849 u8 proto_hdr_len; 2850 int i; 2851 2852 tx_q = &priv->tx_queue[queue]; 2853 2854 /* Compute header lengths */ 2855 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2856 2857 /* Desc availability based on threshold should be enough safe */ 2858 if (unlikely(stmmac_tx_avail(priv, queue) < 2859 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 2860 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 2861 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 2862 queue)); 2863 /* This is a hard error, log it. */ 2864 netdev_err(priv->dev, 2865 "%s: Tx Ring full when queue awake\n", 2866 __func__); 2867 } 2868 return NETDEV_TX_BUSY; 2869 } 2870 2871 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 2872 2873 mss = skb_shinfo(skb)->gso_size; 2874 2875 /* set new MSS value if needed */ 2876 if (mss != tx_q->mss) { 2877 mss_desc = tx_q->dma_tx + tx_q->cur_tx; 2878 stmmac_set_mss(priv, mss_desc, mss); 2879 tx_q->mss = mss; 2880 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2881 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 2882 } 2883 2884 if (netif_msg_tx_queued(priv)) { 2885 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 2886 __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss); 2887 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 2888 skb->data_len); 2889 } 2890 2891 first_entry = tx_q->cur_tx; 2892 WARN_ON(tx_q->tx_skbuff[first_entry]); 2893 2894 desc = tx_q->dma_tx + first_entry; 2895 first = desc; 2896 2897 /* first descriptor: fill Headers on Buf1 */ 2898 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 2899 DMA_TO_DEVICE); 2900 if (dma_mapping_error(priv->device, des)) 2901 goto dma_map_err; 2902 2903 tx_q->tx_skbuff_dma[first_entry].buf = des; 2904 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 2905 2906 first->des0 = cpu_to_le32(des); 2907 2908 /* Fill start of payload in buff2 of first descriptor */ 2909 if (pay_len) 2910 first->des1 = cpu_to_le32(des + proto_hdr_len); 2911 2912 /* If needed take extra descriptors to fill the remaining payload */ 2913 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 2914 2915 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 2916 2917 /* Prepare fragments */ 2918 for (i = 0; i < nfrags; i++) { 2919 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2920 2921 des = skb_frag_dma_map(priv->device, frag, 0, 2922 skb_frag_size(frag), 2923 DMA_TO_DEVICE); 2924 if (dma_mapping_error(priv->device, des)) 2925 goto dma_map_err; 2926 2927 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 2928 (i == nfrags - 1), queue); 2929 2930 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 2931 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 2932 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 2933 } 2934 2935 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 2936 2937 /* Only the last descriptor gets to point to the skb. */ 2938 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 2939 2940 /* We've used all descriptors we need for this skb, however, 2941 * advance cur_tx so that it references a fresh descriptor. 2942 * ndo_start_xmit will fill this descriptor the next time it's 2943 * called and stmmac_tx_clean may clean up to this descriptor. 2944 */ 2945 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE); 2946 2947 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 2948 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 2949 __func__); 2950 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 2951 } 2952 2953 dev->stats.tx_bytes += skb->len; 2954 priv->xstats.tx_tso_frames++; 2955 priv->xstats.tx_tso_nfrags += nfrags; 2956 2957 /* Manage tx mitigation */ 2958 tx_q->tx_count_frames += nfrags + 1; 2959 if (priv->tx_coal_frames <= tx_q->tx_count_frames) { 2960 stmmac_set_tx_ic(priv, desc); 2961 priv->xstats.tx_set_ic_bit++; 2962 tx_q->tx_count_frames = 0; 2963 } else { 2964 stmmac_tx_timer_arm(priv, queue); 2965 } 2966 2967 skb_tx_timestamp(skb); 2968 2969 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 2970 priv->hwts_tx_en)) { 2971 /* declare that device is doing timestamping */ 2972 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2973 stmmac_enable_tx_timestamp(priv, first); 2974 } 2975 2976 /* Complete the first descriptor before granting the DMA */ 2977 stmmac_prepare_tso_tx_desc(priv, first, 1, 2978 proto_hdr_len, 2979 pay_len, 2980 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 2981 tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len)); 2982 2983 /* If context desc is used to change MSS */ 2984 if (mss_desc) { 2985 /* Make sure that first descriptor has been completely 2986 * written, including its own bit. This is because MSS is 2987 * actually before first descriptor, so we need to make 2988 * sure that MSS's own bit is the last thing written. 2989 */ 2990 dma_wmb(); 2991 stmmac_set_tx_owner(priv, mss_desc); 2992 } 2993 2994 /* The own bit must be the latest setting done when prepare the 2995 * descriptor and then barrier is needed to make sure that 2996 * all is coherent before granting the DMA engine. 2997 */ 2998 wmb(); 2999 3000 if (netif_msg_pktdata(priv)) { 3001 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 3002 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 3003 tx_q->cur_tx, first, nfrags); 3004 3005 stmmac_display_ring(priv, (void *)tx_q->dma_tx, DMA_TX_SIZE, 0); 3006 3007 pr_info(">>> frame to be transmitted: "); 3008 print_pkt(skb->data, skb_headlen(skb)); 3009 } 3010 3011 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 3012 3013 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc)); 3014 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 3015 3016 return NETDEV_TX_OK; 3017 3018 dma_map_err: 3019 dev_err(priv->device, "Tx dma map failed\n"); 3020 dev_kfree_skb(skb); 3021 priv->dev->stats.tx_dropped++; 3022 return NETDEV_TX_OK; 3023 } 3024 3025 /** 3026 * stmmac_xmit - Tx entry point of the driver 3027 * @skb : the socket buffer 3028 * @dev : device pointer 3029 * Description : this is the tx entry point of the driver. 3030 * It programs the chain or the ring and supports oversized frames 3031 * and SG feature. 3032 */ 3033 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 3034 { 3035 struct stmmac_priv *priv = netdev_priv(dev); 3036 unsigned int nopaged_len = skb_headlen(skb); 3037 int i, csum_insertion = 0, is_jumbo = 0; 3038 u32 queue = skb_get_queue_mapping(skb); 3039 int nfrags = skb_shinfo(skb)->nr_frags; 3040 int entry; 3041 unsigned int first_entry; 3042 struct dma_desc *desc, *first; 3043 struct stmmac_tx_queue *tx_q; 3044 unsigned int enh_desc; 3045 unsigned int des; 3046 3047 tx_q = &priv->tx_queue[queue]; 3048 3049 /* Manage oversized TCP frames for GMAC4 device */ 3050 if (skb_is_gso(skb) && priv->tso) { 3051 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 3052 return stmmac_tso_xmit(skb, dev); 3053 } 3054 3055 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 3056 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 3057 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 3058 queue)); 3059 /* This is a hard error, log it. */ 3060 netdev_err(priv->dev, 3061 "%s: Tx Ring full when queue awake\n", 3062 __func__); 3063 } 3064 return NETDEV_TX_BUSY; 3065 } 3066 3067 if (priv->tx_path_in_lpi_mode) 3068 stmmac_disable_eee_mode(priv); 3069 3070 entry = tx_q->cur_tx; 3071 first_entry = entry; 3072 WARN_ON(tx_q->tx_skbuff[first_entry]); 3073 3074 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 3075 3076 if (likely(priv->extend_desc)) 3077 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3078 else 3079 desc = tx_q->dma_tx + entry; 3080 3081 first = desc; 3082 3083 enh_desc = priv->plat->enh_desc; 3084 /* To program the descriptors according to the size of the frame */ 3085 if (enh_desc) 3086 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 3087 3088 if (unlikely(is_jumbo)) { 3089 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 3090 if (unlikely(entry < 0) && (entry != -EINVAL)) 3091 goto dma_map_err; 3092 } 3093 3094 for (i = 0; i < nfrags; i++) { 3095 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3096 int len = skb_frag_size(frag); 3097 bool last_segment = (i == (nfrags - 1)); 3098 3099 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3100 WARN_ON(tx_q->tx_skbuff[entry]); 3101 3102 if (likely(priv->extend_desc)) 3103 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 3104 else 3105 desc = tx_q->dma_tx + entry; 3106 3107 des = skb_frag_dma_map(priv->device, frag, 0, len, 3108 DMA_TO_DEVICE); 3109 if (dma_mapping_error(priv->device, des)) 3110 goto dma_map_err; /* should reuse desc w/o issues */ 3111 3112 tx_q->tx_skbuff_dma[entry].buf = des; 3113 3114 stmmac_set_desc_addr(priv, desc, des); 3115 3116 tx_q->tx_skbuff_dma[entry].map_as_page = true; 3117 tx_q->tx_skbuff_dma[entry].len = len; 3118 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 3119 3120 /* Prepare the descriptor and set the own bit too */ 3121 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 3122 priv->mode, 1, last_segment, skb->len); 3123 } 3124 3125 /* Only the last descriptor gets to point to the skb. */ 3126 tx_q->tx_skbuff[entry] = skb; 3127 3128 /* We've used all descriptors we need for this skb, however, 3129 * advance cur_tx so that it references a fresh descriptor. 3130 * ndo_start_xmit will fill this descriptor the next time it's 3131 * called and stmmac_tx_clean may clean up to this descriptor. 3132 */ 3133 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE); 3134 tx_q->cur_tx = entry; 3135 3136 if (netif_msg_pktdata(priv)) { 3137 void *tx_head; 3138 3139 netdev_dbg(priv->dev, 3140 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 3141 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 3142 entry, first, nfrags); 3143 3144 if (priv->extend_desc) 3145 tx_head = (void *)tx_q->dma_etx; 3146 else 3147 tx_head = (void *)tx_q->dma_tx; 3148 3149 stmmac_display_ring(priv, tx_head, DMA_TX_SIZE, false); 3150 3151 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 3152 print_pkt(skb->data, skb->len); 3153 } 3154 3155 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 3156 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 3157 __func__); 3158 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 3159 } 3160 3161 dev->stats.tx_bytes += skb->len; 3162 3163 /* According to the coalesce parameter the IC bit for the latest 3164 * segment is reset and the timer re-started to clean the tx status. 3165 * This approach takes care about the fragments: desc is the first 3166 * element in case of no SG. 3167 */ 3168 tx_q->tx_count_frames += nfrags + 1; 3169 if (priv->tx_coal_frames <= tx_q->tx_count_frames) { 3170 stmmac_set_tx_ic(priv, desc); 3171 priv->xstats.tx_set_ic_bit++; 3172 tx_q->tx_count_frames = 0; 3173 } else { 3174 stmmac_tx_timer_arm(priv, queue); 3175 } 3176 3177 skb_tx_timestamp(skb); 3178 3179 /* Ready to fill the first descriptor and set the OWN bit w/o any 3180 * problems because all the descriptors are actually ready to be 3181 * passed to the DMA engine. 3182 */ 3183 if (likely(!is_jumbo)) { 3184 bool last_segment = (nfrags == 0); 3185 3186 des = dma_map_single(priv->device, skb->data, 3187 nopaged_len, DMA_TO_DEVICE); 3188 if (dma_mapping_error(priv->device, des)) 3189 goto dma_map_err; 3190 3191 tx_q->tx_skbuff_dma[first_entry].buf = des; 3192 3193 stmmac_set_desc_addr(priv, first, des); 3194 3195 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 3196 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 3197 3198 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 3199 priv->hwts_tx_en)) { 3200 /* declare that device is doing timestamping */ 3201 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 3202 stmmac_enable_tx_timestamp(priv, first); 3203 } 3204 3205 /* Prepare the first descriptor setting the OWN bit too */ 3206 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 3207 csum_insertion, priv->mode, 1, last_segment, 3208 skb->len); 3209 3210 /* The own bit must be the latest setting done when prepare the 3211 * descriptor and then barrier is needed to make sure that 3212 * all is coherent before granting the DMA engine. 3213 */ 3214 wmb(); 3215 } 3216 3217 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 3218 3219 stmmac_enable_dma_transmission(priv, priv->ioaddr); 3220 3221 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc)); 3222 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 3223 3224 return NETDEV_TX_OK; 3225 3226 dma_map_err: 3227 netdev_err(priv->dev, "Tx DMA map failed\n"); 3228 dev_kfree_skb(skb); 3229 priv->dev->stats.tx_dropped++; 3230 return NETDEV_TX_OK; 3231 } 3232 3233 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 3234 { 3235 struct vlan_ethhdr *veth; 3236 __be16 vlan_proto; 3237 u16 vlanid; 3238 3239 veth = (struct vlan_ethhdr *)skb->data; 3240 vlan_proto = veth->h_vlan_proto; 3241 3242 if ((vlan_proto == htons(ETH_P_8021Q) && 3243 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 3244 (vlan_proto == htons(ETH_P_8021AD) && 3245 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 3246 /* pop the vlan tag */ 3247 vlanid = ntohs(veth->h_vlan_TCI); 3248 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 3249 skb_pull(skb, VLAN_HLEN); 3250 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 3251 } 3252 } 3253 3254 3255 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q) 3256 { 3257 if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH) 3258 return 0; 3259 3260 return 1; 3261 } 3262 3263 /** 3264 * stmmac_rx_refill - refill used skb preallocated buffers 3265 * @priv: driver private structure 3266 * @queue: RX queue index 3267 * Description : this is to reallocate the skb for the reception process 3268 * that is based on zero-copy. 3269 */ 3270 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 3271 { 3272 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3273 int dirty = stmmac_rx_dirty(priv, queue); 3274 unsigned int entry = rx_q->dirty_rx; 3275 3276 int bfsize = priv->dma_buf_sz; 3277 3278 while (dirty-- > 0) { 3279 struct dma_desc *p; 3280 3281 if (priv->extend_desc) 3282 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3283 else 3284 p = rx_q->dma_rx + entry; 3285 3286 if (likely(!rx_q->rx_skbuff[entry])) { 3287 struct sk_buff *skb; 3288 3289 skb = netdev_alloc_skb_ip_align(priv->dev, bfsize); 3290 if (unlikely(!skb)) { 3291 /* so for a while no zero-copy! */ 3292 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH; 3293 if (unlikely(net_ratelimit())) 3294 dev_err(priv->device, 3295 "fail to alloc skb entry %d\n", 3296 entry); 3297 break; 3298 } 3299 3300 rx_q->rx_skbuff[entry] = skb; 3301 rx_q->rx_skbuff_dma[entry] = 3302 dma_map_single(priv->device, skb->data, bfsize, 3303 DMA_FROM_DEVICE); 3304 if (dma_mapping_error(priv->device, 3305 rx_q->rx_skbuff_dma[entry])) { 3306 netdev_err(priv->dev, "Rx DMA map failed\n"); 3307 dev_kfree_skb(skb); 3308 break; 3309 } 3310 3311 stmmac_set_desc_addr(priv, p, rx_q->rx_skbuff_dma[entry]); 3312 stmmac_refill_desc3(priv, rx_q, p); 3313 3314 if (rx_q->rx_zeroc_thresh > 0) 3315 rx_q->rx_zeroc_thresh--; 3316 3317 netif_dbg(priv, rx_status, priv->dev, 3318 "refill entry #%d\n", entry); 3319 } 3320 dma_wmb(); 3321 3322 stmmac_set_rx_owner(priv, p, priv->use_riwt); 3323 3324 dma_wmb(); 3325 3326 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE); 3327 } 3328 rx_q->dirty_rx = entry; 3329 } 3330 3331 /** 3332 * stmmac_rx - manage the receive process 3333 * @priv: driver private structure 3334 * @limit: napi bugget 3335 * @queue: RX queue index. 3336 * Description : this the function called by the napi poll method. 3337 * It gets all the frames inside the ring. 3338 */ 3339 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 3340 { 3341 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3342 struct stmmac_channel *ch = &priv->channel[queue]; 3343 unsigned int entry = rx_q->cur_rx; 3344 int coe = priv->hw->rx_csum; 3345 unsigned int next_entry; 3346 unsigned int count = 0; 3347 bool xmac; 3348 3349 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 3350 3351 if (netif_msg_rx_status(priv)) { 3352 void *rx_head; 3353 3354 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 3355 if (priv->extend_desc) 3356 rx_head = (void *)rx_q->dma_erx; 3357 else 3358 rx_head = (void *)rx_q->dma_rx; 3359 3360 stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true); 3361 } 3362 while (count < limit) { 3363 int status; 3364 struct dma_desc *p; 3365 struct dma_desc *np; 3366 3367 if (priv->extend_desc) 3368 p = (struct dma_desc *)(rx_q->dma_erx + entry); 3369 else 3370 p = rx_q->dma_rx + entry; 3371 3372 /* read the status of the incoming frame */ 3373 status = stmmac_rx_status(priv, &priv->dev->stats, 3374 &priv->xstats, p); 3375 /* check if managed by the DMA otherwise go ahead */ 3376 if (unlikely(status & dma_own)) 3377 break; 3378 3379 count++; 3380 3381 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE); 3382 next_entry = rx_q->cur_rx; 3383 3384 if (priv->extend_desc) 3385 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 3386 else 3387 np = rx_q->dma_rx + next_entry; 3388 3389 prefetch(np); 3390 3391 if (priv->extend_desc) 3392 stmmac_rx_extended_status(priv, &priv->dev->stats, 3393 &priv->xstats, rx_q->dma_erx + entry); 3394 if (unlikely(status == discard_frame)) { 3395 priv->dev->stats.rx_errors++; 3396 if (priv->hwts_rx_en && !priv->extend_desc) { 3397 /* DESC2 & DESC3 will be overwritten by device 3398 * with timestamp value, hence reinitialize 3399 * them in stmmac_rx_refill() function so that 3400 * device can reuse it. 3401 */ 3402 dev_kfree_skb_any(rx_q->rx_skbuff[entry]); 3403 rx_q->rx_skbuff[entry] = NULL; 3404 dma_unmap_single(priv->device, 3405 rx_q->rx_skbuff_dma[entry], 3406 priv->dma_buf_sz, 3407 DMA_FROM_DEVICE); 3408 } 3409 } else { 3410 struct sk_buff *skb; 3411 int frame_len; 3412 unsigned int des; 3413 3414 stmmac_get_desc_addr(priv, p, &des); 3415 frame_len = stmmac_get_rx_frame_len(priv, p, coe); 3416 3417 /* If frame length is greater than skb buffer size 3418 * (preallocated during init) then the packet is 3419 * ignored 3420 */ 3421 if (frame_len > priv->dma_buf_sz) { 3422 netdev_err(priv->dev, 3423 "len %d larger than size (%d)\n", 3424 frame_len, priv->dma_buf_sz); 3425 priv->dev->stats.rx_length_errors++; 3426 break; 3427 } 3428 3429 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 3430 * Type frames (LLC/LLC-SNAP) 3431 * 3432 * llc_snap is never checked in GMAC >= 4, so this ACS 3433 * feature is always disabled and packets need to be 3434 * stripped manually. 3435 */ 3436 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00) || 3437 unlikely(status != llc_snap)) 3438 frame_len -= ETH_FCS_LEN; 3439 3440 if (netif_msg_rx_status(priv)) { 3441 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n", 3442 p, entry, des); 3443 netdev_dbg(priv->dev, "frame size %d, COE: %d\n", 3444 frame_len, status); 3445 } 3446 3447 /* The zero-copy is always used for all the sizes 3448 * in case of GMAC4 because it needs 3449 * to refill the used descriptors, always. 3450 */ 3451 if (unlikely(!xmac && 3452 ((frame_len < priv->rx_copybreak) || 3453 stmmac_rx_threshold_count(rx_q)))) { 3454 skb = netdev_alloc_skb_ip_align(priv->dev, 3455 frame_len); 3456 if (unlikely(!skb)) { 3457 if (net_ratelimit()) 3458 dev_warn(priv->device, 3459 "packet dropped\n"); 3460 priv->dev->stats.rx_dropped++; 3461 break; 3462 } 3463 3464 dma_sync_single_for_cpu(priv->device, 3465 rx_q->rx_skbuff_dma 3466 [entry], frame_len, 3467 DMA_FROM_DEVICE); 3468 skb_copy_to_linear_data(skb, 3469 rx_q-> 3470 rx_skbuff[entry]->data, 3471 frame_len); 3472 3473 skb_put(skb, frame_len); 3474 dma_sync_single_for_device(priv->device, 3475 rx_q->rx_skbuff_dma 3476 [entry], frame_len, 3477 DMA_FROM_DEVICE); 3478 } else { 3479 skb = rx_q->rx_skbuff[entry]; 3480 if (unlikely(!skb)) { 3481 netdev_err(priv->dev, 3482 "%s: Inconsistent Rx chain\n", 3483 priv->dev->name); 3484 priv->dev->stats.rx_dropped++; 3485 break; 3486 } 3487 prefetch(skb->data - NET_IP_ALIGN); 3488 rx_q->rx_skbuff[entry] = NULL; 3489 rx_q->rx_zeroc_thresh++; 3490 3491 skb_put(skb, frame_len); 3492 dma_unmap_single(priv->device, 3493 rx_q->rx_skbuff_dma[entry], 3494 priv->dma_buf_sz, 3495 DMA_FROM_DEVICE); 3496 } 3497 3498 if (netif_msg_pktdata(priv)) { 3499 netdev_dbg(priv->dev, "frame received (%dbytes)", 3500 frame_len); 3501 print_pkt(skb->data, frame_len); 3502 } 3503 3504 stmmac_get_rx_hwtstamp(priv, p, np, skb); 3505 3506 stmmac_rx_vlan(priv->dev, skb); 3507 3508 skb->protocol = eth_type_trans(skb, priv->dev); 3509 3510 if (unlikely(!coe)) 3511 skb_checksum_none_assert(skb); 3512 else 3513 skb->ip_summed = CHECKSUM_UNNECESSARY; 3514 3515 napi_gro_receive(&ch->napi, skb); 3516 3517 priv->dev->stats.rx_packets++; 3518 priv->dev->stats.rx_bytes += frame_len; 3519 } 3520 entry = next_entry; 3521 } 3522 3523 stmmac_rx_refill(priv, queue); 3524 3525 priv->xstats.rx_pkt_n += count; 3526 3527 return count; 3528 } 3529 3530 /** 3531 * stmmac_poll - stmmac poll method (NAPI) 3532 * @napi : pointer to the napi structure. 3533 * @budget : maximum number of packets that the current CPU can receive from 3534 * all interfaces. 3535 * Description : 3536 * To look at the incoming frames and clear the tx resources. 3537 */ 3538 static int stmmac_napi_poll(struct napi_struct *napi, int budget) 3539 { 3540 struct stmmac_channel *ch = 3541 container_of(napi, struct stmmac_channel, napi); 3542 struct stmmac_priv *priv = ch->priv_data; 3543 int work_done, rx_done = 0, tx_done = 0; 3544 u32 chan = ch->index; 3545 3546 priv->xstats.napi_poll++; 3547 3548 if (ch->has_tx) 3549 tx_done = stmmac_tx_clean(priv, budget, chan); 3550 if (ch->has_rx) 3551 rx_done = stmmac_rx(priv, budget, chan); 3552 3553 work_done = max(rx_done, tx_done); 3554 work_done = min(work_done, budget); 3555 3556 if (work_done < budget && napi_complete_done(napi, work_done)) { 3557 int stat; 3558 3559 stmmac_enable_dma_irq(priv, priv->ioaddr, chan); 3560 stat = stmmac_dma_interrupt_status(priv, priv->ioaddr, 3561 &priv->xstats, chan); 3562 if (stat && napi_reschedule(napi)) 3563 stmmac_disable_dma_irq(priv, priv->ioaddr, chan); 3564 } 3565 3566 return work_done; 3567 } 3568 3569 /** 3570 * stmmac_tx_timeout 3571 * @dev : Pointer to net device structure 3572 * Description: this function is called when a packet transmission fails to 3573 * complete within a reasonable time. The driver will mark the error in the 3574 * netdev structure and arrange for the device to be reset to a sane state 3575 * in order to transmit a new packet. 3576 */ 3577 static void stmmac_tx_timeout(struct net_device *dev) 3578 { 3579 struct stmmac_priv *priv = netdev_priv(dev); 3580 3581 stmmac_global_err(priv); 3582 } 3583 3584 /** 3585 * stmmac_set_rx_mode - entry point for multicast addressing 3586 * @dev : pointer to the device structure 3587 * Description: 3588 * This function is a driver entry point which gets called by the kernel 3589 * whenever multicast addresses must be enabled/disabled. 3590 * Return value: 3591 * void. 3592 */ 3593 static void stmmac_set_rx_mode(struct net_device *dev) 3594 { 3595 struct stmmac_priv *priv = netdev_priv(dev); 3596 3597 stmmac_set_filter(priv, priv->hw, dev); 3598 } 3599 3600 /** 3601 * stmmac_change_mtu - entry point to change MTU size for the device. 3602 * @dev : device pointer. 3603 * @new_mtu : the new MTU size for the device. 3604 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 3605 * to drive packet transmission. Ethernet has an MTU of 1500 octets 3606 * (ETH_DATA_LEN). This value can be changed with ifconfig. 3607 * Return value: 3608 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3609 * file on failure. 3610 */ 3611 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 3612 { 3613 struct stmmac_priv *priv = netdev_priv(dev); 3614 3615 if (netif_running(dev)) { 3616 netdev_err(priv->dev, "must be stopped to change its MTU\n"); 3617 return -EBUSY; 3618 } 3619 3620 dev->mtu = new_mtu; 3621 3622 netdev_update_features(dev); 3623 3624 return 0; 3625 } 3626 3627 static netdev_features_t stmmac_fix_features(struct net_device *dev, 3628 netdev_features_t features) 3629 { 3630 struct stmmac_priv *priv = netdev_priv(dev); 3631 3632 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 3633 features &= ~NETIF_F_RXCSUM; 3634 3635 if (!priv->plat->tx_coe) 3636 features &= ~NETIF_F_CSUM_MASK; 3637 3638 /* Some GMAC devices have a bugged Jumbo frame support that 3639 * needs to have the Tx COE disabled for oversized frames 3640 * (due to limited buffer sizes). In this case we disable 3641 * the TX csum insertion in the TDES and not use SF. 3642 */ 3643 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 3644 features &= ~NETIF_F_CSUM_MASK; 3645 3646 /* Disable tso if asked by ethtool */ 3647 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 3648 if (features & NETIF_F_TSO) 3649 priv->tso = true; 3650 else 3651 priv->tso = false; 3652 } 3653 3654 return features; 3655 } 3656 3657 static int stmmac_set_features(struct net_device *netdev, 3658 netdev_features_t features) 3659 { 3660 struct stmmac_priv *priv = netdev_priv(netdev); 3661 3662 /* Keep the COE Type in case of csum is supporting */ 3663 if (features & NETIF_F_RXCSUM) 3664 priv->hw->rx_csum = priv->plat->rx_coe; 3665 else 3666 priv->hw->rx_csum = 0; 3667 /* No check needed because rx_coe has been set before and it will be 3668 * fixed in case of issue. 3669 */ 3670 stmmac_rx_ipc(priv, priv->hw); 3671 3672 return 0; 3673 } 3674 3675 /** 3676 * stmmac_interrupt - main ISR 3677 * @irq: interrupt number. 3678 * @dev_id: to pass the net device pointer. 3679 * Description: this is the main driver interrupt service routine. 3680 * It can call: 3681 * o DMA service routine (to manage incoming frame reception and transmission 3682 * status) 3683 * o Core interrupts to manage: remote wake-up, management counter, LPI 3684 * interrupts. 3685 */ 3686 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 3687 { 3688 struct net_device *dev = (struct net_device *)dev_id; 3689 struct stmmac_priv *priv = netdev_priv(dev); 3690 u32 rx_cnt = priv->plat->rx_queues_to_use; 3691 u32 tx_cnt = priv->plat->tx_queues_to_use; 3692 u32 queues_count; 3693 u32 queue; 3694 bool xmac; 3695 3696 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 3697 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 3698 3699 if (priv->irq_wake) 3700 pm_wakeup_event(priv->device, 0); 3701 3702 if (unlikely(!dev)) { 3703 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 3704 return IRQ_NONE; 3705 } 3706 3707 /* Check if adapter is up */ 3708 if (test_bit(STMMAC_DOWN, &priv->state)) 3709 return IRQ_HANDLED; 3710 /* Check if a fatal error happened */ 3711 if (stmmac_safety_feat_interrupt(priv)) 3712 return IRQ_HANDLED; 3713 3714 /* To handle GMAC own interrupts */ 3715 if ((priv->plat->has_gmac) || xmac) { 3716 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 3717 int mtl_status; 3718 3719 if (unlikely(status)) { 3720 /* For LPI we need to save the tx status */ 3721 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 3722 priv->tx_path_in_lpi_mode = true; 3723 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 3724 priv->tx_path_in_lpi_mode = false; 3725 } 3726 3727 for (queue = 0; queue < queues_count; queue++) { 3728 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3729 3730 mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw, 3731 queue); 3732 if (mtl_status != -EINVAL) 3733 status |= mtl_status; 3734 3735 if (status & CORE_IRQ_MTL_RX_OVERFLOW) 3736 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 3737 rx_q->rx_tail_addr, 3738 queue); 3739 } 3740 3741 /* PCS link status */ 3742 if (priv->hw->pcs) { 3743 if (priv->xstats.pcs_link) 3744 netif_carrier_on(dev); 3745 else 3746 netif_carrier_off(dev); 3747 } 3748 } 3749 3750 /* To handle DMA interrupts */ 3751 stmmac_dma_interrupt(priv); 3752 3753 return IRQ_HANDLED; 3754 } 3755 3756 #ifdef CONFIG_NET_POLL_CONTROLLER 3757 /* Polling receive - used by NETCONSOLE and other diagnostic tools 3758 * to allow network I/O with interrupts disabled. 3759 */ 3760 static void stmmac_poll_controller(struct net_device *dev) 3761 { 3762 disable_irq(dev->irq); 3763 stmmac_interrupt(dev->irq, dev); 3764 enable_irq(dev->irq); 3765 } 3766 #endif 3767 3768 /** 3769 * stmmac_ioctl - Entry point for the Ioctl 3770 * @dev: Device pointer. 3771 * @rq: An IOCTL specefic structure, that can contain a pointer to 3772 * a proprietary structure used to pass information to the driver. 3773 * @cmd: IOCTL command 3774 * Description: 3775 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 3776 */ 3777 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 3778 { 3779 int ret = -EOPNOTSUPP; 3780 3781 if (!netif_running(dev)) 3782 return -EINVAL; 3783 3784 switch (cmd) { 3785 case SIOCGMIIPHY: 3786 case SIOCGMIIREG: 3787 case SIOCSMIIREG: 3788 if (!dev->phydev) 3789 return -EINVAL; 3790 ret = phy_mii_ioctl(dev->phydev, rq, cmd); 3791 break; 3792 case SIOCSHWTSTAMP: 3793 ret = stmmac_hwtstamp_set(dev, rq); 3794 break; 3795 case SIOCGHWTSTAMP: 3796 ret = stmmac_hwtstamp_get(dev, rq); 3797 break; 3798 default: 3799 break; 3800 } 3801 3802 return ret; 3803 } 3804 3805 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 3806 void *cb_priv) 3807 { 3808 struct stmmac_priv *priv = cb_priv; 3809 int ret = -EOPNOTSUPP; 3810 3811 stmmac_disable_all_queues(priv); 3812 3813 switch (type) { 3814 case TC_SETUP_CLSU32: 3815 if (tc_cls_can_offload_and_chain0(priv->dev, type_data)) 3816 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 3817 break; 3818 default: 3819 break; 3820 } 3821 3822 stmmac_enable_all_queues(priv); 3823 return ret; 3824 } 3825 3826 static int stmmac_setup_tc_block(struct stmmac_priv *priv, 3827 struct tc_block_offload *f) 3828 { 3829 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 3830 return -EOPNOTSUPP; 3831 3832 switch (f->command) { 3833 case TC_BLOCK_BIND: 3834 return tcf_block_cb_register(f->block, stmmac_setup_tc_block_cb, 3835 priv, priv, f->extack); 3836 case TC_BLOCK_UNBIND: 3837 tcf_block_cb_unregister(f->block, stmmac_setup_tc_block_cb, priv); 3838 return 0; 3839 default: 3840 return -EOPNOTSUPP; 3841 } 3842 } 3843 3844 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 3845 void *type_data) 3846 { 3847 struct stmmac_priv *priv = netdev_priv(ndev); 3848 3849 switch (type) { 3850 case TC_SETUP_BLOCK: 3851 return stmmac_setup_tc_block(priv, type_data); 3852 case TC_SETUP_QDISC_CBS: 3853 return stmmac_tc_setup_cbs(priv, priv, type_data); 3854 default: 3855 return -EOPNOTSUPP; 3856 } 3857 } 3858 3859 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 3860 { 3861 struct stmmac_priv *priv = netdev_priv(ndev); 3862 int ret = 0; 3863 3864 ret = eth_mac_addr(ndev, addr); 3865 if (ret) 3866 return ret; 3867 3868 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 3869 3870 return ret; 3871 } 3872 3873 #ifdef CONFIG_DEBUG_FS 3874 static struct dentry *stmmac_fs_dir; 3875 3876 static void sysfs_display_ring(void *head, int size, int extend_desc, 3877 struct seq_file *seq) 3878 { 3879 int i; 3880 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 3881 struct dma_desc *p = (struct dma_desc *)head; 3882 3883 for (i = 0; i < size; i++) { 3884 if (extend_desc) { 3885 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3886 i, (unsigned int)virt_to_phys(ep), 3887 le32_to_cpu(ep->basic.des0), 3888 le32_to_cpu(ep->basic.des1), 3889 le32_to_cpu(ep->basic.des2), 3890 le32_to_cpu(ep->basic.des3)); 3891 ep++; 3892 } else { 3893 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", 3894 i, (unsigned int)virt_to_phys(p), 3895 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 3896 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 3897 p++; 3898 } 3899 seq_printf(seq, "\n"); 3900 } 3901 } 3902 3903 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 3904 { 3905 struct net_device *dev = seq->private; 3906 struct stmmac_priv *priv = netdev_priv(dev); 3907 u32 rx_count = priv->plat->rx_queues_to_use; 3908 u32 tx_count = priv->plat->tx_queues_to_use; 3909 u32 queue; 3910 3911 if ((dev->flags & IFF_UP) == 0) 3912 return 0; 3913 3914 for (queue = 0; queue < rx_count; queue++) { 3915 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 3916 3917 seq_printf(seq, "RX Queue %d:\n", queue); 3918 3919 if (priv->extend_desc) { 3920 seq_printf(seq, "Extended descriptor ring:\n"); 3921 sysfs_display_ring((void *)rx_q->dma_erx, 3922 DMA_RX_SIZE, 1, seq); 3923 } else { 3924 seq_printf(seq, "Descriptor ring:\n"); 3925 sysfs_display_ring((void *)rx_q->dma_rx, 3926 DMA_RX_SIZE, 0, seq); 3927 } 3928 } 3929 3930 for (queue = 0; queue < tx_count; queue++) { 3931 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 3932 3933 seq_printf(seq, "TX Queue %d:\n", queue); 3934 3935 if (priv->extend_desc) { 3936 seq_printf(seq, "Extended descriptor ring:\n"); 3937 sysfs_display_ring((void *)tx_q->dma_etx, 3938 DMA_TX_SIZE, 1, seq); 3939 } else { 3940 seq_printf(seq, "Descriptor ring:\n"); 3941 sysfs_display_ring((void *)tx_q->dma_tx, 3942 DMA_TX_SIZE, 0, seq); 3943 } 3944 } 3945 3946 return 0; 3947 } 3948 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 3949 3950 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 3951 { 3952 struct net_device *dev = seq->private; 3953 struct stmmac_priv *priv = netdev_priv(dev); 3954 3955 if (!priv->hw_cap_support) { 3956 seq_printf(seq, "DMA HW features not supported\n"); 3957 return 0; 3958 } 3959 3960 seq_printf(seq, "==============================\n"); 3961 seq_printf(seq, "\tDMA HW features\n"); 3962 seq_printf(seq, "==============================\n"); 3963 3964 seq_printf(seq, "\t10/100 Mbps: %s\n", 3965 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 3966 seq_printf(seq, "\t1000 Mbps: %s\n", 3967 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 3968 seq_printf(seq, "\tHalf duplex: %s\n", 3969 (priv->dma_cap.half_duplex) ? "Y" : "N"); 3970 seq_printf(seq, "\tHash Filter: %s\n", 3971 (priv->dma_cap.hash_filter) ? "Y" : "N"); 3972 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 3973 (priv->dma_cap.multi_addr) ? "Y" : "N"); 3974 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 3975 (priv->dma_cap.pcs) ? "Y" : "N"); 3976 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 3977 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 3978 seq_printf(seq, "\tPMT Remote wake up: %s\n", 3979 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 3980 seq_printf(seq, "\tPMT Magic Frame: %s\n", 3981 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 3982 seq_printf(seq, "\tRMON module: %s\n", 3983 (priv->dma_cap.rmon) ? "Y" : "N"); 3984 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 3985 (priv->dma_cap.time_stamp) ? "Y" : "N"); 3986 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 3987 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 3988 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 3989 (priv->dma_cap.eee) ? "Y" : "N"); 3990 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 3991 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 3992 (priv->dma_cap.tx_coe) ? "Y" : "N"); 3993 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 3994 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 3995 (priv->dma_cap.rx_coe) ? "Y" : "N"); 3996 } else { 3997 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 3998 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 3999 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 4000 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 4001 } 4002 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 4003 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 4004 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 4005 priv->dma_cap.number_rx_channel); 4006 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 4007 priv->dma_cap.number_tx_channel); 4008 seq_printf(seq, "\tEnhanced descriptors: %s\n", 4009 (priv->dma_cap.enh_desc) ? "Y" : "N"); 4010 4011 return 0; 4012 } 4013 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 4014 4015 static int stmmac_init_fs(struct net_device *dev) 4016 { 4017 struct stmmac_priv *priv = netdev_priv(dev); 4018 4019 /* Create per netdev entries */ 4020 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 4021 4022 if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) { 4023 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n"); 4024 4025 return -ENOMEM; 4026 } 4027 4028 /* Entry to report DMA RX/TX rings */ 4029 priv->dbgfs_rings_status = 4030 debugfs_create_file("descriptors_status", 0444, 4031 priv->dbgfs_dir, dev, 4032 &stmmac_rings_status_fops); 4033 4034 if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) { 4035 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n"); 4036 debugfs_remove_recursive(priv->dbgfs_dir); 4037 4038 return -ENOMEM; 4039 } 4040 4041 /* Entry to report the DMA HW features */ 4042 priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", 0444, 4043 priv->dbgfs_dir, 4044 dev, &stmmac_dma_cap_fops); 4045 4046 if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) { 4047 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n"); 4048 debugfs_remove_recursive(priv->dbgfs_dir); 4049 4050 return -ENOMEM; 4051 } 4052 4053 return 0; 4054 } 4055 4056 static void stmmac_exit_fs(struct net_device *dev) 4057 { 4058 struct stmmac_priv *priv = netdev_priv(dev); 4059 4060 debugfs_remove_recursive(priv->dbgfs_dir); 4061 } 4062 #endif /* CONFIG_DEBUG_FS */ 4063 4064 static const struct net_device_ops stmmac_netdev_ops = { 4065 .ndo_open = stmmac_open, 4066 .ndo_start_xmit = stmmac_xmit, 4067 .ndo_stop = stmmac_release, 4068 .ndo_change_mtu = stmmac_change_mtu, 4069 .ndo_fix_features = stmmac_fix_features, 4070 .ndo_set_features = stmmac_set_features, 4071 .ndo_set_rx_mode = stmmac_set_rx_mode, 4072 .ndo_tx_timeout = stmmac_tx_timeout, 4073 .ndo_do_ioctl = stmmac_ioctl, 4074 .ndo_setup_tc = stmmac_setup_tc, 4075 #ifdef CONFIG_NET_POLL_CONTROLLER 4076 .ndo_poll_controller = stmmac_poll_controller, 4077 #endif 4078 .ndo_set_mac_address = stmmac_set_mac_address, 4079 }; 4080 4081 static void stmmac_reset_subtask(struct stmmac_priv *priv) 4082 { 4083 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 4084 return; 4085 if (test_bit(STMMAC_DOWN, &priv->state)) 4086 return; 4087 4088 netdev_err(priv->dev, "Reset adapter.\n"); 4089 4090 rtnl_lock(); 4091 netif_trans_update(priv->dev); 4092 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 4093 usleep_range(1000, 2000); 4094 4095 set_bit(STMMAC_DOWN, &priv->state); 4096 dev_close(priv->dev); 4097 dev_open(priv->dev, NULL); 4098 clear_bit(STMMAC_DOWN, &priv->state); 4099 clear_bit(STMMAC_RESETING, &priv->state); 4100 rtnl_unlock(); 4101 } 4102 4103 static void stmmac_service_task(struct work_struct *work) 4104 { 4105 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 4106 service_task); 4107 4108 stmmac_reset_subtask(priv); 4109 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 4110 } 4111 4112 /** 4113 * stmmac_hw_init - Init the MAC device 4114 * @priv: driver private structure 4115 * Description: this function is to configure the MAC device according to 4116 * some platform parameters or the HW capability register. It prepares the 4117 * driver to use either ring or chain modes and to setup either enhanced or 4118 * normal descriptors. 4119 */ 4120 static int stmmac_hw_init(struct stmmac_priv *priv) 4121 { 4122 int ret; 4123 4124 /* dwmac-sun8i only work in chain mode */ 4125 if (priv->plat->has_sun8i) 4126 chain_mode = 1; 4127 priv->chain_mode = chain_mode; 4128 4129 /* Initialize HW Interface */ 4130 ret = stmmac_hwif_init(priv); 4131 if (ret) 4132 return ret; 4133 4134 /* Get the HW capability (new GMAC newer than 3.50a) */ 4135 priv->hw_cap_support = stmmac_get_hw_features(priv); 4136 if (priv->hw_cap_support) { 4137 dev_info(priv->device, "DMA HW capability register supported\n"); 4138 4139 /* We can override some gmac/dma configuration fields: e.g. 4140 * enh_desc, tx_coe (e.g. that are passed through the 4141 * platform) with the values from the HW capability 4142 * register (if supported). 4143 */ 4144 priv->plat->enh_desc = priv->dma_cap.enh_desc; 4145 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up; 4146 priv->hw->pmt = priv->plat->pmt; 4147 4148 /* TXCOE doesn't work in thresh DMA mode */ 4149 if (priv->plat->force_thresh_dma_mode) 4150 priv->plat->tx_coe = 0; 4151 else 4152 priv->plat->tx_coe = priv->dma_cap.tx_coe; 4153 4154 /* In case of GMAC4 rx_coe is from HW cap register. */ 4155 priv->plat->rx_coe = priv->dma_cap.rx_coe; 4156 4157 if (priv->dma_cap.rx_coe_type2) 4158 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 4159 else if (priv->dma_cap.rx_coe_type1) 4160 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 4161 4162 } else { 4163 dev_info(priv->device, "No HW DMA feature register supported\n"); 4164 } 4165 4166 if (priv->plat->rx_coe) { 4167 priv->hw->rx_csum = priv->plat->rx_coe; 4168 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 4169 if (priv->synopsys_id < DWMAC_CORE_4_00) 4170 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 4171 } 4172 if (priv->plat->tx_coe) 4173 dev_info(priv->device, "TX Checksum insertion supported\n"); 4174 4175 if (priv->plat->pmt) { 4176 dev_info(priv->device, "Wake-Up On Lan supported\n"); 4177 device_set_wakeup_capable(priv->device, 1); 4178 } 4179 4180 if (priv->dma_cap.tsoen) 4181 dev_info(priv->device, "TSO supported\n"); 4182 4183 /* Run HW quirks, if any */ 4184 if (priv->hwif_quirks) { 4185 ret = priv->hwif_quirks(priv); 4186 if (ret) 4187 return ret; 4188 } 4189 4190 /* Rx Watchdog is available in the COREs newer than the 3.40. 4191 * In some case, for example on bugged HW this feature 4192 * has to be disable and this can be done by passing the 4193 * riwt_off field from the platform. 4194 */ 4195 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 4196 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 4197 priv->use_riwt = 1; 4198 dev_info(priv->device, 4199 "Enable RX Mitigation via HW Watchdog Timer\n"); 4200 } 4201 4202 return 0; 4203 } 4204 4205 /** 4206 * stmmac_dvr_probe 4207 * @device: device pointer 4208 * @plat_dat: platform data pointer 4209 * @res: stmmac resource pointer 4210 * Description: this is the main probe function used to 4211 * call the alloc_etherdev, allocate the priv structure. 4212 * Return: 4213 * returns 0 on success, otherwise errno. 4214 */ 4215 int stmmac_dvr_probe(struct device *device, 4216 struct plat_stmmacenet_data *plat_dat, 4217 struct stmmac_resources *res) 4218 { 4219 struct net_device *ndev = NULL; 4220 struct stmmac_priv *priv; 4221 u32 queue, maxq; 4222 int ret = 0; 4223 4224 ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv), 4225 MTL_MAX_TX_QUEUES, 4226 MTL_MAX_RX_QUEUES); 4227 if (!ndev) 4228 return -ENOMEM; 4229 4230 SET_NETDEV_DEV(ndev, device); 4231 4232 priv = netdev_priv(ndev); 4233 priv->device = device; 4234 priv->dev = ndev; 4235 4236 stmmac_set_ethtool_ops(ndev); 4237 priv->pause = pause; 4238 priv->plat = plat_dat; 4239 priv->ioaddr = res->addr; 4240 priv->dev->base_addr = (unsigned long)res->addr; 4241 4242 priv->dev->irq = res->irq; 4243 priv->wol_irq = res->wol_irq; 4244 priv->lpi_irq = res->lpi_irq; 4245 4246 if (res->mac) 4247 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); 4248 4249 dev_set_drvdata(device, priv->dev); 4250 4251 /* Verify driver arguments */ 4252 stmmac_verify_args(); 4253 4254 /* Allocate workqueue */ 4255 priv->wq = create_singlethread_workqueue("stmmac_wq"); 4256 if (!priv->wq) { 4257 dev_err(priv->device, "failed to create workqueue\n"); 4258 ret = -ENOMEM; 4259 goto error_wq; 4260 } 4261 4262 INIT_WORK(&priv->service_task, stmmac_service_task); 4263 4264 /* Override with kernel parameters if supplied XXX CRS XXX 4265 * this needs to have multiple instances 4266 */ 4267 if ((phyaddr >= 0) && (phyaddr <= 31)) 4268 priv->plat->phy_addr = phyaddr; 4269 4270 if (priv->plat->stmmac_rst) { 4271 ret = reset_control_assert(priv->plat->stmmac_rst); 4272 reset_control_deassert(priv->plat->stmmac_rst); 4273 /* Some reset controllers have only reset callback instead of 4274 * assert + deassert callbacks pair. 4275 */ 4276 if (ret == -ENOTSUPP) 4277 reset_control_reset(priv->plat->stmmac_rst); 4278 } 4279 4280 /* Init MAC and get the capabilities */ 4281 ret = stmmac_hw_init(priv); 4282 if (ret) 4283 goto error_hw_init; 4284 4285 /* Configure real RX and TX queues */ 4286 netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use); 4287 netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use); 4288 4289 ndev->netdev_ops = &stmmac_netdev_ops; 4290 4291 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4292 NETIF_F_RXCSUM; 4293 4294 ret = stmmac_tc_init(priv, priv); 4295 if (!ret) { 4296 ndev->hw_features |= NETIF_F_HW_TC; 4297 } 4298 4299 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 4300 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 4301 priv->tso = true; 4302 dev_info(priv->device, "TSO feature enabled\n"); 4303 } 4304 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 4305 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 4306 #ifdef STMMAC_VLAN_TAG_USED 4307 /* Both mac100 and gmac support receive VLAN tag detection */ 4308 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 4309 #endif 4310 priv->msg_enable = netif_msg_init(debug, default_msg_level); 4311 4312 /* MTU range: 46 - hw-specific max */ 4313 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 4314 if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 4315 ndev->max_mtu = JUMBO_LEN; 4316 else if (priv->plat->has_xgmac) 4317 ndev->max_mtu = XGMAC_JUMBO_LEN; 4318 else 4319 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 4320 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 4321 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 4322 */ 4323 if ((priv->plat->maxmtu < ndev->max_mtu) && 4324 (priv->plat->maxmtu >= ndev->min_mtu)) 4325 ndev->max_mtu = priv->plat->maxmtu; 4326 else if (priv->plat->maxmtu < ndev->min_mtu) 4327 dev_warn(priv->device, 4328 "%s: warning: maxmtu having invalid value (%d)\n", 4329 __func__, priv->plat->maxmtu); 4330 4331 if (flow_ctrl) 4332 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 4333 4334 /* Setup channels NAPI */ 4335 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 4336 4337 for (queue = 0; queue < maxq; queue++) { 4338 struct stmmac_channel *ch = &priv->channel[queue]; 4339 4340 ch->priv_data = priv; 4341 ch->index = queue; 4342 4343 if (queue < priv->plat->rx_queues_to_use) 4344 ch->has_rx = true; 4345 if (queue < priv->plat->tx_queues_to_use) 4346 ch->has_tx = true; 4347 4348 netif_napi_add(ndev, &ch->napi, stmmac_napi_poll, 4349 NAPI_POLL_WEIGHT); 4350 } 4351 4352 mutex_init(&priv->lock); 4353 4354 /* If a specific clk_csr value is passed from the platform 4355 * this means that the CSR Clock Range selection cannot be 4356 * changed at run-time and it is fixed. Viceversa the driver'll try to 4357 * set the MDC clock dynamically according to the csr actual 4358 * clock input. 4359 */ 4360 if (!priv->plat->clk_csr) 4361 stmmac_clk_csr_set(priv); 4362 else 4363 priv->clk_csr = priv->plat->clk_csr; 4364 4365 stmmac_check_pcs_mode(priv); 4366 4367 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4368 priv->hw->pcs != STMMAC_PCS_TBI && 4369 priv->hw->pcs != STMMAC_PCS_RTBI) { 4370 /* MDIO bus Registration */ 4371 ret = stmmac_mdio_register(ndev); 4372 if (ret < 0) { 4373 dev_err(priv->device, 4374 "%s: MDIO bus (id: %d) registration failed", 4375 __func__, priv->plat->bus_id); 4376 goto error_mdio_register; 4377 } 4378 } 4379 4380 ret = register_netdev(ndev); 4381 if (ret) { 4382 dev_err(priv->device, "%s: ERROR %i registering the device\n", 4383 __func__, ret); 4384 goto error_netdev_register; 4385 } 4386 4387 #ifdef CONFIG_DEBUG_FS 4388 ret = stmmac_init_fs(ndev); 4389 if (ret < 0) 4390 netdev_warn(priv->dev, "%s: failed debugFS registration\n", 4391 __func__); 4392 #endif 4393 4394 return ret; 4395 4396 error_netdev_register: 4397 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4398 priv->hw->pcs != STMMAC_PCS_TBI && 4399 priv->hw->pcs != STMMAC_PCS_RTBI) 4400 stmmac_mdio_unregister(ndev); 4401 error_mdio_register: 4402 for (queue = 0; queue < maxq; queue++) { 4403 struct stmmac_channel *ch = &priv->channel[queue]; 4404 4405 netif_napi_del(&ch->napi); 4406 } 4407 error_hw_init: 4408 destroy_workqueue(priv->wq); 4409 error_wq: 4410 free_netdev(ndev); 4411 4412 return ret; 4413 } 4414 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 4415 4416 /** 4417 * stmmac_dvr_remove 4418 * @dev: device pointer 4419 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 4420 * changes the link status, releases the DMA descriptor rings. 4421 */ 4422 int stmmac_dvr_remove(struct device *dev) 4423 { 4424 struct net_device *ndev = dev_get_drvdata(dev); 4425 struct stmmac_priv *priv = netdev_priv(ndev); 4426 4427 netdev_info(priv->dev, "%s: removing driver", __func__); 4428 4429 #ifdef CONFIG_DEBUG_FS 4430 stmmac_exit_fs(ndev); 4431 #endif 4432 stmmac_stop_all_dma(priv); 4433 4434 stmmac_mac_set(priv, priv->ioaddr, false); 4435 netif_carrier_off(ndev); 4436 unregister_netdev(ndev); 4437 if (priv->plat->stmmac_rst) 4438 reset_control_assert(priv->plat->stmmac_rst); 4439 clk_disable_unprepare(priv->plat->pclk); 4440 clk_disable_unprepare(priv->plat->stmmac_clk); 4441 if (priv->hw->pcs != STMMAC_PCS_RGMII && 4442 priv->hw->pcs != STMMAC_PCS_TBI && 4443 priv->hw->pcs != STMMAC_PCS_RTBI) 4444 stmmac_mdio_unregister(ndev); 4445 destroy_workqueue(priv->wq); 4446 mutex_destroy(&priv->lock); 4447 free_netdev(ndev); 4448 4449 return 0; 4450 } 4451 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 4452 4453 /** 4454 * stmmac_suspend - suspend callback 4455 * @dev: device pointer 4456 * Description: this is the function to suspend the device and it is called 4457 * by the platform driver to stop the network queue, release the resources, 4458 * program the PMT register (for WoL), clean and release driver resources. 4459 */ 4460 int stmmac_suspend(struct device *dev) 4461 { 4462 struct net_device *ndev = dev_get_drvdata(dev); 4463 struct stmmac_priv *priv = netdev_priv(ndev); 4464 4465 if (!ndev || !netif_running(ndev)) 4466 return 0; 4467 4468 if (ndev->phydev) 4469 phy_stop(ndev->phydev); 4470 4471 mutex_lock(&priv->lock); 4472 4473 netif_device_detach(ndev); 4474 stmmac_stop_all_queues(priv); 4475 4476 stmmac_disable_all_queues(priv); 4477 4478 /* Stop TX/RX DMA */ 4479 stmmac_stop_all_dma(priv); 4480 4481 /* Enable Power down mode by programming the PMT regs */ 4482 if (device_may_wakeup(priv->device)) { 4483 stmmac_pmt(priv, priv->hw, priv->wolopts); 4484 priv->irq_wake = 1; 4485 } else { 4486 stmmac_mac_set(priv, priv->ioaddr, false); 4487 pinctrl_pm_select_sleep_state(priv->device); 4488 /* Disable clock in case of PWM is off */ 4489 clk_disable(priv->plat->pclk); 4490 clk_disable(priv->plat->stmmac_clk); 4491 } 4492 mutex_unlock(&priv->lock); 4493 4494 priv->oldlink = false; 4495 priv->speed = SPEED_UNKNOWN; 4496 priv->oldduplex = DUPLEX_UNKNOWN; 4497 return 0; 4498 } 4499 EXPORT_SYMBOL_GPL(stmmac_suspend); 4500 4501 /** 4502 * stmmac_reset_queues_param - reset queue parameters 4503 * @dev: device pointer 4504 */ 4505 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 4506 { 4507 u32 rx_cnt = priv->plat->rx_queues_to_use; 4508 u32 tx_cnt = priv->plat->tx_queues_to_use; 4509 u32 queue; 4510 4511 for (queue = 0; queue < rx_cnt; queue++) { 4512 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4513 4514 rx_q->cur_rx = 0; 4515 rx_q->dirty_rx = 0; 4516 } 4517 4518 for (queue = 0; queue < tx_cnt; queue++) { 4519 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 4520 4521 tx_q->cur_tx = 0; 4522 tx_q->dirty_tx = 0; 4523 tx_q->mss = 0; 4524 } 4525 } 4526 4527 /** 4528 * stmmac_resume - resume callback 4529 * @dev: device pointer 4530 * Description: when resume this function is invoked to setup the DMA and CORE 4531 * in a usable state. 4532 */ 4533 int stmmac_resume(struct device *dev) 4534 { 4535 struct net_device *ndev = dev_get_drvdata(dev); 4536 struct stmmac_priv *priv = netdev_priv(ndev); 4537 4538 if (!netif_running(ndev)) 4539 return 0; 4540 4541 /* Power Down bit, into the PM register, is cleared 4542 * automatically as soon as a magic packet or a Wake-up frame 4543 * is received. Anyway, it's better to manually clear 4544 * this bit because it can generate problems while resuming 4545 * from another devices (e.g. serial console). 4546 */ 4547 if (device_may_wakeup(priv->device)) { 4548 mutex_lock(&priv->lock); 4549 stmmac_pmt(priv, priv->hw, 0); 4550 mutex_unlock(&priv->lock); 4551 priv->irq_wake = 0; 4552 } else { 4553 pinctrl_pm_select_default_state(priv->device); 4554 /* enable the clk previously disabled */ 4555 clk_enable(priv->plat->stmmac_clk); 4556 clk_enable(priv->plat->pclk); 4557 /* reset the phy so that it's ready */ 4558 if (priv->mii) 4559 stmmac_mdio_reset(priv->mii); 4560 } 4561 4562 netif_device_attach(ndev); 4563 4564 mutex_lock(&priv->lock); 4565 4566 stmmac_reset_queues_param(priv); 4567 4568 stmmac_clear_descriptors(priv); 4569 4570 stmmac_hw_setup(ndev, false); 4571 stmmac_init_tx_coalesce(priv); 4572 stmmac_set_rx_mode(ndev); 4573 4574 stmmac_enable_all_queues(priv); 4575 4576 stmmac_start_all_queues(priv); 4577 4578 mutex_unlock(&priv->lock); 4579 4580 if (ndev->phydev) 4581 phy_start(ndev->phydev); 4582 4583 return 0; 4584 } 4585 EXPORT_SYMBOL_GPL(stmmac_resume); 4586 4587 #ifndef MODULE 4588 static int __init stmmac_cmdline_opt(char *str) 4589 { 4590 char *opt; 4591 4592 if (!str || !*str) 4593 return -EINVAL; 4594 while ((opt = strsep(&str, ",")) != NULL) { 4595 if (!strncmp(opt, "debug:", 6)) { 4596 if (kstrtoint(opt + 6, 0, &debug)) 4597 goto err; 4598 } else if (!strncmp(opt, "phyaddr:", 8)) { 4599 if (kstrtoint(opt + 8, 0, &phyaddr)) 4600 goto err; 4601 } else if (!strncmp(opt, "buf_sz:", 7)) { 4602 if (kstrtoint(opt + 7, 0, &buf_sz)) 4603 goto err; 4604 } else if (!strncmp(opt, "tc:", 3)) { 4605 if (kstrtoint(opt + 3, 0, &tc)) 4606 goto err; 4607 } else if (!strncmp(opt, "watchdog:", 9)) { 4608 if (kstrtoint(opt + 9, 0, &watchdog)) 4609 goto err; 4610 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 4611 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 4612 goto err; 4613 } else if (!strncmp(opt, "pause:", 6)) { 4614 if (kstrtoint(opt + 6, 0, &pause)) 4615 goto err; 4616 } else if (!strncmp(opt, "eee_timer:", 10)) { 4617 if (kstrtoint(opt + 10, 0, &eee_timer)) 4618 goto err; 4619 } else if (!strncmp(opt, "chain_mode:", 11)) { 4620 if (kstrtoint(opt + 11, 0, &chain_mode)) 4621 goto err; 4622 } 4623 } 4624 return 0; 4625 4626 err: 4627 pr_err("%s: ERROR broken module parameter conversion", __func__); 4628 return -EINVAL; 4629 } 4630 4631 __setup("stmmaceth=", stmmac_cmdline_opt); 4632 #endif /* MODULE */ 4633 4634 static int __init stmmac_init(void) 4635 { 4636 #ifdef CONFIG_DEBUG_FS 4637 /* Create debugfs main directory if it doesn't exist yet */ 4638 if (!stmmac_fs_dir) { 4639 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 4640 4641 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) { 4642 pr_err("ERROR %s, debugfs create directory failed\n", 4643 STMMAC_RESOURCE_NAME); 4644 4645 return -ENOMEM; 4646 } 4647 } 4648 #endif 4649 4650 return 0; 4651 } 4652 4653 static void __exit stmmac_exit(void) 4654 { 4655 #ifdef CONFIG_DEBUG_FS 4656 debugfs_remove_recursive(stmmac_fs_dir); 4657 #endif 4658 } 4659 4660 module_init(stmmac_init) 4661 module_exit(stmmac_exit) 4662 4663 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 4664 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 4665 MODULE_LICENSE("GPL"); 4666