1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/clk.h> 18 #include <linux/kernel.h> 19 #include <linux/interrupt.h> 20 #include <linux/ip.h> 21 #include <linux/tcp.h> 22 #include <linux/skbuff.h> 23 #include <linux/ethtool.h> 24 #include <linux/if_ether.h> 25 #include <linux/crc32.h> 26 #include <linux/mii.h> 27 #include <linux/if.h> 28 #include <linux/if_vlan.h> 29 #include <linux/dma-mapping.h> 30 #include <linux/slab.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/prefetch.h> 33 #include <linux/pinctrl/consumer.h> 34 #ifdef CONFIG_DEBUG_FS 35 #include <linux/debugfs.h> 36 #include <linux/seq_file.h> 37 #endif /* CONFIG_DEBUG_FS */ 38 #include <linux/net_tstamp.h> 39 #include <linux/phylink.h> 40 #include <linux/udp.h> 41 #include <linux/bpf_trace.h> 42 #include <net/page_pool/helpers.h> 43 #include <net/pkt_cls.h> 44 #include <net/xdp_sock_drv.h> 45 #include "stmmac_ptp.h" 46 #include "stmmac.h" 47 #include "stmmac_xdp.h" 48 #include <linux/reset.h> 49 #include <linux/of_mdio.h> 50 #include "dwmac1000.h" 51 #include "dwxgmac2.h" 52 #include "hwif.h" 53 54 /* As long as the interface is active, we keep the timestamping counter enabled 55 * with fine resolution and binary rollover. This avoid non-monotonic behavior 56 * (clock jumps) when changing timestamping settings at runtime. 57 */ 58 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ 59 PTP_TCR_TSCTRLSSR) 60 61 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 62 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 63 64 /* Module parameters */ 65 #define TX_TIMEO 5000 66 static int watchdog = TX_TIMEO; 67 module_param(watchdog, int, 0644); 68 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 69 70 static int debug = -1; 71 module_param(debug, int, 0644); 72 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 73 74 static int phyaddr = -1; 75 module_param(phyaddr, int, 0444); 76 MODULE_PARM_DESC(phyaddr, "Physical device address"); 77 78 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 79 #define STMMAC_RX_THRESH(x) ((x)->dma_conf.dma_rx_size / 4) 80 81 /* Limit to make sure XDP TX and slow path can coexist */ 82 #define STMMAC_XSK_TX_BUDGET_MAX 256 83 #define STMMAC_TX_XSK_AVAIL 16 84 #define STMMAC_RX_FILL_BATCH 16 85 86 #define STMMAC_XDP_PASS 0 87 #define STMMAC_XDP_CONSUMED BIT(0) 88 #define STMMAC_XDP_TX BIT(1) 89 #define STMMAC_XDP_REDIRECT BIT(2) 90 91 static int flow_ctrl = FLOW_AUTO; 92 module_param(flow_ctrl, int, 0644); 93 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 94 95 static int pause = PAUSE_TIME; 96 module_param(pause, int, 0644); 97 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 98 99 #define TC_DEFAULT 64 100 static int tc = TC_DEFAULT; 101 module_param(tc, int, 0644); 102 MODULE_PARM_DESC(tc, "DMA threshold control value"); 103 104 #define DEFAULT_BUFSIZE 1536 105 static int buf_sz = DEFAULT_BUFSIZE; 106 module_param(buf_sz, int, 0644); 107 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 108 109 #define STMMAC_RX_COPYBREAK 256 110 111 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 112 NETIF_MSG_LINK | NETIF_MSG_IFUP | 113 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 114 115 #define STMMAC_DEFAULT_LPI_TIMER 1000 116 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 117 module_param(eee_timer, int, 0644); 118 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 119 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 120 121 /* By default the driver will use the ring mode to manage tx and rx descriptors, 122 * but allow user to force to use the chain instead of the ring 123 */ 124 static unsigned int chain_mode; 125 module_param(chain_mode, int, 0444); 126 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 127 128 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 129 /* For MSI interrupts handling */ 130 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 131 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 132 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 133 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 134 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 135 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 136 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 137 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 138 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 139 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 140 u32 rxmode, u32 chan); 141 142 #ifdef CONFIG_DEBUG_FS 143 static const struct net_device_ops stmmac_netdev_ops; 144 static void stmmac_init_fs(struct net_device *dev); 145 static void stmmac_exit_fs(struct net_device *dev); 146 #endif 147 148 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 149 150 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) 151 { 152 int ret = 0; 153 154 if (enabled) { 155 ret = clk_prepare_enable(priv->plat->stmmac_clk); 156 if (ret) 157 return ret; 158 ret = clk_prepare_enable(priv->plat->pclk); 159 if (ret) { 160 clk_disable_unprepare(priv->plat->stmmac_clk); 161 return ret; 162 } 163 if (priv->plat->clks_config) { 164 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); 165 if (ret) { 166 clk_disable_unprepare(priv->plat->stmmac_clk); 167 clk_disable_unprepare(priv->plat->pclk); 168 return ret; 169 } 170 } 171 } else { 172 clk_disable_unprepare(priv->plat->stmmac_clk); 173 clk_disable_unprepare(priv->plat->pclk); 174 if (priv->plat->clks_config) 175 priv->plat->clks_config(priv->plat->bsp_priv, enabled); 176 } 177 178 return ret; 179 } 180 EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); 181 182 /** 183 * stmmac_verify_args - verify the driver parameters. 184 * Description: it checks the driver parameters and set a default in case of 185 * errors. 186 */ 187 static void stmmac_verify_args(void) 188 { 189 if (unlikely(watchdog < 0)) 190 watchdog = TX_TIMEO; 191 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 192 buf_sz = DEFAULT_BUFSIZE; 193 if (unlikely(flow_ctrl > 1)) 194 flow_ctrl = FLOW_AUTO; 195 else if (likely(flow_ctrl < 0)) 196 flow_ctrl = FLOW_OFF; 197 if (unlikely((pause < 0) || (pause > 0xffff))) 198 pause = PAUSE_TIME; 199 if (eee_timer < 0) 200 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 201 } 202 203 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 204 { 205 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 206 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 207 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 208 u32 queue; 209 210 for (queue = 0; queue < maxq; queue++) { 211 struct stmmac_channel *ch = &priv->channel[queue]; 212 213 if (stmmac_xdp_is_enabled(priv) && 214 test_bit(queue, priv->af_xdp_zc_qps)) { 215 napi_disable(&ch->rxtx_napi); 216 continue; 217 } 218 219 if (queue < rx_queues_cnt) 220 napi_disable(&ch->rx_napi); 221 if (queue < tx_queues_cnt) 222 napi_disable(&ch->tx_napi); 223 } 224 } 225 226 /** 227 * stmmac_disable_all_queues - Disable all queues 228 * @priv: driver private structure 229 */ 230 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 231 { 232 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 233 struct stmmac_rx_queue *rx_q; 234 u32 queue; 235 236 /* synchronize_rcu() needed for pending XDP buffers to drain */ 237 for (queue = 0; queue < rx_queues_cnt; queue++) { 238 rx_q = &priv->dma_conf.rx_queue[queue]; 239 if (rx_q->xsk_pool) { 240 synchronize_rcu(); 241 break; 242 } 243 } 244 245 __stmmac_disable_all_queues(priv); 246 } 247 248 /** 249 * stmmac_enable_all_queues - Enable all queues 250 * @priv: driver private structure 251 */ 252 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 253 { 254 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 255 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 256 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 257 u32 queue; 258 259 for (queue = 0; queue < maxq; queue++) { 260 struct stmmac_channel *ch = &priv->channel[queue]; 261 262 if (stmmac_xdp_is_enabled(priv) && 263 test_bit(queue, priv->af_xdp_zc_qps)) { 264 napi_enable(&ch->rxtx_napi); 265 continue; 266 } 267 268 if (queue < rx_queues_cnt) 269 napi_enable(&ch->rx_napi); 270 if (queue < tx_queues_cnt) 271 napi_enable(&ch->tx_napi); 272 } 273 } 274 275 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 276 { 277 if (!test_bit(STMMAC_DOWN, &priv->state) && 278 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 279 queue_work(priv->wq, &priv->service_task); 280 } 281 282 static void stmmac_global_err(struct stmmac_priv *priv) 283 { 284 netif_carrier_off(priv->dev); 285 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 286 stmmac_service_event_schedule(priv); 287 } 288 289 /** 290 * stmmac_clk_csr_set - dynamically set the MDC clock 291 * @priv: driver private structure 292 * Description: this is to dynamically set the MDC clock according to the csr 293 * clock input. 294 * Note: 295 * If a specific clk_csr value is passed from the platform 296 * this means that the CSR Clock Range selection cannot be 297 * changed at run-time and it is fixed (as reported in the driver 298 * documentation). Viceversa the driver will try to set the MDC 299 * clock dynamically according to the actual clock input. 300 */ 301 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 302 { 303 u32 clk_rate; 304 305 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 306 307 /* Platform provided default clk_csr would be assumed valid 308 * for all other cases except for the below mentioned ones. 309 * For values higher than the IEEE 802.3 specified frequency 310 * we can not estimate the proper divider as it is not known 311 * the frequency of clk_csr_i. So we do not change the default 312 * divider. 313 */ 314 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 315 if (clk_rate < CSR_F_35M) 316 priv->clk_csr = STMMAC_CSR_20_35M; 317 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 318 priv->clk_csr = STMMAC_CSR_35_60M; 319 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 320 priv->clk_csr = STMMAC_CSR_60_100M; 321 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 322 priv->clk_csr = STMMAC_CSR_100_150M; 323 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 324 priv->clk_csr = STMMAC_CSR_150_250M; 325 else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) 326 priv->clk_csr = STMMAC_CSR_250_300M; 327 } 328 329 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { 330 if (clk_rate > 160000000) 331 priv->clk_csr = 0x03; 332 else if (clk_rate > 80000000) 333 priv->clk_csr = 0x02; 334 else if (clk_rate > 40000000) 335 priv->clk_csr = 0x01; 336 else 337 priv->clk_csr = 0; 338 } 339 340 if (priv->plat->has_xgmac) { 341 if (clk_rate > 400000000) 342 priv->clk_csr = 0x5; 343 else if (clk_rate > 350000000) 344 priv->clk_csr = 0x4; 345 else if (clk_rate > 300000000) 346 priv->clk_csr = 0x3; 347 else if (clk_rate > 250000000) 348 priv->clk_csr = 0x2; 349 else if (clk_rate > 150000000) 350 priv->clk_csr = 0x1; 351 else 352 priv->clk_csr = 0x0; 353 } 354 } 355 356 static void print_pkt(unsigned char *buf, int len) 357 { 358 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 359 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 360 } 361 362 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 363 { 364 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 365 u32 avail; 366 367 if (tx_q->dirty_tx > tx_q->cur_tx) 368 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 369 else 370 avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; 371 372 return avail; 373 } 374 375 /** 376 * stmmac_rx_dirty - Get RX queue dirty 377 * @priv: driver private structure 378 * @queue: RX queue index 379 */ 380 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 381 { 382 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 383 u32 dirty; 384 385 if (rx_q->dirty_rx <= rx_q->cur_rx) 386 dirty = rx_q->cur_rx - rx_q->dirty_rx; 387 else 388 dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; 389 390 return dirty; 391 } 392 393 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) 394 { 395 int tx_lpi_timer; 396 397 /* Clear/set the SW EEE timer flag based on LPI ET enablement */ 398 priv->eee_sw_timer_en = en ? 0 : 1; 399 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; 400 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); 401 } 402 403 /** 404 * stmmac_enable_eee_mode - check and enter in LPI mode 405 * @priv: driver private structure 406 * Description: this function is to verify and enter in LPI mode in case of 407 * EEE. 408 */ 409 static int stmmac_enable_eee_mode(struct stmmac_priv *priv) 410 { 411 u32 tx_cnt = priv->plat->tx_queues_to_use; 412 u32 queue; 413 414 /* check if all TX queues have the work finished */ 415 for (queue = 0; queue < tx_cnt; queue++) { 416 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 417 418 if (tx_q->dirty_tx != tx_q->cur_tx) 419 return -EBUSY; /* still unfinished work */ 420 } 421 422 /* Check and enter in LPI mode */ 423 if (!priv->tx_path_in_lpi_mode) 424 stmmac_set_eee_mode(priv, priv->hw, 425 priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING); 426 return 0; 427 } 428 429 /** 430 * stmmac_disable_eee_mode - disable and exit from LPI mode 431 * @priv: driver private structure 432 * Description: this function is to exit and disable EEE in case of 433 * LPI state is true. This is called by the xmit. 434 */ 435 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 436 { 437 if (!priv->eee_sw_timer_en) { 438 stmmac_lpi_entry_timer_config(priv, 0); 439 return; 440 } 441 442 stmmac_reset_eee_mode(priv, priv->hw); 443 del_timer_sync(&priv->eee_ctrl_timer); 444 priv->tx_path_in_lpi_mode = false; 445 } 446 447 /** 448 * stmmac_eee_ctrl_timer - EEE TX SW timer. 449 * @t: timer_list struct containing private info 450 * Description: 451 * if there is no data transfer and if we are not in LPI state, 452 * then MAC Transmitter can be moved to LPI state. 453 */ 454 static void stmmac_eee_ctrl_timer(struct timer_list *t) 455 { 456 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 457 458 if (stmmac_enable_eee_mode(priv)) 459 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 460 } 461 462 /** 463 * stmmac_eee_init - init EEE 464 * @priv: driver private structure 465 * Description: 466 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 467 * can also manage EEE, this function enable the LPI state and start related 468 * timer. 469 */ 470 bool stmmac_eee_init(struct stmmac_priv *priv) 471 { 472 int eee_tw_timer = priv->eee_tw_timer; 473 474 /* Using PCS we cannot dial with the phy registers at this stage 475 * so we do not support extra feature like EEE. 476 */ 477 if (priv->hw->pcs == STMMAC_PCS_TBI || 478 priv->hw->pcs == STMMAC_PCS_RTBI) 479 return false; 480 481 /* Check if MAC core supports the EEE feature. */ 482 if (!priv->dma_cap.eee) 483 return false; 484 485 mutex_lock(&priv->lock); 486 487 /* Check if it needs to be deactivated */ 488 if (!priv->eee_active) { 489 if (priv->eee_enabled) { 490 netdev_dbg(priv->dev, "disable EEE\n"); 491 stmmac_lpi_entry_timer_config(priv, 0); 492 del_timer_sync(&priv->eee_ctrl_timer); 493 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); 494 if (priv->hw->xpcs) 495 xpcs_config_eee(priv->hw->xpcs, 496 priv->plat->mult_fact_100ns, 497 false); 498 } 499 mutex_unlock(&priv->lock); 500 return false; 501 } 502 503 if (priv->eee_active && !priv->eee_enabled) { 504 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 505 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 506 eee_tw_timer); 507 if (priv->hw->xpcs) 508 xpcs_config_eee(priv->hw->xpcs, 509 priv->plat->mult_fact_100ns, 510 true); 511 } 512 513 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { 514 del_timer_sync(&priv->eee_ctrl_timer); 515 priv->tx_path_in_lpi_mode = false; 516 stmmac_lpi_entry_timer_config(priv, 1); 517 } else { 518 stmmac_lpi_entry_timer_config(priv, 0); 519 mod_timer(&priv->eee_ctrl_timer, 520 STMMAC_LPI_T(priv->tx_lpi_timer)); 521 } 522 523 mutex_unlock(&priv->lock); 524 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 525 return true; 526 } 527 528 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 529 * @priv: driver private structure 530 * @p : descriptor pointer 531 * @skb : the socket buffer 532 * Description : 533 * This function will read timestamp from the descriptor & pass it to stack. 534 * and also perform some sanity checks. 535 */ 536 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 537 struct dma_desc *p, struct sk_buff *skb) 538 { 539 struct skb_shared_hwtstamps shhwtstamp; 540 bool found = false; 541 u64 ns = 0; 542 543 if (!priv->hwts_tx_en) 544 return; 545 546 /* exit if skb doesn't support hw tstamp */ 547 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 548 return; 549 550 /* check tx tstamp status */ 551 if (stmmac_get_tx_timestamp_status(priv, p)) { 552 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 553 found = true; 554 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 555 found = true; 556 } 557 558 if (found) { 559 ns -= priv->plat->cdc_error_adj; 560 561 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 562 shhwtstamp.hwtstamp = ns_to_ktime(ns); 563 564 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 565 /* pass tstamp to stack */ 566 skb_tstamp_tx(skb, &shhwtstamp); 567 } 568 } 569 570 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 571 * @priv: driver private structure 572 * @p : descriptor pointer 573 * @np : next descriptor pointer 574 * @skb : the socket buffer 575 * Description : 576 * This function will read received packet's timestamp from the descriptor 577 * and pass it to stack. It also perform some sanity checks. 578 */ 579 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 580 struct dma_desc *np, struct sk_buff *skb) 581 { 582 struct skb_shared_hwtstamps *shhwtstamp = NULL; 583 struct dma_desc *desc = p; 584 u64 ns = 0; 585 586 if (!priv->hwts_rx_en) 587 return; 588 /* For GMAC4, the valid timestamp is from CTX next desc. */ 589 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 590 desc = np; 591 592 /* Check if timestamp is available */ 593 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 594 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 595 596 ns -= priv->plat->cdc_error_adj; 597 598 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 599 shhwtstamp = skb_hwtstamps(skb); 600 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 601 shhwtstamp->hwtstamp = ns_to_ktime(ns); 602 } else { 603 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 604 } 605 } 606 607 /** 608 * stmmac_hwtstamp_set - control hardware timestamping. 609 * @dev: device pointer. 610 * @ifr: An IOCTL specific structure, that can contain a pointer to 611 * a proprietary structure used to pass information to the driver. 612 * Description: 613 * This function configures the MAC to enable/disable both outgoing(TX) 614 * and incoming(RX) packets time stamping based on user input. 615 * Return Value: 616 * 0 on success and an appropriate -ve integer on failure. 617 */ 618 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 619 { 620 struct stmmac_priv *priv = netdev_priv(dev); 621 struct hwtstamp_config config; 622 u32 ptp_v2 = 0; 623 u32 tstamp_all = 0; 624 u32 ptp_over_ipv4_udp = 0; 625 u32 ptp_over_ipv6_udp = 0; 626 u32 ptp_over_ethernet = 0; 627 u32 snap_type_sel = 0; 628 u32 ts_master_en = 0; 629 u32 ts_event_en = 0; 630 631 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 632 netdev_alert(priv->dev, "No support for HW time stamping\n"); 633 priv->hwts_tx_en = 0; 634 priv->hwts_rx_en = 0; 635 636 return -EOPNOTSUPP; 637 } 638 639 if (copy_from_user(&config, ifr->ifr_data, 640 sizeof(config))) 641 return -EFAULT; 642 643 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 644 __func__, config.flags, config.tx_type, config.rx_filter); 645 646 if (config.tx_type != HWTSTAMP_TX_OFF && 647 config.tx_type != HWTSTAMP_TX_ON) 648 return -ERANGE; 649 650 if (priv->adv_ts) { 651 switch (config.rx_filter) { 652 case HWTSTAMP_FILTER_NONE: 653 /* time stamp no incoming packet at all */ 654 config.rx_filter = HWTSTAMP_FILTER_NONE; 655 break; 656 657 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 658 /* PTP v1, UDP, any kind of event packet */ 659 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 660 /* 'xmac' hardware can support Sync, Pdelay_Req and 661 * Pdelay_resp by setting bit14 and bits17/16 to 01 662 * This leaves Delay_Req timestamps out. 663 * Enable all events *and* general purpose message 664 * timestamping 665 */ 666 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 667 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 668 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 669 break; 670 671 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 672 /* PTP v1, UDP, Sync packet */ 673 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 674 /* take time stamp for SYNC messages only */ 675 ts_event_en = PTP_TCR_TSEVNTENA; 676 677 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 678 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 679 break; 680 681 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 682 /* PTP v1, UDP, Delay_req packet */ 683 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 684 /* take time stamp for Delay_Req messages only */ 685 ts_master_en = PTP_TCR_TSMSTRENA; 686 ts_event_en = PTP_TCR_TSEVNTENA; 687 688 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 689 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 690 break; 691 692 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 693 /* PTP v2, UDP, any kind of event packet */ 694 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 695 ptp_v2 = PTP_TCR_TSVER2ENA; 696 /* take time stamp for all event messages */ 697 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 698 699 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 700 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 701 break; 702 703 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 704 /* PTP v2, UDP, Sync packet */ 705 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 706 ptp_v2 = PTP_TCR_TSVER2ENA; 707 /* take time stamp for SYNC messages only */ 708 ts_event_en = PTP_TCR_TSEVNTENA; 709 710 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 711 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 712 break; 713 714 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 715 /* PTP v2, UDP, Delay_req packet */ 716 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 717 ptp_v2 = PTP_TCR_TSVER2ENA; 718 /* take time stamp for Delay_Req messages only */ 719 ts_master_en = PTP_TCR_TSMSTRENA; 720 ts_event_en = PTP_TCR_TSEVNTENA; 721 722 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 723 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 724 break; 725 726 case HWTSTAMP_FILTER_PTP_V2_EVENT: 727 /* PTP v2/802.AS1 any layer, any kind of event packet */ 728 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 729 ptp_v2 = PTP_TCR_TSVER2ENA; 730 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 731 if (priv->synopsys_id < DWMAC_CORE_4_10) 732 ts_event_en = PTP_TCR_TSEVNTENA; 733 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 734 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 735 ptp_over_ethernet = PTP_TCR_TSIPENA; 736 break; 737 738 case HWTSTAMP_FILTER_PTP_V2_SYNC: 739 /* PTP v2/802.AS1, any layer, Sync packet */ 740 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 741 ptp_v2 = PTP_TCR_TSVER2ENA; 742 /* take time stamp for SYNC messages only */ 743 ts_event_en = PTP_TCR_TSEVNTENA; 744 745 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 746 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 747 ptp_over_ethernet = PTP_TCR_TSIPENA; 748 break; 749 750 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 751 /* PTP v2/802.AS1, any layer, Delay_req packet */ 752 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 753 ptp_v2 = PTP_TCR_TSVER2ENA; 754 /* take time stamp for Delay_Req messages only */ 755 ts_master_en = PTP_TCR_TSMSTRENA; 756 ts_event_en = PTP_TCR_TSEVNTENA; 757 758 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 759 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 760 ptp_over_ethernet = PTP_TCR_TSIPENA; 761 break; 762 763 case HWTSTAMP_FILTER_NTP_ALL: 764 case HWTSTAMP_FILTER_ALL: 765 /* time stamp any incoming packet */ 766 config.rx_filter = HWTSTAMP_FILTER_ALL; 767 tstamp_all = PTP_TCR_TSENALL; 768 break; 769 770 default: 771 return -ERANGE; 772 } 773 } else { 774 switch (config.rx_filter) { 775 case HWTSTAMP_FILTER_NONE: 776 config.rx_filter = HWTSTAMP_FILTER_NONE; 777 break; 778 default: 779 /* PTP v1, UDP, any kind of event packet */ 780 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 781 break; 782 } 783 } 784 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 785 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 786 787 priv->systime_flags = STMMAC_HWTS_ACTIVE; 788 789 if (priv->hwts_tx_en || priv->hwts_rx_en) { 790 priv->systime_flags |= tstamp_all | ptp_v2 | 791 ptp_over_ethernet | ptp_over_ipv6_udp | 792 ptp_over_ipv4_udp | ts_event_en | 793 ts_master_en | snap_type_sel; 794 } 795 796 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 797 798 memcpy(&priv->tstamp_config, &config, sizeof(config)); 799 800 return copy_to_user(ifr->ifr_data, &config, 801 sizeof(config)) ? -EFAULT : 0; 802 } 803 804 /** 805 * stmmac_hwtstamp_get - read hardware timestamping. 806 * @dev: device pointer. 807 * @ifr: An IOCTL specific structure, that can contain a pointer to 808 * a proprietary structure used to pass information to the driver. 809 * Description: 810 * This function obtain the current hardware timestamping settings 811 * as requested. 812 */ 813 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 814 { 815 struct stmmac_priv *priv = netdev_priv(dev); 816 struct hwtstamp_config *config = &priv->tstamp_config; 817 818 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 819 return -EOPNOTSUPP; 820 821 return copy_to_user(ifr->ifr_data, config, 822 sizeof(*config)) ? -EFAULT : 0; 823 } 824 825 /** 826 * stmmac_init_tstamp_counter - init hardware timestamping counter 827 * @priv: driver private structure 828 * @systime_flags: timestamping flags 829 * Description: 830 * Initialize hardware counter for packet timestamping. 831 * This is valid as long as the interface is open and not suspended. 832 * Will be rerun after resuming from suspend, case in which the timestamping 833 * flags updated by stmmac_hwtstamp_set() also need to be restored. 834 */ 835 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) 836 { 837 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 838 struct timespec64 now; 839 u32 sec_inc = 0; 840 u64 temp = 0; 841 842 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 843 return -EOPNOTSUPP; 844 845 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 846 priv->systime_flags = systime_flags; 847 848 /* program Sub Second Increment reg */ 849 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 850 priv->plat->clk_ptp_rate, 851 xmac, &sec_inc); 852 temp = div_u64(1000000000ULL, sec_inc); 853 854 /* Store sub second increment for later use */ 855 priv->sub_second_inc = sec_inc; 856 857 /* calculate default added value: 858 * formula is : 859 * addend = (2^32)/freq_div_ratio; 860 * where, freq_div_ratio = 1e9ns/sec_inc 861 */ 862 temp = (u64)(temp << 32); 863 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 864 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 865 866 /* initialize system time */ 867 ktime_get_real_ts64(&now); 868 869 /* lower 32 bits of tv_sec are safe until y2106 */ 870 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 871 872 return 0; 873 } 874 EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); 875 876 /** 877 * stmmac_init_ptp - init PTP 878 * @priv: driver private structure 879 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 880 * This is done by looking at the HW cap. register. 881 * This function also registers the ptp driver. 882 */ 883 static int stmmac_init_ptp(struct stmmac_priv *priv) 884 { 885 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 886 int ret; 887 888 if (priv->plat->ptp_clk_freq_config) 889 priv->plat->ptp_clk_freq_config(priv); 890 891 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); 892 if (ret) 893 return ret; 894 895 priv->adv_ts = 0; 896 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 897 if (xmac && priv->dma_cap.atime_stamp) 898 priv->adv_ts = 1; 899 /* Dwmac 3.x core with extend_desc can support adv_ts */ 900 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 901 priv->adv_ts = 1; 902 903 if (priv->dma_cap.time_stamp) 904 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 905 906 if (priv->adv_ts) 907 netdev_info(priv->dev, 908 "IEEE 1588-2008 Advanced Timestamp supported\n"); 909 910 priv->hwts_tx_en = 0; 911 priv->hwts_rx_en = 0; 912 913 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 914 stmmac_hwtstamp_correct_latency(priv, priv); 915 916 return 0; 917 } 918 919 static void stmmac_release_ptp(struct stmmac_priv *priv) 920 { 921 clk_disable_unprepare(priv->plat->clk_ptp_ref); 922 stmmac_ptp_unregister(priv); 923 } 924 925 /** 926 * stmmac_mac_flow_ctrl - Configure flow control in all queues 927 * @priv: driver private structure 928 * @duplex: duplex passed to the next function 929 * Description: It is used for configuring the flow control in all queues 930 */ 931 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 932 { 933 u32 tx_cnt = priv->plat->tx_queues_to_use; 934 935 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 936 priv->pause, tx_cnt); 937 } 938 939 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 940 phy_interface_t interface) 941 { 942 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 943 944 if (priv->hw->xpcs) 945 return &priv->hw->xpcs->pcs; 946 947 if (priv->hw->lynx_pcs) 948 return priv->hw->lynx_pcs; 949 950 return NULL; 951 } 952 953 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 954 const struct phylink_link_state *state) 955 { 956 /* Nothing to do, xpcs_config() handles everything */ 957 } 958 959 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) 960 { 961 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 962 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 963 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 964 bool *hs_enable = &fpe_cfg->hs_enable; 965 966 if (is_up && *hs_enable) { 967 stmmac_fpe_send_mpacket(priv, priv->ioaddr, fpe_cfg, 968 MPACKET_VERIFY); 969 } else { 970 *lo_state = FPE_STATE_OFF; 971 *lp_state = FPE_STATE_OFF; 972 } 973 } 974 975 static void stmmac_mac_link_down(struct phylink_config *config, 976 unsigned int mode, phy_interface_t interface) 977 { 978 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 979 980 stmmac_mac_set(priv, priv->ioaddr, false); 981 priv->eee_active = false; 982 priv->tx_lpi_enabled = false; 983 priv->eee_enabled = stmmac_eee_init(priv); 984 stmmac_set_eee_pls(priv, priv->hw, false); 985 986 if (priv->dma_cap.fpesel) 987 stmmac_fpe_link_state_handle(priv, false); 988 } 989 990 static void stmmac_mac_link_up(struct phylink_config *config, 991 struct phy_device *phy, 992 unsigned int mode, phy_interface_t interface, 993 int speed, int duplex, 994 bool tx_pause, bool rx_pause) 995 { 996 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 997 u32 old_ctrl, ctrl; 998 999 if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 1000 priv->plat->serdes_powerup) 1001 priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 1002 1003 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1004 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1005 1006 if (interface == PHY_INTERFACE_MODE_USXGMII) { 1007 switch (speed) { 1008 case SPEED_10000: 1009 ctrl |= priv->hw->link.xgmii.speed10000; 1010 break; 1011 case SPEED_5000: 1012 ctrl |= priv->hw->link.xgmii.speed5000; 1013 break; 1014 case SPEED_2500: 1015 ctrl |= priv->hw->link.xgmii.speed2500; 1016 break; 1017 default: 1018 return; 1019 } 1020 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 1021 switch (speed) { 1022 case SPEED_100000: 1023 ctrl |= priv->hw->link.xlgmii.speed100000; 1024 break; 1025 case SPEED_50000: 1026 ctrl |= priv->hw->link.xlgmii.speed50000; 1027 break; 1028 case SPEED_40000: 1029 ctrl |= priv->hw->link.xlgmii.speed40000; 1030 break; 1031 case SPEED_25000: 1032 ctrl |= priv->hw->link.xlgmii.speed25000; 1033 break; 1034 case SPEED_10000: 1035 ctrl |= priv->hw->link.xgmii.speed10000; 1036 break; 1037 case SPEED_2500: 1038 ctrl |= priv->hw->link.speed2500; 1039 break; 1040 case SPEED_1000: 1041 ctrl |= priv->hw->link.speed1000; 1042 break; 1043 default: 1044 return; 1045 } 1046 } else { 1047 switch (speed) { 1048 case SPEED_2500: 1049 ctrl |= priv->hw->link.speed2500; 1050 break; 1051 case SPEED_1000: 1052 ctrl |= priv->hw->link.speed1000; 1053 break; 1054 case SPEED_100: 1055 ctrl |= priv->hw->link.speed100; 1056 break; 1057 case SPEED_10: 1058 ctrl |= priv->hw->link.speed10; 1059 break; 1060 default: 1061 return; 1062 } 1063 } 1064 1065 priv->speed = speed; 1066 1067 if (priv->plat->fix_mac_speed) 1068 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); 1069 1070 if (!duplex) 1071 ctrl &= ~priv->hw->link.duplex; 1072 else 1073 ctrl |= priv->hw->link.duplex; 1074 1075 /* Flow Control operation */ 1076 if (rx_pause && tx_pause) 1077 priv->flow_ctrl = FLOW_AUTO; 1078 else if (rx_pause && !tx_pause) 1079 priv->flow_ctrl = FLOW_RX; 1080 else if (!rx_pause && tx_pause) 1081 priv->flow_ctrl = FLOW_TX; 1082 else 1083 priv->flow_ctrl = FLOW_OFF; 1084 1085 stmmac_mac_flow_ctrl(priv, duplex); 1086 1087 if (ctrl != old_ctrl) 1088 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1089 1090 stmmac_mac_set(priv, priv->ioaddr, true); 1091 if (phy && priv->dma_cap.eee) { 1092 priv->eee_active = 1093 phy_init_eee(phy, !(priv->plat->flags & 1094 STMMAC_FLAG_RX_CLK_RUNS_IN_LPI)) >= 0; 1095 priv->eee_enabled = stmmac_eee_init(priv); 1096 priv->tx_lpi_enabled = priv->eee_enabled; 1097 stmmac_set_eee_pls(priv, priv->hw, true); 1098 } 1099 1100 if (priv->dma_cap.fpesel) 1101 stmmac_fpe_link_state_handle(priv, true); 1102 1103 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1104 stmmac_hwtstamp_correct_latency(priv, priv); 1105 } 1106 1107 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1108 .mac_select_pcs = stmmac_mac_select_pcs, 1109 .mac_config = stmmac_mac_config, 1110 .mac_link_down = stmmac_mac_link_down, 1111 .mac_link_up = stmmac_mac_link_up, 1112 }; 1113 1114 /** 1115 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1116 * @priv: driver private structure 1117 * Description: this is to verify if the HW supports the PCS. 1118 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1119 * configured for the TBI, RTBI, or SGMII PHY interface. 1120 */ 1121 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1122 { 1123 int interface = priv->plat->mac_interface; 1124 1125 if (priv->dma_cap.pcs) { 1126 if ((interface == PHY_INTERFACE_MODE_RGMII) || 1127 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 1128 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 1129 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 1130 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 1131 priv->hw->pcs = STMMAC_PCS_RGMII; 1132 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 1133 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1134 priv->hw->pcs = STMMAC_PCS_SGMII; 1135 } 1136 } 1137 } 1138 1139 /** 1140 * stmmac_init_phy - PHY initialization 1141 * @dev: net device structure 1142 * Description: it initializes the driver's PHY state, and attaches the PHY 1143 * to the mac driver. 1144 * Return value: 1145 * 0 on success 1146 */ 1147 static int stmmac_init_phy(struct net_device *dev) 1148 { 1149 struct stmmac_priv *priv = netdev_priv(dev); 1150 struct fwnode_handle *phy_fwnode; 1151 struct fwnode_handle *fwnode; 1152 int ret; 1153 1154 if (!phylink_expects_phy(priv->phylink)) 1155 return 0; 1156 1157 fwnode = priv->plat->port_node; 1158 if (!fwnode) 1159 fwnode = dev_fwnode(priv->device); 1160 1161 if (fwnode) 1162 phy_fwnode = fwnode_get_phy_node(fwnode); 1163 else 1164 phy_fwnode = NULL; 1165 1166 /* Some DT bindings do not set-up the PHY handle. Let's try to 1167 * manually parse it 1168 */ 1169 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1170 int addr = priv->plat->phy_addr; 1171 struct phy_device *phydev; 1172 1173 if (addr < 0) { 1174 netdev_err(priv->dev, "no phy found\n"); 1175 return -ENODEV; 1176 } 1177 1178 phydev = mdiobus_get_phy(priv->mii, addr); 1179 if (!phydev) { 1180 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1181 return -ENODEV; 1182 } 1183 1184 ret = phylink_connect_phy(priv->phylink, phydev); 1185 } else { 1186 fwnode_handle_put(phy_fwnode); 1187 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); 1188 } 1189 1190 if (!priv->plat->pmt) { 1191 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 1192 1193 phylink_ethtool_get_wol(priv->phylink, &wol); 1194 device_set_wakeup_capable(priv->device, !!wol.supported); 1195 device_set_wakeup_enable(priv->device, !!wol.wolopts); 1196 } 1197 1198 return ret; 1199 } 1200 1201 static int stmmac_phy_setup(struct stmmac_priv *priv) 1202 { 1203 struct stmmac_mdio_bus_data *mdio_bus_data; 1204 int mode = priv->plat->phy_interface; 1205 struct fwnode_handle *fwnode; 1206 struct phylink *phylink; 1207 int max_speed; 1208 1209 priv->phylink_config.dev = &priv->dev->dev; 1210 priv->phylink_config.type = PHYLINK_NETDEV; 1211 priv->phylink_config.mac_managed_pm = true; 1212 1213 mdio_bus_data = priv->plat->mdio_bus_data; 1214 if (mdio_bus_data) 1215 priv->phylink_config.ovr_an_inband = 1216 mdio_bus_data->xpcs_an_inband; 1217 1218 /* Set the platform/firmware specified interface mode. Note, phylink 1219 * deals with the PHY interface mode, not the MAC interface mode. 1220 */ 1221 __set_bit(mode, priv->phylink_config.supported_interfaces); 1222 1223 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1224 if (priv->hw->xpcs) 1225 xpcs_get_interfaces(priv->hw->xpcs, 1226 priv->phylink_config.supported_interfaces); 1227 1228 /* Get the MAC specific capabilities */ 1229 stmmac_mac_phylink_get_caps(priv); 1230 1231 priv->phylink_config.mac_capabilities = priv->hw->link.caps; 1232 1233 max_speed = priv->plat->max_speed; 1234 if (max_speed) 1235 phylink_limit_mac_speed(&priv->phylink_config, max_speed); 1236 1237 fwnode = priv->plat->port_node; 1238 if (!fwnode) 1239 fwnode = dev_fwnode(priv->device); 1240 1241 phylink = phylink_create(&priv->phylink_config, fwnode, 1242 mode, &stmmac_phylink_mac_ops); 1243 if (IS_ERR(phylink)) 1244 return PTR_ERR(phylink); 1245 1246 priv->phylink = phylink; 1247 return 0; 1248 } 1249 1250 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1251 struct stmmac_dma_conf *dma_conf) 1252 { 1253 u32 rx_cnt = priv->plat->rx_queues_to_use; 1254 unsigned int desc_size; 1255 void *head_rx; 1256 u32 queue; 1257 1258 /* Display RX rings */ 1259 for (queue = 0; queue < rx_cnt; queue++) { 1260 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1261 1262 pr_info("\tRX Queue %u rings\n", queue); 1263 1264 if (priv->extend_desc) { 1265 head_rx = (void *)rx_q->dma_erx; 1266 desc_size = sizeof(struct dma_extended_desc); 1267 } else { 1268 head_rx = (void *)rx_q->dma_rx; 1269 desc_size = sizeof(struct dma_desc); 1270 } 1271 1272 /* Display RX ring */ 1273 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1274 rx_q->dma_rx_phy, desc_size); 1275 } 1276 } 1277 1278 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1279 struct stmmac_dma_conf *dma_conf) 1280 { 1281 u32 tx_cnt = priv->plat->tx_queues_to_use; 1282 unsigned int desc_size; 1283 void *head_tx; 1284 u32 queue; 1285 1286 /* Display TX rings */ 1287 for (queue = 0; queue < tx_cnt; queue++) { 1288 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1289 1290 pr_info("\tTX Queue %d rings\n", queue); 1291 1292 if (priv->extend_desc) { 1293 head_tx = (void *)tx_q->dma_etx; 1294 desc_size = sizeof(struct dma_extended_desc); 1295 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1296 head_tx = (void *)tx_q->dma_entx; 1297 desc_size = sizeof(struct dma_edesc); 1298 } else { 1299 head_tx = (void *)tx_q->dma_tx; 1300 desc_size = sizeof(struct dma_desc); 1301 } 1302 1303 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1304 tx_q->dma_tx_phy, desc_size); 1305 } 1306 } 1307 1308 static void stmmac_display_rings(struct stmmac_priv *priv, 1309 struct stmmac_dma_conf *dma_conf) 1310 { 1311 /* Display RX ring */ 1312 stmmac_display_rx_rings(priv, dma_conf); 1313 1314 /* Display TX ring */ 1315 stmmac_display_tx_rings(priv, dma_conf); 1316 } 1317 1318 static int stmmac_set_bfsize(int mtu, int bufsize) 1319 { 1320 int ret = bufsize; 1321 1322 if (mtu >= BUF_SIZE_8KiB) 1323 ret = BUF_SIZE_16KiB; 1324 else if (mtu >= BUF_SIZE_4KiB) 1325 ret = BUF_SIZE_8KiB; 1326 else if (mtu >= BUF_SIZE_2KiB) 1327 ret = BUF_SIZE_4KiB; 1328 else if (mtu > DEFAULT_BUFSIZE) 1329 ret = BUF_SIZE_2KiB; 1330 else 1331 ret = DEFAULT_BUFSIZE; 1332 1333 return ret; 1334 } 1335 1336 /** 1337 * stmmac_clear_rx_descriptors - clear RX descriptors 1338 * @priv: driver private structure 1339 * @dma_conf: structure to take the dma data 1340 * @queue: RX queue index 1341 * Description: this function is called to clear the RX descriptors 1342 * in case of both basic and extended descriptors are used. 1343 */ 1344 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1345 struct stmmac_dma_conf *dma_conf, 1346 u32 queue) 1347 { 1348 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1349 int i; 1350 1351 /* Clear the RX descriptors */ 1352 for (i = 0; i < dma_conf->dma_rx_size; i++) 1353 if (priv->extend_desc) 1354 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1355 priv->use_riwt, priv->mode, 1356 (i == dma_conf->dma_rx_size - 1), 1357 dma_conf->dma_buf_sz); 1358 else 1359 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1360 priv->use_riwt, priv->mode, 1361 (i == dma_conf->dma_rx_size - 1), 1362 dma_conf->dma_buf_sz); 1363 } 1364 1365 /** 1366 * stmmac_clear_tx_descriptors - clear tx descriptors 1367 * @priv: driver private structure 1368 * @dma_conf: structure to take the dma data 1369 * @queue: TX queue index. 1370 * Description: this function is called to clear the TX descriptors 1371 * in case of both basic and extended descriptors are used. 1372 */ 1373 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1374 struct stmmac_dma_conf *dma_conf, 1375 u32 queue) 1376 { 1377 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1378 int i; 1379 1380 /* Clear the TX descriptors */ 1381 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1382 int last = (i == (dma_conf->dma_tx_size - 1)); 1383 struct dma_desc *p; 1384 1385 if (priv->extend_desc) 1386 p = &tx_q->dma_etx[i].basic; 1387 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1388 p = &tx_q->dma_entx[i].basic; 1389 else 1390 p = &tx_q->dma_tx[i]; 1391 1392 stmmac_init_tx_desc(priv, p, priv->mode, last); 1393 } 1394 } 1395 1396 /** 1397 * stmmac_clear_descriptors - clear descriptors 1398 * @priv: driver private structure 1399 * @dma_conf: structure to take the dma data 1400 * Description: this function is called to clear the TX and RX descriptors 1401 * in case of both basic and extended descriptors are used. 1402 */ 1403 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1404 struct stmmac_dma_conf *dma_conf) 1405 { 1406 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1407 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1408 u32 queue; 1409 1410 /* Clear the RX descriptors */ 1411 for (queue = 0; queue < rx_queue_cnt; queue++) 1412 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1413 1414 /* Clear the TX descriptors */ 1415 for (queue = 0; queue < tx_queue_cnt; queue++) 1416 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1417 } 1418 1419 /** 1420 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1421 * @priv: driver private structure 1422 * @dma_conf: structure to take the dma data 1423 * @p: descriptor pointer 1424 * @i: descriptor index 1425 * @flags: gfp flag 1426 * @queue: RX queue index 1427 * Description: this function is called to allocate a receive buffer, perform 1428 * the DMA mapping and init the descriptor. 1429 */ 1430 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1431 struct stmmac_dma_conf *dma_conf, 1432 struct dma_desc *p, 1433 int i, gfp_t flags, u32 queue) 1434 { 1435 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1436 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1437 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1438 1439 if (priv->dma_cap.host_dma_width <= 32) 1440 gfp |= GFP_DMA32; 1441 1442 if (!buf->page) { 1443 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1444 if (!buf->page) 1445 return -ENOMEM; 1446 buf->page_offset = stmmac_rx_offset(priv); 1447 } 1448 1449 if (priv->sph && !buf->sec_page) { 1450 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1451 if (!buf->sec_page) 1452 return -ENOMEM; 1453 1454 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1455 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1456 } else { 1457 buf->sec_page = NULL; 1458 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1459 } 1460 1461 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1462 1463 stmmac_set_desc_addr(priv, p, buf->addr); 1464 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1465 stmmac_init_desc3(priv, p); 1466 1467 return 0; 1468 } 1469 1470 /** 1471 * stmmac_free_rx_buffer - free RX dma buffers 1472 * @priv: private structure 1473 * @rx_q: RX queue 1474 * @i: buffer index. 1475 */ 1476 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1477 struct stmmac_rx_queue *rx_q, 1478 int i) 1479 { 1480 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1481 1482 if (buf->page) 1483 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1484 buf->page = NULL; 1485 1486 if (buf->sec_page) 1487 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1488 buf->sec_page = NULL; 1489 } 1490 1491 /** 1492 * stmmac_free_tx_buffer - free RX dma buffers 1493 * @priv: private structure 1494 * @dma_conf: structure to take the dma data 1495 * @queue: RX queue index 1496 * @i: buffer index. 1497 */ 1498 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1499 struct stmmac_dma_conf *dma_conf, 1500 u32 queue, int i) 1501 { 1502 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1503 1504 if (tx_q->tx_skbuff_dma[i].buf && 1505 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1506 if (tx_q->tx_skbuff_dma[i].map_as_page) 1507 dma_unmap_page(priv->device, 1508 tx_q->tx_skbuff_dma[i].buf, 1509 tx_q->tx_skbuff_dma[i].len, 1510 DMA_TO_DEVICE); 1511 else 1512 dma_unmap_single(priv->device, 1513 tx_q->tx_skbuff_dma[i].buf, 1514 tx_q->tx_skbuff_dma[i].len, 1515 DMA_TO_DEVICE); 1516 } 1517 1518 if (tx_q->xdpf[i] && 1519 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1520 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1521 xdp_return_frame(tx_q->xdpf[i]); 1522 tx_q->xdpf[i] = NULL; 1523 } 1524 1525 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1526 tx_q->xsk_frames_done++; 1527 1528 if (tx_q->tx_skbuff[i] && 1529 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1530 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1531 tx_q->tx_skbuff[i] = NULL; 1532 } 1533 1534 tx_q->tx_skbuff_dma[i].buf = 0; 1535 tx_q->tx_skbuff_dma[i].map_as_page = false; 1536 } 1537 1538 /** 1539 * dma_free_rx_skbufs - free RX dma buffers 1540 * @priv: private structure 1541 * @dma_conf: structure to take the dma data 1542 * @queue: RX queue index 1543 */ 1544 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1545 struct stmmac_dma_conf *dma_conf, 1546 u32 queue) 1547 { 1548 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1549 int i; 1550 1551 for (i = 0; i < dma_conf->dma_rx_size; i++) 1552 stmmac_free_rx_buffer(priv, rx_q, i); 1553 } 1554 1555 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1556 struct stmmac_dma_conf *dma_conf, 1557 u32 queue, gfp_t flags) 1558 { 1559 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1560 int i; 1561 1562 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1563 struct dma_desc *p; 1564 int ret; 1565 1566 if (priv->extend_desc) 1567 p = &((rx_q->dma_erx + i)->basic); 1568 else 1569 p = rx_q->dma_rx + i; 1570 1571 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1572 queue); 1573 if (ret) 1574 return ret; 1575 1576 rx_q->buf_alloc_num++; 1577 } 1578 1579 return 0; 1580 } 1581 1582 /** 1583 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1584 * @priv: private structure 1585 * @dma_conf: structure to take the dma data 1586 * @queue: RX queue index 1587 */ 1588 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1589 struct stmmac_dma_conf *dma_conf, 1590 u32 queue) 1591 { 1592 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1593 int i; 1594 1595 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1596 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1597 1598 if (!buf->xdp) 1599 continue; 1600 1601 xsk_buff_free(buf->xdp); 1602 buf->xdp = NULL; 1603 } 1604 } 1605 1606 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1607 struct stmmac_dma_conf *dma_conf, 1608 u32 queue) 1609 { 1610 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1611 int i; 1612 1613 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1614 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1615 * use this macro to make sure no size violations. 1616 */ 1617 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1618 1619 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1620 struct stmmac_rx_buffer *buf; 1621 dma_addr_t dma_addr; 1622 struct dma_desc *p; 1623 1624 if (priv->extend_desc) 1625 p = (struct dma_desc *)(rx_q->dma_erx + i); 1626 else 1627 p = rx_q->dma_rx + i; 1628 1629 buf = &rx_q->buf_pool[i]; 1630 1631 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1632 if (!buf->xdp) 1633 return -ENOMEM; 1634 1635 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1636 stmmac_set_desc_addr(priv, p, dma_addr); 1637 rx_q->buf_alloc_num++; 1638 } 1639 1640 return 0; 1641 } 1642 1643 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1644 { 1645 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1646 return NULL; 1647 1648 return xsk_get_pool_from_qid(priv->dev, queue); 1649 } 1650 1651 /** 1652 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1653 * @priv: driver private structure 1654 * @dma_conf: structure to take the dma data 1655 * @queue: RX queue index 1656 * @flags: gfp flag. 1657 * Description: this function initializes the DMA RX descriptors 1658 * and allocates the socket buffers. It supports the chained and ring 1659 * modes. 1660 */ 1661 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1662 struct stmmac_dma_conf *dma_conf, 1663 u32 queue, gfp_t flags) 1664 { 1665 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1666 int ret; 1667 1668 netif_dbg(priv, probe, priv->dev, 1669 "(%s) dma_rx_phy=0x%08x\n", __func__, 1670 (u32)rx_q->dma_rx_phy); 1671 1672 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1673 1674 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1675 1676 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1677 1678 if (rx_q->xsk_pool) { 1679 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1680 MEM_TYPE_XSK_BUFF_POOL, 1681 NULL)); 1682 netdev_info(priv->dev, 1683 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1684 rx_q->queue_index); 1685 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1686 } else { 1687 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1688 MEM_TYPE_PAGE_POOL, 1689 rx_q->page_pool)); 1690 netdev_info(priv->dev, 1691 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1692 rx_q->queue_index); 1693 } 1694 1695 if (rx_q->xsk_pool) { 1696 /* RX XDP ZC buffer pool may not be populated, e.g. 1697 * xdpsock TX-only. 1698 */ 1699 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1700 } else { 1701 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1702 if (ret < 0) 1703 return -ENOMEM; 1704 } 1705 1706 /* Setup the chained descriptor addresses */ 1707 if (priv->mode == STMMAC_CHAIN_MODE) { 1708 if (priv->extend_desc) 1709 stmmac_mode_init(priv, rx_q->dma_erx, 1710 rx_q->dma_rx_phy, 1711 dma_conf->dma_rx_size, 1); 1712 else 1713 stmmac_mode_init(priv, rx_q->dma_rx, 1714 rx_q->dma_rx_phy, 1715 dma_conf->dma_rx_size, 0); 1716 } 1717 1718 return 0; 1719 } 1720 1721 static int init_dma_rx_desc_rings(struct net_device *dev, 1722 struct stmmac_dma_conf *dma_conf, 1723 gfp_t flags) 1724 { 1725 struct stmmac_priv *priv = netdev_priv(dev); 1726 u32 rx_count = priv->plat->rx_queues_to_use; 1727 int queue; 1728 int ret; 1729 1730 /* RX INITIALIZATION */ 1731 netif_dbg(priv, probe, priv->dev, 1732 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1733 1734 for (queue = 0; queue < rx_count; queue++) { 1735 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1736 if (ret) 1737 goto err_init_rx_buffers; 1738 } 1739 1740 return 0; 1741 1742 err_init_rx_buffers: 1743 while (queue >= 0) { 1744 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1745 1746 if (rx_q->xsk_pool) 1747 dma_free_rx_xskbufs(priv, dma_conf, queue); 1748 else 1749 dma_free_rx_skbufs(priv, dma_conf, queue); 1750 1751 rx_q->buf_alloc_num = 0; 1752 rx_q->xsk_pool = NULL; 1753 1754 queue--; 1755 } 1756 1757 return ret; 1758 } 1759 1760 /** 1761 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1762 * @priv: driver private structure 1763 * @dma_conf: structure to take the dma data 1764 * @queue: TX queue index 1765 * Description: this function initializes the DMA TX descriptors 1766 * and allocates the socket buffers. It supports the chained and ring 1767 * modes. 1768 */ 1769 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 1770 struct stmmac_dma_conf *dma_conf, 1771 u32 queue) 1772 { 1773 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1774 int i; 1775 1776 netif_dbg(priv, probe, priv->dev, 1777 "(%s) dma_tx_phy=0x%08x\n", __func__, 1778 (u32)tx_q->dma_tx_phy); 1779 1780 /* Setup the chained descriptor addresses */ 1781 if (priv->mode == STMMAC_CHAIN_MODE) { 1782 if (priv->extend_desc) 1783 stmmac_mode_init(priv, tx_q->dma_etx, 1784 tx_q->dma_tx_phy, 1785 dma_conf->dma_tx_size, 1); 1786 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 1787 stmmac_mode_init(priv, tx_q->dma_tx, 1788 tx_q->dma_tx_phy, 1789 dma_conf->dma_tx_size, 0); 1790 } 1791 1792 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1793 1794 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1795 struct dma_desc *p; 1796 1797 if (priv->extend_desc) 1798 p = &((tx_q->dma_etx + i)->basic); 1799 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1800 p = &((tx_q->dma_entx + i)->basic); 1801 else 1802 p = tx_q->dma_tx + i; 1803 1804 stmmac_clear_desc(priv, p); 1805 1806 tx_q->tx_skbuff_dma[i].buf = 0; 1807 tx_q->tx_skbuff_dma[i].map_as_page = false; 1808 tx_q->tx_skbuff_dma[i].len = 0; 1809 tx_q->tx_skbuff_dma[i].last_segment = false; 1810 tx_q->tx_skbuff[i] = NULL; 1811 } 1812 1813 return 0; 1814 } 1815 1816 static int init_dma_tx_desc_rings(struct net_device *dev, 1817 struct stmmac_dma_conf *dma_conf) 1818 { 1819 struct stmmac_priv *priv = netdev_priv(dev); 1820 u32 tx_queue_cnt; 1821 u32 queue; 1822 1823 tx_queue_cnt = priv->plat->tx_queues_to_use; 1824 1825 for (queue = 0; queue < tx_queue_cnt; queue++) 1826 __init_dma_tx_desc_rings(priv, dma_conf, queue); 1827 1828 return 0; 1829 } 1830 1831 /** 1832 * init_dma_desc_rings - init the RX/TX descriptor rings 1833 * @dev: net device structure 1834 * @dma_conf: structure to take the dma data 1835 * @flags: gfp flag. 1836 * Description: this function initializes the DMA RX/TX descriptors 1837 * and allocates the socket buffers. It supports the chained and ring 1838 * modes. 1839 */ 1840 static int init_dma_desc_rings(struct net_device *dev, 1841 struct stmmac_dma_conf *dma_conf, 1842 gfp_t flags) 1843 { 1844 struct stmmac_priv *priv = netdev_priv(dev); 1845 int ret; 1846 1847 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 1848 if (ret) 1849 return ret; 1850 1851 ret = init_dma_tx_desc_rings(dev, dma_conf); 1852 1853 stmmac_clear_descriptors(priv, dma_conf); 1854 1855 if (netif_msg_hw(priv)) 1856 stmmac_display_rings(priv, dma_conf); 1857 1858 return ret; 1859 } 1860 1861 /** 1862 * dma_free_tx_skbufs - free TX dma buffers 1863 * @priv: private structure 1864 * @dma_conf: structure to take the dma data 1865 * @queue: TX queue index 1866 */ 1867 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 1868 struct stmmac_dma_conf *dma_conf, 1869 u32 queue) 1870 { 1871 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1872 int i; 1873 1874 tx_q->xsk_frames_done = 0; 1875 1876 for (i = 0; i < dma_conf->dma_tx_size; i++) 1877 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 1878 1879 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 1880 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 1881 tx_q->xsk_frames_done = 0; 1882 tx_q->xsk_pool = NULL; 1883 } 1884 } 1885 1886 /** 1887 * stmmac_free_tx_skbufs - free TX skb buffers 1888 * @priv: private structure 1889 */ 1890 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 1891 { 1892 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1893 u32 queue; 1894 1895 for (queue = 0; queue < tx_queue_cnt; queue++) 1896 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 1897 } 1898 1899 /** 1900 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 1901 * @priv: private structure 1902 * @dma_conf: structure to take the dma data 1903 * @queue: RX queue index 1904 */ 1905 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 1906 struct stmmac_dma_conf *dma_conf, 1907 u32 queue) 1908 { 1909 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1910 1911 /* Release the DMA RX socket buffers */ 1912 if (rx_q->xsk_pool) 1913 dma_free_rx_xskbufs(priv, dma_conf, queue); 1914 else 1915 dma_free_rx_skbufs(priv, dma_conf, queue); 1916 1917 rx_q->buf_alloc_num = 0; 1918 rx_q->xsk_pool = NULL; 1919 1920 /* Free DMA regions of consistent memory previously allocated */ 1921 if (!priv->extend_desc) 1922 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1923 sizeof(struct dma_desc), 1924 rx_q->dma_rx, rx_q->dma_rx_phy); 1925 else 1926 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1927 sizeof(struct dma_extended_desc), 1928 rx_q->dma_erx, rx_q->dma_rx_phy); 1929 1930 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 1931 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 1932 1933 kfree(rx_q->buf_pool); 1934 if (rx_q->page_pool) 1935 page_pool_destroy(rx_q->page_pool); 1936 } 1937 1938 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 1939 struct stmmac_dma_conf *dma_conf) 1940 { 1941 u32 rx_count = priv->plat->rx_queues_to_use; 1942 u32 queue; 1943 1944 /* Free RX queue resources */ 1945 for (queue = 0; queue < rx_count; queue++) 1946 __free_dma_rx_desc_resources(priv, dma_conf, queue); 1947 } 1948 1949 /** 1950 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 1951 * @priv: private structure 1952 * @dma_conf: structure to take the dma data 1953 * @queue: TX queue index 1954 */ 1955 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 1956 struct stmmac_dma_conf *dma_conf, 1957 u32 queue) 1958 { 1959 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1960 size_t size; 1961 void *addr; 1962 1963 /* Release the DMA TX socket buffers */ 1964 dma_free_tx_skbufs(priv, dma_conf, queue); 1965 1966 if (priv->extend_desc) { 1967 size = sizeof(struct dma_extended_desc); 1968 addr = tx_q->dma_etx; 1969 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1970 size = sizeof(struct dma_edesc); 1971 addr = tx_q->dma_entx; 1972 } else { 1973 size = sizeof(struct dma_desc); 1974 addr = tx_q->dma_tx; 1975 } 1976 1977 size *= dma_conf->dma_tx_size; 1978 1979 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 1980 1981 kfree(tx_q->tx_skbuff_dma); 1982 kfree(tx_q->tx_skbuff); 1983 } 1984 1985 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 1986 struct stmmac_dma_conf *dma_conf) 1987 { 1988 u32 tx_count = priv->plat->tx_queues_to_use; 1989 u32 queue; 1990 1991 /* Free TX queue resources */ 1992 for (queue = 0; queue < tx_count; queue++) 1993 __free_dma_tx_desc_resources(priv, dma_conf, queue); 1994 } 1995 1996 /** 1997 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 1998 * @priv: private structure 1999 * @dma_conf: structure to take the dma data 2000 * @queue: RX queue index 2001 * Description: according to which descriptor can be used (extend or basic) 2002 * this function allocates the resources for TX and RX paths. In case of 2003 * reception, for example, it pre-allocated the RX socket buffer in order to 2004 * allow zero-copy mechanism. 2005 */ 2006 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2007 struct stmmac_dma_conf *dma_conf, 2008 u32 queue) 2009 { 2010 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2011 struct stmmac_channel *ch = &priv->channel[queue]; 2012 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2013 struct page_pool_params pp_params = { 0 }; 2014 unsigned int num_pages; 2015 unsigned int napi_id; 2016 int ret; 2017 2018 rx_q->queue_index = queue; 2019 rx_q->priv_data = priv; 2020 2021 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2022 pp_params.pool_size = dma_conf->dma_rx_size; 2023 num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE); 2024 pp_params.order = ilog2(num_pages); 2025 pp_params.nid = dev_to_node(priv->device); 2026 pp_params.dev = priv->device; 2027 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2028 pp_params.offset = stmmac_rx_offset(priv); 2029 pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages); 2030 2031 rx_q->page_pool = page_pool_create(&pp_params); 2032 if (IS_ERR(rx_q->page_pool)) { 2033 ret = PTR_ERR(rx_q->page_pool); 2034 rx_q->page_pool = NULL; 2035 return ret; 2036 } 2037 2038 rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, 2039 sizeof(*rx_q->buf_pool), 2040 GFP_KERNEL); 2041 if (!rx_q->buf_pool) 2042 return -ENOMEM; 2043 2044 if (priv->extend_desc) { 2045 rx_q->dma_erx = dma_alloc_coherent(priv->device, 2046 dma_conf->dma_rx_size * 2047 sizeof(struct dma_extended_desc), 2048 &rx_q->dma_rx_phy, 2049 GFP_KERNEL); 2050 if (!rx_q->dma_erx) 2051 return -ENOMEM; 2052 2053 } else { 2054 rx_q->dma_rx = dma_alloc_coherent(priv->device, 2055 dma_conf->dma_rx_size * 2056 sizeof(struct dma_desc), 2057 &rx_q->dma_rx_phy, 2058 GFP_KERNEL); 2059 if (!rx_q->dma_rx) 2060 return -ENOMEM; 2061 } 2062 2063 if (stmmac_xdp_is_enabled(priv) && 2064 test_bit(queue, priv->af_xdp_zc_qps)) 2065 napi_id = ch->rxtx_napi.napi_id; 2066 else 2067 napi_id = ch->rx_napi.napi_id; 2068 2069 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, 2070 rx_q->queue_index, 2071 napi_id); 2072 if (ret) { 2073 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2074 return -EINVAL; 2075 } 2076 2077 return 0; 2078 } 2079 2080 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2081 struct stmmac_dma_conf *dma_conf) 2082 { 2083 u32 rx_count = priv->plat->rx_queues_to_use; 2084 u32 queue; 2085 int ret; 2086 2087 /* RX queues buffers and DMA */ 2088 for (queue = 0; queue < rx_count; queue++) { 2089 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2090 if (ret) 2091 goto err_dma; 2092 } 2093 2094 return 0; 2095 2096 err_dma: 2097 free_dma_rx_desc_resources(priv, dma_conf); 2098 2099 return ret; 2100 } 2101 2102 /** 2103 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2104 * @priv: private structure 2105 * @dma_conf: structure to take the dma data 2106 * @queue: TX queue index 2107 * Description: according to which descriptor can be used (extend or basic) 2108 * this function allocates the resources for TX and RX paths. In case of 2109 * reception, for example, it pre-allocated the RX socket buffer in order to 2110 * allow zero-copy mechanism. 2111 */ 2112 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2113 struct stmmac_dma_conf *dma_conf, 2114 u32 queue) 2115 { 2116 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2117 size_t size; 2118 void *addr; 2119 2120 tx_q->queue_index = queue; 2121 tx_q->priv_data = priv; 2122 2123 tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, 2124 sizeof(*tx_q->tx_skbuff_dma), 2125 GFP_KERNEL); 2126 if (!tx_q->tx_skbuff_dma) 2127 return -ENOMEM; 2128 2129 tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, 2130 sizeof(struct sk_buff *), 2131 GFP_KERNEL); 2132 if (!tx_q->tx_skbuff) 2133 return -ENOMEM; 2134 2135 if (priv->extend_desc) 2136 size = sizeof(struct dma_extended_desc); 2137 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2138 size = sizeof(struct dma_edesc); 2139 else 2140 size = sizeof(struct dma_desc); 2141 2142 size *= dma_conf->dma_tx_size; 2143 2144 addr = dma_alloc_coherent(priv->device, size, 2145 &tx_q->dma_tx_phy, GFP_KERNEL); 2146 if (!addr) 2147 return -ENOMEM; 2148 2149 if (priv->extend_desc) 2150 tx_q->dma_etx = addr; 2151 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2152 tx_q->dma_entx = addr; 2153 else 2154 tx_q->dma_tx = addr; 2155 2156 return 0; 2157 } 2158 2159 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2160 struct stmmac_dma_conf *dma_conf) 2161 { 2162 u32 tx_count = priv->plat->tx_queues_to_use; 2163 u32 queue; 2164 int ret; 2165 2166 /* TX queues buffers and DMA */ 2167 for (queue = 0; queue < tx_count; queue++) { 2168 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2169 if (ret) 2170 goto err_dma; 2171 } 2172 2173 return 0; 2174 2175 err_dma: 2176 free_dma_tx_desc_resources(priv, dma_conf); 2177 return ret; 2178 } 2179 2180 /** 2181 * alloc_dma_desc_resources - alloc TX/RX resources. 2182 * @priv: private structure 2183 * @dma_conf: structure to take the dma data 2184 * Description: according to which descriptor can be used (extend or basic) 2185 * this function allocates the resources for TX and RX paths. In case of 2186 * reception, for example, it pre-allocated the RX socket buffer in order to 2187 * allow zero-copy mechanism. 2188 */ 2189 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2190 struct stmmac_dma_conf *dma_conf) 2191 { 2192 /* RX Allocation */ 2193 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2194 2195 if (ret) 2196 return ret; 2197 2198 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2199 2200 return ret; 2201 } 2202 2203 /** 2204 * free_dma_desc_resources - free dma desc resources 2205 * @priv: private structure 2206 * @dma_conf: structure to take the dma data 2207 */ 2208 static void free_dma_desc_resources(struct stmmac_priv *priv, 2209 struct stmmac_dma_conf *dma_conf) 2210 { 2211 /* Release the DMA TX socket buffers */ 2212 free_dma_tx_desc_resources(priv, dma_conf); 2213 2214 /* Release the DMA RX socket buffers later 2215 * to ensure all pending XDP_TX buffers are returned. 2216 */ 2217 free_dma_rx_desc_resources(priv, dma_conf); 2218 } 2219 2220 /** 2221 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2222 * @priv: driver private structure 2223 * Description: It is used for enabling the rx queues in the MAC 2224 */ 2225 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2226 { 2227 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2228 int queue; 2229 u8 mode; 2230 2231 for (queue = 0; queue < rx_queues_count; queue++) { 2232 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2233 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2234 } 2235 } 2236 2237 /** 2238 * stmmac_start_rx_dma - start RX DMA channel 2239 * @priv: driver private structure 2240 * @chan: RX channel index 2241 * Description: 2242 * This starts a RX DMA channel 2243 */ 2244 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2245 { 2246 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2247 stmmac_start_rx(priv, priv->ioaddr, chan); 2248 } 2249 2250 /** 2251 * stmmac_start_tx_dma - start TX DMA channel 2252 * @priv: driver private structure 2253 * @chan: TX channel index 2254 * Description: 2255 * This starts a TX DMA channel 2256 */ 2257 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2258 { 2259 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2260 stmmac_start_tx(priv, priv->ioaddr, chan); 2261 } 2262 2263 /** 2264 * stmmac_stop_rx_dma - stop RX DMA channel 2265 * @priv: driver private structure 2266 * @chan: RX channel index 2267 * Description: 2268 * This stops a RX DMA channel 2269 */ 2270 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2271 { 2272 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2273 stmmac_stop_rx(priv, priv->ioaddr, chan); 2274 } 2275 2276 /** 2277 * stmmac_stop_tx_dma - stop TX DMA channel 2278 * @priv: driver private structure 2279 * @chan: TX channel index 2280 * Description: 2281 * This stops a TX DMA channel 2282 */ 2283 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2284 { 2285 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2286 stmmac_stop_tx(priv, priv->ioaddr, chan); 2287 } 2288 2289 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2290 { 2291 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2292 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2293 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2294 u32 chan; 2295 2296 for (chan = 0; chan < dma_csr_ch; chan++) { 2297 struct stmmac_channel *ch = &priv->channel[chan]; 2298 unsigned long flags; 2299 2300 spin_lock_irqsave(&ch->lock, flags); 2301 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2302 spin_unlock_irqrestore(&ch->lock, flags); 2303 } 2304 } 2305 2306 /** 2307 * stmmac_start_all_dma - start all RX and TX DMA channels 2308 * @priv: driver private structure 2309 * Description: 2310 * This starts all the RX and TX DMA channels 2311 */ 2312 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2313 { 2314 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2315 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2316 u32 chan = 0; 2317 2318 for (chan = 0; chan < rx_channels_count; chan++) 2319 stmmac_start_rx_dma(priv, chan); 2320 2321 for (chan = 0; chan < tx_channels_count; chan++) 2322 stmmac_start_tx_dma(priv, chan); 2323 } 2324 2325 /** 2326 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2327 * @priv: driver private structure 2328 * Description: 2329 * This stops the RX and TX DMA channels 2330 */ 2331 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2332 { 2333 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2334 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2335 u32 chan = 0; 2336 2337 for (chan = 0; chan < rx_channels_count; chan++) 2338 stmmac_stop_rx_dma(priv, chan); 2339 2340 for (chan = 0; chan < tx_channels_count; chan++) 2341 stmmac_stop_tx_dma(priv, chan); 2342 } 2343 2344 /** 2345 * stmmac_dma_operation_mode - HW DMA operation mode 2346 * @priv: driver private structure 2347 * Description: it is used for configuring the DMA operation mode register in 2348 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2349 */ 2350 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2351 { 2352 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2353 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2354 int rxfifosz = priv->plat->rx_fifo_size; 2355 int txfifosz = priv->plat->tx_fifo_size; 2356 u32 txmode = 0; 2357 u32 rxmode = 0; 2358 u32 chan = 0; 2359 u8 qmode = 0; 2360 2361 if (rxfifosz == 0) 2362 rxfifosz = priv->dma_cap.rx_fifo_size; 2363 if (txfifosz == 0) 2364 txfifosz = priv->dma_cap.tx_fifo_size; 2365 2366 /* Adjust for real per queue fifo size */ 2367 rxfifosz /= rx_channels_count; 2368 txfifosz /= tx_channels_count; 2369 2370 if (priv->plat->force_thresh_dma_mode) { 2371 txmode = tc; 2372 rxmode = tc; 2373 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2374 /* 2375 * In case of GMAC, SF mode can be enabled 2376 * to perform the TX COE in HW. This depends on: 2377 * 1) TX COE if actually supported 2378 * 2) There is no bugged Jumbo frame support 2379 * that needs to not insert csum in the TDES. 2380 */ 2381 txmode = SF_DMA_MODE; 2382 rxmode = SF_DMA_MODE; 2383 priv->xstats.threshold = SF_DMA_MODE; 2384 } else { 2385 txmode = tc; 2386 rxmode = SF_DMA_MODE; 2387 } 2388 2389 /* configure all channels */ 2390 for (chan = 0; chan < rx_channels_count; chan++) { 2391 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2392 u32 buf_size; 2393 2394 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2395 2396 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2397 rxfifosz, qmode); 2398 2399 if (rx_q->xsk_pool) { 2400 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 2401 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2402 buf_size, 2403 chan); 2404 } else { 2405 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2406 priv->dma_conf.dma_buf_sz, 2407 chan); 2408 } 2409 } 2410 2411 for (chan = 0; chan < tx_channels_count; chan++) { 2412 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2413 2414 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2415 txfifosz, qmode); 2416 } 2417 } 2418 2419 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2420 { 2421 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2422 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2423 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2424 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2425 unsigned int entry = tx_q->cur_tx; 2426 struct dma_desc *tx_desc = NULL; 2427 struct xdp_desc xdp_desc; 2428 bool work_done = true; 2429 u32 tx_set_ic_bit = 0; 2430 2431 /* Avoids TX time-out as we are sharing with slow path */ 2432 txq_trans_cond_update(nq); 2433 2434 budget = min(budget, stmmac_tx_avail(priv, queue)); 2435 2436 while (budget-- > 0) { 2437 dma_addr_t dma_addr; 2438 bool set_ic; 2439 2440 /* We are sharing with slow path and stop XSK TX desc submission when 2441 * available TX ring is less than threshold. 2442 */ 2443 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2444 !netif_carrier_ok(priv->dev)) { 2445 work_done = false; 2446 break; 2447 } 2448 2449 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2450 break; 2451 2452 if (likely(priv->extend_desc)) 2453 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2454 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2455 tx_desc = &tx_q->dma_entx[entry].basic; 2456 else 2457 tx_desc = tx_q->dma_tx + entry; 2458 2459 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2460 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2461 2462 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; 2463 2464 /* To return XDP buffer to XSK pool, we simple call 2465 * xsk_tx_completed(), so we don't need to fill up 2466 * 'buf' and 'xdpf'. 2467 */ 2468 tx_q->tx_skbuff_dma[entry].buf = 0; 2469 tx_q->xdpf[entry] = NULL; 2470 2471 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2472 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; 2473 tx_q->tx_skbuff_dma[entry].last_segment = true; 2474 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2475 2476 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2477 2478 tx_q->tx_count_frames++; 2479 2480 if (!priv->tx_coal_frames[queue]) 2481 set_ic = false; 2482 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2483 set_ic = true; 2484 else 2485 set_ic = false; 2486 2487 if (set_ic) { 2488 tx_q->tx_count_frames = 0; 2489 stmmac_set_tx_ic(priv, tx_desc); 2490 tx_set_ic_bit++; 2491 } 2492 2493 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2494 true, priv->mode, true, true, 2495 xdp_desc.len); 2496 2497 stmmac_enable_dma_transmission(priv, priv->ioaddr); 2498 2499 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2500 entry = tx_q->cur_tx; 2501 } 2502 u64_stats_update_begin(&txq_stats->napi_syncp); 2503 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); 2504 u64_stats_update_end(&txq_stats->napi_syncp); 2505 2506 if (tx_desc) { 2507 stmmac_flush_tx_descriptors(priv, queue); 2508 xsk_tx_release(pool); 2509 } 2510 2511 /* Return true if all of the 3 conditions are met 2512 * a) TX Budget is still available 2513 * b) work_done = true when XSK TX desc peek is empty (no more 2514 * pending XSK TX for transmission) 2515 */ 2516 return !!budget && work_done; 2517 } 2518 2519 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2520 { 2521 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2522 tc += 64; 2523 2524 if (priv->plat->force_thresh_dma_mode) 2525 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2526 else 2527 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2528 chan); 2529 2530 priv->xstats.threshold = tc; 2531 } 2532 } 2533 2534 /** 2535 * stmmac_tx_clean - to manage the transmission completion 2536 * @priv: driver private structure 2537 * @budget: napi budget limiting this functions packet handling 2538 * @queue: TX queue index 2539 * Description: it reclaims the transmit resources after transmission completes. 2540 */ 2541 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) 2542 { 2543 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2544 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2545 unsigned int bytes_compl = 0, pkts_compl = 0; 2546 unsigned int entry, xmits = 0, count = 0; 2547 u32 tx_packets = 0, tx_errors = 0; 2548 2549 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2550 2551 tx_q->xsk_frames_done = 0; 2552 2553 entry = tx_q->dirty_tx; 2554 2555 /* Try to clean all TX complete frame in 1 shot */ 2556 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2557 struct xdp_frame *xdpf; 2558 struct sk_buff *skb; 2559 struct dma_desc *p; 2560 int status; 2561 2562 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2563 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2564 xdpf = tx_q->xdpf[entry]; 2565 skb = NULL; 2566 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2567 xdpf = NULL; 2568 skb = tx_q->tx_skbuff[entry]; 2569 } else { 2570 xdpf = NULL; 2571 skb = NULL; 2572 } 2573 2574 if (priv->extend_desc) 2575 p = (struct dma_desc *)(tx_q->dma_etx + entry); 2576 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2577 p = &tx_q->dma_entx[entry].basic; 2578 else 2579 p = tx_q->dma_tx + entry; 2580 2581 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2582 /* Check if the descriptor is owned by the DMA */ 2583 if (unlikely(status & tx_dma_own)) 2584 break; 2585 2586 count++; 2587 2588 /* Make sure descriptor fields are read after reading 2589 * the own bit. 2590 */ 2591 dma_rmb(); 2592 2593 /* Just consider the last segment and ...*/ 2594 if (likely(!(status & tx_not_ls))) { 2595 /* ... verify the status error condition */ 2596 if (unlikely(status & tx_err)) { 2597 tx_errors++; 2598 if (unlikely(status & tx_err_bump_tc)) 2599 stmmac_bump_dma_threshold(priv, queue); 2600 } else { 2601 tx_packets++; 2602 } 2603 if (skb) 2604 stmmac_get_tx_hwtstamp(priv, p, skb); 2605 } 2606 2607 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2608 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2609 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2610 dma_unmap_page(priv->device, 2611 tx_q->tx_skbuff_dma[entry].buf, 2612 tx_q->tx_skbuff_dma[entry].len, 2613 DMA_TO_DEVICE); 2614 else 2615 dma_unmap_single(priv->device, 2616 tx_q->tx_skbuff_dma[entry].buf, 2617 tx_q->tx_skbuff_dma[entry].len, 2618 DMA_TO_DEVICE); 2619 tx_q->tx_skbuff_dma[entry].buf = 0; 2620 tx_q->tx_skbuff_dma[entry].len = 0; 2621 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2622 } 2623 2624 stmmac_clean_desc3(priv, tx_q, p); 2625 2626 tx_q->tx_skbuff_dma[entry].last_segment = false; 2627 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2628 2629 if (xdpf && 2630 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2631 xdp_return_frame_rx_napi(xdpf); 2632 tx_q->xdpf[entry] = NULL; 2633 } 2634 2635 if (xdpf && 2636 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2637 xdp_return_frame(xdpf); 2638 tx_q->xdpf[entry] = NULL; 2639 } 2640 2641 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2642 tx_q->xsk_frames_done++; 2643 2644 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2645 if (likely(skb)) { 2646 pkts_compl++; 2647 bytes_compl += skb->len; 2648 dev_consume_skb_any(skb); 2649 tx_q->tx_skbuff[entry] = NULL; 2650 } 2651 } 2652 2653 stmmac_release_tx_desc(priv, p, priv->mode); 2654 2655 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 2656 } 2657 tx_q->dirty_tx = entry; 2658 2659 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2660 pkts_compl, bytes_compl); 2661 2662 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2663 queue))) && 2664 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2665 2666 netif_dbg(priv, tx_done, priv->dev, 2667 "%s: restart transmit\n", __func__); 2668 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2669 } 2670 2671 if (tx_q->xsk_pool) { 2672 bool work_done; 2673 2674 if (tx_q->xsk_frames_done) 2675 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2676 2677 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2678 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2679 2680 /* For XSK TX, we try to send as many as possible. 2681 * If XSK work done (XSK TX desc empty and budget still 2682 * available), return "budget - 1" to reenable TX IRQ. 2683 * Else, return "budget" to make NAPI continue polling. 2684 */ 2685 work_done = stmmac_xdp_xmit_zc(priv, queue, 2686 STMMAC_XSK_TX_BUDGET_MAX); 2687 if (work_done) 2688 xmits = budget - 1; 2689 else 2690 xmits = budget; 2691 } 2692 2693 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && 2694 priv->eee_sw_timer_en) { 2695 if (stmmac_enable_eee_mode(priv)) 2696 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 2697 } 2698 2699 /* We still have pending packets, let's call for a new scheduling */ 2700 if (tx_q->dirty_tx != tx_q->cur_tx) 2701 stmmac_tx_timer_arm(priv, queue); 2702 2703 u64_stats_update_begin(&txq_stats->napi_syncp); 2704 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); 2705 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); 2706 u64_stats_inc(&txq_stats->napi.tx_clean); 2707 u64_stats_update_end(&txq_stats->napi_syncp); 2708 2709 priv->xstats.tx_errors += tx_errors; 2710 2711 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2712 2713 /* Combine decisions from TX clean and XSK TX */ 2714 return max(count, xmits); 2715 } 2716 2717 /** 2718 * stmmac_tx_err - to manage the tx error 2719 * @priv: driver private structure 2720 * @chan: channel index 2721 * Description: it cleans the descriptors and restarts the transmission 2722 * in case of transmission errors. 2723 */ 2724 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 2725 { 2726 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2727 2728 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 2729 2730 stmmac_stop_tx_dma(priv, chan); 2731 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 2732 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 2733 stmmac_reset_tx_queue(priv, chan); 2734 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2735 tx_q->dma_tx_phy, chan); 2736 stmmac_start_tx_dma(priv, chan); 2737 2738 priv->xstats.tx_errors++; 2739 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2740 } 2741 2742 /** 2743 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2744 * @priv: driver private structure 2745 * @txmode: TX operating mode 2746 * @rxmode: RX operating mode 2747 * @chan: channel index 2748 * Description: it is used for configuring of the DMA operation mode in 2749 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2750 * mode. 2751 */ 2752 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2753 u32 rxmode, u32 chan) 2754 { 2755 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2756 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2757 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2758 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2759 int rxfifosz = priv->plat->rx_fifo_size; 2760 int txfifosz = priv->plat->tx_fifo_size; 2761 2762 if (rxfifosz == 0) 2763 rxfifosz = priv->dma_cap.rx_fifo_size; 2764 if (txfifosz == 0) 2765 txfifosz = priv->dma_cap.tx_fifo_size; 2766 2767 /* Adjust for real per queue fifo size */ 2768 rxfifosz /= rx_channels_count; 2769 txfifosz /= tx_channels_count; 2770 2771 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2772 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2773 } 2774 2775 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2776 { 2777 int ret; 2778 2779 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2780 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2781 if (ret && (ret != -EINVAL)) { 2782 stmmac_global_err(priv); 2783 return true; 2784 } 2785 2786 return false; 2787 } 2788 2789 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 2790 { 2791 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2792 &priv->xstats, chan, dir); 2793 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2794 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2795 struct stmmac_channel *ch = &priv->channel[chan]; 2796 struct napi_struct *rx_napi; 2797 struct napi_struct *tx_napi; 2798 unsigned long flags; 2799 2800 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 2801 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 2802 2803 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 2804 if (napi_schedule_prep(rx_napi)) { 2805 spin_lock_irqsave(&ch->lock, flags); 2806 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 2807 spin_unlock_irqrestore(&ch->lock, flags); 2808 __napi_schedule(rx_napi); 2809 } 2810 } 2811 2812 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 2813 if (napi_schedule_prep(tx_napi)) { 2814 spin_lock_irqsave(&ch->lock, flags); 2815 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 2816 spin_unlock_irqrestore(&ch->lock, flags); 2817 __napi_schedule(tx_napi); 2818 } 2819 } 2820 2821 return status; 2822 } 2823 2824 /** 2825 * stmmac_dma_interrupt - DMA ISR 2826 * @priv: driver private structure 2827 * Description: this is the DMA ISR. It is called by the main ISR. 2828 * It calls the dwmac dma routine and schedule poll method in case of some 2829 * work can be done. 2830 */ 2831 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2832 { 2833 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2834 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2835 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2836 tx_channel_count : rx_channel_count; 2837 u32 chan; 2838 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2839 2840 /* Make sure we never check beyond our status buffer. */ 2841 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2842 channels_to_check = ARRAY_SIZE(status); 2843 2844 for (chan = 0; chan < channels_to_check; chan++) 2845 status[chan] = stmmac_napi_check(priv, chan, 2846 DMA_DIR_RXTX); 2847 2848 for (chan = 0; chan < tx_channel_count; chan++) { 2849 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2850 /* Try to bump up the dma threshold on this failure */ 2851 stmmac_bump_dma_threshold(priv, chan); 2852 } else if (unlikely(status[chan] == tx_hard_error)) { 2853 stmmac_tx_err(priv, chan); 2854 } 2855 } 2856 } 2857 2858 /** 2859 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2860 * @priv: driver private structure 2861 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2862 */ 2863 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2864 { 2865 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2866 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2867 2868 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 2869 2870 if (priv->dma_cap.rmon) { 2871 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 2872 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2873 } else 2874 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2875 } 2876 2877 /** 2878 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2879 * @priv: driver private structure 2880 * Description: 2881 * new GMAC chip generations have a new register to indicate the 2882 * presence of the optional feature/functions. 2883 * This can be also used to override the value passed through the 2884 * platform and necessary for old MAC10/100 and GMAC chips. 2885 */ 2886 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2887 { 2888 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 2889 } 2890 2891 /** 2892 * stmmac_check_ether_addr - check if the MAC addr is valid 2893 * @priv: driver private structure 2894 * Description: 2895 * it is to verify if the MAC address is valid, in case of failures it 2896 * generates a random MAC address 2897 */ 2898 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2899 { 2900 u8 addr[ETH_ALEN]; 2901 2902 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2903 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 2904 if (is_valid_ether_addr(addr)) 2905 eth_hw_addr_set(priv->dev, addr); 2906 else 2907 eth_hw_addr_random(priv->dev); 2908 dev_info(priv->device, "device MAC address %pM\n", 2909 priv->dev->dev_addr); 2910 } 2911 } 2912 2913 /** 2914 * stmmac_init_dma_engine - DMA init. 2915 * @priv: driver private structure 2916 * Description: 2917 * It inits the DMA invoking the specific MAC/GMAC callback. 2918 * Some DMA parameters can be passed from the platform; 2919 * in case of these are not passed a default is kept for the MAC or GMAC. 2920 */ 2921 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2922 { 2923 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2924 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2925 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2926 struct stmmac_rx_queue *rx_q; 2927 struct stmmac_tx_queue *tx_q; 2928 u32 chan = 0; 2929 int atds = 0; 2930 int ret = 0; 2931 2932 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2933 dev_err(priv->device, "Invalid DMA configuration\n"); 2934 return -EINVAL; 2935 } 2936 2937 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2938 atds = 1; 2939 2940 ret = stmmac_reset(priv, priv->ioaddr); 2941 if (ret) { 2942 dev_err(priv->device, "Failed to reset the dma\n"); 2943 return ret; 2944 } 2945 2946 /* DMA Configuration */ 2947 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); 2948 2949 if (priv->plat->axi) 2950 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 2951 2952 /* DMA CSR Channel configuration */ 2953 for (chan = 0; chan < dma_csr_ch; chan++) { 2954 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 2955 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2956 } 2957 2958 /* DMA RX Channel Configuration */ 2959 for (chan = 0; chan < rx_channels_count; chan++) { 2960 rx_q = &priv->dma_conf.rx_queue[chan]; 2961 2962 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2963 rx_q->dma_rx_phy, chan); 2964 2965 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2966 (rx_q->buf_alloc_num * 2967 sizeof(struct dma_desc)); 2968 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 2969 rx_q->rx_tail_addr, chan); 2970 } 2971 2972 /* DMA TX Channel Configuration */ 2973 for (chan = 0; chan < tx_channels_count; chan++) { 2974 tx_q = &priv->dma_conf.tx_queue[chan]; 2975 2976 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2977 tx_q->dma_tx_phy, chan); 2978 2979 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 2980 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 2981 tx_q->tx_tail_addr, chan); 2982 } 2983 2984 return ret; 2985 } 2986 2987 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 2988 { 2989 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2990 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 2991 2992 if (!tx_coal_timer) 2993 return; 2994 2995 hrtimer_start(&tx_q->txtimer, 2996 STMMAC_COAL_TIMER(tx_coal_timer), 2997 HRTIMER_MODE_REL); 2998 } 2999 3000 /** 3001 * stmmac_tx_timer - mitigation sw timer for tx. 3002 * @t: data pointer 3003 * Description: 3004 * This is the timer handler to directly invoke the stmmac_tx_clean. 3005 */ 3006 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3007 { 3008 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3009 struct stmmac_priv *priv = tx_q->priv_data; 3010 struct stmmac_channel *ch; 3011 struct napi_struct *napi; 3012 3013 ch = &priv->channel[tx_q->queue_index]; 3014 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3015 3016 if (likely(napi_schedule_prep(napi))) { 3017 unsigned long flags; 3018 3019 spin_lock_irqsave(&ch->lock, flags); 3020 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3021 spin_unlock_irqrestore(&ch->lock, flags); 3022 __napi_schedule(napi); 3023 } 3024 3025 return HRTIMER_NORESTART; 3026 } 3027 3028 /** 3029 * stmmac_init_coalesce - init mitigation options. 3030 * @priv: driver private structure 3031 * Description: 3032 * This inits the coalesce parameters: i.e. timer rate, 3033 * timer handler and default threshold used for enabling the 3034 * interrupt on completion bit. 3035 */ 3036 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3037 { 3038 u32 tx_channel_count = priv->plat->tx_queues_to_use; 3039 u32 rx_channel_count = priv->plat->rx_queues_to_use; 3040 u32 chan; 3041 3042 for (chan = 0; chan < tx_channel_count; chan++) { 3043 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3044 3045 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3046 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3047 3048 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3049 tx_q->txtimer.function = stmmac_tx_timer; 3050 } 3051 3052 for (chan = 0; chan < rx_channel_count; chan++) 3053 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3054 } 3055 3056 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3057 { 3058 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3059 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3060 u32 chan; 3061 3062 /* set TX ring length */ 3063 for (chan = 0; chan < tx_channels_count; chan++) 3064 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3065 (priv->dma_conf.dma_tx_size - 1), chan); 3066 3067 /* set RX ring length */ 3068 for (chan = 0; chan < rx_channels_count; chan++) 3069 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3070 (priv->dma_conf.dma_rx_size - 1), chan); 3071 } 3072 3073 /** 3074 * stmmac_set_tx_queue_weight - Set TX queue weight 3075 * @priv: driver private structure 3076 * Description: It is used for setting TX queues weight 3077 */ 3078 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3079 { 3080 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3081 u32 weight; 3082 u32 queue; 3083 3084 for (queue = 0; queue < tx_queues_count; queue++) { 3085 weight = priv->plat->tx_queues_cfg[queue].weight; 3086 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3087 } 3088 } 3089 3090 /** 3091 * stmmac_configure_cbs - Configure CBS in TX queue 3092 * @priv: driver private structure 3093 * Description: It is used for configuring CBS in AVB TX queues 3094 */ 3095 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3096 { 3097 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3098 u32 mode_to_use; 3099 u32 queue; 3100 3101 /* queue 0 is reserved for legacy traffic */ 3102 for (queue = 1; queue < tx_queues_count; queue++) { 3103 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3104 if (mode_to_use == MTL_QUEUE_DCB) 3105 continue; 3106 3107 stmmac_config_cbs(priv, priv->hw, 3108 priv->plat->tx_queues_cfg[queue].send_slope, 3109 priv->plat->tx_queues_cfg[queue].idle_slope, 3110 priv->plat->tx_queues_cfg[queue].high_credit, 3111 priv->plat->tx_queues_cfg[queue].low_credit, 3112 queue); 3113 } 3114 } 3115 3116 /** 3117 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3118 * @priv: driver private structure 3119 * Description: It is used for mapping RX queues to RX dma channels 3120 */ 3121 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3122 { 3123 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3124 u32 queue; 3125 u32 chan; 3126 3127 for (queue = 0; queue < rx_queues_count; queue++) { 3128 chan = priv->plat->rx_queues_cfg[queue].chan; 3129 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3130 } 3131 } 3132 3133 /** 3134 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3135 * @priv: driver private structure 3136 * Description: It is used for configuring the RX Queue Priority 3137 */ 3138 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3139 { 3140 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3141 u32 queue; 3142 u32 prio; 3143 3144 for (queue = 0; queue < rx_queues_count; queue++) { 3145 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3146 continue; 3147 3148 prio = priv->plat->rx_queues_cfg[queue].prio; 3149 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3150 } 3151 } 3152 3153 /** 3154 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3155 * @priv: driver private structure 3156 * Description: It is used for configuring the TX Queue Priority 3157 */ 3158 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3159 { 3160 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3161 u32 queue; 3162 u32 prio; 3163 3164 for (queue = 0; queue < tx_queues_count; queue++) { 3165 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3166 continue; 3167 3168 prio = priv->plat->tx_queues_cfg[queue].prio; 3169 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3170 } 3171 } 3172 3173 /** 3174 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3175 * @priv: driver private structure 3176 * Description: It is used for configuring the RX queue routing 3177 */ 3178 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3179 { 3180 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3181 u32 queue; 3182 u8 packet; 3183 3184 for (queue = 0; queue < rx_queues_count; queue++) { 3185 /* no specific packet type routing specified for the queue */ 3186 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3187 continue; 3188 3189 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3190 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3191 } 3192 } 3193 3194 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3195 { 3196 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3197 priv->rss.enable = false; 3198 return; 3199 } 3200 3201 if (priv->dev->features & NETIF_F_RXHASH) 3202 priv->rss.enable = true; 3203 else 3204 priv->rss.enable = false; 3205 3206 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3207 priv->plat->rx_queues_to_use); 3208 } 3209 3210 /** 3211 * stmmac_mtl_configuration - Configure MTL 3212 * @priv: driver private structure 3213 * Description: It is used for configurring MTL 3214 */ 3215 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3216 { 3217 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3218 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3219 3220 if (tx_queues_count > 1) 3221 stmmac_set_tx_queue_weight(priv); 3222 3223 /* Configure MTL RX algorithms */ 3224 if (rx_queues_count > 1) 3225 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3226 priv->plat->rx_sched_algorithm); 3227 3228 /* Configure MTL TX algorithms */ 3229 if (tx_queues_count > 1) 3230 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3231 priv->plat->tx_sched_algorithm); 3232 3233 /* Configure CBS in AVB TX queues */ 3234 if (tx_queues_count > 1) 3235 stmmac_configure_cbs(priv); 3236 3237 /* Map RX MTL to DMA channels */ 3238 stmmac_rx_queue_dma_chan_map(priv); 3239 3240 /* Enable MAC RX Queues */ 3241 stmmac_mac_enable_rx_queues(priv); 3242 3243 /* Set RX priorities */ 3244 if (rx_queues_count > 1) 3245 stmmac_mac_config_rx_queues_prio(priv); 3246 3247 /* Set TX priorities */ 3248 if (tx_queues_count > 1) 3249 stmmac_mac_config_tx_queues_prio(priv); 3250 3251 /* Set RX routing */ 3252 if (rx_queues_count > 1) 3253 stmmac_mac_config_rx_queues_routing(priv); 3254 3255 /* Receive Side Scaling */ 3256 if (rx_queues_count > 1) 3257 stmmac_mac_config_rss(priv); 3258 } 3259 3260 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3261 { 3262 if (priv->dma_cap.asp) { 3263 netdev_info(priv->dev, "Enabling Safety Features\n"); 3264 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3265 priv->plat->safety_feat_cfg); 3266 } else { 3267 netdev_info(priv->dev, "No Safety Features support found\n"); 3268 } 3269 } 3270 3271 static int stmmac_fpe_start_wq(struct stmmac_priv *priv) 3272 { 3273 char *name; 3274 3275 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 3276 clear_bit(__FPE_REMOVING, &priv->fpe_task_state); 3277 3278 name = priv->wq_name; 3279 sprintf(name, "%s-fpe", priv->dev->name); 3280 3281 priv->fpe_wq = create_singlethread_workqueue(name); 3282 if (!priv->fpe_wq) { 3283 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); 3284 3285 return -ENOMEM; 3286 } 3287 netdev_info(priv->dev, "FPE workqueue start"); 3288 3289 return 0; 3290 } 3291 3292 /** 3293 * stmmac_hw_setup - setup mac in a usable state. 3294 * @dev : pointer to the device structure. 3295 * @ptp_register: register PTP if set 3296 * Description: 3297 * this is the main function to setup the HW in a usable state because the 3298 * dma engine is reset, the core registers are configured (e.g. AXI, 3299 * Checksum features, timers). The DMA is ready to start receiving and 3300 * transmitting. 3301 * Return value: 3302 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3303 * file on failure. 3304 */ 3305 static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) 3306 { 3307 struct stmmac_priv *priv = netdev_priv(dev); 3308 u32 rx_cnt = priv->plat->rx_queues_to_use; 3309 u32 tx_cnt = priv->plat->tx_queues_to_use; 3310 bool sph_en; 3311 u32 chan; 3312 int ret; 3313 3314 /* DMA initialization and SW reset */ 3315 ret = stmmac_init_dma_engine(priv); 3316 if (ret < 0) { 3317 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3318 __func__); 3319 return ret; 3320 } 3321 3322 /* Copy the MAC addr into the HW */ 3323 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3324 3325 /* PS and related bits will be programmed according to the speed */ 3326 if (priv->hw->pcs) { 3327 int speed = priv->plat->mac_port_sel_speed; 3328 3329 if ((speed == SPEED_10) || (speed == SPEED_100) || 3330 (speed == SPEED_1000)) { 3331 priv->hw->ps = speed; 3332 } else { 3333 dev_warn(priv->device, "invalid port speed\n"); 3334 priv->hw->ps = 0; 3335 } 3336 } 3337 3338 /* Initialize the MAC Core */ 3339 stmmac_core_init(priv, priv->hw, dev); 3340 3341 /* Initialize MTL*/ 3342 stmmac_mtl_configuration(priv); 3343 3344 /* Initialize Safety Features */ 3345 stmmac_safety_feat_configuration(priv); 3346 3347 ret = stmmac_rx_ipc(priv, priv->hw); 3348 if (!ret) { 3349 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3350 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3351 priv->hw->rx_csum = 0; 3352 } 3353 3354 /* Enable the MAC Rx/Tx */ 3355 stmmac_mac_set(priv, priv->ioaddr, true); 3356 3357 /* Set the HW DMA mode and the COE */ 3358 stmmac_dma_operation_mode(priv); 3359 3360 stmmac_mmc_setup(priv); 3361 3362 if (ptp_register) { 3363 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 3364 if (ret < 0) 3365 netdev_warn(priv->dev, 3366 "failed to enable PTP reference clock: %pe\n", 3367 ERR_PTR(ret)); 3368 } 3369 3370 ret = stmmac_init_ptp(priv); 3371 if (ret == -EOPNOTSUPP) 3372 netdev_info(priv->dev, "PTP not supported by HW\n"); 3373 else if (ret) 3374 netdev_warn(priv->dev, "PTP init failed\n"); 3375 else if (ptp_register) 3376 stmmac_ptp_register(priv); 3377 3378 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; 3379 3380 /* Convert the timer from msec to usec */ 3381 if (!priv->tx_lpi_timer) 3382 priv->tx_lpi_timer = eee_timer * 1000; 3383 3384 if (priv->use_riwt) { 3385 u32 queue; 3386 3387 for (queue = 0; queue < rx_cnt; queue++) { 3388 if (!priv->rx_riwt[queue]) 3389 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3390 3391 stmmac_rx_watchdog(priv, priv->ioaddr, 3392 priv->rx_riwt[queue], queue); 3393 } 3394 } 3395 3396 if (priv->hw->pcs) 3397 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); 3398 3399 /* set TX and RX rings length */ 3400 stmmac_set_rings_length(priv); 3401 3402 /* Enable TSO */ 3403 if (priv->tso) { 3404 for (chan = 0; chan < tx_cnt; chan++) { 3405 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3406 3407 /* TSO and TBS cannot co-exist */ 3408 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3409 continue; 3410 3411 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3412 } 3413 } 3414 3415 /* Enable Split Header */ 3416 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 3417 for (chan = 0; chan < rx_cnt; chan++) 3418 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3419 3420 3421 /* VLAN Tag Insertion */ 3422 if (priv->dma_cap.vlins) 3423 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3424 3425 /* TBS */ 3426 for (chan = 0; chan < tx_cnt; chan++) { 3427 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3428 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3429 3430 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3431 } 3432 3433 /* Configure real RX and TX queues */ 3434 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3435 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3436 3437 /* Start the ball rolling... */ 3438 stmmac_start_all_dma(priv); 3439 3440 if (priv->dma_cap.fpesel) { 3441 stmmac_fpe_start_wq(priv); 3442 3443 if (priv->plat->fpe_cfg->enable) 3444 stmmac_fpe_handshake(priv, true); 3445 } 3446 3447 return 0; 3448 } 3449 3450 static void stmmac_hw_teardown(struct net_device *dev) 3451 { 3452 struct stmmac_priv *priv = netdev_priv(dev); 3453 3454 clk_disable_unprepare(priv->plat->clk_ptp_ref); 3455 } 3456 3457 static void stmmac_free_irq(struct net_device *dev, 3458 enum request_irq_err irq_err, int irq_idx) 3459 { 3460 struct stmmac_priv *priv = netdev_priv(dev); 3461 int j; 3462 3463 switch (irq_err) { 3464 case REQ_IRQ_ERR_ALL: 3465 irq_idx = priv->plat->tx_queues_to_use; 3466 fallthrough; 3467 case REQ_IRQ_ERR_TX: 3468 for (j = irq_idx - 1; j >= 0; j--) { 3469 if (priv->tx_irq[j] > 0) { 3470 irq_set_affinity_hint(priv->tx_irq[j], NULL); 3471 free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); 3472 } 3473 } 3474 irq_idx = priv->plat->rx_queues_to_use; 3475 fallthrough; 3476 case REQ_IRQ_ERR_RX: 3477 for (j = irq_idx - 1; j >= 0; j--) { 3478 if (priv->rx_irq[j] > 0) { 3479 irq_set_affinity_hint(priv->rx_irq[j], NULL); 3480 free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); 3481 } 3482 } 3483 3484 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) 3485 free_irq(priv->sfty_ue_irq, dev); 3486 fallthrough; 3487 case REQ_IRQ_ERR_SFTY_UE: 3488 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) 3489 free_irq(priv->sfty_ce_irq, dev); 3490 fallthrough; 3491 case REQ_IRQ_ERR_SFTY_CE: 3492 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) 3493 free_irq(priv->lpi_irq, dev); 3494 fallthrough; 3495 case REQ_IRQ_ERR_LPI: 3496 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3497 free_irq(priv->wol_irq, dev); 3498 fallthrough; 3499 case REQ_IRQ_ERR_WOL: 3500 free_irq(dev->irq, dev); 3501 fallthrough; 3502 case REQ_IRQ_ERR_MAC: 3503 case REQ_IRQ_ERR_NO: 3504 /* If MAC IRQ request error, no more IRQ to free */ 3505 break; 3506 } 3507 } 3508 3509 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3510 { 3511 struct stmmac_priv *priv = netdev_priv(dev); 3512 enum request_irq_err irq_err; 3513 cpumask_t cpu_mask; 3514 int irq_idx = 0; 3515 char *int_name; 3516 int ret; 3517 int i; 3518 3519 /* For common interrupt */ 3520 int_name = priv->int_name_mac; 3521 sprintf(int_name, "%s:%s", dev->name, "mac"); 3522 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3523 0, int_name, dev); 3524 if (unlikely(ret < 0)) { 3525 netdev_err(priv->dev, 3526 "%s: alloc mac MSI %d (error: %d)\n", 3527 __func__, dev->irq, ret); 3528 irq_err = REQ_IRQ_ERR_MAC; 3529 goto irq_error; 3530 } 3531 3532 /* Request the Wake IRQ in case of another line 3533 * is used for WoL 3534 */ 3535 priv->wol_irq_disabled = true; 3536 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3537 int_name = priv->int_name_wol; 3538 sprintf(int_name, "%s:%s", dev->name, "wol"); 3539 ret = request_irq(priv->wol_irq, 3540 stmmac_mac_interrupt, 3541 0, int_name, dev); 3542 if (unlikely(ret < 0)) { 3543 netdev_err(priv->dev, 3544 "%s: alloc wol MSI %d (error: %d)\n", 3545 __func__, priv->wol_irq, ret); 3546 irq_err = REQ_IRQ_ERR_WOL; 3547 goto irq_error; 3548 } 3549 } 3550 3551 /* Request the LPI IRQ in case of another line 3552 * is used for LPI 3553 */ 3554 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3555 int_name = priv->int_name_lpi; 3556 sprintf(int_name, "%s:%s", dev->name, "lpi"); 3557 ret = request_irq(priv->lpi_irq, 3558 stmmac_mac_interrupt, 3559 0, int_name, dev); 3560 if (unlikely(ret < 0)) { 3561 netdev_err(priv->dev, 3562 "%s: alloc lpi MSI %d (error: %d)\n", 3563 __func__, priv->lpi_irq, ret); 3564 irq_err = REQ_IRQ_ERR_LPI; 3565 goto irq_error; 3566 } 3567 } 3568 3569 /* Request the Safety Feature Correctible Error line in 3570 * case of another line is used 3571 */ 3572 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { 3573 int_name = priv->int_name_sfty_ce; 3574 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3575 ret = request_irq(priv->sfty_ce_irq, 3576 stmmac_safety_interrupt, 3577 0, int_name, dev); 3578 if (unlikely(ret < 0)) { 3579 netdev_err(priv->dev, 3580 "%s: alloc sfty ce MSI %d (error: %d)\n", 3581 __func__, priv->sfty_ce_irq, ret); 3582 irq_err = REQ_IRQ_ERR_SFTY_CE; 3583 goto irq_error; 3584 } 3585 } 3586 3587 /* Request the Safety Feature Uncorrectible Error line in 3588 * case of another line is used 3589 */ 3590 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { 3591 int_name = priv->int_name_sfty_ue; 3592 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3593 ret = request_irq(priv->sfty_ue_irq, 3594 stmmac_safety_interrupt, 3595 0, int_name, dev); 3596 if (unlikely(ret < 0)) { 3597 netdev_err(priv->dev, 3598 "%s: alloc sfty ue MSI %d (error: %d)\n", 3599 __func__, priv->sfty_ue_irq, ret); 3600 irq_err = REQ_IRQ_ERR_SFTY_UE; 3601 goto irq_error; 3602 } 3603 } 3604 3605 /* Request Rx MSI irq */ 3606 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3607 if (i >= MTL_MAX_RX_QUEUES) 3608 break; 3609 if (priv->rx_irq[i] == 0) 3610 continue; 3611 3612 int_name = priv->int_name_rx_irq[i]; 3613 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3614 ret = request_irq(priv->rx_irq[i], 3615 stmmac_msi_intr_rx, 3616 0, int_name, &priv->dma_conf.rx_queue[i]); 3617 if (unlikely(ret < 0)) { 3618 netdev_err(priv->dev, 3619 "%s: alloc rx-%d MSI %d (error: %d)\n", 3620 __func__, i, priv->rx_irq[i], ret); 3621 irq_err = REQ_IRQ_ERR_RX; 3622 irq_idx = i; 3623 goto irq_error; 3624 } 3625 cpumask_clear(&cpu_mask); 3626 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3627 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); 3628 } 3629 3630 /* Request Tx MSI irq */ 3631 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3632 if (i >= MTL_MAX_TX_QUEUES) 3633 break; 3634 if (priv->tx_irq[i] == 0) 3635 continue; 3636 3637 int_name = priv->int_name_tx_irq[i]; 3638 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3639 ret = request_irq(priv->tx_irq[i], 3640 stmmac_msi_intr_tx, 3641 0, int_name, &priv->dma_conf.tx_queue[i]); 3642 if (unlikely(ret < 0)) { 3643 netdev_err(priv->dev, 3644 "%s: alloc tx-%d MSI %d (error: %d)\n", 3645 __func__, i, priv->tx_irq[i], ret); 3646 irq_err = REQ_IRQ_ERR_TX; 3647 irq_idx = i; 3648 goto irq_error; 3649 } 3650 cpumask_clear(&cpu_mask); 3651 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3652 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); 3653 } 3654 3655 return 0; 3656 3657 irq_error: 3658 stmmac_free_irq(dev, irq_err, irq_idx); 3659 return ret; 3660 } 3661 3662 static int stmmac_request_irq_single(struct net_device *dev) 3663 { 3664 struct stmmac_priv *priv = netdev_priv(dev); 3665 enum request_irq_err irq_err; 3666 int ret; 3667 3668 ret = request_irq(dev->irq, stmmac_interrupt, 3669 IRQF_SHARED, dev->name, dev); 3670 if (unlikely(ret < 0)) { 3671 netdev_err(priv->dev, 3672 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3673 __func__, dev->irq, ret); 3674 irq_err = REQ_IRQ_ERR_MAC; 3675 goto irq_error; 3676 } 3677 3678 /* Request the Wake IRQ in case of another line 3679 * is used for WoL 3680 */ 3681 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3682 ret = request_irq(priv->wol_irq, stmmac_interrupt, 3683 IRQF_SHARED, dev->name, dev); 3684 if (unlikely(ret < 0)) { 3685 netdev_err(priv->dev, 3686 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 3687 __func__, priv->wol_irq, ret); 3688 irq_err = REQ_IRQ_ERR_WOL; 3689 goto irq_error; 3690 } 3691 } 3692 3693 /* Request the IRQ lines */ 3694 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3695 ret = request_irq(priv->lpi_irq, stmmac_interrupt, 3696 IRQF_SHARED, dev->name, dev); 3697 if (unlikely(ret < 0)) { 3698 netdev_err(priv->dev, 3699 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 3700 __func__, priv->lpi_irq, ret); 3701 irq_err = REQ_IRQ_ERR_LPI; 3702 goto irq_error; 3703 } 3704 } 3705 3706 return 0; 3707 3708 irq_error: 3709 stmmac_free_irq(dev, irq_err, 0); 3710 return ret; 3711 } 3712 3713 static int stmmac_request_irq(struct net_device *dev) 3714 { 3715 struct stmmac_priv *priv = netdev_priv(dev); 3716 int ret; 3717 3718 /* Request the IRQ lines */ 3719 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 3720 ret = stmmac_request_irq_multi_msi(dev); 3721 else 3722 ret = stmmac_request_irq_single(dev); 3723 3724 return ret; 3725 } 3726 3727 /** 3728 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 3729 * @priv: driver private structure 3730 * @mtu: MTU to setup the dma queue and buf with 3731 * Description: Allocate and generate a dma_conf based on the provided MTU. 3732 * Allocate the Tx/Rx DMA queue and init them. 3733 * Return value: 3734 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 3735 */ 3736 static struct stmmac_dma_conf * 3737 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 3738 { 3739 struct stmmac_dma_conf *dma_conf; 3740 int chan, bfsize, ret; 3741 3742 dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); 3743 if (!dma_conf) { 3744 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 3745 __func__); 3746 return ERR_PTR(-ENOMEM); 3747 } 3748 3749 bfsize = stmmac_set_16kib_bfsize(priv, mtu); 3750 if (bfsize < 0) 3751 bfsize = 0; 3752 3753 if (bfsize < BUF_SIZE_16KiB) 3754 bfsize = stmmac_set_bfsize(mtu, 0); 3755 3756 dma_conf->dma_buf_sz = bfsize; 3757 /* Chose the tx/rx size from the already defined one in the 3758 * priv struct. (if defined) 3759 */ 3760 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 3761 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 3762 3763 if (!dma_conf->dma_tx_size) 3764 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 3765 if (!dma_conf->dma_rx_size) 3766 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 3767 3768 /* Earlier check for TBS */ 3769 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 3770 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 3771 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 3772 3773 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 3774 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 3775 } 3776 3777 ret = alloc_dma_desc_resources(priv, dma_conf); 3778 if (ret < 0) { 3779 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 3780 __func__); 3781 goto alloc_error; 3782 } 3783 3784 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 3785 if (ret < 0) { 3786 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 3787 __func__); 3788 goto init_error; 3789 } 3790 3791 return dma_conf; 3792 3793 init_error: 3794 free_dma_desc_resources(priv, dma_conf); 3795 alloc_error: 3796 kfree(dma_conf); 3797 return ERR_PTR(ret); 3798 } 3799 3800 /** 3801 * __stmmac_open - open entry point of the driver 3802 * @dev : pointer to the device structure. 3803 * @dma_conf : structure to take the dma data 3804 * Description: 3805 * This function is the open entry point of the driver. 3806 * Return value: 3807 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3808 * file on failure. 3809 */ 3810 static int __stmmac_open(struct net_device *dev, 3811 struct stmmac_dma_conf *dma_conf) 3812 { 3813 struct stmmac_priv *priv = netdev_priv(dev); 3814 int mode = priv->plat->phy_interface; 3815 u32 chan; 3816 int ret; 3817 3818 ret = pm_runtime_resume_and_get(priv->device); 3819 if (ret < 0) 3820 return ret; 3821 3822 if (priv->hw->pcs != STMMAC_PCS_TBI && 3823 priv->hw->pcs != STMMAC_PCS_RTBI && 3824 (!priv->hw->xpcs || 3825 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73) && 3826 !priv->hw->lynx_pcs) { 3827 ret = stmmac_init_phy(dev); 3828 if (ret) { 3829 netdev_err(priv->dev, 3830 "%s: Cannot attach to PHY (error: %d)\n", 3831 __func__, ret); 3832 goto init_phy_error; 3833 } 3834 } 3835 3836 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 3837 3838 buf_sz = dma_conf->dma_buf_sz; 3839 for (int i = 0; i < MTL_MAX_TX_QUEUES; i++) 3840 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) 3841 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; 3842 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 3843 3844 stmmac_reset_queues_param(priv); 3845 3846 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 3847 priv->plat->serdes_powerup) { 3848 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); 3849 if (ret < 0) { 3850 netdev_err(priv->dev, "%s: Serdes powerup failed\n", 3851 __func__); 3852 goto init_error; 3853 } 3854 } 3855 3856 ret = stmmac_hw_setup(dev, true); 3857 if (ret < 0) { 3858 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 3859 goto init_error; 3860 } 3861 3862 stmmac_init_coalesce(priv); 3863 3864 phylink_start(priv->phylink); 3865 /* We may have called phylink_speed_down before */ 3866 phylink_speed_up(priv->phylink); 3867 3868 ret = stmmac_request_irq(dev); 3869 if (ret) 3870 goto irq_error; 3871 3872 stmmac_enable_all_queues(priv); 3873 netif_tx_start_all_queues(priv->dev); 3874 stmmac_enable_all_dma_irq(priv); 3875 3876 return 0; 3877 3878 irq_error: 3879 phylink_stop(priv->phylink); 3880 3881 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3882 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 3883 3884 stmmac_hw_teardown(dev); 3885 init_error: 3886 phylink_disconnect_phy(priv->phylink); 3887 init_phy_error: 3888 pm_runtime_put(priv->device); 3889 return ret; 3890 } 3891 3892 static int stmmac_open(struct net_device *dev) 3893 { 3894 struct stmmac_priv *priv = netdev_priv(dev); 3895 struct stmmac_dma_conf *dma_conf; 3896 int ret; 3897 3898 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 3899 if (IS_ERR(dma_conf)) 3900 return PTR_ERR(dma_conf); 3901 3902 ret = __stmmac_open(dev, dma_conf); 3903 if (ret) 3904 free_dma_desc_resources(priv, dma_conf); 3905 3906 kfree(dma_conf); 3907 return ret; 3908 } 3909 3910 static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) 3911 { 3912 set_bit(__FPE_REMOVING, &priv->fpe_task_state); 3913 3914 if (priv->fpe_wq) { 3915 destroy_workqueue(priv->fpe_wq); 3916 priv->fpe_wq = NULL; 3917 } 3918 3919 netdev_info(priv->dev, "FPE workqueue stop"); 3920 } 3921 3922 /** 3923 * stmmac_release - close entry point of the driver 3924 * @dev : device pointer. 3925 * Description: 3926 * This is the stop entry point of the driver. 3927 */ 3928 static int stmmac_release(struct net_device *dev) 3929 { 3930 struct stmmac_priv *priv = netdev_priv(dev); 3931 u32 chan; 3932 3933 if (device_may_wakeup(priv->device)) 3934 phylink_speed_down(priv->phylink, false); 3935 /* Stop and disconnect the PHY */ 3936 phylink_stop(priv->phylink); 3937 phylink_disconnect_phy(priv->phylink); 3938 3939 stmmac_disable_all_queues(priv); 3940 3941 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3942 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 3943 3944 netif_tx_disable(dev); 3945 3946 /* Free the IRQ lines */ 3947 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 3948 3949 if (priv->eee_enabled) { 3950 priv->tx_path_in_lpi_mode = false; 3951 del_timer_sync(&priv->eee_ctrl_timer); 3952 } 3953 3954 /* Stop TX/RX DMA and clear the descriptors */ 3955 stmmac_stop_all_dma(priv); 3956 3957 /* Release and free the Rx/Tx resources */ 3958 free_dma_desc_resources(priv, &priv->dma_conf); 3959 3960 /* Disable the MAC Rx/Tx */ 3961 stmmac_mac_set(priv, priv->ioaddr, false); 3962 3963 /* Powerdown Serdes if there is */ 3964 if (priv->plat->serdes_powerdown) 3965 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); 3966 3967 netif_carrier_off(dev); 3968 3969 stmmac_release_ptp(priv); 3970 3971 pm_runtime_put(priv->device); 3972 3973 if (priv->dma_cap.fpesel) 3974 stmmac_fpe_stop_wq(priv); 3975 3976 return 0; 3977 } 3978 3979 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 3980 struct stmmac_tx_queue *tx_q) 3981 { 3982 u16 tag = 0x0, inner_tag = 0x0; 3983 u32 inner_type = 0x0; 3984 struct dma_desc *p; 3985 3986 if (!priv->dma_cap.vlins) 3987 return false; 3988 if (!skb_vlan_tag_present(skb)) 3989 return false; 3990 if (skb->vlan_proto == htons(ETH_P_8021AD)) { 3991 inner_tag = skb_vlan_tag_get(skb); 3992 inner_type = STMMAC_VLAN_INSERT; 3993 } 3994 3995 tag = skb_vlan_tag_get(skb); 3996 3997 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3998 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 3999 else 4000 p = &tx_q->dma_tx[tx_q->cur_tx]; 4001 4002 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) 4003 return false; 4004 4005 stmmac_set_tx_owner(priv, p); 4006 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4007 return true; 4008 } 4009 4010 /** 4011 * stmmac_tso_allocator - close entry point of the driver 4012 * @priv: driver private structure 4013 * @des: buffer start address 4014 * @total_len: total length to fill in descriptors 4015 * @last_segment: condition for the last descriptor 4016 * @queue: TX queue index 4017 * Description: 4018 * This function fills descriptor and request new descriptors according to 4019 * buffer length to fill 4020 */ 4021 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 4022 int total_len, bool last_segment, u32 queue) 4023 { 4024 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4025 struct dma_desc *desc; 4026 u32 buff_size; 4027 int tmp_len; 4028 4029 tmp_len = total_len; 4030 4031 while (tmp_len > 0) { 4032 dma_addr_t curr_addr; 4033 4034 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4035 priv->dma_conf.dma_tx_size); 4036 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4037 4038 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4039 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4040 else 4041 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4042 4043 curr_addr = des + (total_len - tmp_len); 4044 if (priv->dma_cap.addr64 <= 32) 4045 desc->des0 = cpu_to_le32(curr_addr); 4046 else 4047 stmmac_set_desc_addr(priv, desc, curr_addr); 4048 4049 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4050 TSO_MAX_BUFF_SIZE : tmp_len; 4051 4052 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4053 0, 1, 4054 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4055 0, 0); 4056 4057 tmp_len -= TSO_MAX_BUFF_SIZE; 4058 } 4059 } 4060 4061 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4062 { 4063 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4064 int desc_size; 4065 4066 if (likely(priv->extend_desc)) 4067 desc_size = sizeof(struct dma_extended_desc); 4068 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4069 desc_size = sizeof(struct dma_edesc); 4070 else 4071 desc_size = sizeof(struct dma_desc); 4072 4073 /* The own bit must be the latest setting done when prepare the 4074 * descriptor and then barrier is needed to make sure that 4075 * all is coherent before granting the DMA engine. 4076 */ 4077 wmb(); 4078 4079 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); 4080 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 4081 } 4082 4083 /** 4084 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4085 * @skb : the socket buffer 4086 * @dev : device pointer 4087 * Description: this is the transmit function that is called on TSO frames 4088 * (support available on GMAC4 and newer chips). 4089 * Diagram below show the ring programming in case of TSO frames: 4090 * 4091 * First Descriptor 4092 * -------- 4093 * | DES0 |---> buffer1 = L2/L3/L4 header 4094 * | DES1 |---> TCP Payload (can continue on next descr...) 4095 * | DES2 |---> buffer 1 and 2 len 4096 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4097 * -------- 4098 * | 4099 * ... 4100 * | 4101 * -------- 4102 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 4103 * | DES1 | --| 4104 * | DES2 | --> buffer 1 and 2 len 4105 * | DES3 | 4106 * -------- 4107 * 4108 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4109 */ 4110 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4111 { 4112 struct dma_desc *desc, *first, *mss_desc = NULL; 4113 struct stmmac_priv *priv = netdev_priv(dev); 4114 int nfrags = skb_shinfo(skb)->nr_frags; 4115 u32 queue = skb_get_queue_mapping(skb); 4116 unsigned int first_entry, tx_packets; 4117 struct stmmac_txq_stats *txq_stats; 4118 int tmp_pay_len = 0, first_tx; 4119 struct stmmac_tx_queue *tx_q; 4120 bool has_vlan, set_ic; 4121 u8 proto_hdr_len, hdr; 4122 u32 pay_len, mss; 4123 dma_addr_t des; 4124 int i; 4125 4126 tx_q = &priv->dma_conf.tx_queue[queue]; 4127 txq_stats = &priv->xstats.txq_stats[queue]; 4128 first_tx = tx_q->cur_tx; 4129 4130 /* Compute header lengths */ 4131 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 4132 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); 4133 hdr = sizeof(struct udphdr); 4134 } else { 4135 proto_hdr_len = skb_tcp_all_headers(skb); 4136 hdr = tcp_hdrlen(skb); 4137 } 4138 4139 /* Desc availability based on threshold should be enough safe */ 4140 if (unlikely(stmmac_tx_avail(priv, queue) < 4141 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 4142 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4143 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4144 queue)); 4145 /* This is a hard error, log it. */ 4146 netdev_err(priv->dev, 4147 "%s: Tx Ring full when queue awake\n", 4148 __func__); 4149 } 4150 return NETDEV_TX_BUSY; 4151 } 4152 4153 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4154 4155 mss = skb_shinfo(skb)->gso_size; 4156 4157 /* set new MSS value if needed */ 4158 if (mss != tx_q->mss) { 4159 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4160 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4161 else 4162 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4163 4164 stmmac_set_mss(priv, mss_desc, mss); 4165 tx_q->mss = mss; 4166 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4167 priv->dma_conf.dma_tx_size); 4168 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4169 } 4170 4171 if (netif_msg_tx_queued(priv)) { 4172 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 4173 __func__, hdr, proto_hdr_len, pay_len, mss); 4174 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4175 skb->data_len); 4176 } 4177 4178 /* Check if VLAN can be inserted by HW */ 4179 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4180 4181 first_entry = tx_q->cur_tx; 4182 WARN_ON(tx_q->tx_skbuff[first_entry]); 4183 4184 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4185 desc = &tx_q->dma_entx[first_entry].basic; 4186 else 4187 desc = &tx_q->dma_tx[first_entry]; 4188 first = desc; 4189 4190 if (has_vlan) 4191 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4192 4193 /* first descriptor: fill Headers on Buf1 */ 4194 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4195 DMA_TO_DEVICE); 4196 if (dma_mapping_error(priv->device, des)) 4197 goto dma_map_err; 4198 4199 if (priv->dma_cap.addr64 <= 32) { 4200 first->des0 = cpu_to_le32(des); 4201 4202 /* Fill start of payload in buff2 of first descriptor */ 4203 if (pay_len) 4204 first->des1 = cpu_to_le32(des + proto_hdr_len); 4205 4206 /* If needed take extra descriptors to fill the remaining payload */ 4207 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 4208 } else { 4209 stmmac_set_desc_addr(priv, first, des); 4210 tmp_pay_len = pay_len; 4211 des += proto_hdr_len; 4212 pay_len = 0; 4213 } 4214 4215 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 4216 4217 /* In case two or more DMA transmit descriptors are allocated for this 4218 * non-paged SKB data, the DMA buffer address should be saved to 4219 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, 4220 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee 4221 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early 4222 * since the tail areas of the DMA buffer can be accessed by DMA engine 4223 * sooner or later. 4224 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf 4225 * corresponding to the last descriptor, stmmac_tx_clean() will unmap 4226 * this DMA buffer right after the DMA engine completely finishes the 4227 * full buffer transmission. 4228 */ 4229 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4230 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb); 4231 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false; 4232 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4233 4234 /* Prepare fragments */ 4235 for (i = 0; i < nfrags; i++) { 4236 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4237 4238 des = skb_frag_dma_map(priv->device, frag, 0, 4239 skb_frag_size(frag), 4240 DMA_TO_DEVICE); 4241 if (dma_mapping_error(priv->device, des)) 4242 goto dma_map_err; 4243 4244 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4245 (i == nfrags - 1), queue); 4246 4247 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4248 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 4249 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 4250 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4251 } 4252 4253 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 4254 4255 /* Only the last descriptor gets to point to the skb. */ 4256 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4257 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4258 4259 /* Manage tx mitigation */ 4260 tx_packets = (tx_q->cur_tx + 1) - first_tx; 4261 tx_q->tx_count_frames += tx_packets; 4262 4263 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4264 set_ic = true; 4265 else if (!priv->tx_coal_frames[queue]) 4266 set_ic = false; 4267 else if (tx_packets > priv->tx_coal_frames[queue]) 4268 set_ic = true; 4269 else if ((tx_q->tx_count_frames % 4270 priv->tx_coal_frames[queue]) < tx_packets) 4271 set_ic = true; 4272 else 4273 set_ic = false; 4274 4275 if (set_ic) { 4276 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4277 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4278 else 4279 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4280 4281 tx_q->tx_count_frames = 0; 4282 stmmac_set_tx_ic(priv, desc); 4283 } 4284 4285 /* We've used all descriptors we need for this skb, however, 4286 * advance cur_tx so that it references a fresh descriptor. 4287 * ndo_start_xmit will fill this descriptor the next time it's 4288 * called and stmmac_tx_clean may clean up to this descriptor. 4289 */ 4290 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4291 4292 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4293 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4294 __func__); 4295 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4296 } 4297 4298 u64_stats_update_begin(&txq_stats->q_syncp); 4299 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4300 u64_stats_inc(&txq_stats->q.tx_tso_frames); 4301 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); 4302 if (set_ic) 4303 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4304 u64_stats_update_end(&txq_stats->q_syncp); 4305 4306 if (priv->sarc_type) 4307 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4308 4309 skb_tx_timestamp(skb); 4310 4311 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4312 priv->hwts_tx_en)) { 4313 /* declare that device is doing timestamping */ 4314 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4315 stmmac_enable_tx_timestamp(priv, first); 4316 } 4317 4318 /* Complete the first descriptor before granting the DMA */ 4319 stmmac_prepare_tso_tx_desc(priv, first, 1, 4320 proto_hdr_len, 4321 pay_len, 4322 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 4323 hdr / 4, (skb->len - proto_hdr_len)); 4324 4325 /* If context desc is used to change MSS */ 4326 if (mss_desc) { 4327 /* Make sure that first descriptor has been completely 4328 * written, including its own bit. This is because MSS is 4329 * actually before first descriptor, so we need to make 4330 * sure that MSS's own bit is the last thing written. 4331 */ 4332 dma_wmb(); 4333 stmmac_set_tx_owner(priv, mss_desc); 4334 } 4335 4336 if (netif_msg_pktdata(priv)) { 4337 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4338 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4339 tx_q->cur_tx, first, nfrags); 4340 pr_info(">>> frame to be transmitted: "); 4341 print_pkt(skb->data, skb_headlen(skb)); 4342 } 4343 4344 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4345 4346 stmmac_flush_tx_descriptors(priv, queue); 4347 stmmac_tx_timer_arm(priv, queue); 4348 4349 return NETDEV_TX_OK; 4350 4351 dma_map_err: 4352 dev_err(priv->device, "Tx dma map failed\n"); 4353 dev_kfree_skb(skb); 4354 priv->xstats.tx_dropped++; 4355 return NETDEV_TX_OK; 4356 } 4357 4358 /** 4359 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4360 * @skb: socket buffer to check 4361 * 4362 * Check if a packet has an ethertype that will trigger the IP header checks 4363 * and IP/TCP checksum engine of the stmmac core. 4364 * 4365 * Return: true if the ethertype can trigger the checksum engine, false 4366 * otherwise 4367 */ 4368 static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4369 { 4370 int depth = 0; 4371 __be16 proto; 4372 4373 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4374 &depth); 4375 4376 return (depth <= ETH_HLEN) && 4377 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4378 } 4379 4380 /** 4381 * stmmac_xmit - Tx entry point of the driver 4382 * @skb : the socket buffer 4383 * @dev : device pointer 4384 * Description : this is the tx entry point of the driver. 4385 * It programs the chain or the ring and supports oversized frames 4386 * and SG feature. 4387 */ 4388 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4389 { 4390 unsigned int first_entry, tx_packets, enh_desc; 4391 struct stmmac_priv *priv = netdev_priv(dev); 4392 unsigned int nopaged_len = skb_headlen(skb); 4393 int i, csum_insertion = 0, is_jumbo = 0; 4394 u32 queue = skb_get_queue_mapping(skb); 4395 int nfrags = skb_shinfo(skb)->nr_frags; 4396 int gso = skb_shinfo(skb)->gso_type; 4397 struct stmmac_txq_stats *txq_stats; 4398 struct dma_edesc *tbs_desc = NULL; 4399 struct dma_desc *desc, *first; 4400 struct stmmac_tx_queue *tx_q; 4401 bool has_vlan, set_ic; 4402 int entry, first_tx; 4403 dma_addr_t des; 4404 4405 tx_q = &priv->dma_conf.tx_queue[queue]; 4406 txq_stats = &priv->xstats.txq_stats[queue]; 4407 first_tx = tx_q->cur_tx; 4408 4409 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4410 stmmac_disable_eee_mode(priv); 4411 4412 /* Manage oversized TCP frames for GMAC4 device */ 4413 if (skb_is_gso(skb) && priv->tso) { 4414 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 4415 return stmmac_tso_xmit(skb, dev); 4416 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) 4417 return stmmac_tso_xmit(skb, dev); 4418 } 4419 4420 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4421 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4422 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4423 queue)); 4424 /* This is a hard error, log it. */ 4425 netdev_err(priv->dev, 4426 "%s: Tx Ring full when queue awake\n", 4427 __func__); 4428 } 4429 return NETDEV_TX_BUSY; 4430 } 4431 4432 /* Check if VLAN can be inserted by HW */ 4433 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4434 4435 entry = tx_q->cur_tx; 4436 first_entry = entry; 4437 WARN_ON(tx_q->tx_skbuff[first_entry]); 4438 4439 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 4440 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 4441 * queues. In that case, checksum offloading for those queues that don't 4442 * support tx coe needs to fallback to software checksum calculation. 4443 * 4444 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 4445 * also have to be checksummed in software. 4446 */ 4447 if (csum_insertion && 4448 (priv->plat->tx_queues_cfg[queue].coe_unsupported || 4449 !stmmac_has_ip_ethertype(skb))) { 4450 if (unlikely(skb_checksum_help(skb))) 4451 goto dma_map_err; 4452 csum_insertion = !csum_insertion; 4453 } 4454 4455 if (likely(priv->extend_desc)) 4456 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4457 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4458 desc = &tx_q->dma_entx[entry].basic; 4459 else 4460 desc = tx_q->dma_tx + entry; 4461 4462 first = desc; 4463 4464 if (has_vlan) 4465 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4466 4467 enh_desc = priv->plat->enh_desc; 4468 /* To program the descriptors according to the size of the frame */ 4469 if (enh_desc) 4470 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4471 4472 if (unlikely(is_jumbo)) { 4473 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4474 if (unlikely(entry < 0) && (entry != -EINVAL)) 4475 goto dma_map_err; 4476 } 4477 4478 for (i = 0; i < nfrags; i++) { 4479 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4480 int len = skb_frag_size(frag); 4481 bool last_segment = (i == (nfrags - 1)); 4482 4483 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4484 WARN_ON(tx_q->tx_skbuff[entry]); 4485 4486 if (likely(priv->extend_desc)) 4487 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4488 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4489 desc = &tx_q->dma_entx[entry].basic; 4490 else 4491 desc = tx_q->dma_tx + entry; 4492 4493 des = skb_frag_dma_map(priv->device, frag, 0, len, 4494 DMA_TO_DEVICE); 4495 if (dma_mapping_error(priv->device, des)) 4496 goto dma_map_err; /* should reuse desc w/o issues */ 4497 4498 tx_q->tx_skbuff_dma[entry].buf = des; 4499 4500 stmmac_set_desc_addr(priv, desc, des); 4501 4502 tx_q->tx_skbuff_dma[entry].map_as_page = true; 4503 tx_q->tx_skbuff_dma[entry].len = len; 4504 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 4505 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4506 4507 /* Prepare the descriptor and set the own bit too */ 4508 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 4509 priv->mode, 1, last_segment, skb->len); 4510 } 4511 4512 /* Only the last descriptor gets to point to the skb. */ 4513 tx_q->tx_skbuff[entry] = skb; 4514 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4515 4516 /* According to the coalesce parameter the IC bit for the latest 4517 * segment is reset and the timer re-started to clean the tx status. 4518 * This approach takes care about the fragments: desc is the first 4519 * element in case of no SG. 4520 */ 4521 tx_packets = (entry + 1) - first_tx; 4522 tx_q->tx_count_frames += tx_packets; 4523 4524 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4525 set_ic = true; 4526 else if (!priv->tx_coal_frames[queue]) 4527 set_ic = false; 4528 else if (tx_packets > priv->tx_coal_frames[queue]) 4529 set_ic = true; 4530 else if ((tx_q->tx_count_frames % 4531 priv->tx_coal_frames[queue]) < tx_packets) 4532 set_ic = true; 4533 else 4534 set_ic = false; 4535 4536 if (set_ic) { 4537 if (likely(priv->extend_desc)) 4538 desc = &tx_q->dma_etx[entry].basic; 4539 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4540 desc = &tx_q->dma_entx[entry].basic; 4541 else 4542 desc = &tx_q->dma_tx[entry]; 4543 4544 tx_q->tx_count_frames = 0; 4545 stmmac_set_tx_ic(priv, desc); 4546 } 4547 4548 /* We've used all descriptors we need for this skb, however, 4549 * advance cur_tx so that it references a fresh descriptor. 4550 * ndo_start_xmit will fill this descriptor the next time it's 4551 * called and stmmac_tx_clean may clean up to this descriptor. 4552 */ 4553 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4554 tx_q->cur_tx = entry; 4555 4556 if (netif_msg_pktdata(priv)) { 4557 netdev_dbg(priv->dev, 4558 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4559 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4560 entry, first, nfrags); 4561 4562 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4563 print_pkt(skb->data, skb->len); 4564 } 4565 4566 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4567 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4568 __func__); 4569 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4570 } 4571 4572 u64_stats_update_begin(&txq_stats->q_syncp); 4573 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4574 if (set_ic) 4575 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4576 u64_stats_update_end(&txq_stats->q_syncp); 4577 4578 if (priv->sarc_type) 4579 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4580 4581 skb_tx_timestamp(skb); 4582 4583 /* Ready to fill the first descriptor and set the OWN bit w/o any 4584 * problems because all the descriptors are actually ready to be 4585 * passed to the DMA engine. 4586 */ 4587 if (likely(!is_jumbo)) { 4588 bool last_segment = (nfrags == 0); 4589 4590 des = dma_map_single(priv->device, skb->data, 4591 nopaged_len, DMA_TO_DEVICE); 4592 if (dma_mapping_error(priv->device, des)) 4593 goto dma_map_err; 4594 4595 tx_q->tx_skbuff_dma[first_entry].buf = des; 4596 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4597 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4598 4599 stmmac_set_desc_addr(priv, first, des); 4600 4601 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 4602 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 4603 4604 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4605 priv->hwts_tx_en)) { 4606 /* declare that device is doing timestamping */ 4607 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4608 stmmac_enable_tx_timestamp(priv, first); 4609 } 4610 4611 /* Prepare the first descriptor setting the OWN bit too */ 4612 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 4613 csum_insertion, priv->mode, 0, last_segment, 4614 skb->len); 4615 } 4616 4617 if (tx_q->tbs & STMMAC_TBS_EN) { 4618 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4619 4620 tbs_desc = &tx_q->dma_entx[first_entry]; 4621 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); 4622 } 4623 4624 stmmac_set_tx_owner(priv, first); 4625 4626 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4627 4628 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4629 4630 stmmac_flush_tx_descriptors(priv, queue); 4631 stmmac_tx_timer_arm(priv, queue); 4632 4633 return NETDEV_TX_OK; 4634 4635 dma_map_err: 4636 netdev_err(priv->dev, "Tx DMA map failed\n"); 4637 dev_kfree_skb(skb); 4638 priv->xstats.tx_dropped++; 4639 return NETDEV_TX_OK; 4640 } 4641 4642 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 4643 { 4644 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 4645 __be16 vlan_proto = veth->h_vlan_proto; 4646 u16 vlanid; 4647 4648 if ((vlan_proto == htons(ETH_P_8021Q) && 4649 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 4650 (vlan_proto == htons(ETH_P_8021AD) && 4651 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 4652 /* pop the vlan tag */ 4653 vlanid = ntohs(veth->h_vlan_TCI); 4654 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 4655 skb_pull(skb, VLAN_HLEN); 4656 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 4657 } 4658 } 4659 4660 /** 4661 * stmmac_rx_refill - refill used skb preallocated buffers 4662 * @priv: driver private structure 4663 * @queue: RX queue index 4664 * Description : this is to reallocate the skb for the reception process 4665 * that is based on zero-copy. 4666 */ 4667 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 4668 { 4669 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4670 int dirty = stmmac_rx_dirty(priv, queue); 4671 unsigned int entry = rx_q->dirty_rx; 4672 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 4673 4674 if (priv->dma_cap.host_dma_width <= 32) 4675 gfp |= GFP_DMA32; 4676 4677 while (dirty-- > 0) { 4678 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4679 struct dma_desc *p; 4680 bool use_rx_wd; 4681 4682 if (priv->extend_desc) 4683 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4684 else 4685 p = rx_q->dma_rx + entry; 4686 4687 if (!buf->page) { 4688 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4689 if (!buf->page) 4690 break; 4691 } 4692 4693 if (priv->sph && !buf->sec_page) { 4694 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4695 if (!buf->sec_page) 4696 break; 4697 4698 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 4699 } 4700 4701 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 4702 4703 stmmac_set_desc_addr(priv, p, buf->addr); 4704 if (priv->sph) 4705 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 4706 else 4707 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 4708 stmmac_refill_desc3(priv, rx_q, p); 4709 4710 rx_q->rx_count_frames++; 4711 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4712 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4713 rx_q->rx_count_frames = 0; 4714 4715 use_rx_wd = !priv->rx_coal_frames[queue]; 4716 use_rx_wd |= rx_q->rx_count_frames > 0; 4717 if (!priv->use_riwt) 4718 use_rx_wd = false; 4719 4720 dma_wmb(); 4721 stmmac_set_rx_owner(priv, p, use_rx_wd); 4722 4723 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 4724 } 4725 rx_q->dirty_rx = entry; 4726 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4727 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4728 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4729 } 4730 4731 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 4732 struct dma_desc *p, 4733 int status, unsigned int len) 4734 { 4735 unsigned int plen = 0, hlen = 0; 4736 int coe = priv->hw->rx_csum; 4737 4738 /* Not first descriptor, buffer is always zero */ 4739 if (priv->sph && len) 4740 return 0; 4741 4742 /* First descriptor, get split header length */ 4743 stmmac_get_rx_header_len(priv, p, &hlen); 4744 if (priv->sph && hlen) { 4745 priv->xstats.rx_split_hdr_pkt_n++; 4746 return hlen; 4747 } 4748 4749 /* First descriptor, not last descriptor and not split header */ 4750 if (status & rx_not_ls) 4751 return priv->dma_conf.dma_buf_sz; 4752 4753 plen = stmmac_get_rx_frame_len(priv, p, coe); 4754 4755 /* First descriptor and last descriptor and not split header */ 4756 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 4757 } 4758 4759 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 4760 struct dma_desc *p, 4761 int status, unsigned int len) 4762 { 4763 int coe = priv->hw->rx_csum; 4764 unsigned int plen = 0; 4765 4766 /* Not split header, buffer is not available */ 4767 if (!priv->sph) 4768 return 0; 4769 4770 /* Not last descriptor */ 4771 if (status & rx_not_ls) 4772 return priv->dma_conf.dma_buf_sz; 4773 4774 plen = stmmac_get_rx_frame_len(priv, p, coe); 4775 4776 /* Last descriptor */ 4777 return plen - len; 4778 } 4779 4780 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 4781 struct xdp_frame *xdpf, bool dma_map) 4782 { 4783 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 4784 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4785 unsigned int entry = tx_q->cur_tx; 4786 struct dma_desc *tx_desc; 4787 dma_addr_t dma_addr; 4788 bool set_ic; 4789 4790 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 4791 return STMMAC_XDP_CONSUMED; 4792 4793 if (likely(priv->extend_desc)) 4794 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4795 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4796 tx_desc = &tx_q->dma_entx[entry].basic; 4797 else 4798 tx_desc = tx_q->dma_tx + entry; 4799 4800 if (dma_map) { 4801 dma_addr = dma_map_single(priv->device, xdpf->data, 4802 xdpf->len, DMA_TO_DEVICE); 4803 if (dma_mapping_error(priv->device, dma_addr)) 4804 return STMMAC_XDP_CONSUMED; 4805 4806 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; 4807 } else { 4808 struct page *page = virt_to_page(xdpf->data); 4809 4810 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 4811 xdpf->headroom; 4812 dma_sync_single_for_device(priv->device, dma_addr, 4813 xdpf->len, DMA_BIDIRECTIONAL); 4814 4815 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; 4816 } 4817 4818 tx_q->tx_skbuff_dma[entry].buf = dma_addr; 4819 tx_q->tx_skbuff_dma[entry].map_as_page = false; 4820 tx_q->tx_skbuff_dma[entry].len = xdpf->len; 4821 tx_q->tx_skbuff_dma[entry].last_segment = true; 4822 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 4823 4824 tx_q->xdpf[entry] = xdpf; 4825 4826 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 4827 4828 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 4829 true, priv->mode, true, true, 4830 xdpf->len); 4831 4832 tx_q->tx_count_frames++; 4833 4834 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 4835 set_ic = true; 4836 else 4837 set_ic = false; 4838 4839 if (set_ic) { 4840 tx_q->tx_count_frames = 0; 4841 stmmac_set_tx_ic(priv, tx_desc); 4842 u64_stats_update_begin(&txq_stats->q_syncp); 4843 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4844 u64_stats_update_end(&txq_stats->q_syncp); 4845 } 4846 4847 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4848 4849 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4850 tx_q->cur_tx = entry; 4851 4852 return STMMAC_XDP_TX; 4853 } 4854 4855 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 4856 int cpu) 4857 { 4858 int index = cpu; 4859 4860 if (unlikely(index < 0)) 4861 index = 0; 4862 4863 while (index >= priv->plat->tx_queues_to_use) 4864 index -= priv->plat->tx_queues_to_use; 4865 4866 return index; 4867 } 4868 4869 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 4870 struct xdp_buff *xdp) 4871 { 4872 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 4873 int cpu = smp_processor_id(); 4874 struct netdev_queue *nq; 4875 int queue; 4876 int res; 4877 4878 if (unlikely(!xdpf)) 4879 return STMMAC_XDP_CONSUMED; 4880 4881 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4882 nq = netdev_get_tx_queue(priv->dev, queue); 4883 4884 __netif_tx_lock(nq, cpu); 4885 /* Avoids TX time-out as we are sharing with slow path */ 4886 txq_trans_cond_update(nq); 4887 4888 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); 4889 if (res == STMMAC_XDP_TX) 4890 stmmac_flush_tx_descriptors(priv, queue); 4891 4892 __netif_tx_unlock(nq); 4893 4894 return res; 4895 } 4896 4897 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 4898 struct bpf_prog *prog, 4899 struct xdp_buff *xdp) 4900 { 4901 u32 act; 4902 int res; 4903 4904 act = bpf_prog_run_xdp(prog, xdp); 4905 switch (act) { 4906 case XDP_PASS: 4907 res = STMMAC_XDP_PASS; 4908 break; 4909 case XDP_TX: 4910 res = stmmac_xdp_xmit_back(priv, xdp); 4911 break; 4912 case XDP_REDIRECT: 4913 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 4914 res = STMMAC_XDP_CONSUMED; 4915 else 4916 res = STMMAC_XDP_REDIRECT; 4917 break; 4918 default: 4919 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 4920 fallthrough; 4921 case XDP_ABORTED: 4922 trace_xdp_exception(priv->dev, prog, act); 4923 fallthrough; 4924 case XDP_DROP: 4925 res = STMMAC_XDP_CONSUMED; 4926 break; 4927 } 4928 4929 return res; 4930 } 4931 4932 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 4933 struct xdp_buff *xdp) 4934 { 4935 struct bpf_prog *prog; 4936 int res; 4937 4938 prog = READ_ONCE(priv->xdp_prog); 4939 if (!prog) { 4940 res = STMMAC_XDP_PASS; 4941 goto out; 4942 } 4943 4944 res = __stmmac_xdp_run_prog(priv, prog, xdp); 4945 out: 4946 return ERR_PTR(-res); 4947 } 4948 4949 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 4950 int xdp_status) 4951 { 4952 int cpu = smp_processor_id(); 4953 int queue; 4954 4955 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4956 4957 if (xdp_status & STMMAC_XDP_TX) 4958 stmmac_tx_timer_arm(priv, queue); 4959 4960 if (xdp_status & STMMAC_XDP_REDIRECT) 4961 xdp_do_flush(); 4962 } 4963 4964 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 4965 struct xdp_buff *xdp) 4966 { 4967 unsigned int metasize = xdp->data - xdp->data_meta; 4968 unsigned int datasize = xdp->data_end - xdp->data; 4969 struct sk_buff *skb; 4970 4971 skb = __napi_alloc_skb(&ch->rxtx_napi, 4972 xdp->data_end - xdp->data_hard_start, 4973 GFP_ATOMIC | __GFP_NOWARN); 4974 if (unlikely(!skb)) 4975 return NULL; 4976 4977 skb_reserve(skb, xdp->data - xdp->data_hard_start); 4978 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 4979 if (metasize) 4980 skb_metadata_set(skb, metasize); 4981 4982 return skb; 4983 } 4984 4985 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 4986 struct dma_desc *p, struct dma_desc *np, 4987 struct xdp_buff *xdp) 4988 { 4989 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 4990 struct stmmac_channel *ch = &priv->channel[queue]; 4991 unsigned int len = xdp->data_end - xdp->data; 4992 enum pkt_hash_types hash_type; 4993 int coe = priv->hw->rx_csum; 4994 struct sk_buff *skb; 4995 u32 hash; 4996 4997 skb = stmmac_construct_skb_zc(ch, xdp); 4998 if (!skb) { 4999 priv->xstats.rx_dropped++; 5000 return; 5001 } 5002 5003 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5004 stmmac_rx_vlan(priv->dev, skb); 5005 skb->protocol = eth_type_trans(skb, priv->dev); 5006 5007 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5008 skb_checksum_none_assert(skb); 5009 else 5010 skb->ip_summed = CHECKSUM_UNNECESSARY; 5011 5012 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5013 skb_set_hash(skb, hash, hash_type); 5014 5015 skb_record_rx_queue(skb, queue); 5016 napi_gro_receive(&ch->rxtx_napi, skb); 5017 5018 u64_stats_update_begin(&rxq_stats->napi_syncp); 5019 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); 5020 u64_stats_add(&rxq_stats->napi.rx_bytes, len); 5021 u64_stats_update_end(&rxq_stats->napi_syncp); 5022 } 5023 5024 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5025 { 5026 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5027 unsigned int entry = rx_q->dirty_rx; 5028 struct dma_desc *rx_desc = NULL; 5029 bool ret = true; 5030 5031 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5032 5033 while (budget-- > 0 && entry != rx_q->cur_rx) { 5034 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5035 dma_addr_t dma_addr; 5036 bool use_rx_wd; 5037 5038 if (!buf->xdp) { 5039 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5040 if (!buf->xdp) { 5041 ret = false; 5042 break; 5043 } 5044 } 5045 5046 if (priv->extend_desc) 5047 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); 5048 else 5049 rx_desc = rx_q->dma_rx + entry; 5050 5051 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5052 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5053 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5054 stmmac_refill_desc3(priv, rx_q, rx_desc); 5055 5056 rx_q->rx_count_frames++; 5057 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5058 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5059 rx_q->rx_count_frames = 0; 5060 5061 use_rx_wd = !priv->rx_coal_frames[queue]; 5062 use_rx_wd |= rx_q->rx_count_frames > 0; 5063 if (!priv->use_riwt) 5064 use_rx_wd = false; 5065 5066 dma_wmb(); 5067 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5068 5069 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 5070 } 5071 5072 if (rx_desc) { 5073 rx_q->dirty_rx = entry; 5074 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 5075 (rx_q->dirty_rx * sizeof(struct dma_desc)); 5076 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 5077 } 5078 5079 return ret; 5080 } 5081 5082 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5083 { 5084 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5085 * to represent incoming packet, whereas cb field in the same structure 5086 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5087 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5088 */ 5089 return (struct stmmac_xdp_buff *)xdp; 5090 } 5091 5092 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5093 { 5094 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5095 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5096 unsigned int count = 0, error = 0, len = 0; 5097 int dirty = stmmac_rx_dirty(priv, queue); 5098 unsigned int next_entry = rx_q->cur_rx; 5099 u32 rx_errors = 0, rx_dropped = 0; 5100 unsigned int desc_size; 5101 struct bpf_prog *prog; 5102 bool failure = false; 5103 int xdp_status = 0; 5104 int status = 0; 5105 5106 if (netif_msg_rx_status(priv)) { 5107 void *rx_head; 5108 5109 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5110 if (priv->extend_desc) { 5111 rx_head = (void *)rx_q->dma_erx; 5112 desc_size = sizeof(struct dma_extended_desc); 5113 } else { 5114 rx_head = (void *)rx_q->dma_rx; 5115 desc_size = sizeof(struct dma_desc); 5116 } 5117 5118 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5119 rx_q->dma_rx_phy, desc_size); 5120 } 5121 while (count < limit) { 5122 struct stmmac_rx_buffer *buf; 5123 struct stmmac_xdp_buff *ctx; 5124 unsigned int buf1_len = 0; 5125 struct dma_desc *np, *p; 5126 int entry; 5127 int res; 5128 5129 if (!count && rx_q->state_saved) { 5130 error = rx_q->state.error; 5131 len = rx_q->state.len; 5132 } else { 5133 rx_q->state_saved = false; 5134 error = 0; 5135 len = 0; 5136 } 5137 5138 if (count >= limit) 5139 break; 5140 5141 read_again: 5142 buf1_len = 0; 5143 entry = next_entry; 5144 buf = &rx_q->buf_pool[entry]; 5145 5146 if (dirty >= STMMAC_RX_FILL_BATCH) { 5147 failure = failure || 5148 !stmmac_rx_refill_zc(priv, queue, dirty); 5149 dirty = 0; 5150 } 5151 5152 if (priv->extend_desc) 5153 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5154 else 5155 p = rx_q->dma_rx + entry; 5156 5157 /* read the status of the incoming frame */ 5158 status = stmmac_rx_status(priv, &priv->xstats, p); 5159 /* check if managed by the DMA otherwise go ahead */ 5160 if (unlikely(status & dma_own)) 5161 break; 5162 5163 /* Prefetch the next RX descriptor */ 5164 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5165 priv->dma_conf.dma_rx_size); 5166 next_entry = rx_q->cur_rx; 5167 5168 if (priv->extend_desc) 5169 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5170 else 5171 np = rx_q->dma_rx + next_entry; 5172 5173 prefetch(np); 5174 5175 /* Ensure a valid XSK buffer before proceed */ 5176 if (!buf->xdp) 5177 break; 5178 5179 if (priv->extend_desc) 5180 stmmac_rx_extended_status(priv, &priv->xstats, 5181 rx_q->dma_erx + entry); 5182 if (unlikely(status == discard_frame)) { 5183 xsk_buff_free(buf->xdp); 5184 buf->xdp = NULL; 5185 dirty++; 5186 error = 1; 5187 if (!priv->hwts_rx_en) 5188 rx_errors++; 5189 } 5190 5191 if (unlikely(error && (status & rx_not_ls))) 5192 goto read_again; 5193 if (unlikely(error)) { 5194 count++; 5195 continue; 5196 } 5197 5198 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5199 if (likely(status & rx_not_ls)) { 5200 xsk_buff_free(buf->xdp); 5201 buf->xdp = NULL; 5202 dirty++; 5203 count++; 5204 goto read_again; 5205 } 5206 5207 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5208 ctx->priv = priv; 5209 ctx->desc = p; 5210 ctx->ndesc = np; 5211 5212 /* XDP ZC Frame only support primary buffers for now */ 5213 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5214 len += buf1_len; 5215 5216 /* ACS is disabled; strip manually. */ 5217 if (likely(!(status & rx_not_ls))) { 5218 buf1_len -= ETH_FCS_LEN; 5219 len -= ETH_FCS_LEN; 5220 } 5221 5222 /* RX buffer is good and fit into a XSK pool buffer */ 5223 buf->xdp->data_end = buf->xdp->data + buf1_len; 5224 xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); 5225 5226 prog = READ_ONCE(priv->xdp_prog); 5227 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5228 5229 switch (res) { 5230 case STMMAC_XDP_PASS: 5231 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5232 xsk_buff_free(buf->xdp); 5233 break; 5234 case STMMAC_XDP_CONSUMED: 5235 xsk_buff_free(buf->xdp); 5236 rx_dropped++; 5237 break; 5238 case STMMAC_XDP_TX: 5239 case STMMAC_XDP_REDIRECT: 5240 xdp_status |= res; 5241 break; 5242 } 5243 5244 buf->xdp = NULL; 5245 dirty++; 5246 count++; 5247 } 5248 5249 if (status & rx_not_ls) { 5250 rx_q->state_saved = true; 5251 rx_q->state.error = error; 5252 rx_q->state.len = len; 5253 } 5254 5255 stmmac_finalize_xdp_rx(priv, xdp_status); 5256 5257 u64_stats_update_begin(&rxq_stats->napi_syncp); 5258 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5259 u64_stats_update_end(&rxq_stats->napi_syncp); 5260 5261 priv->xstats.rx_dropped += rx_dropped; 5262 priv->xstats.rx_errors += rx_errors; 5263 5264 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5265 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5266 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5267 else 5268 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5269 5270 return (int)count; 5271 } 5272 5273 return failure ? limit : (int)count; 5274 } 5275 5276 /** 5277 * stmmac_rx - manage the receive process 5278 * @priv: driver private structure 5279 * @limit: napi bugget 5280 * @queue: RX queue index. 5281 * Description : this the function called by the napi poll method. 5282 * It gets all the frames inside the ring. 5283 */ 5284 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5285 { 5286 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5287 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5288 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5289 struct stmmac_channel *ch = &priv->channel[queue]; 5290 unsigned int count = 0, error = 0, len = 0; 5291 int status = 0, coe = priv->hw->rx_csum; 5292 unsigned int next_entry = rx_q->cur_rx; 5293 enum dma_data_direction dma_dir; 5294 unsigned int desc_size; 5295 struct sk_buff *skb = NULL; 5296 struct stmmac_xdp_buff ctx; 5297 int xdp_status = 0; 5298 int buf_sz; 5299 5300 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5301 buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5302 limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); 5303 5304 if (netif_msg_rx_status(priv)) { 5305 void *rx_head; 5306 5307 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5308 if (priv->extend_desc) { 5309 rx_head = (void *)rx_q->dma_erx; 5310 desc_size = sizeof(struct dma_extended_desc); 5311 } else { 5312 rx_head = (void *)rx_q->dma_rx; 5313 desc_size = sizeof(struct dma_desc); 5314 } 5315 5316 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5317 rx_q->dma_rx_phy, desc_size); 5318 } 5319 while (count < limit) { 5320 unsigned int buf1_len = 0, buf2_len = 0; 5321 enum pkt_hash_types hash_type; 5322 struct stmmac_rx_buffer *buf; 5323 struct dma_desc *np, *p; 5324 int entry; 5325 u32 hash; 5326 5327 if (!count && rx_q->state_saved) { 5328 skb = rx_q->state.skb; 5329 error = rx_q->state.error; 5330 len = rx_q->state.len; 5331 } else { 5332 rx_q->state_saved = false; 5333 skb = NULL; 5334 error = 0; 5335 len = 0; 5336 } 5337 5338 read_again: 5339 if (count >= limit) 5340 break; 5341 5342 buf1_len = 0; 5343 buf2_len = 0; 5344 entry = next_entry; 5345 buf = &rx_q->buf_pool[entry]; 5346 5347 if (priv->extend_desc) 5348 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5349 else 5350 p = rx_q->dma_rx + entry; 5351 5352 /* read the status of the incoming frame */ 5353 status = stmmac_rx_status(priv, &priv->xstats, p); 5354 /* check if managed by the DMA otherwise go ahead */ 5355 if (unlikely(status & dma_own)) 5356 break; 5357 5358 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5359 priv->dma_conf.dma_rx_size); 5360 next_entry = rx_q->cur_rx; 5361 5362 if (priv->extend_desc) 5363 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5364 else 5365 np = rx_q->dma_rx + next_entry; 5366 5367 prefetch(np); 5368 5369 if (priv->extend_desc) 5370 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5371 if (unlikely(status == discard_frame)) { 5372 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5373 buf->page = NULL; 5374 error = 1; 5375 if (!priv->hwts_rx_en) 5376 rx_errors++; 5377 } 5378 5379 if (unlikely(error && (status & rx_not_ls))) 5380 goto read_again; 5381 if (unlikely(error)) { 5382 dev_kfree_skb(skb); 5383 skb = NULL; 5384 count++; 5385 continue; 5386 } 5387 5388 /* Buffer is good. Go on. */ 5389 5390 prefetch(page_address(buf->page) + buf->page_offset); 5391 if (buf->sec_page) 5392 prefetch(page_address(buf->sec_page)); 5393 5394 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5395 len += buf1_len; 5396 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5397 len += buf2_len; 5398 5399 /* ACS is disabled; strip manually. */ 5400 if (likely(!(status & rx_not_ls))) { 5401 if (buf2_len) { 5402 buf2_len -= ETH_FCS_LEN; 5403 len -= ETH_FCS_LEN; 5404 } else if (buf1_len) { 5405 buf1_len -= ETH_FCS_LEN; 5406 len -= ETH_FCS_LEN; 5407 } 5408 } 5409 5410 if (!skb) { 5411 unsigned int pre_len, sync_len; 5412 5413 dma_sync_single_for_cpu(priv->device, buf->addr, 5414 buf1_len, dma_dir); 5415 5416 xdp_init_buff(&ctx.xdp, buf_sz, &rx_q->xdp_rxq); 5417 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5418 buf->page_offset, buf1_len, true); 5419 5420 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5421 buf->page_offset; 5422 5423 ctx.priv = priv; 5424 ctx.desc = p; 5425 ctx.ndesc = np; 5426 5427 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5428 /* Due xdp_adjust_tail: DMA sync for_device 5429 * cover max len CPU touch 5430 */ 5431 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5432 buf->page_offset; 5433 sync_len = max(sync_len, pre_len); 5434 5435 /* For Not XDP_PASS verdict */ 5436 if (IS_ERR(skb)) { 5437 unsigned int xdp_res = -PTR_ERR(skb); 5438 5439 if (xdp_res & STMMAC_XDP_CONSUMED) { 5440 page_pool_put_page(rx_q->page_pool, 5441 virt_to_head_page(ctx.xdp.data), 5442 sync_len, true); 5443 buf->page = NULL; 5444 rx_dropped++; 5445 5446 /* Clear skb as it was set as 5447 * status by XDP program. 5448 */ 5449 skb = NULL; 5450 5451 if (unlikely((status & rx_not_ls))) 5452 goto read_again; 5453 5454 count++; 5455 continue; 5456 } else if (xdp_res & (STMMAC_XDP_TX | 5457 STMMAC_XDP_REDIRECT)) { 5458 xdp_status |= xdp_res; 5459 buf->page = NULL; 5460 skb = NULL; 5461 count++; 5462 continue; 5463 } 5464 } 5465 } 5466 5467 if (!skb) { 5468 /* XDP program may expand or reduce tail */ 5469 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5470 5471 skb = napi_alloc_skb(&ch->rx_napi, buf1_len); 5472 if (!skb) { 5473 rx_dropped++; 5474 count++; 5475 goto drain_data; 5476 } 5477 5478 /* XDP program may adjust header */ 5479 skb_copy_to_linear_data(skb, ctx.xdp.data, buf1_len); 5480 skb_put(skb, buf1_len); 5481 5482 /* Data payload copied into SKB, page ready for recycle */ 5483 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5484 buf->page = NULL; 5485 } else if (buf1_len) { 5486 dma_sync_single_for_cpu(priv->device, buf->addr, 5487 buf1_len, dma_dir); 5488 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5489 buf->page, buf->page_offset, buf1_len, 5490 priv->dma_conf.dma_buf_sz); 5491 5492 /* Data payload appended into SKB */ 5493 skb_mark_for_recycle(skb); 5494 buf->page = NULL; 5495 } 5496 5497 if (buf2_len) { 5498 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5499 buf2_len, dma_dir); 5500 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5501 buf->sec_page, 0, buf2_len, 5502 priv->dma_conf.dma_buf_sz); 5503 5504 /* Data payload appended into SKB */ 5505 skb_mark_for_recycle(skb); 5506 buf->sec_page = NULL; 5507 } 5508 5509 drain_data: 5510 if (likely(status & rx_not_ls)) 5511 goto read_again; 5512 if (!skb) 5513 continue; 5514 5515 /* Got entire packet into SKB. Finish it. */ 5516 5517 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5518 stmmac_rx_vlan(priv->dev, skb); 5519 skb->protocol = eth_type_trans(skb, priv->dev); 5520 5521 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5522 skb_checksum_none_assert(skb); 5523 else 5524 skb->ip_summed = CHECKSUM_UNNECESSARY; 5525 5526 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5527 skb_set_hash(skb, hash, hash_type); 5528 5529 skb_record_rx_queue(skb, queue); 5530 napi_gro_receive(&ch->rx_napi, skb); 5531 skb = NULL; 5532 5533 rx_packets++; 5534 rx_bytes += len; 5535 count++; 5536 } 5537 5538 if (status & rx_not_ls || skb) { 5539 rx_q->state_saved = true; 5540 rx_q->state.skb = skb; 5541 rx_q->state.error = error; 5542 rx_q->state.len = len; 5543 } 5544 5545 stmmac_finalize_xdp_rx(priv, xdp_status); 5546 5547 stmmac_rx_refill(priv, queue); 5548 5549 u64_stats_update_begin(&rxq_stats->napi_syncp); 5550 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); 5551 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); 5552 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5553 u64_stats_update_end(&rxq_stats->napi_syncp); 5554 5555 priv->xstats.rx_dropped += rx_dropped; 5556 priv->xstats.rx_errors += rx_errors; 5557 5558 return count; 5559 } 5560 5561 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5562 { 5563 struct stmmac_channel *ch = 5564 container_of(napi, struct stmmac_channel, rx_napi); 5565 struct stmmac_priv *priv = ch->priv_data; 5566 struct stmmac_rxq_stats *rxq_stats; 5567 u32 chan = ch->index; 5568 int work_done; 5569 5570 rxq_stats = &priv->xstats.rxq_stats[chan]; 5571 u64_stats_update_begin(&rxq_stats->napi_syncp); 5572 u64_stats_inc(&rxq_stats->napi.poll); 5573 u64_stats_update_end(&rxq_stats->napi_syncp); 5574 5575 work_done = stmmac_rx(priv, budget, chan); 5576 if (work_done < budget && napi_complete_done(napi, work_done)) { 5577 unsigned long flags; 5578 5579 spin_lock_irqsave(&ch->lock, flags); 5580 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5581 spin_unlock_irqrestore(&ch->lock, flags); 5582 } 5583 5584 return work_done; 5585 } 5586 5587 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5588 { 5589 struct stmmac_channel *ch = 5590 container_of(napi, struct stmmac_channel, tx_napi); 5591 struct stmmac_priv *priv = ch->priv_data; 5592 struct stmmac_txq_stats *txq_stats; 5593 u32 chan = ch->index; 5594 int work_done; 5595 5596 txq_stats = &priv->xstats.txq_stats[chan]; 5597 u64_stats_update_begin(&txq_stats->napi_syncp); 5598 u64_stats_inc(&txq_stats->napi.poll); 5599 u64_stats_update_end(&txq_stats->napi_syncp); 5600 5601 work_done = stmmac_tx_clean(priv, budget, chan); 5602 work_done = min(work_done, budget); 5603 5604 if (work_done < budget && napi_complete_done(napi, work_done)) { 5605 unsigned long flags; 5606 5607 spin_lock_irqsave(&ch->lock, flags); 5608 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5609 spin_unlock_irqrestore(&ch->lock, flags); 5610 } 5611 5612 return work_done; 5613 } 5614 5615 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 5616 { 5617 struct stmmac_channel *ch = 5618 container_of(napi, struct stmmac_channel, rxtx_napi); 5619 struct stmmac_priv *priv = ch->priv_data; 5620 int rx_done, tx_done, rxtx_done; 5621 struct stmmac_rxq_stats *rxq_stats; 5622 struct stmmac_txq_stats *txq_stats; 5623 u32 chan = ch->index; 5624 5625 rxq_stats = &priv->xstats.rxq_stats[chan]; 5626 u64_stats_update_begin(&rxq_stats->napi_syncp); 5627 u64_stats_inc(&rxq_stats->napi.poll); 5628 u64_stats_update_end(&rxq_stats->napi_syncp); 5629 5630 txq_stats = &priv->xstats.txq_stats[chan]; 5631 u64_stats_update_begin(&txq_stats->napi_syncp); 5632 u64_stats_inc(&txq_stats->napi.poll); 5633 u64_stats_update_end(&txq_stats->napi_syncp); 5634 5635 tx_done = stmmac_tx_clean(priv, budget, chan); 5636 tx_done = min(tx_done, budget); 5637 5638 rx_done = stmmac_rx_zc(priv, budget, chan); 5639 5640 rxtx_done = max(tx_done, rx_done); 5641 5642 /* If either TX or RX work is not complete, return budget 5643 * and keep pooling 5644 */ 5645 if (rxtx_done >= budget) 5646 return budget; 5647 5648 /* all work done, exit the polling mode */ 5649 if (napi_complete_done(napi, rxtx_done)) { 5650 unsigned long flags; 5651 5652 spin_lock_irqsave(&ch->lock, flags); 5653 /* Both RX and TX work done are compelte, 5654 * so enable both RX & TX IRQs. 5655 */ 5656 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 5657 spin_unlock_irqrestore(&ch->lock, flags); 5658 } 5659 5660 return min(rxtx_done, budget - 1); 5661 } 5662 5663 /** 5664 * stmmac_tx_timeout 5665 * @dev : Pointer to net device structure 5666 * @txqueue: the index of the hanging transmit queue 5667 * Description: this function is called when a packet transmission fails to 5668 * complete within a reasonable time. The driver will mark the error in the 5669 * netdev structure and arrange for the device to be reset to a sane state 5670 * in order to transmit a new packet. 5671 */ 5672 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 5673 { 5674 struct stmmac_priv *priv = netdev_priv(dev); 5675 5676 stmmac_global_err(priv); 5677 } 5678 5679 /** 5680 * stmmac_set_rx_mode - entry point for multicast addressing 5681 * @dev : pointer to the device structure 5682 * Description: 5683 * This function is a driver entry point which gets called by the kernel 5684 * whenever multicast addresses must be enabled/disabled. 5685 * Return value: 5686 * void. 5687 */ 5688 static void stmmac_set_rx_mode(struct net_device *dev) 5689 { 5690 struct stmmac_priv *priv = netdev_priv(dev); 5691 5692 stmmac_set_filter(priv, priv->hw, dev); 5693 } 5694 5695 /** 5696 * stmmac_change_mtu - entry point to change MTU size for the device. 5697 * @dev : device pointer. 5698 * @new_mtu : the new MTU size for the device. 5699 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 5700 * to drive packet transmission. Ethernet has an MTU of 1500 octets 5701 * (ETH_DATA_LEN). This value can be changed with ifconfig. 5702 * Return value: 5703 * 0 on success and an appropriate (-)ve integer as defined in errno.h 5704 * file on failure. 5705 */ 5706 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 5707 { 5708 struct stmmac_priv *priv = netdev_priv(dev); 5709 int txfifosz = priv->plat->tx_fifo_size; 5710 struct stmmac_dma_conf *dma_conf; 5711 const int mtu = new_mtu; 5712 int ret; 5713 5714 if (txfifosz == 0) 5715 txfifosz = priv->dma_cap.tx_fifo_size; 5716 5717 txfifosz /= priv->plat->tx_queues_to_use; 5718 5719 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 5720 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 5721 return -EINVAL; 5722 } 5723 5724 new_mtu = STMMAC_ALIGN(new_mtu); 5725 5726 /* If condition true, FIFO is too small or MTU too large */ 5727 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 5728 return -EINVAL; 5729 5730 if (netif_running(dev)) { 5731 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 5732 /* Try to allocate the new DMA conf with the new mtu */ 5733 dma_conf = stmmac_setup_dma_desc(priv, mtu); 5734 if (IS_ERR(dma_conf)) { 5735 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 5736 mtu); 5737 return PTR_ERR(dma_conf); 5738 } 5739 5740 stmmac_release(dev); 5741 5742 ret = __stmmac_open(dev, dma_conf); 5743 if (ret) { 5744 free_dma_desc_resources(priv, dma_conf); 5745 kfree(dma_conf); 5746 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 5747 return ret; 5748 } 5749 5750 kfree(dma_conf); 5751 5752 stmmac_set_rx_mode(dev); 5753 } 5754 5755 dev->mtu = mtu; 5756 netdev_update_features(dev); 5757 5758 return 0; 5759 } 5760 5761 static netdev_features_t stmmac_fix_features(struct net_device *dev, 5762 netdev_features_t features) 5763 { 5764 struct stmmac_priv *priv = netdev_priv(dev); 5765 5766 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 5767 features &= ~NETIF_F_RXCSUM; 5768 5769 if (!priv->plat->tx_coe) 5770 features &= ~NETIF_F_CSUM_MASK; 5771 5772 /* Some GMAC devices have a bugged Jumbo frame support that 5773 * needs to have the Tx COE disabled for oversized frames 5774 * (due to limited buffer sizes). In this case we disable 5775 * the TX csum insertion in the TDES and not use SF. 5776 */ 5777 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 5778 features &= ~NETIF_F_CSUM_MASK; 5779 5780 /* Disable tso if asked by ethtool */ 5781 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 5782 if (features & NETIF_F_TSO) 5783 priv->tso = true; 5784 else 5785 priv->tso = false; 5786 } 5787 5788 return features; 5789 } 5790 5791 static int stmmac_set_features(struct net_device *netdev, 5792 netdev_features_t features) 5793 { 5794 struct stmmac_priv *priv = netdev_priv(netdev); 5795 5796 /* Keep the COE Type in case of csum is supporting */ 5797 if (features & NETIF_F_RXCSUM) 5798 priv->hw->rx_csum = priv->plat->rx_coe; 5799 else 5800 priv->hw->rx_csum = 0; 5801 /* No check needed because rx_coe has been set before and it will be 5802 * fixed in case of issue. 5803 */ 5804 stmmac_rx_ipc(priv, priv->hw); 5805 5806 if (priv->sph_cap) { 5807 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; 5808 u32 chan; 5809 5810 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 5811 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 5812 } 5813 5814 return 0; 5815 } 5816 5817 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status) 5818 { 5819 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 5820 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 5821 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 5822 bool *hs_enable = &fpe_cfg->hs_enable; 5823 5824 if (status == FPE_EVENT_UNKNOWN || !*hs_enable) 5825 return; 5826 5827 /* If LP has sent verify mPacket, LP is FPE capable */ 5828 if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER) { 5829 if (*lp_state < FPE_STATE_CAPABLE) 5830 *lp_state = FPE_STATE_CAPABLE; 5831 5832 /* If user has requested FPE enable, quickly response */ 5833 if (*hs_enable) 5834 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 5835 fpe_cfg, 5836 MPACKET_RESPONSE); 5837 } 5838 5839 /* If Local has sent verify mPacket, Local is FPE capable */ 5840 if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER) { 5841 if (*lo_state < FPE_STATE_CAPABLE) 5842 *lo_state = FPE_STATE_CAPABLE; 5843 } 5844 5845 /* If LP has sent response mPacket, LP is entering FPE ON */ 5846 if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP) 5847 *lp_state = FPE_STATE_ENTERING_ON; 5848 5849 /* If Local has sent response mPacket, Local is entering FPE ON */ 5850 if ((status & FPE_EVENT_TRSP) == FPE_EVENT_TRSP) 5851 *lo_state = FPE_STATE_ENTERING_ON; 5852 5853 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && 5854 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && 5855 priv->fpe_wq) { 5856 queue_work(priv->fpe_wq, &priv->fpe_task); 5857 } 5858 } 5859 5860 static void stmmac_common_interrupt(struct stmmac_priv *priv) 5861 { 5862 u32 rx_cnt = priv->plat->rx_queues_to_use; 5863 u32 tx_cnt = priv->plat->tx_queues_to_use; 5864 u32 queues_count; 5865 u32 queue; 5866 bool xmac; 5867 5868 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 5869 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 5870 5871 if (priv->irq_wake) 5872 pm_wakeup_event(priv->device, 0); 5873 5874 if (priv->dma_cap.estsel) 5875 stmmac_est_irq_status(priv, priv->ioaddr, priv->dev, 5876 &priv->xstats, tx_cnt); 5877 5878 if (priv->dma_cap.fpesel) { 5879 int status = stmmac_fpe_irq_status(priv, priv->ioaddr, 5880 priv->dev); 5881 5882 stmmac_fpe_event_status(priv, status); 5883 } 5884 5885 /* To handle GMAC own interrupts */ 5886 if ((priv->plat->has_gmac) || xmac) { 5887 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 5888 5889 if (unlikely(status)) { 5890 /* For LPI we need to save the tx status */ 5891 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 5892 priv->tx_path_in_lpi_mode = true; 5893 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 5894 priv->tx_path_in_lpi_mode = false; 5895 } 5896 5897 for (queue = 0; queue < queues_count; queue++) { 5898 status = stmmac_host_mtl_irq_status(priv, priv->hw, 5899 queue); 5900 } 5901 5902 /* PCS link status */ 5903 if (priv->hw->pcs && 5904 !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { 5905 if (priv->xstats.pcs_link) 5906 netif_carrier_on(priv->dev); 5907 else 5908 netif_carrier_off(priv->dev); 5909 } 5910 5911 stmmac_timestamp_interrupt(priv, priv); 5912 } 5913 } 5914 5915 /** 5916 * stmmac_interrupt - main ISR 5917 * @irq: interrupt number. 5918 * @dev_id: to pass the net device pointer. 5919 * Description: this is the main driver interrupt service routine. 5920 * It can call: 5921 * o DMA service routine (to manage incoming frame reception and transmission 5922 * status) 5923 * o Core interrupts to manage: remote wake-up, management counter, LPI 5924 * interrupts. 5925 */ 5926 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 5927 { 5928 struct net_device *dev = (struct net_device *)dev_id; 5929 struct stmmac_priv *priv = netdev_priv(dev); 5930 5931 /* Check if adapter is up */ 5932 if (test_bit(STMMAC_DOWN, &priv->state)) 5933 return IRQ_HANDLED; 5934 5935 /* Check if a fatal error happened */ 5936 if (stmmac_safety_feat_interrupt(priv)) 5937 return IRQ_HANDLED; 5938 5939 /* To handle Common interrupts */ 5940 stmmac_common_interrupt(priv); 5941 5942 /* To handle DMA interrupts */ 5943 stmmac_dma_interrupt(priv); 5944 5945 return IRQ_HANDLED; 5946 } 5947 5948 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 5949 { 5950 struct net_device *dev = (struct net_device *)dev_id; 5951 struct stmmac_priv *priv = netdev_priv(dev); 5952 5953 /* Check if adapter is up */ 5954 if (test_bit(STMMAC_DOWN, &priv->state)) 5955 return IRQ_HANDLED; 5956 5957 /* To handle Common interrupts */ 5958 stmmac_common_interrupt(priv); 5959 5960 return IRQ_HANDLED; 5961 } 5962 5963 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 5964 { 5965 struct net_device *dev = (struct net_device *)dev_id; 5966 struct stmmac_priv *priv = netdev_priv(dev); 5967 5968 /* Check if adapter is up */ 5969 if (test_bit(STMMAC_DOWN, &priv->state)) 5970 return IRQ_HANDLED; 5971 5972 /* Check if a fatal error happened */ 5973 stmmac_safety_feat_interrupt(priv); 5974 5975 return IRQ_HANDLED; 5976 } 5977 5978 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 5979 { 5980 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 5981 struct stmmac_dma_conf *dma_conf; 5982 int chan = tx_q->queue_index; 5983 struct stmmac_priv *priv; 5984 int status; 5985 5986 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 5987 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 5988 5989 /* Check if adapter is up */ 5990 if (test_bit(STMMAC_DOWN, &priv->state)) 5991 return IRQ_HANDLED; 5992 5993 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 5994 5995 if (unlikely(status & tx_hard_error_bump_tc)) { 5996 /* Try to bump up the dma threshold on this failure */ 5997 stmmac_bump_dma_threshold(priv, chan); 5998 } else if (unlikely(status == tx_hard_error)) { 5999 stmmac_tx_err(priv, chan); 6000 } 6001 6002 return IRQ_HANDLED; 6003 } 6004 6005 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6006 { 6007 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6008 struct stmmac_dma_conf *dma_conf; 6009 int chan = rx_q->queue_index; 6010 struct stmmac_priv *priv; 6011 6012 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6013 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6014 6015 /* Check if adapter is up */ 6016 if (test_bit(STMMAC_DOWN, &priv->state)) 6017 return IRQ_HANDLED; 6018 6019 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6020 6021 return IRQ_HANDLED; 6022 } 6023 6024 /** 6025 * stmmac_ioctl - Entry point for the Ioctl 6026 * @dev: Device pointer. 6027 * @rq: An IOCTL specefic structure, that can contain a pointer to 6028 * a proprietary structure used to pass information to the driver. 6029 * @cmd: IOCTL command 6030 * Description: 6031 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 6032 */ 6033 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6034 { 6035 struct stmmac_priv *priv = netdev_priv (dev); 6036 int ret = -EOPNOTSUPP; 6037 6038 if (!netif_running(dev)) 6039 return -EINVAL; 6040 6041 switch (cmd) { 6042 case SIOCGMIIPHY: 6043 case SIOCGMIIREG: 6044 case SIOCSMIIREG: 6045 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 6046 break; 6047 case SIOCSHWTSTAMP: 6048 ret = stmmac_hwtstamp_set(dev, rq); 6049 break; 6050 case SIOCGHWTSTAMP: 6051 ret = stmmac_hwtstamp_get(dev, rq); 6052 break; 6053 default: 6054 break; 6055 } 6056 6057 return ret; 6058 } 6059 6060 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6061 void *cb_priv) 6062 { 6063 struct stmmac_priv *priv = cb_priv; 6064 int ret = -EOPNOTSUPP; 6065 6066 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6067 return ret; 6068 6069 __stmmac_disable_all_queues(priv); 6070 6071 switch (type) { 6072 case TC_SETUP_CLSU32: 6073 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6074 break; 6075 case TC_SETUP_CLSFLOWER: 6076 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6077 break; 6078 default: 6079 break; 6080 } 6081 6082 stmmac_enable_all_queues(priv); 6083 return ret; 6084 } 6085 6086 static LIST_HEAD(stmmac_block_cb_list); 6087 6088 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6089 void *type_data) 6090 { 6091 struct stmmac_priv *priv = netdev_priv(ndev); 6092 6093 switch (type) { 6094 case TC_QUERY_CAPS: 6095 return stmmac_tc_query_caps(priv, priv, type_data); 6096 case TC_SETUP_BLOCK: 6097 return flow_block_cb_setup_simple(type_data, 6098 &stmmac_block_cb_list, 6099 stmmac_setup_tc_block_cb, 6100 priv, priv, true); 6101 case TC_SETUP_QDISC_CBS: 6102 return stmmac_tc_setup_cbs(priv, priv, type_data); 6103 case TC_SETUP_QDISC_TAPRIO: 6104 return stmmac_tc_setup_taprio(priv, priv, type_data); 6105 case TC_SETUP_QDISC_ETF: 6106 return stmmac_tc_setup_etf(priv, priv, type_data); 6107 default: 6108 return -EOPNOTSUPP; 6109 } 6110 } 6111 6112 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6113 struct net_device *sb_dev) 6114 { 6115 int gso = skb_shinfo(skb)->gso_type; 6116 6117 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6118 /* 6119 * There is no way to determine the number of TSO/USO 6120 * capable Queues. Let's use always the Queue 0 6121 * because if TSO/USO is supported then at least this 6122 * one will be capable. 6123 */ 6124 return 0; 6125 } 6126 6127 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6128 } 6129 6130 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6131 { 6132 struct stmmac_priv *priv = netdev_priv(ndev); 6133 int ret = 0; 6134 6135 ret = pm_runtime_resume_and_get(priv->device); 6136 if (ret < 0) 6137 return ret; 6138 6139 ret = eth_mac_addr(ndev, addr); 6140 if (ret) 6141 goto set_mac_error; 6142 6143 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6144 6145 set_mac_error: 6146 pm_runtime_put(priv->device); 6147 6148 return ret; 6149 } 6150 6151 #ifdef CONFIG_DEBUG_FS 6152 static struct dentry *stmmac_fs_dir; 6153 6154 static void sysfs_display_ring(void *head, int size, int extend_desc, 6155 struct seq_file *seq, dma_addr_t dma_phy_addr) 6156 { 6157 int i; 6158 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6159 struct dma_desc *p = (struct dma_desc *)head; 6160 dma_addr_t dma_addr; 6161 6162 for (i = 0; i < size; i++) { 6163 if (extend_desc) { 6164 dma_addr = dma_phy_addr + i * sizeof(*ep); 6165 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6166 i, &dma_addr, 6167 le32_to_cpu(ep->basic.des0), 6168 le32_to_cpu(ep->basic.des1), 6169 le32_to_cpu(ep->basic.des2), 6170 le32_to_cpu(ep->basic.des3)); 6171 ep++; 6172 } else { 6173 dma_addr = dma_phy_addr + i * sizeof(*p); 6174 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6175 i, &dma_addr, 6176 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6177 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6178 p++; 6179 } 6180 seq_printf(seq, "\n"); 6181 } 6182 } 6183 6184 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6185 { 6186 struct net_device *dev = seq->private; 6187 struct stmmac_priv *priv = netdev_priv(dev); 6188 u32 rx_count = priv->plat->rx_queues_to_use; 6189 u32 tx_count = priv->plat->tx_queues_to_use; 6190 u32 queue; 6191 6192 if ((dev->flags & IFF_UP) == 0) 6193 return 0; 6194 6195 for (queue = 0; queue < rx_count; queue++) { 6196 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6197 6198 seq_printf(seq, "RX Queue %d:\n", queue); 6199 6200 if (priv->extend_desc) { 6201 seq_printf(seq, "Extended descriptor ring:\n"); 6202 sysfs_display_ring((void *)rx_q->dma_erx, 6203 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6204 } else { 6205 seq_printf(seq, "Descriptor ring:\n"); 6206 sysfs_display_ring((void *)rx_q->dma_rx, 6207 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6208 } 6209 } 6210 6211 for (queue = 0; queue < tx_count; queue++) { 6212 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6213 6214 seq_printf(seq, "TX Queue %d:\n", queue); 6215 6216 if (priv->extend_desc) { 6217 seq_printf(seq, "Extended descriptor ring:\n"); 6218 sysfs_display_ring((void *)tx_q->dma_etx, 6219 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6220 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6221 seq_printf(seq, "Descriptor ring:\n"); 6222 sysfs_display_ring((void *)tx_q->dma_tx, 6223 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6224 } 6225 } 6226 6227 return 0; 6228 } 6229 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6230 6231 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6232 { 6233 static const char * const dwxgmac_timestamp_source[] = { 6234 "None", 6235 "Internal", 6236 "External", 6237 "Both", 6238 }; 6239 static const char * const dwxgmac_safety_feature_desc[] = { 6240 "No", 6241 "All Safety Features with ECC and Parity", 6242 "All Safety Features without ECC or Parity", 6243 "All Safety Features with Parity Only", 6244 "ECC Only", 6245 "UNDEFINED", 6246 "UNDEFINED", 6247 "UNDEFINED", 6248 }; 6249 struct net_device *dev = seq->private; 6250 struct stmmac_priv *priv = netdev_priv(dev); 6251 6252 if (!priv->hw_cap_support) { 6253 seq_printf(seq, "DMA HW features not supported\n"); 6254 return 0; 6255 } 6256 6257 seq_printf(seq, "==============================\n"); 6258 seq_printf(seq, "\tDMA HW features\n"); 6259 seq_printf(seq, "==============================\n"); 6260 6261 seq_printf(seq, "\t10/100 Mbps: %s\n", 6262 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6263 seq_printf(seq, "\t1000 Mbps: %s\n", 6264 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6265 seq_printf(seq, "\tHalf duplex: %s\n", 6266 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6267 if (priv->plat->has_xgmac) { 6268 seq_printf(seq, 6269 "\tNumber of Additional MAC address registers: %d\n", 6270 priv->dma_cap.multi_addr); 6271 } else { 6272 seq_printf(seq, "\tHash Filter: %s\n", 6273 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6274 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6275 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6276 } 6277 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6278 (priv->dma_cap.pcs) ? "Y" : "N"); 6279 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6280 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6281 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6282 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6283 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6284 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6285 seq_printf(seq, "\tRMON module: %s\n", 6286 (priv->dma_cap.rmon) ? "Y" : "N"); 6287 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6288 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6289 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6290 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6291 if (priv->plat->has_xgmac) 6292 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6293 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6294 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6295 (priv->dma_cap.eee) ? "Y" : "N"); 6296 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6297 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6298 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6299 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6300 priv->plat->has_xgmac) { 6301 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6302 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6303 } else { 6304 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6305 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6306 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6307 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6308 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6309 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6310 } 6311 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6312 priv->dma_cap.number_rx_channel); 6313 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6314 priv->dma_cap.number_tx_channel); 6315 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", 6316 priv->dma_cap.number_rx_queues); 6317 seq_printf(seq, "\tNumber of Additional TX queues: %d\n", 6318 priv->dma_cap.number_tx_queues); 6319 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6320 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6321 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6322 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6323 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6324 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6325 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6326 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6327 priv->dma_cap.pps_out_num); 6328 seq_printf(seq, "\tSafety Features: %s\n", 6329 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6330 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6331 priv->dma_cap.frpsel ? "Y" : "N"); 6332 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6333 priv->dma_cap.host_dma_width); 6334 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6335 priv->dma_cap.rssen ? "Y" : "N"); 6336 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6337 priv->dma_cap.vlhash ? "Y" : "N"); 6338 seq_printf(seq, "\tSplit Header: %s\n", 6339 priv->dma_cap.sphen ? "Y" : "N"); 6340 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6341 priv->dma_cap.vlins ? "Y" : "N"); 6342 seq_printf(seq, "\tDouble VLAN: %s\n", 6343 priv->dma_cap.dvlan ? "Y" : "N"); 6344 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6345 priv->dma_cap.l3l4fnum); 6346 seq_printf(seq, "\tARP Offloading: %s\n", 6347 priv->dma_cap.arpoffsel ? "Y" : "N"); 6348 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6349 priv->dma_cap.estsel ? "Y" : "N"); 6350 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6351 priv->dma_cap.fpesel ? "Y" : "N"); 6352 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6353 priv->dma_cap.tbssel ? "Y" : "N"); 6354 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6355 priv->dma_cap.tbs_ch_num); 6356 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6357 priv->dma_cap.sgfsel ? "Y" : "N"); 6358 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6359 BIT(priv->dma_cap.ttsfd) >> 1); 6360 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6361 priv->dma_cap.numtc); 6362 seq_printf(seq, "\tDCB Feature: %s\n", 6363 priv->dma_cap.dcben ? "Y" : "N"); 6364 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6365 priv->dma_cap.advthword ? "Y" : "N"); 6366 seq_printf(seq, "\tPTP Offload: %s\n", 6367 priv->dma_cap.ptoen ? "Y" : "N"); 6368 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6369 priv->dma_cap.osten ? "Y" : "N"); 6370 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6371 priv->dma_cap.pfcen ? "Y" : "N"); 6372 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6373 BIT(priv->dma_cap.frpes) << 6); 6374 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6375 BIT(priv->dma_cap.frpbs) << 6); 6376 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6377 priv->dma_cap.frppipe_num); 6378 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6379 priv->dma_cap.nrvf_num ? 6380 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6381 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6382 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6383 seq_printf(seq, "\tDepth of GCL: %lu\n", 6384 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6385 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6386 priv->dma_cap.cbtisel ? "Y" : "N"); 6387 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6388 priv->dma_cap.aux_snapshot_n); 6389 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6390 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6391 seq_printf(seq, "\tEnhanced DMA: %s\n", 6392 priv->dma_cap.edma ? "Y" : "N"); 6393 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6394 priv->dma_cap.ediffc ? "Y" : "N"); 6395 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6396 priv->dma_cap.vxn ? "Y" : "N"); 6397 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6398 priv->dma_cap.dbgmem ? "Y" : "N"); 6399 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6400 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6401 return 0; 6402 } 6403 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6404 6405 /* Use network device events to rename debugfs file entries. 6406 */ 6407 static int stmmac_device_event(struct notifier_block *unused, 6408 unsigned long event, void *ptr) 6409 { 6410 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6411 struct stmmac_priv *priv = netdev_priv(dev); 6412 6413 if (dev->netdev_ops != &stmmac_netdev_ops) 6414 goto done; 6415 6416 switch (event) { 6417 case NETDEV_CHANGENAME: 6418 if (priv->dbgfs_dir) 6419 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, 6420 priv->dbgfs_dir, 6421 stmmac_fs_dir, 6422 dev->name); 6423 break; 6424 } 6425 done: 6426 return NOTIFY_DONE; 6427 } 6428 6429 static struct notifier_block stmmac_notifier = { 6430 .notifier_call = stmmac_device_event, 6431 }; 6432 6433 static void stmmac_init_fs(struct net_device *dev) 6434 { 6435 struct stmmac_priv *priv = netdev_priv(dev); 6436 6437 rtnl_lock(); 6438 6439 /* Create per netdev entries */ 6440 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6441 6442 /* Entry to report DMA RX/TX rings */ 6443 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6444 &stmmac_rings_status_fops); 6445 6446 /* Entry to report the DMA HW features */ 6447 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6448 &stmmac_dma_cap_fops); 6449 6450 rtnl_unlock(); 6451 } 6452 6453 static void stmmac_exit_fs(struct net_device *dev) 6454 { 6455 struct stmmac_priv *priv = netdev_priv(dev); 6456 6457 debugfs_remove_recursive(priv->dbgfs_dir); 6458 } 6459 #endif /* CONFIG_DEBUG_FS */ 6460 6461 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6462 { 6463 unsigned char *data = (unsigned char *)&vid_le; 6464 unsigned char data_byte = 0; 6465 u32 crc = ~0x0; 6466 u32 temp = 0; 6467 int i, bits; 6468 6469 bits = get_bitmask_order(VLAN_VID_MASK); 6470 for (i = 0; i < bits; i++) { 6471 if ((i % 8) == 0) 6472 data_byte = data[i / 8]; 6473 6474 temp = ((crc & 1) ^ data_byte) & 1; 6475 crc >>= 1; 6476 data_byte >>= 1; 6477 6478 if (temp) 6479 crc ^= 0xedb88320; 6480 } 6481 6482 return crc; 6483 } 6484 6485 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6486 { 6487 u32 crc, hash = 0; 6488 u16 pmatch = 0; 6489 int count = 0; 6490 u16 vid = 0; 6491 6492 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6493 __le16 vid_le = cpu_to_le16(vid); 6494 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6495 hash |= (1 << crc); 6496 count++; 6497 } 6498 6499 if (!priv->dma_cap.vlhash) { 6500 if (count > 2) /* VID = 0 always passes filter */ 6501 return -EOPNOTSUPP; 6502 6503 pmatch = vid; 6504 hash = 0; 6505 } 6506 6507 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6508 } 6509 6510 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6511 { 6512 struct stmmac_priv *priv = netdev_priv(ndev); 6513 bool is_double = false; 6514 int ret; 6515 6516 ret = pm_runtime_resume_and_get(priv->device); 6517 if (ret < 0) 6518 return ret; 6519 6520 if (be16_to_cpu(proto) == ETH_P_8021AD) 6521 is_double = true; 6522 6523 set_bit(vid, priv->active_vlans); 6524 ret = stmmac_vlan_update(priv, is_double); 6525 if (ret) { 6526 clear_bit(vid, priv->active_vlans); 6527 goto err_pm_put; 6528 } 6529 6530 if (priv->hw->num_vlan) { 6531 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6532 if (ret) 6533 goto err_pm_put; 6534 } 6535 err_pm_put: 6536 pm_runtime_put(priv->device); 6537 6538 return ret; 6539 } 6540 6541 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6542 { 6543 struct stmmac_priv *priv = netdev_priv(ndev); 6544 bool is_double = false; 6545 int ret; 6546 6547 ret = pm_runtime_resume_and_get(priv->device); 6548 if (ret < 0) 6549 return ret; 6550 6551 if (be16_to_cpu(proto) == ETH_P_8021AD) 6552 is_double = true; 6553 6554 clear_bit(vid, priv->active_vlans); 6555 6556 if (priv->hw->num_vlan) { 6557 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6558 if (ret) 6559 goto del_vlan_error; 6560 } 6561 6562 ret = stmmac_vlan_update(priv, is_double); 6563 6564 del_vlan_error: 6565 pm_runtime_put(priv->device); 6566 6567 return ret; 6568 } 6569 6570 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6571 { 6572 struct stmmac_priv *priv = netdev_priv(dev); 6573 6574 switch (bpf->command) { 6575 case XDP_SETUP_PROG: 6576 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6577 case XDP_SETUP_XSK_POOL: 6578 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6579 bpf->xsk.queue_id); 6580 default: 6581 return -EOPNOTSUPP; 6582 } 6583 } 6584 6585 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6586 struct xdp_frame **frames, u32 flags) 6587 { 6588 struct stmmac_priv *priv = netdev_priv(dev); 6589 int cpu = smp_processor_id(); 6590 struct netdev_queue *nq; 6591 int i, nxmit = 0; 6592 int queue; 6593 6594 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6595 return -ENETDOWN; 6596 6597 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6598 return -EINVAL; 6599 6600 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6601 nq = netdev_get_tx_queue(priv->dev, queue); 6602 6603 __netif_tx_lock(nq, cpu); 6604 /* Avoids TX time-out as we are sharing with slow path */ 6605 txq_trans_cond_update(nq); 6606 6607 for (i = 0; i < num_frames; i++) { 6608 int res; 6609 6610 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6611 if (res == STMMAC_XDP_CONSUMED) 6612 break; 6613 6614 nxmit++; 6615 } 6616 6617 if (flags & XDP_XMIT_FLUSH) { 6618 stmmac_flush_tx_descriptors(priv, queue); 6619 stmmac_tx_timer_arm(priv, queue); 6620 } 6621 6622 __netif_tx_unlock(nq); 6623 6624 return nxmit; 6625 } 6626 6627 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6628 { 6629 struct stmmac_channel *ch = &priv->channel[queue]; 6630 unsigned long flags; 6631 6632 spin_lock_irqsave(&ch->lock, flags); 6633 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6634 spin_unlock_irqrestore(&ch->lock, flags); 6635 6636 stmmac_stop_rx_dma(priv, queue); 6637 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6638 } 6639 6640 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 6641 { 6642 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6643 struct stmmac_channel *ch = &priv->channel[queue]; 6644 unsigned long flags; 6645 u32 buf_size; 6646 int ret; 6647 6648 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6649 if (ret) { 6650 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 6651 return; 6652 } 6653 6654 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 6655 if (ret) { 6656 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6657 netdev_err(priv->dev, "Failed to init RX desc.\n"); 6658 return; 6659 } 6660 6661 stmmac_reset_rx_queue(priv, queue); 6662 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 6663 6664 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6665 rx_q->dma_rx_phy, rx_q->queue_index); 6666 6667 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * 6668 sizeof(struct dma_desc)); 6669 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6670 rx_q->rx_tail_addr, rx_q->queue_index); 6671 6672 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6673 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6674 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6675 buf_size, 6676 rx_q->queue_index); 6677 } else { 6678 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6679 priv->dma_conf.dma_buf_sz, 6680 rx_q->queue_index); 6681 } 6682 6683 stmmac_start_rx_dma(priv, queue); 6684 6685 spin_lock_irqsave(&ch->lock, flags); 6686 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6687 spin_unlock_irqrestore(&ch->lock, flags); 6688 } 6689 6690 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 6691 { 6692 struct stmmac_channel *ch = &priv->channel[queue]; 6693 unsigned long flags; 6694 6695 spin_lock_irqsave(&ch->lock, flags); 6696 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6697 spin_unlock_irqrestore(&ch->lock, flags); 6698 6699 stmmac_stop_tx_dma(priv, queue); 6700 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6701 } 6702 6703 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 6704 { 6705 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6706 struct stmmac_channel *ch = &priv->channel[queue]; 6707 unsigned long flags; 6708 int ret; 6709 6710 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6711 if (ret) { 6712 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 6713 return; 6714 } 6715 6716 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 6717 if (ret) { 6718 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6719 netdev_err(priv->dev, "Failed to init TX desc.\n"); 6720 return; 6721 } 6722 6723 stmmac_reset_tx_queue(priv, queue); 6724 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 6725 6726 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6727 tx_q->dma_tx_phy, tx_q->queue_index); 6728 6729 if (tx_q->tbs & STMMAC_TBS_AVAIL) 6730 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); 6731 6732 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6733 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6734 tx_q->tx_tail_addr, tx_q->queue_index); 6735 6736 stmmac_start_tx_dma(priv, queue); 6737 6738 spin_lock_irqsave(&ch->lock, flags); 6739 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6740 spin_unlock_irqrestore(&ch->lock, flags); 6741 } 6742 6743 void stmmac_xdp_release(struct net_device *dev) 6744 { 6745 struct stmmac_priv *priv = netdev_priv(dev); 6746 u32 chan; 6747 6748 /* Ensure tx function is not running */ 6749 netif_tx_disable(dev); 6750 6751 /* Disable NAPI process */ 6752 stmmac_disable_all_queues(priv); 6753 6754 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6755 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6756 6757 /* Free the IRQ lines */ 6758 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 6759 6760 /* Stop TX/RX DMA channels */ 6761 stmmac_stop_all_dma(priv); 6762 6763 /* Release and free the Rx/Tx resources */ 6764 free_dma_desc_resources(priv, &priv->dma_conf); 6765 6766 /* Disable the MAC Rx/Tx */ 6767 stmmac_mac_set(priv, priv->ioaddr, false); 6768 6769 /* set trans_start so we don't get spurious 6770 * watchdogs during reset 6771 */ 6772 netif_trans_update(dev); 6773 netif_carrier_off(dev); 6774 } 6775 6776 int stmmac_xdp_open(struct net_device *dev) 6777 { 6778 struct stmmac_priv *priv = netdev_priv(dev); 6779 u32 rx_cnt = priv->plat->rx_queues_to_use; 6780 u32 tx_cnt = priv->plat->tx_queues_to_use; 6781 u32 dma_csr_ch = max(rx_cnt, tx_cnt); 6782 struct stmmac_rx_queue *rx_q; 6783 struct stmmac_tx_queue *tx_q; 6784 u32 buf_size; 6785 bool sph_en; 6786 u32 chan; 6787 int ret; 6788 6789 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 6790 if (ret < 0) { 6791 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 6792 __func__); 6793 goto dma_desc_error; 6794 } 6795 6796 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 6797 if (ret < 0) { 6798 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 6799 __func__); 6800 goto init_error; 6801 } 6802 6803 stmmac_reset_queues_param(priv); 6804 6805 /* DMA CSR Channel configuration */ 6806 for (chan = 0; chan < dma_csr_ch; chan++) { 6807 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 6808 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6809 } 6810 6811 /* Adjust Split header */ 6812 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 6813 6814 /* DMA RX Channel Configuration */ 6815 for (chan = 0; chan < rx_cnt; chan++) { 6816 rx_q = &priv->dma_conf.rx_queue[chan]; 6817 6818 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6819 rx_q->dma_rx_phy, chan); 6820 6821 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 6822 (rx_q->buf_alloc_num * 6823 sizeof(struct dma_desc)); 6824 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6825 rx_q->rx_tail_addr, chan); 6826 6827 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6828 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6829 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6830 buf_size, 6831 rx_q->queue_index); 6832 } else { 6833 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6834 priv->dma_conf.dma_buf_sz, 6835 rx_q->queue_index); 6836 } 6837 6838 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6839 } 6840 6841 /* DMA TX Channel Configuration */ 6842 for (chan = 0; chan < tx_cnt; chan++) { 6843 tx_q = &priv->dma_conf.tx_queue[chan]; 6844 6845 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6846 tx_q->dma_tx_phy, chan); 6847 6848 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6849 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6850 tx_q->tx_tail_addr, chan); 6851 6852 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6853 tx_q->txtimer.function = stmmac_tx_timer; 6854 } 6855 6856 /* Enable the MAC Rx/Tx */ 6857 stmmac_mac_set(priv, priv->ioaddr, true); 6858 6859 /* Start Rx & Tx DMA Channels */ 6860 stmmac_start_all_dma(priv); 6861 6862 ret = stmmac_request_irq(dev); 6863 if (ret) 6864 goto irq_error; 6865 6866 /* Enable NAPI process*/ 6867 stmmac_enable_all_queues(priv); 6868 netif_carrier_on(dev); 6869 netif_tx_start_all_queues(dev); 6870 stmmac_enable_all_dma_irq(priv); 6871 6872 return 0; 6873 6874 irq_error: 6875 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6876 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6877 6878 stmmac_hw_teardown(dev); 6879 init_error: 6880 free_dma_desc_resources(priv, &priv->dma_conf); 6881 dma_desc_error: 6882 return ret; 6883 } 6884 6885 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 6886 { 6887 struct stmmac_priv *priv = netdev_priv(dev); 6888 struct stmmac_rx_queue *rx_q; 6889 struct stmmac_tx_queue *tx_q; 6890 struct stmmac_channel *ch; 6891 6892 if (test_bit(STMMAC_DOWN, &priv->state) || 6893 !netif_carrier_ok(priv->dev)) 6894 return -ENETDOWN; 6895 6896 if (!stmmac_xdp_is_enabled(priv)) 6897 return -EINVAL; 6898 6899 if (queue >= priv->plat->rx_queues_to_use || 6900 queue >= priv->plat->tx_queues_to_use) 6901 return -EINVAL; 6902 6903 rx_q = &priv->dma_conf.rx_queue[queue]; 6904 tx_q = &priv->dma_conf.tx_queue[queue]; 6905 ch = &priv->channel[queue]; 6906 6907 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 6908 return -EINVAL; 6909 6910 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 6911 /* EQoS does not have per-DMA channel SW interrupt, 6912 * so we schedule RX Napi straight-away. 6913 */ 6914 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 6915 __napi_schedule(&ch->rxtx_napi); 6916 } 6917 6918 return 0; 6919 } 6920 6921 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 6922 { 6923 struct stmmac_priv *priv = netdev_priv(dev); 6924 u32 tx_cnt = priv->plat->tx_queues_to_use; 6925 u32 rx_cnt = priv->plat->rx_queues_to_use; 6926 unsigned int start; 6927 int q; 6928 6929 for (q = 0; q < tx_cnt; q++) { 6930 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 6931 u64 tx_packets; 6932 u64 tx_bytes; 6933 6934 do { 6935 start = u64_stats_fetch_begin(&txq_stats->q_syncp); 6936 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); 6937 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); 6938 do { 6939 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); 6940 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); 6941 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); 6942 6943 stats->tx_packets += tx_packets; 6944 stats->tx_bytes += tx_bytes; 6945 } 6946 6947 for (q = 0; q < rx_cnt; q++) { 6948 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 6949 u64 rx_packets; 6950 u64 rx_bytes; 6951 6952 do { 6953 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); 6954 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); 6955 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); 6956 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); 6957 6958 stats->rx_packets += rx_packets; 6959 stats->rx_bytes += rx_bytes; 6960 } 6961 6962 stats->rx_dropped = priv->xstats.rx_dropped; 6963 stats->rx_errors = priv->xstats.rx_errors; 6964 stats->tx_dropped = priv->xstats.tx_dropped; 6965 stats->tx_errors = priv->xstats.tx_errors; 6966 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 6967 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 6968 stats->rx_length_errors = priv->xstats.rx_length; 6969 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 6970 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 6971 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 6972 } 6973 6974 static const struct net_device_ops stmmac_netdev_ops = { 6975 .ndo_open = stmmac_open, 6976 .ndo_start_xmit = stmmac_xmit, 6977 .ndo_stop = stmmac_release, 6978 .ndo_change_mtu = stmmac_change_mtu, 6979 .ndo_fix_features = stmmac_fix_features, 6980 .ndo_set_features = stmmac_set_features, 6981 .ndo_set_rx_mode = stmmac_set_rx_mode, 6982 .ndo_tx_timeout = stmmac_tx_timeout, 6983 .ndo_eth_ioctl = stmmac_ioctl, 6984 .ndo_get_stats64 = stmmac_get_stats64, 6985 .ndo_setup_tc = stmmac_setup_tc, 6986 .ndo_select_queue = stmmac_select_queue, 6987 .ndo_set_mac_address = stmmac_set_mac_address, 6988 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 6989 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 6990 .ndo_bpf = stmmac_bpf, 6991 .ndo_xdp_xmit = stmmac_xdp_xmit, 6992 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 6993 }; 6994 6995 static void stmmac_reset_subtask(struct stmmac_priv *priv) 6996 { 6997 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 6998 return; 6999 if (test_bit(STMMAC_DOWN, &priv->state)) 7000 return; 7001 7002 netdev_err(priv->dev, "Reset adapter.\n"); 7003 7004 rtnl_lock(); 7005 netif_trans_update(priv->dev); 7006 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7007 usleep_range(1000, 2000); 7008 7009 set_bit(STMMAC_DOWN, &priv->state); 7010 dev_close(priv->dev); 7011 dev_open(priv->dev, NULL); 7012 clear_bit(STMMAC_DOWN, &priv->state); 7013 clear_bit(STMMAC_RESETING, &priv->state); 7014 rtnl_unlock(); 7015 } 7016 7017 static void stmmac_service_task(struct work_struct *work) 7018 { 7019 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7020 service_task); 7021 7022 stmmac_reset_subtask(priv); 7023 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7024 } 7025 7026 /** 7027 * stmmac_hw_init - Init the MAC device 7028 * @priv: driver private structure 7029 * Description: this function is to configure the MAC device according to 7030 * some platform parameters or the HW capability register. It prepares the 7031 * driver to use either ring or chain modes and to setup either enhanced or 7032 * normal descriptors. 7033 */ 7034 static int stmmac_hw_init(struct stmmac_priv *priv) 7035 { 7036 int ret; 7037 7038 /* dwmac-sun8i only work in chain mode */ 7039 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7040 chain_mode = 1; 7041 priv->chain_mode = chain_mode; 7042 7043 /* Initialize HW Interface */ 7044 ret = stmmac_hwif_init(priv); 7045 if (ret) 7046 return ret; 7047 7048 /* Get the HW capability (new GMAC newer than 3.50a) */ 7049 priv->hw_cap_support = stmmac_get_hw_features(priv); 7050 if (priv->hw_cap_support) { 7051 dev_info(priv->device, "DMA HW capability register supported\n"); 7052 7053 /* We can override some gmac/dma configuration fields: e.g. 7054 * enh_desc, tx_coe (e.g. that are passed through the 7055 * platform) with the values from the HW capability 7056 * register (if supported). 7057 */ 7058 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7059 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7060 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7061 priv->hw->pmt = priv->plat->pmt; 7062 if (priv->dma_cap.hash_tb_sz) { 7063 priv->hw->multicast_filter_bins = 7064 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7065 priv->hw->mcast_bits_log2 = 7066 ilog2(priv->hw->multicast_filter_bins); 7067 } 7068 7069 /* TXCOE doesn't work in thresh DMA mode */ 7070 if (priv->plat->force_thresh_dma_mode) 7071 priv->plat->tx_coe = 0; 7072 else 7073 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7074 7075 /* In case of GMAC4 rx_coe is from HW cap register. */ 7076 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7077 7078 if (priv->dma_cap.rx_coe_type2) 7079 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7080 else if (priv->dma_cap.rx_coe_type1) 7081 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7082 7083 } else { 7084 dev_info(priv->device, "No HW DMA feature register supported\n"); 7085 } 7086 7087 if (priv->plat->rx_coe) { 7088 priv->hw->rx_csum = priv->plat->rx_coe; 7089 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7090 if (priv->synopsys_id < DWMAC_CORE_4_00) 7091 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7092 } 7093 if (priv->plat->tx_coe) 7094 dev_info(priv->device, "TX Checksum insertion supported\n"); 7095 7096 if (priv->plat->pmt) { 7097 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7098 device_set_wakeup_capable(priv->device, 1); 7099 } 7100 7101 if (priv->dma_cap.tsoen) 7102 dev_info(priv->device, "TSO supported\n"); 7103 7104 priv->hw->vlan_fail_q_en = 7105 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7106 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7107 7108 /* Run HW quirks, if any */ 7109 if (priv->hwif_quirks) { 7110 ret = priv->hwif_quirks(priv); 7111 if (ret) 7112 return ret; 7113 } 7114 7115 /* Rx Watchdog is available in the COREs newer than the 3.40. 7116 * In some case, for example on bugged HW this feature 7117 * has to be disable and this can be done by passing the 7118 * riwt_off field from the platform. 7119 */ 7120 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 7121 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 7122 priv->use_riwt = 1; 7123 dev_info(priv->device, 7124 "Enable RX Mitigation via HW Watchdog Timer\n"); 7125 } 7126 7127 return 0; 7128 } 7129 7130 static void stmmac_napi_add(struct net_device *dev) 7131 { 7132 struct stmmac_priv *priv = netdev_priv(dev); 7133 u32 queue, maxq; 7134 7135 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7136 7137 for (queue = 0; queue < maxq; queue++) { 7138 struct stmmac_channel *ch = &priv->channel[queue]; 7139 7140 ch->priv_data = priv; 7141 ch->index = queue; 7142 spin_lock_init(&ch->lock); 7143 7144 if (queue < priv->plat->rx_queues_to_use) { 7145 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7146 } 7147 if (queue < priv->plat->tx_queues_to_use) { 7148 netif_napi_add_tx(dev, &ch->tx_napi, 7149 stmmac_napi_poll_tx); 7150 } 7151 if (queue < priv->plat->rx_queues_to_use && 7152 queue < priv->plat->tx_queues_to_use) { 7153 netif_napi_add(dev, &ch->rxtx_napi, 7154 stmmac_napi_poll_rxtx); 7155 } 7156 } 7157 } 7158 7159 static void stmmac_napi_del(struct net_device *dev) 7160 { 7161 struct stmmac_priv *priv = netdev_priv(dev); 7162 u32 queue, maxq; 7163 7164 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7165 7166 for (queue = 0; queue < maxq; queue++) { 7167 struct stmmac_channel *ch = &priv->channel[queue]; 7168 7169 if (queue < priv->plat->rx_queues_to_use) 7170 netif_napi_del(&ch->rx_napi); 7171 if (queue < priv->plat->tx_queues_to_use) 7172 netif_napi_del(&ch->tx_napi); 7173 if (queue < priv->plat->rx_queues_to_use && 7174 queue < priv->plat->tx_queues_to_use) { 7175 netif_napi_del(&ch->rxtx_napi); 7176 } 7177 } 7178 } 7179 7180 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) 7181 { 7182 struct stmmac_priv *priv = netdev_priv(dev); 7183 int ret = 0, i; 7184 int max_speed; 7185 7186 if (netif_running(dev)) 7187 stmmac_release(dev); 7188 7189 stmmac_napi_del(dev); 7190 7191 priv->plat->rx_queues_to_use = rx_cnt; 7192 priv->plat->tx_queues_to_use = tx_cnt; 7193 if (!netif_is_rxfh_configured(dev)) 7194 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7195 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7196 rx_cnt); 7197 7198 stmmac_mac_phylink_get_caps(priv); 7199 7200 priv->phylink_config.mac_capabilities = priv->hw->link.caps; 7201 7202 max_speed = priv->plat->max_speed; 7203 if (max_speed) 7204 phylink_limit_mac_speed(&priv->phylink_config, max_speed); 7205 7206 stmmac_napi_add(dev); 7207 7208 if (netif_running(dev)) 7209 ret = stmmac_open(dev); 7210 7211 return ret; 7212 } 7213 7214 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7215 { 7216 struct stmmac_priv *priv = netdev_priv(dev); 7217 int ret = 0; 7218 7219 if (netif_running(dev)) 7220 stmmac_release(dev); 7221 7222 priv->dma_conf.dma_rx_size = rx_size; 7223 priv->dma_conf.dma_tx_size = tx_size; 7224 7225 if (netif_running(dev)) 7226 ret = stmmac_open(dev); 7227 7228 return ret; 7229 } 7230 7231 #define SEND_VERIFY_MPAKCET_FMT "Send Verify mPacket lo_state=%d lp_state=%d\n" 7232 static void stmmac_fpe_lp_task(struct work_struct *work) 7233 { 7234 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7235 fpe_task); 7236 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 7237 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 7238 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 7239 bool *hs_enable = &fpe_cfg->hs_enable; 7240 bool *enable = &fpe_cfg->enable; 7241 int retries = 20; 7242 7243 while (retries-- > 0) { 7244 /* Bail out immediately if FPE handshake is OFF */ 7245 if (*lo_state == FPE_STATE_OFF || !*hs_enable) 7246 break; 7247 7248 if (*lo_state == FPE_STATE_ENTERING_ON && 7249 *lp_state == FPE_STATE_ENTERING_ON) { 7250 stmmac_fpe_configure(priv, priv->ioaddr, 7251 fpe_cfg, 7252 priv->plat->tx_queues_to_use, 7253 priv->plat->rx_queues_to_use, 7254 *enable); 7255 7256 netdev_info(priv->dev, "configured FPE\n"); 7257 7258 *lo_state = FPE_STATE_ON; 7259 *lp_state = FPE_STATE_ON; 7260 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); 7261 break; 7262 } 7263 7264 if ((*lo_state == FPE_STATE_CAPABLE || 7265 *lo_state == FPE_STATE_ENTERING_ON) && 7266 *lp_state != FPE_STATE_ON) { 7267 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, 7268 *lo_state, *lp_state); 7269 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 7270 fpe_cfg, 7271 MPACKET_VERIFY); 7272 } 7273 /* Sleep then retry */ 7274 msleep(500); 7275 } 7276 7277 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 7278 } 7279 7280 void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable) 7281 { 7282 if (priv->plat->fpe_cfg->hs_enable != enable) { 7283 if (enable) { 7284 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 7285 priv->plat->fpe_cfg, 7286 MPACKET_VERIFY); 7287 } else { 7288 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; 7289 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; 7290 } 7291 7292 priv->plat->fpe_cfg->hs_enable = enable; 7293 } 7294 } 7295 7296 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7297 { 7298 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7299 struct dma_desc *desc_contains_ts = ctx->desc; 7300 struct stmmac_priv *priv = ctx->priv; 7301 struct dma_desc *ndesc = ctx->ndesc; 7302 struct dma_desc *desc = ctx->desc; 7303 u64 ns = 0; 7304 7305 if (!priv->hwts_rx_en) 7306 return -ENODATA; 7307 7308 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7309 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 7310 desc_contains_ts = ndesc; 7311 7312 /* Check if timestamp is available */ 7313 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7314 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7315 ns -= priv->plat->cdc_error_adj; 7316 *timestamp = ns_to_ktime(ns); 7317 return 0; 7318 } 7319 7320 return -ENODATA; 7321 } 7322 7323 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7324 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7325 }; 7326 7327 /** 7328 * stmmac_dvr_probe 7329 * @device: device pointer 7330 * @plat_dat: platform data pointer 7331 * @res: stmmac resource pointer 7332 * Description: this is the main probe function used to 7333 * call the alloc_etherdev, allocate the priv structure. 7334 * Return: 7335 * returns 0 on success, otherwise errno. 7336 */ 7337 int stmmac_dvr_probe(struct device *device, 7338 struct plat_stmmacenet_data *plat_dat, 7339 struct stmmac_resources *res) 7340 { 7341 struct net_device *ndev = NULL; 7342 struct stmmac_priv *priv; 7343 u32 rxq; 7344 int i, ret = 0; 7345 7346 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7347 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7348 if (!ndev) 7349 return -ENOMEM; 7350 7351 SET_NETDEV_DEV(ndev, device); 7352 7353 priv = netdev_priv(ndev); 7354 priv->device = device; 7355 priv->dev = ndev; 7356 7357 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7358 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); 7359 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { 7360 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); 7361 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); 7362 } 7363 7364 priv->xstats.pcpu_stats = 7365 devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); 7366 if (!priv->xstats.pcpu_stats) 7367 return -ENOMEM; 7368 7369 stmmac_set_ethtool_ops(ndev); 7370 priv->pause = pause; 7371 priv->plat = plat_dat; 7372 priv->ioaddr = res->addr; 7373 priv->dev->base_addr = (unsigned long)res->addr; 7374 priv->plat->dma_cfg->multi_msi_en = 7375 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7376 7377 priv->dev->irq = res->irq; 7378 priv->wol_irq = res->wol_irq; 7379 priv->lpi_irq = res->lpi_irq; 7380 priv->sfty_ce_irq = res->sfty_ce_irq; 7381 priv->sfty_ue_irq = res->sfty_ue_irq; 7382 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7383 priv->rx_irq[i] = res->rx_irq[i]; 7384 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7385 priv->tx_irq[i] = res->tx_irq[i]; 7386 7387 if (!is_zero_ether_addr(res->mac)) 7388 eth_hw_addr_set(priv->dev, res->mac); 7389 7390 dev_set_drvdata(device, priv->dev); 7391 7392 /* Verify driver arguments */ 7393 stmmac_verify_args(); 7394 7395 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7396 if (!priv->af_xdp_zc_qps) 7397 return -ENOMEM; 7398 7399 /* Allocate workqueue */ 7400 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7401 if (!priv->wq) { 7402 dev_err(priv->device, "failed to create workqueue\n"); 7403 ret = -ENOMEM; 7404 goto error_wq_init; 7405 } 7406 7407 INIT_WORK(&priv->service_task, stmmac_service_task); 7408 7409 /* Initialize Link Partner FPE workqueue */ 7410 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); 7411 7412 /* Override with kernel parameters if supplied XXX CRS XXX 7413 * this needs to have multiple instances 7414 */ 7415 if ((phyaddr >= 0) && (phyaddr <= 31)) 7416 priv->plat->phy_addr = phyaddr; 7417 7418 if (priv->plat->stmmac_rst) { 7419 ret = reset_control_assert(priv->plat->stmmac_rst); 7420 reset_control_deassert(priv->plat->stmmac_rst); 7421 /* Some reset controllers have only reset callback instead of 7422 * assert + deassert callbacks pair. 7423 */ 7424 if (ret == -ENOTSUPP) 7425 reset_control_reset(priv->plat->stmmac_rst); 7426 } 7427 7428 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7429 if (ret == -ENOTSUPP) 7430 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7431 ERR_PTR(ret)); 7432 7433 /* Wait a bit for the reset to take effect */ 7434 udelay(10); 7435 7436 /* Init MAC and get the capabilities */ 7437 ret = stmmac_hw_init(priv); 7438 if (ret) 7439 goto error_hw_init; 7440 7441 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7442 */ 7443 if (priv->synopsys_id < DWMAC_CORE_5_20) 7444 priv->plat->dma_cfg->dche = false; 7445 7446 stmmac_check_ether_addr(priv); 7447 7448 ndev->netdev_ops = &stmmac_netdev_ops; 7449 7450 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7451 7452 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7453 NETIF_F_RXCSUM; 7454 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7455 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7456 7457 ret = stmmac_tc_init(priv, priv); 7458 if (!ret) { 7459 ndev->hw_features |= NETIF_F_HW_TC; 7460 } 7461 7462 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 7463 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 7464 if (priv->plat->has_gmac4) 7465 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 7466 priv->tso = true; 7467 dev_info(priv->device, "TSO feature enabled\n"); 7468 } 7469 7470 if (priv->dma_cap.sphen && 7471 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7472 ndev->hw_features |= NETIF_F_GRO; 7473 priv->sph_cap = true; 7474 priv->sph = priv->sph_cap; 7475 dev_info(priv->device, "SPH feature enabled\n"); 7476 } 7477 7478 /* Ideally our host DMA address width is the same as for the 7479 * device. However, it may differ and then we have to use our 7480 * host DMA width for allocation and the device DMA width for 7481 * register handling. 7482 */ 7483 if (priv->plat->host_dma_width) 7484 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7485 else 7486 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7487 7488 if (priv->dma_cap.host_dma_width) { 7489 ret = dma_set_mask_and_coherent(device, 7490 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7491 if (!ret) { 7492 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7493 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7494 7495 /* 7496 * If more than 32 bits can be addressed, make sure to 7497 * enable enhanced addressing mode. 7498 */ 7499 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7500 priv->plat->dma_cfg->eame = true; 7501 } else { 7502 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7503 if (ret) { 7504 dev_err(priv->device, "Failed to set DMA Mask\n"); 7505 goto error_hw_init; 7506 } 7507 7508 priv->dma_cap.host_dma_width = 32; 7509 } 7510 } 7511 7512 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7513 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7514 #ifdef STMMAC_VLAN_TAG_USED 7515 /* Both mac100 and gmac support receive VLAN tag detection */ 7516 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7517 if (priv->dma_cap.vlhash) { 7518 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7519 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7520 } 7521 if (priv->dma_cap.vlins) { 7522 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7523 if (priv->dma_cap.dvlan) 7524 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; 7525 } 7526 #endif 7527 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7528 7529 priv->xstats.threshold = tc; 7530 7531 /* Initialize RSS */ 7532 rxq = priv->plat->rx_queues_to_use; 7533 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7534 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7535 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7536 7537 if (priv->dma_cap.rssen && priv->plat->rss_en) 7538 ndev->features |= NETIF_F_RXHASH; 7539 7540 ndev->vlan_features |= ndev->features; 7541 /* TSO doesn't work on VLANs yet */ 7542 ndev->vlan_features &= ~NETIF_F_TSO; 7543 7544 /* MTU range: 46 - hw-specific max */ 7545 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 7546 if (priv->plat->has_xgmac) 7547 ndev->max_mtu = XGMAC_JUMBO_LEN; 7548 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 7549 ndev->max_mtu = JUMBO_LEN; 7550 else 7551 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 7552 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 7553 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 7554 */ 7555 if ((priv->plat->maxmtu < ndev->max_mtu) && 7556 (priv->plat->maxmtu >= ndev->min_mtu)) 7557 ndev->max_mtu = priv->plat->maxmtu; 7558 else if (priv->plat->maxmtu < ndev->min_mtu) 7559 dev_warn(priv->device, 7560 "%s: warning: maxmtu having invalid value (%d)\n", 7561 __func__, priv->plat->maxmtu); 7562 7563 if (flow_ctrl) 7564 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 7565 7566 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 7567 7568 /* Setup channels NAPI */ 7569 stmmac_napi_add(ndev); 7570 7571 mutex_init(&priv->lock); 7572 7573 /* If a specific clk_csr value is passed from the platform 7574 * this means that the CSR Clock Range selection cannot be 7575 * changed at run-time and it is fixed. Viceversa the driver'll try to 7576 * set the MDC clock dynamically according to the csr actual 7577 * clock input. 7578 */ 7579 if (priv->plat->clk_csr >= 0) 7580 priv->clk_csr = priv->plat->clk_csr; 7581 else 7582 stmmac_clk_csr_set(priv); 7583 7584 stmmac_check_pcs_mode(priv); 7585 7586 pm_runtime_get_noresume(device); 7587 pm_runtime_set_active(device); 7588 if (!pm_runtime_enabled(device)) 7589 pm_runtime_enable(device); 7590 7591 if (priv->hw->pcs != STMMAC_PCS_TBI && 7592 priv->hw->pcs != STMMAC_PCS_RTBI) { 7593 /* MDIO bus Registration */ 7594 ret = stmmac_mdio_register(ndev); 7595 if (ret < 0) { 7596 dev_err_probe(priv->device, ret, 7597 "%s: MDIO bus (id: %d) registration failed\n", 7598 __func__, priv->plat->bus_id); 7599 goto error_mdio_register; 7600 } 7601 } 7602 7603 if (priv->plat->speed_mode_2500) 7604 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); 7605 7606 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { 7607 ret = stmmac_xpcs_setup(priv->mii); 7608 if (ret) 7609 goto error_xpcs_setup; 7610 } 7611 7612 ret = stmmac_phy_setup(priv); 7613 if (ret) { 7614 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 7615 goto error_phy_setup; 7616 } 7617 7618 ret = register_netdev(ndev); 7619 if (ret) { 7620 dev_err(priv->device, "%s: ERROR %i registering the device\n", 7621 __func__, ret); 7622 goto error_netdev_register; 7623 } 7624 7625 #ifdef CONFIG_DEBUG_FS 7626 stmmac_init_fs(ndev); 7627 #endif 7628 7629 if (priv->plat->dump_debug_regs) 7630 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 7631 7632 /* Let pm_runtime_put() disable the clocks. 7633 * If CONFIG_PM is not enabled, the clocks will stay powered. 7634 */ 7635 pm_runtime_put(device); 7636 7637 return ret; 7638 7639 error_netdev_register: 7640 phylink_destroy(priv->phylink); 7641 error_xpcs_setup: 7642 error_phy_setup: 7643 if (priv->hw->pcs != STMMAC_PCS_TBI && 7644 priv->hw->pcs != STMMAC_PCS_RTBI) 7645 stmmac_mdio_unregister(ndev); 7646 error_mdio_register: 7647 stmmac_napi_del(ndev); 7648 error_hw_init: 7649 destroy_workqueue(priv->wq); 7650 error_wq_init: 7651 bitmap_free(priv->af_xdp_zc_qps); 7652 7653 return ret; 7654 } 7655 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 7656 7657 /** 7658 * stmmac_dvr_remove 7659 * @dev: device pointer 7660 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 7661 * changes the link status, releases the DMA descriptor rings. 7662 */ 7663 void stmmac_dvr_remove(struct device *dev) 7664 { 7665 struct net_device *ndev = dev_get_drvdata(dev); 7666 struct stmmac_priv *priv = netdev_priv(ndev); 7667 7668 netdev_info(priv->dev, "%s: removing driver", __func__); 7669 7670 pm_runtime_get_sync(dev); 7671 7672 stmmac_stop_all_dma(priv); 7673 stmmac_mac_set(priv, priv->ioaddr, false); 7674 netif_carrier_off(ndev); 7675 unregister_netdev(ndev); 7676 7677 #ifdef CONFIG_DEBUG_FS 7678 stmmac_exit_fs(ndev); 7679 #endif 7680 phylink_destroy(priv->phylink); 7681 if (priv->plat->stmmac_rst) 7682 reset_control_assert(priv->plat->stmmac_rst); 7683 reset_control_assert(priv->plat->stmmac_ahb_rst); 7684 if (priv->hw->pcs != STMMAC_PCS_TBI && 7685 priv->hw->pcs != STMMAC_PCS_RTBI) 7686 stmmac_mdio_unregister(ndev); 7687 destroy_workqueue(priv->wq); 7688 mutex_destroy(&priv->lock); 7689 bitmap_free(priv->af_xdp_zc_qps); 7690 7691 pm_runtime_disable(dev); 7692 pm_runtime_put_noidle(dev); 7693 } 7694 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 7695 7696 /** 7697 * stmmac_suspend - suspend callback 7698 * @dev: device pointer 7699 * Description: this is the function to suspend the device and it is called 7700 * by the platform driver to stop the network queue, release the resources, 7701 * program the PMT register (for WoL), clean and release driver resources. 7702 */ 7703 int stmmac_suspend(struct device *dev) 7704 { 7705 struct net_device *ndev = dev_get_drvdata(dev); 7706 struct stmmac_priv *priv = netdev_priv(ndev); 7707 u32 chan; 7708 7709 if (!ndev || !netif_running(ndev)) 7710 return 0; 7711 7712 mutex_lock(&priv->lock); 7713 7714 netif_device_detach(ndev); 7715 7716 stmmac_disable_all_queues(priv); 7717 7718 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7719 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7720 7721 if (priv->eee_enabled) { 7722 priv->tx_path_in_lpi_mode = false; 7723 del_timer_sync(&priv->eee_ctrl_timer); 7724 } 7725 7726 /* Stop TX/RX DMA */ 7727 stmmac_stop_all_dma(priv); 7728 7729 if (priv->plat->serdes_powerdown) 7730 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 7731 7732 /* Enable Power down mode by programming the PMT regs */ 7733 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7734 stmmac_pmt(priv, priv->hw, priv->wolopts); 7735 priv->irq_wake = 1; 7736 } else { 7737 stmmac_mac_set(priv, priv->ioaddr, false); 7738 pinctrl_pm_select_sleep_state(priv->device); 7739 } 7740 7741 mutex_unlock(&priv->lock); 7742 7743 rtnl_lock(); 7744 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7745 phylink_suspend(priv->phylink, true); 7746 } else { 7747 if (device_may_wakeup(priv->device)) 7748 phylink_speed_down(priv->phylink, false); 7749 phylink_suspend(priv->phylink, false); 7750 } 7751 rtnl_unlock(); 7752 7753 if (priv->dma_cap.fpesel) { 7754 /* Disable FPE */ 7755 stmmac_fpe_configure(priv, priv->ioaddr, 7756 priv->plat->fpe_cfg, 7757 priv->plat->tx_queues_to_use, 7758 priv->plat->rx_queues_to_use, false); 7759 7760 stmmac_fpe_handshake(priv, false); 7761 stmmac_fpe_stop_wq(priv); 7762 } 7763 7764 priv->speed = SPEED_UNKNOWN; 7765 return 0; 7766 } 7767 EXPORT_SYMBOL_GPL(stmmac_suspend); 7768 7769 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 7770 { 7771 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7772 7773 rx_q->cur_rx = 0; 7774 rx_q->dirty_rx = 0; 7775 } 7776 7777 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 7778 { 7779 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7780 7781 tx_q->cur_tx = 0; 7782 tx_q->dirty_tx = 0; 7783 tx_q->mss = 0; 7784 7785 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 7786 } 7787 7788 /** 7789 * stmmac_reset_queues_param - reset queue parameters 7790 * @priv: device pointer 7791 */ 7792 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 7793 { 7794 u32 rx_cnt = priv->plat->rx_queues_to_use; 7795 u32 tx_cnt = priv->plat->tx_queues_to_use; 7796 u32 queue; 7797 7798 for (queue = 0; queue < rx_cnt; queue++) 7799 stmmac_reset_rx_queue(priv, queue); 7800 7801 for (queue = 0; queue < tx_cnt; queue++) 7802 stmmac_reset_tx_queue(priv, queue); 7803 } 7804 7805 /** 7806 * stmmac_resume - resume callback 7807 * @dev: device pointer 7808 * Description: when resume this function is invoked to setup the DMA and CORE 7809 * in a usable state. 7810 */ 7811 int stmmac_resume(struct device *dev) 7812 { 7813 struct net_device *ndev = dev_get_drvdata(dev); 7814 struct stmmac_priv *priv = netdev_priv(ndev); 7815 int ret; 7816 7817 if (!netif_running(ndev)) 7818 return 0; 7819 7820 /* Power Down bit, into the PM register, is cleared 7821 * automatically as soon as a magic packet or a Wake-up frame 7822 * is received. Anyway, it's better to manually clear 7823 * this bit because it can generate problems while resuming 7824 * from another devices (e.g. serial console). 7825 */ 7826 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7827 mutex_lock(&priv->lock); 7828 stmmac_pmt(priv, priv->hw, 0); 7829 mutex_unlock(&priv->lock); 7830 priv->irq_wake = 0; 7831 } else { 7832 pinctrl_pm_select_default_state(priv->device); 7833 /* reset the phy so that it's ready */ 7834 if (priv->mii) 7835 stmmac_mdio_reset(priv->mii); 7836 } 7837 7838 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 7839 priv->plat->serdes_powerup) { 7840 ret = priv->plat->serdes_powerup(ndev, 7841 priv->plat->bsp_priv); 7842 7843 if (ret < 0) 7844 return ret; 7845 } 7846 7847 rtnl_lock(); 7848 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7849 phylink_resume(priv->phylink); 7850 } else { 7851 phylink_resume(priv->phylink); 7852 if (device_may_wakeup(priv->device)) 7853 phylink_speed_up(priv->phylink); 7854 } 7855 rtnl_unlock(); 7856 7857 rtnl_lock(); 7858 mutex_lock(&priv->lock); 7859 7860 stmmac_reset_queues_param(priv); 7861 7862 stmmac_free_tx_skbufs(priv); 7863 stmmac_clear_descriptors(priv, &priv->dma_conf); 7864 7865 stmmac_hw_setup(ndev, false); 7866 stmmac_init_coalesce(priv); 7867 stmmac_set_rx_mode(ndev); 7868 7869 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); 7870 7871 stmmac_enable_all_queues(priv); 7872 stmmac_enable_all_dma_irq(priv); 7873 7874 mutex_unlock(&priv->lock); 7875 rtnl_unlock(); 7876 7877 netif_device_attach(ndev); 7878 7879 return 0; 7880 } 7881 EXPORT_SYMBOL_GPL(stmmac_resume); 7882 7883 #ifndef MODULE 7884 static int __init stmmac_cmdline_opt(char *str) 7885 { 7886 char *opt; 7887 7888 if (!str || !*str) 7889 return 1; 7890 while ((opt = strsep(&str, ",")) != NULL) { 7891 if (!strncmp(opt, "debug:", 6)) { 7892 if (kstrtoint(opt + 6, 0, &debug)) 7893 goto err; 7894 } else if (!strncmp(opt, "phyaddr:", 8)) { 7895 if (kstrtoint(opt + 8, 0, &phyaddr)) 7896 goto err; 7897 } else if (!strncmp(opt, "buf_sz:", 7)) { 7898 if (kstrtoint(opt + 7, 0, &buf_sz)) 7899 goto err; 7900 } else if (!strncmp(opt, "tc:", 3)) { 7901 if (kstrtoint(opt + 3, 0, &tc)) 7902 goto err; 7903 } else if (!strncmp(opt, "watchdog:", 9)) { 7904 if (kstrtoint(opt + 9, 0, &watchdog)) 7905 goto err; 7906 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 7907 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 7908 goto err; 7909 } else if (!strncmp(opt, "pause:", 6)) { 7910 if (kstrtoint(opt + 6, 0, &pause)) 7911 goto err; 7912 } else if (!strncmp(opt, "eee_timer:", 10)) { 7913 if (kstrtoint(opt + 10, 0, &eee_timer)) 7914 goto err; 7915 } else if (!strncmp(opt, "chain_mode:", 11)) { 7916 if (kstrtoint(opt + 11, 0, &chain_mode)) 7917 goto err; 7918 } 7919 } 7920 return 1; 7921 7922 err: 7923 pr_err("%s: ERROR broken module parameter conversion", __func__); 7924 return 1; 7925 } 7926 7927 __setup("stmmaceth=", stmmac_cmdline_opt); 7928 #endif /* MODULE */ 7929 7930 static int __init stmmac_init(void) 7931 { 7932 #ifdef CONFIG_DEBUG_FS 7933 /* Create debugfs main directory if it doesn't exist yet */ 7934 if (!stmmac_fs_dir) 7935 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 7936 register_netdevice_notifier(&stmmac_notifier); 7937 #endif 7938 7939 return 0; 7940 } 7941 7942 static void __exit stmmac_exit(void) 7943 { 7944 #ifdef CONFIG_DEBUG_FS 7945 unregister_netdevice_notifier(&stmmac_notifier); 7946 debugfs_remove_recursive(stmmac_fs_dir); 7947 #endif 7948 } 7949 7950 module_init(stmmac_init) 7951 module_exit(stmmac_exit) 7952 7953 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 7954 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 7955 MODULE_LICENSE("GPL"); 7956