1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/clk.h> 18 #include <linux/kernel.h> 19 #include <linux/interrupt.h> 20 #include <linux/ip.h> 21 #include <linux/tcp.h> 22 #include <linux/skbuff.h> 23 #include <linux/ethtool.h> 24 #include <linux/if_ether.h> 25 #include <linux/crc32.h> 26 #include <linux/mii.h> 27 #include <linux/if.h> 28 #include <linux/if_vlan.h> 29 #include <linux/dma-mapping.h> 30 #include <linux/slab.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/prefetch.h> 33 #include <linux/pinctrl/consumer.h> 34 #ifdef CONFIG_DEBUG_FS 35 #include <linux/debugfs.h> 36 #include <linux/seq_file.h> 37 #endif /* CONFIG_DEBUG_FS */ 38 #include <linux/net_tstamp.h> 39 #include <linux/phylink.h> 40 #include <linux/udp.h> 41 #include <linux/bpf_trace.h> 42 #include <net/pkt_cls.h> 43 #include <net/xdp_sock_drv.h> 44 #include "stmmac_ptp.h" 45 #include "stmmac.h" 46 #include "stmmac_xdp.h" 47 #include <linux/reset.h> 48 #include <linux/of_mdio.h> 49 #include "dwmac1000.h" 50 #include "dwxgmac2.h" 51 #include "hwif.h" 52 53 /* As long as the interface is active, we keep the timestamping counter enabled 54 * with fine resolution and binary rollover. This avoid non-monotonic behavior 55 * (clock jumps) when changing timestamping settings at runtime. 56 */ 57 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ 58 PTP_TCR_TSCTRLSSR) 59 60 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 61 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 62 63 /* Module parameters */ 64 #define TX_TIMEO 5000 65 static int watchdog = TX_TIMEO; 66 module_param(watchdog, int, 0644); 67 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 68 69 static int debug = -1; 70 module_param(debug, int, 0644); 71 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 72 73 static int phyaddr = -1; 74 module_param(phyaddr, int, 0444); 75 MODULE_PARM_DESC(phyaddr, "Physical device address"); 76 77 #define STMMAC_TX_THRESH(x) ((x)->dma_tx_size / 4) 78 #define STMMAC_RX_THRESH(x) ((x)->dma_rx_size / 4) 79 80 /* Limit to make sure XDP TX and slow path can coexist */ 81 #define STMMAC_XSK_TX_BUDGET_MAX 256 82 #define STMMAC_TX_XSK_AVAIL 16 83 #define STMMAC_RX_FILL_BATCH 16 84 85 #define STMMAC_XDP_PASS 0 86 #define STMMAC_XDP_CONSUMED BIT(0) 87 #define STMMAC_XDP_TX BIT(1) 88 #define STMMAC_XDP_REDIRECT BIT(2) 89 90 static int flow_ctrl = FLOW_AUTO; 91 module_param(flow_ctrl, int, 0644); 92 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 93 94 static int pause = PAUSE_TIME; 95 module_param(pause, int, 0644); 96 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 97 98 #define TC_DEFAULT 64 99 static int tc = TC_DEFAULT; 100 module_param(tc, int, 0644); 101 MODULE_PARM_DESC(tc, "DMA threshold control value"); 102 103 #define DEFAULT_BUFSIZE 1536 104 static int buf_sz = DEFAULT_BUFSIZE; 105 module_param(buf_sz, int, 0644); 106 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 107 108 #define STMMAC_RX_COPYBREAK 256 109 110 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 111 NETIF_MSG_LINK | NETIF_MSG_IFUP | 112 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 113 114 #define STMMAC_DEFAULT_LPI_TIMER 1000 115 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 116 module_param(eee_timer, int, 0644); 117 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 118 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 119 120 /* By default the driver will use the ring mode to manage tx and rx descriptors, 121 * but allow user to force to use the chain instead of the ring 122 */ 123 static unsigned int chain_mode; 124 module_param(chain_mode, int, 0444); 125 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 126 127 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 128 /* For MSI interrupts handling */ 129 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 130 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 131 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 132 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 133 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 134 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 135 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 136 u32 rxmode, u32 chan); 137 138 #ifdef CONFIG_DEBUG_FS 139 static const struct net_device_ops stmmac_netdev_ops; 140 static void stmmac_init_fs(struct net_device *dev); 141 static void stmmac_exit_fs(struct net_device *dev); 142 #endif 143 144 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 145 146 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) 147 { 148 int ret = 0; 149 150 if (enabled) { 151 ret = clk_prepare_enable(priv->plat->stmmac_clk); 152 if (ret) 153 return ret; 154 ret = clk_prepare_enable(priv->plat->pclk); 155 if (ret) { 156 clk_disable_unprepare(priv->plat->stmmac_clk); 157 return ret; 158 } 159 if (priv->plat->clks_config) { 160 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); 161 if (ret) { 162 clk_disable_unprepare(priv->plat->stmmac_clk); 163 clk_disable_unprepare(priv->plat->pclk); 164 return ret; 165 } 166 } 167 } else { 168 clk_disable_unprepare(priv->plat->stmmac_clk); 169 clk_disable_unprepare(priv->plat->pclk); 170 if (priv->plat->clks_config) 171 priv->plat->clks_config(priv->plat->bsp_priv, enabled); 172 } 173 174 return ret; 175 } 176 EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); 177 178 /** 179 * stmmac_verify_args - verify the driver parameters. 180 * Description: it checks the driver parameters and set a default in case of 181 * errors. 182 */ 183 static void stmmac_verify_args(void) 184 { 185 if (unlikely(watchdog < 0)) 186 watchdog = TX_TIMEO; 187 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 188 buf_sz = DEFAULT_BUFSIZE; 189 if (unlikely(flow_ctrl > 1)) 190 flow_ctrl = FLOW_AUTO; 191 else if (likely(flow_ctrl < 0)) 192 flow_ctrl = FLOW_OFF; 193 if (unlikely((pause < 0) || (pause > 0xffff))) 194 pause = PAUSE_TIME; 195 if (eee_timer < 0) 196 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 197 } 198 199 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 200 { 201 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 202 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 203 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 204 u32 queue; 205 206 for (queue = 0; queue < maxq; queue++) { 207 struct stmmac_channel *ch = &priv->channel[queue]; 208 209 if (stmmac_xdp_is_enabled(priv) && 210 test_bit(queue, priv->af_xdp_zc_qps)) { 211 napi_disable(&ch->rxtx_napi); 212 continue; 213 } 214 215 if (queue < rx_queues_cnt) 216 napi_disable(&ch->rx_napi); 217 if (queue < tx_queues_cnt) 218 napi_disable(&ch->tx_napi); 219 } 220 } 221 222 /** 223 * stmmac_disable_all_queues - Disable all queues 224 * @priv: driver private structure 225 */ 226 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 227 { 228 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 229 struct stmmac_rx_queue *rx_q; 230 u32 queue; 231 232 /* synchronize_rcu() needed for pending XDP buffers to drain */ 233 for (queue = 0; queue < rx_queues_cnt; queue++) { 234 rx_q = &priv->rx_queue[queue]; 235 if (rx_q->xsk_pool) { 236 synchronize_rcu(); 237 break; 238 } 239 } 240 241 __stmmac_disable_all_queues(priv); 242 } 243 244 /** 245 * stmmac_enable_all_queues - Enable all queues 246 * @priv: driver private structure 247 */ 248 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 249 { 250 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 251 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 252 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 253 u32 queue; 254 255 for (queue = 0; queue < maxq; queue++) { 256 struct stmmac_channel *ch = &priv->channel[queue]; 257 258 if (stmmac_xdp_is_enabled(priv) && 259 test_bit(queue, priv->af_xdp_zc_qps)) { 260 napi_enable(&ch->rxtx_napi); 261 continue; 262 } 263 264 if (queue < rx_queues_cnt) 265 napi_enable(&ch->rx_napi); 266 if (queue < tx_queues_cnt) 267 napi_enable(&ch->tx_napi); 268 } 269 } 270 271 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 272 { 273 if (!test_bit(STMMAC_DOWN, &priv->state) && 274 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 275 queue_work(priv->wq, &priv->service_task); 276 } 277 278 static void stmmac_global_err(struct stmmac_priv *priv) 279 { 280 netif_carrier_off(priv->dev); 281 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 282 stmmac_service_event_schedule(priv); 283 } 284 285 /** 286 * stmmac_clk_csr_set - dynamically set the MDC clock 287 * @priv: driver private structure 288 * Description: this is to dynamically set the MDC clock according to the csr 289 * clock input. 290 * Note: 291 * If a specific clk_csr value is passed from the platform 292 * this means that the CSR Clock Range selection cannot be 293 * changed at run-time and it is fixed (as reported in the driver 294 * documentation). Viceversa the driver will try to set the MDC 295 * clock dynamically according to the actual clock input. 296 */ 297 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 298 { 299 u32 clk_rate; 300 301 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 302 303 /* Platform provided default clk_csr would be assumed valid 304 * for all other cases except for the below mentioned ones. 305 * For values higher than the IEEE 802.3 specified frequency 306 * we can not estimate the proper divider as it is not known 307 * the frequency of clk_csr_i. So we do not change the default 308 * divider. 309 */ 310 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 311 if (clk_rate < CSR_F_35M) 312 priv->clk_csr = STMMAC_CSR_20_35M; 313 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 314 priv->clk_csr = STMMAC_CSR_35_60M; 315 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 316 priv->clk_csr = STMMAC_CSR_60_100M; 317 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 318 priv->clk_csr = STMMAC_CSR_100_150M; 319 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 320 priv->clk_csr = STMMAC_CSR_150_250M; 321 else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) 322 priv->clk_csr = STMMAC_CSR_250_300M; 323 } 324 325 if (priv->plat->has_sun8i) { 326 if (clk_rate > 160000000) 327 priv->clk_csr = 0x03; 328 else if (clk_rate > 80000000) 329 priv->clk_csr = 0x02; 330 else if (clk_rate > 40000000) 331 priv->clk_csr = 0x01; 332 else 333 priv->clk_csr = 0; 334 } 335 336 if (priv->plat->has_xgmac) { 337 if (clk_rate > 400000000) 338 priv->clk_csr = 0x5; 339 else if (clk_rate > 350000000) 340 priv->clk_csr = 0x4; 341 else if (clk_rate > 300000000) 342 priv->clk_csr = 0x3; 343 else if (clk_rate > 250000000) 344 priv->clk_csr = 0x2; 345 else if (clk_rate > 150000000) 346 priv->clk_csr = 0x1; 347 else 348 priv->clk_csr = 0x0; 349 } 350 } 351 352 static void print_pkt(unsigned char *buf, int len) 353 { 354 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 355 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 356 } 357 358 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 359 { 360 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 361 u32 avail; 362 363 if (tx_q->dirty_tx > tx_q->cur_tx) 364 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 365 else 366 avail = priv->dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; 367 368 return avail; 369 } 370 371 /** 372 * stmmac_rx_dirty - Get RX queue dirty 373 * @priv: driver private structure 374 * @queue: RX queue index 375 */ 376 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 377 { 378 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 379 u32 dirty; 380 381 if (rx_q->dirty_rx <= rx_q->cur_rx) 382 dirty = rx_q->cur_rx - rx_q->dirty_rx; 383 else 384 dirty = priv->dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; 385 386 return dirty; 387 } 388 389 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) 390 { 391 int tx_lpi_timer; 392 393 /* Clear/set the SW EEE timer flag based on LPI ET enablement */ 394 priv->eee_sw_timer_en = en ? 0 : 1; 395 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; 396 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); 397 } 398 399 /** 400 * stmmac_enable_eee_mode - check and enter in LPI mode 401 * @priv: driver private structure 402 * Description: this function is to verify and enter in LPI mode in case of 403 * EEE. 404 */ 405 static int stmmac_enable_eee_mode(struct stmmac_priv *priv) 406 { 407 u32 tx_cnt = priv->plat->tx_queues_to_use; 408 u32 queue; 409 410 /* check if all TX queues have the work finished */ 411 for (queue = 0; queue < tx_cnt; queue++) { 412 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 413 414 if (tx_q->dirty_tx != tx_q->cur_tx) 415 return -EBUSY; /* still unfinished work */ 416 } 417 418 /* Check and enter in LPI mode */ 419 if (!priv->tx_path_in_lpi_mode) 420 stmmac_set_eee_mode(priv, priv->hw, 421 priv->plat->en_tx_lpi_clockgating); 422 return 0; 423 } 424 425 /** 426 * stmmac_disable_eee_mode - disable and exit from LPI mode 427 * @priv: driver private structure 428 * Description: this function is to exit and disable EEE in case of 429 * LPI state is true. This is called by the xmit. 430 */ 431 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 432 { 433 if (!priv->eee_sw_timer_en) { 434 stmmac_lpi_entry_timer_config(priv, 0); 435 return; 436 } 437 438 stmmac_reset_eee_mode(priv, priv->hw); 439 del_timer_sync(&priv->eee_ctrl_timer); 440 priv->tx_path_in_lpi_mode = false; 441 } 442 443 /** 444 * stmmac_eee_ctrl_timer - EEE TX SW timer. 445 * @t: timer_list struct containing private info 446 * Description: 447 * if there is no data transfer and if we are not in LPI state, 448 * then MAC Transmitter can be moved to LPI state. 449 */ 450 static void stmmac_eee_ctrl_timer(struct timer_list *t) 451 { 452 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 453 454 if (stmmac_enable_eee_mode(priv)) 455 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 456 } 457 458 /** 459 * stmmac_eee_init - init EEE 460 * @priv: driver private structure 461 * Description: 462 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 463 * can also manage EEE, this function enable the LPI state and start related 464 * timer. 465 */ 466 bool stmmac_eee_init(struct stmmac_priv *priv) 467 { 468 int eee_tw_timer = priv->eee_tw_timer; 469 470 /* Using PCS we cannot dial with the phy registers at this stage 471 * so we do not support extra feature like EEE. 472 */ 473 if (priv->hw->pcs == STMMAC_PCS_TBI || 474 priv->hw->pcs == STMMAC_PCS_RTBI) 475 return false; 476 477 /* Check if MAC core supports the EEE feature. */ 478 if (!priv->dma_cap.eee) 479 return false; 480 481 mutex_lock(&priv->lock); 482 483 /* Check if it needs to be deactivated */ 484 if (!priv->eee_active) { 485 if (priv->eee_enabled) { 486 netdev_dbg(priv->dev, "disable EEE\n"); 487 stmmac_lpi_entry_timer_config(priv, 0); 488 del_timer_sync(&priv->eee_ctrl_timer); 489 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); 490 if (priv->hw->xpcs) 491 xpcs_config_eee(priv->hw->xpcs, 492 priv->plat->mult_fact_100ns, 493 false); 494 } 495 mutex_unlock(&priv->lock); 496 return false; 497 } 498 499 if (priv->eee_active && !priv->eee_enabled) { 500 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 501 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 502 eee_tw_timer); 503 if (priv->hw->xpcs) 504 xpcs_config_eee(priv->hw->xpcs, 505 priv->plat->mult_fact_100ns, 506 true); 507 } 508 509 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { 510 del_timer_sync(&priv->eee_ctrl_timer); 511 priv->tx_path_in_lpi_mode = false; 512 stmmac_lpi_entry_timer_config(priv, 1); 513 } else { 514 stmmac_lpi_entry_timer_config(priv, 0); 515 mod_timer(&priv->eee_ctrl_timer, 516 STMMAC_LPI_T(priv->tx_lpi_timer)); 517 } 518 519 mutex_unlock(&priv->lock); 520 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 521 return true; 522 } 523 524 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 525 * @priv: driver private structure 526 * @p : descriptor pointer 527 * @skb : the socket buffer 528 * Description : 529 * This function will read timestamp from the descriptor & pass it to stack. 530 * and also perform some sanity checks. 531 */ 532 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 533 struct dma_desc *p, struct sk_buff *skb) 534 { 535 struct skb_shared_hwtstamps shhwtstamp; 536 bool found = false; 537 u64 ns = 0; 538 539 if (!priv->hwts_tx_en) 540 return; 541 542 /* exit if skb doesn't support hw tstamp */ 543 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 544 return; 545 546 /* check tx tstamp status */ 547 if (stmmac_get_tx_timestamp_status(priv, p)) { 548 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 549 found = true; 550 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 551 found = true; 552 } 553 554 if (found) { 555 ns -= priv->plat->cdc_error_adj; 556 557 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 558 shhwtstamp.hwtstamp = ns_to_ktime(ns); 559 560 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 561 /* pass tstamp to stack */ 562 skb_tstamp_tx(skb, &shhwtstamp); 563 } 564 } 565 566 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 567 * @priv: driver private structure 568 * @p : descriptor pointer 569 * @np : next descriptor pointer 570 * @skb : the socket buffer 571 * Description : 572 * This function will read received packet's timestamp from the descriptor 573 * and pass it to stack. It also perform some sanity checks. 574 */ 575 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 576 struct dma_desc *np, struct sk_buff *skb) 577 { 578 struct skb_shared_hwtstamps *shhwtstamp = NULL; 579 struct dma_desc *desc = p; 580 u64 ns = 0; 581 582 if (!priv->hwts_rx_en) 583 return; 584 /* For GMAC4, the valid timestamp is from CTX next desc. */ 585 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 586 desc = np; 587 588 /* Check if timestamp is available */ 589 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 590 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 591 592 ns -= priv->plat->cdc_error_adj; 593 594 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 595 shhwtstamp = skb_hwtstamps(skb); 596 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 597 shhwtstamp->hwtstamp = ns_to_ktime(ns); 598 } else { 599 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 600 } 601 } 602 603 /** 604 * stmmac_hwtstamp_set - control hardware timestamping. 605 * @dev: device pointer. 606 * @ifr: An IOCTL specific structure, that can contain a pointer to 607 * a proprietary structure used to pass information to the driver. 608 * Description: 609 * This function configures the MAC to enable/disable both outgoing(TX) 610 * and incoming(RX) packets time stamping based on user input. 611 * Return Value: 612 * 0 on success and an appropriate -ve integer on failure. 613 */ 614 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 615 { 616 struct stmmac_priv *priv = netdev_priv(dev); 617 struct hwtstamp_config config; 618 u32 ptp_v2 = 0; 619 u32 tstamp_all = 0; 620 u32 ptp_over_ipv4_udp = 0; 621 u32 ptp_over_ipv6_udp = 0; 622 u32 ptp_over_ethernet = 0; 623 u32 snap_type_sel = 0; 624 u32 ts_master_en = 0; 625 u32 ts_event_en = 0; 626 627 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 628 netdev_alert(priv->dev, "No support for HW time stamping\n"); 629 priv->hwts_tx_en = 0; 630 priv->hwts_rx_en = 0; 631 632 return -EOPNOTSUPP; 633 } 634 635 if (copy_from_user(&config, ifr->ifr_data, 636 sizeof(config))) 637 return -EFAULT; 638 639 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 640 __func__, config.flags, config.tx_type, config.rx_filter); 641 642 if (config.tx_type != HWTSTAMP_TX_OFF && 643 config.tx_type != HWTSTAMP_TX_ON) 644 return -ERANGE; 645 646 if (priv->adv_ts) { 647 switch (config.rx_filter) { 648 case HWTSTAMP_FILTER_NONE: 649 /* time stamp no incoming packet at all */ 650 config.rx_filter = HWTSTAMP_FILTER_NONE; 651 break; 652 653 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 654 /* PTP v1, UDP, any kind of event packet */ 655 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 656 /* 'xmac' hardware can support Sync, Pdelay_Req and 657 * Pdelay_resp by setting bit14 and bits17/16 to 01 658 * This leaves Delay_Req timestamps out. 659 * Enable all events *and* general purpose message 660 * timestamping 661 */ 662 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 663 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 664 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 665 break; 666 667 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 668 /* PTP v1, UDP, Sync packet */ 669 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 670 /* take time stamp for SYNC messages only */ 671 ts_event_en = PTP_TCR_TSEVNTENA; 672 673 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 674 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 675 break; 676 677 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 678 /* PTP v1, UDP, Delay_req packet */ 679 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 680 /* take time stamp for Delay_Req messages only */ 681 ts_master_en = PTP_TCR_TSMSTRENA; 682 ts_event_en = PTP_TCR_TSEVNTENA; 683 684 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 685 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 686 break; 687 688 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 689 /* PTP v2, UDP, any kind of event packet */ 690 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 691 ptp_v2 = PTP_TCR_TSVER2ENA; 692 /* take time stamp for all event messages */ 693 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 694 695 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 696 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 697 break; 698 699 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 700 /* PTP v2, UDP, Sync packet */ 701 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 702 ptp_v2 = PTP_TCR_TSVER2ENA; 703 /* take time stamp for SYNC messages only */ 704 ts_event_en = PTP_TCR_TSEVNTENA; 705 706 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 707 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 708 break; 709 710 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 711 /* PTP v2, UDP, Delay_req packet */ 712 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 713 ptp_v2 = PTP_TCR_TSVER2ENA; 714 /* take time stamp for Delay_Req messages only */ 715 ts_master_en = PTP_TCR_TSMSTRENA; 716 ts_event_en = PTP_TCR_TSEVNTENA; 717 718 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 719 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 720 break; 721 722 case HWTSTAMP_FILTER_PTP_V2_EVENT: 723 /* PTP v2/802.AS1 any layer, any kind of event packet */ 724 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 725 ptp_v2 = PTP_TCR_TSVER2ENA; 726 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 727 if (priv->synopsys_id < DWMAC_CORE_4_10) 728 ts_event_en = PTP_TCR_TSEVNTENA; 729 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 730 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 731 ptp_over_ethernet = PTP_TCR_TSIPENA; 732 break; 733 734 case HWTSTAMP_FILTER_PTP_V2_SYNC: 735 /* PTP v2/802.AS1, any layer, Sync packet */ 736 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 737 ptp_v2 = PTP_TCR_TSVER2ENA; 738 /* take time stamp for SYNC messages only */ 739 ts_event_en = PTP_TCR_TSEVNTENA; 740 741 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 742 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 743 ptp_over_ethernet = PTP_TCR_TSIPENA; 744 break; 745 746 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 747 /* PTP v2/802.AS1, any layer, Delay_req packet */ 748 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 749 ptp_v2 = PTP_TCR_TSVER2ENA; 750 /* take time stamp for Delay_Req messages only */ 751 ts_master_en = PTP_TCR_TSMSTRENA; 752 ts_event_en = PTP_TCR_TSEVNTENA; 753 754 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 755 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 756 ptp_over_ethernet = PTP_TCR_TSIPENA; 757 break; 758 759 case HWTSTAMP_FILTER_NTP_ALL: 760 case HWTSTAMP_FILTER_ALL: 761 /* time stamp any incoming packet */ 762 config.rx_filter = HWTSTAMP_FILTER_ALL; 763 tstamp_all = PTP_TCR_TSENALL; 764 break; 765 766 default: 767 return -ERANGE; 768 } 769 } else { 770 switch (config.rx_filter) { 771 case HWTSTAMP_FILTER_NONE: 772 config.rx_filter = HWTSTAMP_FILTER_NONE; 773 break; 774 default: 775 /* PTP v1, UDP, any kind of event packet */ 776 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 777 break; 778 } 779 } 780 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 781 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 782 783 priv->systime_flags = STMMAC_HWTS_ACTIVE; 784 785 if (priv->hwts_tx_en || priv->hwts_rx_en) { 786 priv->systime_flags |= tstamp_all | ptp_v2 | 787 ptp_over_ethernet | ptp_over_ipv6_udp | 788 ptp_over_ipv4_udp | ts_event_en | 789 ts_master_en | snap_type_sel; 790 } 791 792 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 793 794 memcpy(&priv->tstamp_config, &config, sizeof(config)); 795 796 return copy_to_user(ifr->ifr_data, &config, 797 sizeof(config)) ? -EFAULT : 0; 798 } 799 800 /** 801 * stmmac_hwtstamp_get - read hardware timestamping. 802 * @dev: device pointer. 803 * @ifr: An IOCTL specific structure, that can contain a pointer to 804 * a proprietary structure used to pass information to the driver. 805 * Description: 806 * This function obtain the current hardware timestamping settings 807 * as requested. 808 */ 809 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 810 { 811 struct stmmac_priv *priv = netdev_priv(dev); 812 struct hwtstamp_config *config = &priv->tstamp_config; 813 814 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 815 return -EOPNOTSUPP; 816 817 return copy_to_user(ifr->ifr_data, config, 818 sizeof(*config)) ? -EFAULT : 0; 819 } 820 821 /** 822 * stmmac_init_tstamp_counter - init hardware timestamping counter 823 * @priv: driver private structure 824 * @systime_flags: timestamping flags 825 * Description: 826 * Initialize hardware counter for packet timestamping. 827 * This is valid as long as the interface is open and not suspended. 828 * Will be rerun after resuming from suspend, case in which the timestamping 829 * flags updated by stmmac_hwtstamp_set() also need to be restored. 830 */ 831 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) 832 { 833 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 834 struct timespec64 now; 835 u32 sec_inc = 0; 836 u64 temp = 0; 837 838 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 839 return -EOPNOTSUPP; 840 841 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 842 priv->systime_flags = systime_flags; 843 844 /* program Sub Second Increment reg */ 845 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 846 priv->plat->clk_ptp_rate, 847 xmac, &sec_inc); 848 temp = div_u64(1000000000ULL, sec_inc); 849 850 /* Store sub second increment for later use */ 851 priv->sub_second_inc = sec_inc; 852 853 /* calculate default added value: 854 * formula is : 855 * addend = (2^32)/freq_div_ratio; 856 * where, freq_div_ratio = 1e9ns/sec_inc 857 */ 858 temp = (u64)(temp << 32); 859 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 860 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 861 862 /* initialize system time */ 863 ktime_get_real_ts64(&now); 864 865 /* lower 32 bits of tv_sec are safe until y2106 */ 866 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 867 868 return 0; 869 } 870 EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); 871 872 /** 873 * stmmac_init_ptp - init PTP 874 * @priv: driver private structure 875 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 876 * This is done by looking at the HW cap. register. 877 * This function also registers the ptp driver. 878 */ 879 static int stmmac_init_ptp(struct stmmac_priv *priv) 880 { 881 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 882 int ret; 883 884 if (priv->plat->ptp_clk_freq_config) 885 priv->plat->ptp_clk_freq_config(priv); 886 887 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); 888 if (ret) 889 return ret; 890 891 priv->adv_ts = 0; 892 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 893 if (xmac && priv->dma_cap.atime_stamp) 894 priv->adv_ts = 1; 895 /* Dwmac 3.x core with extend_desc can support adv_ts */ 896 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 897 priv->adv_ts = 1; 898 899 if (priv->dma_cap.time_stamp) 900 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 901 902 if (priv->adv_ts) 903 netdev_info(priv->dev, 904 "IEEE 1588-2008 Advanced Timestamp supported\n"); 905 906 priv->hwts_tx_en = 0; 907 priv->hwts_rx_en = 0; 908 909 return 0; 910 } 911 912 static void stmmac_release_ptp(struct stmmac_priv *priv) 913 { 914 clk_disable_unprepare(priv->plat->clk_ptp_ref); 915 stmmac_ptp_unregister(priv); 916 } 917 918 /** 919 * stmmac_mac_flow_ctrl - Configure flow control in all queues 920 * @priv: driver private structure 921 * @duplex: duplex passed to the next function 922 * Description: It is used for configuring the flow control in all queues 923 */ 924 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 925 { 926 u32 tx_cnt = priv->plat->tx_queues_to_use; 927 928 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 929 priv->pause, tx_cnt); 930 } 931 932 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 933 phy_interface_t interface) 934 { 935 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 936 937 if (!priv->hw->xpcs) 938 return NULL; 939 940 return &priv->hw->xpcs->pcs; 941 } 942 943 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 944 const struct phylink_link_state *state) 945 { 946 /* Nothing to do, xpcs_config() handles everything */ 947 } 948 949 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) 950 { 951 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 952 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 953 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 954 bool *hs_enable = &fpe_cfg->hs_enable; 955 956 if (is_up && *hs_enable) { 957 stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY); 958 } else { 959 *lo_state = FPE_STATE_OFF; 960 *lp_state = FPE_STATE_OFF; 961 } 962 } 963 964 static void stmmac_mac_link_down(struct phylink_config *config, 965 unsigned int mode, phy_interface_t interface) 966 { 967 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 968 969 stmmac_mac_set(priv, priv->ioaddr, false); 970 priv->eee_active = false; 971 priv->tx_lpi_enabled = false; 972 priv->eee_enabled = stmmac_eee_init(priv); 973 stmmac_set_eee_pls(priv, priv->hw, false); 974 975 if (priv->dma_cap.fpesel) 976 stmmac_fpe_link_state_handle(priv, false); 977 } 978 979 static void stmmac_mac_link_up(struct phylink_config *config, 980 struct phy_device *phy, 981 unsigned int mode, phy_interface_t interface, 982 int speed, int duplex, 983 bool tx_pause, bool rx_pause) 984 { 985 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 986 u32 ctrl; 987 988 ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 989 ctrl &= ~priv->hw->link.speed_mask; 990 991 if (interface == PHY_INTERFACE_MODE_USXGMII) { 992 switch (speed) { 993 case SPEED_10000: 994 ctrl |= priv->hw->link.xgmii.speed10000; 995 break; 996 case SPEED_5000: 997 ctrl |= priv->hw->link.xgmii.speed5000; 998 break; 999 case SPEED_2500: 1000 ctrl |= priv->hw->link.xgmii.speed2500; 1001 break; 1002 default: 1003 return; 1004 } 1005 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 1006 switch (speed) { 1007 case SPEED_100000: 1008 ctrl |= priv->hw->link.xlgmii.speed100000; 1009 break; 1010 case SPEED_50000: 1011 ctrl |= priv->hw->link.xlgmii.speed50000; 1012 break; 1013 case SPEED_40000: 1014 ctrl |= priv->hw->link.xlgmii.speed40000; 1015 break; 1016 case SPEED_25000: 1017 ctrl |= priv->hw->link.xlgmii.speed25000; 1018 break; 1019 case SPEED_10000: 1020 ctrl |= priv->hw->link.xgmii.speed10000; 1021 break; 1022 case SPEED_2500: 1023 ctrl |= priv->hw->link.speed2500; 1024 break; 1025 case SPEED_1000: 1026 ctrl |= priv->hw->link.speed1000; 1027 break; 1028 default: 1029 return; 1030 } 1031 } else { 1032 switch (speed) { 1033 case SPEED_2500: 1034 ctrl |= priv->hw->link.speed2500; 1035 break; 1036 case SPEED_1000: 1037 ctrl |= priv->hw->link.speed1000; 1038 break; 1039 case SPEED_100: 1040 ctrl |= priv->hw->link.speed100; 1041 break; 1042 case SPEED_10: 1043 ctrl |= priv->hw->link.speed10; 1044 break; 1045 default: 1046 return; 1047 } 1048 } 1049 1050 priv->speed = speed; 1051 1052 if (priv->plat->fix_mac_speed) 1053 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed); 1054 1055 if (!duplex) 1056 ctrl &= ~priv->hw->link.duplex; 1057 else 1058 ctrl |= priv->hw->link.duplex; 1059 1060 /* Flow Control operation */ 1061 if (tx_pause && rx_pause) 1062 stmmac_mac_flow_ctrl(priv, duplex); 1063 1064 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1065 1066 stmmac_mac_set(priv, priv->ioaddr, true); 1067 if (phy && priv->dma_cap.eee) { 1068 priv->eee_active = phy_init_eee(phy, 1) >= 0; 1069 priv->eee_enabled = stmmac_eee_init(priv); 1070 priv->tx_lpi_enabled = priv->eee_enabled; 1071 stmmac_set_eee_pls(priv, priv->hw, true); 1072 } 1073 1074 if (priv->dma_cap.fpesel) 1075 stmmac_fpe_link_state_handle(priv, true); 1076 } 1077 1078 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1079 .validate = phylink_generic_validate, 1080 .mac_select_pcs = stmmac_mac_select_pcs, 1081 .mac_config = stmmac_mac_config, 1082 .mac_link_down = stmmac_mac_link_down, 1083 .mac_link_up = stmmac_mac_link_up, 1084 }; 1085 1086 /** 1087 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1088 * @priv: driver private structure 1089 * Description: this is to verify if the HW supports the PCS. 1090 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1091 * configured for the TBI, RTBI, or SGMII PHY interface. 1092 */ 1093 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1094 { 1095 int interface = priv->plat->interface; 1096 1097 if (priv->dma_cap.pcs) { 1098 if ((interface == PHY_INTERFACE_MODE_RGMII) || 1099 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 1100 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 1101 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 1102 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 1103 priv->hw->pcs = STMMAC_PCS_RGMII; 1104 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 1105 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1106 priv->hw->pcs = STMMAC_PCS_SGMII; 1107 } 1108 } 1109 } 1110 1111 /** 1112 * stmmac_init_phy - PHY initialization 1113 * @dev: net device structure 1114 * Description: it initializes the driver's PHY state, and attaches the PHY 1115 * to the mac driver. 1116 * Return value: 1117 * 0 on success 1118 */ 1119 static int stmmac_init_phy(struct net_device *dev) 1120 { 1121 struct stmmac_priv *priv = netdev_priv(dev); 1122 struct device_node *node; 1123 int ret; 1124 1125 node = priv->plat->phylink_node; 1126 1127 if (node) 1128 ret = phylink_of_phy_connect(priv->phylink, node, 0); 1129 1130 /* Some DT bindings do not set-up the PHY handle. Let's try to 1131 * manually parse it 1132 */ 1133 if (!node || ret) { 1134 int addr = priv->plat->phy_addr; 1135 struct phy_device *phydev; 1136 1137 phydev = mdiobus_get_phy(priv->mii, addr); 1138 if (!phydev) { 1139 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1140 return -ENODEV; 1141 } 1142 1143 ret = phylink_connect_phy(priv->phylink, phydev); 1144 } 1145 1146 if (!priv->plat->pmt) { 1147 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 1148 1149 phylink_ethtool_get_wol(priv->phylink, &wol); 1150 device_set_wakeup_capable(priv->device, !!wol.supported); 1151 } 1152 1153 return ret; 1154 } 1155 1156 static int stmmac_phy_setup(struct stmmac_priv *priv) 1157 { 1158 struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data; 1159 struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node); 1160 int max_speed = priv->plat->max_speed; 1161 int mode = priv->plat->phy_interface; 1162 struct phylink *phylink; 1163 1164 priv->phylink_config.dev = &priv->dev->dev; 1165 priv->phylink_config.type = PHYLINK_NETDEV; 1166 if (priv->plat->mdio_bus_data) 1167 priv->phylink_config.ovr_an_inband = 1168 mdio_bus_data->xpcs_an_inband; 1169 1170 if (!fwnode) 1171 fwnode = dev_fwnode(priv->device); 1172 1173 /* Set the platform/firmware specified interface mode */ 1174 __set_bit(mode, priv->phylink_config.supported_interfaces); 1175 1176 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1177 if (priv->hw->xpcs) 1178 xpcs_get_interfaces(priv->hw->xpcs, 1179 priv->phylink_config.supported_interfaces); 1180 1181 priv->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | 1182 MAC_10 | MAC_100; 1183 1184 if (!max_speed || max_speed >= 1000) 1185 priv->phylink_config.mac_capabilities |= MAC_1000; 1186 1187 if (priv->plat->has_gmac4) { 1188 if (!max_speed || max_speed >= 2500) 1189 priv->phylink_config.mac_capabilities |= MAC_2500FD; 1190 } else if (priv->plat->has_xgmac) { 1191 if (!max_speed || max_speed >= 2500) 1192 priv->phylink_config.mac_capabilities |= MAC_2500FD; 1193 if (!max_speed || max_speed >= 5000) 1194 priv->phylink_config.mac_capabilities |= MAC_5000FD; 1195 if (!max_speed || max_speed >= 10000) 1196 priv->phylink_config.mac_capabilities |= MAC_10000FD; 1197 if (!max_speed || max_speed >= 25000) 1198 priv->phylink_config.mac_capabilities |= MAC_25000FD; 1199 if (!max_speed || max_speed >= 40000) 1200 priv->phylink_config.mac_capabilities |= MAC_40000FD; 1201 if (!max_speed || max_speed >= 50000) 1202 priv->phylink_config.mac_capabilities |= MAC_50000FD; 1203 if (!max_speed || max_speed >= 100000) 1204 priv->phylink_config.mac_capabilities |= MAC_100000FD; 1205 } 1206 1207 /* Half-Duplex can only work with single queue */ 1208 if (priv->plat->tx_queues_to_use > 1) 1209 priv->phylink_config.mac_capabilities &= 1210 ~(MAC_10HD | MAC_100HD | MAC_1000HD); 1211 1212 phylink = phylink_create(&priv->phylink_config, fwnode, 1213 mode, &stmmac_phylink_mac_ops); 1214 if (IS_ERR(phylink)) 1215 return PTR_ERR(phylink); 1216 1217 priv->phylink = phylink; 1218 return 0; 1219 } 1220 1221 static void stmmac_display_rx_rings(struct stmmac_priv *priv) 1222 { 1223 u32 rx_cnt = priv->plat->rx_queues_to_use; 1224 unsigned int desc_size; 1225 void *head_rx; 1226 u32 queue; 1227 1228 /* Display RX rings */ 1229 for (queue = 0; queue < rx_cnt; queue++) { 1230 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1231 1232 pr_info("\tRX Queue %u rings\n", queue); 1233 1234 if (priv->extend_desc) { 1235 head_rx = (void *)rx_q->dma_erx; 1236 desc_size = sizeof(struct dma_extended_desc); 1237 } else { 1238 head_rx = (void *)rx_q->dma_rx; 1239 desc_size = sizeof(struct dma_desc); 1240 } 1241 1242 /* Display RX ring */ 1243 stmmac_display_ring(priv, head_rx, priv->dma_rx_size, true, 1244 rx_q->dma_rx_phy, desc_size); 1245 } 1246 } 1247 1248 static void stmmac_display_tx_rings(struct stmmac_priv *priv) 1249 { 1250 u32 tx_cnt = priv->plat->tx_queues_to_use; 1251 unsigned int desc_size; 1252 void *head_tx; 1253 u32 queue; 1254 1255 /* Display TX rings */ 1256 for (queue = 0; queue < tx_cnt; queue++) { 1257 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1258 1259 pr_info("\tTX Queue %d rings\n", queue); 1260 1261 if (priv->extend_desc) { 1262 head_tx = (void *)tx_q->dma_etx; 1263 desc_size = sizeof(struct dma_extended_desc); 1264 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1265 head_tx = (void *)tx_q->dma_entx; 1266 desc_size = sizeof(struct dma_edesc); 1267 } else { 1268 head_tx = (void *)tx_q->dma_tx; 1269 desc_size = sizeof(struct dma_desc); 1270 } 1271 1272 stmmac_display_ring(priv, head_tx, priv->dma_tx_size, false, 1273 tx_q->dma_tx_phy, desc_size); 1274 } 1275 } 1276 1277 static void stmmac_display_rings(struct stmmac_priv *priv) 1278 { 1279 /* Display RX ring */ 1280 stmmac_display_rx_rings(priv); 1281 1282 /* Display TX ring */ 1283 stmmac_display_tx_rings(priv); 1284 } 1285 1286 static int stmmac_set_bfsize(int mtu, int bufsize) 1287 { 1288 int ret = bufsize; 1289 1290 if (mtu >= BUF_SIZE_8KiB) 1291 ret = BUF_SIZE_16KiB; 1292 else if (mtu >= BUF_SIZE_4KiB) 1293 ret = BUF_SIZE_8KiB; 1294 else if (mtu >= BUF_SIZE_2KiB) 1295 ret = BUF_SIZE_4KiB; 1296 else if (mtu > DEFAULT_BUFSIZE) 1297 ret = BUF_SIZE_2KiB; 1298 else 1299 ret = DEFAULT_BUFSIZE; 1300 1301 return ret; 1302 } 1303 1304 /** 1305 * stmmac_clear_rx_descriptors - clear RX descriptors 1306 * @priv: driver private structure 1307 * @queue: RX queue index 1308 * Description: this function is called to clear the RX descriptors 1309 * in case of both basic and extended descriptors are used. 1310 */ 1311 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) 1312 { 1313 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1314 int i; 1315 1316 /* Clear the RX descriptors */ 1317 for (i = 0; i < priv->dma_rx_size; i++) 1318 if (priv->extend_desc) 1319 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1320 priv->use_riwt, priv->mode, 1321 (i == priv->dma_rx_size - 1), 1322 priv->dma_buf_sz); 1323 else 1324 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1325 priv->use_riwt, priv->mode, 1326 (i == priv->dma_rx_size - 1), 1327 priv->dma_buf_sz); 1328 } 1329 1330 /** 1331 * stmmac_clear_tx_descriptors - clear tx descriptors 1332 * @priv: driver private structure 1333 * @queue: TX queue index. 1334 * Description: this function is called to clear the TX descriptors 1335 * in case of both basic and extended descriptors are used. 1336 */ 1337 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) 1338 { 1339 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1340 int i; 1341 1342 /* Clear the TX descriptors */ 1343 for (i = 0; i < priv->dma_tx_size; i++) { 1344 int last = (i == (priv->dma_tx_size - 1)); 1345 struct dma_desc *p; 1346 1347 if (priv->extend_desc) 1348 p = &tx_q->dma_etx[i].basic; 1349 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1350 p = &tx_q->dma_entx[i].basic; 1351 else 1352 p = &tx_q->dma_tx[i]; 1353 1354 stmmac_init_tx_desc(priv, p, priv->mode, last); 1355 } 1356 } 1357 1358 /** 1359 * stmmac_clear_descriptors - clear descriptors 1360 * @priv: driver private structure 1361 * Description: this function is called to clear the TX and RX descriptors 1362 * in case of both basic and extended descriptors are used. 1363 */ 1364 static void stmmac_clear_descriptors(struct stmmac_priv *priv) 1365 { 1366 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1367 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1368 u32 queue; 1369 1370 /* Clear the RX descriptors */ 1371 for (queue = 0; queue < rx_queue_cnt; queue++) 1372 stmmac_clear_rx_descriptors(priv, queue); 1373 1374 /* Clear the TX descriptors */ 1375 for (queue = 0; queue < tx_queue_cnt; queue++) 1376 stmmac_clear_tx_descriptors(priv, queue); 1377 } 1378 1379 /** 1380 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1381 * @priv: driver private structure 1382 * @p: descriptor pointer 1383 * @i: descriptor index 1384 * @flags: gfp flag 1385 * @queue: RX queue index 1386 * Description: this function is called to allocate a receive buffer, perform 1387 * the DMA mapping and init the descriptor. 1388 */ 1389 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p, 1390 int i, gfp_t flags, u32 queue) 1391 { 1392 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1393 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1394 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1395 1396 if (priv->dma_cap.addr64 <= 32) 1397 gfp |= GFP_DMA32; 1398 1399 if (!buf->page) { 1400 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1401 if (!buf->page) 1402 return -ENOMEM; 1403 buf->page_offset = stmmac_rx_offset(priv); 1404 } 1405 1406 if (priv->sph && !buf->sec_page) { 1407 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1408 if (!buf->sec_page) 1409 return -ENOMEM; 1410 1411 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1412 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1413 } else { 1414 buf->sec_page = NULL; 1415 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1416 } 1417 1418 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1419 1420 stmmac_set_desc_addr(priv, p, buf->addr); 1421 if (priv->dma_buf_sz == BUF_SIZE_16KiB) 1422 stmmac_init_desc3(priv, p); 1423 1424 return 0; 1425 } 1426 1427 /** 1428 * stmmac_free_rx_buffer - free RX dma buffers 1429 * @priv: private structure 1430 * @queue: RX queue index 1431 * @i: buffer index. 1432 */ 1433 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1434 { 1435 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1436 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1437 1438 if (buf->page) 1439 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1440 buf->page = NULL; 1441 1442 if (buf->sec_page) 1443 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1444 buf->sec_page = NULL; 1445 } 1446 1447 /** 1448 * stmmac_free_tx_buffer - free RX dma buffers 1449 * @priv: private structure 1450 * @queue: RX queue index 1451 * @i: buffer index. 1452 */ 1453 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) 1454 { 1455 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1456 1457 if (tx_q->tx_skbuff_dma[i].buf && 1458 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1459 if (tx_q->tx_skbuff_dma[i].map_as_page) 1460 dma_unmap_page(priv->device, 1461 tx_q->tx_skbuff_dma[i].buf, 1462 tx_q->tx_skbuff_dma[i].len, 1463 DMA_TO_DEVICE); 1464 else 1465 dma_unmap_single(priv->device, 1466 tx_q->tx_skbuff_dma[i].buf, 1467 tx_q->tx_skbuff_dma[i].len, 1468 DMA_TO_DEVICE); 1469 } 1470 1471 if (tx_q->xdpf[i] && 1472 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1473 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1474 xdp_return_frame(tx_q->xdpf[i]); 1475 tx_q->xdpf[i] = NULL; 1476 } 1477 1478 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1479 tx_q->xsk_frames_done++; 1480 1481 if (tx_q->tx_skbuff[i] && 1482 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1483 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1484 tx_q->tx_skbuff[i] = NULL; 1485 } 1486 1487 tx_q->tx_skbuff_dma[i].buf = 0; 1488 tx_q->tx_skbuff_dma[i].map_as_page = false; 1489 } 1490 1491 /** 1492 * dma_free_rx_skbufs - free RX dma buffers 1493 * @priv: private structure 1494 * @queue: RX queue index 1495 */ 1496 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) 1497 { 1498 int i; 1499 1500 for (i = 0; i < priv->dma_rx_size; i++) 1501 stmmac_free_rx_buffer(priv, queue, i); 1502 } 1503 1504 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, u32 queue, 1505 gfp_t flags) 1506 { 1507 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1508 int i; 1509 1510 for (i = 0; i < priv->dma_rx_size; i++) { 1511 struct dma_desc *p; 1512 int ret; 1513 1514 if (priv->extend_desc) 1515 p = &((rx_q->dma_erx + i)->basic); 1516 else 1517 p = rx_q->dma_rx + i; 1518 1519 ret = stmmac_init_rx_buffers(priv, p, i, flags, 1520 queue); 1521 if (ret) 1522 return ret; 1523 1524 rx_q->buf_alloc_num++; 1525 } 1526 1527 return 0; 1528 } 1529 1530 /** 1531 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1532 * @priv: private structure 1533 * @queue: RX queue index 1534 */ 1535 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, u32 queue) 1536 { 1537 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1538 int i; 1539 1540 for (i = 0; i < priv->dma_rx_size; i++) { 1541 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1542 1543 if (!buf->xdp) 1544 continue; 1545 1546 xsk_buff_free(buf->xdp); 1547 buf->xdp = NULL; 1548 } 1549 } 1550 1551 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, u32 queue) 1552 { 1553 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1554 int i; 1555 1556 for (i = 0; i < priv->dma_rx_size; i++) { 1557 struct stmmac_rx_buffer *buf; 1558 dma_addr_t dma_addr; 1559 struct dma_desc *p; 1560 1561 if (priv->extend_desc) 1562 p = (struct dma_desc *)(rx_q->dma_erx + i); 1563 else 1564 p = rx_q->dma_rx + i; 1565 1566 buf = &rx_q->buf_pool[i]; 1567 1568 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1569 if (!buf->xdp) 1570 return -ENOMEM; 1571 1572 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1573 stmmac_set_desc_addr(priv, p, dma_addr); 1574 rx_q->buf_alloc_num++; 1575 } 1576 1577 return 0; 1578 } 1579 1580 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1581 { 1582 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1583 return NULL; 1584 1585 return xsk_get_pool_from_qid(priv->dev, queue); 1586 } 1587 1588 /** 1589 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1590 * @priv: driver private structure 1591 * @queue: RX queue index 1592 * @flags: gfp flag. 1593 * Description: this function initializes the DMA RX descriptors 1594 * and allocates the socket buffers. It supports the chained and ring 1595 * modes. 1596 */ 1597 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, u32 queue, gfp_t flags) 1598 { 1599 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1600 int ret; 1601 1602 netif_dbg(priv, probe, priv->dev, 1603 "(%s) dma_rx_phy=0x%08x\n", __func__, 1604 (u32)rx_q->dma_rx_phy); 1605 1606 stmmac_clear_rx_descriptors(priv, queue); 1607 1608 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1609 1610 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1611 1612 if (rx_q->xsk_pool) { 1613 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1614 MEM_TYPE_XSK_BUFF_POOL, 1615 NULL)); 1616 netdev_info(priv->dev, 1617 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1618 rx_q->queue_index); 1619 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1620 } else { 1621 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1622 MEM_TYPE_PAGE_POOL, 1623 rx_q->page_pool)); 1624 netdev_info(priv->dev, 1625 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1626 rx_q->queue_index); 1627 } 1628 1629 if (rx_q->xsk_pool) { 1630 /* RX XDP ZC buffer pool may not be populated, e.g. 1631 * xdpsock TX-only. 1632 */ 1633 stmmac_alloc_rx_buffers_zc(priv, queue); 1634 } else { 1635 ret = stmmac_alloc_rx_buffers(priv, queue, flags); 1636 if (ret < 0) 1637 return -ENOMEM; 1638 } 1639 1640 rx_q->cur_rx = 0; 1641 rx_q->dirty_rx = 0; 1642 1643 /* Setup the chained descriptor addresses */ 1644 if (priv->mode == STMMAC_CHAIN_MODE) { 1645 if (priv->extend_desc) 1646 stmmac_mode_init(priv, rx_q->dma_erx, 1647 rx_q->dma_rx_phy, 1648 priv->dma_rx_size, 1); 1649 else 1650 stmmac_mode_init(priv, rx_q->dma_rx, 1651 rx_q->dma_rx_phy, 1652 priv->dma_rx_size, 0); 1653 } 1654 1655 return 0; 1656 } 1657 1658 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) 1659 { 1660 struct stmmac_priv *priv = netdev_priv(dev); 1661 u32 rx_count = priv->plat->rx_queues_to_use; 1662 int queue; 1663 int ret; 1664 1665 /* RX INITIALIZATION */ 1666 netif_dbg(priv, probe, priv->dev, 1667 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1668 1669 for (queue = 0; queue < rx_count; queue++) { 1670 ret = __init_dma_rx_desc_rings(priv, queue, flags); 1671 if (ret) 1672 goto err_init_rx_buffers; 1673 } 1674 1675 return 0; 1676 1677 err_init_rx_buffers: 1678 while (queue >= 0) { 1679 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1680 1681 if (rx_q->xsk_pool) 1682 dma_free_rx_xskbufs(priv, queue); 1683 else 1684 dma_free_rx_skbufs(priv, queue); 1685 1686 rx_q->buf_alloc_num = 0; 1687 rx_q->xsk_pool = NULL; 1688 1689 queue--; 1690 } 1691 1692 return ret; 1693 } 1694 1695 /** 1696 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1697 * @priv: driver private structure 1698 * @queue : TX queue index 1699 * Description: this function initializes the DMA TX descriptors 1700 * and allocates the socket buffers. It supports the chained and ring 1701 * modes. 1702 */ 1703 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, u32 queue) 1704 { 1705 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1706 int i; 1707 1708 netif_dbg(priv, probe, priv->dev, 1709 "(%s) dma_tx_phy=0x%08x\n", __func__, 1710 (u32)tx_q->dma_tx_phy); 1711 1712 /* Setup the chained descriptor addresses */ 1713 if (priv->mode == STMMAC_CHAIN_MODE) { 1714 if (priv->extend_desc) 1715 stmmac_mode_init(priv, tx_q->dma_etx, 1716 tx_q->dma_tx_phy, 1717 priv->dma_tx_size, 1); 1718 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 1719 stmmac_mode_init(priv, tx_q->dma_tx, 1720 tx_q->dma_tx_phy, 1721 priv->dma_tx_size, 0); 1722 } 1723 1724 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1725 1726 for (i = 0; i < priv->dma_tx_size; i++) { 1727 struct dma_desc *p; 1728 1729 if (priv->extend_desc) 1730 p = &((tx_q->dma_etx + i)->basic); 1731 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1732 p = &((tx_q->dma_entx + i)->basic); 1733 else 1734 p = tx_q->dma_tx + i; 1735 1736 stmmac_clear_desc(priv, p); 1737 1738 tx_q->tx_skbuff_dma[i].buf = 0; 1739 tx_q->tx_skbuff_dma[i].map_as_page = false; 1740 tx_q->tx_skbuff_dma[i].len = 0; 1741 tx_q->tx_skbuff_dma[i].last_segment = false; 1742 tx_q->tx_skbuff[i] = NULL; 1743 } 1744 1745 tx_q->dirty_tx = 0; 1746 tx_q->cur_tx = 0; 1747 tx_q->mss = 0; 1748 1749 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 1750 1751 return 0; 1752 } 1753 1754 static int init_dma_tx_desc_rings(struct net_device *dev) 1755 { 1756 struct stmmac_priv *priv = netdev_priv(dev); 1757 u32 tx_queue_cnt; 1758 u32 queue; 1759 1760 tx_queue_cnt = priv->plat->tx_queues_to_use; 1761 1762 for (queue = 0; queue < tx_queue_cnt; queue++) 1763 __init_dma_tx_desc_rings(priv, queue); 1764 1765 return 0; 1766 } 1767 1768 /** 1769 * init_dma_desc_rings - init the RX/TX descriptor rings 1770 * @dev: net device structure 1771 * @flags: gfp flag. 1772 * Description: this function initializes the DMA RX/TX descriptors 1773 * and allocates the socket buffers. It supports the chained and ring 1774 * modes. 1775 */ 1776 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags) 1777 { 1778 struct stmmac_priv *priv = netdev_priv(dev); 1779 int ret; 1780 1781 ret = init_dma_rx_desc_rings(dev, flags); 1782 if (ret) 1783 return ret; 1784 1785 ret = init_dma_tx_desc_rings(dev); 1786 1787 stmmac_clear_descriptors(priv); 1788 1789 if (netif_msg_hw(priv)) 1790 stmmac_display_rings(priv); 1791 1792 return ret; 1793 } 1794 1795 /** 1796 * dma_free_tx_skbufs - free TX dma buffers 1797 * @priv: private structure 1798 * @queue: TX queue index 1799 */ 1800 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) 1801 { 1802 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1803 int i; 1804 1805 tx_q->xsk_frames_done = 0; 1806 1807 for (i = 0; i < priv->dma_tx_size; i++) 1808 stmmac_free_tx_buffer(priv, queue, i); 1809 1810 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 1811 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 1812 tx_q->xsk_frames_done = 0; 1813 tx_q->xsk_pool = NULL; 1814 } 1815 } 1816 1817 /** 1818 * stmmac_free_tx_skbufs - free TX skb buffers 1819 * @priv: private structure 1820 */ 1821 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 1822 { 1823 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1824 u32 queue; 1825 1826 for (queue = 0; queue < tx_queue_cnt; queue++) 1827 dma_free_tx_skbufs(priv, queue); 1828 } 1829 1830 /** 1831 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 1832 * @priv: private structure 1833 * @queue: RX queue index 1834 */ 1835 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue) 1836 { 1837 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1838 1839 /* Release the DMA RX socket buffers */ 1840 if (rx_q->xsk_pool) 1841 dma_free_rx_xskbufs(priv, queue); 1842 else 1843 dma_free_rx_skbufs(priv, queue); 1844 1845 rx_q->buf_alloc_num = 0; 1846 rx_q->xsk_pool = NULL; 1847 1848 /* Free DMA regions of consistent memory previously allocated */ 1849 if (!priv->extend_desc) 1850 dma_free_coherent(priv->device, priv->dma_rx_size * 1851 sizeof(struct dma_desc), 1852 rx_q->dma_rx, rx_q->dma_rx_phy); 1853 else 1854 dma_free_coherent(priv->device, priv->dma_rx_size * 1855 sizeof(struct dma_extended_desc), 1856 rx_q->dma_erx, rx_q->dma_rx_phy); 1857 1858 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 1859 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 1860 1861 kfree(rx_q->buf_pool); 1862 if (rx_q->page_pool) 1863 page_pool_destroy(rx_q->page_pool); 1864 } 1865 1866 static void free_dma_rx_desc_resources(struct stmmac_priv *priv) 1867 { 1868 u32 rx_count = priv->plat->rx_queues_to_use; 1869 u32 queue; 1870 1871 /* Free RX queue resources */ 1872 for (queue = 0; queue < rx_count; queue++) 1873 __free_dma_rx_desc_resources(priv, queue); 1874 } 1875 1876 /** 1877 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 1878 * @priv: private structure 1879 * @queue: TX queue index 1880 */ 1881 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue) 1882 { 1883 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 1884 size_t size; 1885 void *addr; 1886 1887 /* Release the DMA TX socket buffers */ 1888 dma_free_tx_skbufs(priv, queue); 1889 1890 if (priv->extend_desc) { 1891 size = sizeof(struct dma_extended_desc); 1892 addr = tx_q->dma_etx; 1893 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1894 size = sizeof(struct dma_edesc); 1895 addr = tx_q->dma_entx; 1896 } else { 1897 size = sizeof(struct dma_desc); 1898 addr = tx_q->dma_tx; 1899 } 1900 1901 size *= priv->dma_tx_size; 1902 1903 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 1904 1905 kfree(tx_q->tx_skbuff_dma); 1906 kfree(tx_q->tx_skbuff); 1907 } 1908 1909 static void free_dma_tx_desc_resources(struct stmmac_priv *priv) 1910 { 1911 u32 tx_count = priv->plat->tx_queues_to_use; 1912 u32 queue; 1913 1914 /* Free TX queue resources */ 1915 for (queue = 0; queue < tx_count; queue++) 1916 __free_dma_tx_desc_resources(priv, queue); 1917 } 1918 1919 /** 1920 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 1921 * @priv: private structure 1922 * @queue: RX queue index 1923 * Description: according to which descriptor can be used (extend or basic) 1924 * this function allocates the resources for TX and RX paths. In case of 1925 * reception, for example, it pre-allocated the RX socket buffer in order to 1926 * allow zero-copy mechanism. 1927 */ 1928 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue) 1929 { 1930 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 1931 struct stmmac_channel *ch = &priv->channel[queue]; 1932 bool xdp_prog = stmmac_xdp_is_enabled(priv); 1933 struct page_pool_params pp_params = { 0 }; 1934 unsigned int num_pages; 1935 unsigned int napi_id; 1936 int ret; 1937 1938 rx_q->queue_index = queue; 1939 rx_q->priv_data = priv; 1940 1941 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 1942 pp_params.pool_size = priv->dma_rx_size; 1943 num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE); 1944 pp_params.order = ilog2(num_pages); 1945 pp_params.nid = dev_to_node(priv->device); 1946 pp_params.dev = priv->device; 1947 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 1948 pp_params.offset = stmmac_rx_offset(priv); 1949 pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages); 1950 1951 rx_q->page_pool = page_pool_create(&pp_params); 1952 if (IS_ERR(rx_q->page_pool)) { 1953 ret = PTR_ERR(rx_q->page_pool); 1954 rx_q->page_pool = NULL; 1955 return ret; 1956 } 1957 1958 rx_q->buf_pool = kcalloc(priv->dma_rx_size, 1959 sizeof(*rx_q->buf_pool), 1960 GFP_KERNEL); 1961 if (!rx_q->buf_pool) 1962 return -ENOMEM; 1963 1964 if (priv->extend_desc) { 1965 rx_q->dma_erx = dma_alloc_coherent(priv->device, 1966 priv->dma_rx_size * 1967 sizeof(struct dma_extended_desc), 1968 &rx_q->dma_rx_phy, 1969 GFP_KERNEL); 1970 if (!rx_q->dma_erx) 1971 return -ENOMEM; 1972 1973 } else { 1974 rx_q->dma_rx = dma_alloc_coherent(priv->device, 1975 priv->dma_rx_size * 1976 sizeof(struct dma_desc), 1977 &rx_q->dma_rx_phy, 1978 GFP_KERNEL); 1979 if (!rx_q->dma_rx) 1980 return -ENOMEM; 1981 } 1982 1983 if (stmmac_xdp_is_enabled(priv) && 1984 test_bit(queue, priv->af_xdp_zc_qps)) 1985 napi_id = ch->rxtx_napi.napi_id; 1986 else 1987 napi_id = ch->rx_napi.napi_id; 1988 1989 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, 1990 rx_q->queue_index, 1991 napi_id); 1992 if (ret) { 1993 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 1994 return -EINVAL; 1995 } 1996 1997 return 0; 1998 } 1999 2000 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv) 2001 { 2002 u32 rx_count = priv->plat->rx_queues_to_use; 2003 u32 queue; 2004 int ret; 2005 2006 /* RX queues buffers and DMA */ 2007 for (queue = 0; queue < rx_count; queue++) { 2008 ret = __alloc_dma_rx_desc_resources(priv, queue); 2009 if (ret) 2010 goto err_dma; 2011 } 2012 2013 return 0; 2014 2015 err_dma: 2016 free_dma_rx_desc_resources(priv); 2017 2018 return ret; 2019 } 2020 2021 /** 2022 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2023 * @priv: private structure 2024 * @queue: TX queue index 2025 * Description: according to which descriptor can be used (extend or basic) 2026 * this function allocates the resources for TX and RX paths. In case of 2027 * reception, for example, it pre-allocated the RX socket buffer in order to 2028 * allow zero-copy mechanism. 2029 */ 2030 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue) 2031 { 2032 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2033 size_t size; 2034 void *addr; 2035 2036 tx_q->queue_index = queue; 2037 tx_q->priv_data = priv; 2038 2039 tx_q->tx_skbuff_dma = kcalloc(priv->dma_tx_size, 2040 sizeof(*tx_q->tx_skbuff_dma), 2041 GFP_KERNEL); 2042 if (!tx_q->tx_skbuff_dma) 2043 return -ENOMEM; 2044 2045 tx_q->tx_skbuff = kcalloc(priv->dma_tx_size, 2046 sizeof(struct sk_buff *), 2047 GFP_KERNEL); 2048 if (!tx_q->tx_skbuff) 2049 return -ENOMEM; 2050 2051 if (priv->extend_desc) 2052 size = sizeof(struct dma_extended_desc); 2053 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2054 size = sizeof(struct dma_edesc); 2055 else 2056 size = sizeof(struct dma_desc); 2057 2058 size *= priv->dma_tx_size; 2059 2060 addr = dma_alloc_coherent(priv->device, size, 2061 &tx_q->dma_tx_phy, GFP_KERNEL); 2062 if (!addr) 2063 return -ENOMEM; 2064 2065 if (priv->extend_desc) 2066 tx_q->dma_etx = addr; 2067 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2068 tx_q->dma_entx = addr; 2069 else 2070 tx_q->dma_tx = addr; 2071 2072 return 0; 2073 } 2074 2075 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv) 2076 { 2077 u32 tx_count = priv->plat->tx_queues_to_use; 2078 u32 queue; 2079 int ret; 2080 2081 /* TX queues buffers and DMA */ 2082 for (queue = 0; queue < tx_count; queue++) { 2083 ret = __alloc_dma_tx_desc_resources(priv, queue); 2084 if (ret) 2085 goto err_dma; 2086 } 2087 2088 return 0; 2089 2090 err_dma: 2091 free_dma_tx_desc_resources(priv); 2092 return ret; 2093 } 2094 2095 /** 2096 * alloc_dma_desc_resources - alloc TX/RX resources. 2097 * @priv: private structure 2098 * Description: according to which descriptor can be used (extend or basic) 2099 * this function allocates the resources for TX and RX paths. In case of 2100 * reception, for example, it pre-allocated the RX socket buffer in order to 2101 * allow zero-copy mechanism. 2102 */ 2103 static int alloc_dma_desc_resources(struct stmmac_priv *priv) 2104 { 2105 /* RX Allocation */ 2106 int ret = alloc_dma_rx_desc_resources(priv); 2107 2108 if (ret) 2109 return ret; 2110 2111 ret = alloc_dma_tx_desc_resources(priv); 2112 2113 return ret; 2114 } 2115 2116 /** 2117 * free_dma_desc_resources - free dma desc resources 2118 * @priv: private structure 2119 */ 2120 static void free_dma_desc_resources(struct stmmac_priv *priv) 2121 { 2122 /* Release the DMA TX socket buffers */ 2123 free_dma_tx_desc_resources(priv); 2124 2125 /* Release the DMA RX socket buffers later 2126 * to ensure all pending XDP_TX buffers are returned. 2127 */ 2128 free_dma_rx_desc_resources(priv); 2129 } 2130 2131 /** 2132 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2133 * @priv: driver private structure 2134 * Description: It is used for enabling the rx queues in the MAC 2135 */ 2136 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2137 { 2138 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2139 int queue; 2140 u8 mode; 2141 2142 for (queue = 0; queue < rx_queues_count; queue++) { 2143 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2144 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2145 } 2146 } 2147 2148 /** 2149 * stmmac_start_rx_dma - start RX DMA channel 2150 * @priv: driver private structure 2151 * @chan: RX channel index 2152 * Description: 2153 * This starts a RX DMA channel 2154 */ 2155 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2156 { 2157 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2158 stmmac_start_rx(priv, priv->ioaddr, chan); 2159 } 2160 2161 /** 2162 * stmmac_start_tx_dma - start TX DMA channel 2163 * @priv: driver private structure 2164 * @chan: TX channel index 2165 * Description: 2166 * This starts a TX DMA channel 2167 */ 2168 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2169 { 2170 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2171 stmmac_start_tx(priv, priv->ioaddr, chan); 2172 } 2173 2174 /** 2175 * stmmac_stop_rx_dma - stop RX DMA channel 2176 * @priv: driver private structure 2177 * @chan: RX channel index 2178 * Description: 2179 * This stops a RX DMA channel 2180 */ 2181 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2182 { 2183 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2184 stmmac_stop_rx(priv, priv->ioaddr, chan); 2185 } 2186 2187 /** 2188 * stmmac_stop_tx_dma - stop TX DMA channel 2189 * @priv: driver private structure 2190 * @chan: TX channel index 2191 * Description: 2192 * This stops a TX DMA channel 2193 */ 2194 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2195 { 2196 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2197 stmmac_stop_tx(priv, priv->ioaddr, chan); 2198 } 2199 2200 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2201 { 2202 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2203 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2204 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2205 u32 chan; 2206 2207 for (chan = 0; chan < dma_csr_ch; chan++) { 2208 struct stmmac_channel *ch = &priv->channel[chan]; 2209 unsigned long flags; 2210 2211 spin_lock_irqsave(&ch->lock, flags); 2212 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2213 spin_unlock_irqrestore(&ch->lock, flags); 2214 } 2215 } 2216 2217 /** 2218 * stmmac_start_all_dma - start all RX and TX DMA channels 2219 * @priv: driver private structure 2220 * Description: 2221 * This starts all the RX and TX DMA channels 2222 */ 2223 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2224 { 2225 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2226 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2227 u32 chan = 0; 2228 2229 for (chan = 0; chan < rx_channels_count; chan++) 2230 stmmac_start_rx_dma(priv, chan); 2231 2232 for (chan = 0; chan < tx_channels_count; chan++) 2233 stmmac_start_tx_dma(priv, chan); 2234 } 2235 2236 /** 2237 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2238 * @priv: driver private structure 2239 * Description: 2240 * This stops the RX and TX DMA channels 2241 */ 2242 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2243 { 2244 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2245 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2246 u32 chan = 0; 2247 2248 for (chan = 0; chan < rx_channels_count; chan++) 2249 stmmac_stop_rx_dma(priv, chan); 2250 2251 for (chan = 0; chan < tx_channels_count; chan++) 2252 stmmac_stop_tx_dma(priv, chan); 2253 } 2254 2255 /** 2256 * stmmac_dma_operation_mode - HW DMA operation mode 2257 * @priv: driver private structure 2258 * Description: it is used for configuring the DMA operation mode register in 2259 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2260 */ 2261 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2262 { 2263 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2264 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2265 int rxfifosz = priv->plat->rx_fifo_size; 2266 int txfifosz = priv->plat->tx_fifo_size; 2267 u32 txmode = 0; 2268 u32 rxmode = 0; 2269 u32 chan = 0; 2270 u8 qmode = 0; 2271 2272 if (rxfifosz == 0) 2273 rxfifosz = priv->dma_cap.rx_fifo_size; 2274 if (txfifosz == 0) 2275 txfifosz = priv->dma_cap.tx_fifo_size; 2276 2277 /* Adjust for real per queue fifo size */ 2278 rxfifosz /= rx_channels_count; 2279 txfifosz /= tx_channels_count; 2280 2281 if (priv->plat->force_thresh_dma_mode) { 2282 txmode = tc; 2283 rxmode = tc; 2284 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2285 /* 2286 * In case of GMAC, SF mode can be enabled 2287 * to perform the TX COE in HW. This depends on: 2288 * 1) TX COE if actually supported 2289 * 2) There is no bugged Jumbo frame support 2290 * that needs to not insert csum in the TDES. 2291 */ 2292 txmode = SF_DMA_MODE; 2293 rxmode = SF_DMA_MODE; 2294 priv->xstats.threshold = SF_DMA_MODE; 2295 } else { 2296 txmode = tc; 2297 rxmode = SF_DMA_MODE; 2298 } 2299 2300 /* configure all channels */ 2301 for (chan = 0; chan < rx_channels_count; chan++) { 2302 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; 2303 u32 buf_size; 2304 2305 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2306 2307 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2308 rxfifosz, qmode); 2309 2310 if (rx_q->xsk_pool) { 2311 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 2312 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2313 buf_size, 2314 chan); 2315 } else { 2316 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2317 priv->dma_buf_sz, 2318 chan); 2319 } 2320 } 2321 2322 for (chan = 0; chan < tx_channels_count; chan++) { 2323 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2324 2325 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2326 txfifosz, qmode); 2327 } 2328 } 2329 2330 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2331 { 2332 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2333 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2334 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2335 unsigned int entry = tx_q->cur_tx; 2336 struct dma_desc *tx_desc = NULL; 2337 struct xdp_desc xdp_desc; 2338 bool work_done = true; 2339 2340 /* Avoids TX time-out as we are sharing with slow path */ 2341 txq_trans_cond_update(nq); 2342 2343 budget = min(budget, stmmac_tx_avail(priv, queue)); 2344 2345 while (budget-- > 0) { 2346 dma_addr_t dma_addr; 2347 bool set_ic; 2348 2349 /* We are sharing with slow path and stop XSK TX desc submission when 2350 * available TX ring is less than threshold. 2351 */ 2352 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2353 !netif_carrier_ok(priv->dev)) { 2354 work_done = false; 2355 break; 2356 } 2357 2358 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2359 break; 2360 2361 if (likely(priv->extend_desc)) 2362 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2363 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2364 tx_desc = &tx_q->dma_entx[entry].basic; 2365 else 2366 tx_desc = tx_q->dma_tx + entry; 2367 2368 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2369 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2370 2371 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; 2372 2373 /* To return XDP buffer to XSK pool, we simple call 2374 * xsk_tx_completed(), so we don't need to fill up 2375 * 'buf' and 'xdpf'. 2376 */ 2377 tx_q->tx_skbuff_dma[entry].buf = 0; 2378 tx_q->xdpf[entry] = NULL; 2379 2380 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2381 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; 2382 tx_q->tx_skbuff_dma[entry].last_segment = true; 2383 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2384 2385 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2386 2387 tx_q->tx_count_frames++; 2388 2389 if (!priv->tx_coal_frames[queue]) 2390 set_ic = false; 2391 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2392 set_ic = true; 2393 else 2394 set_ic = false; 2395 2396 if (set_ic) { 2397 tx_q->tx_count_frames = 0; 2398 stmmac_set_tx_ic(priv, tx_desc); 2399 priv->xstats.tx_set_ic_bit++; 2400 } 2401 2402 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2403 true, priv->mode, true, true, 2404 xdp_desc.len); 2405 2406 stmmac_enable_dma_transmission(priv, priv->ioaddr); 2407 2408 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); 2409 entry = tx_q->cur_tx; 2410 } 2411 2412 if (tx_desc) { 2413 stmmac_flush_tx_descriptors(priv, queue); 2414 xsk_tx_release(pool); 2415 } 2416 2417 /* Return true if all of the 3 conditions are met 2418 * a) TX Budget is still available 2419 * b) work_done = true when XSK TX desc peek is empty (no more 2420 * pending XSK TX for transmission) 2421 */ 2422 return !!budget && work_done; 2423 } 2424 2425 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2426 { 2427 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2428 tc += 64; 2429 2430 if (priv->plat->force_thresh_dma_mode) 2431 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2432 else 2433 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2434 chan); 2435 2436 priv->xstats.threshold = tc; 2437 } 2438 } 2439 2440 /** 2441 * stmmac_tx_clean - to manage the transmission completion 2442 * @priv: driver private structure 2443 * @budget: napi budget limiting this functions packet handling 2444 * @queue: TX queue index 2445 * Description: it reclaims the transmit resources after transmission completes. 2446 */ 2447 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) 2448 { 2449 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2450 unsigned int bytes_compl = 0, pkts_compl = 0; 2451 unsigned int entry, xmits = 0, count = 0; 2452 2453 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2454 2455 priv->xstats.tx_clean++; 2456 2457 tx_q->xsk_frames_done = 0; 2458 2459 entry = tx_q->dirty_tx; 2460 2461 /* Try to clean all TX complete frame in 1 shot */ 2462 while ((entry != tx_q->cur_tx) && count < priv->dma_tx_size) { 2463 struct xdp_frame *xdpf; 2464 struct sk_buff *skb; 2465 struct dma_desc *p; 2466 int status; 2467 2468 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2469 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2470 xdpf = tx_q->xdpf[entry]; 2471 skb = NULL; 2472 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2473 xdpf = NULL; 2474 skb = tx_q->tx_skbuff[entry]; 2475 } else { 2476 xdpf = NULL; 2477 skb = NULL; 2478 } 2479 2480 if (priv->extend_desc) 2481 p = (struct dma_desc *)(tx_q->dma_etx + entry); 2482 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2483 p = &tx_q->dma_entx[entry].basic; 2484 else 2485 p = tx_q->dma_tx + entry; 2486 2487 status = stmmac_tx_status(priv, &priv->dev->stats, 2488 &priv->xstats, p, priv->ioaddr); 2489 /* Check if the descriptor is owned by the DMA */ 2490 if (unlikely(status & tx_dma_own)) 2491 break; 2492 2493 count++; 2494 2495 /* Make sure descriptor fields are read after reading 2496 * the own bit. 2497 */ 2498 dma_rmb(); 2499 2500 /* Just consider the last segment and ...*/ 2501 if (likely(!(status & tx_not_ls))) { 2502 /* ... verify the status error condition */ 2503 if (unlikely(status & tx_err)) { 2504 priv->dev->stats.tx_errors++; 2505 if (unlikely(status & tx_err_bump_tc)) 2506 stmmac_bump_dma_threshold(priv, queue); 2507 } else { 2508 priv->dev->stats.tx_packets++; 2509 priv->xstats.tx_pkt_n++; 2510 priv->xstats.txq_stats[queue].tx_pkt_n++; 2511 } 2512 if (skb) 2513 stmmac_get_tx_hwtstamp(priv, p, skb); 2514 } 2515 2516 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2517 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2518 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2519 dma_unmap_page(priv->device, 2520 tx_q->tx_skbuff_dma[entry].buf, 2521 tx_q->tx_skbuff_dma[entry].len, 2522 DMA_TO_DEVICE); 2523 else 2524 dma_unmap_single(priv->device, 2525 tx_q->tx_skbuff_dma[entry].buf, 2526 tx_q->tx_skbuff_dma[entry].len, 2527 DMA_TO_DEVICE); 2528 tx_q->tx_skbuff_dma[entry].buf = 0; 2529 tx_q->tx_skbuff_dma[entry].len = 0; 2530 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2531 } 2532 2533 stmmac_clean_desc3(priv, tx_q, p); 2534 2535 tx_q->tx_skbuff_dma[entry].last_segment = false; 2536 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2537 2538 if (xdpf && 2539 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2540 xdp_return_frame_rx_napi(xdpf); 2541 tx_q->xdpf[entry] = NULL; 2542 } 2543 2544 if (xdpf && 2545 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2546 xdp_return_frame(xdpf); 2547 tx_q->xdpf[entry] = NULL; 2548 } 2549 2550 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2551 tx_q->xsk_frames_done++; 2552 2553 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2554 if (likely(skb)) { 2555 pkts_compl++; 2556 bytes_compl += skb->len; 2557 dev_consume_skb_any(skb); 2558 tx_q->tx_skbuff[entry] = NULL; 2559 } 2560 } 2561 2562 stmmac_release_tx_desc(priv, p, priv->mode); 2563 2564 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); 2565 } 2566 tx_q->dirty_tx = entry; 2567 2568 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2569 pkts_compl, bytes_compl); 2570 2571 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2572 queue))) && 2573 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2574 2575 netif_dbg(priv, tx_done, priv->dev, 2576 "%s: restart transmit\n", __func__); 2577 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2578 } 2579 2580 if (tx_q->xsk_pool) { 2581 bool work_done; 2582 2583 if (tx_q->xsk_frames_done) 2584 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2585 2586 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2587 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2588 2589 /* For XSK TX, we try to send as many as possible. 2590 * If XSK work done (XSK TX desc empty and budget still 2591 * available), return "budget - 1" to reenable TX IRQ. 2592 * Else, return "budget" to make NAPI continue polling. 2593 */ 2594 work_done = stmmac_xdp_xmit_zc(priv, queue, 2595 STMMAC_XSK_TX_BUDGET_MAX); 2596 if (work_done) 2597 xmits = budget - 1; 2598 else 2599 xmits = budget; 2600 } 2601 2602 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && 2603 priv->eee_sw_timer_en) { 2604 if (stmmac_enable_eee_mode(priv)) 2605 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 2606 } 2607 2608 /* We still have pending packets, let's call for a new scheduling */ 2609 if (tx_q->dirty_tx != tx_q->cur_tx) 2610 hrtimer_start(&tx_q->txtimer, 2611 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), 2612 HRTIMER_MODE_REL); 2613 2614 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2615 2616 /* Combine decisions from TX clean and XSK TX */ 2617 return max(count, xmits); 2618 } 2619 2620 /** 2621 * stmmac_tx_err - to manage the tx error 2622 * @priv: driver private structure 2623 * @chan: channel index 2624 * Description: it cleans the descriptors and restarts the transmission 2625 * in case of transmission errors. 2626 */ 2627 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 2628 { 2629 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 2630 2631 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 2632 2633 stmmac_stop_tx_dma(priv, chan); 2634 dma_free_tx_skbufs(priv, chan); 2635 stmmac_clear_tx_descriptors(priv, chan); 2636 tx_q->dirty_tx = 0; 2637 tx_q->cur_tx = 0; 2638 tx_q->mss = 0; 2639 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); 2640 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2641 tx_q->dma_tx_phy, chan); 2642 stmmac_start_tx_dma(priv, chan); 2643 2644 priv->dev->stats.tx_errors++; 2645 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2646 } 2647 2648 /** 2649 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2650 * @priv: driver private structure 2651 * @txmode: TX operating mode 2652 * @rxmode: RX operating mode 2653 * @chan: channel index 2654 * Description: it is used for configuring of the DMA operation mode in 2655 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2656 * mode. 2657 */ 2658 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2659 u32 rxmode, u32 chan) 2660 { 2661 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2662 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2663 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2664 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2665 int rxfifosz = priv->plat->rx_fifo_size; 2666 int txfifosz = priv->plat->tx_fifo_size; 2667 2668 if (rxfifosz == 0) 2669 rxfifosz = priv->dma_cap.rx_fifo_size; 2670 if (txfifosz == 0) 2671 txfifosz = priv->dma_cap.tx_fifo_size; 2672 2673 /* Adjust for real per queue fifo size */ 2674 rxfifosz /= rx_channels_count; 2675 txfifosz /= tx_channels_count; 2676 2677 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2678 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2679 } 2680 2681 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2682 { 2683 int ret; 2684 2685 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2686 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2687 if (ret && (ret != -EINVAL)) { 2688 stmmac_global_err(priv); 2689 return true; 2690 } 2691 2692 return false; 2693 } 2694 2695 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 2696 { 2697 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2698 &priv->xstats, chan, dir); 2699 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; 2700 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 2701 struct stmmac_channel *ch = &priv->channel[chan]; 2702 struct napi_struct *rx_napi; 2703 struct napi_struct *tx_napi; 2704 unsigned long flags; 2705 2706 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 2707 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 2708 2709 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 2710 if (napi_schedule_prep(rx_napi)) { 2711 spin_lock_irqsave(&ch->lock, flags); 2712 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 2713 spin_unlock_irqrestore(&ch->lock, flags); 2714 __napi_schedule(rx_napi); 2715 } 2716 } 2717 2718 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 2719 if (napi_schedule_prep(tx_napi)) { 2720 spin_lock_irqsave(&ch->lock, flags); 2721 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 2722 spin_unlock_irqrestore(&ch->lock, flags); 2723 __napi_schedule(tx_napi); 2724 } 2725 } 2726 2727 return status; 2728 } 2729 2730 /** 2731 * stmmac_dma_interrupt - DMA ISR 2732 * @priv: driver private structure 2733 * Description: this is the DMA ISR. It is called by the main ISR. 2734 * It calls the dwmac dma routine and schedule poll method in case of some 2735 * work can be done. 2736 */ 2737 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2738 { 2739 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2740 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2741 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2742 tx_channel_count : rx_channel_count; 2743 u32 chan; 2744 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2745 2746 /* Make sure we never check beyond our status buffer. */ 2747 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2748 channels_to_check = ARRAY_SIZE(status); 2749 2750 for (chan = 0; chan < channels_to_check; chan++) 2751 status[chan] = stmmac_napi_check(priv, chan, 2752 DMA_DIR_RXTX); 2753 2754 for (chan = 0; chan < tx_channel_count; chan++) { 2755 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2756 /* Try to bump up the dma threshold on this failure */ 2757 stmmac_bump_dma_threshold(priv, chan); 2758 } else if (unlikely(status[chan] == tx_hard_error)) { 2759 stmmac_tx_err(priv, chan); 2760 } 2761 } 2762 } 2763 2764 /** 2765 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2766 * @priv: driver private structure 2767 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2768 */ 2769 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2770 { 2771 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2772 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2773 2774 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 2775 2776 if (priv->dma_cap.rmon) { 2777 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 2778 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2779 } else 2780 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2781 } 2782 2783 /** 2784 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2785 * @priv: driver private structure 2786 * Description: 2787 * new GMAC chip generations have a new register to indicate the 2788 * presence of the optional feature/functions. 2789 * This can be also used to override the value passed through the 2790 * platform and necessary for old MAC10/100 and GMAC chips. 2791 */ 2792 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2793 { 2794 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 2795 } 2796 2797 /** 2798 * stmmac_check_ether_addr - check if the MAC addr is valid 2799 * @priv: driver private structure 2800 * Description: 2801 * it is to verify if the MAC address is valid, in case of failures it 2802 * generates a random MAC address 2803 */ 2804 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2805 { 2806 u8 addr[ETH_ALEN]; 2807 2808 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2809 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 2810 if (is_valid_ether_addr(addr)) 2811 eth_hw_addr_set(priv->dev, addr); 2812 else 2813 eth_hw_addr_random(priv->dev); 2814 dev_info(priv->device, "device MAC address %pM\n", 2815 priv->dev->dev_addr); 2816 } 2817 } 2818 2819 /** 2820 * stmmac_init_dma_engine - DMA init. 2821 * @priv: driver private structure 2822 * Description: 2823 * It inits the DMA invoking the specific MAC/GMAC callback. 2824 * Some DMA parameters can be passed from the platform; 2825 * in case of these are not passed a default is kept for the MAC or GMAC. 2826 */ 2827 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2828 { 2829 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2830 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2831 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2832 struct stmmac_rx_queue *rx_q; 2833 struct stmmac_tx_queue *tx_q; 2834 u32 chan = 0; 2835 int atds = 0; 2836 int ret = 0; 2837 2838 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2839 dev_err(priv->device, "Invalid DMA configuration\n"); 2840 return -EINVAL; 2841 } 2842 2843 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2844 atds = 1; 2845 2846 ret = stmmac_reset(priv, priv->ioaddr); 2847 if (ret) { 2848 dev_err(priv->device, "Failed to reset the dma\n"); 2849 return ret; 2850 } 2851 2852 /* DMA Configuration */ 2853 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); 2854 2855 if (priv->plat->axi) 2856 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 2857 2858 /* DMA CSR Channel configuration */ 2859 for (chan = 0; chan < dma_csr_ch; chan++) { 2860 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 2861 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2862 } 2863 2864 /* DMA RX Channel Configuration */ 2865 for (chan = 0; chan < rx_channels_count; chan++) { 2866 rx_q = &priv->rx_queue[chan]; 2867 2868 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2869 rx_q->dma_rx_phy, chan); 2870 2871 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2872 (rx_q->buf_alloc_num * 2873 sizeof(struct dma_desc)); 2874 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 2875 rx_q->rx_tail_addr, chan); 2876 } 2877 2878 /* DMA TX Channel Configuration */ 2879 for (chan = 0; chan < tx_channels_count; chan++) { 2880 tx_q = &priv->tx_queue[chan]; 2881 2882 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2883 tx_q->dma_tx_phy, chan); 2884 2885 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 2886 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 2887 tx_q->tx_tail_addr, chan); 2888 } 2889 2890 return ret; 2891 } 2892 2893 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 2894 { 2895 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 2896 2897 hrtimer_start(&tx_q->txtimer, 2898 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), 2899 HRTIMER_MODE_REL); 2900 } 2901 2902 /** 2903 * stmmac_tx_timer - mitigation sw timer for tx. 2904 * @t: data pointer 2905 * Description: 2906 * This is the timer handler to directly invoke the stmmac_tx_clean. 2907 */ 2908 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 2909 { 2910 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 2911 struct stmmac_priv *priv = tx_q->priv_data; 2912 struct stmmac_channel *ch; 2913 struct napi_struct *napi; 2914 2915 ch = &priv->channel[tx_q->queue_index]; 2916 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 2917 2918 if (likely(napi_schedule_prep(napi))) { 2919 unsigned long flags; 2920 2921 spin_lock_irqsave(&ch->lock, flags); 2922 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 2923 spin_unlock_irqrestore(&ch->lock, flags); 2924 __napi_schedule(napi); 2925 } 2926 2927 return HRTIMER_NORESTART; 2928 } 2929 2930 /** 2931 * stmmac_init_coalesce - init mitigation options. 2932 * @priv: driver private structure 2933 * Description: 2934 * This inits the coalesce parameters: i.e. timer rate, 2935 * timer handler and default threshold used for enabling the 2936 * interrupt on completion bit. 2937 */ 2938 static void stmmac_init_coalesce(struct stmmac_priv *priv) 2939 { 2940 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2941 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2942 u32 chan; 2943 2944 for (chan = 0; chan < tx_channel_count; chan++) { 2945 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 2946 2947 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 2948 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 2949 2950 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 2951 tx_q->txtimer.function = stmmac_tx_timer; 2952 } 2953 2954 for (chan = 0; chan < rx_channel_count; chan++) 2955 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 2956 } 2957 2958 static void stmmac_set_rings_length(struct stmmac_priv *priv) 2959 { 2960 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2961 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2962 u32 chan; 2963 2964 /* set TX ring length */ 2965 for (chan = 0; chan < tx_channels_count; chan++) 2966 stmmac_set_tx_ring_len(priv, priv->ioaddr, 2967 (priv->dma_tx_size - 1), chan); 2968 2969 /* set RX ring length */ 2970 for (chan = 0; chan < rx_channels_count; chan++) 2971 stmmac_set_rx_ring_len(priv, priv->ioaddr, 2972 (priv->dma_rx_size - 1), chan); 2973 } 2974 2975 /** 2976 * stmmac_set_tx_queue_weight - Set TX queue weight 2977 * @priv: driver private structure 2978 * Description: It is used for setting TX queues weight 2979 */ 2980 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 2981 { 2982 u32 tx_queues_count = priv->plat->tx_queues_to_use; 2983 u32 weight; 2984 u32 queue; 2985 2986 for (queue = 0; queue < tx_queues_count; queue++) { 2987 weight = priv->plat->tx_queues_cfg[queue].weight; 2988 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 2989 } 2990 } 2991 2992 /** 2993 * stmmac_configure_cbs - Configure CBS in TX queue 2994 * @priv: driver private structure 2995 * Description: It is used for configuring CBS in AVB TX queues 2996 */ 2997 static void stmmac_configure_cbs(struct stmmac_priv *priv) 2998 { 2999 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3000 u32 mode_to_use; 3001 u32 queue; 3002 3003 /* queue 0 is reserved for legacy traffic */ 3004 for (queue = 1; queue < tx_queues_count; queue++) { 3005 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3006 if (mode_to_use == MTL_QUEUE_DCB) 3007 continue; 3008 3009 stmmac_config_cbs(priv, priv->hw, 3010 priv->plat->tx_queues_cfg[queue].send_slope, 3011 priv->plat->tx_queues_cfg[queue].idle_slope, 3012 priv->plat->tx_queues_cfg[queue].high_credit, 3013 priv->plat->tx_queues_cfg[queue].low_credit, 3014 queue); 3015 } 3016 } 3017 3018 /** 3019 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3020 * @priv: driver private structure 3021 * Description: It is used for mapping RX queues to RX dma channels 3022 */ 3023 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3024 { 3025 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3026 u32 queue; 3027 u32 chan; 3028 3029 for (queue = 0; queue < rx_queues_count; queue++) { 3030 chan = priv->plat->rx_queues_cfg[queue].chan; 3031 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3032 } 3033 } 3034 3035 /** 3036 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3037 * @priv: driver private structure 3038 * Description: It is used for configuring the RX Queue Priority 3039 */ 3040 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3041 { 3042 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3043 u32 queue; 3044 u32 prio; 3045 3046 for (queue = 0; queue < rx_queues_count; queue++) { 3047 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3048 continue; 3049 3050 prio = priv->plat->rx_queues_cfg[queue].prio; 3051 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3052 } 3053 } 3054 3055 /** 3056 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3057 * @priv: driver private structure 3058 * Description: It is used for configuring the TX Queue Priority 3059 */ 3060 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3061 { 3062 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3063 u32 queue; 3064 u32 prio; 3065 3066 for (queue = 0; queue < tx_queues_count; queue++) { 3067 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3068 continue; 3069 3070 prio = priv->plat->tx_queues_cfg[queue].prio; 3071 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3072 } 3073 } 3074 3075 /** 3076 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3077 * @priv: driver private structure 3078 * Description: It is used for configuring the RX queue routing 3079 */ 3080 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3081 { 3082 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3083 u32 queue; 3084 u8 packet; 3085 3086 for (queue = 0; queue < rx_queues_count; queue++) { 3087 /* no specific packet type routing specified for the queue */ 3088 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3089 continue; 3090 3091 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3092 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3093 } 3094 } 3095 3096 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3097 { 3098 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3099 priv->rss.enable = false; 3100 return; 3101 } 3102 3103 if (priv->dev->features & NETIF_F_RXHASH) 3104 priv->rss.enable = true; 3105 else 3106 priv->rss.enable = false; 3107 3108 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3109 priv->plat->rx_queues_to_use); 3110 } 3111 3112 /** 3113 * stmmac_mtl_configuration - Configure MTL 3114 * @priv: driver private structure 3115 * Description: It is used for configurring MTL 3116 */ 3117 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3118 { 3119 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3120 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3121 3122 if (tx_queues_count > 1) 3123 stmmac_set_tx_queue_weight(priv); 3124 3125 /* Configure MTL RX algorithms */ 3126 if (rx_queues_count > 1) 3127 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3128 priv->plat->rx_sched_algorithm); 3129 3130 /* Configure MTL TX algorithms */ 3131 if (tx_queues_count > 1) 3132 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3133 priv->plat->tx_sched_algorithm); 3134 3135 /* Configure CBS in AVB TX queues */ 3136 if (tx_queues_count > 1) 3137 stmmac_configure_cbs(priv); 3138 3139 /* Map RX MTL to DMA channels */ 3140 stmmac_rx_queue_dma_chan_map(priv); 3141 3142 /* Enable MAC RX Queues */ 3143 stmmac_mac_enable_rx_queues(priv); 3144 3145 /* Set RX priorities */ 3146 if (rx_queues_count > 1) 3147 stmmac_mac_config_rx_queues_prio(priv); 3148 3149 /* Set TX priorities */ 3150 if (tx_queues_count > 1) 3151 stmmac_mac_config_tx_queues_prio(priv); 3152 3153 /* Set RX routing */ 3154 if (rx_queues_count > 1) 3155 stmmac_mac_config_rx_queues_routing(priv); 3156 3157 /* Receive Side Scaling */ 3158 if (rx_queues_count > 1) 3159 stmmac_mac_config_rss(priv); 3160 } 3161 3162 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3163 { 3164 if (priv->dma_cap.asp) { 3165 netdev_info(priv->dev, "Enabling Safety Features\n"); 3166 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3167 priv->plat->safety_feat_cfg); 3168 } else { 3169 netdev_info(priv->dev, "No Safety Features support found\n"); 3170 } 3171 } 3172 3173 static int stmmac_fpe_start_wq(struct stmmac_priv *priv) 3174 { 3175 char *name; 3176 3177 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 3178 clear_bit(__FPE_REMOVING, &priv->fpe_task_state); 3179 3180 name = priv->wq_name; 3181 sprintf(name, "%s-fpe", priv->dev->name); 3182 3183 priv->fpe_wq = create_singlethread_workqueue(name); 3184 if (!priv->fpe_wq) { 3185 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); 3186 3187 return -ENOMEM; 3188 } 3189 netdev_info(priv->dev, "FPE workqueue start"); 3190 3191 return 0; 3192 } 3193 3194 /** 3195 * stmmac_hw_setup - setup mac in a usable state. 3196 * @dev : pointer to the device structure. 3197 * @ptp_register: register PTP if set 3198 * Description: 3199 * this is the main function to setup the HW in a usable state because the 3200 * dma engine is reset, the core registers are configured (e.g. AXI, 3201 * Checksum features, timers). The DMA is ready to start receiving and 3202 * transmitting. 3203 * Return value: 3204 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3205 * file on failure. 3206 */ 3207 static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) 3208 { 3209 struct stmmac_priv *priv = netdev_priv(dev); 3210 u32 rx_cnt = priv->plat->rx_queues_to_use; 3211 u32 tx_cnt = priv->plat->tx_queues_to_use; 3212 bool sph_en; 3213 u32 chan; 3214 int ret; 3215 3216 /* DMA initialization and SW reset */ 3217 ret = stmmac_init_dma_engine(priv); 3218 if (ret < 0) { 3219 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3220 __func__); 3221 return ret; 3222 } 3223 3224 /* Copy the MAC addr into the HW */ 3225 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3226 3227 /* PS and related bits will be programmed according to the speed */ 3228 if (priv->hw->pcs) { 3229 int speed = priv->plat->mac_port_sel_speed; 3230 3231 if ((speed == SPEED_10) || (speed == SPEED_100) || 3232 (speed == SPEED_1000)) { 3233 priv->hw->ps = speed; 3234 } else { 3235 dev_warn(priv->device, "invalid port speed\n"); 3236 priv->hw->ps = 0; 3237 } 3238 } 3239 3240 /* Initialize the MAC Core */ 3241 stmmac_core_init(priv, priv->hw, dev); 3242 3243 /* Initialize MTL*/ 3244 stmmac_mtl_configuration(priv); 3245 3246 /* Initialize Safety Features */ 3247 stmmac_safety_feat_configuration(priv); 3248 3249 ret = stmmac_rx_ipc(priv, priv->hw); 3250 if (!ret) { 3251 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3252 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3253 priv->hw->rx_csum = 0; 3254 } 3255 3256 /* Enable the MAC Rx/Tx */ 3257 stmmac_mac_set(priv, priv->ioaddr, true); 3258 3259 /* Set the HW DMA mode and the COE */ 3260 stmmac_dma_operation_mode(priv); 3261 3262 stmmac_mmc_setup(priv); 3263 3264 if (ptp_register) { 3265 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 3266 if (ret < 0) 3267 netdev_warn(priv->dev, 3268 "failed to enable PTP reference clock: %pe\n", 3269 ERR_PTR(ret)); 3270 } 3271 3272 ret = stmmac_init_ptp(priv); 3273 if (ret == -EOPNOTSUPP) 3274 netdev_info(priv->dev, "PTP not supported by HW\n"); 3275 else if (ret) 3276 netdev_warn(priv->dev, "PTP init failed\n"); 3277 else if (ptp_register) 3278 stmmac_ptp_register(priv); 3279 3280 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; 3281 3282 /* Convert the timer from msec to usec */ 3283 if (!priv->tx_lpi_timer) 3284 priv->tx_lpi_timer = eee_timer * 1000; 3285 3286 if (priv->use_riwt) { 3287 u32 queue; 3288 3289 for (queue = 0; queue < rx_cnt; queue++) { 3290 if (!priv->rx_riwt[queue]) 3291 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3292 3293 stmmac_rx_watchdog(priv, priv->ioaddr, 3294 priv->rx_riwt[queue], queue); 3295 } 3296 } 3297 3298 if (priv->hw->pcs) 3299 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); 3300 3301 /* set TX and RX rings length */ 3302 stmmac_set_rings_length(priv); 3303 3304 /* Enable TSO */ 3305 if (priv->tso) { 3306 for (chan = 0; chan < tx_cnt; chan++) { 3307 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 3308 3309 /* TSO and TBS cannot co-exist */ 3310 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3311 continue; 3312 3313 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3314 } 3315 } 3316 3317 /* Enable Split Header */ 3318 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 3319 for (chan = 0; chan < rx_cnt; chan++) 3320 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3321 3322 3323 /* VLAN Tag Insertion */ 3324 if (priv->dma_cap.vlins) 3325 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3326 3327 /* TBS */ 3328 for (chan = 0; chan < tx_cnt; chan++) { 3329 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 3330 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3331 3332 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3333 } 3334 3335 /* Configure real RX and TX queues */ 3336 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3337 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3338 3339 /* Start the ball rolling... */ 3340 stmmac_start_all_dma(priv); 3341 3342 if (priv->dma_cap.fpesel) { 3343 stmmac_fpe_start_wq(priv); 3344 3345 if (priv->plat->fpe_cfg->enable) 3346 stmmac_fpe_handshake(priv, true); 3347 } 3348 3349 return 0; 3350 } 3351 3352 static void stmmac_hw_teardown(struct net_device *dev) 3353 { 3354 struct stmmac_priv *priv = netdev_priv(dev); 3355 3356 clk_disable_unprepare(priv->plat->clk_ptp_ref); 3357 } 3358 3359 static void stmmac_free_irq(struct net_device *dev, 3360 enum request_irq_err irq_err, int irq_idx) 3361 { 3362 struct stmmac_priv *priv = netdev_priv(dev); 3363 int j; 3364 3365 switch (irq_err) { 3366 case REQ_IRQ_ERR_ALL: 3367 irq_idx = priv->plat->tx_queues_to_use; 3368 fallthrough; 3369 case REQ_IRQ_ERR_TX: 3370 for (j = irq_idx - 1; j >= 0; j--) { 3371 if (priv->tx_irq[j] > 0) { 3372 irq_set_affinity_hint(priv->tx_irq[j], NULL); 3373 free_irq(priv->tx_irq[j], &priv->tx_queue[j]); 3374 } 3375 } 3376 irq_idx = priv->plat->rx_queues_to_use; 3377 fallthrough; 3378 case REQ_IRQ_ERR_RX: 3379 for (j = irq_idx - 1; j >= 0; j--) { 3380 if (priv->rx_irq[j] > 0) { 3381 irq_set_affinity_hint(priv->rx_irq[j], NULL); 3382 free_irq(priv->rx_irq[j], &priv->rx_queue[j]); 3383 } 3384 } 3385 3386 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) 3387 free_irq(priv->sfty_ue_irq, dev); 3388 fallthrough; 3389 case REQ_IRQ_ERR_SFTY_UE: 3390 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) 3391 free_irq(priv->sfty_ce_irq, dev); 3392 fallthrough; 3393 case REQ_IRQ_ERR_SFTY_CE: 3394 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) 3395 free_irq(priv->lpi_irq, dev); 3396 fallthrough; 3397 case REQ_IRQ_ERR_LPI: 3398 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3399 free_irq(priv->wol_irq, dev); 3400 fallthrough; 3401 case REQ_IRQ_ERR_WOL: 3402 free_irq(dev->irq, dev); 3403 fallthrough; 3404 case REQ_IRQ_ERR_MAC: 3405 case REQ_IRQ_ERR_NO: 3406 /* If MAC IRQ request error, no more IRQ to free */ 3407 break; 3408 } 3409 } 3410 3411 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3412 { 3413 struct stmmac_priv *priv = netdev_priv(dev); 3414 enum request_irq_err irq_err; 3415 cpumask_t cpu_mask; 3416 int irq_idx = 0; 3417 char *int_name; 3418 int ret; 3419 int i; 3420 3421 /* For common interrupt */ 3422 int_name = priv->int_name_mac; 3423 sprintf(int_name, "%s:%s", dev->name, "mac"); 3424 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3425 0, int_name, dev); 3426 if (unlikely(ret < 0)) { 3427 netdev_err(priv->dev, 3428 "%s: alloc mac MSI %d (error: %d)\n", 3429 __func__, dev->irq, ret); 3430 irq_err = REQ_IRQ_ERR_MAC; 3431 goto irq_error; 3432 } 3433 3434 /* Request the Wake IRQ in case of another line 3435 * is used for WoL 3436 */ 3437 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3438 int_name = priv->int_name_wol; 3439 sprintf(int_name, "%s:%s", dev->name, "wol"); 3440 ret = request_irq(priv->wol_irq, 3441 stmmac_mac_interrupt, 3442 0, int_name, dev); 3443 if (unlikely(ret < 0)) { 3444 netdev_err(priv->dev, 3445 "%s: alloc wol MSI %d (error: %d)\n", 3446 __func__, priv->wol_irq, ret); 3447 irq_err = REQ_IRQ_ERR_WOL; 3448 goto irq_error; 3449 } 3450 } 3451 3452 /* Request the LPI IRQ in case of another line 3453 * is used for LPI 3454 */ 3455 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3456 int_name = priv->int_name_lpi; 3457 sprintf(int_name, "%s:%s", dev->name, "lpi"); 3458 ret = request_irq(priv->lpi_irq, 3459 stmmac_mac_interrupt, 3460 0, int_name, dev); 3461 if (unlikely(ret < 0)) { 3462 netdev_err(priv->dev, 3463 "%s: alloc lpi MSI %d (error: %d)\n", 3464 __func__, priv->lpi_irq, ret); 3465 irq_err = REQ_IRQ_ERR_LPI; 3466 goto irq_error; 3467 } 3468 } 3469 3470 /* Request the Safety Feature Correctible Error line in 3471 * case of another line is used 3472 */ 3473 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { 3474 int_name = priv->int_name_sfty_ce; 3475 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3476 ret = request_irq(priv->sfty_ce_irq, 3477 stmmac_safety_interrupt, 3478 0, int_name, dev); 3479 if (unlikely(ret < 0)) { 3480 netdev_err(priv->dev, 3481 "%s: alloc sfty ce MSI %d (error: %d)\n", 3482 __func__, priv->sfty_ce_irq, ret); 3483 irq_err = REQ_IRQ_ERR_SFTY_CE; 3484 goto irq_error; 3485 } 3486 } 3487 3488 /* Request the Safety Feature Uncorrectible Error line in 3489 * case of another line is used 3490 */ 3491 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { 3492 int_name = priv->int_name_sfty_ue; 3493 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3494 ret = request_irq(priv->sfty_ue_irq, 3495 stmmac_safety_interrupt, 3496 0, int_name, dev); 3497 if (unlikely(ret < 0)) { 3498 netdev_err(priv->dev, 3499 "%s: alloc sfty ue MSI %d (error: %d)\n", 3500 __func__, priv->sfty_ue_irq, ret); 3501 irq_err = REQ_IRQ_ERR_SFTY_UE; 3502 goto irq_error; 3503 } 3504 } 3505 3506 /* Request Rx MSI irq */ 3507 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3508 if (i >= MTL_MAX_RX_QUEUES) 3509 break; 3510 if (priv->rx_irq[i] == 0) 3511 continue; 3512 3513 int_name = priv->int_name_rx_irq[i]; 3514 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3515 ret = request_irq(priv->rx_irq[i], 3516 stmmac_msi_intr_rx, 3517 0, int_name, &priv->rx_queue[i]); 3518 if (unlikely(ret < 0)) { 3519 netdev_err(priv->dev, 3520 "%s: alloc rx-%d MSI %d (error: %d)\n", 3521 __func__, i, priv->rx_irq[i], ret); 3522 irq_err = REQ_IRQ_ERR_RX; 3523 irq_idx = i; 3524 goto irq_error; 3525 } 3526 cpumask_clear(&cpu_mask); 3527 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3528 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); 3529 } 3530 3531 /* Request Tx MSI irq */ 3532 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3533 if (i >= MTL_MAX_TX_QUEUES) 3534 break; 3535 if (priv->tx_irq[i] == 0) 3536 continue; 3537 3538 int_name = priv->int_name_tx_irq[i]; 3539 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3540 ret = request_irq(priv->tx_irq[i], 3541 stmmac_msi_intr_tx, 3542 0, int_name, &priv->tx_queue[i]); 3543 if (unlikely(ret < 0)) { 3544 netdev_err(priv->dev, 3545 "%s: alloc tx-%d MSI %d (error: %d)\n", 3546 __func__, i, priv->tx_irq[i], ret); 3547 irq_err = REQ_IRQ_ERR_TX; 3548 irq_idx = i; 3549 goto irq_error; 3550 } 3551 cpumask_clear(&cpu_mask); 3552 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3553 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); 3554 } 3555 3556 return 0; 3557 3558 irq_error: 3559 stmmac_free_irq(dev, irq_err, irq_idx); 3560 return ret; 3561 } 3562 3563 static int stmmac_request_irq_single(struct net_device *dev) 3564 { 3565 struct stmmac_priv *priv = netdev_priv(dev); 3566 enum request_irq_err irq_err; 3567 int ret; 3568 3569 ret = request_irq(dev->irq, stmmac_interrupt, 3570 IRQF_SHARED, dev->name, dev); 3571 if (unlikely(ret < 0)) { 3572 netdev_err(priv->dev, 3573 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3574 __func__, dev->irq, ret); 3575 irq_err = REQ_IRQ_ERR_MAC; 3576 goto irq_error; 3577 } 3578 3579 /* Request the Wake IRQ in case of another line 3580 * is used for WoL 3581 */ 3582 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3583 ret = request_irq(priv->wol_irq, stmmac_interrupt, 3584 IRQF_SHARED, dev->name, dev); 3585 if (unlikely(ret < 0)) { 3586 netdev_err(priv->dev, 3587 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 3588 __func__, priv->wol_irq, ret); 3589 irq_err = REQ_IRQ_ERR_WOL; 3590 goto irq_error; 3591 } 3592 } 3593 3594 /* Request the IRQ lines */ 3595 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3596 ret = request_irq(priv->lpi_irq, stmmac_interrupt, 3597 IRQF_SHARED, dev->name, dev); 3598 if (unlikely(ret < 0)) { 3599 netdev_err(priv->dev, 3600 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 3601 __func__, priv->lpi_irq, ret); 3602 irq_err = REQ_IRQ_ERR_LPI; 3603 goto irq_error; 3604 } 3605 } 3606 3607 return 0; 3608 3609 irq_error: 3610 stmmac_free_irq(dev, irq_err, 0); 3611 return ret; 3612 } 3613 3614 static int stmmac_request_irq(struct net_device *dev) 3615 { 3616 struct stmmac_priv *priv = netdev_priv(dev); 3617 int ret; 3618 3619 /* Request the IRQ lines */ 3620 if (priv->plat->multi_msi_en) 3621 ret = stmmac_request_irq_multi_msi(dev); 3622 else 3623 ret = stmmac_request_irq_single(dev); 3624 3625 return ret; 3626 } 3627 3628 /** 3629 * stmmac_open - open entry point of the driver 3630 * @dev : pointer to the device structure. 3631 * Description: 3632 * This function is the open entry point of the driver. 3633 * Return value: 3634 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3635 * file on failure. 3636 */ 3637 static int stmmac_open(struct net_device *dev) 3638 { 3639 struct stmmac_priv *priv = netdev_priv(dev); 3640 int mode = priv->plat->phy_interface; 3641 int bfsize = 0; 3642 u32 chan; 3643 int ret; 3644 3645 ret = pm_runtime_resume_and_get(priv->device); 3646 if (ret < 0) 3647 return ret; 3648 3649 if (priv->hw->pcs != STMMAC_PCS_TBI && 3650 priv->hw->pcs != STMMAC_PCS_RTBI && 3651 (!priv->hw->xpcs || 3652 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) { 3653 ret = stmmac_init_phy(dev); 3654 if (ret) { 3655 netdev_err(priv->dev, 3656 "%s: Cannot attach to PHY (error: %d)\n", 3657 __func__, ret); 3658 goto init_phy_error; 3659 } 3660 } 3661 3662 /* Extra statistics */ 3663 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); 3664 priv->xstats.threshold = tc; 3665 3666 bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu); 3667 if (bfsize < 0) 3668 bfsize = 0; 3669 3670 if (bfsize < BUF_SIZE_16KiB) 3671 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); 3672 3673 priv->dma_buf_sz = bfsize; 3674 buf_sz = bfsize; 3675 3676 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 3677 3678 if (!priv->dma_tx_size) 3679 priv->dma_tx_size = DMA_DEFAULT_TX_SIZE; 3680 if (!priv->dma_rx_size) 3681 priv->dma_rx_size = DMA_DEFAULT_RX_SIZE; 3682 3683 /* Earlier check for TBS */ 3684 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 3685 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; 3686 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 3687 3688 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 3689 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 3690 } 3691 3692 ret = alloc_dma_desc_resources(priv); 3693 if (ret < 0) { 3694 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 3695 __func__); 3696 goto dma_desc_error; 3697 } 3698 3699 ret = init_dma_desc_rings(dev, GFP_KERNEL); 3700 if (ret < 0) { 3701 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 3702 __func__); 3703 goto init_error; 3704 } 3705 3706 ret = stmmac_hw_setup(dev, true); 3707 if (ret < 0) { 3708 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 3709 goto init_error; 3710 } 3711 3712 stmmac_init_coalesce(priv); 3713 3714 phylink_start(priv->phylink); 3715 /* We may have called phylink_speed_down before */ 3716 phylink_speed_up(priv->phylink); 3717 3718 ret = stmmac_request_irq(dev); 3719 if (ret) 3720 goto irq_error; 3721 3722 stmmac_enable_all_queues(priv); 3723 netif_tx_start_all_queues(priv->dev); 3724 stmmac_enable_all_dma_irq(priv); 3725 3726 return 0; 3727 3728 irq_error: 3729 phylink_stop(priv->phylink); 3730 3731 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3732 hrtimer_cancel(&priv->tx_queue[chan].txtimer); 3733 3734 stmmac_hw_teardown(dev); 3735 init_error: 3736 free_dma_desc_resources(priv); 3737 dma_desc_error: 3738 phylink_disconnect_phy(priv->phylink); 3739 init_phy_error: 3740 pm_runtime_put(priv->device); 3741 return ret; 3742 } 3743 3744 static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) 3745 { 3746 set_bit(__FPE_REMOVING, &priv->fpe_task_state); 3747 3748 if (priv->fpe_wq) 3749 destroy_workqueue(priv->fpe_wq); 3750 3751 netdev_info(priv->dev, "FPE workqueue stop"); 3752 } 3753 3754 /** 3755 * stmmac_release - close entry point of the driver 3756 * @dev : device pointer. 3757 * Description: 3758 * This is the stop entry point of the driver. 3759 */ 3760 static int stmmac_release(struct net_device *dev) 3761 { 3762 struct stmmac_priv *priv = netdev_priv(dev); 3763 u32 chan; 3764 3765 netif_tx_disable(dev); 3766 3767 if (device_may_wakeup(priv->device)) 3768 phylink_speed_down(priv->phylink, false); 3769 /* Stop and disconnect the PHY */ 3770 phylink_stop(priv->phylink); 3771 phylink_disconnect_phy(priv->phylink); 3772 3773 stmmac_disable_all_queues(priv); 3774 3775 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3776 hrtimer_cancel(&priv->tx_queue[chan].txtimer); 3777 3778 /* Free the IRQ lines */ 3779 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 3780 3781 if (priv->eee_enabled) { 3782 priv->tx_path_in_lpi_mode = false; 3783 del_timer_sync(&priv->eee_ctrl_timer); 3784 } 3785 3786 /* Stop TX/RX DMA and clear the descriptors */ 3787 stmmac_stop_all_dma(priv); 3788 3789 /* Release and free the Rx/Tx resources */ 3790 free_dma_desc_resources(priv); 3791 3792 /* Disable the MAC Rx/Tx */ 3793 stmmac_mac_set(priv, priv->ioaddr, false); 3794 3795 netif_carrier_off(dev); 3796 3797 stmmac_release_ptp(priv); 3798 3799 pm_runtime_put(priv->device); 3800 3801 if (priv->dma_cap.fpesel) 3802 stmmac_fpe_stop_wq(priv); 3803 3804 return 0; 3805 } 3806 3807 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 3808 struct stmmac_tx_queue *tx_q) 3809 { 3810 u16 tag = 0x0, inner_tag = 0x0; 3811 u32 inner_type = 0x0; 3812 struct dma_desc *p; 3813 3814 if (!priv->dma_cap.vlins) 3815 return false; 3816 if (!skb_vlan_tag_present(skb)) 3817 return false; 3818 if (skb->vlan_proto == htons(ETH_P_8021AD)) { 3819 inner_tag = skb_vlan_tag_get(skb); 3820 inner_type = STMMAC_VLAN_INSERT; 3821 } 3822 3823 tag = skb_vlan_tag_get(skb); 3824 3825 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3826 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 3827 else 3828 p = &tx_q->dma_tx[tx_q->cur_tx]; 3829 3830 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) 3831 return false; 3832 3833 stmmac_set_tx_owner(priv, p); 3834 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); 3835 return true; 3836 } 3837 3838 /** 3839 * stmmac_tso_allocator - close entry point of the driver 3840 * @priv: driver private structure 3841 * @des: buffer start address 3842 * @total_len: total length to fill in descriptors 3843 * @last_segment: condition for the last descriptor 3844 * @queue: TX queue index 3845 * Description: 3846 * This function fills descriptor and request new descriptors according to 3847 * buffer length to fill 3848 */ 3849 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 3850 int total_len, bool last_segment, u32 queue) 3851 { 3852 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 3853 struct dma_desc *desc; 3854 u32 buff_size; 3855 int tmp_len; 3856 3857 tmp_len = total_len; 3858 3859 while (tmp_len > 0) { 3860 dma_addr_t curr_addr; 3861 3862 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 3863 priv->dma_tx_size); 3864 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 3865 3866 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3867 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 3868 else 3869 desc = &tx_q->dma_tx[tx_q->cur_tx]; 3870 3871 curr_addr = des + (total_len - tmp_len); 3872 if (priv->dma_cap.addr64 <= 32) 3873 desc->des0 = cpu_to_le32(curr_addr); 3874 else 3875 stmmac_set_desc_addr(priv, desc, curr_addr); 3876 3877 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 3878 TSO_MAX_BUFF_SIZE : tmp_len; 3879 3880 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 3881 0, 1, 3882 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 3883 0, 0); 3884 3885 tmp_len -= TSO_MAX_BUFF_SIZE; 3886 } 3887 } 3888 3889 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 3890 { 3891 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 3892 int desc_size; 3893 3894 if (likely(priv->extend_desc)) 3895 desc_size = sizeof(struct dma_extended_desc); 3896 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 3897 desc_size = sizeof(struct dma_edesc); 3898 else 3899 desc_size = sizeof(struct dma_desc); 3900 3901 /* The own bit must be the latest setting done when prepare the 3902 * descriptor and then barrier is needed to make sure that 3903 * all is coherent before granting the DMA engine. 3904 */ 3905 wmb(); 3906 3907 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); 3908 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 3909 } 3910 3911 /** 3912 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 3913 * @skb : the socket buffer 3914 * @dev : device pointer 3915 * Description: this is the transmit function that is called on TSO frames 3916 * (support available on GMAC4 and newer chips). 3917 * Diagram below show the ring programming in case of TSO frames: 3918 * 3919 * First Descriptor 3920 * -------- 3921 * | DES0 |---> buffer1 = L2/L3/L4 header 3922 * | DES1 |---> TCP Payload (can continue on next descr...) 3923 * | DES2 |---> buffer 1 and 2 len 3924 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 3925 * -------- 3926 * | 3927 * ... 3928 * | 3929 * -------- 3930 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 3931 * | DES1 | --| 3932 * | DES2 | --> buffer 1 and 2 len 3933 * | DES3 | 3934 * -------- 3935 * 3936 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 3937 */ 3938 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 3939 { 3940 struct dma_desc *desc, *first, *mss_desc = NULL; 3941 struct stmmac_priv *priv = netdev_priv(dev); 3942 int nfrags = skb_shinfo(skb)->nr_frags; 3943 u32 queue = skb_get_queue_mapping(skb); 3944 unsigned int first_entry, tx_packets; 3945 int tmp_pay_len = 0, first_tx; 3946 struct stmmac_tx_queue *tx_q; 3947 bool has_vlan, set_ic; 3948 u8 proto_hdr_len, hdr; 3949 u32 pay_len, mss; 3950 dma_addr_t des; 3951 int i; 3952 3953 tx_q = &priv->tx_queue[queue]; 3954 first_tx = tx_q->cur_tx; 3955 3956 /* Compute header lengths */ 3957 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 3958 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); 3959 hdr = sizeof(struct udphdr); 3960 } else { 3961 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 3962 hdr = tcp_hdrlen(skb); 3963 } 3964 3965 /* Desc availability based on threshold should be enough safe */ 3966 if (unlikely(stmmac_tx_avail(priv, queue) < 3967 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 3968 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 3969 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 3970 queue)); 3971 /* This is a hard error, log it. */ 3972 netdev_err(priv->dev, 3973 "%s: Tx Ring full when queue awake\n", 3974 __func__); 3975 } 3976 return NETDEV_TX_BUSY; 3977 } 3978 3979 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 3980 3981 mss = skb_shinfo(skb)->gso_size; 3982 3983 /* set new MSS value if needed */ 3984 if (mss != tx_q->mss) { 3985 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3986 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 3987 else 3988 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 3989 3990 stmmac_set_mss(priv, mss_desc, mss); 3991 tx_q->mss = mss; 3992 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 3993 priv->dma_tx_size); 3994 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 3995 } 3996 3997 if (netif_msg_tx_queued(priv)) { 3998 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 3999 __func__, hdr, proto_hdr_len, pay_len, mss); 4000 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4001 skb->data_len); 4002 } 4003 4004 /* Check if VLAN can be inserted by HW */ 4005 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4006 4007 first_entry = tx_q->cur_tx; 4008 WARN_ON(tx_q->tx_skbuff[first_entry]); 4009 4010 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4011 desc = &tx_q->dma_entx[first_entry].basic; 4012 else 4013 desc = &tx_q->dma_tx[first_entry]; 4014 first = desc; 4015 4016 if (has_vlan) 4017 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4018 4019 /* first descriptor: fill Headers on Buf1 */ 4020 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4021 DMA_TO_DEVICE); 4022 if (dma_mapping_error(priv->device, des)) 4023 goto dma_map_err; 4024 4025 tx_q->tx_skbuff_dma[first_entry].buf = des; 4026 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 4027 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4028 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4029 4030 if (priv->dma_cap.addr64 <= 32) { 4031 first->des0 = cpu_to_le32(des); 4032 4033 /* Fill start of payload in buff2 of first descriptor */ 4034 if (pay_len) 4035 first->des1 = cpu_to_le32(des + proto_hdr_len); 4036 4037 /* If needed take extra descriptors to fill the remaining payload */ 4038 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 4039 } else { 4040 stmmac_set_desc_addr(priv, first, des); 4041 tmp_pay_len = pay_len; 4042 des += proto_hdr_len; 4043 pay_len = 0; 4044 } 4045 4046 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 4047 4048 /* Prepare fragments */ 4049 for (i = 0; i < nfrags; i++) { 4050 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4051 4052 des = skb_frag_dma_map(priv->device, frag, 0, 4053 skb_frag_size(frag), 4054 DMA_TO_DEVICE); 4055 if (dma_mapping_error(priv->device, des)) 4056 goto dma_map_err; 4057 4058 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4059 (i == nfrags - 1), queue); 4060 4061 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4062 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 4063 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 4064 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4065 } 4066 4067 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 4068 4069 /* Only the last descriptor gets to point to the skb. */ 4070 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4071 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4072 4073 /* Manage tx mitigation */ 4074 tx_packets = (tx_q->cur_tx + 1) - first_tx; 4075 tx_q->tx_count_frames += tx_packets; 4076 4077 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4078 set_ic = true; 4079 else if (!priv->tx_coal_frames[queue]) 4080 set_ic = false; 4081 else if (tx_packets > priv->tx_coal_frames[queue]) 4082 set_ic = true; 4083 else if ((tx_q->tx_count_frames % 4084 priv->tx_coal_frames[queue]) < tx_packets) 4085 set_ic = true; 4086 else 4087 set_ic = false; 4088 4089 if (set_ic) { 4090 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4091 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4092 else 4093 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4094 4095 tx_q->tx_count_frames = 0; 4096 stmmac_set_tx_ic(priv, desc); 4097 priv->xstats.tx_set_ic_bit++; 4098 } 4099 4100 /* We've used all descriptors we need for this skb, however, 4101 * advance cur_tx so that it references a fresh descriptor. 4102 * ndo_start_xmit will fill this descriptor the next time it's 4103 * called and stmmac_tx_clean may clean up to this descriptor. 4104 */ 4105 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); 4106 4107 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4108 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4109 __func__); 4110 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4111 } 4112 4113 dev->stats.tx_bytes += skb->len; 4114 priv->xstats.tx_tso_frames++; 4115 priv->xstats.tx_tso_nfrags += nfrags; 4116 4117 if (priv->sarc_type) 4118 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4119 4120 skb_tx_timestamp(skb); 4121 4122 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4123 priv->hwts_tx_en)) { 4124 /* declare that device is doing timestamping */ 4125 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4126 stmmac_enable_tx_timestamp(priv, first); 4127 } 4128 4129 /* Complete the first descriptor before granting the DMA */ 4130 stmmac_prepare_tso_tx_desc(priv, first, 1, 4131 proto_hdr_len, 4132 pay_len, 4133 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 4134 hdr / 4, (skb->len - proto_hdr_len)); 4135 4136 /* If context desc is used to change MSS */ 4137 if (mss_desc) { 4138 /* Make sure that first descriptor has been completely 4139 * written, including its own bit. This is because MSS is 4140 * actually before first descriptor, so we need to make 4141 * sure that MSS's own bit is the last thing written. 4142 */ 4143 dma_wmb(); 4144 stmmac_set_tx_owner(priv, mss_desc); 4145 } 4146 4147 if (netif_msg_pktdata(priv)) { 4148 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4149 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4150 tx_q->cur_tx, first, nfrags); 4151 pr_info(">>> frame to be transmitted: "); 4152 print_pkt(skb->data, skb_headlen(skb)); 4153 } 4154 4155 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4156 4157 stmmac_flush_tx_descriptors(priv, queue); 4158 stmmac_tx_timer_arm(priv, queue); 4159 4160 return NETDEV_TX_OK; 4161 4162 dma_map_err: 4163 dev_err(priv->device, "Tx dma map failed\n"); 4164 dev_kfree_skb(skb); 4165 priv->dev->stats.tx_dropped++; 4166 return NETDEV_TX_OK; 4167 } 4168 4169 /** 4170 * stmmac_xmit - Tx entry point of the driver 4171 * @skb : the socket buffer 4172 * @dev : device pointer 4173 * Description : this is the tx entry point of the driver. 4174 * It programs the chain or the ring and supports oversized frames 4175 * and SG feature. 4176 */ 4177 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4178 { 4179 unsigned int first_entry, tx_packets, enh_desc; 4180 struct stmmac_priv *priv = netdev_priv(dev); 4181 unsigned int nopaged_len = skb_headlen(skb); 4182 int i, csum_insertion = 0, is_jumbo = 0; 4183 u32 queue = skb_get_queue_mapping(skb); 4184 int nfrags = skb_shinfo(skb)->nr_frags; 4185 int gso = skb_shinfo(skb)->gso_type; 4186 struct dma_edesc *tbs_desc = NULL; 4187 struct dma_desc *desc, *first; 4188 struct stmmac_tx_queue *tx_q; 4189 bool has_vlan, set_ic; 4190 int entry, first_tx; 4191 dma_addr_t des; 4192 4193 tx_q = &priv->tx_queue[queue]; 4194 first_tx = tx_q->cur_tx; 4195 4196 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4197 stmmac_disable_eee_mode(priv); 4198 4199 /* Manage oversized TCP frames for GMAC4 device */ 4200 if (skb_is_gso(skb) && priv->tso) { 4201 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 4202 return stmmac_tso_xmit(skb, dev); 4203 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) 4204 return stmmac_tso_xmit(skb, dev); 4205 } 4206 4207 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4208 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4209 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4210 queue)); 4211 /* This is a hard error, log it. */ 4212 netdev_err(priv->dev, 4213 "%s: Tx Ring full when queue awake\n", 4214 __func__); 4215 } 4216 return NETDEV_TX_BUSY; 4217 } 4218 4219 /* Check if VLAN can be inserted by HW */ 4220 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4221 4222 entry = tx_q->cur_tx; 4223 first_entry = entry; 4224 WARN_ON(tx_q->tx_skbuff[first_entry]); 4225 4226 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 4227 4228 if (likely(priv->extend_desc)) 4229 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4230 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4231 desc = &tx_q->dma_entx[entry].basic; 4232 else 4233 desc = tx_q->dma_tx + entry; 4234 4235 first = desc; 4236 4237 if (has_vlan) 4238 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4239 4240 enh_desc = priv->plat->enh_desc; 4241 /* To program the descriptors according to the size of the frame */ 4242 if (enh_desc) 4243 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4244 4245 if (unlikely(is_jumbo)) { 4246 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4247 if (unlikely(entry < 0) && (entry != -EINVAL)) 4248 goto dma_map_err; 4249 } 4250 4251 for (i = 0; i < nfrags; i++) { 4252 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4253 int len = skb_frag_size(frag); 4254 bool last_segment = (i == (nfrags - 1)); 4255 4256 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); 4257 WARN_ON(tx_q->tx_skbuff[entry]); 4258 4259 if (likely(priv->extend_desc)) 4260 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4261 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4262 desc = &tx_q->dma_entx[entry].basic; 4263 else 4264 desc = tx_q->dma_tx + entry; 4265 4266 des = skb_frag_dma_map(priv->device, frag, 0, len, 4267 DMA_TO_DEVICE); 4268 if (dma_mapping_error(priv->device, des)) 4269 goto dma_map_err; /* should reuse desc w/o issues */ 4270 4271 tx_q->tx_skbuff_dma[entry].buf = des; 4272 4273 stmmac_set_desc_addr(priv, desc, des); 4274 4275 tx_q->tx_skbuff_dma[entry].map_as_page = true; 4276 tx_q->tx_skbuff_dma[entry].len = len; 4277 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 4278 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4279 4280 /* Prepare the descriptor and set the own bit too */ 4281 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 4282 priv->mode, 1, last_segment, skb->len); 4283 } 4284 4285 /* Only the last descriptor gets to point to the skb. */ 4286 tx_q->tx_skbuff[entry] = skb; 4287 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4288 4289 /* According to the coalesce parameter the IC bit for the latest 4290 * segment is reset and the timer re-started to clean the tx status. 4291 * This approach takes care about the fragments: desc is the first 4292 * element in case of no SG. 4293 */ 4294 tx_packets = (entry + 1) - first_tx; 4295 tx_q->tx_count_frames += tx_packets; 4296 4297 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4298 set_ic = true; 4299 else if (!priv->tx_coal_frames[queue]) 4300 set_ic = false; 4301 else if (tx_packets > priv->tx_coal_frames[queue]) 4302 set_ic = true; 4303 else if ((tx_q->tx_count_frames % 4304 priv->tx_coal_frames[queue]) < tx_packets) 4305 set_ic = true; 4306 else 4307 set_ic = false; 4308 4309 if (set_ic) { 4310 if (likely(priv->extend_desc)) 4311 desc = &tx_q->dma_etx[entry].basic; 4312 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4313 desc = &tx_q->dma_entx[entry].basic; 4314 else 4315 desc = &tx_q->dma_tx[entry]; 4316 4317 tx_q->tx_count_frames = 0; 4318 stmmac_set_tx_ic(priv, desc); 4319 priv->xstats.tx_set_ic_bit++; 4320 } 4321 4322 /* We've used all descriptors we need for this skb, however, 4323 * advance cur_tx so that it references a fresh descriptor. 4324 * ndo_start_xmit will fill this descriptor the next time it's 4325 * called and stmmac_tx_clean may clean up to this descriptor. 4326 */ 4327 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); 4328 tx_q->cur_tx = entry; 4329 4330 if (netif_msg_pktdata(priv)) { 4331 netdev_dbg(priv->dev, 4332 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4333 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4334 entry, first, nfrags); 4335 4336 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4337 print_pkt(skb->data, skb->len); 4338 } 4339 4340 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4341 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4342 __func__); 4343 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4344 } 4345 4346 dev->stats.tx_bytes += skb->len; 4347 4348 if (priv->sarc_type) 4349 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4350 4351 skb_tx_timestamp(skb); 4352 4353 /* Ready to fill the first descriptor and set the OWN bit w/o any 4354 * problems because all the descriptors are actually ready to be 4355 * passed to the DMA engine. 4356 */ 4357 if (likely(!is_jumbo)) { 4358 bool last_segment = (nfrags == 0); 4359 4360 des = dma_map_single(priv->device, skb->data, 4361 nopaged_len, DMA_TO_DEVICE); 4362 if (dma_mapping_error(priv->device, des)) 4363 goto dma_map_err; 4364 4365 tx_q->tx_skbuff_dma[first_entry].buf = des; 4366 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4367 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4368 4369 stmmac_set_desc_addr(priv, first, des); 4370 4371 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 4372 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 4373 4374 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4375 priv->hwts_tx_en)) { 4376 /* declare that device is doing timestamping */ 4377 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4378 stmmac_enable_tx_timestamp(priv, first); 4379 } 4380 4381 /* Prepare the first descriptor setting the OWN bit too */ 4382 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 4383 csum_insertion, priv->mode, 0, last_segment, 4384 skb->len); 4385 } 4386 4387 if (tx_q->tbs & STMMAC_TBS_EN) { 4388 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4389 4390 tbs_desc = &tx_q->dma_entx[first_entry]; 4391 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); 4392 } 4393 4394 stmmac_set_tx_owner(priv, first); 4395 4396 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4397 4398 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4399 4400 stmmac_flush_tx_descriptors(priv, queue); 4401 stmmac_tx_timer_arm(priv, queue); 4402 4403 return NETDEV_TX_OK; 4404 4405 dma_map_err: 4406 netdev_err(priv->dev, "Tx DMA map failed\n"); 4407 dev_kfree_skb(skb); 4408 priv->dev->stats.tx_dropped++; 4409 return NETDEV_TX_OK; 4410 } 4411 4412 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 4413 { 4414 struct vlan_ethhdr *veth; 4415 __be16 vlan_proto; 4416 u16 vlanid; 4417 4418 veth = (struct vlan_ethhdr *)skb->data; 4419 vlan_proto = veth->h_vlan_proto; 4420 4421 if ((vlan_proto == htons(ETH_P_8021Q) && 4422 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 4423 (vlan_proto == htons(ETH_P_8021AD) && 4424 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 4425 /* pop the vlan tag */ 4426 vlanid = ntohs(veth->h_vlan_TCI); 4427 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 4428 skb_pull(skb, VLAN_HLEN); 4429 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 4430 } 4431 } 4432 4433 /** 4434 * stmmac_rx_refill - refill used skb preallocated buffers 4435 * @priv: driver private structure 4436 * @queue: RX queue index 4437 * Description : this is to reallocate the skb for the reception process 4438 * that is based on zero-copy. 4439 */ 4440 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 4441 { 4442 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4443 int dirty = stmmac_rx_dirty(priv, queue); 4444 unsigned int entry = rx_q->dirty_rx; 4445 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 4446 4447 if (priv->dma_cap.addr64 <= 32) 4448 gfp |= GFP_DMA32; 4449 4450 while (dirty-- > 0) { 4451 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4452 struct dma_desc *p; 4453 bool use_rx_wd; 4454 4455 if (priv->extend_desc) 4456 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4457 else 4458 p = rx_q->dma_rx + entry; 4459 4460 if (!buf->page) { 4461 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4462 if (!buf->page) 4463 break; 4464 } 4465 4466 if (priv->sph && !buf->sec_page) { 4467 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4468 if (!buf->sec_page) 4469 break; 4470 4471 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 4472 } 4473 4474 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 4475 4476 stmmac_set_desc_addr(priv, p, buf->addr); 4477 if (priv->sph) 4478 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 4479 else 4480 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 4481 stmmac_refill_desc3(priv, rx_q, p); 4482 4483 rx_q->rx_count_frames++; 4484 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4485 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4486 rx_q->rx_count_frames = 0; 4487 4488 use_rx_wd = !priv->rx_coal_frames[queue]; 4489 use_rx_wd |= rx_q->rx_count_frames > 0; 4490 if (!priv->use_riwt) 4491 use_rx_wd = false; 4492 4493 dma_wmb(); 4494 stmmac_set_rx_owner(priv, p, use_rx_wd); 4495 4496 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size); 4497 } 4498 rx_q->dirty_rx = entry; 4499 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4500 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4501 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4502 } 4503 4504 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 4505 struct dma_desc *p, 4506 int status, unsigned int len) 4507 { 4508 unsigned int plen = 0, hlen = 0; 4509 int coe = priv->hw->rx_csum; 4510 4511 /* Not first descriptor, buffer is always zero */ 4512 if (priv->sph && len) 4513 return 0; 4514 4515 /* First descriptor, get split header length */ 4516 stmmac_get_rx_header_len(priv, p, &hlen); 4517 if (priv->sph && hlen) { 4518 priv->xstats.rx_split_hdr_pkt_n++; 4519 return hlen; 4520 } 4521 4522 /* First descriptor, not last descriptor and not split header */ 4523 if (status & rx_not_ls) 4524 return priv->dma_buf_sz; 4525 4526 plen = stmmac_get_rx_frame_len(priv, p, coe); 4527 4528 /* First descriptor and last descriptor and not split header */ 4529 return min_t(unsigned int, priv->dma_buf_sz, plen); 4530 } 4531 4532 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 4533 struct dma_desc *p, 4534 int status, unsigned int len) 4535 { 4536 int coe = priv->hw->rx_csum; 4537 unsigned int plen = 0; 4538 4539 /* Not split header, buffer is not available */ 4540 if (!priv->sph) 4541 return 0; 4542 4543 /* Not last descriptor */ 4544 if (status & rx_not_ls) 4545 return priv->dma_buf_sz; 4546 4547 plen = stmmac_get_rx_frame_len(priv, p, coe); 4548 4549 /* Last descriptor */ 4550 return plen - len; 4551 } 4552 4553 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 4554 struct xdp_frame *xdpf, bool dma_map) 4555 { 4556 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 4557 unsigned int entry = tx_q->cur_tx; 4558 struct dma_desc *tx_desc; 4559 dma_addr_t dma_addr; 4560 bool set_ic; 4561 4562 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 4563 return STMMAC_XDP_CONSUMED; 4564 4565 if (likely(priv->extend_desc)) 4566 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4567 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4568 tx_desc = &tx_q->dma_entx[entry].basic; 4569 else 4570 tx_desc = tx_q->dma_tx + entry; 4571 4572 if (dma_map) { 4573 dma_addr = dma_map_single(priv->device, xdpf->data, 4574 xdpf->len, DMA_TO_DEVICE); 4575 if (dma_mapping_error(priv->device, dma_addr)) 4576 return STMMAC_XDP_CONSUMED; 4577 4578 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; 4579 } else { 4580 struct page *page = virt_to_page(xdpf->data); 4581 4582 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 4583 xdpf->headroom; 4584 dma_sync_single_for_device(priv->device, dma_addr, 4585 xdpf->len, DMA_BIDIRECTIONAL); 4586 4587 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; 4588 } 4589 4590 tx_q->tx_skbuff_dma[entry].buf = dma_addr; 4591 tx_q->tx_skbuff_dma[entry].map_as_page = false; 4592 tx_q->tx_skbuff_dma[entry].len = xdpf->len; 4593 tx_q->tx_skbuff_dma[entry].last_segment = true; 4594 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 4595 4596 tx_q->xdpf[entry] = xdpf; 4597 4598 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 4599 4600 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 4601 true, priv->mode, true, true, 4602 xdpf->len); 4603 4604 tx_q->tx_count_frames++; 4605 4606 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 4607 set_ic = true; 4608 else 4609 set_ic = false; 4610 4611 if (set_ic) { 4612 tx_q->tx_count_frames = 0; 4613 stmmac_set_tx_ic(priv, tx_desc); 4614 priv->xstats.tx_set_ic_bit++; 4615 } 4616 4617 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4618 4619 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); 4620 tx_q->cur_tx = entry; 4621 4622 return STMMAC_XDP_TX; 4623 } 4624 4625 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 4626 int cpu) 4627 { 4628 int index = cpu; 4629 4630 if (unlikely(index < 0)) 4631 index = 0; 4632 4633 while (index >= priv->plat->tx_queues_to_use) 4634 index -= priv->plat->tx_queues_to_use; 4635 4636 return index; 4637 } 4638 4639 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 4640 struct xdp_buff *xdp) 4641 { 4642 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 4643 int cpu = smp_processor_id(); 4644 struct netdev_queue *nq; 4645 int queue; 4646 int res; 4647 4648 if (unlikely(!xdpf)) 4649 return STMMAC_XDP_CONSUMED; 4650 4651 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4652 nq = netdev_get_tx_queue(priv->dev, queue); 4653 4654 __netif_tx_lock(nq, cpu); 4655 /* Avoids TX time-out as we are sharing with slow path */ 4656 txq_trans_cond_update(nq); 4657 4658 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); 4659 if (res == STMMAC_XDP_TX) 4660 stmmac_flush_tx_descriptors(priv, queue); 4661 4662 __netif_tx_unlock(nq); 4663 4664 return res; 4665 } 4666 4667 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 4668 struct bpf_prog *prog, 4669 struct xdp_buff *xdp) 4670 { 4671 u32 act; 4672 int res; 4673 4674 act = bpf_prog_run_xdp(prog, xdp); 4675 switch (act) { 4676 case XDP_PASS: 4677 res = STMMAC_XDP_PASS; 4678 break; 4679 case XDP_TX: 4680 res = stmmac_xdp_xmit_back(priv, xdp); 4681 break; 4682 case XDP_REDIRECT: 4683 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 4684 res = STMMAC_XDP_CONSUMED; 4685 else 4686 res = STMMAC_XDP_REDIRECT; 4687 break; 4688 default: 4689 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 4690 fallthrough; 4691 case XDP_ABORTED: 4692 trace_xdp_exception(priv->dev, prog, act); 4693 fallthrough; 4694 case XDP_DROP: 4695 res = STMMAC_XDP_CONSUMED; 4696 break; 4697 } 4698 4699 return res; 4700 } 4701 4702 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 4703 struct xdp_buff *xdp) 4704 { 4705 struct bpf_prog *prog; 4706 int res; 4707 4708 prog = READ_ONCE(priv->xdp_prog); 4709 if (!prog) { 4710 res = STMMAC_XDP_PASS; 4711 goto out; 4712 } 4713 4714 res = __stmmac_xdp_run_prog(priv, prog, xdp); 4715 out: 4716 return ERR_PTR(-res); 4717 } 4718 4719 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 4720 int xdp_status) 4721 { 4722 int cpu = smp_processor_id(); 4723 int queue; 4724 4725 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4726 4727 if (xdp_status & STMMAC_XDP_TX) 4728 stmmac_tx_timer_arm(priv, queue); 4729 4730 if (xdp_status & STMMAC_XDP_REDIRECT) 4731 xdp_do_flush(); 4732 } 4733 4734 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 4735 struct xdp_buff *xdp) 4736 { 4737 unsigned int metasize = xdp->data - xdp->data_meta; 4738 unsigned int datasize = xdp->data_end - xdp->data; 4739 struct sk_buff *skb; 4740 4741 skb = __napi_alloc_skb(&ch->rxtx_napi, 4742 xdp->data_end - xdp->data_hard_start, 4743 GFP_ATOMIC | __GFP_NOWARN); 4744 if (unlikely(!skb)) 4745 return NULL; 4746 4747 skb_reserve(skb, xdp->data - xdp->data_hard_start); 4748 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 4749 if (metasize) 4750 skb_metadata_set(skb, metasize); 4751 4752 return skb; 4753 } 4754 4755 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 4756 struct dma_desc *p, struct dma_desc *np, 4757 struct xdp_buff *xdp) 4758 { 4759 struct stmmac_channel *ch = &priv->channel[queue]; 4760 unsigned int len = xdp->data_end - xdp->data; 4761 enum pkt_hash_types hash_type; 4762 int coe = priv->hw->rx_csum; 4763 struct sk_buff *skb; 4764 u32 hash; 4765 4766 skb = stmmac_construct_skb_zc(ch, xdp); 4767 if (!skb) { 4768 priv->dev->stats.rx_dropped++; 4769 return; 4770 } 4771 4772 stmmac_get_rx_hwtstamp(priv, p, np, skb); 4773 stmmac_rx_vlan(priv->dev, skb); 4774 skb->protocol = eth_type_trans(skb, priv->dev); 4775 4776 if (unlikely(!coe)) 4777 skb_checksum_none_assert(skb); 4778 else 4779 skb->ip_summed = CHECKSUM_UNNECESSARY; 4780 4781 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 4782 skb_set_hash(skb, hash, hash_type); 4783 4784 skb_record_rx_queue(skb, queue); 4785 napi_gro_receive(&ch->rxtx_napi, skb); 4786 4787 priv->dev->stats.rx_packets++; 4788 priv->dev->stats.rx_bytes += len; 4789 } 4790 4791 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 4792 { 4793 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4794 unsigned int entry = rx_q->dirty_rx; 4795 struct dma_desc *rx_desc = NULL; 4796 bool ret = true; 4797 4798 budget = min(budget, stmmac_rx_dirty(priv, queue)); 4799 4800 while (budget-- > 0 && entry != rx_q->cur_rx) { 4801 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4802 dma_addr_t dma_addr; 4803 bool use_rx_wd; 4804 4805 if (!buf->xdp) { 4806 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 4807 if (!buf->xdp) { 4808 ret = false; 4809 break; 4810 } 4811 } 4812 4813 if (priv->extend_desc) 4814 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); 4815 else 4816 rx_desc = rx_q->dma_rx + entry; 4817 4818 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 4819 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 4820 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 4821 stmmac_refill_desc3(priv, rx_q, rx_desc); 4822 4823 rx_q->rx_count_frames++; 4824 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4825 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4826 rx_q->rx_count_frames = 0; 4827 4828 use_rx_wd = !priv->rx_coal_frames[queue]; 4829 use_rx_wd |= rx_q->rx_count_frames > 0; 4830 if (!priv->use_riwt) 4831 use_rx_wd = false; 4832 4833 dma_wmb(); 4834 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 4835 4836 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size); 4837 } 4838 4839 if (rx_desc) { 4840 rx_q->dirty_rx = entry; 4841 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4842 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4843 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4844 } 4845 4846 return ret; 4847 } 4848 4849 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 4850 { 4851 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 4852 unsigned int count = 0, error = 0, len = 0; 4853 int dirty = stmmac_rx_dirty(priv, queue); 4854 unsigned int next_entry = rx_q->cur_rx; 4855 unsigned int desc_size; 4856 struct bpf_prog *prog; 4857 bool failure = false; 4858 int xdp_status = 0; 4859 int status = 0; 4860 4861 if (netif_msg_rx_status(priv)) { 4862 void *rx_head; 4863 4864 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 4865 if (priv->extend_desc) { 4866 rx_head = (void *)rx_q->dma_erx; 4867 desc_size = sizeof(struct dma_extended_desc); 4868 } else { 4869 rx_head = (void *)rx_q->dma_rx; 4870 desc_size = sizeof(struct dma_desc); 4871 } 4872 4873 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true, 4874 rx_q->dma_rx_phy, desc_size); 4875 } 4876 while (count < limit) { 4877 struct stmmac_rx_buffer *buf; 4878 unsigned int buf1_len = 0; 4879 struct dma_desc *np, *p; 4880 int entry; 4881 int res; 4882 4883 if (!count && rx_q->state_saved) { 4884 error = rx_q->state.error; 4885 len = rx_q->state.len; 4886 } else { 4887 rx_q->state_saved = false; 4888 error = 0; 4889 len = 0; 4890 } 4891 4892 if (count >= limit) 4893 break; 4894 4895 read_again: 4896 buf1_len = 0; 4897 entry = next_entry; 4898 buf = &rx_q->buf_pool[entry]; 4899 4900 if (dirty >= STMMAC_RX_FILL_BATCH) { 4901 failure = failure || 4902 !stmmac_rx_refill_zc(priv, queue, dirty); 4903 dirty = 0; 4904 } 4905 4906 if (priv->extend_desc) 4907 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4908 else 4909 p = rx_q->dma_rx + entry; 4910 4911 /* read the status of the incoming frame */ 4912 status = stmmac_rx_status(priv, &priv->dev->stats, 4913 &priv->xstats, p); 4914 /* check if managed by the DMA otherwise go ahead */ 4915 if (unlikely(status & dma_own)) 4916 break; 4917 4918 /* Prefetch the next RX descriptor */ 4919 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 4920 priv->dma_rx_size); 4921 next_entry = rx_q->cur_rx; 4922 4923 if (priv->extend_desc) 4924 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 4925 else 4926 np = rx_q->dma_rx + next_entry; 4927 4928 prefetch(np); 4929 4930 /* Ensure a valid XSK buffer before proceed */ 4931 if (!buf->xdp) 4932 break; 4933 4934 if (priv->extend_desc) 4935 stmmac_rx_extended_status(priv, &priv->dev->stats, 4936 &priv->xstats, 4937 rx_q->dma_erx + entry); 4938 if (unlikely(status == discard_frame)) { 4939 xsk_buff_free(buf->xdp); 4940 buf->xdp = NULL; 4941 dirty++; 4942 error = 1; 4943 if (!priv->hwts_rx_en) 4944 priv->dev->stats.rx_errors++; 4945 } 4946 4947 if (unlikely(error && (status & rx_not_ls))) 4948 goto read_again; 4949 if (unlikely(error)) { 4950 count++; 4951 continue; 4952 } 4953 4954 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 4955 if (likely(status & rx_not_ls)) { 4956 xsk_buff_free(buf->xdp); 4957 buf->xdp = NULL; 4958 dirty++; 4959 count++; 4960 goto read_again; 4961 } 4962 4963 /* XDP ZC Frame only support primary buffers for now */ 4964 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 4965 len += buf1_len; 4966 4967 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 4968 * Type frames (LLC/LLC-SNAP) 4969 * 4970 * llc_snap is never checked in GMAC >= 4, so this ACS 4971 * feature is always disabled and packets need to be 4972 * stripped manually. 4973 */ 4974 if (likely(!(status & rx_not_ls)) && 4975 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || 4976 unlikely(status != llc_snap))) { 4977 buf1_len -= ETH_FCS_LEN; 4978 len -= ETH_FCS_LEN; 4979 } 4980 4981 /* RX buffer is good and fit into a XSK pool buffer */ 4982 buf->xdp->data_end = buf->xdp->data + buf1_len; 4983 xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); 4984 4985 prog = READ_ONCE(priv->xdp_prog); 4986 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 4987 4988 switch (res) { 4989 case STMMAC_XDP_PASS: 4990 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 4991 xsk_buff_free(buf->xdp); 4992 break; 4993 case STMMAC_XDP_CONSUMED: 4994 xsk_buff_free(buf->xdp); 4995 priv->dev->stats.rx_dropped++; 4996 break; 4997 case STMMAC_XDP_TX: 4998 case STMMAC_XDP_REDIRECT: 4999 xdp_status |= res; 5000 break; 5001 } 5002 5003 buf->xdp = NULL; 5004 dirty++; 5005 count++; 5006 } 5007 5008 if (status & rx_not_ls) { 5009 rx_q->state_saved = true; 5010 rx_q->state.error = error; 5011 rx_q->state.len = len; 5012 } 5013 5014 stmmac_finalize_xdp_rx(priv, xdp_status); 5015 5016 priv->xstats.rx_pkt_n += count; 5017 priv->xstats.rxq_stats[queue].rx_pkt_n += count; 5018 5019 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5020 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5021 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5022 else 5023 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5024 5025 return (int)count; 5026 } 5027 5028 return failure ? limit : (int)count; 5029 } 5030 5031 /** 5032 * stmmac_rx - manage the receive process 5033 * @priv: driver private structure 5034 * @limit: napi bugget 5035 * @queue: RX queue index. 5036 * Description : this the function called by the napi poll method. 5037 * It gets all the frames inside the ring. 5038 */ 5039 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5040 { 5041 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 5042 struct stmmac_channel *ch = &priv->channel[queue]; 5043 unsigned int count = 0, error = 0, len = 0; 5044 int status = 0, coe = priv->hw->rx_csum; 5045 unsigned int next_entry = rx_q->cur_rx; 5046 enum dma_data_direction dma_dir; 5047 unsigned int desc_size; 5048 struct sk_buff *skb = NULL; 5049 struct xdp_buff xdp; 5050 int xdp_status = 0; 5051 int buf_sz; 5052 5053 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5054 buf_sz = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5055 5056 if (netif_msg_rx_status(priv)) { 5057 void *rx_head; 5058 5059 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5060 if (priv->extend_desc) { 5061 rx_head = (void *)rx_q->dma_erx; 5062 desc_size = sizeof(struct dma_extended_desc); 5063 } else { 5064 rx_head = (void *)rx_q->dma_rx; 5065 desc_size = sizeof(struct dma_desc); 5066 } 5067 5068 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true, 5069 rx_q->dma_rx_phy, desc_size); 5070 } 5071 while (count < limit) { 5072 unsigned int buf1_len = 0, buf2_len = 0; 5073 enum pkt_hash_types hash_type; 5074 struct stmmac_rx_buffer *buf; 5075 struct dma_desc *np, *p; 5076 int entry; 5077 u32 hash; 5078 5079 if (!count && rx_q->state_saved) { 5080 skb = rx_q->state.skb; 5081 error = rx_q->state.error; 5082 len = rx_q->state.len; 5083 } else { 5084 rx_q->state_saved = false; 5085 skb = NULL; 5086 error = 0; 5087 len = 0; 5088 } 5089 5090 if (count >= limit) 5091 break; 5092 5093 read_again: 5094 buf1_len = 0; 5095 buf2_len = 0; 5096 entry = next_entry; 5097 buf = &rx_q->buf_pool[entry]; 5098 5099 if (priv->extend_desc) 5100 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5101 else 5102 p = rx_q->dma_rx + entry; 5103 5104 /* read the status of the incoming frame */ 5105 status = stmmac_rx_status(priv, &priv->dev->stats, 5106 &priv->xstats, p); 5107 /* check if managed by the DMA otherwise go ahead */ 5108 if (unlikely(status & dma_own)) 5109 break; 5110 5111 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5112 priv->dma_rx_size); 5113 next_entry = rx_q->cur_rx; 5114 5115 if (priv->extend_desc) 5116 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5117 else 5118 np = rx_q->dma_rx + next_entry; 5119 5120 prefetch(np); 5121 5122 if (priv->extend_desc) 5123 stmmac_rx_extended_status(priv, &priv->dev->stats, 5124 &priv->xstats, rx_q->dma_erx + entry); 5125 if (unlikely(status == discard_frame)) { 5126 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5127 buf->page = NULL; 5128 error = 1; 5129 if (!priv->hwts_rx_en) 5130 priv->dev->stats.rx_errors++; 5131 } 5132 5133 if (unlikely(error && (status & rx_not_ls))) 5134 goto read_again; 5135 if (unlikely(error)) { 5136 dev_kfree_skb(skb); 5137 skb = NULL; 5138 count++; 5139 continue; 5140 } 5141 5142 /* Buffer is good. Go on. */ 5143 5144 prefetch(page_address(buf->page) + buf->page_offset); 5145 if (buf->sec_page) 5146 prefetch(page_address(buf->sec_page)); 5147 5148 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5149 len += buf1_len; 5150 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5151 len += buf2_len; 5152 5153 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 5154 * Type frames (LLC/LLC-SNAP) 5155 * 5156 * llc_snap is never checked in GMAC >= 4, so this ACS 5157 * feature is always disabled and packets need to be 5158 * stripped manually. 5159 */ 5160 if (likely(!(status & rx_not_ls)) && 5161 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || 5162 unlikely(status != llc_snap))) { 5163 if (buf2_len) { 5164 buf2_len -= ETH_FCS_LEN; 5165 len -= ETH_FCS_LEN; 5166 } else if (buf1_len) { 5167 buf1_len -= ETH_FCS_LEN; 5168 len -= ETH_FCS_LEN; 5169 } 5170 } 5171 5172 if (!skb) { 5173 unsigned int pre_len, sync_len; 5174 5175 dma_sync_single_for_cpu(priv->device, buf->addr, 5176 buf1_len, dma_dir); 5177 5178 xdp_init_buff(&xdp, buf_sz, &rx_q->xdp_rxq); 5179 xdp_prepare_buff(&xdp, page_address(buf->page), 5180 buf->page_offset, buf1_len, false); 5181 5182 pre_len = xdp.data_end - xdp.data_hard_start - 5183 buf->page_offset; 5184 skb = stmmac_xdp_run_prog(priv, &xdp); 5185 /* Due xdp_adjust_tail: DMA sync for_device 5186 * cover max len CPU touch 5187 */ 5188 sync_len = xdp.data_end - xdp.data_hard_start - 5189 buf->page_offset; 5190 sync_len = max(sync_len, pre_len); 5191 5192 /* For Not XDP_PASS verdict */ 5193 if (IS_ERR(skb)) { 5194 unsigned int xdp_res = -PTR_ERR(skb); 5195 5196 if (xdp_res & STMMAC_XDP_CONSUMED) { 5197 page_pool_put_page(rx_q->page_pool, 5198 virt_to_head_page(xdp.data), 5199 sync_len, true); 5200 buf->page = NULL; 5201 priv->dev->stats.rx_dropped++; 5202 5203 /* Clear skb as it was set as 5204 * status by XDP program. 5205 */ 5206 skb = NULL; 5207 5208 if (unlikely((status & rx_not_ls))) 5209 goto read_again; 5210 5211 count++; 5212 continue; 5213 } else if (xdp_res & (STMMAC_XDP_TX | 5214 STMMAC_XDP_REDIRECT)) { 5215 xdp_status |= xdp_res; 5216 buf->page = NULL; 5217 skb = NULL; 5218 count++; 5219 continue; 5220 } 5221 } 5222 } 5223 5224 if (!skb) { 5225 /* XDP program may expand or reduce tail */ 5226 buf1_len = xdp.data_end - xdp.data; 5227 5228 skb = napi_alloc_skb(&ch->rx_napi, buf1_len); 5229 if (!skb) { 5230 priv->dev->stats.rx_dropped++; 5231 count++; 5232 goto drain_data; 5233 } 5234 5235 /* XDP program may adjust header */ 5236 skb_copy_to_linear_data(skb, xdp.data, buf1_len); 5237 skb_put(skb, buf1_len); 5238 5239 /* Data payload copied into SKB, page ready for recycle */ 5240 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5241 buf->page = NULL; 5242 } else if (buf1_len) { 5243 dma_sync_single_for_cpu(priv->device, buf->addr, 5244 buf1_len, dma_dir); 5245 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5246 buf->page, buf->page_offset, buf1_len, 5247 priv->dma_buf_sz); 5248 5249 /* Data payload appended into SKB */ 5250 page_pool_release_page(rx_q->page_pool, buf->page); 5251 buf->page = NULL; 5252 } 5253 5254 if (buf2_len) { 5255 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5256 buf2_len, dma_dir); 5257 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5258 buf->sec_page, 0, buf2_len, 5259 priv->dma_buf_sz); 5260 5261 /* Data payload appended into SKB */ 5262 page_pool_release_page(rx_q->page_pool, buf->sec_page); 5263 buf->sec_page = NULL; 5264 } 5265 5266 drain_data: 5267 if (likely(status & rx_not_ls)) 5268 goto read_again; 5269 if (!skb) 5270 continue; 5271 5272 /* Got entire packet into SKB. Finish it. */ 5273 5274 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5275 stmmac_rx_vlan(priv->dev, skb); 5276 skb->protocol = eth_type_trans(skb, priv->dev); 5277 5278 if (unlikely(!coe)) 5279 skb_checksum_none_assert(skb); 5280 else 5281 skb->ip_summed = CHECKSUM_UNNECESSARY; 5282 5283 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5284 skb_set_hash(skb, hash, hash_type); 5285 5286 skb_record_rx_queue(skb, queue); 5287 napi_gro_receive(&ch->rx_napi, skb); 5288 skb = NULL; 5289 5290 priv->dev->stats.rx_packets++; 5291 priv->dev->stats.rx_bytes += len; 5292 count++; 5293 } 5294 5295 if (status & rx_not_ls || skb) { 5296 rx_q->state_saved = true; 5297 rx_q->state.skb = skb; 5298 rx_q->state.error = error; 5299 rx_q->state.len = len; 5300 } 5301 5302 stmmac_finalize_xdp_rx(priv, xdp_status); 5303 5304 stmmac_rx_refill(priv, queue); 5305 5306 priv->xstats.rx_pkt_n += count; 5307 priv->xstats.rxq_stats[queue].rx_pkt_n += count; 5308 5309 return count; 5310 } 5311 5312 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5313 { 5314 struct stmmac_channel *ch = 5315 container_of(napi, struct stmmac_channel, rx_napi); 5316 struct stmmac_priv *priv = ch->priv_data; 5317 u32 chan = ch->index; 5318 int work_done; 5319 5320 priv->xstats.napi_poll++; 5321 5322 work_done = stmmac_rx(priv, budget, chan); 5323 if (work_done < budget && napi_complete_done(napi, work_done)) { 5324 unsigned long flags; 5325 5326 spin_lock_irqsave(&ch->lock, flags); 5327 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5328 spin_unlock_irqrestore(&ch->lock, flags); 5329 } 5330 5331 return work_done; 5332 } 5333 5334 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5335 { 5336 struct stmmac_channel *ch = 5337 container_of(napi, struct stmmac_channel, tx_napi); 5338 struct stmmac_priv *priv = ch->priv_data; 5339 u32 chan = ch->index; 5340 int work_done; 5341 5342 priv->xstats.napi_poll++; 5343 5344 work_done = stmmac_tx_clean(priv, budget, chan); 5345 work_done = min(work_done, budget); 5346 5347 if (work_done < budget && napi_complete_done(napi, work_done)) { 5348 unsigned long flags; 5349 5350 spin_lock_irqsave(&ch->lock, flags); 5351 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5352 spin_unlock_irqrestore(&ch->lock, flags); 5353 } 5354 5355 return work_done; 5356 } 5357 5358 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 5359 { 5360 struct stmmac_channel *ch = 5361 container_of(napi, struct stmmac_channel, rxtx_napi); 5362 struct stmmac_priv *priv = ch->priv_data; 5363 int rx_done, tx_done, rxtx_done; 5364 u32 chan = ch->index; 5365 5366 priv->xstats.napi_poll++; 5367 5368 tx_done = stmmac_tx_clean(priv, budget, chan); 5369 tx_done = min(tx_done, budget); 5370 5371 rx_done = stmmac_rx_zc(priv, budget, chan); 5372 5373 rxtx_done = max(tx_done, rx_done); 5374 5375 /* If either TX or RX work is not complete, return budget 5376 * and keep pooling 5377 */ 5378 if (rxtx_done >= budget) 5379 return budget; 5380 5381 /* all work done, exit the polling mode */ 5382 if (napi_complete_done(napi, rxtx_done)) { 5383 unsigned long flags; 5384 5385 spin_lock_irqsave(&ch->lock, flags); 5386 /* Both RX and TX work done are compelte, 5387 * so enable both RX & TX IRQs. 5388 */ 5389 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 5390 spin_unlock_irqrestore(&ch->lock, flags); 5391 } 5392 5393 return min(rxtx_done, budget - 1); 5394 } 5395 5396 /** 5397 * stmmac_tx_timeout 5398 * @dev : Pointer to net device structure 5399 * @txqueue: the index of the hanging transmit queue 5400 * Description: this function is called when a packet transmission fails to 5401 * complete within a reasonable time. The driver will mark the error in the 5402 * netdev structure and arrange for the device to be reset to a sane state 5403 * in order to transmit a new packet. 5404 */ 5405 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 5406 { 5407 struct stmmac_priv *priv = netdev_priv(dev); 5408 5409 stmmac_global_err(priv); 5410 } 5411 5412 /** 5413 * stmmac_set_rx_mode - entry point for multicast addressing 5414 * @dev : pointer to the device structure 5415 * Description: 5416 * This function is a driver entry point which gets called by the kernel 5417 * whenever multicast addresses must be enabled/disabled. 5418 * Return value: 5419 * void. 5420 */ 5421 static void stmmac_set_rx_mode(struct net_device *dev) 5422 { 5423 struct stmmac_priv *priv = netdev_priv(dev); 5424 5425 stmmac_set_filter(priv, priv->hw, dev); 5426 } 5427 5428 /** 5429 * stmmac_change_mtu - entry point to change MTU size for the device. 5430 * @dev : device pointer. 5431 * @new_mtu : the new MTU size for the device. 5432 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 5433 * to drive packet transmission. Ethernet has an MTU of 1500 octets 5434 * (ETH_DATA_LEN). This value can be changed with ifconfig. 5435 * Return value: 5436 * 0 on success and an appropriate (-)ve integer as defined in errno.h 5437 * file on failure. 5438 */ 5439 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 5440 { 5441 struct stmmac_priv *priv = netdev_priv(dev); 5442 int txfifosz = priv->plat->tx_fifo_size; 5443 const int mtu = new_mtu; 5444 5445 if (txfifosz == 0) 5446 txfifosz = priv->dma_cap.tx_fifo_size; 5447 5448 txfifosz /= priv->plat->tx_queues_to_use; 5449 5450 if (netif_running(dev)) { 5451 netdev_err(priv->dev, "must be stopped to change its MTU\n"); 5452 return -EBUSY; 5453 } 5454 5455 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 5456 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 5457 return -EINVAL; 5458 } 5459 5460 new_mtu = STMMAC_ALIGN(new_mtu); 5461 5462 /* If condition true, FIFO is too small or MTU too large */ 5463 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 5464 return -EINVAL; 5465 5466 dev->mtu = mtu; 5467 5468 netdev_update_features(dev); 5469 5470 return 0; 5471 } 5472 5473 static netdev_features_t stmmac_fix_features(struct net_device *dev, 5474 netdev_features_t features) 5475 { 5476 struct stmmac_priv *priv = netdev_priv(dev); 5477 5478 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 5479 features &= ~NETIF_F_RXCSUM; 5480 5481 if (!priv->plat->tx_coe) 5482 features &= ~NETIF_F_CSUM_MASK; 5483 5484 /* Some GMAC devices have a bugged Jumbo frame support that 5485 * needs to have the Tx COE disabled for oversized frames 5486 * (due to limited buffer sizes). In this case we disable 5487 * the TX csum insertion in the TDES and not use SF. 5488 */ 5489 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 5490 features &= ~NETIF_F_CSUM_MASK; 5491 5492 /* Disable tso if asked by ethtool */ 5493 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 5494 if (features & NETIF_F_TSO) 5495 priv->tso = true; 5496 else 5497 priv->tso = false; 5498 } 5499 5500 return features; 5501 } 5502 5503 static int stmmac_set_features(struct net_device *netdev, 5504 netdev_features_t features) 5505 { 5506 struct stmmac_priv *priv = netdev_priv(netdev); 5507 5508 /* Keep the COE Type in case of csum is supporting */ 5509 if (features & NETIF_F_RXCSUM) 5510 priv->hw->rx_csum = priv->plat->rx_coe; 5511 else 5512 priv->hw->rx_csum = 0; 5513 /* No check needed because rx_coe has been set before and it will be 5514 * fixed in case of issue. 5515 */ 5516 stmmac_rx_ipc(priv, priv->hw); 5517 5518 if (priv->sph_cap) { 5519 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; 5520 u32 chan; 5521 5522 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 5523 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 5524 } 5525 5526 return 0; 5527 } 5528 5529 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status) 5530 { 5531 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 5532 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 5533 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 5534 bool *hs_enable = &fpe_cfg->hs_enable; 5535 5536 if (status == FPE_EVENT_UNKNOWN || !*hs_enable) 5537 return; 5538 5539 /* If LP has sent verify mPacket, LP is FPE capable */ 5540 if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER) { 5541 if (*lp_state < FPE_STATE_CAPABLE) 5542 *lp_state = FPE_STATE_CAPABLE; 5543 5544 /* If user has requested FPE enable, quickly response */ 5545 if (*hs_enable) 5546 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 5547 MPACKET_RESPONSE); 5548 } 5549 5550 /* If Local has sent verify mPacket, Local is FPE capable */ 5551 if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER) { 5552 if (*lo_state < FPE_STATE_CAPABLE) 5553 *lo_state = FPE_STATE_CAPABLE; 5554 } 5555 5556 /* If LP has sent response mPacket, LP is entering FPE ON */ 5557 if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP) 5558 *lp_state = FPE_STATE_ENTERING_ON; 5559 5560 /* If Local has sent response mPacket, Local is entering FPE ON */ 5561 if ((status & FPE_EVENT_TRSP) == FPE_EVENT_TRSP) 5562 *lo_state = FPE_STATE_ENTERING_ON; 5563 5564 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && 5565 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && 5566 priv->fpe_wq) { 5567 queue_work(priv->fpe_wq, &priv->fpe_task); 5568 } 5569 } 5570 5571 static void stmmac_common_interrupt(struct stmmac_priv *priv) 5572 { 5573 u32 rx_cnt = priv->plat->rx_queues_to_use; 5574 u32 tx_cnt = priv->plat->tx_queues_to_use; 5575 u32 queues_count; 5576 u32 queue; 5577 bool xmac; 5578 5579 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 5580 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 5581 5582 if (priv->irq_wake) 5583 pm_wakeup_event(priv->device, 0); 5584 5585 if (priv->dma_cap.estsel) 5586 stmmac_est_irq_status(priv, priv->ioaddr, priv->dev, 5587 &priv->xstats, tx_cnt); 5588 5589 if (priv->dma_cap.fpesel) { 5590 int status = stmmac_fpe_irq_status(priv, priv->ioaddr, 5591 priv->dev); 5592 5593 stmmac_fpe_event_status(priv, status); 5594 } 5595 5596 /* To handle GMAC own interrupts */ 5597 if ((priv->plat->has_gmac) || xmac) { 5598 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 5599 5600 if (unlikely(status)) { 5601 /* For LPI we need to save the tx status */ 5602 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 5603 priv->tx_path_in_lpi_mode = true; 5604 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 5605 priv->tx_path_in_lpi_mode = false; 5606 } 5607 5608 for (queue = 0; queue < queues_count; queue++) { 5609 status = stmmac_host_mtl_irq_status(priv, priv->hw, 5610 queue); 5611 } 5612 5613 /* PCS link status */ 5614 if (priv->hw->pcs) { 5615 if (priv->xstats.pcs_link) 5616 netif_carrier_on(priv->dev); 5617 else 5618 netif_carrier_off(priv->dev); 5619 } 5620 5621 stmmac_timestamp_interrupt(priv, priv); 5622 } 5623 } 5624 5625 /** 5626 * stmmac_interrupt - main ISR 5627 * @irq: interrupt number. 5628 * @dev_id: to pass the net device pointer. 5629 * Description: this is the main driver interrupt service routine. 5630 * It can call: 5631 * o DMA service routine (to manage incoming frame reception and transmission 5632 * status) 5633 * o Core interrupts to manage: remote wake-up, management counter, LPI 5634 * interrupts. 5635 */ 5636 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 5637 { 5638 struct net_device *dev = (struct net_device *)dev_id; 5639 struct stmmac_priv *priv = netdev_priv(dev); 5640 5641 /* Check if adapter is up */ 5642 if (test_bit(STMMAC_DOWN, &priv->state)) 5643 return IRQ_HANDLED; 5644 5645 /* Check if a fatal error happened */ 5646 if (stmmac_safety_feat_interrupt(priv)) 5647 return IRQ_HANDLED; 5648 5649 /* To handle Common interrupts */ 5650 stmmac_common_interrupt(priv); 5651 5652 /* To handle DMA interrupts */ 5653 stmmac_dma_interrupt(priv); 5654 5655 return IRQ_HANDLED; 5656 } 5657 5658 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 5659 { 5660 struct net_device *dev = (struct net_device *)dev_id; 5661 struct stmmac_priv *priv = netdev_priv(dev); 5662 5663 if (unlikely(!dev)) { 5664 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5665 return IRQ_NONE; 5666 } 5667 5668 /* Check if adapter is up */ 5669 if (test_bit(STMMAC_DOWN, &priv->state)) 5670 return IRQ_HANDLED; 5671 5672 /* To handle Common interrupts */ 5673 stmmac_common_interrupt(priv); 5674 5675 return IRQ_HANDLED; 5676 } 5677 5678 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 5679 { 5680 struct net_device *dev = (struct net_device *)dev_id; 5681 struct stmmac_priv *priv = netdev_priv(dev); 5682 5683 if (unlikely(!dev)) { 5684 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5685 return IRQ_NONE; 5686 } 5687 5688 /* Check if adapter is up */ 5689 if (test_bit(STMMAC_DOWN, &priv->state)) 5690 return IRQ_HANDLED; 5691 5692 /* Check if a fatal error happened */ 5693 stmmac_safety_feat_interrupt(priv); 5694 5695 return IRQ_HANDLED; 5696 } 5697 5698 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 5699 { 5700 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 5701 int chan = tx_q->queue_index; 5702 struct stmmac_priv *priv; 5703 int status; 5704 5705 priv = container_of(tx_q, struct stmmac_priv, tx_queue[chan]); 5706 5707 if (unlikely(!data)) { 5708 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5709 return IRQ_NONE; 5710 } 5711 5712 /* Check if adapter is up */ 5713 if (test_bit(STMMAC_DOWN, &priv->state)) 5714 return IRQ_HANDLED; 5715 5716 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 5717 5718 if (unlikely(status & tx_hard_error_bump_tc)) { 5719 /* Try to bump up the dma threshold on this failure */ 5720 stmmac_bump_dma_threshold(priv, chan); 5721 } else if (unlikely(status == tx_hard_error)) { 5722 stmmac_tx_err(priv, chan); 5723 } 5724 5725 return IRQ_HANDLED; 5726 } 5727 5728 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 5729 { 5730 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 5731 int chan = rx_q->queue_index; 5732 struct stmmac_priv *priv; 5733 5734 priv = container_of(rx_q, struct stmmac_priv, rx_queue[chan]); 5735 5736 if (unlikely(!data)) { 5737 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5738 return IRQ_NONE; 5739 } 5740 5741 /* Check if adapter is up */ 5742 if (test_bit(STMMAC_DOWN, &priv->state)) 5743 return IRQ_HANDLED; 5744 5745 stmmac_napi_check(priv, chan, DMA_DIR_RX); 5746 5747 return IRQ_HANDLED; 5748 } 5749 5750 #ifdef CONFIG_NET_POLL_CONTROLLER 5751 /* Polling receive - used by NETCONSOLE and other diagnostic tools 5752 * to allow network I/O with interrupts disabled. 5753 */ 5754 static void stmmac_poll_controller(struct net_device *dev) 5755 { 5756 struct stmmac_priv *priv = netdev_priv(dev); 5757 int i; 5758 5759 /* If adapter is down, do nothing */ 5760 if (test_bit(STMMAC_DOWN, &priv->state)) 5761 return; 5762 5763 if (priv->plat->multi_msi_en) { 5764 for (i = 0; i < priv->plat->rx_queues_to_use; i++) 5765 stmmac_msi_intr_rx(0, &priv->rx_queue[i]); 5766 5767 for (i = 0; i < priv->plat->tx_queues_to_use; i++) 5768 stmmac_msi_intr_tx(0, &priv->tx_queue[i]); 5769 } else { 5770 disable_irq(dev->irq); 5771 stmmac_interrupt(dev->irq, dev); 5772 enable_irq(dev->irq); 5773 } 5774 } 5775 #endif 5776 5777 /** 5778 * stmmac_ioctl - Entry point for the Ioctl 5779 * @dev: Device pointer. 5780 * @rq: An IOCTL specefic structure, that can contain a pointer to 5781 * a proprietary structure used to pass information to the driver. 5782 * @cmd: IOCTL command 5783 * Description: 5784 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 5785 */ 5786 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 5787 { 5788 struct stmmac_priv *priv = netdev_priv (dev); 5789 int ret = -EOPNOTSUPP; 5790 5791 if (!netif_running(dev)) 5792 return -EINVAL; 5793 5794 switch (cmd) { 5795 case SIOCGMIIPHY: 5796 case SIOCGMIIREG: 5797 case SIOCSMIIREG: 5798 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 5799 break; 5800 case SIOCSHWTSTAMP: 5801 ret = stmmac_hwtstamp_set(dev, rq); 5802 break; 5803 case SIOCGHWTSTAMP: 5804 ret = stmmac_hwtstamp_get(dev, rq); 5805 break; 5806 default: 5807 break; 5808 } 5809 5810 return ret; 5811 } 5812 5813 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 5814 void *cb_priv) 5815 { 5816 struct stmmac_priv *priv = cb_priv; 5817 int ret = -EOPNOTSUPP; 5818 5819 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 5820 return ret; 5821 5822 __stmmac_disable_all_queues(priv); 5823 5824 switch (type) { 5825 case TC_SETUP_CLSU32: 5826 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 5827 break; 5828 case TC_SETUP_CLSFLOWER: 5829 ret = stmmac_tc_setup_cls(priv, priv, type_data); 5830 break; 5831 default: 5832 break; 5833 } 5834 5835 stmmac_enable_all_queues(priv); 5836 return ret; 5837 } 5838 5839 static LIST_HEAD(stmmac_block_cb_list); 5840 5841 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 5842 void *type_data) 5843 { 5844 struct stmmac_priv *priv = netdev_priv(ndev); 5845 5846 switch (type) { 5847 case TC_SETUP_BLOCK: 5848 return flow_block_cb_setup_simple(type_data, 5849 &stmmac_block_cb_list, 5850 stmmac_setup_tc_block_cb, 5851 priv, priv, true); 5852 case TC_SETUP_QDISC_CBS: 5853 return stmmac_tc_setup_cbs(priv, priv, type_data); 5854 case TC_SETUP_QDISC_TAPRIO: 5855 return stmmac_tc_setup_taprio(priv, priv, type_data); 5856 case TC_SETUP_QDISC_ETF: 5857 return stmmac_tc_setup_etf(priv, priv, type_data); 5858 default: 5859 return -EOPNOTSUPP; 5860 } 5861 } 5862 5863 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 5864 struct net_device *sb_dev) 5865 { 5866 int gso = skb_shinfo(skb)->gso_type; 5867 5868 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 5869 /* 5870 * There is no way to determine the number of TSO/USO 5871 * capable Queues. Let's use always the Queue 0 5872 * because if TSO/USO is supported then at least this 5873 * one will be capable. 5874 */ 5875 return 0; 5876 } 5877 5878 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 5879 } 5880 5881 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 5882 { 5883 struct stmmac_priv *priv = netdev_priv(ndev); 5884 int ret = 0; 5885 5886 ret = pm_runtime_resume_and_get(priv->device); 5887 if (ret < 0) 5888 return ret; 5889 5890 ret = eth_mac_addr(ndev, addr); 5891 if (ret) 5892 goto set_mac_error; 5893 5894 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 5895 5896 set_mac_error: 5897 pm_runtime_put(priv->device); 5898 5899 return ret; 5900 } 5901 5902 #ifdef CONFIG_DEBUG_FS 5903 static struct dentry *stmmac_fs_dir; 5904 5905 static void sysfs_display_ring(void *head, int size, int extend_desc, 5906 struct seq_file *seq, dma_addr_t dma_phy_addr) 5907 { 5908 int i; 5909 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 5910 struct dma_desc *p = (struct dma_desc *)head; 5911 dma_addr_t dma_addr; 5912 5913 for (i = 0; i < size; i++) { 5914 if (extend_desc) { 5915 dma_addr = dma_phy_addr + i * sizeof(*ep); 5916 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 5917 i, &dma_addr, 5918 le32_to_cpu(ep->basic.des0), 5919 le32_to_cpu(ep->basic.des1), 5920 le32_to_cpu(ep->basic.des2), 5921 le32_to_cpu(ep->basic.des3)); 5922 ep++; 5923 } else { 5924 dma_addr = dma_phy_addr + i * sizeof(*p); 5925 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 5926 i, &dma_addr, 5927 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 5928 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 5929 p++; 5930 } 5931 seq_printf(seq, "\n"); 5932 } 5933 } 5934 5935 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 5936 { 5937 struct net_device *dev = seq->private; 5938 struct stmmac_priv *priv = netdev_priv(dev); 5939 u32 rx_count = priv->plat->rx_queues_to_use; 5940 u32 tx_count = priv->plat->tx_queues_to_use; 5941 u32 queue; 5942 5943 if ((dev->flags & IFF_UP) == 0) 5944 return 0; 5945 5946 for (queue = 0; queue < rx_count; queue++) { 5947 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 5948 5949 seq_printf(seq, "RX Queue %d:\n", queue); 5950 5951 if (priv->extend_desc) { 5952 seq_printf(seq, "Extended descriptor ring:\n"); 5953 sysfs_display_ring((void *)rx_q->dma_erx, 5954 priv->dma_rx_size, 1, seq, rx_q->dma_rx_phy); 5955 } else { 5956 seq_printf(seq, "Descriptor ring:\n"); 5957 sysfs_display_ring((void *)rx_q->dma_rx, 5958 priv->dma_rx_size, 0, seq, rx_q->dma_rx_phy); 5959 } 5960 } 5961 5962 for (queue = 0; queue < tx_count; queue++) { 5963 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 5964 5965 seq_printf(seq, "TX Queue %d:\n", queue); 5966 5967 if (priv->extend_desc) { 5968 seq_printf(seq, "Extended descriptor ring:\n"); 5969 sysfs_display_ring((void *)tx_q->dma_etx, 5970 priv->dma_tx_size, 1, seq, tx_q->dma_tx_phy); 5971 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 5972 seq_printf(seq, "Descriptor ring:\n"); 5973 sysfs_display_ring((void *)tx_q->dma_tx, 5974 priv->dma_tx_size, 0, seq, tx_q->dma_tx_phy); 5975 } 5976 } 5977 5978 return 0; 5979 } 5980 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 5981 5982 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 5983 { 5984 struct net_device *dev = seq->private; 5985 struct stmmac_priv *priv = netdev_priv(dev); 5986 5987 if (!priv->hw_cap_support) { 5988 seq_printf(seq, "DMA HW features not supported\n"); 5989 return 0; 5990 } 5991 5992 seq_printf(seq, "==============================\n"); 5993 seq_printf(seq, "\tDMA HW features\n"); 5994 seq_printf(seq, "==============================\n"); 5995 5996 seq_printf(seq, "\t10/100 Mbps: %s\n", 5997 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 5998 seq_printf(seq, "\t1000 Mbps: %s\n", 5999 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6000 seq_printf(seq, "\tHalf duplex: %s\n", 6001 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6002 seq_printf(seq, "\tHash Filter: %s\n", 6003 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6004 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6005 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6006 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6007 (priv->dma_cap.pcs) ? "Y" : "N"); 6008 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6009 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6010 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6011 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6012 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6013 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6014 seq_printf(seq, "\tRMON module: %s\n", 6015 (priv->dma_cap.rmon) ? "Y" : "N"); 6016 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6017 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6018 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6019 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6020 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6021 (priv->dma_cap.eee) ? "Y" : "N"); 6022 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6023 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6024 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6025 if (priv->synopsys_id >= DWMAC_CORE_4_00) { 6026 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6027 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6028 } else { 6029 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6030 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6031 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6032 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6033 } 6034 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6035 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6036 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6037 priv->dma_cap.number_rx_channel); 6038 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6039 priv->dma_cap.number_tx_channel); 6040 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", 6041 priv->dma_cap.number_rx_queues); 6042 seq_printf(seq, "\tNumber of Additional TX queues: %d\n", 6043 priv->dma_cap.number_tx_queues); 6044 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6045 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6046 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6047 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6048 seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz); 6049 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6050 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6051 priv->dma_cap.pps_out_num); 6052 seq_printf(seq, "\tSafety Features: %s\n", 6053 priv->dma_cap.asp ? "Y" : "N"); 6054 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6055 priv->dma_cap.frpsel ? "Y" : "N"); 6056 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6057 priv->dma_cap.addr64); 6058 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6059 priv->dma_cap.rssen ? "Y" : "N"); 6060 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6061 priv->dma_cap.vlhash ? "Y" : "N"); 6062 seq_printf(seq, "\tSplit Header: %s\n", 6063 priv->dma_cap.sphen ? "Y" : "N"); 6064 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6065 priv->dma_cap.vlins ? "Y" : "N"); 6066 seq_printf(seq, "\tDouble VLAN: %s\n", 6067 priv->dma_cap.dvlan ? "Y" : "N"); 6068 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6069 priv->dma_cap.l3l4fnum); 6070 seq_printf(seq, "\tARP Offloading: %s\n", 6071 priv->dma_cap.arpoffsel ? "Y" : "N"); 6072 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6073 priv->dma_cap.estsel ? "Y" : "N"); 6074 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6075 priv->dma_cap.fpesel ? "Y" : "N"); 6076 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6077 priv->dma_cap.tbssel ? "Y" : "N"); 6078 return 0; 6079 } 6080 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6081 6082 /* Use network device events to rename debugfs file entries. 6083 */ 6084 static int stmmac_device_event(struct notifier_block *unused, 6085 unsigned long event, void *ptr) 6086 { 6087 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6088 struct stmmac_priv *priv = netdev_priv(dev); 6089 6090 if (dev->netdev_ops != &stmmac_netdev_ops) 6091 goto done; 6092 6093 switch (event) { 6094 case NETDEV_CHANGENAME: 6095 if (priv->dbgfs_dir) 6096 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, 6097 priv->dbgfs_dir, 6098 stmmac_fs_dir, 6099 dev->name); 6100 break; 6101 } 6102 done: 6103 return NOTIFY_DONE; 6104 } 6105 6106 static struct notifier_block stmmac_notifier = { 6107 .notifier_call = stmmac_device_event, 6108 }; 6109 6110 static void stmmac_init_fs(struct net_device *dev) 6111 { 6112 struct stmmac_priv *priv = netdev_priv(dev); 6113 6114 rtnl_lock(); 6115 6116 /* Create per netdev entries */ 6117 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6118 6119 /* Entry to report DMA RX/TX rings */ 6120 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6121 &stmmac_rings_status_fops); 6122 6123 /* Entry to report the DMA HW features */ 6124 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6125 &stmmac_dma_cap_fops); 6126 6127 rtnl_unlock(); 6128 } 6129 6130 static void stmmac_exit_fs(struct net_device *dev) 6131 { 6132 struct stmmac_priv *priv = netdev_priv(dev); 6133 6134 debugfs_remove_recursive(priv->dbgfs_dir); 6135 } 6136 #endif /* CONFIG_DEBUG_FS */ 6137 6138 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6139 { 6140 unsigned char *data = (unsigned char *)&vid_le; 6141 unsigned char data_byte = 0; 6142 u32 crc = ~0x0; 6143 u32 temp = 0; 6144 int i, bits; 6145 6146 bits = get_bitmask_order(VLAN_VID_MASK); 6147 for (i = 0; i < bits; i++) { 6148 if ((i % 8) == 0) 6149 data_byte = data[i / 8]; 6150 6151 temp = ((crc & 1) ^ data_byte) & 1; 6152 crc >>= 1; 6153 data_byte >>= 1; 6154 6155 if (temp) 6156 crc ^= 0xedb88320; 6157 } 6158 6159 return crc; 6160 } 6161 6162 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6163 { 6164 u32 crc, hash = 0; 6165 __le16 pmatch = 0; 6166 int count = 0; 6167 u16 vid = 0; 6168 6169 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6170 __le16 vid_le = cpu_to_le16(vid); 6171 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6172 hash |= (1 << crc); 6173 count++; 6174 } 6175 6176 if (!priv->dma_cap.vlhash) { 6177 if (count > 2) /* VID = 0 always passes filter */ 6178 return -EOPNOTSUPP; 6179 6180 pmatch = cpu_to_le16(vid); 6181 hash = 0; 6182 } 6183 6184 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6185 } 6186 6187 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6188 { 6189 struct stmmac_priv *priv = netdev_priv(ndev); 6190 bool is_double = false; 6191 int ret; 6192 6193 if (be16_to_cpu(proto) == ETH_P_8021AD) 6194 is_double = true; 6195 6196 set_bit(vid, priv->active_vlans); 6197 ret = stmmac_vlan_update(priv, is_double); 6198 if (ret) { 6199 clear_bit(vid, priv->active_vlans); 6200 return ret; 6201 } 6202 6203 if (priv->hw->num_vlan) { 6204 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6205 if (ret) 6206 return ret; 6207 } 6208 6209 return 0; 6210 } 6211 6212 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6213 { 6214 struct stmmac_priv *priv = netdev_priv(ndev); 6215 bool is_double = false; 6216 int ret; 6217 6218 ret = pm_runtime_resume_and_get(priv->device); 6219 if (ret < 0) 6220 return ret; 6221 6222 if (be16_to_cpu(proto) == ETH_P_8021AD) 6223 is_double = true; 6224 6225 clear_bit(vid, priv->active_vlans); 6226 6227 if (priv->hw->num_vlan) { 6228 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6229 if (ret) 6230 goto del_vlan_error; 6231 } 6232 6233 ret = stmmac_vlan_update(priv, is_double); 6234 6235 del_vlan_error: 6236 pm_runtime_put(priv->device); 6237 6238 return ret; 6239 } 6240 6241 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6242 { 6243 struct stmmac_priv *priv = netdev_priv(dev); 6244 6245 switch (bpf->command) { 6246 case XDP_SETUP_PROG: 6247 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6248 case XDP_SETUP_XSK_POOL: 6249 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6250 bpf->xsk.queue_id); 6251 default: 6252 return -EOPNOTSUPP; 6253 } 6254 } 6255 6256 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6257 struct xdp_frame **frames, u32 flags) 6258 { 6259 struct stmmac_priv *priv = netdev_priv(dev); 6260 int cpu = smp_processor_id(); 6261 struct netdev_queue *nq; 6262 int i, nxmit = 0; 6263 int queue; 6264 6265 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6266 return -ENETDOWN; 6267 6268 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6269 return -EINVAL; 6270 6271 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6272 nq = netdev_get_tx_queue(priv->dev, queue); 6273 6274 __netif_tx_lock(nq, cpu); 6275 /* Avoids TX time-out as we are sharing with slow path */ 6276 txq_trans_cond_update(nq); 6277 6278 for (i = 0; i < num_frames; i++) { 6279 int res; 6280 6281 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6282 if (res == STMMAC_XDP_CONSUMED) 6283 break; 6284 6285 nxmit++; 6286 } 6287 6288 if (flags & XDP_XMIT_FLUSH) { 6289 stmmac_flush_tx_descriptors(priv, queue); 6290 stmmac_tx_timer_arm(priv, queue); 6291 } 6292 6293 __netif_tx_unlock(nq); 6294 6295 return nxmit; 6296 } 6297 6298 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6299 { 6300 struct stmmac_channel *ch = &priv->channel[queue]; 6301 unsigned long flags; 6302 6303 spin_lock_irqsave(&ch->lock, flags); 6304 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6305 spin_unlock_irqrestore(&ch->lock, flags); 6306 6307 stmmac_stop_rx_dma(priv, queue); 6308 __free_dma_rx_desc_resources(priv, queue); 6309 } 6310 6311 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 6312 { 6313 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 6314 struct stmmac_channel *ch = &priv->channel[queue]; 6315 unsigned long flags; 6316 u32 buf_size; 6317 int ret; 6318 6319 ret = __alloc_dma_rx_desc_resources(priv, queue); 6320 if (ret) { 6321 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 6322 return; 6323 } 6324 6325 ret = __init_dma_rx_desc_rings(priv, queue, GFP_KERNEL); 6326 if (ret) { 6327 __free_dma_rx_desc_resources(priv, queue); 6328 netdev_err(priv->dev, "Failed to init RX desc.\n"); 6329 return; 6330 } 6331 6332 stmmac_clear_rx_descriptors(priv, queue); 6333 6334 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6335 rx_q->dma_rx_phy, rx_q->queue_index); 6336 6337 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * 6338 sizeof(struct dma_desc)); 6339 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6340 rx_q->rx_tail_addr, rx_q->queue_index); 6341 6342 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6343 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6344 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6345 buf_size, 6346 rx_q->queue_index); 6347 } else { 6348 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6349 priv->dma_buf_sz, 6350 rx_q->queue_index); 6351 } 6352 6353 stmmac_start_rx_dma(priv, queue); 6354 6355 spin_lock_irqsave(&ch->lock, flags); 6356 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6357 spin_unlock_irqrestore(&ch->lock, flags); 6358 } 6359 6360 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 6361 { 6362 struct stmmac_channel *ch = &priv->channel[queue]; 6363 unsigned long flags; 6364 6365 spin_lock_irqsave(&ch->lock, flags); 6366 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6367 spin_unlock_irqrestore(&ch->lock, flags); 6368 6369 stmmac_stop_tx_dma(priv, queue); 6370 __free_dma_tx_desc_resources(priv, queue); 6371 } 6372 6373 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 6374 { 6375 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 6376 struct stmmac_channel *ch = &priv->channel[queue]; 6377 unsigned long flags; 6378 int ret; 6379 6380 ret = __alloc_dma_tx_desc_resources(priv, queue); 6381 if (ret) { 6382 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 6383 return; 6384 } 6385 6386 ret = __init_dma_tx_desc_rings(priv, queue); 6387 if (ret) { 6388 __free_dma_tx_desc_resources(priv, queue); 6389 netdev_err(priv->dev, "Failed to init TX desc.\n"); 6390 return; 6391 } 6392 6393 stmmac_clear_tx_descriptors(priv, queue); 6394 6395 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6396 tx_q->dma_tx_phy, tx_q->queue_index); 6397 6398 if (tx_q->tbs & STMMAC_TBS_AVAIL) 6399 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); 6400 6401 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6402 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6403 tx_q->tx_tail_addr, tx_q->queue_index); 6404 6405 stmmac_start_tx_dma(priv, queue); 6406 6407 spin_lock_irqsave(&ch->lock, flags); 6408 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6409 spin_unlock_irqrestore(&ch->lock, flags); 6410 } 6411 6412 void stmmac_xdp_release(struct net_device *dev) 6413 { 6414 struct stmmac_priv *priv = netdev_priv(dev); 6415 u32 chan; 6416 6417 /* Disable NAPI process */ 6418 stmmac_disable_all_queues(priv); 6419 6420 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6421 hrtimer_cancel(&priv->tx_queue[chan].txtimer); 6422 6423 /* Free the IRQ lines */ 6424 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 6425 6426 /* Stop TX/RX DMA channels */ 6427 stmmac_stop_all_dma(priv); 6428 6429 /* Release and free the Rx/Tx resources */ 6430 free_dma_desc_resources(priv); 6431 6432 /* Disable the MAC Rx/Tx */ 6433 stmmac_mac_set(priv, priv->ioaddr, false); 6434 6435 /* set trans_start so we don't get spurious 6436 * watchdogs during reset 6437 */ 6438 netif_trans_update(dev); 6439 netif_carrier_off(dev); 6440 } 6441 6442 int stmmac_xdp_open(struct net_device *dev) 6443 { 6444 struct stmmac_priv *priv = netdev_priv(dev); 6445 u32 rx_cnt = priv->plat->rx_queues_to_use; 6446 u32 tx_cnt = priv->plat->tx_queues_to_use; 6447 u32 dma_csr_ch = max(rx_cnt, tx_cnt); 6448 struct stmmac_rx_queue *rx_q; 6449 struct stmmac_tx_queue *tx_q; 6450 u32 buf_size; 6451 bool sph_en; 6452 u32 chan; 6453 int ret; 6454 6455 ret = alloc_dma_desc_resources(priv); 6456 if (ret < 0) { 6457 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 6458 __func__); 6459 goto dma_desc_error; 6460 } 6461 6462 ret = init_dma_desc_rings(dev, GFP_KERNEL); 6463 if (ret < 0) { 6464 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 6465 __func__); 6466 goto init_error; 6467 } 6468 6469 /* DMA CSR Channel configuration */ 6470 for (chan = 0; chan < dma_csr_ch; chan++) { 6471 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 6472 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6473 } 6474 6475 /* Adjust Split header */ 6476 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 6477 6478 /* DMA RX Channel Configuration */ 6479 for (chan = 0; chan < rx_cnt; chan++) { 6480 rx_q = &priv->rx_queue[chan]; 6481 6482 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6483 rx_q->dma_rx_phy, chan); 6484 6485 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 6486 (rx_q->buf_alloc_num * 6487 sizeof(struct dma_desc)); 6488 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6489 rx_q->rx_tail_addr, chan); 6490 6491 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6492 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6493 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6494 buf_size, 6495 rx_q->queue_index); 6496 } else { 6497 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6498 priv->dma_buf_sz, 6499 rx_q->queue_index); 6500 } 6501 6502 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6503 } 6504 6505 /* DMA TX Channel Configuration */ 6506 for (chan = 0; chan < tx_cnt; chan++) { 6507 tx_q = &priv->tx_queue[chan]; 6508 6509 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6510 tx_q->dma_tx_phy, chan); 6511 6512 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6513 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6514 tx_q->tx_tail_addr, chan); 6515 6516 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6517 tx_q->txtimer.function = stmmac_tx_timer; 6518 } 6519 6520 /* Enable the MAC Rx/Tx */ 6521 stmmac_mac_set(priv, priv->ioaddr, true); 6522 6523 /* Start Rx & Tx DMA Channels */ 6524 stmmac_start_all_dma(priv); 6525 6526 ret = stmmac_request_irq(dev); 6527 if (ret) 6528 goto irq_error; 6529 6530 /* Enable NAPI process*/ 6531 stmmac_enable_all_queues(priv); 6532 netif_carrier_on(dev); 6533 netif_tx_start_all_queues(dev); 6534 stmmac_enable_all_dma_irq(priv); 6535 6536 return 0; 6537 6538 irq_error: 6539 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6540 hrtimer_cancel(&priv->tx_queue[chan].txtimer); 6541 6542 stmmac_hw_teardown(dev); 6543 init_error: 6544 free_dma_desc_resources(priv); 6545 dma_desc_error: 6546 return ret; 6547 } 6548 6549 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 6550 { 6551 struct stmmac_priv *priv = netdev_priv(dev); 6552 struct stmmac_rx_queue *rx_q; 6553 struct stmmac_tx_queue *tx_q; 6554 struct stmmac_channel *ch; 6555 6556 if (test_bit(STMMAC_DOWN, &priv->state) || 6557 !netif_carrier_ok(priv->dev)) 6558 return -ENETDOWN; 6559 6560 if (!stmmac_xdp_is_enabled(priv)) 6561 return -EINVAL; 6562 6563 if (queue >= priv->plat->rx_queues_to_use || 6564 queue >= priv->plat->tx_queues_to_use) 6565 return -EINVAL; 6566 6567 rx_q = &priv->rx_queue[queue]; 6568 tx_q = &priv->tx_queue[queue]; 6569 ch = &priv->channel[queue]; 6570 6571 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 6572 return -EINVAL; 6573 6574 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 6575 /* EQoS does not have per-DMA channel SW interrupt, 6576 * so we schedule RX Napi straight-away. 6577 */ 6578 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 6579 __napi_schedule(&ch->rxtx_napi); 6580 } 6581 6582 return 0; 6583 } 6584 6585 static const struct net_device_ops stmmac_netdev_ops = { 6586 .ndo_open = stmmac_open, 6587 .ndo_start_xmit = stmmac_xmit, 6588 .ndo_stop = stmmac_release, 6589 .ndo_change_mtu = stmmac_change_mtu, 6590 .ndo_fix_features = stmmac_fix_features, 6591 .ndo_set_features = stmmac_set_features, 6592 .ndo_set_rx_mode = stmmac_set_rx_mode, 6593 .ndo_tx_timeout = stmmac_tx_timeout, 6594 .ndo_eth_ioctl = stmmac_ioctl, 6595 .ndo_setup_tc = stmmac_setup_tc, 6596 .ndo_select_queue = stmmac_select_queue, 6597 #ifdef CONFIG_NET_POLL_CONTROLLER 6598 .ndo_poll_controller = stmmac_poll_controller, 6599 #endif 6600 .ndo_set_mac_address = stmmac_set_mac_address, 6601 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 6602 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 6603 .ndo_bpf = stmmac_bpf, 6604 .ndo_xdp_xmit = stmmac_xdp_xmit, 6605 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 6606 }; 6607 6608 static void stmmac_reset_subtask(struct stmmac_priv *priv) 6609 { 6610 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 6611 return; 6612 if (test_bit(STMMAC_DOWN, &priv->state)) 6613 return; 6614 6615 netdev_err(priv->dev, "Reset adapter.\n"); 6616 6617 rtnl_lock(); 6618 netif_trans_update(priv->dev); 6619 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 6620 usleep_range(1000, 2000); 6621 6622 set_bit(STMMAC_DOWN, &priv->state); 6623 dev_close(priv->dev); 6624 dev_open(priv->dev, NULL); 6625 clear_bit(STMMAC_DOWN, &priv->state); 6626 clear_bit(STMMAC_RESETING, &priv->state); 6627 rtnl_unlock(); 6628 } 6629 6630 static void stmmac_service_task(struct work_struct *work) 6631 { 6632 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 6633 service_task); 6634 6635 stmmac_reset_subtask(priv); 6636 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 6637 } 6638 6639 /** 6640 * stmmac_hw_init - Init the MAC device 6641 * @priv: driver private structure 6642 * Description: this function is to configure the MAC device according to 6643 * some platform parameters or the HW capability register. It prepares the 6644 * driver to use either ring or chain modes and to setup either enhanced or 6645 * normal descriptors. 6646 */ 6647 static int stmmac_hw_init(struct stmmac_priv *priv) 6648 { 6649 int ret; 6650 6651 /* dwmac-sun8i only work in chain mode */ 6652 if (priv->plat->has_sun8i) 6653 chain_mode = 1; 6654 priv->chain_mode = chain_mode; 6655 6656 /* Initialize HW Interface */ 6657 ret = stmmac_hwif_init(priv); 6658 if (ret) 6659 return ret; 6660 6661 /* Get the HW capability (new GMAC newer than 3.50a) */ 6662 priv->hw_cap_support = stmmac_get_hw_features(priv); 6663 if (priv->hw_cap_support) { 6664 dev_info(priv->device, "DMA HW capability register supported\n"); 6665 6666 /* We can override some gmac/dma configuration fields: e.g. 6667 * enh_desc, tx_coe (e.g. that are passed through the 6668 * platform) with the values from the HW capability 6669 * register (if supported). 6670 */ 6671 priv->plat->enh_desc = priv->dma_cap.enh_desc; 6672 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 6673 !priv->plat->use_phy_wol; 6674 priv->hw->pmt = priv->plat->pmt; 6675 if (priv->dma_cap.hash_tb_sz) { 6676 priv->hw->multicast_filter_bins = 6677 (BIT(priv->dma_cap.hash_tb_sz) << 5); 6678 priv->hw->mcast_bits_log2 = 6679 ilog2(priv->hw->multicast_filter_bins); 6680 } 6681 6682 /* TXCOE doesn't work in thresh DMA mode */ 6683 if (priv->plat->force_thresh_dma_mode) 6684 priv->plat->tx_coe = 0; 6685 else 6686 priv->plat->tx_coe = priv->dma_cap.tx_coe; 6687 6688 /* In case of GMAC4 rx_coe is from HW cap register. */ 6689 priv->plat->rx_coe = priv->dma_cap.rx_coe; 6690 6691 if (priv->dma_cap.rx_coe_type2) 6692 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 6693 else if (priv->dma_cap.rx_coe_type1) 6694 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 6695 6696 } else { 6697 dev_info(priv->device, "No HW DMA feature register supported\n"); 6698 } 6699 6700 if (priv->plat->rx_coe) { 6701 priv->hw->rx_csum = priv->plat->rx_coe; 6702 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 6703 if (priv->synopsys_id < DWMAC_CORE_4_00) 6704 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 6705 } 6706 if (priv->plat->tx_coe) 6707 dev_info(priv->device, "TX Checksum insertion supported\n"); 6708 6709 if (priv->plat->pmt) { 6710 dev_info(priv->device, "Wake-Up On Lan supported\n"); 6711 device_set_wakeup_capable(priv->device, 1); 6712 } 6713 6714 if (priv->dma_cap.tsoen) 6715 dev_info(priv->device, "TSO supported\n"); 6716 6717 priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en; 6718 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 6719 6720 /* Run HW quirks, if any */ 6721 if (priv->hwif_quirks) { 6722 ret = priv->hwif_quirks(priv); 6723 if (ret) 6724 return ret; 6725 } 6726 6727 /* Rx Watchdog is available in the COREs newer than the 3.40. 6728 * In some case, for example on bugged HW this feature 6729 * has to be disable and this can be done by passing the 6730 * riwt_off field from the platform. 6731 */ 6732 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 6733 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 6734 priv->use_riwt = 1; 6735 dev_info(priv->device, 6736 "Enable RX Mitigation via HW Watchdog Timer\n"); 6737 } 6738 6739 return 0; 6740 } 6741 6742 static void stmmac_napi_add(struct net_device *dev) 6743 { 6744 struct stmmac_priv *priv = netdev_priv(dev); 6745 u32 queue, maxq; 6746 6747 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 6748 6749 for (queue = 0; queue < maxq; queue++) { 6750 struct stmmac_channel *ch = &priv->channel[queue]; 6751 6752 ch->priv_data = priv; 6753 ch->index = queue; 6754 spin_lock_init(&ch->lock); 6755 6756 if (queue < priv->plat->rx_queues_to_use) { 6757 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx, 6758 NAPI_POLL_WEIGHT); 6759 } 6760 if (queue < priv->plat->tx_queues_to_use) { 6761 netif_napi_add_tx(dev, &ch->tx_napi, 6762 stmmac_napi_poll_tx); 6763 } 6764 if (queue < priv->plat->rx_queues_to_use && 6765 queue < priv->plat->tx_queues_to_use) { 6766 netif_napi_add(dev, &ch->rxtx_napi, 6767 stmmac_napi_poll_rxtx, 6768 NAPI_POLL_WEIGHT); 6769 } 6770 } 6771 } 6772 6773 static void stmmac_napi_del(struct net_device *dev) 6774 { 6775 struct stmmac_priv *priv = netdev_priv(dev); 6776 u32 queue, maxq; 6777 6778 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 6779 6780 for (queue = 0; queue < maxq; queue++) { 6781 struct stmmac_channel *ch = &priv->channel[queue]; 6782 6783 if (queue < priv->plat->rx_queues_to_use) 6784 netif_napi_del(&ch->rx_napi); 6785 if (queue < priv->plat->tx_queues_to_use) 6786 netif_napi_del(&ch->tx_napi); 6787 if (queue < priv->plat->rx_queues_to_use && 6788 queue < priv->plat->tx_queues_to_use) { 6789 netif_napi_del(&ch->rxtx_napi); 6790 } 6791 } 6792 } 6793 6794 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) 6795 { 6796 struct stmmac_priv *priv = netdev_priv(dev); 6797 int ret = 0; 6798 6799 if (netif_running(dev)) 6800 stmmac_release(dev); 6801 6802 stmmac_napi_del(dev); 6803 6804 priv->plat->rx_queues_to_use = rx_cnt; 6805 priv->plat->tx_queues_to_use = tx_cnt; 6806 6807 stmmac_napi_add(dev); 6808 6809 if (netif_running(dev)) 6810 ret = stmmac_open(dev); 6811 6812 return ret; 6813 } 6814 6815 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 6816 { 6817 struct stmmac_priv *priv = netdev_priv(dev); 6818 int ret = 0; 6819 6820 if (netif_running(dev)) 6821 stmmac_release(dev); 6822 6823 priv->dma_rx_size = rx_size; 6824 priv->dma_tx_size = tx_size; 6825 6826 if (netif_running(dev)) 6827 ret = stmmac_open(dev); 6828 6829 return ret; 6830 } 6831 6832 #define SEND_VERIFY_MPAKCET_FMT "Send Verify mPacket lo_state=%d lp_state=%d\n" 6833 static void stmmac_fpe_lp_task(struct work_struct *work) 6834 { 6835 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 6836 fpe_task); 6837 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 6838 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 6839 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 6840 bool *hs_enable = &fpe_cfg->hs_enable; 6841 bool *enable = &fpe_cfg->enable; 6842 int retries = 20; 6843 6844 while (retries-- > 0) { 6845 /* Bail out immediately if FPE handshake is OFF */ 6846 if (*lo_state == FPE_STATE_OFF || !*hs_enable) 6847 break; 6848 6849 if (*lo_state == FPE_STATE_ENTERING_ON && 6850 *lp_state == FPE_STATE_ENTERING_ON) { 6851 stmmac_fpe_configure(priv, priv->ioaddr, 6852 priv->plat->tx_queues_to_use, 6853 priv->plat->rx_queues_to_use, 6854 *enable); 6855 6856 netdev_info(priv->dev, "configured FPE\n"); 6857 6858 *lo_state = FPE_STATE_ON; 6859 *lp_state = FPE_STATE_ON; 6860 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); 6861 break; 6862 } 6863 6864 if ((*lo_state == FPE_STATE_CAPABLE || 6865 *lo_state == FPE_STATE_ENTERING_ON) && 6866 *lp_state != FPE_STATE_ON) { 6867 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, 6868 *lo_state, *lp_state); 6869 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 6870 MPACKET_VERIFY); 6871 } 6872 /* Sleep then retry */ 6873 msleep(500); 6874 } 6875 6876 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 6877 } 6878 6879 void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable) 6880 { 6881 if (priv->plat->fpe_cfg->hs_enable != enable) { 6882 if (enable) { 6883 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 6884 MPACKET_VERIFY); 6885 } else { 6886 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; 6887 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; 6888 } 6889 6890 priv->plat->fpe_cfg->hs_enable = enable; 6891 } 6892 } 6893 6894 /** 6895 * stmmac_dvr_probe 6896 * @device: device pointer 6897 * @plat_dat: platform data pointer 6898 * @res: stmmac resource pointer 6899 * Description: this is the main probe function used to 6900 * call the alloc_etherdev, allocate the priv structure. 6901 * Return: 6902 * returns 0 on success, otherwise errno. 6903 */ 6904 int stmmac_dvr_probe(struct device *device, 6905 struct plat_stmmacenet_data *plat_dat, 6906 struct stmmac_resources *res) 6907 { 6908 struct net_device *ndev = NULL; 6909 struct stmmac_priv *priv; 6910 u32 rxq; 6911 int i, ret = 0; 6912 6913 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 6914 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 6915 if (!ndev) 6916 return -ENOMEM; 6917 6918 SET_NETDEV_DEV(ndev, device); 6919 6920 priv = netdev_priv(ndev); 6921 priv->device = device; 6922 priv->dev = ndev; 6923 6924 stmmac_set_ethtool_ops(ndev); 6925 priv->pause = pause; 6926 priv->plat = plat_dat; 6927 priv->ioaddr = res->addr; 6928 priv->dev->base_addr = (unsigned long)res->addr; 6929 priv->plat->dma_cfg->multi_msi_en = priv->plat->multi_msi_en; 6930 6931 priv->dev->irq = res->irq; 6932 priv->wol_irq = res->wol_irq; 6933 priv->lpi_irq = res->lpi_irq; 6934 priv->sfty_ce_irq = res->sfty_ce_irq; 6935 priv->sfty_ue_irq = res->sfty_ue_irq; 6936 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 6937 priv->rx_irq[i] = res->rx_irq[i]; 6938 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 6939 priv->tx_irq[i] = res->tx_irq[i]; 6940 6941 if (!is_zero_ether_addr(res->mac)) 6942 eth_hw_addr_set(priv->dev, res->mac); 6943 6944 dev_set_drvdata(device, priv->dev); 6945 6946 /* Verify driver arguments */ 6947 stmmac_verify_args(); 6948 6949 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 6950 if (!priv->af_xdp_zc_qps) 6951 return -ENOMEM; 6952 6953 /* Allocate workqueue */ 6954 priv->wq = create_singlethread_workqueue("stmmac_wq"); 6955 if (!priv->wq) { 6956 dev_err(priv->device, "failed to create workqueue\n"); 6957 return -ENOMEM; 6958 } 6959 6960 INIT_WORK(&priv->service_task, stmmac_service_task); 6961 6962 /* Initialize Link Partner FPE workqueue */ 6963 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); 6964 6965 /* Override with kernel parameters if supplied XXX CRS XXX 6966 * this needs to have multiple instances 6967 */ 6968 if ((phyaddr >= 0) && (phyaddr <= 31)) 6969 priv->plat->phy_addr = phyaddr; 6970 6971 if (priv->plat->stmmac_rst) { 6972 ret = reset_control_assert(priv->plat->stmmac_rst); 6973 reset_control_deassert(priv->plat->stmmac_rst); 6974 /* Some reset controllers have only reset callback instead of 6975 * assert + deassert callbacks pair. 6976 */ 6977 if (ret == -ENOTSUPP) 6978 reset_control_reset(priv->plat->stmmac_rst); 6979 } 6980 6981 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 6982 if (ret == -ENOTSUPP) 6983 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 6984 ERR_PTR(ret)); 6985 6986 /* Init MAC and get the capabilities */ 6987 ret = stmmac_hw_init(priv); 6988 if (ret) 6989 goto error_hw_init; 6990 6991 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 6992 */ 6993 if (priv->synopsys_id < DWMAC_CORE_5_20) 6994 priv->plat->dma_cfg->dche = false; 6995 6996 stmmac_check_ether_addr(priv); 6997 6998 ndev->netdev_ops = &stmmac_netdev_ops; 6999 7000 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7001 NETIF_F_RXCSUM; 7002 7003 ret = stmmac_tc_init(priv, priv); 7004 if (!ret) { 7005 ndev->hw_features |= NETIF_F_HW_TC; 7006 } 7007 7008 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { 7009 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 7010 if (priv->plat->has_gmac4) 7011 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 7012 priv->tso = true; 7013 dev_info(priv->device, "TSO feature enabled\n"); 7014 } 7015 7016 if (priv->dma_cap.sphen && !priv->plat->sph_disable) { 7017 ndev->hw_features |= NETIF_F_GRO; 7018 priv->sph_cap = true; 7019 priv->sph = priv->sph_cap; 7020 dev_info(priv->device, "SPH feature enabled\n"); 7021 } 7022 7023 /* The current IP register MAC_HW_Feature1[ADDR64] only define 7024 * 32/40/64 bit width, but some SOC support others like i.MX8MP 7025 * support 34 bits but it map to 40 bits width in MAC_HW_Feature1[ADDR64]. 7026 * So overwrite dma_cap.addr64 according to HW real design. 7027 */ 7028 if (priv->plat->addr64) 7029 priv->dma_cap.addr64 = priv->plat->addr64; 7030 7031 if (priv->dma_cap.addr64) { 7032 ret = dma_set_mask_and_coherent(device, 7033 DMA_BIT_MASK(priv->dma_cap.addr64)); 7034 if (!ret) { 7035 dev_info(priv->device, "Using %d bits DMA width\n", 7036 priv->dma_cap.addr64); 7037 7038 /* 7039 * If more than 32 bits can be addressed, make sure to 7040 * enable enhanced addressing mode. 7041 */ 7042 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7043 priv->plat->dma_cfg->eame = true; 7044 } else { 7045 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7046 if (ret) { 7047 dev_err(priv->device, "Failed to set DMA Mask\n"); 7048 goto error_hw_init; 7049 } 7050 7051 priv->dma_cap.addr64 = 32; 7052 } 7053 } 7054 7055 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7056 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7057 #ifdef STMMAC_VLAN_TAG_USED 7058 /* Both mac100 and gmac support receive VLAN tag detection */ 7059 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7060 if (priv->dma_cap.vlhash) { 7061 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7062 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7063 } 7064 if (priv->dma_cap.vlins) { 7065 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7066 if (priv->dma_cap.dvlan) 7067 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; 7068 } 7069 #endif 7070 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7071 7072 /* Initialize RSS */ 7073 rxq = priv->plat->rx_queues_to_use; 7074 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7075 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7076 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7077 7078 if (priv->dma_cap.rssen && priv->plat->rss_en) 7079 ndev->features |= NETIF_F_RXHASH; 7080 7081 /* MTU range: 46 - hw-specific max */ 7082 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 7083 if (priv->plat->has_xgmac) 7084 ndev->max_mtu = XGMAC_JUMBO_LEN; 7085 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 7086 ndev->max_mtu = JUMBO_LEN; 7087 else 7088 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 7089 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 7090 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 7091 */ 7092 if ((priv->plat->maxmtu < ndev->max_mtu) && 7093 (priv->plat->maxmtu >= ndev->min_mtu)) 7094 ndev->max_mtu = priv->plat->maxmtu; 7095 else if (priv->plat->maxmtu < ndev->min_mtu) 7096 dev_warn(priv->device, 7097 "%s: warning: maxmtu having invalid value (%d)\n", 7098 __func__, priv->plat->maxmtu); 7099 7100 if (flow_ctrl) 7101 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 7102 7103 /* Setup channels NAPI */ 7104 stmmac_napi_add(ndev); 7105 7106 mutex_init(&priv->lock); 7107 7108 /* If a specific clk_csr value is passed from the platform 7109 * this means that the CSR Clock Range selection cannot be 7110 * changed at run-time and it is fixed. Viceversa the driver'll try to 7111 * set the MDC clock dynamically according to the csr actual 7112 * clock input. 7113 */ 7114 if (priv->plat->clk_csr >= 0) 7115 priv->clk_csr = priv->plat->clk_csr; 7116 else 7117 stmmac_clk_csr_set(priv); 7118 7119 stmmac_check_pcs_mode(priv); 7120 7121 pm_runtime_get_noresume(device); 7122 pm_runtime_set_active(device); 7123 if (!pm_runtime_enabled(device)) 7124 pm_runtime_enable(device); 7125 7126 if (priv->hw->pcs != STMMAC_PCS_TBI && 7127 priv->hw->pcs != STMMAC_PCS_RTBI) { 7128 /* MDIO bus Registration */ 7129 ret = stmmac_mdio_register(ndev); 7130 if (ret < 0) { 7131 dev_err_probe(priv->device, ret, 7132 "%s: MDIO bus (id: %d) registration failed\n", 7133 __func__, priv->plat->bus_id); 7134 goto error_mdio_register; 7135 } 7136 } 7137 7138 if (priv->plat->speed_mode_2500) 7139 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); 7140 7141 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { 7142 ret = stmmac_xpcs_setup(priv->mii); 7143 if (ret) 7144 goto error_xpcs_setup; 7145 } 7146 7147 ret = stmmac_phy_setup(priv); 7148 if (ret) { 7149 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 7150 goto error_phy_setup; 7151 } 7152 7153 ret = register_netdev(ndev); 7154 if (ret) { 7155 dev_err(priv->device, "%s: ERROR %i registering the device\n", 7156 __func__, ret); 7157 goto error_netdev_register; 7158 } 7159 7160 if (priv->plat->serdes_powerup) { 7161 ret = priv->plat->serdes_powerup(ndev, 7162 priv->plat->bsp_priv); 7163 7164 if (ret < 0) 7165 goto error_serdes_powerup; 7166 } 7167 7168 #ifdef CONFIG_DEBUG_FS 7169 stmmac_init_fs(ndev); 7170 #endif 7171 7172 if (priv->plat->dump_debug_regs) 7173 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 7174 7175 /* Let pm_runtime_put() disable the clocks. 7176 * If CONFIG_PM is not enabled, the clocks will stay powered. 7177 */ 7178 pm_runtime_put(device); 7179 7180 return ret; 7181 7182 error_serdes_powerup: 7183 unregister_netdev(ndev); 7184 error_netdev_register: 7185 phylink_destroy(priv->phylink); 7186 error_xpcs_setup: 7187 error_phy_setup: 7188 if (priv->hw->pcs != STMMAC_PCS_TBI && 7189 priv->hw->pcs != STMMAC_PCS_RTBI) 7190 stmmac_mdio_unregister(ndev); 7191 error_mdio_register: 7192 stmmac_napi_del(ndev); 7193 error_hw_init: 7194 destroy_workqueue(priv->wq); 7195 bitmap_free(priv->af_xdp_zc_qps); 7196 7197 return ret; 7198 } 7199 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 7200 7201 /** 7202 * stmmac_dvr_remove 7203 * @dev: device pointer 7204 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 7205 * changes the link status, releases the DMA descriptor rings. 7206 */ 7207 int stmmac_dvr_remove(struct device *dev) 7208 { 7209 struct net_device *ndev = dev_get_drvdata(dev); 7210 struct stmmac_priv *priv = netdev_priv(ndev); 7211 7212 netdev_info(priv->dev, "%s: removing driver", __func__); 7213 7214 pm_runtime_get_sync(dev); 7215 7216 stmmac_stop_all_dma(priv); 7217 stmmac_mac_set(priv, priv->ioaddr, false); 7218 netif_carrier_off(ndev); 7219 unregister_netdev(ndev); 7220 7221 /* Serdes power down needs to happen after VLAN filter 7222 * is deleted that is triggered by unregister_netdev(). 7223 */ 7224 if (priv->plat->serdes_powerdown) 7225 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 7226 7227 #ifdef CONFIG_DEBUG_FS 7228 stmmac_exit_fs(ndev); 7229 #endif 7230 phylink_destroy(priv->phylink); 7231 if (priv->plat->stmmac_rst) 7232 reset_control_assert(priv->plat->stmmac_rst); 7233 reset_control_assert(priv->plat->stmmac_ahb_rst); 7234 if (priv->hw->pcs != STMMAC_PCS_TBI && 7235 priv->hw->pcs != STMMAC_PCS_RTBI) 7236 stmmac_mdio_unregister(ndev); 7237 destroy_workqueue(priv->wq); 7238 mutex_destroy(&priv->lock); 7239 bitmap_free(priv->af_xdp_zc_qps); 7240 7241 pm_runtime_disable(dev); 7242 pm_runtime_put_noidle(dev); 7243 7244 return 0; 7245 } 7246 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 7247 7248 /** 7249 * stmmac_suspend - suspend callback 7250 * @dev: device pointer 7251 * Description: this is the function to suspend the device and it is called 7252 * by the platform driver to stop the network queue, release the resources, 7253 * program the PMT register (for WoL), clean and release driver resources. 7254 */ 7255 int stmmac_suspend(struct device *dev) 7256 { 7257 struct net_device *ndev = dev_get_drvdata(dev); 7258 struct stmmac_priv *priv = netdev_priv(ndev); 7259 u32 chan; 7260 7261 if (!ndev || !netif_running(ndev)) 7262 return 0; 7263 7264 mutex_lock(&priv->lock); 7265 7266 netif_device_detach(ndev); 7267 7268 stmmac_disable_all_queues(priv); 7269 7270 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7271 hrtimer_cancel(&priv->tx_queue[chan].txtimer); 7272 7273 if (priv->eee_enabled) { 7274 priv->tx_path_in_lpi_mode = false; 7275 del_timer_sync(&priv->eee_ctrl_timer); 7276 } 7277 7278 /* Stop TX/RX DMA */ 7279 stmmac_stop_all_dma(priv); 7280 7281 if (priv->plat->serdes_powerdown) 7282 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 7283 7284 /* Enable Power down mode by programming the PMT regs */ 7285 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7286 stmmac_pmt(priv, priv->hw, priv->wolopts); 7287 priv->irq_wake = 1; 7288 } else { 7289 stmmac_mac_set(priv, priv->ioaddr, false); 7290 pinctrl_pm_select_sleep_state(priv->device); 7291 } 7292 7293 mutex_unlock(&priv->lock); 7294 7295 rtnl_lock(); 7296 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7297 phylink_suspend(priv->phylink, true); 7298 } else { 7299 if (device_may_wakeup(priv->device)) 7300 phylink_speed_down(priv->phylink, false); 7301 phylink_suspend(priv->phylink, false); 7302 } 7303 rtnl_unlock(); 7304 7305 if (priv->dma_cap.fpesel) { 7306 /* Disable FPE */ 7307 stmmac_fpe_configure(priv, priv->ioaddr, 7308 priv->plat->tx_queues_to_use, 7309 priv->plat->rx_queues_to_use, false); 7310 7311 stmmac_fpe_handshake(priv, false); 7312 stmmac_fpe_stop_wq(priv); 7313 } 7314 7315 priv->speed = SPEED_UNKNOWN; 7316 return 0; 7317 } 7318 EXPORT_SYMBOL_GPL(stmmac_suspend); 7319 7320 /** 7321 * stmmac_reset_queues_param - reset queue parameters 7322 * @priv: device pointer 7323 */ 7324 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 7325 { 7326 u32 rx_cnt = priv->plat->rx_queues_to_use; 7327 u32 tx_cnt = priv->plat->tx_queues_to_use; 7328 u32 queue; 7329 7330 for (queue = 0; queue < rx_cnt; queue++) { 7331 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; 7332 7333 rx_q->cur_rx = 0; 7334 rx_q->dirty_rx = 0; 7335 } 7336 7337 for (queue = 0; queue < tx_cnt; queue++) { 7338 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; 7339 7340 tx_q->cur_tx = 0; 7341 tx_q->dirty_tx = 0; 7342 tx_q->mss = 0; 7343 7344 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 7345 } 7346 } 7347 7348 /** 7349 * stmmac_resume - resume callback 7350 * @dev: device pointer 7351 * Description: when resume this function is invoked to setup the DMA and CORE 7352 * in a usable state. 7353 */ 7354 int stmmac_resume(struct device *dev) 7355 { 7356 struct net_device *ndev = dev_get_drvdata(dev); 7357 struct stmmac_priv *priv = netdev_priv(ndev); 7358 int ret; 7359 7360 if (!netif_running(ndev)) 7361 return 0; 7362 7363 /* Power Down bit, into the PM register, is cleared 7364 * automatically as soon as a magic packet or a Wake-up frame 7365 * is received. Anyway, it's better to manually clear 7366 * this bit because it can generate problems while resuming 7367 * from another devices (e.g. serial console). 7368 */ 7369 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7370 mutex_lock(&priv->lock); 7371 stmmac_pmt(priv, priv->hw, 0); 7372 mutex_unlock(&priv->lock); 7373 priv->irq_wake = 0; 7374 } else { 7375 pinctrl_pm_select_default_state(priv->device); 7376 /* reset the phy so that it's ready */ 7377 if (priv->mii) 7378 stmmac_mdio_reset(priv->mii); 7379 } 7380 7381 if (priv->plat->serdes_powerup) { 7382 ret = priv->plat->serdes_powerup(ndev, 7383 priv->plat->bsp_priv); 7384 7385 if (ret < 0) 7386 return ret; 7387 } 7388 7389 rtnl_lock(); 7390 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7391 phylink_resume(priv->phylink); 7392 } else { 7393 phylink_resume(priv->phylink); 7394 if (device_may_wakeup(priv->device)) 7395 phylink_speed_up(priv->phylink); 7396 } 7397 rtnl_unlock(); 7398 7399 rtnl_lock(); 7400 mutex_lock(&priv->lock); 7401 7402 stmmac_reset_queues_param(priv); 7403 7404 stmmac_free_tx_skbufs(priv); 7405 stmmac_clear_descriptors(priv); 7406 7407 stmmac_hw_setup(ndev, false); 7408 stmmac_init_coalesce(priv); 7409 stmmac_set_rx_mode(ndev); 7410 7411 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); 7412 7413 stmmac_enable_all_queues(priv); 7414 stmmac_enable_all_dma_irq(priv); 7415 7416 mutex_unlock(&priv->lock); 7417 rtnl_unlock(); 7418 7419 netif_device_attach(ndev); 7420 7421 return 0; 7422 } 7423 EXPORT_SYMBOL_GPL(stmmac_resume); 7424 7425 #ifndef MODULE 7426 static int __init stmmac_cmdline_opt(char *str) 7427 { 7428 char *opt; 7429 7430 if (!str || !*str) 7431 return 1; 7432 while ((opt = strsep(&str, ",")) != NULL) { 7433 if (!strncmp(opt, "debug:", 6)) { 7434 if (kstrtoint(opt + 6, 0, &debug)) 7435 goto err; 7436 } else if (!strncmp(opt, "phyaddr:", 8)) { 7437 if (kstrtoint(opt + 8, 0, &phyaddr)) 7438 goto err; 7439 } else if (!strncmp(opt, "buf_sz:", 7)) { 7440 if (kstrtoint(opt + 7, 0, &buf_sz)) 7441 goto err; 7442 } else if (!strncmp(opt, "tc:", 3)) { 7443 if (kstrtoint(opt + 3, 0, &tc)) 7444 goto err; 7445 } else if (!strncmp(opt, "watchdog:", 9)) { 7446 if (kstrtoint(opt + 9, 0, &watchdog)) 7447 goto err; 7448 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 7449 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 7450 goto err; 7451 } else if (!strncmp(opt, "pause:", 6)) { 7452 if (kstrtoint(opt + 6, 0, &pause)) 7453 goto err; 7454 } else if (!strncmp(opt, "eee_timer:", 10)) { 7455 if (kstrtoint(opt + 10, 0, &eee_timer)) 7456 goto err; 7457 } else if (!strncmp(opt, "chain_mode:", 11)) { 7458 if (kstrtoint(opt + 11, 0, &chain_mode)) 7459 goto err; 7460 } 7461 } 7462 return 1; 7463 7464 err: 7465 pr_err("%s: ERROR broken module parameter conversion", __func__); 7466 return 1; 7467 } 7468 7469 __setup("stmmaceth=", stmmac_cmdline_opt); 7470 #endif /* MODULE */ 7471 7472 static int __init stmmac_init(void) 7473 { 7474 #ifdef CONFIG_DEBUG_FS 7475 /* Create debugfs main directory if it doesn't exist yet */ 7476 if (!stmmac_fs_dir) 7477 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 7478 register_netdevice_notifier(&stmmac_notifier); 7479 #endif 7480 7481 return 0; 7482 } 7483 7484 static void __exit stmmac_exit(void) 7485 { 7486 #ifdef CONFIG_DEBUG_FS 7487 unregister_netdevice_notifier(&stmmac_notifier); 7488 debugfs_remove_recursive(stmmac_fs_dir); 7489 #endif 7490 } 7491 7492 module_init(stmmac_init) 7493 module_exit(stmmac_exit) 7494 7495 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 7496 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 7497 MODULE_LICENSE("GPL"); 7498