1 // SPDX-License-Identifier: GPL-2.0-only 2 /******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15 *******************************************************************************/ 16 17 #include <linux/clk.h> 18 #include <linux/kernel.h> 19 #include <linux/interrupt.h> 20 #include <linux/ip.h> 21 #include <linux/tcp.h> 22 #include <linux/skbuff.h> 23 #include <linux/ethtool.h> 24 #include <linux/if_ether.h> 25 #include <linux/crc32.h> 26 #include <linux/mii.h> 27 #include <linux/if.h> 28 #include <linux/if_vlan.h> 29 #include <linux/dma-mapping.h> 30 #include <linux/slab.h> 31 #include <linux/pm_runtime.h> 32 #include <linux/prefetch.h> 33 #include <linux/pinctrl/consumer.h> 34 #ifdef CONFIG_DEBUG_FS 35 #include <linux/debugfs.h> 36 #include <linux/seq_file.h> 37 #endif /* CONFIG_DEBUG_FS */ 38 #include <linux/net_tstamp.h> 39 #include <linux/phylink.h> 40 #include <linux/udp.h> 41 #include <linux/bpf_trace.h> 42 #include <net/page_pool/helpers.h> 43 #include <net/pkt_cls.h> 44 #include <net/xdp_sock_drv.h> 45 #include "stmmac_ptp.h" 46 #include "stmmac.h" 47 #include "stmmac_xdp.h" 48 #include <linux/reset.h> 49 #include <linux/of_mdio.h> 50 #include "dwmac1000.h" 51 #include "dwxgmac2.h" 52 #include "hwif.h" 53 54 /* As long as the interface is active, we keep the timestamping counter enabled 55 * with fine resolution and binary rollover. This avoid non-monotonic behavior 56 * (clock jumps) when changing timestamping settings at runtime. 57 */ 58 #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ 59 PTP_TCR_TSCTRLSSR) 60 61 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 62 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 63 64 /* Module parameters */ 65 #define TX_TIMEO 5000 66 static int watchdog = TX_TIMEO; 67 module_param(watchdog, int, 0644); 68 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 69 70 static int debug = -1; 71 module_param(debug, int, 0644); 72 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 73 74 static int phyaddr = -1; 75 module_param(phyaddr, int, 0444); 76 MODULE_PARM_DESC(phyaddr, "Physical device address"); 77 78 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 79 #define STMMAC_RX_THRESH(x) ((x)->dma_conf.dma_rx_size / 4) 80 81 /* Limit to make sure XDP TX and slow path can coexist */ 82 #define STMMAC_XSK_TX_BUDGET_MAX 256 83 #define STMMAC_TX_XSK_AVAIL 16 84 #define STMMAC_RX_FILL_BATCH 16 85 86 #define STMMAC_XDP_PASS 0 87 #define STMMAC_XDP_CONSUMED BIT(0) 88 #define STMMAC_XDP_TX BIT(1) 89 #define STMMAC_XDP_REDIRECT BIT(2) 90 91 static int flow_ctrl = FLOW_AUTO; 92 module_param(flow_ctrl, int, 0644); 93 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]"); 94 95 static int pause = PAUSE_TIME; 96 module_param(pause, int, 0644); 97 MODULE_PARM_DESC(pause, "Flow Control Pause Time"); 98 99 #define TC_DEFAULT 64 100 static int tc = TC_DEFAULT; 101 module_param(tc, int, 0644); 102 MODULE_PARM_DESC(tc, "DMA threshold control value"); 103 104 #define DEFAULT_BUFSIZE 1536 105 static int buf_sz = DEFAULT_BUFSIZE; 106 module_param(buf_sz, int, 0644); 107 MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 108 109 #define STMMAC_RX_COPYBREAK 256 110 111 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 112 NETIF_MSG_LINK | NETIF_MSG_IFUP | 113 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 114 115 #define STMMAC_DEFAULT_LPI_TIMER 1000 116 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 117 module_param(eee_timer, int, 0644); 118 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 119 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 120 121 /* By default the driver will use the ring mode to manage tx and rx descriptors, 122 * but allow user to force to use the chain instead of the ring 123 */ 124 static unsigned int chain_mode; 125 module_param(chain_mode, int, 0444); 126 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 127 128 static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 129 /* For MSI interrupts handling */ 130 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 131 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 132 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 133 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 134 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 135 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 136 static void stmmac_reset_queues_param(struct stmmac_priv *priv); 137 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 138 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 139 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 140 u32 rxmode, u32 chan); 141 142 #ifdef CONFIG_DEBUG_FS 143 static const struct net_device_ops stmmac_netdev_ops; 144 static void stmmac_init_fs(struct net_device *dev); 145 static void stmmac_exit_fs(struct net_device *dev); 146 #endif 147 148 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 149 150 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) 151 { 152 int ret = 0; 153 154 if (enabled) { 155 ret = clk_prepare_enable(priv->plat->stmmac_clk); 156 if (ret) 157 return ret; 158 ret = clk_prepare_enable(priv->plat->pclk); 159 if (ret) { 160 clk_disable_unprepare(priv->plat->stmmac_clk); 161 return ret; 162 } 163 if (priv->plat->clks_config) { 164 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); 165 if (ret) { 166 clk_disable_unprepare(priv->plat->stmmac_clk); 167 clk_disable_unprepare(priv->plat->pclk); 168 return ret; 169 } 170 } 171 } else { 172 clk_disable_unprepare(priv->plat->stmmac_clk); 173 clk_disable_unprepare(priv->plat->pclk); 174 if (priv->plat->clks_config) 175 priv->plat->clks_config(priv->plat->bsp_priv, enabled); 176 } 177 178 return ret; 179 } 180 EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); 181 182 /** 183 * stmmac_verify_args - verify the driver parameters. 184 * Description: it checks the driver parameters and set a default in case of 185 * errors. 186 */ 187 static void stmmac_verify_args(void) 188 { 189 if (unlikely(watchdog < 0)) 190 watchdog = TX_TIMEO; 191 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB))) 192 buf_sz = DEFAULT_BUFSIZE; 193 if (unlikely(flow_ctrl > 1)) 194 flow_ctrl = FLOW_AUTO; 195 else if (likely(flow_ctrl < 0)) 196 flow_ctrl = FLOW_OFF; 197 if (unlikely((pause < 0) || (pause > 0xffff))) 198 pause = PAUSE_TIME; 199 if (eee_timer < 0) 200 eee_timer = STMMAC_DEFAULT_LPI_TIMER; 201 } 202 203 static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 204 { 205 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 206 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 207 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 208 u32 queue; 209 210 for (queue = 0; queue < maxq; queue++) { 211 struct stmmac_channel *ch = &priv->channel[queue]; 212 213 if (stmmac_xdp_is_enabled(priv) && 214 test_bit(queue, priv->af_xdp_zc_qps)) { 215 napi_disable(&ch->rxtx_napi); 216 continue; 217 } 218 219 if (queue < rx_queues_cnt) 220 napi_disable(&ch->rx_napi); 221 if (queue < tx_queues_cnt) 222 napi_disable(&ch->tx_napi); 223 } 224 } 225 226 /** 227 * stmmac_disable_all_queues - Disable all queues 228 * @priv: driver private structure 229 */ 230 static void stmmac_disable_all_queues(struct stmmac_priv *priv) 231 { 232 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 233 struct stmmac_rx_queue *rx_q; 234 u32 queue; 235 236 /* synchronize_rcu() needed for pending XDP buffers to drain */ 237 for (queue = 0; queue < rx_queues_cnt; queue++) { 238 rx_q = &priv->dma_conf.rx_queue[queue]; 239 if (rx_q->xsk_pool) { 240 synchronize_rcu(); 241 break; 242 } 243 } 244 245 __stmmac_disable_all_queues(priv); 246 } 247 248 /** 249 * stmmac_enable_all_queues - Enable all queues 250 * @priv: driver private structure 251 */ 252 static void stmmac_enable_all_queues(struct stmmac_priv *priv) 253 { 254 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 255 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 256 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 257 u32 queue; 258 259 for (queue = 0; queue < maxq; queue++) { 260 struct stmmac_channel *ch = &priv->channel[queue]; 261 262 if (stmmac_xdp_is_enabled(priv) && 263 test_bit(queue, priv->af_xdp_zc_qps)) { 264 napi_enable(&ch->rxtx_napi); 265 continue; 266 } 267 268 if (queue < rx_queues_cnt) 269 napi_enable(&ch->rx_napi); 270 if (queue < tx_queues_cnt) 271 napi_enable(&ch->tx_napi); 272 } 273 } 274 275 static void stmmac_service_event_schedule(struct stmmac_priv *priv) 276 { 277 if (!test_bit(STMMAC_DOWN, &priv->state) && 278 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 279 queue_work(priv->wq, &priv->service_task); 280 } 281 282 static void stmmac_global_err(struct stmmac_priv *priv) 283 { 284 netif_carrier_off(priv->dev); 285 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 286 stmmac_service_event_schedule(priv); 287 } 288 289 /** 290 * stmmac_clk_csr_set - dynamically set the MDC clock 291 * @priv: driver private structure 292 * Description: this is to dynamically set the MDC clock according to the csr 293 * clock input. 294 * Note: 295 * If a specific clk_csr value is passed from the platform 296 * this means that the CSR Clock Range selection cannot be 297 * changed at run-time and it is fixed (as reported in the driver 298 * documentation). Viceversa the driver will try to set the MDC 299 * clock dynamically according to the actual clock input. 300 */ 301 static void stmmac_clk_csr_set(struct stmmac_priv *priv) 302 { 303 u32 clk_rate; 304 305 clk_rate = clk_get_rate(priv->plat->stmmac_clk); 306 307 /* Platform provided default clk_csr would be assumed valid 308 * for all other cases except for the below mentioned ones. 309 * For values higher than the IEEE 802.3 specified frequency 310 * we can not estimate the proper divider as it is not known 311 * the frequency of clk_csr_i. So we do not change the default 312 * divider. 313 */ 314 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { 315 if (clk_rate < CSR_F_35M) 316 priv->clk_csr = STMMAC_CSR_20_35M; 317 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) 318 priv->clk_csr = STMMAC_CSR_35_60M; 319 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) 320 priv->clk_csr = STMMAC_CSR_60_100M; 321 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) 322 priv->clk_csr = STMMAC_CSR_100_150M; 323 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) 324 priv->clk_csr = STMMAC_CSR_150_250M; 325 else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) 326 priv->clk_csr = STMMAC_CSR_250_300M; 327 } 328 329 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { 330 if (clk_rate > 160000000) 331 priv->clk_csr = 0x03; 332 else if (clk_rate > 80000000) 333 priv->clk_csr = 0x02; 334 else if (clk_rate > 40000000) 335 priv->clk_csr = 0x01; 336 else 337 priv->clk_csr = 0; 338 } 339 340 if (priv->plat->has_xgmac) { 341 if (clk_rate > 400000000) 342 priv->clk_csr = 0x5; 343 else if (clk_rate > 350000000) 344 priv->clk_csr = 0x4; 345 else if (clk_rate > 300000000) 346 priv->clk_csr = 0x3; 347 else if (clk_rate > 250000000) 348 priv->clk_csr = 0x2; 349 else if (clk_rate > 150000000) 350 priv->clk_csr = 0x1; 351 else 352 priv->clk_csr = 0x0; 353 } 354 } 355 356 static void print_pkt(unsigned char *buf, int len) 357 { 358 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 359 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 360 } 361 362 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 363 { 364 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 365 u32 avail; 366 367 if (tx_q->dirty_tx > tx_q->cur_tx) 368 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 369 else 370 avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; 371 372 return avail; 373 } 374 375 /** 376 * stmmac_rx_dirty - Get RX queue dirty 377 * @priv: driver private structure 378 * @queue: RX queue index 379 */ 380 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 381 { 382 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 383 u32 dirty; 384 385 if (rx_q->dirty_rx <= rx_q->cur_rx) 386 dirty = rx_q->cur_rx - rx_q->dirty_rx; 387 else 388 dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; 389 390 return dirty; 391 } 392 393 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en) 394 { 395 int tx_lpi_timer; 396 397 /* Clear/set the SW EEE timer flag based on LPI ET enablement */ 398 priv->eee_sw_timer_en = en ? 0 : 1; 399 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; 400 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); 401 } 402 403 /** 404 * stmmac_enable_eee_mode - check and enter in LPI mode 405 * @priv: driver private structure 406 * Description: this function is to verify and enter in LPI mode in case of 407 * EEE. 408 */ 409 static int stmmac_enable_eee_mode(struct stmmac_priv *priv) 410 { 411 u32 tx_cnt = priv->plat->tx_queues_to_use; 412 u32 queue; 413 414 /* check if all TX queues have the work finished */ 415 for (queue = 0; queue < tx_cnt; queue++) { 416 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 417 418 if (tx_q->dirty_tx != tx_q->cur_tx) 419 return -EBUSY; /* still unfinished work */ 420 } 421 422 /* Check and enter in LPI mode */ 423 if (!priv->tx_path_in_lpi_mode) 424 stmmac_set_eee_mode(priv, priv->hw, 425 priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING); 426 return 0; 427 } 428 429 /** 430 * stmmac_disable_eee_mode - disable and exit from LPI mode 431 * @priv: driver private structure 432 * Description: this function is to exit and disable EEE in case of 433 * LPI state is true. This is called by the xmit. 434 */ 435 void stmmac_disable_eee_mode(struct stmmac_priv *priv) 436 { 437 if (!priv->eee_sw_timer_en) { 438 stmmac_lpi_entry_timer_config(priv, 0); 439 return; 440 } 441 442 stmmac_reset_eee_mode(priv, priv->hw); 443 del_timer_sync(&priv->eee_ctrl_timer); 444 priv->tx_path_in_lpi_mode = false; 445 } 446 447 /** 448 * stmmac_eee_ctrl_timer - EEE TX SW timer. 449 * @t: timer_list struct containing private info 450 * Description: 451 * if there is no data transfer and if we are not in LPI state, 452 * then MAC Transmitter can be moved to LPI state. 453 */ 454 static void stmmac_eee_ctrl_timer(struct timer_list *t) 455 { 456 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer); 457 458 if (stmmac_enable_eee_mode(priv)) 459 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 460 } 461 462 /** 463 * stmmac_eee_init - init EEE 464 * @priv: driver private structure 465 * Description: 466 * if the GMAC supports the EEE (from the HW cap reg) and the phy device 467 * can also manage EEE, this function enable the LPI state and start related 468 * timer. 469 */ 470 bool stmmac_eee_init(struct stmmac_priv *priv) 471 { 472 int eee_tw_timer = priv->eee_tw_timer; 473 474 /* Using PCS we cannot dial with the phy registers at this stage 475 * so we do not support extra feature like EEE. 476 */ 477 if (priv->hw->pcs == STMMAC_PCS_TBI || 478 priv->hw->pcs == STMMAC_PCS_RTBI) 479 return false; 480 481 /* Check if MAC core supports the EEE feature. */ 482 if (!priv->dma_cap.eee) 483 return false; 484 485 mutex_lock(&priv->lock); 486 487 /* Check if it needs to be deactivated */ 488 if (!priv->eee_active) { 489 if (priv->eee_enabled) { 490 netdev_dbg(priv->dev, "disable EEE\n"); 491 stmmac_lpi_entry_timer_config(priv, 0); 492 del_timer_sync(&priv->eee_ctrl_timer); 493 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); 494 if (priv->hw->xpcs) 495 xpcs_config_eee(priv->hw->xpcs, 496 priv->plat->mult_fact_100ns, 497 false); 498 } 499 mutex_unlock(&priv->lock); 500 return false; 501 } 502 503 if (priv->eee_active && !priv->eee_enabled) { 504 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 505 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 506 eee_tw_timer); 507 if (priv->hw->xpcs) 508 xpcs_config_eee(priv->hw->xpcs, 509 priv->plat->mult_fact_100ns, 510 true); 511 } 512 513 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { 514 del_timer_sync(&priv->eee_ctrl_timer); 515 priv->tx_path_in_lpi_mode = false; 516 stmmac_lpi_entry_timer_config(priv, 1); 517 } else { 518 stmmac_lpi_entry_timer_config(priv, 0); 519 mod_timer(&priv->eee_ctrl_timer, 520 STMMAC_LPI_T(priv->tx_lpi_timer)); 521 } 522 523 mutex_unlock(&priv->lock); 524 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 525 return true; 526 } 527 528 /* stmmac_get_tx_hwtstamp - get HW TX timestamps 529 * @priv: driver private structure 530 * @p : descriptor pointer 531 * @skb : the socket buffer 532 * Description : 533 * This function will read timestamp from the descriptor & pass it to stack. 534 * and also perform some sanity checks. 535 */ 536 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 537 struct dma_desc *p, struct sk_buff *skb) 538 { 539 struct skb_shared_hwtstamps shhwtstamp; 540 bool found = false; 541 u64 ns = 0; 542 543 if (!priv->hwts_tx_en) 544 return; 545 546 /* exit if skb doesn't support hw tstamp */ 547 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 548 return; 549 550 /* check tx tstamp status */ 551 if (stmmac_get_tx_timestamp_status(priv, p)) { 552 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 553 found = true; 554 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 555 found = true; 556 } 557 558 if (found) { 559 ns -= priv->plat->cdc_error_adj; 560 561 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 562 shhwtstamp.hwtstamp = ns_to_ktime(ns); 563 564 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 565 /* pass tstamp to stack */ 566 skb_tstamp_tx(skb, &shhwtstamp); 567 } 568 } 569 570 /* stmmac_get_rx_hwtstamp - get HW RX timestamps 571 * @priv: driver private structure 572 * @p : descriptor pointer 573 * @np : next descriptor pointer 574 * @skb : the socket buffer 575 * Description : 576 * This function will read received packet's timestamp from the descriptor 577 * and pass it to stack. It also perform some sanity checks. 578 */ 579 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 580 struct dma_desc *np, struct sk_buff *skb) 581 { 582 struct skb_shared_hwtstamps *shhwtstamp = NULL; 583 struct dma_desc *desc = p; 584 u64 ns = 0; 585 586 if (!priv->hwts_rx_en) 587 return; 588 /* For GMAC4, the valid timestamp is from CTX next desc. */ 589 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 590 desc = np; 591 592 /* Check if timestamp is available */ 593 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 594 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 595 596 ns -= priv->plat->cdc_error_adj; 597 598 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 599 shhwtstamp = skb_hwtstamps(skb); 600 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 601 shhwtstamp->hwtstamp = ns_to_ktime(ns); 602 } else { 603 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 604 } 605 } 606 607 /** 608 * stmmac_hwtstamp_set - control hardware timestamping. 609 * @dev: device pointer. 610 * @ifr: An IOCTL specific structure, that can contain a pointer to 611 * a proprietary structure used to pass information to the driver. 612 * Description: 613 * This function configures the MAC to enable/disable both outgoing(TX) 614 * and incoming(RX) packets time stamping based on user input. 615 * Return Value: 616 * 0 on success and an appropriate -ve integer on failure. 617 */ 618 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 619 { 620 struct stmmac_priv *priv = netdev_priv(dev); 621 struct hwtstamp_config config; 622 u32 ptp_v2 = 0; 623 u32 tstamp_all = 0; 624 u32 ptp_over_ipv4_udp = 0; 625 u32 ptp_over_ipv6_udp = 0; 626 u32 ptp_over_ethernet = 0; 627 u32 snap_type_sel = 0; 628 u32 ts_master_en = 0; 629 u32 ts_event_en = 0; 630 631 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 632 netdev_alert(priv->dev, "No support for HW time stamping\n"); 633 priv->hwts_tx_en = 0; 634 priv->hwts_rx_en = 0; 635 636 return -EOPNOTSUPP; 637 } 638 639 if (copy_from_user(&config, ifr->ifr_data, 640 sizeof(config))) 641 return -EFAULT; 642 643 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 644 __func__, config.flags, config.tx_type, config.rx_filter); 645 646 if (config.tx_type != HWTSTAMP_TX_OFF && 647 config.tx_type != HWTSTAMP_TX_ON) 648 return -ERANGE; 649 650 if (priv->adv_ts) { 651 switch (config.rx_filter) { 652 case HWTSTAMP_FILTER_NONE: 653 /* time stamp no incoming packet at all */ 654 config.rx_filter = HWTSTAMP_FILTER_NONE; 655 break; 656 657 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 658 /* PTP v1, UDP, any kind of event packet */ 659 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 660 /* 'xmac' hardware can support Sync, Pdelay_Req and 661 * Pdelay_resp by setting bit14 and bits17/16 to 01 662 * This leaves Delay_Req timestamps out. 663 * Enable all events *and* general purpose message 664 * timestamping 665 */ 666 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 667 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 668 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 669 break; 670 671 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 672 /* PTP v1, UDP, Sync packet */ 673 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 674 /* take time stamp for SYNC messages only */ 675 ts_event_en = PTP_TCR_TSEVNTENA; 676 677 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 678 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 679 break; 680 681 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 682 /* PTP v1, UDP, Delay_req packet */ 683 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 684 /* take time stamp for Delay_Req messages only */ 685 ts_master_en = PTP_TCR_TSMSTRENA; 686 ts_event_en = PTP_TCR_TSEVNTENA; 687 688 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 689 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 690 break; 691 692 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 693 /* PTP v2, UDP, any kind of event packet */ 694 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 695 ptp_v2 = PTP_TCR_TSVER2ENA; 696 /* take time stamp for all event messages */ 697 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 698 699 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 700 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 701 break; 702 703 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 704 /* PTP v2, UDP, Sync packet */ 705 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 706 ptp_v2 = PTP_TCR_TSVER2ENA; 707 /* take time stamp for SYNC messages only */ 708 ts_event_en = PTP_TCR_TSEVNTENA; 709 710 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 711 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 712 break; 713 714 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 715 /* PTP v2, UDP, Delay_req packet */ 716 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 717 ptp_v2 = PTP_TCR_TSVER2ENA; 718 /* take time stamp for Delay_Req messages only */ 719 ts_master_en = PTP_TCR_TSMSTRENA; 720 ts_event_en = PTP_TCR_TSEVNTENA; 721 722 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 723 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 724 break; 725 726 case HWTSTAMP_FILTER_PTP_V2_EVENT: 727 /* PTP v2/802.AS1 any layer, any kind of event packet */ 728 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 729 ptp_v2 = PTP_TCR_TSVER2ENA; 730 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 731 if (priv->synopsys_id < DWMAC_CORE_4_10) 732 ts_event_en = PTP_TCR_TSEVNTENA; 733 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 734 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 735 ptp_over_ethernet = PTP_TCR_TSIPENA; 736 break; 737 738 case HWTSTAMP_FILTER_PTP_V2_SYNC: 739 /* PTP v2/802.AS1, any layer, Sync packet */ 740 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 741 ptp_v2 = PTP_TCR_TSVER2ENA; 742 /* take time stamp for SYNC messages only */ 743 ts_event_en = PTP_TCR_TSEVNTENA; 744 745 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 746 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 747 ptp_over_ethernet = PTP_TCR_TSIPENA; 748 break; 749 750 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 751 /* PTP v2/802.AS1, any layer, Delay_req packet */ 752 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 753 ptp_v2 = PTP_TCR_TSVER2ENA; 754 /* take time stamp for Delay_Req messages only */ 755 ts_master_en = PTP_TCR_TSMSTRENA; 756 ts_event_en = PTP_TCR_TSEVNTENA; 757 758 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 759 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 760 ptp_over_ethernet = PTP_TCR_TSIPENA; 761 break; 762 763 case HWTSTAMP_FILTER_NTP_ALL: 764 case HWTSTAMP_FILTER_ALL: 765 /* time stamp any incoming packet */ 766 config.rx_filter = HWTSTAMP_FILTER_ALL; 767 tstamp_all = PTP_TCR_TSENALL; 768 break; 769 770 default: 771 return -ERANGE; 772 } 773 } else { 774 switch (config.rx_filter) { 775 case HWTSTAMP_FILTER_NONE: 776 config.rx_filter = HWTSTAMP_FILTER_NONE; 777 break; 778 default: 779 /* PTP v1, UDP, any kind of event packet */ 780 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 781 break; 782 } 783 } 784 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); 785 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; 786 787 priv->systime_flags = STMMAC_HWTS_ACTIVE; 788 789 if (priv->hwts_tx_en || priv->hwts_rx_en) { 790 priv->systime_flags |= tstamp_all | ptp_v2 | 791 ptp_over_ethernet | ptp_over_ipv6_udp | 792 ptp_over_ipv4_udp | ts_event_en | 793 ts_master_en | snap_type_sel; 794 } 795 796 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 797 798 memcpy(&priv->tstamp_config, &config, sizeof(config)); 799 800 return copy_to_user(ifr->ifr_data, &config, 801 sizeof(config)) ? -EFAULT : 0; 802 } 803 804 /** 805 * stmmac_hwtstamp_get - read hardware timestamping. 806 * @dev: device pointer. 807 * @ifr: An IOCTL specific structure, that can contain a pointer to 808 * a proprietary structure used to pass information to the driver. 809 * Description: 810 * This function obtain the current hardware timestamping settings 811 * as requested. 812 */ 813 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 814 { 815 struct stmmac_priv *priv = netdev_priv(dev); 816 struct hwtstamp_config *config = &priv->tstamp_config; 817 818 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 819 return -EOPNOTSUPP; 820 821 return copy_to_user(ifr->ifr_data, config, 822 sizeof(*config)) ? -EFAULT : 0; 823 } 824 825 /** 826 * stmmac_init_tstamp_counter - init hardware timestamping counter 827 * @priv: driver private structure 828 * @systime_flags: timestamping flags 829 * Description: 830 * Initialize hardware counter for packet timestamping. 831 * This is valid as long as the interface is open and not suspended. 832 * Will be rerun after resuming from suspend, case in which the timestamping 833 * flags updated by stmmac_hwtstamp_set() also need to be restored. 834 */ 835 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) 836 { 837 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 838 struct timespec64 now; 839 u32 sec_inc = 0; 840 u64 temp = 0; 841 842 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 843 return -EOPNOTSUPP; 844 845 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 846 priv->systime_flags = systime_flags; 847 848 /* program Sub Second Increment reg */ 849 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 850 priv->plat->clk_ptp_rate, 851 xmac, &sec_inc); 852 temp = div_u64(1000000000ULL, sec_inc); 853 854 /* Store sub second increment for later use */ 855 priv->sub_second_inc = sec_inc; 856 857 /* calculate default added value: 858 * formula is : 859 * addend = (2^32)/freq_div_ratio; 860 * where, freq_div_ratio = 1e9ns/sec_inc 861 */ 862 temp = (u64)(temp << 32); 863 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 864 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 865 866 /* initialize system time */ 867 ktime_get_real_ts64(&now); 868 869 /* lower 32 bits of tv_sec are safe until y2106 */ 870 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 871 872 return 0; 873 } 874 EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); 875 876 /** 877 * stmmac_init_ptp - init PTP 878 * @priv: driver private structure 879 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 880 * This is done by looking at the HW cap. register. 881 * This function also registers the ptp driver. 882 */ 883 static int stmmac_init_ptp(struct stmmac_priv *priv) 884 { 885 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 886 int ret; 887 888 if (priv->plat->ptp_clk_freq_config) 889 priv->plat->ptp_clk_freq_config(priv); 890 891 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); 892 if (ret) 893 return ret; 894 895 priv->adv_ts = 0; 896 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 897 if (xmac && priv->dma_cap.atime_stamp) 898 priv->adv_ts = 1; 899 /* Dwmac 3.x core with extend_desc can support adv_ts */ 900 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 901 priv->adv_ts = 1; 902 903 if (priv->dma_cap.time_stamp) 904 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 905 906 if (priv->adv_ts) 907 netdev_info(priv->dev, 908 "IEEE 1588-2008 Advanced Timestamp supported\n"); 909 910 priv->hwts_tx_en = 0; 911 priv->hwts_rx_en = 0; 912 913 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 914 stmmac_hwtstamp_correct_latency(priv, priv); 915 916 return 0; 917 } 918 919 static void stmmac_release_ptp(struct stmmac_priv *priv) 920 { 921 clk_disable_unprepare(priv->plat->clk_ptp_ref); 922 stmmac_ptp_unregister(priv); 923 } 924 925 /** 926 * stmmac_mac_flow_ctrl - Configure flow control in all queues 927 * @priv: driver private structure 928 * @duplex: duplex passed to the next function 929 * Description: It is used for configuring the flow control in all queues 930 */ 931 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex) 932 { 933 u32 tx_cnt = priv->plat->tx_queues_to_use; 934 935 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, 936 priv->pause, tx_cnt); 937 } 938 939 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 940 phy_interface_t interface) 941 { 942 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 943 944 if (priv->hw->xpcs) 945 return &priv->hw->xpcs->pcs; 946 947 if (priv->hw->lynx_pcs) 948 return priv->hw->lynx_pcs; 949 950 return NULL; 951 } 952 953 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 954 const struct phylink_link_state *state) 955 { 956 /* Nothing to do, xpcs_config() handles everything */ 957 } 958 959 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) 960 { 961 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 962 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 963 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 964 bool *hs_enable = &fpe_cfg->hs_enable; 965 966 if (is_up && *hs_enable) { 967 stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY); 968 } else { 969 *lo_state = FPE_STATE_OFF; 970 *lp_state = FPE_STATE_OFF; 971 } 972 } 973 974 static void stmmac_mac_link_down(struct phylink_config *config, 975 unsigned int mode, phy_interface_t interface) 976 { 977 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 978 979 stmmac_mac_set(priv, priv->ioaddr, false); 980 priv->eee_active = false; 981 priv->tx_lpi_enabled = false; 982 priv->eee_enabled = stmmac_eee_init(priv); 983 stmmac_set_eee_pls(priv, priv->hw, false); 984 985 if (priv->dma_cap.fpesel) 986 stmmac_fpe_link_state_handle(priv, false); 987 } 988 989 static void stmmac_mac_link_up(struct phylink_config *config, 990 struct phy_device *phy, 991 unsigned int mode, phy_interface_t interface, 992 int speed, int duplex, 993 bool tx_pause, bool rx_pause) 994 { 995 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 996 u32 old_ctrl, ctrl; 997 998 if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 999 priv->plat->serdes_powerup) 1000 priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 1001 1002 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 1003 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 1004 1005 if (interface == PHY_INTERFACE_MODE_USXGMII) { 1006 switch (speed) { 1007 case SPEED_10000: 1008 ctrl |= priv->hw->link.xgmii.speed10000; 1009 break; 1010 case SPEED_5000: 1011 ctrl |= priv->hw->link.xgmii.speed5000; 1012 break; 1013 case SPEED_2500: 1014 ctrl |= priv->hw->link.xgmii.speed2500; 1015 break; 1016 default: 1017 return; 1018 } 1019 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 1020 switch (speed) { 1021 case SPEED_100000: 1022 ctrl |= priv->hw->link.xlgmii.speed100000; 1023 break; 1024 case SPEED_50000: 1025 ctrl |= priv->hw->link.xlgmii.speed50000; 1026 break; 1027 case SPEED_40000: 1028 ctrl |= priv->hw->link.xlgmii.speed40000; 1029 break; 1030 case SPEED_25000: 1031 ctrl |= priv->hw->link.xlgmii.speed25000; 1032 break; 1033 case SPEED_10000: 1034 ctrl |= priv->hw->link.xgmii.speed10000; 1035 break; 1036 case SPEED_2500: 1037 ctrl |= priv->hw->link.speed2500; 1038 break; 1039 case SPEED_1000: 1040 ctrl |= priv->hw->link.speed1000; 1041 break; 1042 default: 1043 return; 1044 } 1045 } else { 1046 switch (speed) { 1047 case SPEED_2500: 1048 ctrl |= priv->hw->link.speed2500; 1049 break; 1050 case SPEED_1000: 1051 ctrl |= priv->hw->link.speed1000; 1052 break; 1053 case SPEED_100: 1054 ctrl |= priv->hw->link.speed100; 1055 break; 1056 case SPEED_10: 1057 ctrl |= priv->hw->link.speed10; 1058 break; 1059 default: 1060 return; 1061 } 1062 } 1063 1064 priv->speed = speed; 1065 1066 if (priv->plat->fix_mac_speed) 1067 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); 1068 1069 if (!duplex) 1070 ctrl &= ~priv->hw->link.duplex; 1071 else 1072 ctrl |= priv->hw->link.duplex; 1073 1074 /* Flow Control operation */ 1075 if (rx_pause && tx_pause) 1076 priv->flow_ctrl = FLOW_AUTO; 1077 else if (rx_pause && !tx_pause) 1078 priv->flow_ctrl = FLOW_RX; 1079 else if (!rx_pause && tx_pause) 1080 priv->flow_ctrl = FLOW_TX; 1081 else 1082 priv->flow_ctrl = FLOW_OFF; 1083 1084 stmmac_mac_flow_ctrl(priv, duplex); 1085 1086 if (ctrl != old_ctrl) 1087 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1088 1089 stmmac_mac_set(priv, priv->ioaddr, true); 1090 if (phy && priv->dma_cap.eee) { 1091 priv->eee_active = 1092 phy_init_eee(phy, !(priv->plat->flags & 1093 STMMAC_FLAG_RX_CLK_RUNS_IN_LPI)) >= 0; 1094 priv->eee_enabled = stmmac_eee_init(priv); 1095 priv->tx_lpi_enabled = priv->eee_enabled; 1096 stmmac_set_eee_pls(priv, priv->hw, true); 1097 } 1098 1099 if (priv->dma_cap.fpesel) 1100 stmmac_fpe_link_state_handle(priv, true); 1101 1102 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1103 stmmac_hwtstamp_correct_latency(priv, priv); 1104 } 1105 1106 static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1107 .mac_select_pcs = stmmac_mac_select_pcs, 1108 .mac_config = stmmac_mac_config, 1109 .mac_link_down = stmmac_mac_link_down, 1110 .mac_link_up = stmmac_mac_link_up, 1111 }; 1112 1113 /** 1114 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1115 * @priv: driver private structure 1116 * Description: this is to verify if the HW supports the PCS. 1117 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1118 * configured for the TBI, RTBI, or SGMII PHY interface. 1119 */ 1120 static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1121 { 1122 int interface = priv->plat->interface; 1123 1124 if (priv->dma_cap.pcs) { 1125 if ((interface == PHY_INTERFACE_MODE_RGMII) || 1126 (interface == PHY_INTERFACE_MODE_RGMII_ID) || 1127 (interface == PHY_INTERFACE_MODE_RGMII_RXID) || 1128 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { 1129 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); 1130 priv->hw->pcs = STMMAC_PCS_RGMII; 1131 } else if (interface == PHY_INTERFACE_MODE_SGMII) { 1132 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1133 priv->hw->pcs = STMMAC_PCS_SGMII; 1134 } 1135 } 1136 } 1137 1138 /** 1139 * stmmac_init_phy - PHY initialization 1140 * @dev: net device structure 1141 * Description: it initializes the driver's PHY state, and attaches the PHY 1142 * to the mac driver. 1143 * Return value: 1144 * 0 on success 1145 */ 1146 static int stmmac_init_phy(struct net_device *dev) 1147 { 1148 struct stmmac_priv *priv = netdev_priv(dev); 1149 struct fwnode_handle *phy_fwnode; 1150 struct fwnode_handle *fwnode; 1151 int ret; 1152 1153 if (!phylink_expects_phy(priv->phylink)) 1154 return 0; 1155 1156 fwnode = of_fwnode_handle(priv->plat->phylink_node); 1157 if (!fwnode) 1158 fwnode = dev_fwnode(priv->device); 1159 1160 if (fwnode) 1161 phy_fwnode = fwnode_get_phy_node(fwnode); 1162 else 1163 phy_fwnode = NULL; 1164 1165 /* Some DT bindings do not set-up the PHY handle. Let's try to 1166 * manually parse it 1167 */ 1168 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1169 int addr = priv->plat->phy_addr; 1170 struct phy_device *phydev; 1171 1172 if (addr < 0) { 1173 netdev_err(priv->dev, "no phy found\n"); 1174 return -ENODEV; 1175 } 1176 1177 phydev = mdiobus_get_phy(priv->mii, addr); 1178 if (!phydev) { 1179 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1180 return -ENODEV; 1181 } 1182 1183 ret = phylink_connect_phy(priv->phylink, phydev); 1184 } else { 1185 fwnode_handle_put(phy_fwnode); 1186 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); 1187 } 1188 1189 if (!priv->plat->pmt) { 1190 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 1191 1192 phylink_ethtool_get_wol(priv->phylink, &wol); 1193 device_set_wakeup_capable(priv->device, !!wol.supported); 1194 device_set_wakeup_enable(priv->device, !!wol.wolopts); 1195 } 1196 1197 return ret; 1198 } 1199 1200 static int stmmac_phy_setup(struct stmmac_priv *priv) 1201 { 1202 struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data; 1203 struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node); 1204 int max_speed = priv->plat->max_speed; 1205 int mode = priv->plat->phy_interface; 1206 struct phylink *phylink; 1207 1208 priv->phylink_config.dev = &priv->dev->dev; 1209 priv->phylink_config.type = PHYLINK_NETDEV; 1210 if (priv->plat->mdio_bus_data) 1211 priv->phylink_config.ovr_an_inband = 1212 mdio_bus_data->xpcs_an_inband; 1213 1214 if (!fwnode) 1215 fwnode = dev_fwnode(priv->device); 1216 1217 /* Set the platform/firmware specified interface mode */ 1218 __set_bit(mode, priv->phylink_config.supported_interfaces); 1219 1220 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1221 if (priv->hw->xpcs) 1222 xpcs_get_interfaces(priv->hw->xpcs, 1223 priv->phylink_config.supported_interfaces); 1224 1225 priv->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | 1226 MAC_10 | MAC_100; 1227 1228 if (!max_speed || max_speed >= 1000) 1229 priv->phylink_config.mac_capabilities |= MAC_1000; 1230 1231 if (priv->plat->has_gmac4) { 1232 if (!max_speed || max_speed >= 2500) 1233 priv->phylink_config.mac_capabilities |= MAC_2500FD; 1234 } else if (priv->plat->has_xgmac) { 1235 if (!max_speed || max_speed >= 2500) 1236 priv->phylink_config.mac_capabilities |= MAC_2500FD; 1237 if (!max_speed || max_speed >= 5000) 1238 priv->phylink_config.mac_capabilities |= MAC_5000FD; 1239 if (!max_speed || max_speed >= 10000) 1240 priv->phylink_config.mac_capabilities |= MAC_10000FD; 1241 if (!max_speed || max_speed >= 25000) 1242 priv->phylink_config.mac_capabilities |= MAC_25000FD; 1243 if (!max_speed || max_speed >= 40000) 1244 priv->phylink_config.mac_capabilities |= MAC_40000FD; 1245 if (!max_speed || max_speed >= 50000) 1246 priv->phylink_config.mac_capabilities |= MAC_50000FD; 1247 if (!max_speed || max_speed >= 100000) 1248 priv->phylink_config.mac_capabilities |= MAC_100000FD; 1249 } 1250 1251 /* Half-Duplex can only work with single queue */ 1252 if (priv->plat->tx_queues_to_use > 1) 1253 priv->phylink_config.mac_capabilities &= 1254 ~(MAC_10HD | MAC_100HD | MAC_1000HD); 1255 priv->phylink_config.mac_managed_pm = true; 1256 1257 phylink = phylink_create(&priv->phylink_config, fwnode, 1258 mode, &stmmac_phylink_mac_ops); 1259 if (IS_ERR(phylink)) 1260 return PTR_ERR(phylink); 1261 1262 priv->phylink = phylink; 1263 return 0; 1264 } 1265 1266 static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1267 struct stmmac_dma_conf *dma_conf) 1268 { 1269 u32 rx_cnt = priv->plat->rx_queues_to_use; 1270 unsigned int desc_size; 1271 void *head_rx; 1272 u32 queue; 1273 1274 /* Display RX rings */ 1275 for (queue = 0; queue < rx_cnt; queue++) { 1276 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1277 1278 pr_info("\tRX Queue %u rings\n", queue); 1279 1280 if (priv->extend_desc) { 1281 head_rx = (void *)rx_q->dma_erx; 1282 desc_size = sizeof(struct dma_extended_desc); 1283 } else { 1284 head_rx = (void *)rx_q->dma_rx; 1285 desc_size = sizeof(struct dma_desc); 1286 } 1287 1288 /* Display RX ring */ 1289 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1290 rx_q->dma_rx_phy, desc_size); 1291 } 1292 } 1293 1294 static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1295 struct stmmac_dma_conf *dma_conf) 1296 { 1297 u32 tx_cnt = priv->plat->tx_queues_to_use; 1298 unsigned int desc_size; 1299 void *head_tx; 1300 u32 queue; 1301 1302 /* Display TX rings */ 1303 for (queue = 0; queue < tx_cnt; queue++) { 1304 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1305 1306 pr_info("\tTX Queue %d rings\n", queue); 1307 1308 if (priv->extend_desc) { 1309 head_tx = (void *)tx_q->dma_etx; 1310 desc_size = sizeof(struct dma_extended_desc); 1311 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1312 head_tx = (void *)tx_q->dma_entx; 1313 desc_size = sizeof(struct dma_edesc); 1314 } else { 1315 head_tx = (void *)tx_q->dma_tx; 1316 desc_size = sizeof(struct dma_desc); 1317 } 1318 1319 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1320 tx_q->dma_tx_phy, desc_size); 1321 } 1322 } 1323 1324 static void stmmac_display_rings(struct stmmac_priv *priv, 1325 struct stmmac_dma_conf *dma_conf) 1326 { 1327 /* Display RX ring */ 1328 stmmac_display_rx_rings(priv, dma_conf); 1329 1330 /* Display TX ring */ 1331 stmmac_display_tx_rings(priv, dma_conf); 1332 } 1333 1334 static int stmmac_set_bfsize(int mtu, int bufsize) 1335 { 1336 int ret = bufsize; 1337 1338 if (mtu >= BUF_SIZE_8KiB) 1339 ret = BUF_SIZE_16KiB; 1340 else if (mtu >= BUF_SIZE_4KiB) 1341 ret = BUF_SIZE_8KiB; 1342 else if (mtu >= BUF_SIZE_2KiB) 1343 ret = BUF_SIZE_4KiB; 1344 else if (mtu > DEFAULT_BUFSIZE) 1345 ret = BUF_SIZE_2KiB; 1346 else 1347 ret = DEFAULT_BUFSIZE; 1348 1349 return ret; 1350 } 1351 1352 /** 1353 * stmmac_clear_rx_descriptors - clear RX descriptors 1354 * @priv: driver private structure 1355 * @dma_conf: structure to take the dma data 1356 * @queue: RX queue index 1357 * Description: this function is called to clear the RX descriptors 1358 * in case of both basic and extended descriptors are used. 1359 */ 1360 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1361 struct stmmac_dma_conf *dma_conf, 1362 u32 queue) 1363 { 1364 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1365 int i; 1366 1367 /* Clear the RX descriptors */ 1368 for (i = 0; i < dma_conf->dma_rx_size; i++) 1369 if (priv->extend_desc) 1370 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1371 priv->use_riwt, priv->mode, 1372 (i == dma_conf->dma_rx_size - 1), 1373 dma_conf->dma_buf_sz); 1374 else 1375 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1376 priv->use_riwt, priv->mode, 1377 (i == dma_conf->dma_rx_size - 1), 1378 dma_conf->dma_buf_sz); 1379 } 1380 1381 /** 1382 * stmmac_clear_tx_descriptors - clear tx descriptors 1383 * @priv: driver private structure 1384 * @dma_conf: structure to take the dma data 1385 * @queue: TX queue index. 1386 * Description: this function is called to clear the TX descriptors 1387 * in case of both basic and extended descriptors are used. 1388 */ 1389 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1390 struct stmmac_dma_conf *dma_conf, 1391 u32 queue) 1392 { 1393 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1394 int i; 1395 1396 /* Clear the TX descriptors */ 1397 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1398 int last = (i == (dma_conf->dma_tx_size - 1)); 1399 struct dma_desc *p; 1400 1401 if (priv->extend_desc) 1402 p = &tx_q->dma_etx[i].basic; 1403 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1404 p = &tx_q->dma_entx[i].basic; 1405 else 1406 p = &tx_q->dma_tx[i]; 1407 1408 stmmac_init_tx_desc(priv, p, priv->mode, last); 1409 } 1410 } 1411 1412 /** 1413 * stmmac_clear_descriptors - clear descriptors 1414 * @priv: driver private structure 1415 * @dma_conf: structure to take the dma data 1416 * Description: this function is called to clear the TX and RX descriptors 1417 * in case of both basic and extended descriptors are used. 1418 */ 1419 static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1420 struct stmmac_dma_conf *dma_conf) 1421 { 1422 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1423 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1424 u32 queue; 1425 1426 /* Clear the RX descriptors */ 1427 for (queue = 0; queue < rx_queue_cnt; queue++) 1428 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1429 1430 /* Clear the TX descriptors */ 1431 for (queue = 0; queue < tx_queue_cnt; queue++) 1432 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1433 } 1434 1435 /** 1436 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1437 * @priv: driver private structure 1438 * @dma_conf: structure to take the dma data 1439 * @p: descriptor pointer 1440 * @i: descriptor index 1441 * @flags: gfp flag 1442 * @queue: RX queue index 1443 * Description: this function is called to allocate a receive buffer, perform 1444 * the DMA mapping and init the descriptor. 1445 */ 1446 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1447 struct stmmac_dma_conf *dma_conf, 1448 struct dma_desc *p, 1449 int i, gfp_t flags, u32 queue) 1450 { 1451 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1452 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1453 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1454 1455 if (priv->dma_cap.host_dma_width <= 32) 1456 gfp |= GFP_DMA32; 1457 1458 if (!buf->page) { 1459 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1460 if (!buf->page) 1461 return -ENOMEM; 1462 buf->page_offset = stmmac_rx_offset(priv); 1463 } 1464 1465 if (priv->sph && !buf->sec_page) { 1466 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1467 if (!buf->sec_page) 1468 return -ENOMEM; 1469 1470 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1471 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1472 } else { 1473 buf->sec_page = NULL; 1474 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1475 } 1476 1477 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1478 1479 stmmac_set_desc_addr(priv, p, buf->addr); 1480 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1481 stmmac_init_desc3(priv, p); 1482 1483 return 0; 1484 } 1485 1486 /** 1487 * stmmac_free_rx_buffer - free RX dma buffers 1488 * @priv: private structure 1489 * @rx_q: RX queue 1490 * @i: buffer index. 1491 */ 1492 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1493 struct stmmac_rx_queue *rx_q, 1494 int i) 1495 { 1496 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1497 1498 if (buf->page) 1499 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1500 buf->page = NULL; 1501 1502 if (buf->sec_page) 1503 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1504 buf->sec_page = NULL; 1505 } 1506 1507 /** 1508 * stmmac_free_tx_buffer - free RX dma buffers 1509 * @priv: private structure 1510 * @dma_conf: structure to take the dma data 1511 * @queue: RX queue index 1512 * @i: buffer index. 1513 */ 1514 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1515 struct stmmac_dma_conf *dma_conf, 1516 u32 queue, int i) 1517 { 1518 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1519 1520 if (tx_q->tx_skbuff_dma[i].buf && 1521 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1522 if (tx_q->tx_skbuff_dma[i].map_as_page) 1523 dma_unmap_page(priv->device, 1524 tx_q->tx_skbuff_dma[i].buf, 1525 tx_q->tx_skbuff_dma[i].len, 1526 DMA_TO_DEVICE); 1527 else 1528 dma_unmap_single(priv->device, 1529 tx_q->tx_skbuff_dma[i].buf, 1530 tx_q->tx_skbuff_dma[i].len, 1531 DMA_TO_DEVICE); 1532 } 1533 1534 if (tx_q->xdpf[i] && 1535 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1536 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1537 xdp_return_frame(tx_q->xdpf[i]); 1538 tx_q->xdpf[i] = NULL; 1539 } 1540 1541 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1542 tx_q->xsk_frames_done++; 1543 1544 if (tx_q->tx_skbuff[i] && 1545 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1546 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1547 tx_q->tx_skbuff[i] = NULL; 1548 } 1549 1550 tx_q->tx_skbuff_dma[i].buf = 0; 1551 tx_q->tx_skbuff_dma[i].map_as_page = false; 1552 } 1553 1554 /** 1555 * dma_free_rx_skbufs - free RX dma buffers 1556 * @priv: private structure 1557 * @dma_conf: structure to take the dma data 1558 * @queue: RX queue index 1559 */ 1560 static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1561 struct stmmac_dma_conf *dma_conf, 1562 u32 queue) 1563 { 1564 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1565 int i; 1566 1567 for (i = 0; i < dma_conf->dma_rx_size; i++) 1568 stmmac_free_rx_buffer(priv, rx_q, i); 1569 } 1570 1571 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1572 struct stmmac_dma_conf *dma_conf, 1573 u32 queue, gfp_t flags) 1574 { 1575 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1576 int i; 1577 1578 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1579 struct dma_desc *p; 1580 int ret; 1581 1582 if (priv->extend_desc) 1583 p = &((rx_q->dma_erx + i)->basic); 1584 else 1585 p = rx_q->dma_rx + i; 1586 1587 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1588 queue); 1589 if (ret) 1590 return ret; 1591 1592 rx_q->buf_alloc_num++; 1593 } 1594 1595 return 0; 1596 } 1597 1598 /** 1599 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1600 * @priv: private structure 1601 * @dma_conf: structure to take the dma data 1602 * @queue: RX queue index 1603 */ 1604 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1605 struct stmmac_dma_conf *dma_conf, 1606 u32 queue) 1607 { 1608 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1609 int i; 1610 1611 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1612 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1613 1614 if (!buf->xdp) 1615 continue; 1616 1617 xsk_buff_free(buf->xdp); 1618 buf->xdp = NULL; 1619 } 1620 } 1621 1622 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1623 struct stmmac_dma_conf *dma_conf, 1624 u32 queue) 1625 { 1626 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1627 int i; 1628 1629 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1630 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1631 * use this macro to make sure no size violations. 1632 */ 1633 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1634 1635 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1636 struct stmmac_rx_buffer *buf; 1637 dma_addr_t dma_addr; 1638 struct dma_desc *p; 1639 1640 if (priv->extend_desc) 1641 p = (struct dma_desc *)(rx_q->dma_erx + i); 1642 else 1643 p = rx_q->dma_rx + i; 1644 1645 buf = &rx_q->buf_pool[i]; 1646 1647 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1648 if (!buf->xdp) 1649 return -ENOMEM; 1650 1651 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1652 stmmac_set_desc_addr(priv, p, dma_addr); 1653 rx_q->buf_alloc_num++; 1654 } 1655 1656 return 0; 1657 } 1658 1659 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1660 { 1661 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1662 return NULL; 1663 1664 return xsk_get_pool_from_qid(priv->dev, queue); 1665 } 1666 1667 /** 1668 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1669 * @priv: driver private structure 1670 * @dma_conf: structure to take the dma data 1671 * @queue: RX queue index 1672 * @flags: gfp flag. 1673 * Description: this function initializes the DMA RX descriptors 1674 * and allocates the socket buffers. It supports the chained and ring 1675 * modes. 1676 */ 1677 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1678 struct stmmac_dma_conf *dma_conf, 1679 u32 queue, gfp_t flags) 1680 { 1681 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1682 int ret; 1683 1684 netif_dbg(priv, probe, priv->dev, 1685 "(%s) dma_rx_phy=0x%08x\n", __func__, 1686 (u32)rx_q->dma_rx_phy); 1687 1688 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1689 1690 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1691 1692 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1693 1694 if (rx_q->xsk_pool) { 1695 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1696 MEM_TYPE_XSK_BUFF_POOL, 1697 NULL)); 1698 netdev_info(priv->dev, 1699 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1700 rx_q->queue_index); 1701 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1702 } else { 1703 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1704 MEM_TYPE_PAGE_POOL, 1705 rx_q->page_pool)); 1706 netdev_info(priv->dev, 1707 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1708 rx_q->queue_index); 1709 } 1710 1711 if (rx_q->xsk_pool) { 1712 /* RX XDP ZC buffer pool may not be populated, e.g. 1713 * xdpsock TX-only. 1714 */ 1715 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1716 } else { 1717 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1718 if (ret < 0) 1719 return -ENOMEM; 1720 } 1721 1722 /* Setup the chained descriptor addresses */ 1723 if (priv->mode == STMMAC_CHAIN_MODE) { 1724 if (priv->extend_desc) 1725 stmmac_mode_init(priv, rx_q->dma_erx, 1726 rx_q->dma_rx_phy, 1727 dma_conf->dma_rx_size, 1); 1728 else 1729 stmmac_mode_init(priv, rx_q->dma_rx, 1730 rx_q->dma_rx_phy, 1731 dma_conf->dma_rx_size, 0); 1732 } 1733 1734 return 0; 1735 } 1736 1737 static int init_dma_rx_desc_rings(struct net_device *dev, 1738 struct stmmac_dma_conf *dma_conf, 1739 gfp_t flags) 1740 { 1741 struct stmmac_priv *priv = netdev_priv(dev); 1742 u32 rx_count = priv->plat->rx_queues_to_use; 1743 int queue; 1744 int ret; 1745 1746 /* RX INITIALIZATION */ 1747 netif_dbg(priv, probe, priv->dev, 1748 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1749 1750 for (queue = 0; queue < rx_count; queue++) { 1751 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1752 if (ret) 1753 goto err_init_rx_buffers; 1754 } 1755 1756 return 0; 1757 1758 err_init_rx_buffers: 1759 while (queue >= 0) { 1760 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1761 1762 if (rx_q->xsk_pool) 1763 dma_free_rx_xskbufs(priv, dma_conf, queue); 1764 else 1765 dma_free_rx_skbufs(priv, dma_conf, queue); 1766 1767 rx_q->buf_alloc_num = 0; 1768 rx_q->xsk_pool = NULL; 1769 1770 queue--; 1771 } 1772 1773 return ret; 1774 } 1775 1776 /** 1777 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1778 * @priv: driver private structure 1779 * @dma_conf: structure to take the dma data 1780 * @queue: TX queue index 1781 * Description: this function initializes the DMA TX descriptors 1782 * and allocates the socket buffers. It supports the chained and ring 1783 * modes. 1784 */ 1785 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 1786 struct stmmac_dma_conf *dma_conf, 1787 u32 queue) 1788 { 1789 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1790 int i; 1791 1792 netif_dbg(priv, probe, priv->dev, 1793 "(%s) dma_tx_phy=0x%08x\n", __func__, 1794 (u32)tx_q->dma_tx_phy); 1795 1796 /* Setup the chained descriptor addresses */ 1797 if (priv->mode == STMMAC_CHAIN_MODE) { 1798 if (priv->extend_desc) 1799 stmmac_mode_init(priv, tx_q->dma_etx, 1800 tx_q->dma_tx_phy, 1801 dma_conf->dma_tx_size, 1); 1802 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 1803 stmmac_mode_init(priv, tx_q->dma_tx, 1804 tx_q->dma_tx_phy, 1805 dma_conf->dma_tx_size, 0); 1806 } 1807 1808 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1809 1810 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1811 struct dma_desc *p; 1812 1813 if (priv->extend_desc) 1814 p = &((tx_q->dma_etx + i)->basic); 1815 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1816 p = &((tx_q->dma_entx + i)->basic); 1817 else 1818 p = tx_q->dma_tx + i; 1819 1820 stmmac_clear_desc(priv, p); 1821 1822 tx_q->tx_skbuff_dma[i].buf = 0; 1823 tx_q->tx_skbuff_dma[i].map_as_page = false; 1824 tx_q->tx_skbuff_dma[i].len = 0; 1825 tx_q->tx_skbuff_dma[i].last_segment = false; 1826 tx_q->tx_skbuff[i] = NULL; 1827 } 1828 1829 return 0; 1830 } 1831 1832 static int init_dma_tx_desc_rings(struct net_device *dev, 1833 struct stmmac_dma_conf *dma_conf) 1834 { 1835 struct stmmac_priv *priv = netdev_priv(dev); 1836 u32 tx_queue_cnt; 1837 u32 queue; 1838 1839 tx_queue_cnt = priv->plat->tx_queues_to_use; 1840 1841 for (queue = 0; queue < tx_queue_cnt; queue++) 1842 __init_dma_tx_desc_rings(priv, dma_conf, queue); 1843 1844 return 0; 1845 } 1846 1847 /** 1848 * init_dma_desc_rings - init the RX/TX descriptor rings 1849 * @dev: net device structure 1850 * @dma_conf: structure to take the dma data 1851 * @flags: gfp flag. 1852 * Description: this function initializes the DMA RX/TX descriptors 1853 * and allocates the socket buffers. It supports the chained and ring 1854 * modes. 1855 */ 1856 static int init_dma_desc_rings(struct net_device *dev, 1857 struct stmmac_dma_conf *dma_conf, 1858 gfp_t flags) 1859 { 1860 struct stmmac_priv *priv = netdev_priv(dev); 1861 int ret; 1862 1863 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 1864 if (ret) 1865 return ret; 1866 1867 ret = init_dma_tx_desc_rings(dev, dma_conf); 1868 1869 stmmac_clear_descriptors(priv, dma_conf); 1870 1871 if (netif_msg_hw(priv)) 1872 stmmac_display_rings(priv, dma_conf); 1873 1874 return ret; 1875 } 1876 1877 /** 1878 * dma_free_tx_skbufs - free TX dma buffers 1879 * @priv: private structure 1880 * @dma_conf: structure to take the dma data 1881 * @queue: TX queue index 1882 */ 1883 static void dma_free_tx_skbufs(struct stmmac_priv *priv, 1884 struct stmmac_dma_conf *dma_conf, 1885 u32 queue) 1886 { 1887 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1888 int i; 1889 1890 tx_q->xsk_frames_done = 0; 1891 1892 for (i = 0; i < dma_conf->dma_tx_size; i++) 1893 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 1894 1895 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 1896 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 1897 tx_q->xsk_frames_done = 0; 1898 tx_q->xsk_pool = NULL; 1899 } 1900 } 1901 1902 /** 1903 * stmmac_free_tx_skbufs - free TX skb buffers 1904 * @priv: private structure 1905 */ 1906 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 1907 { 1908 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1909 u32 queue; 1910 1911 for (queue = 0; queue < tx_queue_cnt; queue++) 1912 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 1913 } 1914 1915 /** 1916 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 1917 * @priv: private structure 1918 * @dma_conf: structure to take the dma data 1919 * @queue: RX queue index 1920 */ 1921 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 1922 struct stmmac_dma_conf *dma_conf, 1923 u32 queue) 1924 { 1925 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1926 1927 /* Release the DMA RX socket buffers */ 1928 if (rx_q->xsk_pool) 1929 dma_free_rx_xskbufs(priv, dma_conf, queue); 1930 else 1931 dma_free_rx_skbufs(priv, dma_conf, queue); 1932 1933 rx_q->buf_alloc_num = 0; 1934 rx_q->xsk_pool = NULL; 1935 1936 /* Free DMA regions of consistent memory previously allocated */ 1937 if (!priv->extend_desc) 1938 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1939 sizeof(struct dma_desc), 1940 rx_q->dma_rx, rx_q->dma_rx_phy); 1941 else 1942 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 1943 sizeof(struct dma_extended_desc), 1944 rx_q->dma_erx, rx_q->dma_rx_phy); 1945 1946 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 1947 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 1948 1949 kfree(rx_q->buf_pool); 1950 if (rx_q->page_pool) 1951 page_pool_destroy(rx_q->page_pool); 1952 } 1953 1954 static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 1955 struct stmmac_dma_conf *dma_conf) 1956 { 1957 u32 rx_count = priv->plat->rx_queues_to_use; 1958 u32 queue; 1959 1960 /* Free RX queue resources */ 1961 for (queue = 0; queue < rx_count; queue++) 1962 __free_dma_rx_desc_resources(priv, dma_conf, queue); 1963 } 1964 1965 /** 1966 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 1967 * @priv: private structure 1968 * @dma_conf: structure to take the dma data 1969 * @queue: TX queue index 1970 */ 1971 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 1972 struct stmmac_dma_conf *dma_conf, 1973 u32 queue) 1974 { 1975 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1976 size_t size; 1977 void *addr; 1978 1979 /* Release the DMA TX socket buffers */ 1980 dma_free_tx_skbufs(priv, dma_conf, queue); 1981 1982 if (priv->extend_desc) { 1983 size = sizeof(struct dma_extended_desc); 1984 addr = tx_q->dma_etx; 1985 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1986 size = sizeof(struct dma_edesc); 1987 addr = tx_q->dma_entx; 1988 } else { 1989 size = sizeof(struct dma_desc); 1990 addr = tx_q->dma_tx; 1991 } 1992 1993 size *= dma_conf->dma_tx_size; 1994 1995 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 1996 1997 kfree(tx_q->tx_skbuff_dma); 1998 kfree(tx_q->tx_skbuff); 1999 } 2000 2001 static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 2002 struct stmmac_dma_conf *dma_conf) 2003 { 2004 u32 tx_count = priv->plat->tx_queues_to_use; 2005 u32 queue; 2006 2007 /* Free TX queue resources */ 2008 for (queue = 0; queue < tx_count; queue++) 2009 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2010 } 2011 2012 /** 2013 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2014 * @priv: private structure 2015 * @dma_conf: structure to take the dma data 2016 * @queue: RX queue index 2017 * Description: according to which descriptor can be used (extend or basic) 2018 * this function allocates the resources for TX and RX paths. In case of 2019 * reception, for example, it pre-allocated the RX socket buffer in order to 2020 * allow zero-copy mechanism. 2021 */ 2022 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2023 struct stmmac_dma_conf *dma_conf, 2024 u32 queue) 2025 { 2026 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2027 struct stmmac_channel *ch = &priv->channel[queue]; 2028 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2029 struct page_pool_params pp_params = { 0 }; 2030 unsigned int num_pages; 2031 unsigned int napi_id; 2032 int ret; 2033 2034 rx_q->queue_index = queue; 2035 rx_q->priv_data = priv; 2036 2037 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2038 pp_params.pool_size = dma_conf->dma_rx_size; 2039 num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE); 2040 pp_params.order = ilog2(num_pages); 2041 pp_params.nid = dev_to_node(priv->device); 2042 pp_params.dev = priv->device; 2043 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2044 pp_params.offset = stmmac_rx_offset(priv); 2045 pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages); 2046 2047 rx_q->page_pool = page_pool_create(&pp_params); 2048 if (IS_ERR(rx_q->page_pool)) { 2049 ret = PTR_ERR(rx_q->page_pool); 2050 rx_q->page_pool = NULL; 2051 return ret; 2052 } 2053 2054 rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, 2055 sizeof(*rx_q->buf_pool), 2056 GFP_KERNEL); 2057 if (!rx_q->buf_pool) 2058 return -ENOMEM; 2059 2060 if (priv->extend_desc) { 2061 rx_q->dma_erx = dma_alloc_coherent(priv->device, 2062 dma_conf->dma_rx_size * 2063 sizeof(struct dma_extended_desc), 2064 &rx_q->dma_rx_phy, 2065 GFP_KERNEL); 2066 if (!rx_q->dma_erx) 2067 return -ENOMEM; 2068 2069 } else { 2070 rx_q->dma_rx = dma_alloc_coherent(priv->device, 2071 dma_conf->dma_rx_size * 2072 sizeof(struct dma_desc), 2073 &rx_q->dma_rx_phy, 2074 GFP_KERNEL); 2075 if (!rx_q->dma_rx) 2076 return -ENOMEM; 2077 } 2078 2079 if (stmmac_xdp_is_enabled(priv) && 2080 test_bit(queue, priv->af_xdp_zc_qps)) 2081 napi_id = ch->rxtx_napi.napi_id; 2082 else 2083 napi_id = ch->rx_napi.napi_id; 2084 2085 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, 2086 rx_q->queue_index, 2087 napi_id); 2088 if (ret) { 2089 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2090 return -EINVAL; 2091 } 2092 2093 return 0; 2094 } 2095 2096 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2097 struct stmmac_dma_conf *dma_conf) 2098 { 2099 u32 rx_count = priv->plat->rx_queues_to_use; 2100 u32 queue; 2101 int ret; 2102 2103 /* RX queues buffers and DMA */ 2104 for (queue = 0; queue < rx_count; queue++) { 2105 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2106 if (ret) 2107 goto err_dma; 2108 } 2109 2110 return 0; 2111 2112 err_dma: 2113 free_dma_rx_desc_resources(priv, dma_conf); 2114 2115 return ret; 2116 } 2117 2118 /** 2119 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2120 * @priv: private structure 2121 * @dma_conf: structure to take the dma data 2122 * @queue: TX queue index 2123 * Description: according to which descriptor can be used (extend or basic) 2124 * this function allocates the resources for TX and RX paths. In case of 2125 * reception, for example, it pre-allocated the RX socket buffer in order to 2126 * allow zero-copy mechanism. 2127 */ 2128 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2129 struct stmmac_dma_conf *dma_conf, 2130 u32 queue) 2131 { 2132 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2133 size_t size; 2134 void *addr; 2135 2136 tx_q->queue_index = queue; 2137 tx_q->priv_data = priv; 2138 2139 tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, 2140 sizeof(*tx_q->tx_skbuff_dma), 2141 GFP_KERNEL); 2142 if (!tx_q->tx_skbuff_dma) 2143 return -ENOMEM; 2144 2145 tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, 2146 sizeof(struct sk_buff *), 2147 GFP_KERNEL); 2148 if (!tx_q->tx_skbuff) 2149 return -ENOMEM; 2150 2151 if (priv->extend_desc) 2152 size = sizeof(struct dma_extended_desc); 2153 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2154 size = sizeof(struct dma_edesc); 2155 else 2156 size = sizeof(struct dma_desc); 2157 2158 size *= dma_conf->dma_tx_size; 2159 2160 addr = dma_alloc_coherent(priv->device, size, 2161 &tx_q->dma_tx_phy, GFP_KERNEL); 2162 if (!addr) 2163 return -ENOMEM; 2164 2165 if (priv->extend_desc) 2166 tx_q->dma_etx = addr; 2167 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2168 tx_q->dma_entx = addr; 2169 else 2170 tx_q->dma_tx = addr; 2171 2172 return 0; 2173 } 2174 2175 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2176 struct stmmac_dma_conf *dma_conf) 2177 { 2178 u32 tx_count = priv->plat->tx_queues_to_use; 2179 u32 queue; 2180 int ret; 2181 2182 /* TX queues buffers and DMA */ 2183 for (queue = 0; queue < tx_count; queue++) { 2184 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2185 if (ret) 2186 goto err_dma; 2187 } 2188 2189 return 0; 2190 2191 err_dma: 2192 free_dma_tx_desc_resources(priv, dma_conf); 2193 return ret; 2194 } 2195 2196 /** 2197 * alloc_dma_desc_resources - alloc TX/RX resources. 2198 * @priv: private structure 2199 * @dma_conf: structure to take the dma data 2200 * Description: according to which descriptor can be used (extend or basic) 2201 * this function allocates the resources for TX and RX paths. In case of 2202 * reception, for example, it pre-allocated the RX socket buffer in order to 2203 * allow zero-copy mechanism. 2204 */ 2205 static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2206 struct stmmac_dma_conf *dma_conf) 2207 { 2208 /* RX Allocation */ 2209 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2210 2211 if (ret) 2212 return ret; 2213 2214 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2215 2216 return ret; 2217 } 2218 2219 /** 2220 * free_dma_desc_resources - free dma desc resources 2221 * @priv: private structure 2222 * @dma_conf: structure to take the dma data 2223 */ 2224 static void free_dma_desc_resources(struct stmmac_priv *priv, 2225 struct stmmac_dma_conf *dma_conf) 2226 { 2227 /* Release the DMA TX socket buffers */ 2228 free_dma_tx_desc_resources(priv, dma_conf); 2229 2230 /* Release the DMA RX socket buffers later 2231 * to ensure all pending XDP_TX buffers are returned. 2232 */ 2233 free_dma_rx_desc_resources(priv, dma_conf); 2234 } 2235 2236 /** 2237 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2238 * @priv: driver private structure 2239 * Description: It is used for enabling the rx queues in the MAC 2240 */ 2241 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2242 { 2243 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2244 int queue; 2245 u8 mode; 2246 2247 for (queue = 0; queue < rx_queues_count; queue++) { 2248 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2249 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2250 } 2251 } 2252 2253 /** 2254 * stmmac_start_rx_dma - start RX DMA channel 2255 * @priv: driver private structure 2256 * @chan: RX channel index 2257 * Description: 2258 * This starts a RX DMA channel 2259 */ 2260 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2261 { 2262 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2263 stmmac_start_rx(priv, priv->ioaddr, chan); 2264 } 2265 2266 /** 2267 * stmmac_start_tx_dma - start TX DMA channel 2268 * @priv: driver private structure 2269 * @chan: TX channel index 2270 * Description: 2271 * This starts a TX DMA channel 2272 */ 2273 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2274 { 2275 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2276 stmmac_start_tx(priv, priv->ioaddr, chan); 2277 } 2278 2279 /** 2280 * stmmac_stop_rx_dma - stop RX DMA channel 2281 * @priv: driver private structure 2282 * @chan: RX channel index 2283 * Description: 2284 * This stops a RX DMA channel 2285 */ 2286 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2287 { 2288 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2289 stmmac_stop_rx(priv, priv->ioaddr, chan); 2290 } 2291 2292 /** 2293 * stmmac_stop_tx_dma - stop TX DMA channel 2294 * @priv: driver private structure 2295 * @chan: TX channel index 2296 * Description: 2297 * This stops a TX DMA channel 2298 */ 2299 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2300 { 2301 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2302 stmmac_stop_tx(priv, priv->ioaddr, chan); 2303 } 2304 2305 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2306 { 2307 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2308 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2309 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2310 u32 chan; 2311 2312 for (chan = 0; chan < dma_csr_ch; chan++) { 2313 struct stmmac_channel *ch = &priv->channel[chan]; 2314 unsigned long flags; 2315 2316 spin_lock_irqsave(&ch->lock, flags); 2317 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2318 spin_unlock_irqrestore(&ch->lock, flags); 2319 } 2320 } 2321 2322 /** 2323 * stmmac_start_all_dma - start all RX and TX DMA channels 2324 * @priv: driver private structure 2325 * Description: 2326 * This starts all the RX and TX DMA channels 2327 */ 2328 static void stmmac_start_all_dma(struct stmmac_priv *priv) 2329 { 2330 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2331 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2332 u32 chan = 0; 2333 2334 for (chan = 0; chan < rx_channels_count; chan++) 2335 stmmac_start_rx_dma(priv, chan); 2336 2337 for (chan = 0; chan < tx_channels_count; chan++) 2338 stmmac_start_tx_dma(priv, chan); 2339 } 2340 2341 /** 2342 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2343 * @priv: driver private structure 2344 * Description: 2345 * This stops the RX and TX DMA channels 2346 */ 2347 static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2348 { 2349 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2350 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2351 u32 chan = 0; 2352 2353 for (chan = 0; chan < rx_channels_count; chan++) 2354 stmmac_stop_rx_dma(priv, chan); 2355 2356 for (chan = 0; chan < tx_channels_count; chan++) 2357 stmmac_stop_tx_dma(priv, chan); 2358 } 2359 2360 /** 2361 * stmmac_dma_operation_mode - HW DMA operation mode 2362 * @priv: driver private structure 2363 * Description: it is used for configuring the DMA operation mode register in 2364 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2365 */ 2366 static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2367 { 2368 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2369 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2370 int rxfifosz = priv->plat->rx_fifo_size; 2371 int txfifosz = priv->plat->tx_fifo_size; 2372 u32 txmode = 0; 2373 u32 rxmode = 0; 2374 u32 chan = 0; 2375 u8 qmode = 0; 2376 2377 if (rxfifosz == 0) 2378 rxfifosz = priv->dma_cap.rx_fifo_size; 2379 if (txfifosz == 0) 2380 txfifosz = priv->dma_cap.tx_fifo_size; 2381 2382 /* Adjust for real per queue fifo size */ 2383 rxfifosz /= rx_channels_count; 2384 txfifosz /= tx_channels_count; 2385 2386 if (priv->plat->force_thresh_dma_mode) { 2387 txmode = tc; 2388 rxmode = tc; 2389 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2390 /* 2391 * In case of GMAC, SF mode can be enabled 2392 * to perform the TX COE in HW. This depends on: 2393 * 1) TX COE if actually supported 2394 * 2) There is no bugged Jumbo frame support 2395 * that needs to not insert csum in the TDES. 2396 */ 2397 txmode = SF_DMA_MODE; 2398 rxmode = SF_DMA_MODE; 2399 priv->xstats.threshold = SF_DMA_MODE; 2400 } else { 2401 txmode = tc; 2402 rxmode = SF_DMA_MODE; 2403 } 2404 2405 /* configure all channels */ 2406 for (chan = 0; chan < rx_channels_count; chan++) { 2407 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2408 u32 buf_size; 2409 2410 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2411 2412 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2413 rxfifosz, qmode); 2414 2415 if (rx_q->xsk_pool) { 2416 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 2417 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2418 buf_size, 2419 chan); 2420 } else { 2421 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2422 priv->dma_conf.dma_buf_sz, 2423 chan); 2424 } 2425 } 2426 2427 for (chan = 0; chan < tx_channels_count; chan++) { 2428 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2429 2430 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2431 txfifosz, qmode); 2432 } 2433 } 2434 2435 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2436 { 2437 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2438 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2439 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2440 unsigned int entry = tx_q->cur_tx; 2441 struct dma_desc *tx_desc = NULL; 2442 struct xdp_desc xdp_desc; 2443 bool work_done = true; 2444 u32 tx_set_ic_bit = 0; 2445 unsigned long flags; 2446 2447 /* Avoids TX time-out as we are sharing with slow path */ 2448 txq_trans_cond_update(nq); 2449 2450 budget = min(budget, stmmac_tx_avail(priv, queue)); 2451 2452 while (budget-- > 0) { 2453 dma_addr_t dma_addr; 2454 bool set_ic; 2455 2456 /* We are sharing with slow path and stop XSK TX desc submission when 2457 * available TX ring is less than threshold. 2458 */ 2459 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2460 !netif_carrier_ok(priv->dev)) { 2461 work_done = false; 2462 break; 2463 } 2464 2465 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2466 break; 2467 2468 if (likely(priv->extend_desc)) 2469 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2470 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2471 tx_desc = &tx_q->dma_entx[entry].basic; 2472 else 2473 tx_desc = tx_q->dma_tx + entry; 2474 2475 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2476 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2477 2478 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; 2479 2480 /* To return XDP buffer to XSK pool, we simple call 2481 * xsk_tx_completed(), so we don't need to fill up 2482 * 'buf' and 'xdpf'. 2483 */ 2484 tx_q->tx_skbuff_dma[entry].buf = 0; 2485 tx_q->xdpf[entry] = NULL; 2486 2487 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2488 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; 2489 tx_q->tx_skbuff_dma[entry].last_segment = true; 2490 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2491 2492 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2493 2494 tx_q->tx_count_frames++; 2495 2496 if (!priv->tx_coal_frames[queue]) 2497 set_ic = false; 2498 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2499 set_ic = true; 2500 else 2501 set_ic = false; 2502 2503 if (set_ic) { 2504 tx_q->tx_count_frames = 0; 2505 stmmac_set_tx_ic(priv, tx_desc); 2506 tx_set_ic_bit++; 2507 } 2508 2509 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2510 true, priv->mode, true, true, 2511 xdp_desc.len); 2512 2513 stmmac_enable_dma_transmission(priv, priv->ioaddr); 2514 2515 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2516 entry = tx_q->cur_tx; 2517 } 2518 flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); 2519 tx_q->txq_stats.tx_set_ic_bit += tx_set_ic_bit; 2520 u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); 2521 2522 if (tx_desc) { 2523 stmmac_flush_tx_descriptors(priv, queue); 2524 xsk_tx_release(pool); 2525 } 2526 2527 /* Return true if all of the 3 conditions are met 2528 * a) TX Budget is still available 2529 * b) work_done = true when XSK TX desc peek is empty (no more 2530 * pending XSK TX for transmission) 2531 */ 2532 return !!budget && work_done; 2533 } 2534 2535 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2536 { 2537 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2538 tc += 64; 2539 2540 if (priv->plat->force_thresh_dma_mode) 2541 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2542 else 2543 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2544 chan); 2545 2546 priv->xstats.threshold = tc; 2547 } 2548 } 2549 2550 /** 2551 * stmmac_tx_clean - to manage the transmission completion 2552 * @priv: driver private structure 2553 * @budget: napi budget limiting this functions packet handling 2554 * @queue: TX queue index 2555 * Description: it reclaims the transmit resources after transmission completes. 2556 */ 2557 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) 2558 { 2559 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2560 unsigned int bytes_compl = 0, pkts_compl = 0; 2561 unsigned int entry, xmits = 0, count = 0; 2562 u32 tx_packets = 0, tx_errors = 0; 2563 unsigned long flags; 2564 2565 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2566 2567 tx_q->xsk_frames_done = 0; 2568 2569 entry = tx_q->dirty_tx; 2570 2571 /* Try to clean all TX complete frame in 1 shot */ 2572 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2573 struct xdp_frame *xdpf; 2574 struct sk_buff *skb; 2575 struct dma_desc *p; 2576 int status; 2577 2578 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2579 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2580 xdpf = tx_q->xdpf[entry]; 2581 skb = NULL; 2582 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2583 xdpf = NULL; 2584 skb = tx_q->tx_skbuff[entry]; 2585 } else { 2586 xdpf = NULL; 2587 skb = NULL; 2588 } 2589 2590 if (priv->extend_desc) 2591 p = (struct dma_desc *)(tx_q->dma_etx + entry); 2592 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2593 p = &tx_q->dma_entx[entry].basic; 2594 else 2595 p = tx_q->dma_tx + entry; 2596 2597 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2598 /* Check if the descriptor is owned by the DMA */ 2599 if (unlikely(status & tx_dma_own)) 2600 break; 2601 2602 count++; 2603 2604 /* Make sure descriptor fields are read after reading 2605 * the own bit. 2606 */ 2607 dma_rmb(); 2608 2609 /* Just consider the last segment and ...*/ 2610 if (likely(!(status & tx_not_ls))) { 2611 /* ... verify the status error condition */ 2612 if (unlikely(status & tx_err)) { 2613 tx_errors++; 2614 if (unlikely(status & tx_err_bump_tc)) 2615 stmmac_bump_dma_threshold(priv, queue); 2616 } else { 2617 tx_packets++; 2618 } 2619 if (skb) 2620 stmmac_get_tx_hwtstamp(priv, p, skb); 2621 } 2622 2623 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2624 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2625 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2626 dma_unmap_page(priv->device, 2627 tx_q->tx_skbuff_dma[entry].buf, 2628 tx_q->tx_skbuff_dma[entry].len, 2629 DMA_TO_DEVICE); 2630 else 2631 dma_unmap_single(priv->device, 2632 tx_q->tx_skbuff_dma[entry].buf, 2633 tx_q->tx_skbuff_dma[entry].len, 2634 DMA_TO_DEVICE); 2635 tx_q->tx_skbuff_dma[entry].buf = 0; 2636 tx_q->tx_skbuff_dma[entry].len = 0; 2637 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2638 } 2639 2640 stmmac_clean_desc3(priv, tx_q, p); 2641 2642 tx_q->tx_skbuff_dma[entry].last_segment = false; 2643 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2644 2645 if (xdpf && 2646 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2647 xdp_return_frame_rx_napi(xdpf); 2648 tx_q->xdpf[entry] = NULL; 2649 } 2650 2651 if (xdpf && 2652 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2653 xdp_return_frame(xdpf); 2654 tx_q->xdpf[entry] = NULL; 2655 } 2656 2657 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2658 tx_q->xsk_frames_done++; 2659 2660 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2661 if (likely(skb)) { 2662 pkts_compl++; 2663 bytes_compl += skb->len; 2664 dev_consume_skb_any(skb); 2665 tx_q->tx_skbuff[entry] = NULL; 2666 } 2667 } 2668 2669 stmmac_release_tx_desc(priv, p, priv->mode); 2670 2671 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 2672 } 2673 tx_q->dirty_tx = entry; 2674 2675 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2676 pkts_compl, bytes_compl); 2677 2678 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2679 queue))) && 2680 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2681 2682 netif_dbg(priv, tx_done, priv->dev, 2683 "%s: restart transmit\n", __func__); 2684 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2685 } 2686 2687 if (tx_q->xsk_pool) { 2688 bool work_done; 2689 2690 if (tx_q->xsk_frames_done) 2691 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2692 2693 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2694 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2695 2696 /* For XSK TX, we try to send as many as possible. 2697 * If XSK work done (XSK TX desc empty and budget still 2698 * available), return "budget - 1" to reenable TX IRQ. 2699 * Else, return "budget" to make NAPI continue polling. 2700 */ 2701 work_done = stmmac_xdp_xmit_zc(priv, queue, 2702 STMMAC_XSK_TX_BUDGET_MAX); 2703 if (work_done) 2704 xmits = budget - 1; 2705 else 2706 xmits = budget; 2707 } 2708 2709 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && 2710 priv->eee_sw_timer_en) { 2711 if (stmmac_enable_eee_mode(priv)) 2712 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 2713 } 2714 2715 /* We still have pending packets, let's call for a new scheduling */ 2716 if (tx_q->dirty_tx != tx_q->cur_tx) 2717 hrtimer_start(&tx_q->txtimer, 2718 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), 2719 HRTIMER_MODE_REL); 2720 2721 flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); 2722 tx_q->txq_stats.tx_packets += tx_packets; 2723 tx_q->txq_stats.tx_pkt_n += tx_packets; 2724 tx_q->txq_stats.tx_clean++; 2725 u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); 2726 2727 priv->xstats.tx_errors += tx_errors; 2728 2729 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2730 2731 /* Combine decisions from TX clean and XSK TX */ 2732 return max(count, xmits); 2733 } 2734 2735 /** 2736 * stmmac_tx_err - to manage the tx error 2737 * @priv: driver private structure 2738 * @chan: channel index 2739 * Description: it cleans the descriptors and restarts the transmission 2740 * in case of transmission errors. 2741 */ 2742 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 2743 { 2744 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2745 2746 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 2747 2748 stmmac_stop_tx_dma(priv, chan); 2749 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 2750 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 2751 stmmac_reset_tx_queue(priv, chan); 2752 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2753 tx_q->dma_tx_phy, chan); 2754 stmmac_start_tx_dma(priv, chan); 2755 2756 priv->xstats.tx_errors++; 2757 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2758 } 2759 2760 /** 2761 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2762 * @priv: driver private structure 2763 * @txmode: TX operating mode 2764 * @rxmode: RX operating mode 2765 * @chan: channel index 2766 * Description: it is used for configuring of the DMA operation mode in 2767 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2768 * mode. 2769 */ 2770 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2771 u32 rxmode, u32 chan) 2772 { 2773 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2774 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2775 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2776 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2777 int rxfifosz = priv->plat->rx_fifo_size; 2778 int txfifosz = priv->plat->tx_fifo_size; 2779 2780 if (rxfifosz == 0) 2781 rxfifosz = priv->dma_cap.rx_fifo_size; 2782 if (txfifosz == 0) 2783 txfifosz = priv->dma_cap.tx_fifo_size; 2784 2785 /* Adjust for real per queue fifo size */ 2786 rxfifosz /= rx_channels_count; 2787 txfifosz /= tx_channels_count; 2788 2789 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2790 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2791 } 2792 2793 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2794 { 2795 int ret; 2796 2797 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2798 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2799 if (ret && (ret != -EINVAL)) { 2800 stmmac_global_err(priv); 2801 return true; 2802 } 2803 2804 return false; 2805 } 2806 2807 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 2808 { 2809 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 2810 &priv->xstats, chan, dir); 2811 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2812 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2813 struct stmmac_channel *ch = &priv->channel[chan]; 2814 struct napi_struct *rx_napi; 2815 struct napi_struct *tx_napi; 2816 unsigned long flags; 2817 2818 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 2819 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 2820 2821 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 2822 if (napi_schedule_prep(rx_napi)) { 2823 spin_lock_irqsave(&ch->lock, flags); 2824 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 2825 spin_unlock_irqrestore(&ch->lock, flags); 2826 __napi_schedule(rx_napi); 2827 } 2828 } 2829 2830 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 2831 if (napi_schedule_prep(tx_napi)) { 2832 spin_lock_irqsave(&ch->lock, flags); 2833 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 2834 spin_unlock_irqrestore(&ch->lock, flags); 2835 __napi_schedule(tx_napi); 2836 } 2837 } 2838 2839 return status; 2840 } 2841 2842 /** 2843 * stmmac_dma_interrupt - DMA ISR 2844 * @priv: driver private structure 2845 * Description: this is the DMA ISR. It is called by the main ISR. 2846 * It calls the dwmac dma routine and schedule poll method in case of some 2847 * work can be done. 2848 */ 2849 static void stmmac_dma_interrupt(struct stmmac_priv *priv) 2850 { 2851 u32 tx_channel_count = priv->plat->tx_queues_to_use; 2852 u32 rx_channel_count = priv->plat->rx_queues_to_use; 2853 u32 channels_to_check = tx_channel_count > rx_channel_count ? 2854 tx_channel_count : rx_channel_count; 2855 u32 chan; 2856 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 2857 2858 /* Make sure we never check beyond our status buffer. */ 2859 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 2860 channels_to_check = ARRAY_SIZE(status); 2861 2862 for (chan = 0; chan < channels_to_check; chan++) 2863 status[chan] = stmmac_napi_check(priv, chan, 2864 DMA_DIR_RXTX); 2865 2866 for (chan = 0; chan < tx_channel_count; chan++) { 2867 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 2868 /* Try to bump up the dma threshold on this failure */ 2869 stmmac_bump_dma_threshold(priv, chan); 2870 } else if (unlikely(status[chan] == tx_hard_error)) { 2871 stmmac_tx_err(priv, chan); 2872 } 2873 } 2874 } 2875 2876 /** 2877 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 2878 * @priv: driver private structure 2879 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 2880 */ 2881 static void stmmac_mmc_setup(struct stmmac_priv *priv) 2882 { 2883 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 2884 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 2885 2886 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 2887 2888 if (priv->dma_cap.rmon) { 2889 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 2890 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 2891 } else 2892 netdev_info(priv->dev, "No MAC Management Counters available\n"); 2893 } 2894 2895 /** 2896 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 2897 * @priv: driver private structure 2898 * Description: 2899 * new GMAC chip generations have a new register to indicate the 2900 * presence of the optional feature/functions. 2901 * This can be also used to override the value passed through the 2902 * platform and necessary for old MAC10/100 and GMAC chips. 2903 */ 2904 static int stmmac_get_hw_features(struct stmmac_priv *priv) 2905 { 2906 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 2907 } 2908 2909 /** 2910 * stmmac_check_ether_addr - check if the MAC addr is valid 2911 * @priv: driver private structure 2912 * Description: 2913 * it is to verify if the MAC address is valid, in case of failures it 2914 * generates a random MAC address 2915 */ 2916 static void stmmac_check_ether_addr(struct stmmac_priv *priv) 2917 { 2918 u8 addr[ETH_ALEN]; 2919 2920 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 2921 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 2922 if (is_valid_ether_addr(addr)) 2923 eth_hw_addr_set(priv->dev, addr); 2924 else 2925 eth_hw_addr_random(priv->dev); 2926 dev_info(priv->device, "device MAC address %pM\n", 2927 priv->dev->dev_addr); 2928 } 2929 } 2930 2931 /** 2932 * stmmac_init_dma_engine - DMA init. 2933 * @priv: driver private structure 2934 * Description: 2935 * It inits the DMA invoking the specific MAC/GMAC callback. 2936 * Some DMA parameters can be passed from the platform; 2937 * in case of these are not passed a default is kept for the MAC or GMAC. 2938 */ 2939 static int stmmac_init_dma_engine(struct stmmac_priv *priv) 2940 { 2941 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2942 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2943 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2944 struct stmmac_rx_queue *rx_q; 2945 struct stmmac_tx_queue *tx_q; 2946 u32 chan = 0; 2947 int atds = 0; 2948 int ret = 0; 2949 2950 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 2951 dev_err(priv->device, "Invalid DMA configuration\n"); 2952 return -EINVAL; 2953 } 2954 2955 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 2956 atds = 1; 2957 2958 ret = stmmac_reset(priv, priv->ioaddr); 2959 if (ret) { 2960 dev_err(priv->device, "Failed to reset the dma\n"); 2961 return ret; 2962 } 2963 2964 /* DMA Configuration */ 2965 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); 2966 2967 if (priv->plat->axi) 2968 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 2969 2970 /* DMA CSR Channel configuration */ 2971 for (chan = 0; chan < dma_csr_ch; chan++) { 2972 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 2973 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2974 } 2975 2976 /* DMA RX Channel Configuration */ 2977 for (chan = 0; chan < rx_channels_count; chan++) { 2978 rx_q = &priv->dma_conf.rx_queue[chan]; 2979 2980 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2981 rx_q->dma_rx_phy, chan); 2982 2983 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 2984 (rx_q->buf_alloc_num * 2985 sizeof(struct dma_desc)); 2986 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 2987 rx_q->rx_tail_addr, chan); 2988 } 2989 2990 /* DMA TX Channel Configuration */ 2991 for (chan = 0; chan < tx_channels_count; chan++) { 2992 tx_q = &priv->dma_conf.tx_queue[chan]; 2993 2994 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2995 tx_q->dma_tx_phy, chan); 2996 2997 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 2998 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 2999 tx_q->tx_tail_addr, chan); 3000 } 3001 3002 return ret; 3003 } 3004 3005 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3006 { 3007 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3008 3009 hrtimer_start(&tx_q->txtimer, 3010 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), 3011 HRTIMER_MODE_REL); 3012 } 3013 3014 /** 3015 * stmmac_tx_timer - mitigation sw timer for tx. 3016 * @t: data pointer 3017 * Description: 3018 * This is the timer handler to directly invoke the stmmac_tx_clean. 3019 */ 3020 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3021 { 3022 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3023 struct stmmac_priv *priv = tx_q->priv_data; 3024 struct stmmac_channel *ch; 3025 struct napi_struct *napi; 3026 3027 ch = &priv->channel[tx_q->queue_index]; 3028 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3029 3030 if (likely(napi_schedule_prep(napi))) { 3031 unsigned long flags; 3032 3033 spin_lock_irqsave(&ch->lock, flags); 3034 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3035 spin_unlock_irqrestore(&ch->lock, flags); 3036 __napi_schedule(napi); 3037 } 3038 3039 return HRTIMER_NORESTART; 3040 } 3041 3042 /** 3043 * stmmac_init_coalesce - init mitigation options. 3044 * @priv: driver private structure 3045 * Description: 3046 * This inits the coalesce parameters: i.e. timer rate, 3047 * timer handler and default threshold used for enabling the 3048 * interrupt on completion bit. 3049 */ 3050 static void stmmac_init_coalesce(struct stmmac_priv *priv) 3051 { 3052 u32 tx_channel_count = priv->plat->tx_queues_to_use; 3053 u32 rx_channel_count = priv->plat->rx_queues_to_use; 3054 u32 chan; 3055 3056 for (chan = 0; chan < tx_channel_count; chan++) { 3057 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3058 3059 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3060 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3061 3062 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3063 tx_q->txtimer.function = stmmac_tx_timer; 3064 } 3065 3066 for (chan = 0; chan < rx_channel_count; chan++) 3067 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3068 } 3069 3070 static void stmmac_set_rings_length(struct stmmac_priv *priv) 3071 { 3072 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3073 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3074 u32 chan; 3075 3076 /* set TX ring length */ 3077 for (chan = 0; chan < tx_channels_count; chan++) 3078 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3079 (priv->dma_conf.dma_tx_size - 1), chan); 3080 3081 /* set RX ring length */ 3082 for (chan = 0; chan < rx_channels_count; chan++) 3083 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3084 (priv->dma_conf.dma_rx_size - 1), chan); 3085 } 3086 3087 /** 3088 * stmmac_set_tx_queue_weight - Set TX queue weight 3089 * @priv: driver private structure 3090 * Description: It is used for setting TX queues weight 3091 */ 3092 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3093 { 3094 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3095 u32 weight; 3096 u32 queue; 3097 3098 for (queue = 0; queue < tx_queues_count; queue++) { 3099 weight = priv->plat->tx_queues_cfg[queue].weight; 3100 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3101 } 3102 } 3103 3104 /** 3105 * stmmac_configure_cbs - Configure CBS in TX queue 3106 * @priv: driver private structure 3107 * Description: It is used for configuring CBS in AVB TX queues 3108 */ 3109 static void stmmac_configure_cbs(struct stmmac_priv *priv) 3110 { 3111 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3112 u32 mode_to_use; 3113 u32 queue; 3114 3115 /* queue 0 is reserved for legacy traffic */ 3116 for (queue = 1; queue < tx_queues_count; queue++) { 3117 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3118 if (mode_to_use == MTL_QUEUE_DCB) 3119 continue; 3120 3121 stmmac_config_cbs(priv, priv->hw, 3122 priv->plat->tx_queues_cfg[queue].send_slope, 3123 priv->plat->tx_queues_cfg[queue].idle_slope, 3124 priv->plat->tx_queues_cfg[queue].high_credit, 3125 priv->plat->tx_queues_cfg[queue].low_credit, 3126 queue); 3127 } 3128 } 3129 3130 /** 3131 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3132 * @priv: driver private structure 3133 * Description: It is used for mapping RX queues to RX dma channels 3134 */ 3135 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3136 { 3137 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3138 u32 queue; 3139 u32 chan; 3140 3141 for (queue = 0; queue < rx_queues_count; queue++) { 3142 chan = priv->plat->rx_queues_cfg[queue].chan; 3143 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3144 } 3145 } 3146 3147 /** 3148 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3149 * @priv: driver private structure 3150 * Description: It is used for configuring the RX Queue Priority 3151 */ 3152 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3153 { 3154 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3155 u32 queue; 3156 u32 prio; 3157 3158 for (queue = 0; queue < rx_queues_count; queue++) { 3159 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3160 continue; 3161 3162 prio = priv->plat->rx_queues_cfg[queue].prio; 3163 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3164 } 3165 } 3166 3167 /** 3168 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3169 * @priv: driver private structure 3170 * Description: It is used for configuring the TX Queue Priority 3171 */ 3172 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3173 { 3174 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3175 u32 queue; 3176 u32 prio; 3177 3178 for (queue = 0; queue < tx_queues_count; queue++) { 3179 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3180 continue; 3181 3182 prio = priv->plat->tx_queues_cfg[queue].prio; 3183 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3184 } 3185 } 3186 3187 /** 3188 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3189 * @priv: driver private structure 3190 * Description: It is used for configuring the RX queue routing 3191 */ 3192 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3193 { 3194 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3195 u32 queue; 3196 u8 packet; 3197 3198 for (queue = 0; queue < rx_queues_count; queue++) { 3199 /* no specific packet type routing specified for the queue */ 3200 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3201 continue; 3202 3203 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3204 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3205 } 3206 } 3207 3208 static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3209 { 3210 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3211 priv->rss.enable = false; 3212 return; 3213 } 3214 3215 if (priv->dev->features & NETIF_F_RXHASH) 3216 priv->rss.enable = true; 3217 else 3218 priv->rss.enable = false; 3219 3220 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3221 priv->plat->rx_queues_to_use); 3222 } 3223 3224 /** 3225 * stmmac_mtl_configuration - Configure MTL 3226 * @priv: driver private structure 3227 * Description: It is used for configurring MTL 3228 */ 3229 static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3230 { 3231 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3232 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3233 3234 if (tx_queues_count > 1) 3235 stmmac_set_tx_queue_weight(priv); 3236 3237 /* Configure MTL RX algorithms */ 3238 if (rx_queues_count > 1) 3239 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3240 priv->plat->rx_sched_algorithm); 3241 3242 /* Configure MTL TX algorithms */ 3243 if (tx_queues_count > 1) 3244 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3245 priv->plat->tx_sched_algorithm); 3246 3247 /* Configure CBS in AVB TX queues */ 3248 if (tx_queues_count > 1) 3249 stmmac_configure_cbs(priv); 3250 3251 /* Map RX MTL to DMA channels */ 3252 stmmac_rx_queue_dma_chan_map(priv); 3253 3254 /* Enable MAC RX Queues */ 3255 stmmac_mac_enable_rx_queues(priv); 3256 3257 /* Set RX priorities */ 3258 if (rx_queues_count > 1) 3259 stmmac_mac_config_rx_queues_prio(priv); 3260 3261 /* Set TX priorities */ 3262 if (tx_queues_count > 1) 3263 stmmac_mac_config_tx_queues_prio(priv); 3264 3265 /* Set RX routing */ 3266 if (rx_queues_count > 1) 3267 stmmac_mac_config_rx_queues_routing(priv); 3268 3269 /* Receive Side Scaling */ 3270 if (rx_queues_count > 1) 3271 stmmac_mac_config_rss(priv); 3272 } 3273 3274 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3275 { 3276 if (priv->dma_cap.asp) { 3277 netdev_info(priv->dev, "Enabling Safety Features\n"); 3278 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3279 priv->plat->safety_feat_cfg); 3280 } else { 3281 netdev_info(priv->dev, "No Safety Features support found\n"); 3282 } 3283 } 3284 3285 static int stmmac_fpe_start_wq(struct stmmac_priv *priv) 3286 { 3287 char *name; 3288 3289 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 3290 clear_bit(__FPE_REMOVING, &priv->fpe_task_state); 3291 3292 name = priv->wq_name; 3293 sprintf(name, "%s-fpe", priv->dev->name); 3294 3295 priv->fpe_wq = create_singlethread_workqueue(name); 3296 if (!priv->fpe_wq) { 3297 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); 3298 3299 return -ENOMEM; 3300 } 3301 netdev_info(priv->dev, "FPE workqueue start"); 3302 3303 return 0; 3304 } 3305 3306 /** 3307 * stmmac_hw_setup - setup mac in a usable state. 3308 * @dev : pointer to the device structure. 3309 * @ptp_register: register PTP if set 3310 * Description: 3311 * this is the main function to setup the HW in a usable state because the 3312 * dma engine is reset, the core registers are configured (e.g. AXI, 3313 * Checksum features, timers). The DMA is ready to start receiving and 3314 * transmitting. 3315 * Return value: 3316 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3317 * file on failure. 3318 */ 3319 static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) 3320 { 3321 struct stmmac_priv *priv = netdev_priv(dev); 3322 u32 rx_cnt = priv->plat->rx_queues_to_use; 3323 u32 tx_cnt = priv->plat->tx_queues_to_use; 3324 bool sph_en; 3325 u32 chan; 3326 int ret; 3327 3328 /* DMA initialization and SW reset */ 3329 ret = stmmac_init_dma_engine(priv); 3330 if (ret < 0) { 3331 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3332 __func__); 3333 return ret; 3334 } 3335 3336 /* Copy the MAC addr into the HW */ 3337 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3338 3339 /* PS and related bits will be programmed according to the speed */ 3340 if (priv->hw->pcs) { 3341 int speed = priv->plat->mac_port_sel_speed; 3342 3343 if ((speed == SPEED_10) || (speed == SPEED_100) || 3344 (speed == SPEED_1000)) { 3345 priv->hw->ps = speed; 3346 } else { 3347 dev_warn(priv->device, "invalid port speed\n"); 3348 priv->hw->ps = 0; 3349 } 3350 } 3351 3352 /* Initialize the MAC Core */ 3353 stmmac_core_init(priv, priv->hw, dev); 3354 3355 /* Initialize MTL*/ 3356 stmmac_mtl_configuration(priv); 3357 3358 /* Initialize Safety Features */ 3359 stmmac_safety_feat_configuration(priv); 3360 3361 ret = stmmac_rx_ipc(priv, priv->hw); 3362 if (!ret) { 3363 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3364 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3365 priv->hw->rx_csum = 0; 3366 } 3367 3368 /* Enable the MAC Rx/Tx */ 3369 stmmac_mac_set(priv, priv->ioaddr, true); 3370 3371 /* Set the HW DMA mode and the COE */ 3372 stmmac_dma_operation_mode(priv); 3373 3374 stmmac_mmc_setup(priv); 3375 3376 if (ptp_register) { 3377 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 3378 if (ret < 0) 3379 netdev_warn(priv->dev, 3380 "failed to enable PTP reference clock: %pe\n", 3381 ERR_PTR(ret)); 3382 } 3383 3384 ret = stmmac_init_ptp(priv); 3385 if (ret == -EOPNOTSUPP) 3386 netdev_info(priv->dev, "PTP not supported by HW\n"); 3387 else if (ret) 3388 netdev_warn(priv->dev, "PTP init failed\n"); 3389 else if (ptp_register) 3390 stmmac_ptp_register(priv); 3391 3392 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; 3393 3394 /* Convert the timer from msec to usec */ 3395 if (!priv->tx_lpi_timer) 3396 priv->tx_lpi_timer = eee_timer * 1000; 3397 3398 if (priv->use_riwt) { 3399 u32 queue; 3400 3401 for (queue = 0; queue < rx_cnt; queue++) { 3402 if (!priv->rx_riwt[queue]) 3403 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3404 3405 stmmac_rx_watchdog(priv, priv->ioaddr, 3406 priv->rx_riwt[queue], queue); 3407 } 3408 } 3409 3410 if (priv->hw->pcs) 3411 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); 3412 3413 /* set TX and RX rings length */ 3414 stmmac_set_rings_length(priv); 3415 3416 /* Enable TSO */ 3417 if (priv->tso) { 3418 for (chan = 0; chan < tx_cnt; chan++) { 3419 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3420 3421 /* TSO and TBS cannot co-exist */ 3422 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3423 continue; 3424 3425 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3426 } 3427 } 3428 3429 /* Enable Split Header */ 3430 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 3431 for (chan = 0; chan < rx_cnt; chan++) 3432 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3433 3434 3435 /* VLAN Tag Insertion */ 3436 if (priv->dma_cap.vlins) 3437 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3438 3439 /* TBS */ 3440 for (chan = 0; chan < tx_cnt; chan++) { 3441 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3442 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3443 3444 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3445 } 3446 3447 /* Configure real RX and TX queues */ 3448 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3449 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3450 3451 /* Start the ball rolling... */ 3452 stmmac_start_all_dma(priv); 3453 3454 if (priv->dma_cap.fpesel) { 3455 stmmac_fpe_start_wq(priv); 3456 3457 if (priv->plat->fpe_cfg->enable) 3458 stmmac_fpe_handshake(priv, true); 3459 } 3460 3461 return 0; 3462 } 3463 3464 static void stmmac_hw_teardown(struct net_device *dev) 3465 { 3466 struct stmmac_priv *priv = netdev_priv(dev); 3467 3468 clk_disable_unprepare(priv->plat->clk_ptp_ref); 3469 } 3470 3471 static void stmmac_free_irq(struct net_device *dev, 3472 enum request_irq_err irq_err, int irq_idx) 3473 { 3474 struct stmmac_priv *priv = netdev_priv(dev); 3475 int j; 3476 3477 switch (irq_err) { 3478 case REQ_IRQ_ERR_ALL: 3479 irq_idx = priv->plat->tx_queues_to_use; 3480 fallthrough; 3481 case REQ_IRQ_ERR_TX: 3482 for (j = irq_idx - 1; j >= 0; j--) { 3483 if (priv->tx_irq[j] > 0) { 3484 irq_set_affinity_hint(priv->tx_irq[j], NULL); 3485 free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); 3486 } 3487 } 3488 irq_idx = priv->plat->rx_queues_to_use; 3489 fallthrough; 3490 case REQ_IRQ_ERR_RX: 3491 for (j = irq_idx - 1; j >= 0; j--) { 3492 if (priv->rx_irq[j] > 0) { 3493 irq_set_affinity_hint(priv->rx_irq[j], NULL); 3494 free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); 3495 } 3496 } 3497 3498 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) 3499 free_irq(priv->sfty_ue_irq, dev); 3500 fallthrough; 3501 case REQ_IRQ_ERR_SFTY_UE: 3502 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) 3503 free_irq(priv->sfty_ce_irq, dev); 3504 fallthrough; 3505 case REQ_IRQ_ERR_SFTY_CE: 3506 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) 3507 free_irq(priv->lpi_irq, dev); 3508 fallthrough; 3509 case REQ_IRQ_ERR_LPI: 3510 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3511 free_irq(priv->wol_irq, dev); 3512 fallthrough; 3513 case REQ_IRQ_ERR_WOL: 3514 free_irq(dev->irq, dev); 3515 fallthrough; 3516 case REQ_IRQ_ERR_MAC: 3517 case REQ_IRQ_ERR_NO: 3518 /* If MAC IRQ request error, no more IRQ to free */ 3519 break; 3520 } 3521 } 3522 3523 static int stmmac_request_irq_multi_msi(struct net_device *dev) 3524 { 3525 struct stmmac_priv *priv = netdev_priv(dev); 3526 enum request_irq_err irq_err; 3527 cpumask_t cpu_mask; 3528 int irq_idx = 0; 3529 char *int_name; 3530 int ret; 3531 int i; 3532 3533 /* For common interrupt */ 3534 int_name = priv->int_name_mac; 3535 sprintf(int_name, "%s:%s", dev->name, "mac"); 3536 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3537 0, int_name, dev); 3538 if (unlikely(ret < 0)) { 3539 netdev_err(priv->dev, 3540 "%s: alloc mac MSI %d (error: %d)\n", 3541 __func__, dev->irq, ret); 3542 irq_err = REQ_IRQ_ERR_MAC; 3543 goto irq_error; 3544 } 3545 3546 /* Request the Wake IRQ in case of another line 3547 * is used for WoL 3548 */ 3549 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3550 int_name = priv->int_name_wol; 3551 sprintf(int_name, "%s:%s", dev->name, "wol"); 3552 ret = request_irq(priv->wol_irq, 3553 stmmac_mac_interrupt, 3554 0, int_name, dev); 3555 if (unlikely(ret < 0)) { 3556 netdev_err(priv->dev, 3557 "%s: alloc wol MSI %d (error: %d)\n", 3558 __func__, priv->wol_irq, ret); 3559 irq_err = REQ_IRQ_ERR_WOL; 3560 goto irq_error; 3561 } 3562 } 3563 3564 /* Request the LPI IRQ in case of another line 3565 * is used for LPI 3566 */ 3567 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3568 int_name = priv->int_name_lpi; 3569 sprintf(int_name, "%s:%s", dev->name, "lpi"); 3570 ret = request_irq(priv->lpi_irq, 3571 stmmac_mac_interrupt, 3572 0, int_name, dev); 3573 if (unlikely(ret < 0)) { 3574 netdev_err(priv->dev, 3575 "%s: alloc lpi MSI %d (error: %d)\n", 3576 __func__, priv->lpi_irq, ret); 3577 irq_err = REQ_IRQ_ERR_LPI; 3578 goto irq_error; 3579 } 3580 } 3581 3582 /* Request the Safety Feature Correctible Error line in 3583 * case of another line is used 3584 */ 3585 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { 3586 int_name = priv->int_name_sfty_ce; 3587 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3588 ret = request_irq(priv->sfty_ce_irq, 3589 stmmac_safety_interrupt, 3590 0, int_name, dev); 3591 if (unlikely(ret < 0)) { 3592 netdev_err(priv->dev, 3593 "%s: alloc sfty ce MSI %d (error: %d)\n", 3594 __func__, priv->sfty_ce_irq, ret); 3595 irq_err = REQ_IRQ_ERR_SFTY_CE; 3596 goto irq_error; 3597 } 3598 } 3599 3600 /* Request the Safety Feature Uncorrectible Error line in 3601 * case of another line is used 3602 */ 3603 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { 3604 int_name = priv->int_name_sfty_ue; 3605 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3606 ret = request_irq(priv->sfty_ue_irq, 3607 stmmac_safety_interrupt, 3608 0, int_name, dev); 3609 if (unlikely(ret < 0)) { 3610 netdev_err(priv->dev, 3611 "%s: alloc sfty ue MSI %d (error: %d)\n", 3612 __func__, priv->sfty_ue_irq, ret); 3613 irq_err = REQ_IRQ_ERR_SFTY_UE; 3614 goto irq_error; 3615 } 3616 } 3617 3618 /* Request Rx MSI irq */ 3619 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3620 if (i >= MTL_MAX_RX_QUEUES) 3621 break; 3622 if (priv->rx_irq[i] == 0) 3623 continue; 3624 3625 int_name = priv->int_name_rx_irq[i]; 3626 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3627 ret = request_irq(priv->rx_irq[i], 3628 stmmac_msi_intr_rx, 3629 0, int_name, &priv->dma_conf.rx_queue[i]); 3630 if (unlikely(ret < 0)) { 3631 netdev_err(priv->dev, 3632 "%s: alloc rx-%d MSI %d (error: %d)\n", 3633 __func__, i, priv->rx_irq[i], ret); 3634 irq_err = REQ_IRQ_ERR_RX; 3635 irq_idx = i; 3636 goto irq_error; 3637 } 3638 cpumask_clear(&cpu_mask); 3639 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3640 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); 3641 } 3642 3643 /* Request Tx MSI irq */ 3644 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3645 if (i >= MTL_MAX_TX_QUEUES) 3646 break; 3647 if (priv->tx_irq[i] == 0) 3648 continue; 3649 3650 int_name = priv->int_name_tx_irq[i]; 3651 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3652 ret = request_irq(priv->tx_irq[i], 3653 stmmac_msi_intr_tx, 3654 0, int_name, &priv->dma_conf.tx_queue[i]); 3655 if (unlikely(ret < 0)) { 3656 netdev_err(priv->dev, 3657 "%s: alloc tx-%d MSI %d (error: %d)\n", 3658 __func__, i, priv->tx_irq[i], ret); 3659 irq_err = REQ_IRQ_ERR_TX; 3660 irq_idx = i; 3661 goto irq_error; 3662 } 3663 cpumask_clear(&cpu_mask); 3664 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask); 3665 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); 3666 } 3667 3668 return 0; 3669 3670 irq_error: 3671 stmmac_free_irq(dev, irq_err, irq_idx); 3672 return ret; 3673 } 3674 3675 static int stmmac_request_irq_single(struct net_device *dev) 3676 { 3677 struct stmmac_priv *priv = netdev_priv(dev); 3678 enum request_irq_err irq_err; 3679 int ret; 3680 3681 ret = request_irq(dev->irq, stmmac_interrupt, 3682 IRQF_SHARED, dev->name, dev); 3683 if (unlikely(ret < 0)) { 3684 netdev_err(priv->dev, 3685 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3686 __func__, dev->irq, ret); 3687 irq_err = REQ_IRQ_ERR_MAC; 3688 goto irq_error; 3689 } 3690 3691 /* Request the Wake IRQ in case of another line 3692 * is used for WoL 3693 */ 3694 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3695 ret = request_irq(priv->wol_irq, stmmac_interrupt, 3696 IRQF_SHARED, dev->name, dev); 3697 if (unlikely(ret < 0)) { 3698 netdev_err(priv->dev, 3699 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 3700 __func__, priv->wol_irq, ret); 3701 irq_err = REQ_IRQ_ERR_WOL; 3702 goto irq_error; 3703 } 3704 } 3705 3706 /* Request the IRQ lines */ 3707 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { 3708 ret = request_irq(priv->lpi_irq, stmmac_interrupt, 3709 IRQF_SHARED, dev->name, dev); 3710 if (unlikely(ret < 0)) { 3711 netdev_err(priv->dev, 3712 "%s: ERROR: allocating the LPI IRQ %d (%d)\n", 3713 __func__, priv->lpi_irq, ret); 3714 irq_err = REQ_IRQ_ERR_LPI; 3715 goto irq_error; 3716 } 3717 } 3718 3719 return 0; 3720 3721 irq_error: 3722 stmmac_free_irq(dev, irq_err, 0); 3723 return ret; 3724 } 3725 3726 static int stmmac_request_irq(struct net_device *dev) 3727 { 3728 struct stmmac_priv *priv = netdev_priv(dev); 3729 int ret; 3730 3731 /* Request the IRQ lines */ 3732 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 3733 ret = stmmac_request_irq_multi_msi(dev); 3734 else 3735 ret = stmmac_request_irq_single(dev); 3736 3737 return ret; 3738 } 3739 3740 /** 3741 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 3742 * @priv: driver private structure 3743 * @mtu: MTU to setup the dma queue and buf with 3744 * Description: Allocate and generate a dma_conf based on the provided MTU. 3745 * Allocate the Tx/Rx DMA queue and init them. 3746 * Return value: 3747 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 3748 */ 3749 static struct stmmac_dma_conf * 3750 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 3751 { 3752 struct stmmac_dma_conf *dma_conf; 3753 int chan, bfsize, ret; 3754 3755 dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); 3756 if (!dma_conf) { 3757 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 3758 __func__); 3759 return ERR_PTR(-ENOMEM); 3760 } 3761 3762 bfsize = stmmac_set_16kib_bfsize(priv, mtu); 3763 if (bfsize < 0) 3764 bfsize = 0; 3765 3766 if (bfsize < BUF_SIZE_16KiB) 3767 bfsize = stmmac_set_bfsize(mtu, 0); 3768 3769 dma_conf->dma_buf_sz = bfsize; 3770 /* Chose the tx/rx size from the already defined one in the 3771 * priv struct. (if defined) 3772 */ 3773 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 3774 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 3775 3776 if (!dma_conf->dma_tx_size) 3777 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 3778 if (!dma_conf->dma_rx_size) 3779 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 3780 3781 /* Earlier check for TBS */ 3782 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 3783 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 3784 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 3785 3786 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 3787 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 3788 } 3789 3790 ret = alloc_dma_desc_resources(priv, dma_conf); 3791 if (ret < 0) { 3792 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 3793 __func__); 3794 goto alloc_error; 3795 } 3796 3797 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 3798 if (ret < 0) { 3799 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 3800 __func__); 3801 goto init_error; 3802 } 3803 3804 return dma_conf; 3805 3806 init_error: 3807 free_dma_desc_resources(priv, dma_conf); 3808 alloc_error: 3809 kfree(dma_conf); 3810 return ERR_PTR(ret); 3811 } 3812 3813 /** 3814 * __stmmac_open - open entry point of the driver 3815 * @dev : pointer to the device structure. 3816 * @dma_conf : structure to take the dma data 3817 * Description: 3818 * This function is the open entry point of the driver. 3819 * Return value: 3820 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3821 * file on failure. 3822 */ 3823 static int __stmmac_open(struct net_device *dev, 3824 struct stmmac_dma_conf *dma_conf) 3825 { 3826 struct stmmac_priv *priv = netdev_priv(dev); 3827 int mode = priv->plat->phy_interface; 3828 u32 chan; 3829 int ret; 3830 3831 ret = pm_runtime_resume_and_get(priv->device); 3832 if (ret < 0) 3833 return ret; 3834 3835 if (priv->hw->pcs != STMMAC_PCS_TBI && 3836 priv->hw->pcs != STMMAC_PCS_RTBI && 3837 (!priv->hw->xpcs || 3838 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73) && 3839 !priv->hw->lynx_pcs) { 3840 ret = stmmac_init_phy(dev); 3841 if (ret) { 3842 netdev_err(priv->dev, 3843 "%s: Cannot attach to PHY (error: %d)\n", 3844 __func__, ret); 3845 goto init_phy_error; 3846 } 3847 } 3848 3849 priv->rx_copybreak = STMMAC_RX_COPYBREAK; 3850 3851 buf_sz = dma_conf->dma_buf_sz; 3852 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 3853 3854 stmmac_reset_queues_param(priv); 3855 3856 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 3857 priv->plat->serdes_powerup) { 3858 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); 3859 if (ret < 0) { 3860 netdev_err(priv->dev, "%s: Serdes powerup failed\n", 3861 __func__); 3862 goto init_error; 3863 } 3864 } 3865 3866 ret = stmmac_hw_setup(dev, true); 3867 if (ret < 0) { 3868 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 3869 goto init_error; 3870 } 3871 3872 stmmac_init_coalesce(priv); 3873 3874 phylink_start(priv->phylink); 3875 /* We may have called phylink_speed_down before */ 3876 phylink_speed_up(priv->phylink); 3877 3878 ret = stmmac_request_irq(dev); 3879 if (ret) 3880 goto irq_error; 3881 3882 stmmac_enable_all_queues(priv); 3883 netif_tx_start_all_queues(priv->dev); 3884 stmmac_enable_all_dma_irq(priv); 3885 3886 return 0; 3887 3888 irq_error: 3889 phylink_stop(priv->phylink); 3890 3891 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3892 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 3893 3894 stmmac_hw_teardown(dev); 3895 init_error: 3896 phylink_disconnect_phy(priv->phylink); 3897 init_phy_error: 3898 pm_runtime_put(priv->device); 3899 return ret; 3900 } 3901 3902 static int stmmac_open(struct net_device *dev) 3903 { 3904 struct stmmac_priv *priv = netdev_priv(dev); 3905 struct stmmac_dma_conf *dma_conf; 3906 int ret; 3907 3908 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 3909 if (IS_ERR(dma_conf)) 3910 return PTR_ERR(dma_conf); 3911 3912 ret = __stmmac_open(dev, dma_conf); 3913 if (ret) 3914 free_dma_desc_resources(priv, dma_conf); 3915 3916 kfree(dma_conf); 3917 return ret; 3918 } 3919 3920 static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) 3921 { 3922 set_bit(__FPE_REMOVING, &priv->fpe_task_state); 3923 3924 if (priv->fpe_wq) 3925 destroy_workqueue(priv->fpe_wq); 3926 3927 netdev_info(priv->dev, "FPE workqueue stop"); 3928 } 3929 3930 /** 3931 * stmmac_release - close entry point of the driver 3932 * @dev : device pointer. 3933 * Description: 3934 * This is the stop entry point of the driver. 3935 */ 3936 static int stmmac_release(struct net_device *dev) 3937 { 3938 struct stmmac_priv *priv = netdev_priv(dev); 3939 u32 chan; 3940 3941 if (device_may_wakeup(priv->device)) 3942 phylink_speed_down(priv->phylink, false); 3943 /* Stop and disconnect the PHY */ 3944 phylink_stop(priv->phylink); 3945 phylink_disconnect_phy(priv->phylink); 3946 3947 stmmac_disable_all_queues(priv); 3948 3949 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 3950 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 3951 3952 netif_tx_disable(dev); 3953 3954 /* Free the IRQ lines */ 3955 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 3956 3957 if (priv->eee_enabled) { 3958 priv->tx_path_in_lpi_mode = false; 3959 del_timer_sync(&priv->eee_ctrl_timer); 3960 } 3961 3962 /* Stop TX/RX DMA and clear the descriptors */ 3963 stmmac_stop_all_dma(priv); 3964 3965 /* Release and free the Rx/Tx resources */ 3966 free_dma_desc_resources(priv, &priv->dma_conf); 3967 3968 /* Disable the MAC Rx/Tx */ 3969 stmmac_mac_set(priv, priv->ioaddr, false); 3970 3971 /* Powerdown Serdes if there is */ 3972 if (priv->plat->serdes_powerdown) 3973 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); 3974 3975 netif_carrier_off(dev); 3976 3977 stmmac_release_ptp(priv); 3978 3979 pm_runtime_put(priv->device); 3980 3981 if (priv->dma_cap.fpesel) 3982 stmmac_fpe_stop_wq(priv); 3983 3984 return 0; 3985 } 3986 3987 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 3988 struct stmmac_tx_queue *tx_q) 3989 { 3990 u16 tag = 0x0, inner_tag = 0x0; 3991 u32 inner_type = 0x0; 3992 struct dma_desc *p; 3993 3994 if (!priv->dma_cap.vlins) 3995 return false; 3996 if (!skb_vlan_tag_present(skb)) 3997 return false; 3998 if (skb->vlan_proto == htons(ETH_P_8021AD)) { 3999 inner_tag = skb_vlan_tag_get(skb); 4000 inner_type = STMMAC_VLAN_INSERT; 4001 } 4002 4003 tag = skb_vlan_tag_get(skb); 4004 4005 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4006 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4007 else 4008 p = &tx_q->dma_tx[tx_q->cur_tx]; 4009 4010 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) 4011 return false; 4012 4013 stmmac_set_tx_owner(priv, p); 4014 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4015 return true; 4016 } 4017 4018 /** 4019 * stmmac_tso_allocator - close entry point of the driver 4020 * @priv: driver private structure 4021 * @des: buffer start address 4022 * @total_len: total length to fill in descriptors 4023 * @last_segment: condition for the last descriptor 4024 * @queue: TX queue index 4025 * Description: 4026 * This function fills descriptor and request new descriptors according to 4027 * buffer length to fill 4028 */ 4029 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 4030 int total_len, bool last_segment, u32 queue) 4031 { 4032 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4033 struct dma_desc *desc; 4034 u32 buff_size; 4035 int tmp_len; 4036 4037 tmp_len = total_len; 4038 4039 while (tmp_len > 0) { 4040 dma_addr_t curr_addr; 4041 4042 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4043 priv->dma_conf.dma_tx_size); 4044 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4045 4046 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4047 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4048 else 4049 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4050 4051 curr_addr = des + (total_len - tmp_len); 4052 if (priv->dma_cap.addr64 <= 32) 4053 desc->des0 = cpu_to_le32(curr_addr); 4054 else 4055 stmmac_set_desc_addr(priv, desc, curr_addr); 4056 4057 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4058 TSO_MAX_BUFF_SIZE : tmp_len; 4059 4060 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4061 0, 1, 4062 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4063 0, 0); 4064 4065 tmp_len -= TSO_MAX_BUFF_SIZE; 4066 } 4067 } 4068 4069 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4070 { 4071 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4072 int desc_size; 4073 4074 if (likely(priv->extend_desc)) 4075 desc_size = sizeof(struct dma_extended_desc); 4076 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4077 desc_size = sizeof(struct dma_edesc); 4078 else 4079 desc_size = sizeof(struct dma_desc); 4080 4081 /* The own bit must be the latest setting done when prepare the 4082 * descriptor and then barrier is needed to make sure that 4083 * all is coherent before granting the DMA engine. 4084 */ 4085 wmb(); 4086 4087 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); 4088 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 4089 } 4090 4091 /** 4092 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4093 * @skb : the socket buffer 4094 * @dev : device pointer 4095 * Description: this is the transmit function that is called on TSO frames 4096 * (support available on GMAC4 and newer chips). 4097 * Diagram below show the ring programming in case of TSO frames: 4098 * 4099 * First Descriptor 4100 * -------- 4101 * | DES0 |---> buffer1 = L2/L3/L4 header 4102 * | DES1 |---> TCP Payload (can continue on next descr...) 4103 * | DES2 |---> buffer 1 and 2 len 4104 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4105 * -------- 4106 * | 4107 * ... 4108 * | 4109 * -------- 4110 * | DES0 | --| Split TCP Payload on Buffers 1 and 2 4111 * | DES1 | --| 4112 * | DES2 | --> buffer 1 and 2 len 4113 * | DES3 | 4114 * -------- 4115 * 4116 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4117 */ 4118 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4119 { 4120 struct dma_desc *desc, *first, *mss_desc = NULL; 4121 struct stmmac_priv *priv = netdev_priv(dev); 4122 int nfrags = skb_shinfo(skb)->nr_frags; 4123 u32 queue = skb_get_queue_mapping(skb); 4124 unsigned int first_entry, tx_packets; 4125 int tmp_pay_len = 0, first_tx; 4126 struct stmmac_tx_queue *tx_q; 4127 bool has_vlan, set_ic; 4128 u8 proto_hdr_len, hdr; 4129 unsigned long flags; 4130 u32 pay_len, mss; 4131 dma_addr_t des; 4132 int i; 4133 4134 tx_q = &priv->dma_conf.tx_queue[queue]; 4135 first_tx = tx_q->cur_tx; 4136 4137 /* Compute header lengths */ 4138 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 4139 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); 4140 hdr = sizeof(struct udphdr); 4141 } else { 4142 proto_hdr_len = skb_tcp_all_headers(skb); 4143 hdr = tcp_hdrlen(skb); 4144 } 4145 4146 /* Desc availability based on threshold should be enough safe */ 4147 if (unlikely(stmmac_tx_avail(priv, queue) < 4148 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 4149 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4150 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4151 queue)); 4152 /* This is a hard error, log it. */ 4153 netdev_err(priv->dev, 4154 "%s: Tx Ring full when queue awake\n", 4155 __func__); 4156 } 4157 return NETDEV_TX_BUSY; 4158 } 4159 4160 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4161 4162 mss = skb_shinfo(skb)->gso_size; 4163 4164 /* set new MSS value if needed */ 4165 if (mss != tx_q->mss) { 4166 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4167 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4168 else 4169 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4170 4171 stmmac_set_mss(priv, mss_desc, mss); 4172 tx_q->mss = mss; 4173 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4174 priv->dma_conf.dma_tx_size); 4175 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4176 } 4177 4178 if (netif_msg_tx_queued(priv)) { 4179 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 4180 __func__, hdr, proto_hdr_len, pay_len, mss); 4181 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4182 skb->data_len); 4183 } 4184 4185 /* Check if VLAN can be inserted by HW */ 4186 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4187 4188 first_entry = tx_q->cur_tx; 4189 WARN_ON(tx_q->tx_skbuff[first_entry]); 4190 4191 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4192 desc = &tx_q->dma_entx[first_entry].basic; 4193 else 4194 desc = &tx_q->dma_tx[first_entry]; 4195 first = desc; 4196 4197 if (has_vlan) 4198 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4199 4200 /* first descriptor: fill Headers on Buf1 */ 4201 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4202 DMA_TO_DEVICE); 4203 if (dma_mapping_error(priv->device, des)) 4204 goto dma_map_err; 4205 4206 tx_q->tx_skbuff_dma[first_entry].buf = des; 4207 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); 4208 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4209 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4210 4211 if (priv->dma_cap.addr64 <= 32) { 4212 first->des0 = cpu_to_le32(des); 4213 4214 /* Fill start of payload in buff2 of first descriptor */ 4215 if (pay_len) 4216 first->des1 = cpu_to_le32(des + proto_hdr_len); 4217 4218 /* If needed take extra descriptors to fill the remaining payload */ 4219 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; 4220 } else { 4221 stmmac_set_desc_addr(priv, first, des); 4222 tmp_pay_len = pay_len; 4223 des += proto_hdr_len; 4224 pay_len = 0; 4225 } 4226 4227 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); 4228 4229 /* Prepare fragments */ 4230 for (i = 0; i < nfrags; i++) { 4231 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4232 4233 des = skb_frag_dma_map(priv->device, frag, 0, 4234 skb_frag_size(frag), 4235 DMA_TO_DEVICE); 4236 if (dma_mapping_error(priv->device, des)) 4237 goto dma_map_err; 4238 4239 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4240 (i == nfrags - 1), queue); 4241 4242 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4243 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 4244 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 4245 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4246 } 4247 4248 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 4249 4250 /* Only the last descriptor gets to point to the skb. */ 4251 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4252 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4253 4254 /* Manage tx mitigation */ 4255 tx_packets = (tx_q->cur_tx + 1) - first_tx; 4256 tx_q->tx_count_frames += tx_packets; 4257 4258 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4259 set_ic = true; 4260 else if (!priv->tx_coal_frames[queue]) 4261 set_ic = false; 4262 else if (tx_packets > priv->tx_coal_frames[queue]) 4263 set_ic = true; 4264 else if ((tx_q->tx_count_frames % 4265 priv->tx_coal_frames[queue]) < tx_packets) 4266 set_ic = true; 4267 else 4268 set_ic = false; 4269 4270 if (set_ic) { 4271 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4272 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4273 else 4274 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4275 4276 tx_q->tx_count_frames = 0; 4277 stmmac_set_tx_ic(priv, desc); 4278 } 4279 4280 /* We've used all descriptors we need for this skb, however, 4281 * advance cur_tx so that it references a fresh descriptor. 4282 * ndo_start_xmit will fill this descriptor the next time it's 4283 * called and stmmac_tx_clean may clean up to this descriptor. 4284 */ 4285 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4286 4287 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4288 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4289 __func__); 4290 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4291 } 4292 4293 flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); 4294 tx_q->txq_stats.tx_bytes += skb->len; 4295 tx_q->txq_stats.tx_tso_frames++; 4296 tx_q->txq_stats.tx_tso_nfrags += nfrags; 4297 if (set_ic) 4298 tx_q->txq_stats.tx_set_ic_bit++; 4299 u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); 4300 4301 if (priv->sarc_type) 4302 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4303 4304 skb_tx_timestamp(skb); 4305 4306 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4307 priv->hwts_tx_en)) { 4308 /* declare that device is doing timestamping */ 4309 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4310 stmmac_enable_tx_timestamp(priv, first); 4311 } 4312 4313 /* Complete the first descriptor before granting the DMA */ 4314 stmmac_prepare_tso_tx_desc(priv, first, 1, 4315 proto_hdr_len, 4316 pay_len, 4317 1, tx_q->tx_skbuff_dma[first_entry].last_segment, 4318 hdr / 4, (skb->len - proto_hdr_len)); 4319 4320 /* If context desc is used to change MSS */ 4321 if (mss_desc) { 4322 /* Make sure that first descriptor has been completely 4323 * written, including its own bit. This is because MSS is 4324 * actually before first descriptor, so we need to make 4325 * sure that MSS's own bit is the last thing written. 4326 */ 4327 dma_wmb(); 4328 stmmac_set_tx_owner(priv, mss_desc); 4329 } 4330 4331 if (netif_msg_pktdata(priv)) { 4332 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4333 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4334 tx_q->cur_tx, first, nfrags); 4335 pr_info(">>> frame to be transmitted: "); 4336 print_pkt(skb->data, skb_headlen(skb)); 4337 } 4338 4339 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4340 4341 stmmac_flush_tx_descriptors(priv, queue); 4342 stmmac_tx_timer_arm(priv, queue); 4343 4344 return NETDEV_TX_OK; 4345 4346 dma_map_err: 4347 dev_err(priv->device, "Tx dma map failed\n"); 4348 dev_kfree_skb(skb); 4349 priv->xstats.tx_dropped++; 4350 return NETDEV_TX_OK; 4351 } 4352 4353 /** 4354 * stmmac_xmit - Tx entry point of the driver 4355 * @skb : the socket buffer 4356 * @dev : device pointer 4357 * Description : this is the tx entry point of the driver. 4358 * It programs the chain or the ring and supports oversized frames 4359 * and SG feature. 4360 */ 4361 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4362 { 4363 unsigned int first_entry, tx_packets, enh_desc; 4364 struct stmmac_priv *priv = netdev_priv(dev); 4365 unsigned int nopaged_len = skb_headlen(skb); 4366 int i, csum_insertion = 0, is_jumbo = 0; 4367 u32 queue = skb_get_queue_mapping(skb); 4368 int nfrags = skb_shinfo(skb)->nr_frags; 4369 int gso = skb_shinfo(skb)->gso_type; 4370 struct dma_edesc *tbs_desc = NULL; 4371 struct dma_desc *desc, *first; 4372 struct stmmac_tx_queue *tx_q; 4373 bool has_vlan, set_ic; 4374 int entry, first_tx; 4375 unsigned long flags; 4376 dma_addr_t des; 4377 4378 tx_q = &priv->dma_conf.tx_queue[queue]; 4379 first_tx = tx_q->cur_tx; 4380 4381 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4382 stmmac_disable_eee_mode(priv); 4383 4384 /* Manage oversized TCP frames for GMAC4 device */ 4385 if (skb_is_gso(skb) && priv->tso) { 4386 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 4387 return stmmac_tso_xmit(skb, dev); 4388 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) 4389 return stmmac_tso_xmit(skb, dev); 4390 } 4391 4392 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4393 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4394 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4395 queue)); 4396 /* This is a hard error, log it. */ 4397 netdev_err(priv->dev, 4398 "%s: Tx Ring full when queue awake\n", 4399 __func__); 4400 } 4401 return NETDEV_TX_BUSY; 4402 } 4403 4404 /* Check if VLAN can be inserted by HW */ 4405 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4406 4407 entry = tx_q->cur_tx; 4408 first_entry = entry; 4409 WARN_ON(tx_q->tx_skbuff[first_entry]); 4410 4411 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 4412 4413 if (likely(priv->extend_desc)) 4414 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4415 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4416 desc = &tx_q->dma_entx[entry].basic; 4417 else 4418 desc = tx_q->dma_tx + entry; 4419 4420 first = desc; 4421 4422 if (has_vlan) 4423 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4424 4425 enh_desc = priv->plat->enh_desc; 4426 /* To program the descriptors according to the size of the frame */ 4427 if (enh_desc) 4428 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4429 4430 if (unlikely(is_jumbo)) { 4431 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4432 if (unlikely(entry < 0) && (entry != -EINVAL)) 4433 goto dma_map_err; 4434 } 4435 4436 for (i = 0; i < nfrags; i++) { 4437 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4438 int len = skb_frag_size(frag); 4439 bool last_segment = (i == (nfrags - 1)); 4440 4441 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4442 WARN_ON(tx_q->tx_skbuff[entry]); 4443 4444 if (likely(priv->extend_desc)) 4445 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4446 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4447 desc = &tx_q->dma_entx[entry].basic; 4448 else 4449 desc = tx_q->dma_tx + entry; 4450 4451 des = skb_frag_dma_map(priv->device, frag, 0, len, 4452 DMA_TO_DEVICE); 4453 if (dma_mapping_error(priv->device, des)) 4454 goto dma_map_err; /* should reuse desc w/o issues */ 4455 4456 tx_q->tx_skbuff_dma[entry].buf = des; 4457 4458 stmmac_set_desc_addr(priv, desc, des); 4459 4460 tx_q->tx_skbuff_dma[entry].map_as_page = true; 4461 tx_q->tx_skbuff_dma[entry].len = len; 4462 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 4463 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4464 4465 /* Prepare the descriptor and set the own bit too */ 4466 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 4467 priv->mode, 1, last_segment, skb->len); 4468 } 4469 4470 /* Only the last descriptor gets to point to the skb. */ 4471 tx_q->tx_skbuff[entry] = skb; 4472 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4473 4474 /* According to the coalesce parameter the IC bit for the latest 4475 * segment is reset and the timer re-started to clean the tx status. 4476 * This approach takes care about the fragments: desc is the first 4477 * element in case of no SG. 4478 */ 4479 tx_packets = (entry + 1) - first_tx; 4480 tx_q->tx_count_frames += tx_packets; 4481 4482 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4483 set_ic = true; 4484 else if (!priv->tx_coal_frames[queue]) 4485 set_ic = false; 4486 else if (tx_packets > priv->tx_coal_frames[queue]) 4487 set_ic = true; 4488 else if ((tx_q->tx_count_frames % 4489 priv->tx_coal_frames[queue]) < tx_packets) 4490 set_ic = true; 4491 else 4492 set_ic = false; 4493 4494 if (set_ic) { 4495 if (likely(priv->extend_desc)) 4496 desc = &tx_q->dma_etx[entry].basic; 4497 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4498 desc = &tx_q->dma_entx[entry].basic; 4499 else 4500 desc = &tx_q->dma_tx[entry]; 4501 4502 tx_q->tx_count_frames = 0; 4503 stmmac_set_tx_ic(priv, desc); 4504 } 4505 4506 /* We've used all descriptors we need for this skb, however, 4507 * advance cur_tx so that it references a fresh descriptor. 4508 * ndo_start_xmit will fill this descriptor the next time it's 4509 * called and stmmac_tx_clean may clean up to this descriptor. 4510 */ 4511 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4512 tx_q->cur_tx = entry; 4513 4514 if (netif_msg_pktdata(priv)) { 4515 netdev_dbg(priv->dev, 4516 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4517 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4518 entry, first, nfrags); 4519 4520 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4521 print_pkt(skb->data, skb->len); 4522 } 4523 4524 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4525 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4526 __func__); 4527 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4528 } 4529 4530 flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); 4531 tx_q->txq_stats.tx_bytes += skb->len; 4532 if (set_ic) 4533 tx_q->txq_stats.tx_set_ic_bit++; 4534 u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); 4535 4536 if (priv->sarc_type) 4537 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4538 4539 skb_tx_timestamp(skb); 4540 4541 /* Ready to fill the first descriptor and set the OWN bit w/o any 4542 * problems because all the descriptors are actually ready to be 4543 * passed to the DMA engine. 4544 */ 4545 if (likely(!is_jumbo)) { 4546 bool last_segment = (nfrags == 0); 4547 4548 des = dma_map_single(priv->device, skb->data, 4549 nopaged_len, DMA_TO_DEVICE); 4550 if (dma_mapping_error(priv->device, des)) 4551 goto dma_map_err; 4552 4553 tx_q->tx_skbuff_dma[first_entry].buf = des; 4554 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4555 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4556 4557 stmmac_set_desc_addr(priv, first, des); 4558 4559 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 4560 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 4561 4562 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4563 priv->hwts_tx_en)) { 4564 /* declare that device is doing timestamping */ 4565 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4566 stmmac_enable_tx_timestamp(priv, first); 4567 } 4568 4569 /* Prepare the first descriptor setting the OWN bit too */ 4570 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 4571 csum_insertion, priv->mode, 0, last_segment, 4572 skb->len); 4573 } 4574 4575 if (tx_q->tbs & STMMAC_TBS_EN) { 4576 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4577 4578 tbs_desc = &tx_q->dma_entx[first_entry]; 4579 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); 4580 } 4581 4582 stmmac_set_tx_owner(priv, first); 4583 4584 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4585 4586 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4587 4588 stmmac_flush_tx_descriptors(priv, queue); 4589 stmmac_tx_timer_arm(priv, queue); 4590 4591 return NETDEV_TX_OK; 4592 4593 dma_map_err: 4594 netdev_err(priv->dev, "Tx DMA map failed\n"); 4595 dev_kfree_skb(skb); 4596 priv->xstats.tx_dropped++; 4597 return NETDEV_TX_OK; 4598 } 4599 4600 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 4601 { 4602 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 4603 __be16 vlan_proto = veth->h_vlan_proto; 4604 u16 vlanid; 4605 4606 if ((vlan_proto == htons(ETH_P_8021Q) && 4607 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 4608 (vlan_proto == htons(ETH_P_8021AD) && 4609 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 4610 /* pop the vlan tag */ 4611 vlanid = ntohs(veth->h_vlan_TCI); 4612 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 4613 skb_pull(skb, VLAN_HLEN); 4614 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 4615 } 4616 } 4617 4618 /** 4619 * stmmac_rx_refill - refill used skb preallocated buffers 4620 * @priv: driver private structure 4621 * @queue: RX queue index 4622 * Description : this is to reallocate the skb for the reception process 4623 * that is based on zero-copy. 4624 */ 4625 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 4626 { 4627 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4628 int dirty = stmmac_rx_dirty(priv, queue); 4629 unsigned int entry = rx_q->dirty_rx; 4630 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 4631 4632 if (priv->dma_cap.host_dma_width <= 32) 4633 gfp |= GFP_DMA32; 4634 4635 while (dirty-- > 0) { 4636 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4637 struct dma_desc *p; 4638 bool use_rx_wd; 4639 4640 if (priv->extend_desc) 4641 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4642 else 4643 p = rx_q->dma_rx + entry; 4644 4645 if (!buf->page) { 4646 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4647 if (!buf->page) 4648 break; 4649 } 4650 4651 if (priv->sph && !buf->sec_page) { 4652 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4653 if (!buf->sec_page) 4654 break; 4655 4656 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 4657 } 4658 4659 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 4660 4661 stmmac_set_desc_addr(priv, p, buf->addr); 4662 if (priv->sph) 4663 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 4664 else 4665 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 4666 stmmac_refill_desc3(priv, rx_q, p); 4667 4668 rx_q->rx_count_frames++; 4669 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4670 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4671 rx_q->rx_count_frames = 0; 4672 4673 use_rx_wd = !priv->rx_coal_frames[queue]; 4674 use_rx_wd |= rx_q->rx_count_frames > 0; 4675 if (!priv->use_riwt) 4676 use_rx_wd = false; 4677 4678 dma_wmb(); 4679 stmmac_set_rx_owner(priv, p, use_rx_wd); 4680 4681 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 4682 } 4683 rx_q->dirty_rx = entry; 4684 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4685 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4686 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4687 } 4688 4689 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 4690 struct dma_desc *p, 4691 int status, unsigned int len) 4692 { 4693 unsigned int plen = 0, hlen = 0; 4694 int coe = priv->hw->rx_csum; 4695 4696 /* Not first descriptor, buffer is always zero */ 4697 if (priv->sph && len) 4698 return 0; 4699 4700 /* First descriptor, get split header length */ 4701 stmmac_get_rx_header_len(priv, p, &hlen); 4702 if (priv->sph && hlen) { 4703 priv->xstats.rx_split_hdr_pkt_n++; 4704 return hlen; 4705 } 4706 4707 /* First descriptor, not last descriptor and not split header */ 4708 if (status & rx_not_ls) 4709 return priv->dma_conf.dma_buf_sz; 4710 4711 plen = stmmac_get_rx_frame_len(priv, p, coe); 4712 4713 /* First descriptor and last descriptor and not split header */ 4714 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 4715 } 4716 4717 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 4718 struct dma_desc *p, 4719 int status, unsigned int len) 4720 { 4721 int coe = priv->hw->rx_csum; 4722 unsigned int plen = 0; 4723 4724 /* Not split header, buffer is not available */ 4725 if (!priv->sph) 4726 return 0; 4727 4728 /* Not last descriptor */ 4729 if (status & rx_not_ls) 4730 return priv->dma_conf.dma_buf_sz; 4731 4732 plen = stmmac_get_rx_frame_len(priv, p, coe); 4733 4734 /* Last descriptor */ 4735 return plen - len; 4736 } 4737 4738 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 4739 struct xdp_frame *xdpf, bool dma_map) 4740 { 4741 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4742 unsigned int entry = tx_q->cur_tx; 4743 struct dma_desc *tx_desc; 4744 dma_addr_t dma_addr; 4745 bool set_ic; 4746 4747 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 4748 return STMMAC_XDP_CONSUMED; 4749 4750 if (likely(priv->extend_desc)) 4751 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4752 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4753 tx_desc = &tx_q->dma_entx[entry].basic; 4754 else 4755 tx_desc = tx_q->dma_tx + entry; 4756 4757 if (dma_map) { 4758 dma_addr = dma_map_single(priv->device, xdpf->data, 4759 xdpf->len, DMA_TO_DEVICE); 4760 if (dma_mapping_error(priv->device, dma_addr)) 4761 return STMMAC_XDP_CONSUMED; 4762 4763 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; 4764 } else { 4765 struct page *page = virt_to_page(xdpf->data); 4766 4767 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 4768 xdpf->headroom; 4769 dma_sync_single_for_device(priv->device, dma_addr, 4770 xdpf->len, DMA_BIDIRECTIONAL); 4771 4772 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; 4773 } 4774 4775 tx_q->tx_skbuff_dma[entry].buf = dma_addr; 4776 tx_q->tx_skbuff_dma[entry].map_as_page = false; 4777 tx_q->tx_skbuff_dma[entry].len = xdpf->len; 4778 tx_q->tx_skbuff_dma[entry].last_segment = true; 4779 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 4780 4781 tx_q->xdpf[entry] = xdpf; 4782 4783 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 4784 4785 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 4786 true, priv->mode, true, true, 4787 xdpf->len); 4788 4789 tx_q->tx_count_frames++; 4790 4791 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 4792 set_ic = true; 4793 else 4794 set_ic = false; 4795 4796 if (set_ic) { 4797 unsigned long flags; 4798 tx_q->tx_count_frames = 0; 4799 stmmac_set_tx_ic(priv, tx_desc); 4800 flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); 4801 tx_q->txq_stats.tx_set_ic_bit++; 4802 u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); 4803 } 4804 4805 stmmac_enable_dma_transmission(priv, priv->ioaddr); 4806 4807 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4808 tx_q->cur_tx = entry; 4809 4810 return STMMAC_XDP_TX; 4811 } 4812 4813 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 4814 int cpu) 4815 { 4816 int index = cpu; 4817 4818 if (unlikely(index < 0)) 4819 index = 0; 4820 4821 while (index >= priv->plat->tx_queues_to_use) 4822 index -= priv->plat->tx_queues_to_use; 4823 4824 return index; 4825 } 4826 4827 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 4828 struct xdp_buff *xdp) 4829 { 4830 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 4831 int cpu = smp_processor_id(); 4832 struct netdev_queue *nq; 4833 int queue; 4834 int res; 4835 4836 if (unlikely(!xdpf)) 4837 return STMMAC_XDP_CONSUMED; 4838 4839 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4840 nq = netdev_get_tx_queue(priv->dev, queue); 4841 4842 __netif_tx_lock(nq, cpu); 4843 /* Avoids TX time-out as we are sharing with slow path */ 4844 txq_trans_cond_update(nq); 4845 4846 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); 4847 if (res == STMMAC_XDP_TX) 4848 stmmac_flush_tx_descriptors(priv, queue); 4849 4850 __netif_tx_unlock(nq); 4851 4852 return res; 4853 } 4854 4855 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 4856 struct bpf_prog *prog, 4857 struct xdp_buff *xdp) 4858 { 4859 u32 act; 4860 int res; 4861 4862 act = bpf_prog_run_xdp(prog, xdp); 4863 switch (act) { 4864 case XDP_PASS: 4865 res = STMMAC_XDP_PASS; 4866 break; 4867 case XDP_TX: 4868 res = stmmac_xdp_xmit_back(priv, xdp); 4869 break; 4870 case XDP_REDIRECT: 4871 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 4872 res = STMMAC_XDP_CONSUMED; 4873 else 4874 res = STMMAC_XDP_REDIRECT; 4875 break; 4876 default: 4877 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 4878 fallthrough; 4879 case XDP_ABORTED: 4880 trace_xdp_exception(priv->dev, prog, act); 4881 fallthrough; 4882 case XDP_DROP: 4883 res = STMMAC_XDP_CONSUMED; 4884 break; 4885 } 4886 4887 return res; 4888 } 4889 4890 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 4891 struct xdp_buff *xdp) 4892 { 4893 struct bpf_prog *prog; 4894 int res; 4895 4896 prog = READ_ONCE(priv->xdp_prog); 4897 if (!prog) { 4898 res = STMMAC_XDP_PASS; 4899 goto out; 4900 } 4901 4902 res = __stmmac_xdp_run_prog(priv, prog, xdp); 4903 out: 4904 return ERR_PTR(-res); 4905 } 4906 4907 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 4908 int xdp_status) 4909 { 4910 int cpu = smp_processor_id(); 4911 int queue; 4912 4913 queue = stmmac_xdp_get_tx_queue(priv, cpu); 4914 4915 if (xdp_status & STMMAC_XDP_TX) 4916 stmmac_tx_timer_arm(priv, queue); 4917 4918 if (xdp_status & STMMAC_XDP_REDIRECT) 4919 xdp_do_flush(); 4920 } 4921 4922 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 4923 struct xdp_buff *xdp) 4924 { 4925 unsigned int metasize = xdp->data - xdp->data_meta; 4926 unsigned int datasize = xdp->data_end - xdp->data; 4927 struct sk_buff *skb; 4928 4929 skb = __napi_alloc_skb(&ch->rxtx_napi, 4930 xdp->data_end - xdp->data_hard_start, 4931 GFP_ATOMIC | __GFP_NOWARN); 4932 if (unlikely(!skb)) 4933 return NULL; 4934 4935 skb_reserve(skb, xdp->data - xdp->data_hard_start); 4936 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 4937 if (metasize) 4938 skb_metadata_set(skb, metasize); 4939 4940 return skb; 4941 } 4942 4943 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 4944 struct dma_desc *p, struct dma_desc *np, 4945 struct xdp_buff *xdp) 4946 { 4947 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4948 struct stmmac_channel *ch = &priv->channel[queue]; 4949 unsigned int len = xdp->data_end - xdp->data; 4950 enum pkt_hash_types hash_type; 4951 int coe = priv->hw->rx_csum; 4952 unsigned long flags; 4953 struct sk_buff *skb; 4954 u32 hash; 4955 4956 skb = stmmac_construct_skb_zc(ch, xdp); 4957 if (!skb) { 4958 priv->xstats.rx_dropped++; 4959 return; 4960 } 4961 4962 stmmac_get_rx_hwtstamp(priv, p, np, skb); 4963 stmmac_rx_vlan(priv->dev, skb); 4964 skb->protocol = eth_type_trans(skb, priv->dev); 4965 4966 if (unlikely(!coe)) 4967 skb_checksum_none_assert(skb); 4968 else 4969 skb->ip_summed = CHECKSUM_UNNECESSARY; 4970 4971 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 4972 skb_set_hash(skb, hash, hash_type); 4973 4974 skb_record_rx_queue(skb, queue); 4975 napi_gro_receive(&ch->rxtx_napi, skb); 4976 4977 flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); 4978 rx_q->rxq_stats.rx_pkt_n++; 4979 rx_q->rxq_stats.rx_bytes += len; 4980 u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); 4981 } 4982 4983 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 4984 { 4985 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4986 unsigned int entry = rx_q->dirty_rx; 4987 struct dma_desc *rx_desc = NULL; 4988 bool ret = true; 4989 4990 budget = min(budget, stmmac_rx_dirty(priv, queue)); 4991 4992 while (budget-- > 0 && entry != rx_q->cur_rx) { 4993 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4994 dma_addr_t dma_addr; 4995 bool use_rx_wd; 4996 4997 if (!buf->xdp) { 4998 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 4999 if (!buf->xdp) { 5000 ret = false; 5001 break; 5002 } 5003 } 5004 5005 if (priv->extend_desc) 5006 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); 5007 else 5008 rx_desc = rx_q->dma_rx + entry; 5009 5010 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5011 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5012 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5013 stmmac_refill_desc3(priv, rx_q, rx_desc); 5014 5015 rx_q->rx_count_frames++; 5016 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5017 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5018 rx_q->rx_count_frames = 0; 5019 5020 use_rx_wd = !priv->rx_coal_frames[queue]; 5021 use_rx_wd |= rx_q->rx_count_frames > 0; 5022 if (!priv->use_riwt) 5023 use_rx_wd = false; 5024 5025 dma_wmb(); 5026 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5027 5028 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 5029 } 5030 5031 if (rx_desc) { 5032 rx_q->dirty_rx = entry; 5033 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 5034 (rx_q->dirty_rx * sizeof(struct dma_desc)); 5035 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 5036 } 5037 5038 return ret; 5039 } 5040 5041 static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5042 { 5043 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5044 * to represent incoming packet, whereas cb field in the same structure 5045 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5046 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5047 */ 5048 return (struct stmmac_xdp_buff *)xdp; 5049 } 5050 5051 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5052 { 5053 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5054 unsigned int count = 0, error = 0, len = 0; 5055 int dirty = stmmac_rx_dirty(priv, queue); 5056 unsigned int next_entry = rx_q->cur_rx; 5057 u32 rx_errors = 0, rx_dropped = 0; 5058 unsigned int desc_size; 5059 struct bpf_prog *prog; 5060 bool failure = false; 5061 unsigned long flags; 5062 int xdp_status = 0; 5063 int status = 0; 5064 5065 if (netif_msg_rx_status(priv)) { 5066 void *rx_head; 5067 5068 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5069 if (priv->extend_desc) { 5070 rx_head = (void *)rx_q->dma_erx; 5071 desc_size = sizeof(struct dma_extended_desc); 5072 } else { 5073 rx_head = (void *)rx_q->dma_rx; 5074 desc_size = sizeof(struct dma_desc); 5075 } 5076 5077 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5078 rx_q->dma_rx_phy, desc_size); 5079 } 5080 while (count < limit) { 5081 struct stmmac_rx_buffer *buf; 5082 struct stmmac_xdp_buff *ctx; 5083 unsigned int buf1_len = 0; 5084 struct dma_desc *np, *p; 5085 int entry; 5086 int res; 5087 5088 if (!count && rx_q->state_saved) { 5089 error = rx_q->state.error; 5090 len = rx_q->state.len; 5091 } else { 5092 rx_q->state_saved = false; 5093 error = 0; 5094 len = 0; 5095 } 5096 5097 if (count >= limit) 5098 break; 5099 5100 read_again: 5101 buf1_len = 0; 5102 entry = next_entry; 5103 buf = &rx_q->buf_pool[entry]; 5104 5105 if (dirty >= STMMAC_RX_FILL_BATCH) { 5106 failure = failure || 5107 !stmmac_rx_refill_zc(priv, queue, dirty); 5108 dirty = 0; 5109 } 5110 5111 if (priv->extend_desc) 5112 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5113 else 5114 p = rx_q->dma_rx + entry; 5115 5116 /* read the status of the incoming frame */ 5117 status = stmmac_rx_status(priv, &priv->xstats, p); 5118 /* check if managed by the DMA otherwise go ahead */ 5119 if (unlikely(status & dma_own)) 5120 break; 5121 5122 /* Prefetch the next RX descriptor */ 5123 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5124 priv->dma_conf.dma_rx_size); 5125 next_entry = rx_q->cur_rx; 5126 5127 if (priv->extend_desc) 5128 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5129 else 5130 np = rx_q->dma_rx + next_entry; 5131 5132 prefetch(np); 5133 5134 /* Ensure a valid XSK buffer before proceed */ 5135 if (!buf->xdp) 5136 break; 5137 5138 if (priv->extend_desc) 5139 stmmac_rx_extended_status(priv, &priv->xstats, 5140 rx_q->dma_erx + entry); 5141 if (unlikely(status == discard_frame)) { 5142 xsk_buff_free(buf->xdp); 5143 buf->xdp = NULL; 5144 dirty++; 5145 error = 1; 5146 if (!priv->hwts_rx_en) 5147 rx_errors++; 5148 } 5149 5150 if (unlikely(error && (status & rx_not_ls))) 5151 goto read_again; 5152 if (unlikely(error)) { 5153 count++; 5154 continue; 5155 } 5156 5157 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5158 if (likely(status & rx_not_ls)) { 5159 xsk_buff_free(buf->xdp); 5160 buf->xdp = NULL; 5161 dirty++; 5162 count++; 5163 goto read_again; 5164 } 5165 5166 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5167 ctx->priv = priv; 5168 ctx->desc = p; 5169 ctx->ndesc = np; 5170 5171 /* XDP ZC Frame only support primary buffers for now */ 5172 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5173 len += buf1_len; 5174 5175 /* ACS is disabled; strip manually. */ 5176 if (likely(!(status & rx_not_ls))) { 5177 buf1_len -= ETH_FCS_LEN; 5178 len -= ETH_FCS_LEN; 5179 } 5180 5181 /* RX buffer is good and fit into a XSK pool buffer */ 5182 buf->xdp->data_end = buf->xdp->data + buf1_len; 5183 xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); 5184 5185 prog = READ_ONCE(priv->xdp_prog); 5186 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5187 5188 switch (res) { 5189 case STMMAC_XDP_PASS: 5190 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5191 xsk_buff_free(buf->xdp); 5192 break; 5193 case STMMAC_XDP_CONSUMED: 5194 xsk_buff_free(buf->xdp); 5195 rx_dropped++; 5196 break; 5197 case STMMAC_XDP_TX: 5198 case STMMAC_XDP_REDIRECT: 5199 xdp_status |= res; 5200 break; 5201 } 5202 5203 buf->xdp = NULL; 5204 dirty++; 5205 count++; 5206 } 5207 5208 if (status & rx_not_ls) { 5209 rx_q->state_saved = true; 5210 rx_q->state.error = error; 5211 rx_q->state.len = len; 5212 } 5213 5214 stmmac_finalize_xdp_rx(priv, xdp_status); 5215 5216 flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); 5217 rx_q->rxq_stats.rx_pkt_n += count; 5218 u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); 5219 5220 priv->xstats.rx_dropped += rx_dropped; 5221 priv->xstats.rx_errors += rx_errors; 5222 5223 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5224 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5225 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5226 else 5227 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5228 5229 return (int)count; 5230 } 5231 5232 return failure ? limit : (int)count; 5233 } 5234 5235 /** 5236 * stmmac_rx - manage the receive process 5237 * @priv: driver private structure 5238 * @limit: napi bugget 5239 * @queue: RX queue index. 5240 * Description : this the function called by the napi poll method. 5241 * It gets all the frames inside the ring. 5242 */ 5243 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5244 { 5245 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5246 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5247 struct stmmac_channel *ch = &priv->channel[queue]; 5248 unsigned int count = 0, error = 0, len = 0; 5249 int status = 0, coe = priv->hw->rx_csum; 5250 unsigned int next_entry = rx_q->cur_rx; 5251 enum dma_data_direction dma_dir; 5252 unsigned int desc_size; 5253 struct sk_buff *skb = NULL; 5254 struct stmmac_xdp_buff ctx; 5255 unsigned long flags; 5256 int xdp_status = 0; 5257 int buf_sz; 5258 5259 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5260 buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5261 5262 if (netif_msg_rx_status(priv)) { 5263 void *rx_head; 5264 5265 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5266 if (priv->extend_desc) { 5267 rx_head = (void *)rx_q->dma_erx; 5268 desc_size = sizeof(struct dma_extended_desc); 5269 } else { 5270 rx_head = (void *)rx_q->dma_rx; 5271 desc_size = sizeof(struct dma_desc); 5272 } 5273 5274 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5275 rx_q->dma_rx_phy, desc_size); 5276 } 5277 while (count < limit) { 5278 unsigned int buf1_len = 0, buf2_len = 0; 5279 enum pkt_hash_types hash_type; 5280 struct stmmac_rx_buffer *buf; 5281 struct dma_desc *np, *p; 5282 int entry; 5283 u32 hash; 5284 5285 if (!count && rx_q->state_saved) { 5286 skb = rx_q->state.skb; 5287 error = rx_q->state.error; 5288 len = rx_q->state.len; 5289 } else { 5290 rx_q->state_saved = false; 5291 skb = NULL; 5292 error = 0; 5293 len = 0; 5294 } 5295 5296 if (count >= limit) 5297 break; 5298 5299 read_again: 5300 buf1_len = 0; 5301 buf2_len = 0; 5302 entry = next_entry; 5303 buf = &rx_q->buf_pool[entry]; 5304 5305 if (priv->extend_desc) 5306 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5307 else 5308 p = rx_q->dma_rx + entry; 5309 5310 /* read the status of the incoming frame */ 5311 status = stmmac_rx_status(priv, &priv->xstats, p); 5312 /* check if managed by the DMA otherwise go ahead */ 5313 if (unlikely(status & dma_own)) 5314 break; 5315 5316 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5317 priv->dma_conf.dma_rx_size); 5318 next_entry = rx_q->cur_rx; 5319 5320 if (priv->extend_desc) 5321 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5322 else 5323 np = rx_q->dma_rx + next_entry; 5324 5325 prefetch(np); 5326 5327 if (priv->extend_desc) 5328 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5329 if (unlikely(status == discard_frame)) { 5330 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5331 buf->page = NULL; 5332 error = 1; 5333 if (!priv->hwts_rx_en) 5334 rx_errors++; 5335 } 5336 5337 if (unlikely(error && (status & rx_not_ls))) 5338 goto read_again; 5339 if (unlikely(error)) { 5340 dev_kfree_skb(skb); 5341 skb = NULL; 5342 count++; 5343 continue; 5344 } 5345 5346 /* Buffer is good. Go on. */ 5347 5348 prefetch(page_address(buf->page) + buf->page_offset); 5349 if (buf->sec_page) 5350 prefetch(page_address(buf->sec_page)); 5351 5352 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5353 len += buf1_len; 5354 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5355 len += buf2_len; 5356 5357 /* ACS is disabled; strip manually. */ 5358 if (likely(!(status & rx_not_ls))) { 5359 if (buf2_len) { 5360 buf2_len -= ETH_FCS_LEN; 5361 len -= ETH_FCS_LEN; 5362 } else if (buf1_len) { 5363 buf1_len -= ETH_FCS_LEN; 5364 len -= ETH_FCS_LEN; 5365 } 5366 } 5367 5368 if (!skb) { 5369 unsigned int pre_len, sync_len; 5370 5371 dma_sync_single_for_cpu(priv->device, buf->addr, 5372 buf1_len, dma_dir); 5373 5374 xdp_init_buff(&ctx.xdp, buf_sz, &rx_q->xdp_rxq); 5375 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5376 buf->page_offset, buf1_len, true); 5377 5378 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5379 buf->page_offset; 5380 5381 ctx.priv = priv; 5382 ctx.desc = p; 5383 ctx.ndesc = np; 5384 5385 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5386 /* Due xdp_adjust_tail: DMA sync for_device 5387 * cover max len CPU touch 5388 */ 5389 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5390 buf->page_offset; 5391 sync_len = max(sync_len, pre_len); 5392 5393 /* For Not XDP_PASS verdict */ 5394 if (IS_ERR(skb)) { 5395 unsigned int xdp_res = -PTR_ERR(skb); 5396 5397 if (xdp_res & STMMAC_XDP_CONSUMED) { 5398 page_pool_put_page(rx_q->page_pool, 5399 virt_to_head_page(ctx.xdp.data), 5400 sync_len, true); 5401 buf->page = NULL; 5402 rx_dropped++; 5403 5404 /* Clear skb as it was set as 5405 * status by XDP program. 5406 */ 5407 skb = NULL; 5408 5409 if (unlikely((status & rx_not_ls))) 5410 goto read_again; 5411 5412 count++; 5413 continue; 5414 } else if (xdp_res & (STMMAC_XDP_TX | 5415 STMMAC_XDP_REDIRECT)) { 5416 xdp_status |= xdp_res; 5417 buf->page = NULL; 5418 skb = NULL; 5419 count++; 5420 continue; 5421 } 5422 } 5423 } 5424 5425 if (!skb) { 5426 /* XDP program may expand or reduce tail */ 5427 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5428 5429 skb = napi_alloc_skb(&ch->rx_napi, buf1_len); 5430 if (!skb) { 5431 rx_dropped++; 5432 count++; 5433 goto drain_data; 5434 } 5435 5436 /* XDP program may adjust header */ 5437 skb_copy_to_linear_data(skb, ctx.xdp.data, buf1_len); 5438 skb_put(skb, buf1_len); 5439 5440 /* Data payload copied into SKB, page ready for recycle */ 5441 page_pool_recycle_direct(rx_q->page_pool, buf->page); 5442 buf->page = NULL; 5443 } else if (buf1_len) { 5444 dma_sync_single_for_cpu(priv->device, buf->addr, 5445 buf1_len, dma_dir); 5446 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5447 buf->page, buf->page_offset, buf1_len, 5448 priv->dma_conf.dma_buf_sz); 5449 5450 /* Data payload appended into SKB */ 5451 skb_mark_for_recycle(skb); 5452 buf->page = NULL; 5453 } 5454 5455 if (buf2_len) { 5456 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5457 buf2_len, dma_dir); 5458 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5459 buf->sec_page, 0, buf2_len, 5460 priv->dma_conf.dma_buf_sz); 5461 5462 /* Data payload appended into SKB */ 5463 skb_mark_for_recycle(skb); 5464 buf->sec_page = NULL; 5465 } 5466 5467 drain_data: 5468 if (likely(status & rx_not_ls)) 5469 goto read_again; 5470 if (!skb) 5471 continue; 5472 5473 /* Got entire packet into SKB. Finish it. */ 5474 5475 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5476 stmmac_rx_vlan(priv->dev, skb); 5477 skb->protocol = eth_type_trans(skb, priv->dev); 5478 5479 if (unlikely(!coe)) 5480 skb_checksum_none_assert(skb); 5481 else 5482 skb->ip_summed = CHECKSUM_UNNECESSARY; 5483 5484 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5485 skb_set_hash(skb, hash, hash_type); 5486 5487 skb_record_rx_queue(skb, queue); 5488 napi_gro_receive(&ch->rx_napi, skb); 5489 skb = NULL; 5490 5491 rx_packets++; 5492 rx_bytes += len; 5493 count++; 5494 } 5495 5496 if (status & rx_not_ls || skb) { 5497 rx_q->state_saved = true; 5498 rx_q->state.skb = skb; 5499 rx_q->state.error = error; 5500 rx_q->state.len = len; 5501 } 5502 5503 stmmac_finalize_xdp_rx(priv, xdp_status); 5504 5505 stmmac_rx_refill(priv, queue); 5506 5507 flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); 5508 rx_q->rxq_stats.rx_packets += rx_packets; 5509 rx_q->rxq_stats.rx_bytes += rx_bytes; 5510 rx_q->rxq_stats.rx_pkt_n += count; 5511 u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); 5512 5513 priv->xstats.rx_dropped += rx_dropped; 5514 priv->xstats.rx_errors += rx_errors; 5515 5516 return count; 5517 } 5518 5519 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5520 { 5521 struct stmmac_channel *ch = 5522 container_of(napi, struct stmmac_channel, rx_napi); 5523 struct stmmac_priv *priv = ch->priv_data; 5524 struct stmmac_rx_queue *rx_q; 5525 u32 chan = ch->index; 5526 unsigned long flags; 5527 int work_done; 5528 5529 rx_q = &priv->dma_conf.rx_queue[chan]; 5530 flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); 5531 rx_q->rxq_stats.napi_poll++; 5532 u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); 5533 5534 work_done = stmmac_rx(priv, budget, chan); 5535 if (work_done < budget && napi_complete_done(napi, work_done)) { 5536 unsigned long flags; 5537 5538 spin_lock_irqsave(&ch->lock, flags); 5539 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5540 spin_unlock_irqrestore(&ch->lock, flags); 5541 } 5542 5543 return work_done; 5544 } 5545 5546 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5547 { 5548 struct stmmac_channel *ch = 5549 container_of(napi, struct stmmac_channel, tx_napi); 5550 struct stmmac_priv *priv = ch->priv_data; 5551 struct stmmac_tx_queue *tx_q; 5552 u32 chan = ch->index; 5553 unsigned long flags; 5554 int work_done; 5555 5556 tx_q = &priv->dma_conf.tx_queue[chan]; 5557 flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); 5558 tx_q->txq_stats.napi_poll++; 5559 u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); 5560 5561 work_done = stmmac_tx_clean(priv, budget, chan); 5562 work_done = min(work_done, budget); 5563 5564 if (work_done < budget && napi_complete_done(napi, work_done)) { 5565 unsigned long flags; 5566 5567 spin_lock_irqsave(&ch->lock, flags); 5568 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5569 spin_unlock_irqrestore(&ch->lock, flags); 5570 } 5571 5572 return work_done; 5573 } 5574 5575 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 5576 { 5577 struct stmmac_channel *ch = 5578 container_of(napi, struct stmmac_channel, rxtx_napi); 5579 struct stmmac_priv *priv = ch->priv_data; 5580 int rx_done, tx_done, rxtx_done; 5581 struct stmmac_rx_queue *rx_q; 5582 struct stmmac_tx_queue *tx_q; 5583 u32 chan = ch->index; 5584 unsigned long flags; 5585 5586 rx_q = &priv->dma_conf.rx_queue[chan]; 5587 flags = u64_stats_update_begin_irqsave(&rx_q->rxq_stats.syncp); 5588 rx_q->rxq_stats.napi_poll++; 5589 u64_stats_update_end_irqrestore(&rx_q->rxq_stats.syncp, flags); 5590 5591 tx_q = &priv->dma_conf.tx_queue[chan]; 5592 flags = u64_stats_update_begin_irqsave(&tx_q->txq_stats.syncp); 5593 tx_q->txq_stats.napi_poll++; 5594 u64_stats_update_end_irqrestore(&tx_q->txq_stats.syncp, flags); 5595 5596 tx_done = stmmac_tx_clean(priv, budget, chan); 5597 tx_done = min(tx_done, budget); 5598 5599 rx_done = stmmac_rx_zc(priv, budget, chan); 5600 5601 rxtx_done = max(tx_done, rx_done); 5602 5603 /* If either TX or RX work is not complete, return budget 5604 * and keep pooling 5605 */ 5606 if (rxtx_done >= budget) 5607 return budget; 5608 5609 /* all work done, exit the polling mode */ 5610 if (napi_complete_done(napi, rxtx_done)) { 5611 unsigned long flags; 5612 5613 spin_lock_irqsave(&ch->lock, flags); 5614 /* Both RX and TX work done are compelte, 5615 * so enable both RX & TX IRQs. 5616 */ 5617 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 5618 spin_unlock_irqrestore(&ch->lock, flags); 5619 } 5620 5621 return min(rxtx_done, budget - 1); 5622 } 5623 5624 /** 5625 * stmmac_tx_timeout 5626 * @dev : Pointer to net device structure 5627 * @txqueue: the index of the hanging transmit queue 5628 * Description: this function is called when a packet transmission fails to 5629 * complete within a reasonable time. The driver will mark the error in the 5630 * netdev structure and arrange for the device to be reset to a sane state 5631 * in order to transmit a new packet. 5632 */ 5633 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 5634 { 5635 struct stmmac_priv *priv = netdev_priv(dev); 5636 5637 stmmac_global_err(priv); 5638 } 5639 5640 /** 5641 * stmmac_set_rx_mode - entry point for multicast addressing 5642 * @dev : pointer to the device structure 5643 * Description: 5644 * This function is a driver entry point which gets called by the kernel 5645 * whenever multicast addresses must be enabled/disabled. 5646 * Return value: 5647 * void. 5648 */ 5649 static void stmmac_set_rx_mode(struct net_device *dev) 5650 { 5651 struct stmmac_priv *priv = netdev_priv(dev); 5652 5653 stmmac_set_filter(priv, priv->hw, dev); 5654 } 5655 5656 /** 5657 * stmmac_change_mtu - entry point to change MTU size for the device. 5658 * @dev : device pointer. 5659 * @new_mtu : the new MTU size for the device. 5660 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 5661 * to drive packet transmission. Ethernet has an MTU of 1500 octets 5662 * (ETH_DATA_LEN). This value can be changed with ifconfig. 5663 * Return value: 5664 * 0 on success and an appropriate (-)ve integer as defined in errno.h 5665 * file on failure. 5666 */ 5667 static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 5668 { 5669 struct stmmac_priv *priv = netdev_priv(dev); 5670 int txfifosz = priv->plat->tx_fifo_size; 5671 struct stmmac_dma_conf *dma_conf; 5672 const int mtu = new_mtu; 5673 int ret; 5674 5675 if (txfifosz == 0) 5676 txfifosz = priv->dma_cap.tx_fifo_size; 5677 5678 txfifosz /= priv->plat->tx_queues_to_use; 5679 5680 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 5681 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 5682 return -EINVAL; 5683 } 5684 5685 new_mtu = STMMAC_ALIGN(new_mtu); 5686 5687 /* If condition true, FIFO is too small or MTU too large */ 5688 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 5689 return -EINVAL; 5690 5691 if (netif_running(dev)) { 5692 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 5693 /* Try to allocate the new DMA conf with the new mtu */ 5694 dma_conf = stmmac_setup_dma_desc(priv, mtu); 5695 if (IS_ERR(dma_conf)) { 5696 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 5697 mtu); 5698 return PTR_ERR(dma_conf); 5699 } 5700 5701 stmmac_release(dev); 5702 5703 ret = __stmmac_open(dev, dma_conf); 5704 if (ret) { 5705 free_dma_desc_resources(priv, dma_conf); 5706 kfree(dma_conf); 5707 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 5708 return ret; 5709 } 5710 5711 kfree(dma_conf); 5712 5713 stmmac_set_rx_mode(dev); 5714 } 5715 5716 dev->mtu = mtu; 5717 netdev_update_features(dev); 5718 5719 return 0; 5720 } 5721 5722 static netdev_features_t stmmac_fix_features(struct net_device *dev, 5723 netdev_features_t features) 5724 { 5725 struct stmmac_priv *priv = netdev_priv(dev); 5726 5727 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 5728 features &= ~NETIF_F_RXCSUM; 5729 5730 if (!priv->plat->tx_coe) 5731 features &= ~NETIF_F_CSUM_MASK; 5732 5733 /* Some GMAC devices have a bugged Jumbo frame support that 5734 * needs to have the Tx COE disabled for oversized frames 5735 * (due to limited buffer sizes). In this case we disable 5736 * the TX csum insertion in the TDES and not use SF. 5737 */ 5738 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 5739 features &= ~NETIF_F_CSUM_MASK; 5740 5741 /* Disable tso if asked by ethtool */ 5742 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 5743 if (features & NETIF_F_TSO) 5744 priv->tso = true; 5745 else 5746 priv->tso = false; 5747 } 5748 5749 return features; 5750 } 5751 5752 static int stmmac_set_features(struct net_device *netdev, 5753 netdev_features_t features) 5754 { 5755 struct stmmac_priv *priv = netdev_priv(netdev); 5756 5757 /* Keep the COE Type in case of csum is supporting */ 5758 if (features & NETIF_F_RXCSUM) 5759 priv->hw->rx_csum = priv->plat->rx_coe; 5760 else 5761 priv->hw->rx_csum = 0; 5762 /* No check needed because rx_coe has been set before and it will be 5763 * fixed in case of issue. 5764 */ 5765 stmmac_rx_ipc(priv, priv->hw); 5766 5767 if (priv->sph_cap) { 5768 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; 5769 u32 chan; 5770 5771 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 5772 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 5773 } 5774 5775 return 0; 5776 } 5777 5778 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status) 5779 { 5780 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 5781 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 5782 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 5783 bool *hs_enable = &fpe_cfg->hs_enable; 5784 5785 if (status == FPE_EVENT_UNKNOWN || !*hs_enable) 5786 return; 5787 5788 /* If LP has sent verify mPacket, LP is FPE capable */ 5789 if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER) { 5790 if (*lp_state < FPE_STATE_CAPABLE) 5791 *lp_state = FPE_STATE_CAPABLE; 5792 5793 /* If user has requested FPE enable, quickly response */ 5794 if (*hs_enable) 5795 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 5796 MPACKET_RESPONSE); 5797 } 5798 5799 /* If Local has sent verify mPacket, Local is FPE capable */ 5800 if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER) { 5801 if (*lo_state < FPE_STATE_CAPABLE) 5802 *lo_state = FPE_STATE_CAPABLE; 5803 } 5804 5805 /* If LP has sent response mPacket, LP is entering FPE ON */ 5806 if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP) 5807 *lp_state = FPE_STATE_ENTERING_ON; 5808 5809 /* If Local has sent response mPacket, Local is entering FPE ON */ 5810 if ((status & FPE_EVENT_TRSP) == FPE_EVENT_TRSP) 5811 *lo_state = FPE_STATE_ENTERING_ON; 5812 5813 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && 5814 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && 5815 priv->fpe_wq) { 5816 queue_work(priv->fpe_wq, &priv->fpe_task); 5817 } 5818 } 5819 5820 static void stmmac_common_interrupt(struct stmmac_priv *priv) 5821 { 5822 u32 rx_cnt = priv->plat->rx_queues_to_use; 5823 u32 tx_cnt = priv->plat->tx_queues_to_use; 5824 u32 queues_count; 5825 u32 queue; 5826 bool xmac; 5827 5828 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; 5829 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 5830 5831 if (priv->irq_wake) 5832 pm_wakeup_event(priv->device, 0); 5833 5834 if (priv->dma_cap.estsel) 5835 stmmac_est_irq_status(priv, priv->ioaddr, priv->dev, 5836 &priv->xstats, tx_cnt); 5837 5838 if (priv->dma_cap.fpesel) { 5839 int status = stmmac_fpe_irq_status(priv, priv->ioaddr, 5840 priv->dev); 5841 5842 stmmac_fpe_event_status(priv, status); 5843 } 5844 5845 /* To handle GMAC own interrupts */ 5846 if ((priv->plat->has_gmac) || xmac) { 5847 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 5848 5849 if (unlikely(status)) { 5850 /* For LPI we need to save the tx status */ 5851 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 5852 priv->tx_path_in_lpi_mode = true; 5853 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 5854 priv->tx_path_in_lpi_mode = false; 5855 } 5856 5857 for (queue = 0; queue < queues_count; queue++) { 5858 status = stmmac_host_mtl_irq_status(priv, priv->hw, 5859 queue); 5860 } 5861 5862 /* PCS link status */ 5863 if (priv->hw->pcs && 5864 !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { 5865 if (priv->xstats.pcs_link) 5866 netif_carrier_on(priv->dev); 5867 else 5868 netif_carrier_off(priv->dev); 5869 } 5870 5871 stmmac_timestamp_interrupt(priv, priv); 5872 } 5873 } 5874 5875 /** 5876 * stmmac_interrupt - main ISR 5877 * @irq: interrupt number. 5878 * @dev_id: to pass the net device pointer. 5879 * Description: this is the main driver interrupt service routine. 5880 * It can call: 5881 * o DMA service routine (to manage incoming frame reception and transmission 5882 * status) 5883 * o Core interrupts to manage: remote wake-up, management counter, LPI 5884 * interrupts. 5885 */ 5886 static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 5887 { 5888 struct net_device *dev = (struct net_device *)dev_id; 5889 struct stmmac_priv *priv = netdev_priv(dev); 5890 5891 /* Check if adapter is up */ 5892 if (test_bit(STMMAC_DOWN, &priv->state)) 5893 return IRQ_HANDLED; 5894 5895 /* Check if a fatal error happened */ 5896 if (stmmac_safety_feat_interrupt(priv)) 5897 return IRQ_HANDLED; 5898 5899 /* To handle Common interrupts */ 5900 stmmac_common_interrupt(priv); 5901 5902 /* To handle DMA interrupts */ 5903 stmmac_dma_interrupt(priv); 5904 5905 return IRQ_HANDLED; 5906 } 5907 5908 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 5909 { 5910 struct net_device *dev = (struct net_device *)dev_id; 5911 struct stmmac_priv *priv = netdev_priv(dev); 5912 5913 if (unlikely(!dev)) { 5914 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5915 return IRQ_NONE; 5916 } 5917 5918 /* Check if adapter is up */ 5919 if (test_bit(STMMAC_DOWN, &priv->state)) 5920 return IRQ_HANDLED; 5921 5922 /* To handle Common interrupts */ 5923 stmmac_common_interrupt(priv); 5924 5925 return IRQ_HANDLED; 5926 } 5927 5928 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 5929 { 5930 struct net_device *dev = (struct net_device *)dev_id; 5931 struct stmmac_priv *priv = netdev_priv(dev); 5932 5933 if (unlikely(!dev)) { 5934 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5935 return IRQ_NONE; 5936 } 5937 5938 /* Check if adapter is up */ 5939 if (test_bit(STMMAC_DOWN, &priv->state)) 5940 return IRQ_HANDLED; 5941 5942 /* Check if a fatal error happened */ 5943 stmmac_safety_feat_interrupt(priv); 5944 5945 return IRQ_HANDLED; 5946 } 5947 5948 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 5949 { 5950 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 5951 struct stmmac_dma_conf *dma_conf; 5952 int chan = tx_q->queue_index; 5953 struct stmmac_priv *priv; 5954 int status; 5955 5956 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 5957 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 5958 5959 if (unlikely(!data)) { 5960 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5961 return IRQ_NONE; 5962 } 5963 5964 /* Check if adapter is up */ 5965 if (test_bit(STMMAC_DOWN, &priv->state)) 5966 return IRQ_HANDLED; 5967 5968 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 5969 5970 if (unlikely(status & tx_hard_error_bump_tc)) { 5971 /* Try to bump up the dma threshold on this failure */ 5972 stmmac_bump_dma_threshold(priv, chan); 5973 } else if (unlikely(status == tx_hard_error)) { 5974 stmmac_tx_err(priv, chan); 5975 } 5976 5977 return IRQ_HANDLED; 5978 } 5979 5980 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 5981 { 5982 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 5983 struct stmmac_dma_conf *dma_conf; 5984 int chan = rx_q->queue_index; 5985 struct stmmac_priv *priv; 5986 5987 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 5988 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 5989 5990 if (unlikely(!data)) { 5991 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); 5992 return IRQ_NONE; 5993 } 5994 5995 /* Check if adapter is up */ 5996 if (test_bit(STMMAC_DOWN, &priv->state)) 5997 return IRQ_HANDLED; 5998 5999 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6000 6001 return IRQ_HANDLED; 6002 } 6003 6004 #ifdef CONFIG_NET_POLL_CONTROLLER 6005 /* Polling receive - used by NETCONSOLE and other diagnostic tools 6006 * to allow network I/O with interrupts disabled. 6007 */ 6008 static void stmmac_poll_controller(struct net_device *dev) 6009 { 6010 struct stmmac_priv *priv = netdev_priv(dev); 6011 int i; 6012 6013 /* If adapter is down, do nothing */ 6014 if (test_bit(STMMAC_DOWN, &priv->state)) 6015 return; 6016 6017 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) { 6018 for (i = 0; i < priv->plat->rx_queues_to_use; i++) 6019 stmmac_msi_intr_rx(0, &priv->dma_conf.rx_queue[i]); 6020 6021 for (i = 0; i < priv->plat->tx_queues_to_use; i++) 6022 stmmac_msi_intr_tx(0, &priv->dma_conf.tx_queue[i]); 6023 } else { 6024 disable_irq(dev->irq); 6025 stmmac_interrupt(dev->irq, dev); 6026 enable_irq(dev->irq); 6027 } 6028 } 6029 #endif 6030 6031 /** 6032 * stmmac_ioctl - Entry point for the Ioctl 6033 * @dev: Device pointer. 6034 * @rq: An IOCTL specefic structure, that can contain a pointer to 6035 * a proprietary structure used to pass information to the driver. 6036 * @cmd: IOCTL command 6037 * Description: 6038 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 6039 */ 6040 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6041 { 6042 struct stmmac_priv *priv = netdev_priv (dev); 6043 int ret = -EOPNOTSUPP; 6044 6045 if (!netif_running(dev)) 6046 return -EINVAL; 6047 6048 switch (cmd) { 6049 case SIOCGMIIPHY: 6050 case SIOCGMIIREG: 6051 case SIOCSMIIREG: 6052 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 6053 break; 6054 case SIOCSHWTSTAMP: 6055 ret = stmmac_hwtstamp_set(dev, rq); 6056 break; 6057 case SIOCGHWTSTAMP: 6058 ret = stmmac_hwtstamp_get(dev, rq); 6059 break; 6060 default: 6061 break; 6062 } 6063 6064 return ret; 6065 } 6066 6067 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6068 void *cb_priv) 6069 { 6070 struct stmmac_priv *priv = cb_priv; 6071 int ret = -EOPNOTSUPP; 6072 6073 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6074 return ret; 6075 6076 __stmmac_disable_all_queues(priv); 6077 6078 switch (type) { 6079 case TC_SETUP_CLSU32: 6080 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6081 break; 6082 case TC_SETUP_CLSFLOWER: 6083 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6084 break; 6085 default: 6086 break; 6087 } 6088 6089 stmmac_enable_all_queues(priv); 6090 return ret; 6091 } 6092 6093 static LIST_HEAD(stmmac_block_cb_list); 6094 6095 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6096 void *type_data) 6097 { 6098 struct stmmac_priv *priv = netdev_priv(ndev); 6099 6100 switch (type) { 6101 case TC_QUERY_CAPS: 6102 return stmmac_tc_query_caps(priv, priv, type_data); 6103 case TC_SETUP_BLOCK: 6104 return flow_block_cb_setup_simple(type_data, 6105 &stmmac_block_cb_list, 6106 stmmac_setup_tc_block_cb, 6107 priv, priv, true); 6108 case TC_SETUP_QDISC_CBS: 6109 return stmmac_tc_setup_cbs(priv, priv, type_data); 6110 case TC_SETUP_QDISC_TAPRIO: 6111 return stmmac_tc_setup_taprio(priv, priv, type_data); 6112 case TC_SETUP_QDISC_ETF: 6113 return stmmac_tc_setup_etf(priv, priv, type_data); 6114 default: 6115 return -EOPNOTSUPP; 6116 } 6117 } 6118 6119 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6120 struct net_device *sb_dev) 6121 { 6122 int gso = skb_shinfo(skb)->gso_type; 6123 6124 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6125 /* 6126 * There is no way to determine the number of TSO/USO 6127 * capable Queues. Let's use always the Queue 0 6128 * because if TSO/USO is supported then at least this 6129 * one will be capable. 6130 */ 6131 return 0; 6132 } 6133 6134 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6135 } 6136 6137 static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6138 { 6139 struct stmmac_priv *priv = netdev_priv(ndev); 6140 int ret = 0; 6141 6142 ret = pm_runtime_resume_and_get(priv->device); 6143 if (ret < 0) 6144 return ret; 6145 6146 ret = eth_mac_addr(ndev, addr); 6147 if (ret) 6148 goto set_mac_error; 6149 6150 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6151 6152 set_mac_error: 6153 pm_runtime_put(priv->device); 6154 6155 return ret; 6156 } 6157 6158 #ifdef CONFIG_DEBUG_FS 6159 static struct dentry *stmmac_fs_dir; 6160 6161 static void sysfs_display_ring(void *head, int size, int extend_desc, 6162 struct seq_file *seq, dma_addr_t dma_phy_addr) 6163 { 6164 int i; 6165 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6166 struct dma_desc *p = (struct dma_desc *)head; 6167 dma_addr_t dma_addr; 6168 6169 for (i = 0; i < size; i++) { 6170 if (extend_desc) { 6171 dma_addr = dma_phy_addr + i * sizeof(*ep); 6172 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6173 i, &dma_addr, 6174 le32_to_cpu(ep->basic.des0), 6175 le32_to_cpu(ep->basic.des1), 6176 le32_to_cpu(ep->basic.des2), 6177 le32_to_cpu(ep->basic.des3)); 6178 ep++; 6179 } else { 6180 dma_addr = dma_phy_addr + i * sizeof(*p); 6181 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6182 i, &dma_addr, 6183 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6184 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6185 p++; 6186 } 6187 seq_printf(seq, "\n"); 6188 } 6189 } 6190 6191 static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6192 { 6193 struct net_device *dev = seq->private; 6194 struct stmmac_priv *priv = netdev_priv(dev); 6195 u32 rx_count = priv->plat->rx_queues_to_use; 6196 u32 tx_count = priv->plat->tx_queues_to_use; 6197 u32 queue; 6198 6199 if ((dev->flags & IFF_UP) == 0) 6200 return 0; 6201 6202 for (queue = 0; queue < rx_count; queue++) { 6203 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6204 6205 seq_printf(seq, "RX Queue %d:\n", queue); 6206 6207 if (priv->extend_desc) { 6208 seq_printf(seq, "Extended descriptor ring:\n"); 6209 sysfs_display_ring((void *)rx_q->dma_erx, 6210 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6211 } else { 6212 seq_printf(seq, "Descriptor ring:\n"); 6213 sysfs_display_ring((void *)rx_q->dma_rx, 6214 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6215 } 6216 } 6217 6218 for (queue = 0; queue < tx_count; queue++) { 6219 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6220 6221 seq_printf(seq, "TX Queue %d:\n", queue); 6222 6223 if (priv->extend_desc) { 6224 seq_printf(seq, "Extended descriptor ring:\n"); 6225 sysfs_display_ring((void *)tx_q->dma_etx, 6226 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6227 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6228 seq_printf(seq, "Descriptor ring:\n"); 6229 sysfs_display_ring((void *)tx_q->dma_tx, 6230 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6231 } 6232 } 6233 6234 return 0; 6235 } 6236 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6237 6238 static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6239 { 6240 static const char * const dwxgmac_timestamp_source[] = { 6241 "None", 6242 "Internal", 6243 "External", 6244 "Both", 6245 }; 6246 struct net_device *dev = seq->private; 6247 struct stmmac_priv *priv = netdev_priv(dev); 6248 6249 if (!priv->hw_cap_support) { 6250 seq_printf(seq, "DMA HW features not supported\n"); 6251 return 0; 6252 } 6253 6254 seq_printf(seq, "==============================\n"); 6255 seq_printf(seq, "\tDMA HW features\n"); 6256 seq_printf(seq, "==============================\n"); 6257 6258 seq_printf(seq, "\t10/100 Mbps: %s\n", 6259 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6260 seq_printf(seq, "\t1000 Mbps: %s\n", 6261 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6262 seq_printf(seq, "\tHalf duplex: %s\n", 6263 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6264 seq_printf(seq, "\tHash Filter: %s\n", 6265 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6266 if (priv->plat->has_xgmac) 6267 seq_printf(seq, 6268 "\tNumber of Additional MAC address registers: %d\n", 6269 priv->dma_cap.multi_addr); 6270 else 6271 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6272 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6273 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6274 (priv->dma_cap.pcs) ? "Y" : "N"); 6275 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6276 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6277 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6278 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6279 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6280 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6281 seq_printf(seq, "\tRMON module: %s\n", 6282 (priv->dma_cap.rmon) ? "Y" : "N"); 6283 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6284 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6285 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6286 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6287 if (priv->plat->has_xgmac) 6288 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6289 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6290 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6291 (priv->dma_cap.eee) ? "Y" : "N"); 6292 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6293 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6294 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6295 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6296 priv->plat->has_xgmac) { 6297 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6298 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6299 } else { 6300 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6301 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6302 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6303 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6304 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6305 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6306 } 6307 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6308 priv->dma_cap.number_rx_channel); 6309 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6310 priv->dma_cap.number_tx_channel); 6311 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", 6312 priv->dma_cap.number_rx_queues); 6313 seq_printf(seq, "\tNumber of Additional TX queues: %d\n", 6314 priv->dma_cap.number_tx_queues); 6315 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6316 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6317 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6318 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6319 seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz); 6320 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6321 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6322 priv->dma_cap.pps_out_num); 6323 seq_printf(seq, "\tSafety Features: %s\n", 6324 priv->dma_cap.asp ? "Y" : "N"); 6325 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6326 priv->dma_cap.frpsel ? "Y" : "N"); 6327 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6328 priv->dma_cap.host_dma_width); 6329 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6330 priv->dma_cap.rssen ? "Y" : "N"); 6331 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6332 priv->dma_cap.vlhash ? "Y" : "N"); 6333 seq_printf(seq, "\tSplit Header: %s\n", 6334 priv->dma_cap.sphen ? "Y" : "N"); 6335 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6336 priv->dma_cap.vlins ? "Y" : "N"); 6337 seq_printf(seq, "\tDouble VLAN: %s\n", 6338 priv->dma_cap.dvlan ? "Y" : "N"); 6339 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6340 priv->dma_cap.l3l4fnum); 6341 seq_printf(seq, "\tARP Offloading: %s\n", 6342 priv->dma_cap.arpoffsel ? "Y" : "N"); 6343 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6344 priv->dma_cap.estsel ? "Y" : "N"); 6345 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6346 priv->dma_cap.fpesel ? "Y" : "N"); 6347 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6348 priv->dma_cap.tbssel ? "Y" : "N"); 6349 return 0; 6350 } 6351 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6352 6353 /* Use network device events to rename debugfs file entries. 6354 */ 6355 static int stmmac_device_event(struct notifier_block *unused, 6356 unsigned long event, void *ptr) 6357 { 6358 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6359 struct stmmac_priv *priv = netdev_priv(dev); 6360 6361 if (dev->netdev_ops != &stmmac_netdev_ops) 6362 goto done; 6363 6364 switch (event) { 6365 case NETDEV_CHANGENAME: 6366 if (priv->dbgfs_dir) 6367 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, 6368 priv->dbgfs_dir, 6369 stmmac_fs_dir, 6370 dev->name); 6371 break; 6372 } 6373 done: 6374 return NOTIFY_DONE; 6375 } 6376 6377 static struct notifier_block stmmac_notifier = { 6378 .notifier_call = stmmac_device_event, 6379 }; 6380 6381 static void stmmac_init_fs(struct net_device *dev) 6382 { 6383 struct stmmac_priv *priv = netdev_priv(dev); 6384 6385 rtnl_lock(); 6386 6387 /* Create per netdev entries */ 6388 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6389 6390 /* Entry to report DMA RX/TX rings */ 6391 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6392 &stmmac_rings_status_fops); 6393 6394 /* Entry to report the DMA HW features */ 6395 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6396 &stmmac_dma_cap_fops); 6397 6398 rtnl_unlock(); 6399 } 6400 6401 static void stmmac_exit_fs(struct net_device *dev) 6402 { 6403 struct stmmac_priv *priv = netdev_priv(dev); 6404 6405 debugfs_remove_recursive(priv->dbgfs_dir); 6406 } 6407 #endif /* CONFIG_DEBUG_FS */ 6408 6409 static u32 stmmac_vid_crc32_le(__le16 vid_le) 6410 { 6411 unsigned char *data = (unsigned char *)&vid_le; 6412 unsigned char data_byte = 0; 6413 u32 crc = ~0x0; 6414 u32 temp = 0; 6415 int i, bits; 6416 6417 bits = get_bitmask_order(VLAN_VID_MASK); 6418 for (i = 0; i < bits; i++) { 6419 if ((i % 8) == 0) 6420 data_byte = data[i / 8]; 6421 6422 temp = ((crc & 1) ^ data_byte) & 1; 6423 crc >>= 1; 6424 data_byte >>= 1; 6425 6426 if (temp) 6427 crc ^= 0xedb88320; 6428 } 6429 6430 return crc; 6431 } 6432 6433 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6434 { 6435 u32 crc, hash = 0; 6436 __le16 pmatch = 0; 6437 int count = 0; 6438 u16 vid = 0; 6439 6440 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6441 __le16 vid_le = cpu_to_le16(vid); 6442 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6443 hash |= (1 << crc); 6444 count++; 6445 } 6446 6447 if (!priv->dma_cap.vlhash) { 6448 if (count > 2) /* VID = 0 always passes filter */ 6449 return -EOPNOTSUPP; 6450 6451 pmatch = cpu_to_le16(vid); 6452 hash = 0; 6453 } 6454 6455 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6456 } 6457 6458 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6459 { 6460 struct stmmac_priv *priv = netdev_priv(ndev); 6461 bool is_double = false; 6462 int ret; 6463 6464 ret = pm_runtime_resume_and_get(priv->device); 6465 if (ret < 0) 6466 return ret; 6467 6468 if (be16_to_cpu(proto) == ETH_P_8021AD) 6469 is_double = true; 6470 6471 set_bit(vid, priv->active_vlans); 6472 ret = stmmac_vlan_update(priv, is_double); 6473 if (ret) { 6474 clear_bit(vid, priv->active_vlans); 6475 goto err_pm_put; 6476 } 6477 6478 if (priv->hw->num_vlan) { 6479 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6480 if (ret) 6481 goto err_pm_put; 6482 } 6483 err_pm_put: 6484 pm_runtime_put(priv->device); 6485 6486 return ret; 6487 } 6488 6489 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6490 { 6491 struct stmmac_priv *priv = netdev_priv(ndev); 6492 bool is_double = false; 6493 int ret; 6494 6495 ret = pm_runtime_resume_and_get(priv->device); 6496 if (ret < 0) 6497 return ret; 6498 6499 if (be16_to_cpu(proto) == ETH_P_8021AD) 6500 is_double = true; 6501 6502 clear_bit(vid, priv->active_vlans); 6503 6504 if (priv->hw->num_vlan) { 6505 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6506 if (ret) 6507 goto del_vlan_error; 6508 } 6509 6510 ret = stmmac_vlan_update(priv, is_double); 6511 6512 del_vlan_error: 6513 pm_runtime_put(priv->device); 6514 6515 return ret; 6516 } 6517 6518 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6519 { 6520 struct stmmac_priv *priv = netdev_priv(dev); 6521 6522 switch (bpf->command) { 6523 case XDP_SETUP_PROG: 6524 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6525 case XDP_SETUP_XSK_POOL: 6526 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6527 bpf->xsk.queue_id); 6528 default: 6529 return -EOPNOTSUPP; 6530 } 6531 } 6532 6533 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6534 struct xdp_frame **frames, u32 flags) 6535 { 6536 struct stmmac_priv *priv = netdev_priv(dev); 6537 int cpu = smp_processor_id(); 6538 struct netdev_queue *nq; 6539 int i, nxmit = 0; 6540 int queue; 6541 6542 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6543 return -ENETDOWN; 6544 6545 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6546 return -EINVAL; 6547 6548 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6549 nq = netdev_get_tx_queue(priv->dev, queue); 6550 6551 __netif_tx_lock(nq, cpu); 6552 /* Avoids TX time-out as we are sharing with slow path */ 6553 txq_trans_cond_update(nq); 6554 6555 for (i = 0; i < num_frames; i++) { 6556 int res; 6557 6558 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6559 if (res == STMMAC_XDP_CONSUMED) 6560 break; 6561 6562 nxmit++; 6563 } 6564 6565 if (flags & XDP_XMIT_FLUSH) { 6566 stmmac_flush_tx_descriptors(priv, queue); 6567 stmmac_tx_timer_arm(priv, queue); 6568 } 6569 6570 __netif_tx_unlock(nq); 6571 6572 return nxmit; 6573 } 6574 6575 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6576 { 6577 struct stmmac_channel *ch = &priv->channel[queue]; 6578 unsigned long flags; 6579 6580 spin_lock_irqsave(&ch->lock, flags); 6581 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6582 spin_unlock_irqrestore(&ch->lock, flags); 6583 6584 stmmac_stop_rx_dma(priv, queue); 6585 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6586 } 6587 6588 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 6589 { 6590 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6591 struct stmmac_channel *ch = &priv->channel[queue]; 6592 unsigned long flags; 6593 u32 buf_size; 6594 int ret; 6595 6596 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6597 if (ret) { 6598 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 6599 return; 6600 } 6601 6602 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 6603 if (ret) { 6604 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6605 netdev_err(priv->dev, "Failed to init RX desc.\n"); 6606 return; 6607 } 6608 6609 stmmac_reset_rx_queue(priv, queue); 6610 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 6611 6612 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6613 rx_q->dma_rx_phy, rx_q->queue_index); 6614 6615 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * 6616 sizeof(struct dma_desc)); 6617 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6618 rx_q->rx_tail_addr, rx_q->queue_index); 6619 6620 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6621 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6622 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6623 buf_size, 6624 rx_q->queue_index); 6625 } else { 6626 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6627 priv->dma_conf.dma_buf_sz, 6628 rx_q->queue_index); 6629 } 6630 6631 stmmac_start_rx_dma(priv, queue); 6632 6633 spin_lock_irqsave(&ch->lock, flags); 6634 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6635 spin_unlock_irqrestore(&ch->lock, flags); 6636 } 6637 6638 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 6639 { 6640 struct stmmac_channel *ch = &priv->channel[queue]; 6641 unsigned long flags; 6642 6643 spin_lock_irqsave(&ch->lock, flags); 6644 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6645 spin_unlock_irqrestore(&ch->lock, flags); 6646 6647 stmmac_stop_tx_dma(priv, queue); 6648 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6649 } 6650 6651 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 6652 { 6653 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6654 struct stmmac_channel *ch = &priv->channel[queue]; 6655 unsigned long flags; 6656 int ret; 6657 6658 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6659 if (ret) { 6660 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 6661 return; 6662 } 6663 6664 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 6665 if (ret) { 6666 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6667 netdev_err(priv->dev, "Failed to init TX desc.\n"); 6668 return; 6669 } 6670 6671 stmmac_reset_tx_queue(priv, queue); 6672 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 6673 6674 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6675 tx_q->dma_tx_phy, tx_q->queue_index); 6676 6677 if (tx_q->tbs & STMMAC_TBS_AVAIL) 6678 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); 6679 6680 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6681 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6682 tx_q->tx_tail_addr, tx_q->queue_index); 6683 6684 stmmac_start_tx_dma(priv, queue); 6685 6686 spin_lock_irqsave(&ch->lock, flags); 6687 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6688 spin_unlock_irqrestore(&ch->lock, flags); 6689 } 6690 6691 void stmmac_xdp_release(struct net_device *dev) 6692 { 6693 struct stmmac_priv *priv = netdev_priv(dev); 6694 u32 chan; 6695 6696 /* Ensure tx function is not running */ 6697 netif_tx_disable(dev); 6698 6699 /* Disable NAPI process */ 6700 stmmac_disable_all_queues(priv); 6701 6702 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6703 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6704 6705 /* Free the IRQ lines */ 6706 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 6707 6708 /* Stop TX/RX DMA channels */ 6709 stmmac_stop_all_dma(priv); 6710 6711 /* Release and free the Rx/Tx resources */ 6712 free_dma_desc_resources(priv, &priv->dma_conf); 6713 6714 /* Disable the MAC Rx/Tx */ 6715 stmmac_mac_set(priv, priv->ioaddr, false); 6716 6717 /* set trans_start so we don't get spurious 6718 * watchdogs during reset 6719 */ 6720 netif_trans_update(dev); 6721 netif_carrier_off(dev); 6722 } 6723 6724 int stmmac_xdp_open(struct net_device *dev) 6725 { 6726 struct stmmac_priv *priv = netdev_priv(dev); 6727 u32 rx_cnt = priv->plat->rx_queues_to_use; 6728 u32 tx_cnt = priv->plat->tx_queues_to_use; 6729 u32 dma_csr_ch = max(rx_cnt, tx_cnt); 6730 struct stmmac_rx_queue *rx_q; 6731 struct stmmac_tx_queue *tx_q; 6732 u32 buf_size; 6733 bool sph_en; 6734 u32 chan; 6735 int ret; 6736 6737 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 6738 if (ret < 0) { 6739 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 6740 __func__); 6741 goto dma_desc_error; 6742 } 6743 6744 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 6745 if (ret < 0) { 6746 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 6747 __func__); 6748 goto init_error; 6749 } 6750 6751 stmmac_reset_queues_param(priv); 6752 6753 /* DMA CSR Channel configuration */ 6754 for (chan = 0; chan < dma_csr_ch; chan++) { 6755 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 6756 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 6757 } 6758 6759 /* Adjust Split header */ 6760 sph_en = (priv->hw->rx_csum > 0) && priv->sph; 6761 6762 /* DMA RX Channel Configuration */ 6763 for (chan = 0; chan < rx_cnt; chan++) { 6764 rx_q = &priv->dma_conf.rx_queue[chan]; 6765 6766 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6767 rx_q->dma_rx_phy, chan); 6768 6769 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 6770 (rx_q->buf_alloc_num * 6771 sizeof(struct dma_desc)); 6772 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6773 rx_q->rx_tail_addr, chan); 6774 6775 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6776 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6777 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6778 buf_size, 6779 rx_q->queue_index); 6780 } else { 6781 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6782 priv->dma_conf.dma_buf_sz, 6783 rx_q->queue_index); 6784 } 6785 6786 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6787 } 6788 6789 /* DMA TX Channel Configuration */ 6790 for (chan = 0; chan < tx_cnt; chan++) { 6791 tx_q = &priv->dma_conf.tx_queue[chan]; 6792 6793 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6794 tx_q->dma_tx_phy, chan); 6795 6796 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6797 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6798 tx_q->tx_tail_addr, chan); 6799 6800 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6801 tx_q->txtimer.function = stmmac_tx_timer; 6802 } 6803 6804 /* Enable the MAC Rx/Tx */ 6805 stmmac_mac_set(priv, priv->ioaddr, true); 6806 6807 /* Start Rx & Tx DMA Channels */ 6808 stmmac_start_all_dma(priv); 6809 6810 ret = stmmac_request_irq(dev); 6811 if (ret) 6812 goto irq_error; 6813 6814 /* Enable NAPI process*/ 6815 stmmac_enable_all_queues(priv); 6816 netif_carrier_on(dev); 6817 netif_tx_start_all_queues(dev); 6818 stmmac_enable_all_dma_irq(priv); 6819 6820 return 0; 6821 6822 irq_error: 6823 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 6824 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 6825 6826 stmmac_hw_teardown(dev); 6827 init_error: 6828 free_dma_desc_resources(priv, &priv->dma_conf); 6829 dma_desc_error: 6830 return ret; 6831 } 6832 6833 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 6834 { 6835 struct stmmac_priv *priv = netdev_priv(dev); 6836 struct stmmac_rx_queue *rx_q; 6837 struct stmmac_tx_queue *tx_q; 6838 struct stmmac_channel *ch; 6839 6840 if (test_bit(STMMAC_DOWN, &priv->state) || 6841 !netif_carrier_ok(priv->dev)) 6842 return -ENETDOWN; 6843 6844 if (!stmmac_xdp_is_enabled(priv)) 6845 return -EINVAL; 6846 6847 if (queue >= priv->plat->rx_queues_to_use || 6848 queue >= priv->plat->tx_queues_to_use) 6849 return -EINVAL; 6850 6851 rx_q = &priv->dma_conf.rx_queue[queue]; 6852 tx_q = &priv->dma_conf.tx_queue[queue]; 6853 ch = &priv->channel[queue]; 6854 6855 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 6856 return -EINVAL; 6857 6858 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 6859 /* EQoS does not have per-DMA channel SW interrupt, 6860 * so we schedule RX Napi straight-away. 6861 */ 6862 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 6863 __napi_schedule(&ch->rxtx_napi); 6864 } 6865 6866 return 0; 6867 } 6868 6869 static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 6870 { 6871 struct stmmac_priv *priv = netdev_priv(dev); 6872 u32 tx_cnt = priv->plat->tx_queues_to_use; 6873 u32 rx_cnt = priv->plat->rx_queues_to_use; 6874 unsigned int start; 6875 int q; 6876 6877 for (q = 0; q < tx_cnt; q++) { 6878 struct stmmac_txq_stats *txq_stats = &priv->dma_conf.tx_queue[q].txq_stats; 6879 u64 tx_packets; 6880 u64 tx_bytes; 6881 6882 do { 6883 start = u64_stats_fetch_begin(&txq_stats->syncp); 6884 tx_packets = txq_stats->tx_packets; 6885 tx_bytes = txq_stats->tx_bytes; 6886 } while (u64_stats_fetch_retry(&txq_stats->syncp, start)); 6887 6888 stats->tx_packets += tx_packets; 6889 stats->tx_bytes += tx_bytes; 6890 } 6891 6892 for (q = 0; q < rx_cnt; q++) { 6893 struct stmmac_rxq_stats *rxq_stats = &priv->dma_conf.rx_queue[q].rxq_stats; 6894 u64 rx_packets; 6895 u64 rx_bytes; 6896 6897 do { 6898 start = u64_stats_fetch_begin(&rxq_stats->syncp); 6899 rx_packets = rxq_stats->rx_packets; 6900 rx_bytes = rxq_stats->rx_bytes; 6901 } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); 6902 6903 stats->rx_packets += rx_packets; 6904 stats->rx_bytes += rx_bytes; 6905 } 6906 6907 stats->rx_dropped = priv->xstats.rx_dropped; 6908 stats->rx_errors = priv->xstats.rx_errors; 6909 stats->tx_dropped = priv->xstats.tx_dropped; 6910 stats->tx_errors = priv->xstats.tx_errors; 6911 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 6912 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 6913 stats->rx_length_errors = priv->xstats.rx_length; 6914 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 6915 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 6916 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 6917 } 6918 6919 static const struct net_device_ops stmmac_netdev_ops = { 6920 .ndo_open = stmmac_open, 6921 .ndo_start_xmit = stmmac_xmit, 6922 .ndo_stop = stmmac_release, 6923 .ndo_change_mtu = stmmac_change_mtu, 6924 .ndo_fix_features = stmmac_fix_features, 6925 .ndo_set_features = stmmac_set_features, 6926 .ndo_set_rx_mode = stmmac_set_rx_mode, 6927 .ndo_tx_timeout = stmmac_tx_timeout, 6928 .ndo_eth_ioctl = stmmac_ioctl, 6929 .ndo_get_stats64 = stmmac_get_stats64, 6930 .ndo_setup_tc = stmmac_setup_tc, 6931 .ndo_select_queue = stmmac_select_queue, 6932 #ifdef CONFIG_NET_POLL_CONTROLLER 6933 .ndo_poll_controller = stmmac_poll_controller, 6934 #endif 6935 .ndo_set_mac_address = stmmac_set_mac_address, 6936 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 6937 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 6938 .ndo_bpf = stmmac_bpf, 6939 .ndo_xdp_xmit = stmmac_xdp_xmit, 6940 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 6941 }; 6942 6943 static void stmmac_reset_subtask(struct stmmac_priv *priv) 6944 { 6945 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 6946 return; 6947 if (test_bit(STMMAC_DOWN, &priv->state)) 6948 return; 6949 6950 netdev_err(priv->dev, "Reset adapter.\n"); 6951 6952 rtnl_lock(); 6953 netif_trans_update(priv->dev); 6954 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 6955 usleep_range(1000, 2000); 6956 6957 set_bit(STMMAC_DOWN, &priv->state); 6958 dev_close(priv->dev); 6959 dev_open(priv->dev, NULL); 6960 clear_bit(STMMAC_DOWN, &priv->state); 6961 clear_bit(STMMAC_RESETING, &priv->state); 6962 rtnl_unlock(); 6963 } 6964 6965 static void stmmac_service_task(struct work_struct *work) 6966 { 6967 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 6968 service_task); 6969 6970 stmmac_reset_subtask(priv); 6971 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 6972 } 6973 6974 /** 6975 * stmmac_hw_init - Init the MAC device 6976 * @priv: driver private structure 6977 * Description: this function is to configure the MAC device according to 6978 * some platform parameters or the HW capability register. It prepares the 6979 * driver to use either ring or chain modes and to setup either enhanced or 6980 * normal descriptors. 6981 */ 6982 static int stmmac_hw_init(struct stmmac_priv *priv) 6983 { 6984 int ret; 6985 6986 /* dwmac-sun8i only work in chain mode */ 6987 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 6988 chain_mode = 1; 6989 priv->chain_mode = chain_mode; 6990 6991 /* Initialize HW Interface */ 6992 ret = stmmac_hwif_init(priv); 6993 if (ret) 6994 return ret; 6995 6996 /* Get the HW capability (new GMAC newer than 3.50a) */ 6997 priv->hw_cap_support = stmmac_get_hw_features(priv); 6998 if (priv->hw_cap_support) { 6999 dev_info(priv->device, "DMA HW capability register supported\n"); 7000 7001 /* We can override some gmac/dma configuration fields: e.g. 7002 * enh_desc, tx_coe (e.g. that are passed through the 7003 * platform) with the values from the HW capability 7004 * register (if supported). 7005 */ 7006 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7007 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7008 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7009 priv->hw->pmt = priv->plat->pmt; 7010 if (priv->dma_cap.hash_tb_sz) { 7011 priv->hw->multicast_filter_bins = 7012 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7013 priv->hw->mcast_bits_log2 = 7014 ilog2(priv->hw->multicast_filter_bins); 7015 } 7016 7017 /* TXCOE doesn't work in thresh DMA mode */ 7018 if (priv->plat->force_thresh_dma_mode) 7019 priv->plat->tx_coe = 0; 7020 else 7021 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7022 7023 /* In case of GMAC4 rx_coe is from HW cap register. */ 7024 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7025 7026 if (priv->dma_cap.rx_coe_type2) 7027 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7028 else if (priv->dma_cap.rx_coe_type1) 7029 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7030 7031 } else { 7032 dev_info(priv->device, "No HW DMA feature register supported\n"); 7033 } 7034 7035 if (priv->plat->rx_coe) { 7036 priv->hw->rx_csum = priv->plat->rx_coe; 7037 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7038 if (priv->synopsys_id < DWMAC_CORE_4_00) 7039 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7040 } 7041 if (priv->plat->tx_coe) 7042 dev_info(priv->device, "TX Checksum insertion supported\n"); 7043 7044 if (priv->plat->pmt) { 7045 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7046 device_set_wakeup_capable(priv->device, 1); 7047 } 7048 7049 if (priv->dma_cap.tsoen) 7050 dev_info(priv->device, "TSO supported\n"); 7051 7052 priv->hw->vlan_fail_q_en = 7053 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7054 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7055 7056 /* Run HW quirks, if any */ 7057 if (priv->hwif_quirks) { 7058 ret = priv->hwif_quirks(priv); 7059 if (ret) 7060 return ret; 7061 } 7062 7063 /* Rx Watchdog is available in the COREs newer than the 3.40. 7064 * In some case, for example on bugged HW this feature 7065 * has to be disable and this can be done by passing the 7066 * riwt_off field from the platform. 7067 */ 7068 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || 7069 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { 7070 priv->use_riwt = 1; 7071 dev_info(priv->device, 7072 "Enable RX Mitigation via HW Watchdog Timer\n"); 7073 } 7074 7075 return 0; 7076 } 7077 7078 static void stmmac_napi_add(struct net_device *dev) 7079 { 7080 struct stmmac_priv *priv = netdev_priv(dev); 7081 u32 queue, maxq; 7082 7083 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7084 7085 for (queue = 0; queue < maxq; queue++) { 7086 struct stmmac_channel *ch = &priv->channel[queue]; 7087 7088 ch->priv_data = priv; 7089 ch->index = queue; 7090 spin_lock_init(&ch->lock); 7091 7092 if (queue < priv->plat->rx_queues_to_use) { 7093 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7094 } 7095 if (queue < priv->plat->tx_queues_to_use) { 7096 netif_napi_add_tx(dev, &ch->tx_napi, 7097 stmmac_napi_poll_tx); 7098 } 7099 if (queue < priv->plat->rx_queues_to_use && 7100 queue < priv->plat->tx_queues_to_use) { 7101 netif_napi_add(dev, &ch->rxtx_napi, 7102 stmmac_napi_poll_rxtx); 7103 } 7104 } 7105 } 7106 7107 static void stmmac_napi_del(struct net_device *dev) 7108 { 7109 struct stmmac_priv *priv = netdev_priv(dev); 7110 u32 queue, maxq; 7111 7112 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7113 7114 for (queue = 0; queue < maxq; queue++) { 7115 struct stmmac_channel *ch = &priv->channel[queue]; 7116 7117 if (queue < priv->plat->rx_queues_to_use) 7118 netif_napi_del(&ch->rx_napi); 7119 if (queue < priv->plat->tx_queues_to_use) 7120 netif_napi_del(&ch->tx_napi); 7121 if (queue < priv->plat->rx_queues_to_use && 7122 queue < priv->plat->tx_queues_to_use) { 7123 netif_napi_del(&ch->rxtx_napi); 7124 } 7125 } 7126 } 7127 7128 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) 7129 { 7130 struct stmmac_priv *priv = netdev_priv(dev); 7131 int ret = 0, i; 7132 7133 if (netif_running(dev)) 7134 stmmac_release(dev); 7135 7136 stmmac_napi_del(dev); 7137 7138 priv->plat->rx_queues_to_use = rx_cnt; 7139 priv->plat->tx_queues_to_use = tx_cnt; 7140 if (!netif_is_rxfh_configured(dev)) 7141 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7142 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7143 rx_cnt); 7144 7145 stmmac_napi_add(dev); 7146 7147 if (netif_running(dev)) 7148 ret = stmmac_open(dev); 7149 7150 return ret; 7151 } 7152 7153 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7154 { 7155 struct stmmac_priv *priv = netdev_priv(dev); 7156 int ret = 0; 7157 7158 if (netif_running(dev)) 7159 stmmac_release(dev); 7160 7161 priv->dma_conf.dma_rx_size = rx_size; 7162 priv->dma_conf.dma_tx_size = tx_size; 7163 7164 if (netif_running(dev)) 7165 ret = stmmac_open(dev); 7166 7167 return ret; 7168 } 7169 7170 #define SEND_VERIFY_MPAKCET_FMT "Send Verify mPacket lo_state=%d lp_state=%d\n" 7171 static void stmmac_fpe_lp_task(struct work_struct *work) 7172 { 7173 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7174 fpe_task); 7175 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; 7176 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; 7177 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; 7178 bool *hs_enable = &fpe_cfg->hs_enable; 7179 bool *enable = &fpe_cfg->enable; 7180 int retries = 20; 7181 7182 while (retries-- > 0) { 7183 /* Bail out immediately if FPE handshake is OFF */ 7184 if (*lo_state == FPE_STATE_OFF || !*hs_enable) 7185 break; 7186 7187 if (*lo_state == FPE_STATE_ENTERING_ON && 7188 *lp_state == FPE_STATE_ENTERING_ON) { 7189 stmmac_fpe_configure(priv, priv->ioaddr, 7190 priv->plat->tx_queues_to_use, 7191 priv->plat->rx_queues_to_use, 7192 *enable); 7193 7194 netdev_info(priv->dev, "configured FPE\n"); 7195 7196 *lo_state = FPE_STATE_ON; 7197 *lp_state = FPE_STATE_ON; 7198 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); 7199 break; 7200 } 7201 7202 if ((*lo_state == FPE_STATE_CAPABLE || 7203 *lo_state == FPE_STATE_ENTERING_ON) && 7204 *lp_state != FPE_STATE_ON) { 7205 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, 7206 *lo_state, *lp_state); 7207 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 7208 MPACKET_VERIFY); 7209 } 7210 /* Sleep then retry */ 7211 msleep(500); 7212 } 7213 7214 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); 7215 } 7216 7217 void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable) 7218 { 7219 if (priv->plat->fpe_cfg->hs_enable != enable) { 7220 if (enable) { 7221 stmmac_fpe_send_mpacket(priv, priv->ioaddr, 7222 MPACKET_VERIFY); 7223 } else { 7224 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; 7225 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; 7226 } 7227 7228 priv->plat->fpe_cfg->hs_enable = enable; 7229 } 7230 } 7231 7232 static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7233 { 7234 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7235 struct dma_desc *desc_contains_ts = ctx->desc; 7236 struct stmmac_priv *priv = ctx->priv; 7237 struct dma_desc *ndesc = ctx->ndesc; 7238 struct dma_desc *desc = ctx->desc; 7239 u64 ns = 0; 7240 7241 if (!priv->hwts_rx_en) 7242 return -ENODATA; 7243 7244 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7245 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) 7246 desc_contains_ts = ndesc; 7247 7248 /* Check if timestamp is available */ 7249 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7250 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7251 ns -= priv->plat->cdc_error_adj; 7252 *timestamp = ns_to_ktime(ns); 7253 return 0; 7254 } 7255 7256 return -ENODATA; 7257 } 7258 7259 static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7260 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7261 }; 7262 7263 /** 7264 * stmmac_dvr_probe 7265 * @device: device pointer 7266 * @plat_dat: platform data pointer 7267 * @res: stmmac resource pointer 7268 * Description: this is the main probe function used to 7269 * call the alloc_etherdev, allocate the priv structure. 7270 * Return: 7271 * returns 0 on success, otherwise errno. 7272 */ 7273 int stmmac_dvr_probe(struct device *device, 7274 struct plat_stmmacenet_data *plat_dat, 7275 struct stmmac_resources *res) 7276 { 7277 struct net_device *ndev = NULL; 7278 struct stmmac_priv *priv; 7279 u32 rxq; 7280 int i, ret = 0; 7281 7282 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7283 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7284 if (!ndev) 7285 return -ENOMEM; 7286 7287 SET_NETDEV_DEV(ndev, device); 7288 7289 priv = netdev_priv(ndev); 7290 priv->device = device; 7291 priv->dev = ndev; 7292 7293 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7294 u64_stats_init(&priv->dma_conf.rx_queue[i].rxq_stats.syncp); 7295 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7296 u64_stats_init(&priv->dma_conf.tx_queue[i].txq_stats.syncp); 7297 7298 stmmac_set_ethtool_ops(ndev); 7299 priv->pause = pause; 7300 priv->plat = plat_dat; 7301 priv->ioaddr = res->addr; 7302 priv->dev->base_addr = (unsigned long)res->addr; 7303 priv->plat->dma_cfg->multi_msi_en = 7304 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7305 7306 priv->dev->irq = res->irq; 7307 priv->wol_irq = res->wol_irq; 7308 priv->lpi_irq = res->lpi_irq; 7309 priv->sfty_ce_irq = res->sfty_ce_irq; 7310 priv->sfty_ue_irq = res->sfty_ue_irq; 7311 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7312 priv->rx_irq[i] = res->rx_irq[i]; 7313 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7314 priv->tx_irq[i] = res->tx_irq[i]; 7315 7316 if (!is_zero_ether_addr(res->mac)) 7317 eth_hw_addr_set(priv->dev, res->mac); 7318 7319 dev_set_drvdata(device, priv->dev); 7320 7321 /* Verify driver arguments */ 7322 stmmac_verify_args(); 7323 7324 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7325 if (!priv->af_xdp_zc_qps) 7326 return -ENOMEM; 7327 7328 /* Allocate workqueue */ 7329 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7330 if (!priv->wq) { 7331 dev_err(priv->device, "failed to create workqueue\n"); 7332 ret = -ENOMEM; 7333 goto error_wq_init; 7334 } 7335 7336 INIT_WORK(&priv->service_task, stmmac_service_task); 7337 7338 /* Initialize Link Partner FPE workqueue */ 7339 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); 7340 7341 /* Override with kernel parameters if supplied XXX CRS XXX 7342 * this needs to have multiple instances 7343 */ 7344 if ((phyaddr >= 0) && (phyaddr <= 31)) 7345 priv->plat->phy_addr = phyaddr; 7346 7347 if (priv->plat->stmmac_rst) { 7348 ret = reset_control_assert(priv->plat->stmmac_rst); 7349 reset_control_deassert(priv->plat->stmmac_rst); 7350 /* Some reset controllers have only reset callback instead of 7351 * assert + deassert callbacks pair. 7352 */ 7353 if (ret == -ENOTSUPP) 7354 reset_control_reset(priv->plat->stmmac_rst); 7355 } 7356 7357 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7358 if (ret == -ENOTSUPP) 7359 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7360 ERR_PTR(ret)); 7361 7362 /* Init MAC and get the capabilities */ 7363 ret = stmmac_hw_init(priv); 7364 if (ret) 7365 goto error_hw_init; 7366 7367 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7368 */ 7369 if (priv->synopsys_id < DWMAC_CORE_5_20) 7370 priv->plat->dma_cfg->dche = false; 7371 7372 stmmac_check_ether_addr(priv); 7373 7374 ndev->netdev_ops = &stmmac_netdev_ops; 7375 7376 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7377 7378 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7379 NETIF_F_RXCSUM; 7380 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7381 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7382 7383 ret = stmmac_tc_init(priv, priv); 7384 if (!ret) { 7385 ndev->hw_features |= NETIF_F_HW_TC; 7386 } 7387 7388 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 7389 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 7390 if (priv->plat->has_gmac4) 7391 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 7392 priv->tso = true; 7393 dev_info(priv->device, "TSO feature enabled\n"); 7394 } 7395 7396 if (priv->dma_cap.sphen && 7397 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7398 ndev->hw_features |= NETIF_F_GRO; 7399 priv->sph_cap = true; 7400 priv->sph = priv->sph_cap; 7401 dev_info(priv->device, "SPH feature enabled\n"); 7402 } 7403 7404 /* Ideally our host DMA address width is the same as for the 7405 * device. However, it may differ and then we have to use our 7406 * host DMA width for allocation and the device DMA width for 7407 * register handling. 7408 */ 7409 if (priv->plat->host_dma_width) 7410 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7411 else 7412 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7413 7414 if (priv->dma_cap.host_dma_width) { 7415 ret = dma_set_mask_and_coherent(device, 7416 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7417 if (!ret) { 7418 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7419 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7420 7421 /* 7422 * If more than 32 bits can be addressed, make sure to 7423 * enable enhanced addressing mode. 7424 */ 7425 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7426 priv->plat->dma_cfg->eame = true; 7427 } else { 7428 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7429 if (ret) { 7430 dev_err(priv->device, "Failed to set DMA Mask\n"); 7431 goto error_hw_init; 7432 } 7433 7434 priv->dma_cap.host_dma_width = 32; 7435 } 7436 } 7437 7438 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7439 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7440 #ifdef STMMAC_VLAN_TAG_USED 7441 /* Both mac100 and gmac support receive VLAN tag detection */ 7442 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7443 if (priv->dma_cap.vlhash) { 7444 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7445 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7446 } 7447 if (priv->dma_cap.vlins) { 7448 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7449 if (priv->dma_cap.dvlan) 7450 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; 7451 } 7452 #endif 7453 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7454 7455 priv->xstats.threshold = tc; 7456 7457 /* Initialize RSS */ 7458 rxq = priv->plat->rx_queues_to_use; 7459 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7460 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7461 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7462 7463 if (priv->dma_cap.rssen && priv->plat->rss_en) 7464 ndev->features |= NETIF_F_RXHASH; 7465 7466 ndev->vlan_features |= ndev->features; 7467 /* TSO doesn't work on VLANs yet */ 7468 ndev->vlan_features &= ~NETIF_F_TSO; 7469 7470 /* MTU range: 46 - hw-specific max */ 7471 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 7472 if (priv->plat->has_xgmac) 7473 ndev->max_mtu = XGMAC_JUMBO_LEN; 7474 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) 7475 ndev->max_mtu = JUMBO_LEN; 7476 else 7477 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 7478 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu 7479 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. 7480 */ 7481 if ((priv->plat->maxmtu < ndev->max_mtu) && 7482 (priv->plat->maxmtu >= ndev->min_mtu)) 7483 ndev->max_mtu = priv->plat->maxmtu; 7484 else if (priv->plat->maxmtu < ndev->min_mtu) 7485 dev_warn(priv->device, 7486 "%s: warning: maxmtu having invalid value (%d)\n", 7487 __func__, priv->plat->maxmtu); 7488 7489 if (flow_ctrl) 7490 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ 7491 7492 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 7493 7494 /* Setup channels NAPI */ 7495 stmmac_napi_add(ndev); 7496 7497 mutex_init(&priv->lock); 7498 7499 /* If a specific clk_csr value is passed from the platform 7500 * this means that the CSR Clock Range selection cannot be 7501 * changed at run-time and it is fixed. Viceversa the driver'll try to 7502 * set the MDC clock dynamically according to the csr actual 7503 * clock input. 7504 */ 7505 if (priv->plat->clk_csr >= 0) 7506 priv->clk_csr = priv->plat->clk_csr; 7507 else 7508 stmmac_clk_csr_set(priv); 7509 7510 stmmac_check_pcs_mode(priv); 7511 7512 pm_runtime_get_noresume(device); 7513 pm_runtime_set_active(device); 7514 if (!pm_runtime_enabled(device)) 7515 pm_runtime_enable(device); 7516 7517 if (priv->hw->pcs != STMMAC_PCS_TBI && 7518 priv->hw->pcs != STMMAC_PCS_RTBI) { 7519 /* MDIO bus Registration */ 7520 ret = stmmac_mdio_register(ndev); 7521 if (ret < 0) { 7522 dev_err_probe(priv->device, ret, 7523 "%s: MDIO bus (id: %d) registration failed\n", 7524 __func__, priv->plat->bus_id); 7525 goto error_mdio_register; 7526 } 7527 } 7528 7529 if (priv->plat->speed_mode_2500) 7530 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); 7531 7532 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { 7533 ret = stmmac_xpcs_setup(priv->mii); 7534 if (ret) 7535 goto error_xpcs_setup; 7536 } 7537 7538 ret = stmmac_phy_setup(priv); 7539 if (ret) { 7540 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 7541 goto error_phy_setup; 7542 } 7543 7544 ret = register_netdev(ndev); 7545 if (ret) { 7546 dev_err(priv->device, "%s: ERROR %i registering the device\n", 7547 __func__, ret); 7548 goto error_netdev_register; 7549 } 7550 7551 #ifdef CONFIG_DEBUG_FS 7552 stmmac_init_fs(ndev); 7553 #endif 7554 7555 if (priv->plat->dump_debug_regs) 7556 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 7557 7558 /* Let pm_runtime_put() disable the clocks. 7559 * If CONFIG_PM is not enabled, the clocks will stay powered. 7560 */ 7561 pm_runtime_put(device); 7562 7563 return ret; 7564 7565 error_netdev_register: 7566 phylink_destroy(priv->phylink); 7567 error_xpcs_setup: 7568 error_phy_setup: 7569 if (priv->hw->pcs != STMMAC_PCS_TBI && 7570 priv->hw->pcs != STMMAC_PCS_RTBI) 7571 stmmac_mdio_unregister(ndev); 7572 error_mdio_register: 7573 stmmac_napi_del(ndev); 7574 error_hw_init: 7575 destroy_workqueue(priv->wq); 7576 error_wq_init: 7577 bitmap_free(priv->af_xdp_zc_qps); 7578 7579 return ret; 7580 } 7581 EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 7582 7583 /** 7584 * stmmac_dvr_remove 7585 * @dev: device pointer 7586 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 7587 * changes the link status, releases the DMA descriptor rings. 7588 */ 7589 void stmmac_dvr_remove(struct device *dev) 7590 { 7591 struct net_device *ndev = dev_get_drvdata(dev); 7592 struct stmmac_priv *priv = netdev_priv(ndev); 7593 7594 netdev_info(priv->dev, "%s: removing driver", __func__); 7595 7596 pm_runtime_get_sync(dev); 7597 7598 stmmac_stop_all_dma(priv); 7599 stmmac_mac_set(priv, priv->ioaddr, false); 7600 netif_carrier_off(ndev); 7601 unregister_netdev(ndev); 7602 7603 #ifdef CONFIG_DEBUG_FS 7604 stmmac_exit_fs(ndev); 7605 #endif 7606 phylink_destroy(priv->phylink); 7607 if (priv->plat->stmmac_rst) 7608 reset_control_assert(priv->plat->stmmac_rst); 7609 reset_control_assert(priv->plat->stmmac_ahb_rst); 7610 if (priv->hw->pcs != STMMAC_PCS_TBI && 7611 priv->hw->pcs != STMMAC_PCS_RTBI) 7612 stmmac_mdio_unregister(ndev); 7613 destroy_workqueue(priv->wq); 7614 mutex_destroy(&priv->lock); 7615 bitmap_free(priv->af_xdp_zc_qps); 7616 7617 pm_runtime_disable(dev); 7618 pm_runtime_put_noidle(dev); 7619 } 7620 EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 7621 7622 /** 7623 * stmmac_suspend - suspend callback 7624 * @dev: device pointer 7625 * Description: this is the function to suspend the device and it is called 7626 * by the platform driver to stop the network queue, release the resources, 7627 * program the PMT register (for WoL), clean and release driver resources. 7628 */ 7629 int stmmac_suspend(struct device *dev) 7630 { 7631 struct net_device *ndev = dev_get_drvdata(dev); 7632 struct stmmac_priv *priv = netdev_priv(ndev); 7633 u32 chan; 7634 7635 if (!ndev || !netif_running(ndev)) 7636 return 0; 7637 7638 mutex_lock(&priv->lock); 7639 7640 netif_device_detach(ndev); 7641 7642 stmmac_disable_all_queues(priv); 7643 7644 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7645 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7646 7647 if (priv->eee_enabled) { 7648 priv->tx_path_in_lpi_mode = false; 7649 del_timer_sync(&priv->eee_ctrl_timer); 7650 } 7651 7652 /* Stop TX/RX DMA */ 7653 stmmac_stop_all_dma(priv); 7654 7655 if (priv->plat->serdes_powerdown) 7656 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 7657 7658 /* Enable Power down mode by programming the PMT regs */ 7659 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7660 stmmac_pmt(priv, priv->hw, priv->wolopts); 7661 priv->irq_wake = 1; 7662 } else { 7663 stmmac_mac_set(priv, priv->ioaddr, false); 7664 pinctrl_pm_select_sleep_state(priv->device); 7665 } 7666 7667 mutex_unlock(&priv->lock); 7668 7669 rtnl_lock(); 7670 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7671 phylink_suspend(priv->phylink, true); 7672 } else { 7673 if (device_may_wakeup(priv->device)) 7674 phylink_speed_down(priv->phylink, false); 7675 phylink_suspend(priv->phylink, false); 7676 } 7677 rtnl_unlock(); 7678 7679 if (priv->dma_cap.fpesel) { 7680 /* Disable FPE */ 7681 stmmac_fpe_configure(priv, priv->ioaddr, 7682 priv->plat->tx_queues_to_use, 7683 priv->plat->rx_queues_to_use, false); 7684 7685 stmmac_fpe_handshake(priv, false); 7686 stmmac_fpe_stop_wq(priv); 7687 } 7688 7689 priv->speed = SPEED_UNKNOWN; 7690 return 0; 7691 } 7692 EXPORT_SYMBOL_GPL(stmmac_suspend); 7693 7694 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 7695 { 7696 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 7697 7698 rx_q->cur_rx = 0; 7699 rx_q->dirty_rx = 0; 7700 } 7701 7702 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 7703 { 7704 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 7705 7706 tx_q->cur_tx = 0; 7707 tx_q->dirty_tx = 0; 7708 tx_q->mss = 0; 7709 7710 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 7711 } 7712 7713 /** 7714 * stmmac_reset_queues_param - reset queue parameters 7715 * @priv: device pointer 7716 */ 7717 static void stmmac_reset_queues_param(struct stmmac_priv *priv) 7718 { 7719 u32 rx_cnt = priv->plat->rx_queues_to_use; 7720 u32 tx_cnt = priv->plat->tx_queues_to_use; 7721 u32 queue; 7722 7723 for (queue = 0; queue < rx_cnt; queue++) 7724 stmmac_reset_rx_queue(priv, queue); 7725 7726 for (queue = 0; queue < tx_cnt; queue++) 7727 stmmac_reset_tx_queue(priv, queue); 7728 } 7729 7730 /** 7731 * stmmac_resume - resume callback 7732 * @dev: device pointer 7733 * Description: when resume this function is invoked to setup the DMA and CORE 7734 * in a usable state. 7735 */ 7736 int stmmac_resume(struct device *dev) 7737 { 7738 struct net_device *ndev = dev_get_drvdata(dev); 7739 struct stmmac_priv *priv = netdev_priv(ndev); 7740 int ret; 7741 7742 if (!netif_running(ndev)) 7743 return 0; 7744 7745 /* Power Down bit, into the PM register, is cleared 7746 * automatically as soon as a magic packet or a Wake-up frame 7747 * is received. Anyway, it's better to manually clear 7748 * this bit because it can generate problems while resuming 7749 * from another devices (e.g. serial console). 7750 */ 7751 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7752 mutex_lock(&priv->lock); 7753 stmmac_pmt(priv, priv->hw, 0); 7754 mutex_unlock(&priv->lock); 7755 priv->irq_wake = 0; 7756 } else { 7757 pinctrl_pm_select_default_state(priv->device); 7758 /* reset the phy so that it's ready */ 7759 if (priv->mii) 7760 stmmac_mdio_reset(priv->mii); 7761 } 7762 7763 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 7764 priv->plat->serdes_powerup) { 7765 ret = priv->plat->serdes_powerup(ndev, 7766 priv->plat->bsp_priv); 7767 7768 if (ret < 0) 7769 return ret; 7770 } 7771 7772 rtnl_lock(); 7773 if (device_may_wakeup(priv->device) && priv->plat->pmt) { 7774 phylink_resume(priv->phylink); 7775 } else { 7776 phylink_resume(priv->phylink); 7777 if (device_may_wakeup(priv->device)) 7778 phylink_speed_up(priv->phylink); 7779 } 7780 rtnl_unlock(); 7781 7782 rtnl_lock(); 7783 mutex_lock(&priv->lock); 7784 7785 stmmac_reset_queues_param(priv); 7786 7787 stmmac_free_tx_skbufs(priv); 7788 stmmac_clear_descriptors(priv, &priv->dma_conf); 7789 7790 stmmac_hw_setup(ndev, false); 7791 stmmac_init_coalesce(priv); 7792 stmmac_set_rx_mode(ndev); 7793 7794 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); 7795 7796 stmmac_enable_all_queues(priv); 7797 stmmac_enable_all_dma_irq(priv); 7798 7799 mutex_unlock(&priv->lock); 7800 rtnl_unlock(); 7801 7802 netif_device_attach(ndev); 7803 7804 return 0; 7805 } 7806 EXPORT_SYMBOL_GPL(stmmac_resume); 7807 7808 #ifndef MODULE 7809 static int __init stmmac_cmdline_opt(char *str) 7810 { 7811 char *opt; 7812 7813 if (!str || !*str) 7814 return 1; 7815 while ((opt = strsep(&str, ",")) != NULL) { 7816 if (!strncmp(opt, "debug:", 6)) { 7817 if (kstrtoint(opt + 6, 0, &debug)) 7818 goto err; 7819 } else if (!strncmp(opt, "phyaddr:", 8)) { 7820 if (kstrtoint(opt + 8, 0, &phyaddr)) 7821 goto err; 7822 } else if (!strncmp(opt, "buf_sz:", 7)) { 7823 if (kstrtoint(opt + 7, 0, &buf_sz)) 7824 goto err; 7825 } else if (!strncmp(opt, "tc:", 3)) { 7826 if (kstrtoint(opt + 3, 0, &tc)) 7827 goto err; 7828 } else if (!strncmp(opt, "watchdog:", 9)) { 7829 if (kstrtoint(opt + 9, 0, &watchdog)) 7830 goto err; 7831 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 7832 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 7833 goto err; 7834 } else if (!strncmp(opt, "pause:", 6)) { 7835 if (kstrtoint(opt + 6, 0, &pause)) 7836 goto err; 7837 } else if (!strncmp(opt, "eee_timer:", 10)) { 7838 if (kstrtoint(opt + 10, 0, &eee_timer)) 7839 goto err; 7840 } else if (!strncmp(opt, "chain_mode:", 11)) { 7841 if (kstrtoint(opt + 11, 0, &chain_mode)) 7842 goto err; 7843 } 7844 } 7845 return 1; 7846 7847 err: 7848 pr_err("%s: ERROR broken module parameter conversion", __func__); 7849 return 1; 7850 } 7851 7852 __setup("stmmaceth=", stmmac_cmdline_opt); 7853 #endif /* MODULE */ 7854 7855 static int __init stmmac_init(void) 7856 { 7857 #ifdef CONFIG_DEBUG_FS 7858 /* Create debugfs main directory if it doesn't exist yet */ 7859 if (!stmmac_fs_dir) 7860 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 7861 register_netdevice_notifier(&stmmac_notifier); 7862 #endif 7863 7864 return 0; 7865 } 7866 7867 static void __exit stmmac_exit(void) 7868 { 7869 #ifdef CONFIG_DEBUG_FS 7870 unregister_netdevice_notifier(&stmmac_notifier); 7871 debugfs_remove_recursive(stmmac_fs_dir); 7872 #endif 7873 } 7874 7875 module_init(stmmac_init) 7876 module_exit(stmmac_exit) 7877 7878 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 7879 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 7880 MODULE_LICENSE("GPL"); 7881