Lines Matching +full:asp +full:- +full:v2
1 // SPDX-License-Identifier: GPL-2.0-only
3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
6 Copyright(C) 2007-2011 STMicroelectronics Ltd
29 #include <linux/dma-mapping.h>
55 * with fine resolution and binary rollover. This avoid non-monotonic behavior
62 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1)
70 static int debug = -1;
72 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
74 static int phyaddr = -1;
78 #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4)
79 #define STMMAC_RX_THRESH(x) ((x)->dma_conf.dma_rx_size / 4)
155 ret = clk_prepare_enable(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
158 ret = clk_prepare_enable(priv->plat->pclk); in stmmac_bus_clks_config()
160 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
163 if (priv->plat->clks_config) { in stmmac_bus_clks_config()
164 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); in stmmac_bus_clks_config()
166 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
167 clk_disable_unprepare(priv->plat->pclk); in stmmac_bus_clks_config()
172 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
173 clk_disable_unprepare(priv->plat->pclk); in stmmac_bus_clks_config()
174 if (priv->plat->clks_config) in stmmac_bus_clks_config()
175 priv->plat->clks_config(priv->plat->bsp_priv, enabled); in stmmac_bus_clks_config()
183 * stmmac_verify_args - verify the driver parameters.
205 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in __stmmac_disable_all_queues()
206 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; in __stmmac_disable_all_queues()
211 struct stmmac_channel *ch = &priv->channel[queue]; in __stmmac_disable_all_queues()
214 test_bit(queue, priv->af_xdp_zc_qps)) { in __stmmac_disable_all_queues()
215 napi_disable(&ch->rxtx_napi); in __stmmac_disable_all_queues()
220 napi_disable(&ch->rx_napi); in __stmmac_disable_all_queues()
222 napi_disable(&ch->tx_napi); in __stmmac_disable_all_queues()
227 * stmmac_disable_all_queues - Disable all queues
232 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in stmmac_disable_all_queues()
238 rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_disable_all_queues()
239 if (rx_q->xsk_pool) { in stmmac_disable_all_queues()
249 * stmmac_enable_all_queues - Enable all queues
254 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in stmmac_enable_all_queues()
255 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; in stmmac_enable_all_queues()
260 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_all_queues()
263 test_bit(queue, priv->af_xdp_zc_qps)) { in stmmac_enable_all_queues()
264 napi_enable(&ch->rxtx_napi); in stmmac_enable_all_queues()
269 napi_enable(&ch->rx_napi); in stmmac_enable_all_queues()
271 napi_enable(&ch->tx_napi); in stmmac_enable_all_queues()
277 if (!test_bit(STMMAC_DOWN, &priv->state) && in stmmac_service_event_schedule()
278 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) in stmmac_service_event_schedule()
279 queue_work(priv->wq, &priv->service_task); in stmmac_service_event_schedule()
284 netif_carrier_off(priv->dev); in stmmac_global_err()
285 set_bit(STMMAC_RESET_REQUESTED, &priv->state); in stmmac_global_err()
290 * stmmac_clk_csr_set - dynamically set the MDC clock
297 * changed at run-time and it is fixed (as reported in the driver
305 clk_rate = clk_get_rate(priv->plat->stmmac_clk); in stmmac_clk_csr_set()
314 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { in stmmac_clk_csr_set()
316 priv->clk_csr = STMMAC_CSR_20_35M; in stmmac_clk_csr_set()
318 priv->clk_csr = STMMAC_CSR_35_60M; in stmmac_clk_csr_set()
320 priv->clk_csr = STMMAC_CSR_60_100M; in stmmac_clk_csr_set()
322 priv->clk_csr = STMMAC_CSR_100_150M; in stmmac_clk_csr_set()
324 priv->clk_csr = STMMAC_CSR_150_250M; in stmmac_clk_csr_set()
326 priv->clk_csr = STMMAC_CSR_250_300M; in stmmac_clk_csr_set()
329 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { in stmmac_clk_csr_set()
331 priv->clk_csr = 0x03; in stmmac_clk_csr_set()
333 priv->clk_csr = 0x02; in stmmac_clk_csr_set()
335 priv->clk_csr = 0x01; in stmmac_clk_csr_set()
337 priv->clk_csr = 0; in stmmac_clk_csr_set()
340 if (priv->plat->has_xgmac) { in stmmac_clk_csr_set()
342 priv->clk_csr = 0x5; in stmmac_clk_csr_set()
344 priv->clk_csr = 0x4; in stmmac_clk_csr_set()
346 priv->clk_csr = 0x3; in stmmac_clk_csr_set()
348 priv->clk_csr = 0x2; in stmmac_clk_csr_set()
350 priv->clk_csr = 0x1; in stmmac_clk_csr_set()
352 priv->clk_csr = 0x0; in stmmac_clk_csr_set()
364 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_tx_avail()
367 if (tx_q->dirty_tx > tx_q->cur_tx) in stmmac_tx_avail()
368 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; in stmmac_tx_avail()
370 avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; in stmmac_tx_avail()
376 * stmmac_rx_dirty - Get RX queue dirty
382 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_rx_dirty()
385 if (rx_q->dirty_rx <= rx_q->cur_rx) in stmmac_rx_dirty()
386 dirty = rx_q->cur_rx - rx_q->dirty_rx; in stmmac_rx_dirty()
388 dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; in stmmac_rx_dirty()
398 priv->eee_sw_timer_en = en ? 0 : 1; in stmmac_lpi_entry_timer_config()
399 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; in stmmac_lpi_entry_timer_config()
400 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); in stmmac_lpi_entry_timer_config()
404 * stmmac_enable_eee_mode - check and enter in LPI mode
411 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_enable_eee_mode()
416 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_enable_eee_mode()
418 if (tx_q->dirty_tx != tx_q->cur_tx) in stmmac_enable_eee_mode()
419 return -EBUSY; /* still unfinished work */ in stmmac_enable_eee_mode()
423 if (!priv->tx_path_in_lpi_mode) in stmmac_enable_eee_mode()
424 stmmac_set_eee_mode(priv, priv->hw, in stmmac_enable_eee_mode()
425 priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING); in stmmac_enable_eee_mode()
430 * stmmac_disable_eee_mode - disable and exit from LPI mode
437 if (!priv->eee_sw_timer_en) { in stmmac_disable_eee_mode()
442 stmmac_reset_eee_mode(priv, priv->hw); in stmmac_disable_eee_mode()
443 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_disable_eee_mode()
444 priv->tx_path_in_lpi_mode = false; in stmmac_disable_eee_mode()
448 * stmmac_eee_ctrl_timer - EEE TX SW timer.
459 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_eee_ctrl_timer()
463 * stmmac_eee_init - init EEE
472 int eee_tw_timer = priv->eee_tw_timer; in stmmac_eee_init()
477 if (priv->hw->pcs == STMMAC_PCS_TBI || in stmmac_eee_init()
478 priv->hw->pcs == STMMAC_PCS_RTBI) in stmmac_eee_init()
482 if (!priv->dma_cap.eee) in stmmac_eee_init()
485 mutex_lock(&priv->lock); in stmmac_eee_init()
488 if (!priv->eee_active) { in stmmac_eee_init()
489 if (priv->eee_enabled) { in stmmac_eee_init()
490 netdev_dbg(priv->dev, "disable EEE\n"); in stmmac_eee_init()
492 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_eee_init()
493 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); in stmmac_eee_init()
494 if (priv->hw->xpcs) in stmmac_eee_init()
495 xpcs_config_eee(priv->hw->xpcs, in stmmac_eee_init()
496 priv->plat->mult_fact_100ns, in stmmac_eee_init()
499 mutex_unlock(&priv->lock); in stmmac_eee_init()
503 if (priv->eee_active && !priv->eee_enabled) { in stmmac_eee_init()
504 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); in stmmac_eee_init()
505 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, in stmmac_eee_init()
507 if (priv->hw->xpcs) in stmmac_eee_init()
508 xpcs_config_eee(priv->hw->xpcs, in stmmac_eee_init()
509 priv->plat->mult_fact_100ns, in stmmac_eee_init()
513 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { in stmmac_eee_init()
514 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_eee_init()
515 priv->tx_path_in_lpi_mode = false; in stmmac_eee_init()
519 mod_timer(&priv->eee_ctrl_timer, in stmmac_eee_init()
520 STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_eee_init()
523 mutex_unlock(&priv->lock); in stmmac_eee_init()
524 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); in stmmac_eee_init()
528 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
543 if (!priv->hwts_tx_en) in stmmac_get_tx_hwtstamp()
547 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) in stmmac_get_tx_hwtstamp()
552 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); in stmmac_get_tx_hwtstamp()
554 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { in stmmac_get_tx_hwtstamp()
559 ns -= priv->plat->cdc_error_adj; in stmmac_get_tx_hwtstamp()
564 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); in stmmac_get_tx_hwtstamp()
570 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
586 if (!priv->hwts_rx_en) in stmmac_get_rx_hwtstamp()
589 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) in stmmac_get_rx_hwtstamp()
593 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { in stmmac_get_rx_hwtstamp()
594 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); in stmmac_get_rx_hwtstamp()
596 ns -= priv->plat->cdc_error_adj; in stmmac_get_rx_hwtstamp()
598 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); in stmmac_get_rx_hwtstamp()
601 shhwtstamp->hwtstamp = ns_to_ktime(ns); in stmmac_get_rx_hwtstamp()
603 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); in stmmac_get_rx_hwtstamp()
608 * stmmac_hwtstamp_set - control hardware timestamping.
616 * 0 on success and an appropriate -ve integer on failure.
631 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { in stmmac_hwtstamp_set()
632 netdev_alert(priv->dev, "No support for HW time stamping\n"); in stmmac_hwtstamp_set()
633 priv->hwts_tx_en = 0; in stmmac_hwtstamp_set()
634 priv->hwts_rx_en = 0; in stmmac_hwtstamp_set()
636 return -EOPNOTSUPP; in stmmac_hwtstamp_set()
639 if (copy_from_user(&config, ifr->ifr_data, in stmmac_hwtstamp_set()
641 return -EFAULT; in stmmac_hwtstamp_set()
643 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", in stmmac_hwtstamp_set()
648 return -ERANGE; in stmmac_hwtstamp_set()
650 if (priv->adv_ts) { in stmmac_hwtstamp_set()
693 /* PTP v2, UDP, any kind of event packet */ in stmmac_hwtstamp_set()
704 /* PTP v2, UDP, Sync packet */ in stmmac_hwtstamp_set()
715 /* PTP v2, UDP, Delay_req packet */ in stmmac_hwtstamp_set()
727 /* PTP v2/802.AS1 any layer, any kind of event packet */ in stmmac_hwtstamp_set()
731 if (priv->synopsys_id < DWMAC_CORE_4_10) in stmmac_hwtstamp_set()
739 /* PTP v2/802.AS1, any layer, Sync packet */ in stmmac_hwtstamp_set()
751 /* PTP v2/802.AS1, any layer, Delay_req packet */ in stmmac_hwtstamp_set()
771 return -ERANGE; in stmmac_hwtstamp_set()
784 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); in stmmac_hwtstamp_set()
785 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; in stmmac_hwtstamp_set()
787 priv->systime_flags = STMMAC_HWTS_ACTIVE; in stmmac_hwtstamp_set()
789 if (priv->hwts_tx_en || priv->hwts_rx_en) { in stmmac_hwtstamp_set()
790 priv->systime_flags |= tstamp_all | ptp_v2 | in stmmac_hwtstamp_set()
796 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); in stmmac_hwtstamp_set()
798 memcpy(&priv->tstamp_config, &config, sizeof(config)); in stmmac_hwtstamp_set()
800 return copy_to_user(ifr->ifr_data, &config, in stmmac_hwtstamp_set()
801 sizeof(config)) ? -EFAULT : 0; in stmmac_hwtstamp_set()
805 * stmmac_hwtstamp_get - read hardware timestamping.
816 struct hwtstamp_config *config = &priv->tstamp_config; in stmmac_hwtstamp_get()
818 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) in stmmac_hwtstamp_get()
819 return -EOPNOTSUPP; in stmmac_hwtstamp_get()
821 return copy_to_user(ifr->ifr_data, config, in stmmac_hwtstamp_get()
822 sizeof(*config)) ? -EFAULT : 0; in stmmac_hwtstamp_get()
826 * stmmac_init_tstamp_counter - init hardware timestamping counter
837 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_init_tstamp_counter()
842 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) in stmmac_init_tstamp_counter()
843 return -EOPNOTSUPP; in stmmac_init_tstamp_counter()
845 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); in stmmac_init_tstamp_counter()
846 priv->systime_flags = systime_flags; in stmmac_init_tstamp_counter()
849 stmmac_config_sub_second_increment(priv, priv->ptpaddr, in stmmac_init_tstamp_counter()
850 priv->plat->clk_ptp_rate, in stmmac_init_tstamp_counter()
855 priv->sub_second_inc = sec_inc; in stmmac_init_tstamp_counter()
863 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); in stmmac_init_tstamp_counter()
864 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); in stmmac_init_tstamp_counter()
870 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); in stmmac_init_tstamp_counter()
877 * stmmac_init_ptp - init PTP
885 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_init_ptp()
888 if (priv->plat->ptp_clk_freq_config) in stmmac_init_ptp()
889 priv->plat->ptp_clk_freq_config(priv); in stmmac_init_ptp()
895 priv->adv_ts = 0; in stmmac_init_ptp()
897 if (xmac && priv->dma_cap.atime_stamp) in stmmac_init_ptp()
898 priv->adv_ts = 1; in stmmac_init_ptp()
900 else if (priv->extend_desc && priv->dma_cap.atime_stamp) in stmmac_init_ptp()
901 priv->adv_ts = 1; in stmmac_init_ptp()
903 if (priv->dma_cap.time_stamp) in stmmac_init_ptp()
904 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); in stmmac_init_ptp()
906 if (priv->adv_ts) in stmmac_init_ptp()
907 netdev_info(priv->dev, in stmmac_init_ptp()
908 "IEEE 1588-2008 Advanced Timestamp supported\n"); in stmmac_init_ptp()
910 priv->hwts_tx_en = 0; in stmmac_init_ptp()
911 priv->hwts_rx_en = 0; in stmmac_init_ptp()
913 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) in stmmac_init_ptp()
921 clk_disable_unprepare(priv->plat->clk_ptp_ref); in stmmac_release_ptp()
926 * stmmac_mac_flow_ctrl - Configure flow control in all queues
933 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_mac_flow_ctrl()
935 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, in stmmac_mac_flow_ctrl()
936 priv->pause, tx_cnt); in stmmac_mac_flow_ctrl()
942 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_mac_select_pcs()
944 if (priv->hw->xpcs) in stmmac_mac_select_pcs()
945 return &priv->hw->xpcs->pcs; in stmmac_mac_select_pcs()
947 if (priv->hw->lynx_pcs) in stmmac_mac_select_pcs()
948 return priv->hw->lynx_pcs; in stmmac_mac_select_pcs()
961 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_link_state_handle()
962 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; in stmmac_fpe_link_state_handle()
963 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; in stmmac_fpe_link_state_handle()
964 bool *hs_enable = &fpe_cfg->hs_enable; in stmmac_fpe_link_state_handle()
967 stmmac_fpe_send_mpacket(priv, priv->ioaddr, fpe_cfg, in stmmac_fpe_link_state_handle()
978 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_mac_link_down()
980 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_mac_link_down()
981 priv->eee_active = false; in stmmac_mac_link_down()
982 priv->tx_lpi_enabled = false; in stmmac_mac_link_down()
983 priv->eee_enabled = stmmac_eee_init(priv); in stmmac_mac_link_down()
984 stmmac_set_eee_pls(priv, priv->hw, false); in stmmac_mac_link_down()
986 if (priv->dma_cap.fpesel) in stmmac_mac_link_down()
996 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_mac_link_up()
999 if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && in stmmac_mac_link_up()
1000 priv->plat->serdes_powerup) in stmmac_mac_link_up()
1001 priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); in stmmac_mac_link_up()
1003 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); in stmmac_mac_link_up()
1004 ctrl = old_ctrl & ~priv->hw->link.speed_mask; in stmmac_mac_link_up()
1009 ctrl |= priv->hw->link.xgmii.speed10000; in stmmac_mac_link_up()
1012 ctrl |= priv->hw->link.xgmii.speed5000; in stmmac_mac_link_up()
1015 ctrl |= priv->hw->link.xgmii.speed2500; in stmmac_mac_link_up()
1023 ctrl |= priv->hw->link.xlgmii.speed100000; in stmmac_mac_link_up()
1026 ctrl |= priv->hw->link.xlgmii.speed50000; in stmmac_mac_link_up()
1029 ctrl |= priv->hw->link.xlgmii.speed40000; in stmmac_mac_link_up()
1032 ctrl |= priv->hw->link.xlgmii.speed25000; in stmmac_mac_link_up()
1035 ctrl |= priv->hw->link.xgmii.speed10000; in stmmac_mac_link_up()
1038 ctrl |= priv->hw->link.speed2500; in stmmac_mac_link_up()
1041 ctrl |= priv->hw->link.speed1000; in stmmac_mac_link_up()
1049 ctrl |= priv->hw->link.speed2500; in stmmac_mac_link_up()
1052 ctrl |= priv->hw->link.speed1000; in stmmac_mac_link_up()
1055 ctrl |= priv->hw->link.speed100; in stmmac_mac_link_up()
1058 ctrl |= priv->hw->link.speed10; in stmmac_mac_link_up()
1065 priv->speed = speed; in stmmac_mac_link_up()
1067 if (priv->plat->fix_mac_speed) in stmmac_mac_link_up()
1068 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); in stmmac_mac_link_up()
1071 ctrl &= ~priv->hw->link.duplex; in stmmac_mac_link_up()
1073 ctrl |= priv->hw->link.duplex; in stmmac_mac_link_up()
1077 priv->flow_ctrl = FLOW_AUTO; in stmmac_mac_link_up()
1079 priv->flow_ctrl = FLOW_RX; in stmmac_mac_link_up()
1081 priv->flow_ctrl = FLOW_TX; in stmmac_mac_link_up()
1083 priv->flow_ctrl = FLOW_OFF; in stmmac_mac_link_up()
1088 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); in stmmac_mac_link_up()
1090 stmmac_mac_set(priv, priv->ioaddr, true); in stmmac_mac_link_up()
1091 if (phy && priv->dma_cap.eee) { in stmmac_mac_link_up()
1092 priv->eee_active = in stmmac_mac_link_up()
1093 phy_init_eee(phy, !(priv->plat->flags & in stmmac_mac_link_up()
1095 priv->eee_enabled = stmmac_eee_init(priv); in stmmac_mac_link_up()
1096 priv->tx_lpi_enabled = priv->eee_enabled; in stmmac_mac_link_up()
1097 stmmac_set_eee_pls(priv, priv->hw, true); in stmmac_mac_link_up()
1100 if (priv->dma_cap.fpesel) in stmmac_mac_link_up()
1103 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) in stmmac_mac_link_up()
1115 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
1123 int interface = priv->plat->mac_interface; in stmmac_check_pcs_mode()
1125 if (priv->dma_cap.pcs) { in stmmac_check_pcs_mode()
1130 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); in stmmac_check_pcs_mode()
1131 priv->hw->pcs = STMMAC_PCS_RGMII; in stmmac_check_pcs_mode()
1133 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); in stmmac_check_pcs_mode()
1134 priv->hw->pcs = STMMAC_PCS_SGMII; in stmmac_check_pcs_mode()
1140 * stmmac_init_phy - PHY initialization
1154 if (!phylink_expects_phy(priv->phylink)) in stmmac_init_phy()
1157 fwnode = priv->plat->port_node; in stmmac_init_phy()
1159 fwnode = dev_fwnode(priv->device); in stmmac_init_phy()
1166 /* Some DT bindings do not set-up the PHY handle. Let's try to in stmmac_init_phy()
1170 int addr = priv->plat->phy_addr; in stmmac_init_phy()
1174 netdev_err(priv->dev, "no phy found\n"); in stmmac_init_phy()
1175 return -ENODEV; in stmmac_init_phy()
1178 phydev = mdiobus_get_phy(priv->mii, addr); in stmmac_init_phy()
1180 netdev_err(priv->dev, "no phy at addr %d\n", addr); in stmmac_init_phy()
1181 return -ENODEV; in stmmac_init_phy()
1184 ret = phylink_connect_phy(priv->phylink, phydev); in stmmac_init_phy()
1187 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); in stmmac_init_phy()
1190 if (!priv->plat->pmt) { in stmmac_init_phy()
1193 phylink_ethtool_get_wol(priv->phylink, &wol); in stmmac_init_phy()
1194 device_set_wakeup_capable(priv->device, !!wol.supported); in stmmac_init_phy()
1195 device_set_wakeup_enable(priv->device, !!wol.wolopts); in stmmac_init_phy()
1204 int mode = priv->plat->phy_interface; in stmmac_phy_setup()
1209 priv->phylink_config.dev = &priv->dev->dev; in stmmac_phy_setup()
1210 priv->phylink_config.type = PHYLINK_NETDEV; in stmmac_phy_setup()
1211 priv->phylink_config.mac_managed_pm = true; in stmmac_phy_setup()
1213 mdio_bus_data = priv->plat->mdio_bus_data; in stmmac_phy_setup()
1215 priv->phylink_config.ovr_an_inband = in stmmac_phy_setup()
1216 mdio_bus_data->xpcs_an_inband; in stmmac_phy_setup()
1221 __set_bit(mode, priv->phylink_config.supported_interfaces); in stmmac_phy_setup()
1224 if (priv->hw->xpcs) in stmmac_phy_setup()
1225 xpcs_get_interfaces(priv->hw->xpcs, in stmmac_phy_setup()
1226 priv->phylink_config.supported_interfaces); in stmmac_phy_setup()
1231 priv->phylink_config.mac_capabilities = priv->hw->link.caps; in stmmac_phy_setup()
1233 max_speed = priv->plat->max_speed; in stmmac_phy_setup()
1235 phylink_limit_mac_speed(&priv->phylink_config, max_speed); in stmmac_phy_setup()
1237 fwnode = priv->plat->port_node; in stmmac_phy_setup()
1239 fwnode = dev_fwnode(priv->device); in stmmac_phy_setup()
1241 phylink = phylink_create(&priv->phylink_config, fwnode, in stmmac_phy_setup()
1246 priv->phylink = phylink; in stmmac_phy_setup()
1253 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_display_rx_rings()
1260 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in stmmac_display_rx_rings()
1264 if (priv->extend_desc) { in stmmac_display_rx_rings()
1265 head_rx = (void *)rx_q->dma_erx; in stmmac_display_rx_rings()
1268 head_rx = (void *)rx_q->dma_rx; in stmmac_display_rx_rings()
1273 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, in stmmac_display_rx_rings()
1274 rx_q->dma_rx_phy, desc_size); in stmmac_display_rx_rings()
1281 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_display_tx_rings()
1288 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; in stmmac_display_tx_rings()
1292 if (priv->extend_desc) { in stmmac_display_tx_rings()
1293 head_tx = (void *)tx_q->dma_etx; in stmmac_display_tx_rings()
1295 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { in stmmac_display_tx_rings()
1296 head_tx = (void *)tx_q->dma_entx; in stmmac_display_tx_rings()
1299 head_tx = (void *)tx_q->dma_tx; in stmmac_display_tx_rings()
1303 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, in stmmac_display_tx_rings()
1304 tx_q->dma_tx_phy, desc_size); in stmmac_display_tx_rings()
1337 * stmmac_clear_rx_descriptors - clear RX descriptors
1348 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in stmmac_clear_rx_descriptors()
1352 for (i = 0; i < dma_conf->dma_rx_size; i++) in stmmac_clear_rx_descriptors()
1353 if (priv->extend_desc) in stmmac_clear_rx_descriptors()
1354 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, in stmmac_clear_rx_descriptors()
1355 priv->use_riwt, priv->mode, in stmmac_clear_rx_descriptors()
1356 (i == dma_conf->dma_rx_size - 1), in stmmac_clear_rx_descriptors()
1357 dma_conf->dma_buf_sz); in stmmac_clear_rx_descriptors()
1359 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], in stmmac_clear_rx_descriptors()
1360 priv->use_riwt, priv->mode, in stmmac_clear_rx_descriptors()
1361 (i == dma_conf->dma_rx_size - 1), in stmmac_clear_rx_descriptors()
1362 dma_conf->dma_buf_sz); in stmmac_clear_rx_descriptors()
1366 * stmmac_clear_tx_descriptors - clear tx descriptors
1377 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; in stmmac_clear_tx_descriptors()
1381 for (i = 0; i < dma_conf->dma_tx_size; i++) { in stmmac_clear_tx_descriptors()
1382 int last = (i == (dma_conf->dma_tx_size - 1)); in stmmac_clear_tx_descriptors()
1385 if (priv->extend_desc) in stmmac_clear_tx_descriptors()
1386 p = &tx_q->dma_etx[i].basic; in stmmac_clear_tx_descriptors()
1387 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_clear_tx_descriptors()
1388 p = &tx_q->dma_entx[i].basic; in stmmac_clear_tx_descriptors()
1390 p = &tx_q->dma_tx[i]; in stmmac_clear_tx_descriptors()
1392 stmmac_init_tx_desc(priv, p, priv->mode, last); in stmmac_clear_tx_descriptors()
1397 * stmmac_clear_descriptors - clear descriptors
1406 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; in stmmac_clear_descriptors()
1407 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; in stmmac_clear_descriptors()
1420 * stmmac_init_rx_buffers - init the RX descriptor buffer.
1435 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in stmmac_init_rx_buffers()
1436 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; in stmmac_init_rx_buffers()
1439 if (priv->dma_cap.host_dma_width <= 32) in stmmac_init_rx_buffers()
1442 if (!buf->page) { in stmmac_init_rx_buffers()
1443 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); in stmmac_init_rx_buffers()
1444 if (!buf->page) in stmmac_init_rx_buffers()
1445 return -ENOMEM; in stmmac_init_rx_buffers()
1446 buf->page_offset = stmmac_rx_offset(priv); in stmmac_init_rx_buffers()
1449 if (priv->sph && !buf->sec_page) { in stmmac_init_rx_buffers()
1450 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); in stmmac_init_rx_buffers()
1451 if (!buf->sec_page) in stmmac_init_rx_buffers()
1452 return -ENOMEM; in stmmac_init_rx_buffers()
1454 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); in stmmac_init_rx_buffers()
1455 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); in stmmac_init_rx_buffers()
1457 buf->sec_page = NULL; in stmmac_init_rx_buffers()
1458 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); in stmmac_init_rx_buffers()
1461 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; in stmmac_init_rx_buffers()
1463 stmmac_set_desc_addr(priv, p, buf->addr); in stmmac_init_rx_buffers()
1464 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) in stmmac_init_rx_buffers()
1471 * stmmac_free_rx_buffer - free RX dma buffers
1480 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; in stmmac_free_rx_buffer()
1482 if (buf->page) in stmmac_free_rx_buffer()
1483 page_pool_put_full_page(rx_q->page_pool, buf->page, false); in stmmac_free_rx_buffer()
1484 buf->page = NULL; in stmmac_free_rx_buffer()
1486 if (buf->sec_page) in stmmac_free_rx_buffer()
1487 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); in stmmac_free_rx_buffer()
1488 buf->sec_page = NULL; in stmmac_free_rx_buffer()
1492 * stmmac_free_tx_buffer - free RX dma buffers
1502 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; in stmmac_free_tx_buffer()
1504 if (tx_q->tx_skbuff_dma[i].buf && in stmmac_free_tx_buffer()
1505 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { in stmmac_free_tx_buffer()
1506 if (tx_q->tx_skbuff_dma[i].map_as_page) in stmmac_free_tx_buffer()
1507 dma_unmap_page(priv->device, in stmmac_free_tx_buffer()
1508 tx_q->tx_skbuff_dma[i].buf, in stmmac_free_tx_buffer()
1509 tx_q->tx_skbuff_dma[i].len, in stmmac_free_tx_buffer()
1512 dma_unmap_single(priv->device, in stmmac_free_tx_buffer()
1513 tx_q->tx_skbuff_dma[i].buf, in stmmac_free_tx_buffer()
1514 tx_q->tx_skbuff_dma[i].len, in stmmac_free_tx_buffer()
1518 if (tx_q->xdpf[i] && in stmmac_free_tx_buffer()
1519 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || in stmmac_free_tx_buffer()
1520 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { in stmmac_free_tx_buffer()
1521 xdp_return_frame(tx_q->xdpf[i]); in stmmac_free_tx_buffer()
1522 tx_q->xdpf[i] = NULL; in stmmac_free_tx_buffer()
1525 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) in stmmac_free_tx_buffer()
1526 tx_q->xsk_frames_done++; in stmmac_free_tx_buffer()
1528 if (tx_q->tx_skbuff[i] && in stmmac_free_tx_buffer()
1529 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { in stmmac_free_tx_buffer()
1530 dev_kfree_skb_any(tx_q->tx_skbuff[i]); in stmmac_free_tx_buffer()
1531 tx_q->tx_skbuff[i] = NULL; in stmmac_free_tx_buffer()
1534 tx_q->tx_skbuff_dma[i].buf = 0; in stmmac_free_tx_buffer()
1535 tx_q->tx_skbuff_dma[i].map_as_page = false; in stmmac_free_tx_buffer()
1539 * dma_free_rx_skbufs - free RX dma buffers
1548 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in dma_free_rx_skbufs()
1551 for (i = 0; i < dma_conf->dma_rx_size; i++) in dma_free_rx_skbufs()
1559 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in stmmac_alloc_rx_buffers()
1562 for (i = 0; i < dma_conf->dma_rx_size; i++) { in stmmac_alloc_rx_buffers()
1566 if (priv->extend_desc) in stmmac_alloc_rx_buffers()
1567 p = &((rx_q->dma_erx + i)->basic); in stmmac_alloc_rx_buffers()
1569 p = rx_q->dma_rx + i; in stmmac_alloc_rx_buffers()
1576 rx_q->buf_alloc_num++; in stmmac_alloc_rx_buffers()
1583 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool
1592 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in dma_free_rx_xskbufs()
1595 for (i = 0; i < dma_conf->dma_rx_size; i++) { in dma_free_rx_xskbufs()
1596 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; in dma_free_rx_xskbufs()
1598 if (!buf->xdp) in dma_free_rx_xskbufs()
1601 xsk_buff_free(buf->xdp); in dma_free_rx_xskbufs()
1602 buf->xdp = NULL; in dma_free_rx_xskbufs()
1610 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in stmmac_alloc_rx_buffers_zc()
1619 for (i = 0; i < dma_conf->dma_rx_size; i++) { in stmmac_alloc_rx_buffers_zc()
1624 if (priv->extend_desc) in stmmac_alloc_rx_buffers_zc()
1625 p = (struct dma_desc *)(rx_q->dma_erx + i); in stmmac_alloc_rx_buffers_zc()
1627 p = rx_q->dma_rx + i; in stmmac_alloc_rx_buffers_zc()
1629 buf = &rx_q->buf_pool[i]; in stmmac_alloc_rx_buffers_zc()
1631 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); in stmmac_alloc_rx_buffers_zc()
1632 if (!buf->xdp) in stmmac_alloc_rx_buffers_zc()
1633 return -ENOMEM; in stmmac_alloc_rx_buffers_zc()
1635 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); in stmmac_alloc_rx_buffers_zc()
1637 rx_q->buf_alloc_num++; in stmmac_alloc_rx_buffers_zc()
1645 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) in stmmac_get_xsk_pool()
1648 return xsk_get_pool_from_qid(priv->dev, queue); in stmmac_get_xsk_pool()
1652 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue)
1665 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in __init_dma_rx_desc_rings()
1668 netif_dbg(priv, probe, priv->dev, in __init_dma_rx_desc_rings()
1670 (u32)rx_q->dma_rx_phy); in __init_dma_rx_desc_rings()
1674 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); in __init_dma_rx_desc_rings()
1676 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); in __init_dma_rx_desc_rings()
1678 if (rx_q->xsk_pool) { in __init_dma_rx_desc_rings()
1679 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, in __init_dma_rx_desc_rings()
1682 netdev_info(priv->dev, in __init_dma_rx_desc_rings()
1683 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", in __init_dma_rx_desc_rings()
1684 rx_q->queue_index); in __init_dma_rx_desc_rings()
1685 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); in __init_dma_rx_desc_rings()
1687 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, in __init_dma_rx_desc_rings()
1689 rx_q->page_pool)); in __init_dma_rx_desc_rings()
1690 netdev_info(priv->dev, in __init_dma_rx_desc_rings()
1691 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", in __init_dma_rx_desc_rings()
1692 rx_q->queue_index); in __init_dma_rx_desc_rings()
1695 if (rx_q->xsk_pool) { in __init_dma_rx_desc_rings()
1697 * xdpsock TX-only. in __init_dma_rx_desc_rings()
1703 return -ENOMEM; in __init_dma_rx_desc_rings()
1707 if (priv->mode == STMMAC_CHAIN_MODE) { in __init_dma_rx_desc_rings()
1708 if (priv->extend_desc) in __init_dma_rx_desc_rings()
1709 stmmac_mode_init(priv, rx_q->dma_erx, in __init_dma_rx_desc_rings()
1710 rx_q->dma_rx_phy, in __init_dma_rx_desc_rings()
1711 dma_conf->dma_rx_size, 1); in __init_dma_rx_desc_rings()
1713 stmmac_mode_init(priv, rx_q->dma_rx, in __init_dma_rx_desc_rings()
1714 rx_q->dma_rx_phy, in __init_dma_rx_desc_rings()
1715 dma_conf->dma_rx_size, 0); in __init_dma_rx_desc_rings()
1726 u32 rx_count = priv->plat->rx_queues_to_use; in init_dma_rx_desc_rings()
1731 netif_dbg(priv, probe, priv->dev, in init_dma_rx_desc_rings()
1744 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in init_dma_rx_desc_rings()
1746 if (rx_q->xsk_pool) in init_dma_rx_desc_rings()
1751 rx_q->buf_alloc_num = 0; in init_dma_rx_desc_rings()
1752 rx_q->xsk_pool = NULL; in init_dma_rx_desc_rings()
1754 queue--; in init_dma_rx_desc_rings()
1761 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue)
1773 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; in __init_dma_tx_desc_rings()
1776 netif_dbg(priv, probe, priv->dev, in __init_dma_tx_desc_rings()
1778 (u32)tx_q->dma_tx_phy); in __init_dma_tx_desc_rings()
1781 if (priv->mode == STMMAC_CHAIN_MODE) { in __init_dma_tx_desc_rings()
1782 if (priv->extend_desc) in __init_dma_tx_desc_rings()
1783 stmmac_mode_init(priv, tx_q->dma_etx, in __init_dma_tx_desc_rings()
1784 tx_q->dma_tx_phy, in __init_dma_tx_desc_rings()
1785 dma_conf->dma_tx_size, 1); in __init_dma_tx_desc_rings()
1786 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) in __init_dma_tx_desc_rings()
1787 stmmac_mode_init(priv, tx_q->dma_tx, in __init_dma_tx_desc_rings()
1788 tx_q->dma_tx_phy, in __init_dma_tx_desc_rings()
1789 dma_conf->dma_tx_size, 0); in __init_dma_tx_desc_rings()
1792 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); in __init_dma_tx_desc_rings()
1794 for (i = 0; i < dma_conf->dma_tx_size; i++) { in __init_dma_tx_desc_rings()
1797 if (priv->extend_desc) in __init_dma_tx_desc_rings()
1798 p = &((tx_q->dma_etx + i)->basic); in __init_dma_tx_desc_rings()
1799 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in __init_dma_tx_desc_rings()
1800 p = &((tx_q->dma_entx + i)->basic); in __init_dma_tx_desc_rings()
1802 p = tx_q->dma_tx + i; in __init_dma_tx_desc_rings()
1806 tx_q->tx_skbuff_dma[i].buf = 0; in __init_dma_tx_desc_rings()
1807 tx_q->tx_skbuff_dma[i].map_as_page = false; in __init_dma_tx_desc_rings()
1808 tx_q->tx_skbuff_dma[i].len = 0; in __init_dma_tx_desc_rings()
1809 tx_q->tx_skbuff_dma[i].last_segment = false; in __init_dma_tx_desc_rings()
1810 tx_q->tx_skbuff[i] = NULL; in __init_dma_tx_desc_rings()
1823 tx_queue_cnt = priv->plat->tx_queues_to_use; in init_dma_tx_desc_rings()
1832 * init_dma_desc_rings - init the RX/TX descriptor rings
1862 * dma_free_tx_skbufs - free TX dma buffers
1871 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; in dma_free_tx_skbufs()
1874 tx_q->xsk_frames_done = 0; in dma_free_tx_skbufs()
1876 for (i = 0; i < dma_conf->dma_tx_size; i++) in dma_free_tx_skbufs()
1879 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { in dma_free_tx_skbufs()
1880 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); in dma_free_tx_skbufs()
1881 tx_q->xsk_frames_done = 0; in dma_free_tx_skbufs()
1882 tx_q->xsk_pool = NULL; in dma_free_tx_skbufs()
1887 * stmmac_free_tx_skbufs - free TX skb buffers
1892 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; in stmmac_free_tx_skbufs()
1896 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); in stmmac_free_tx_skbufs()
1900 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue)
1909 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in __free_dma_rx_desc_resources()
1912 if (rx_q->xsk_pool) in __free_dma_rx_desc_resources()
1917 rx_q->buf_alloc_num = 0; in __free_dma_rx_desc_resources()
1918 rx_q->xsk_pool = NULL; in __free_dma_rx_desc_resources()
1921 if (!priv->extend_desc) in __free_dma_rx_desc_resources()
1922 dma_free_coherent(priv->device, dma_conf->dma_rx_size * in __free_dma_rx_desc_resources()
1924 rx_q->dma_rx, rx_q->dma_rx_phy); in __free_dma_rx_desc_resources()
1926 dma_free_coherent(priv->device, dma_conf->dma_rx_size * in __free_dma_rx_desc_resources()
1928 rx_q->dma_erx, rx_q->dma_rx_phy); in __free_dma_rx_desc_resources()
1930 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) in __free_dma_rx_desc_resources()
1931 xdp_rxq_info_unreg(&rx_q->xdp_rxq); in __free_dma_rx_desc_resources()
1933 kfree(rx_q->buf_pool); in __free_dma_rx_desc_resources()
1934 if (rx_q->page_pool) in __free_dma_rx_desc_resources()
1935 page_pool_destroy(rx_q->page_pool); in __free_dma_rx_desc_resources()
1941 u32 rx_count = priv->plat->rx_queues_to_use; in free_dma_rx_desc_resources()
1950 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue)
1959 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; in __free_dma_tx_desc_resources()
1966 if (priv->extend_desc) { in __free_dma_tx_desc_resources()
1968 addr = tx_q->dma_etx; in __free_dma_tx_desc_resources()
1969 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { in __free_dma_tx_desc_resources()
1971 addr = tx_q->dma_entx; in __free_dma_tx_desc_resources()
1974 addr = tx_q->dma_tx; in __free_dma_tx_desc_resources()
1977 size *= dma_conf->dma_tx_size; in __free_dma_tx_desc_resources()
1979 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); in __free_dma_tx_desc_resources()
1981 kfree(tx_q->tx_skbuff_dma); in __free_dma_tx_desc_resources()
1982 kfree(tx_q->tx_skbuff); in __free_dma_tx_desc_resources()
1988 u32 tx_count = priv->plat->tx_queues_to_use; in free_dma_tx_desc_resources()
1997 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue).
2003 * reception, for example, it pre-allocated the RX socket buffer in order to
2004 * allow zero-copy mechanism.
2010 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; in __alloc_dma_rx_desc_resources()
2011 struct stmmac_channel *ch = &priv->channel[queue]; in __alloc_dma_rx_desc_resources()
2018 rx_q->queue_index = queue; in __alloc_dma_rx_desc_resources()
2019 rx_q->priv_data = priv; in __alloc_dma_rx_desc_resources()
2022 pp_params.pool_size = dma_conf->dma_rx_size; in __alloc_dma_rx_desc_resources()
2023 num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE); in __alloc_dma_rx_desc_resources()
2025 pp_params.nid = dev_to_node(priv->device); in __alloc_dma_rx_desc_resources()
2026 pp_params.dev = priv->device; in __alloc_dma_rx_desc_resources()
2031 rx_q->page_pool = page_pool_create(&pp_params); in __alloc_dma_rx_desc_resources()
2032 if (IS_ERR(rx_q->page_pool)) { in __alloc_dma_rx_desc_resources()
2033 ret = PTR_ERR(rx_q->page_pool); in __alloc_dma_rx_desc_resources()
2034 rx_q->page_pool = NULL; in __alloc_dma_rx_desc_resources()
2038 rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, in __alloc_dma_rx_desc_resources()
2039 sizeof(*rx_q->buf_pool), in __alloc_dma_rx_desc_resources()
2041 if (!rx_q->buf_pool) in __alloc_dma_rx_desc_resources()
2042 return -ENOMEM; in __alloc_dma_rx_desc_resources()
2044 if (priv->extend_desc) { in __alloc_dma_rx_desc_resources()
2045 rx_q->dma_erx = dma_alloc_coherent(priv->device, in __alloc_dma_rx_desc_resources()
2046 dma_conf->dma_rx_size * in __alloc_dma_rx_desc_resources()
2048 &rx_q->dma_rx_phy, in __alloc_dma_rx_desc_resources()
2050 if (!rx_q->dma_erx) in __alloc_dma_rx_desc_resources()
2051 return -ENOMEM; in __alloc_dma_rx_desc_resources()
2054 rx_q->dma_rx = dma_alloc_coherent(priv->device, in __alloc_dma_rx_desc_resources()
2055 dma_conf->dma_rx_size * in __alloc_dma_rx_desc_resources()
2057 &rx_q->dma_rx_phy, in __alloc_dma_rx_desc_resources()
2059 if (!rx_q->dma_rx) in __alloc_dma_rx_desc_resources()
2060 return -ENOMEM; in __alloc_dma_rx_desc_resources()
2064 test_bit(queue, priv->af_xdp_zc_qps)) in __alloc_dma_rx_desc_resources()
2065 napi_id = ch->rxtx_napi.napi_id; in __alloc_dma_rx_desc_resources()
2067 napi_id = ch->rx_napi.napi_id; in __alloc_dma_rx_desc_resources()
2069 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, in __alloc_dma_rx_desc_resources()
2070 rx_q->queue_index, in __alloc_dma_rx_desc_resources()
2073 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); in __alloc_dma_rx_desc_resources()
2074 return -EINVAL; in __alloc_dma_rx_desc_resources()
2083 u32 rx_count = priv->plat->rx_queues_to_use; in alloc_dma_rx_desc_resources()
2103 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue).
2109 * reception, for example, it pre-allocated the RX socket buffer in order to
2110 * allow zero-copy mechanism.
2116 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; in __alloc_dma_tx_desc_resources()
2120 tx_q->queue_index = queue; in __alloc_dma_tx_desc_resources()
2121 tx_q->priv_data = priv; in __alloc_dma_tx_desc_resources()
2123 tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, in __alloc_dma_tx_desc_resources()
2124 sizeof(*tx_q->tx_skbuff_dma), in __alloc_dma_tx_desc_resources()
2126 if (!tx_q->tx_skbuff_dma) in __alloc_dma_tx_desc_resources()
2127 return -ENOMEM; in __alloc_dma_tx_desc_resources()
2129 tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, in __alloc_dma_tx_desc_resources()
2132 if (!tx_q->tx_skbuff) in __alloc_dma_tx_desc_resources()
2133 return -ENOMEM; in __alloc_dma_tx_desc_resources()
2135 if (priv->extend_desc) in __alloc_dma_tx_desc_resources()
2137 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in __alloc_dma_tx_desc_resources()
2142 size *= dma_conf->dma_tx_size; in __alloc_dma_tx_desc_resources()
2144 addr = dma_alloc_coherent(priv->device, size, in __alloc_dma_tx_desc_resources()
2145 &tx_q->dma_tx_phy, GFP_KERNEL); in __alloc_dma_tx_desc_resources()
2147 return -ENOMEM; in __alloc_dma_tx_desc_resources()
2149 if (priv->extend_desc) in __alloc_dma_tx_desc_resources()
2150 tx_q->dma_etx = addr; in __alloc_dma_tx_desc_resources()
2151 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in __alloc_dma_tx_desc_resources()
2152 tx_q->dma_entx = addr; in __alloc_dma_tx_desc_resources()
2154 tx_q->dma_tx = addr; in __alloc_dma_tx_desc_resources()
2162 u32 tx_count = priv->plat->tx_queues_to_use; in alloc_dma_tx_desc_resources()
2181 * alloc_dma_desc_resources - alloc TX/RX resources.
2186 * reception, for example, it pre-allocated the RX socket buffer in order to
2187 * allow zero-copy mechanism.
2204 * free_dma_desc_resources - free dma desc resources
2221 * stmmac_mac_enable_rx_queues - Enable MAC rx queues
2227 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_enable_rx_queues()
2232 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; in stmmac_mac_enable_rx_queues()
2233 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); in stmmac_mac_enable_rx_queues()
2238 * stmmac_start_rx_dma - start RX DMA channel
2246 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); in stmmac_start_rx_dma()
2247 stmmac_start_rx(priv, priv->ioaddr, chan); in stmmac_start_rx_dma()
2251 * stmmac_start_tx_dma - start TX DMA channel
2259 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); in stmmac_start_tx_dma()
2260 stmmac_start_tx(priv, priv->ioaddr, chan); in stmmac_start_tx_dma()
2264 * stmmac_stop_rx_dma - stop RX DMA channel
2272 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); in stmmac_stop_rx_dma()
2273 stmmac_stop_rx(priv, priv->ioaddr, chan); in stmmac_stop_rx_dma()
2277 * stmmac_stop_tx_dma - stop TX DMA channel
2285 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); in stmmac_stop_tx_dma()
2286 stmmac_stop_tx(priv, priv->ioaddr, chan); in stmmac_stop_tx_dma()
2291 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_enable_all_dma_irq()
2292 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_enable_all_dma_irq()
2297 struct stmmac_channel *ch = &priv->channel[chan]; in stmmac_enable_all_dma_irq()
2300 spin_lock_irqsave(&ch->lock, flags); in stmmac_enable_all_dma_irq()
2301 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_enable_all_dma_irq()
2302 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_enable_all_dma_irq()
2307 * stmmac_start_all_dma - start all RX and TX DMA channels
2314 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_start_all_dma()
2315 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_start_all_dma()
2326 * stmmac_stop_all_dma - stop all RX and TX DMA channels
2333 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_stop_all_dma()
2334 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_stop_all_dma()
2345 * stmmac_dma_operation_mode - HW DMA operation mode
2348 * order to program the tx/rx DMA thresholds or Store-And-Forward mode.
2352 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_dma_operation_mode()
2353 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_dma_operation_mode()
2354 int rxfifosz = priv->plat->rx_fifo_size; in stmmac_dma_operation_mode()
2355 int txfifosz = priv->plat->tx_fifo_size; in stmmac_dma_operation_mode()
2362 rxfifosz = priv->dma_cap.rx_fifo_size; in stmmac_dma_operation_mode()
2364 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_dma_operation_mode()
2370 if (priv->plat->force_thresh_dma_mode) { in stmmac_dma_operation_mode()
2373 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { in stmmac_dma_operation_mode()
2383 priv->xstats.threshold = SF_DMA_MODE; in stmmac_dma_operation_mode()
2391 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; in stmmac_dma_operation_mode()
2394 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; in stmmac_dma_operation_mode()
2396 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, in stmmac_dma_operation_mode()
2399 if (rx_q->xsk_pool) { in stmmac_dma_operation_mode()
2400 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); in stmmac_dma_operation_mode()
2401 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_dma_operation_mode()
2405 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_dma_operation_mode()
2406 priv->dma_conf.dma_buf_sz, in stmmac_dma_operation_mode()
2412 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; in stmmac_dma_operation_mode()
2414 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, in stmmac_dma_operation_mode()
2421 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit_zc()
2422 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_xdp_xmit_zc()
2423 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; in stmmac_xdp_xmit_zc()
2424 struct xsk_buff_pool *pool = tx_q->xsk_pool; in stmmac_xdp_xmit_zc()
2425 unsigned int entry = tx_q->cur_tx; in stmmac_xdp_xmit_zc()
2431 /* Avoids TX time-out as we are sharing with slow path */ in stmmac_xdp_xmit_zc()
2436 while (budget-- > 0) { in stmmac_xdp_xmit_zc()
2444 !netif_carrier_ok(priv->dev)) { in stmmac_xdp_xmit_zc()
2452 if (likely(priv->extend_desc)) in stmmac_xdp_xmit_zc()
2453 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_xdp_xmit_zc()
2454 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xdp_xmit_zc()
2455 tx_desc = &tx_q->dma_entx[entry].basic; in stmmac_xdp_xmit_zc()
2457 tx_desc = tx_q->dma_tx + entry; in stmmac_xdp_xmit_zc()
2462 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; in stmmac_xdp_xmit_zc()
2468 tx_q->tx_skbuff_dma[entry].buf = 0; in stmmac_xdp_xmit_zc()
2469 tx_q->xdpf[entry] = NULL; in stmmac_xdp_xmit_zc()
2471 tx_q->tx_skbuff_dma[entry].map_as_page = false; in stmmac_xdp_xmit_zc()
2472 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; in stmmac_xdp_xmit_zc()
2473 tx_q->tx_skbuff_dma[entry].last_segment = true; in stmmac_xdp_xmit_zc()
2474 tx_q->tx_skbuff_dma[entry].is_jumbo = false; in stmmac_xdp_xmit_zc()
2478 tx_q->tx_count_frames++; in stmmac_xdp_xmit_zc()
2480 if (!priv->tx_coal_frames[queue]) in stmmac_xdp_xmit_zc()
2482 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) in stmmac_xdp_xmit_zc()
2488 tx_q->tx_count_frames = 0; in stmmac_xdp_xmit_zc()
2494 true, priv->mode, true, true, in stmmac_xdp_xmit_zc()
2497 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xdp_xmit_zc()
2499 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); in stmmac_xdp_xmit_zc()
2500 entry = tx_q->cur_tx; in stmmac_xdp_xmit_zc()
2502 u64_stats_update_begin(&txq_stats->napi_syncp); in stmmac_xdp_xmit_zc()
2503 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); in stmmac_xdp_xmit_zc()
2504 u64_stats_update_end(&txq_stats->napi_syncp); in stmmac_xdp_xmit_zc()
2521 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { in stmmac_bump_dma_threshold()
2524 if (priv->plat->force_thresh_dma_mode) in stmmac_bump_dma_threshold()
2530 priv->xstats.threshold = tc; in stmmac_bump_dma_threshold()
2535 * stmmac_tx_clean - to manage the transmission completion
2543 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_tx_clean()
2544 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; in stmmac_tx_clean()
2549 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2551 tx_q->xsk_frames_done = 0; in stmmac_tx_clean()
2553 entry = tx_q->dirty_tx; in stmmac_tx_clean()
2556 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { in stmmac_tx_clean()
2562 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || in stmmac_tx_clean()
2563 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { in stmmac_tx_clean()
2564 xdpf = tx_q->xdpf[entry]; in stmmac_tx_clean()
2566 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { in stmmac_tx_clean()
2568 skb = tx_q->tx_skbuff[entry]; in stmmac_tx_clean()
2574 if (priv->extend_desc) in stmmac_tx_clean()
2575 p = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_tx_clean()
2576 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tx_clean()
2577 p = &tx_q->dma_entx[entry].basic; in stmmac_tx_clean()
2579 p = tx_q->dma_tx + entry; in stmmac_tx_clean()
2581 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); in stmmac_tx_clean()
2607 if (likely(tx_q->tx_skbuff_dma[entry].buf && in stmmac_tx_clean()
2608 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { in stmmac_tx_clean()
2609 if (tx_q->tx_skbuff_dma[entry].map_as_page) in stmmac_tx_clean()
2610 dma_unmap_page(priv->device, in stmmac_tx_clean()
2611 tx_q->tx_skbuff_dma[entry].buf, in stmmac_tx_clean()
2612 tx_q->tx_skbuff_dma[entry].len, in stmmac_tx_clean()
2615 dma_unmap_single(priv->device, in stmmac_tx_clean()
2616 tx_q->tx_skbuff_dma[entry].buf, in stmmac_tx_clean()
2617 tx_q->tx_skbuff_dma[entry].len, in stmmac_tx_clean()
2619 tx_q->tx_skbuff_dma[entry].buf = 0; in stmmac_tx_clean()
2620 tx_q->tx_skbuff_dma[entry].len = 0; in stmmac_tx_clean()
2621 tx_q->tx_skbuff_dma[entry].map_as_page = false; in stmmac_tx_clean()
2626 tx_q->tx_skbuff_dma[entry].last_segment = false; in stmmac_tx_clean()
2627 tx_q->tx_skbuff_dma[entry].is_jumbo = false; in stmmac_tx_clean()
2630 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { in stmmac_tx_clean()
2632 tx_q->xdpf[entry] = NULL; in stmmac_tx_clean()
2636 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { in stmmac_tx_clean()
2638 tx_q->xdpf[entry] = NULL; in stmmac_tx_clean()
2641 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) in stmmac_tx_clean()
2642 tx_q->xsk_frames_done++; in stmmac_tx_clean()
2644 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { in stmmac_tx_clean()
2647 bytes_compl += skb->len; in stmmac_tx_clean()
2649 tx_q->tx_skbuff[entry] = NULL; in stmmac_tx_clean()
2653 stmmac_release_tx_desc(priv, p, priv->mode); in stmmac_tx_clean()
2655 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); in stmmac_tx_clean()
2657 tx_q->dirty_tx = entry; in stmmac_tx_clean()
2659 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), in stmmac_tx_clean()
2662 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, in stmmac_tx_clean()
2666 netif_dbg(priv, tx_done, priv->dev, in stmmac_tx_clean()
2668 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2671 if (tx_q->xsk_pool) { in stmmac_tx_clean()
2674 if (tx_q->xsk_frames_done) in stmmac_tx_clean()
2675 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); in stmmac_tx_clean()
2677 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) in stmmac_tx_clean()
2678 xsk_set_tx_need_wakeup(tx_q->xsk_pool); in stmmac_tx_clean()
2682 * available), return "budget - 1" to reenable TX IRQ. in stmmac_tx_clean()
2688 xmits = budget - 1; in stmmac_tx_clean()
2693 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && in stmmac_tx_clean()
2694 priv->eee_sw_timer_en) { in stmmac_tx_clean()
2696 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_tx_clean()
2700 if (tx_q->dirty_tx != tx_q->cur_tx) in stmmac_tx_clean()
2703 u64_stats_update_begin(&txq_stats->napi_syncp); in stmmac_tx_clean()
2704 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); in stmmac_tx_clean()
2705 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); in stmmac_tx_clean()
2706 u64_stats_inc(&txq_stats->napi.tx_clean); in stmmac_tx_clean()
2707 u64_stats_update_end(&txq_stats->napi_syncp); in stmmac_tx_clean()
2709 priv->xstats.tx_errors += tx_errors; in stmmac_tx_clean()
2711 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2718 * stmmac_tx_err - to manage the tx error
2726 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; in stmmac_tx_err()
2728 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); in stmmac_tx_err()
2731 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); in stmmac_tx_err()
2732 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); in stmmac_tx_err()
2734 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_tx_err()
2735 tx_q->dma_tx_phy, chan); in stmmac_tx_err()
2738 priv->xstats.tx_errors++; in stmmac_tx_err()
2739 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); in stmmac_tx_err()
2743 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2749 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2755 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; in stmmac_set_dma_operation_mode()
2756 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; in stmmac_set_dma_operation_mode()
2757 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_set_dma_operation_mode()
2758 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_set_dma_operation_mode()
2759 int rxfifosz = priv->plat->rx_fifo_size; in stmmac_set_dma_operation_mode()
2760 int txfifosz = priv->plat->tx_fifo_size; in stmmac_set_dma_operation_mode()
2763 rxfifosz = priv->dma_cap.rx_fifo_size; in stmmac_set_dma_operation_mode()
2765 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_set_dma_operation_mode()
2771 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); in stmmac_set_dma_operation_mode()
2772 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); in stmmac_set_dma_operation_mode()
2779 ret = stmmac_safety_feat_irq_status(priv, priv->dev, in stmmac_safety_feat_interrupt()
2780 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); in stmmac_safety_feat_interrupt()
2781 if (ret && (ret != -EINVAL)) { in stmmac_safety_feat_interrupt()
2791 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, in stmmac_napi_check()
2792 &priv->xstats, chan, dir); in stmmac_napi_check()
2793 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; in stmmac_napi_check()
2794 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; in stmmac_napi_check()
2795 struct stmmac_channel *ch = &priv->channel[chan]; in stmmac_napi_check()
2800 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; in stmmac_napi_check()
2801 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; in stmmac_napi_check()
2803 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { in stmmac_napi_check()
2805 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_check()
2806 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); in stmmac_napi_check()
2807 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_check()
2812 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { in stmmac_napi_check()
2814 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_check()
2815 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); in stmmac_napi_check()
2816 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_check()
2825 * stmmac_dma_interrupt - DMA ISR
2833 u32 tx_channel_count = priv->plat->tx_queues_to_use; in stmmac_dma_interrupt()
2834 u32 rx_channel_count = priv->plat->rx_queues_to_use; in stmmac_dma_interrupt()
2868 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); in stmmac_mmc_setup()
2870 if (priv->dma_cap.rmon) { in stmmac_mmc_setup()
2871 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); in stmmac_mmc_setup()
2872 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); in stmmac_mmc_setup()
2874 netdev_info(priv->dev, "No MAC Management Counters available\n"); in stmmac_mmc_setup()
2878 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2888 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; in stmmac_get_hw_features()
2892 * stmmac_check_ether_addr - check if the MAC addr is valid
2902 if (!is_valid_ether_addr(priv->dev->dev_addr)) { in stmmac_check_ether_addr()
2903 stmmac_get_umac_addr(priv, priv->hw, addr, 0); in stmmac_check_ether_addr()
2905 eth_hw_addr_set(priv->dev, addr); in stmmac_check_ether_addr()
2907 eth_hw_addr_random(priv->dev); in stmmac_check_ether_addr()
2908 dev_info(priv->device, "device MAC address %pM\n", in stmmac_check_ether_addr()
2909 priv->dev->dev_addr); in stmmac_check_ether_addr()
2914 * stmmac_init_dma_engine - DMA init.
2923 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_init_dma_engine()
2924 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_init_dma_engine()
2932 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { in stmmac_init_dma_engine()
2933 dev_err(priv->device, "Invalid DMA configuration\n"); in stmmac_init_dma_engine()
2934 return -EINVAL; in stmmac_init_dma_engine()
2937 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) in stmmac_init_dma_engine()
2940 ret = stmmac_reset(priv, priv->ioaddr); in stmmac_init_dma_engine()
2942 dev_err(priv->device, "Failed to reset the dma\n"); in stmmac_init_dma_engine()
2947 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); in stmmac_init_dma_engine()
2949 if (priv->plat->axi) in stmmac_init_dma_engine()
2950 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); in stmmac_init_dma_engine()
2954 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); in stmmac_init_dma_engine()
2955 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_init_dma_engine()
2960 rx_q = &priv->dma_conf.rx_queue[chan]; in stmmac_init_dma_engine()
2962 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_init_dma_engine()
2963 rx_q->dma_rx_phy, chan); in stmmac_init_dma_engine()
2965 rx_q->rx_tail_addr = rx_q->dma_rx_phy + in stmmac_init_dma_engine()
2966 (rx_q->buf_alloc_num * in stmmac_init_dma_engine()
2968 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, in stmmac_init_dma_engine()
2969 rx_q->rx_tail_addr, chan); in stmmac_init_dma_engine()
2974 tx_q = &priv->dma_conf.tx_queue[chan]; in stmmac_init_dma_engine()
2976 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_init_dma_engine()
2977 tx_q->dma_tx_phy, chan); in stmmac_init_dma_engine()
2979 tx_q->tx_tail_addr = tx_q->dma_tx_phy; in stmmac_init_dma_engine()
2980 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, in stmmac_init_dma_engine()
2981 tx_q->tx_tail_addr, chan); in stmmac_init_dma_engine()
2989 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_tx_timer_arm()
2990 u32 tx_coal_timer = priv->tx_coal_timer[queue]; in stmmac_tx_timer_arm()
2995 hrtimer_start(&tx_q->txtimer, in stmmac_tx_timer_arm()
3001 * stmmac_tx_timer - mitigation sw timer for tx.
3009 struct stmmac_priv *priv = tx_q->priv_data; in stmmac_tx_timer()
3013 ch = &priv->channel[tx_q->queue_index]; in stmmac_tx_timer()
3014 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; in stmmac_tx_timer()
3019 spin_lock_irqsave(&ch->lock, flags); in stmmac_tx_timer()
3020 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); in stmmac_tx_timer()
3021 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_tx_timer()
3029 * stmmac_init_coalesce - init mitigation options.
3038 u32 tx_channel_count = priv->plat->tx_queues_to_use; in stmmac_init_coalesce()
3039 u32 rx_channel_count = priv->plat->rx_queues_to_use; in stmmac_init_coalesce()
3043 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; in stmmac_init_coalesce()
3045 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; in stmmac_init_coalesce()
3046 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; in stmmac_init_coalesce()
3048 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in stmmac_init_coalesce()
3049 tx_q->txtimer.function = stmmac_tx_timer; in stmmac_init_coalesce()
3053 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; in stmmac_init_coalesce()
3058 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_set_rings_length()
3059 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_set_rings_length()
3064 stmmac_set_tx_ring_len(priv, priv->ioaddr, in stmmac_set_rings_length()
3065 (priv->dma_conf.dma_tx_size - 1), chan); in stmmac_set_rings_length()
3069 stmmac_set_rx_ring_len(priv, priv->ioaddr, in stmmac_set_rings_length()
3070 (priv->dma_conf.dma_rx_size - 1), chan); in stmmac_set_rings_length()
3074 * stmmac_set_tx_queue_weight - Set TX queue weight
3080 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_set_tx_queue_weight()
3085 weight = priv->plat->tx_queues_cfg[queue].weight; in stmmac_set_tx_queue_weight()
3086 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); in stmmac_set_tx_queue_weight()
3091 * stmmac_configure_cbs - Configure CBS in TX queue
3097 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_configure_cbs()
3103 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; in stmmac_configure_cbs()
3107 stmmac_config_cbs(priv, priv->hw, in stmmac_configure_cbs()
3108 priv->plat->tx_queues_cfg[queue].send_slope, in stmmac_configure_cbs()
3109 priv->plat->tx_queues_cfg[queue].idle_slope, in stmmac_configure_cbs()
3110 priv->plat->tx_queues_cfg[queue].high_credit, in stmmac_configure_cbs()
3111 priv->plat->tx_queues_cfg[queue].low_credit, in stmmac_configure_cbs()
3117 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
3123 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_rx_queue_dma_chan_map()
3128 chan = priv->plat->rx_queues_cfg[queue].chan; in stmmac_rx_queue_dma_chan_map()
3129 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); in stmmac_rx_queue_dma_chan_map()
3134 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
3140 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_config_rx_queues_prio()
3145 if (!priv->plat->rx_queues_cfg[queue].use_prio) in stmmac_mac_config_rx_queues_prio()
3148 prio = priv->plat->rx_queues_cfg[queue].prio; in stmmac_mac_config_rx_queues_prio()
3149 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); in stmmac_mac_config_rx_queues_prio()
3154 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
3160 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_mac_config_tx_queues_prio()
3165 if (!priv->plat->tx_queues_cfg[queue].use_prio) in stmmac_mac_config_tx_queues_prio()
3168 prio = priv->plat->tx_queues_cfg[queue].prio; in stmmac_mac_config_tx_queues_prio()
3169 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); in stmmac_mac_config_tx_queues_prio()
3174 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
3180 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_config_rx_queues_routing()
3186 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) in stmmac_mac_config_rx_queues_routing()
3189 packet = priv->plat->rx_queues_cfg[queue].pkt_route; in stmmac_mac_config_rx_queues_routing()
3190 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); in stmmac_mac_config_rx_queues_routing()
3196 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { in stmmac_mac_config_rss()
3197 priv->rss.enable = false; in stmmac_mac_config_rss()
3201 if (priv->dev->features & NETIF_F_RXHASH) in stmmac_mac_config_rss()
3202 priv->rss.enable = true; in stmmac_mac_config_rss()
3204 priv->rss.enable = false; in stmmac_mac_config_rss()
3206 stmmac_rss_configure(priv, priv->hw, &priv->rss, in stmmac_mac_config_rss()
3207 priv->plat->rx_queues_to_use); in stmmac_mac_config_rss()
3211 * stmmac_mtl_configuration - Configure MTL
3217 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mtl_configuration()
3218 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_mtl_configuration()
3225 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, in stmmac_mtl_configuration()
3226 priv->plat->rx_sched_algorithm); in stmmac_mtl_configuration()
3230 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, in stmmac_mtl_configuration()
3231 priv->plat->tx_sched_algorithm); in stmmac_mtl_configuration()
3262 if (priv->dma_cap.asp) { in stmmac_safety_feat_configuration()
3263 netdev_info(priv->dev, "Enabling Safety Features\n"); in stmmac_safety_feat_configuration()
3264 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, in stmmac_safety_feat_configuration()
3265 priv->plat->safety_feat_cfg); in stmmac_safety_feat_configuration()
3267 netdev_info(priv->dev, "No Safety Features support found\n"); in stmmac_safety_feat_configuration()
3275 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); in stmmac_fpe_start_wq()
3276 clear_bit(__FPE_REMOVING, &priv->fpe_task_state); in stmmac_fpe_start_wq()
3278 name = priv->wq_name; in stmmac_fpe_start_wq()
3279 sprintf(name, "%s-fpe", priv->dev->name); in stmmac_fpe_start_wq()
3281 priv->fpe_wq = create_singlethread_workqueue(name); in stmmac_fpe_start_wq()
3282 if (!priv->fpe_wq) { in stmmac_fpe_start_wq()
3283 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); in stmmac_fpe_start_wq()
3285 return -ENOMEM; in stmmac_fpe_start_wq()
3287 netdev_info(priv->dev, "FPE workqueue start"); in stmmac_fpe_start_wq()
3293 * stmmac_hw_setup - setup mac in a usable state.
3302 * 0 on success and an appropriate (-)ve integer as defined in errno.h
3308 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_hw_setup()
3309 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_hw_setup()
3317 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", in stmmac_hw_setup()
3323 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); in stmmac_hw_setup()
3326 if (priv->hw->pcs) { in stmmac_hw_setup()
3327 int speed = priv->plat->mac_port_sel_speed; in stmmac_hw_setup()
3331 priv->hw->ps = speed; in stmmac_hw_setup()
3333 dev_warn(priv->device, "invalid port speed\n"); in stmmac_hw_setup()
3334 priv->hw->ps = 0; in stmmac_hw_setup()
3339 stmmac_core_init(priv, priv->hw, dev); in stmmac_hw_setup()
3347 ret = stmmac_rx_ipc(priv, priv->hw); in stmmac_hw_setup()
3349 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); in stmmac_hw_setup()
3350 priv->plat->rx_coe = STMMAC_RX_COE_NONE; in stmmac_hw_setup()
3351 priv->hw->rx_csum = 0; in stmmac_hw_setup()
3355 stmmac_mac_set(priv, priv->ioaddr, true); in stmmac_hw_setup()
3363 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); in stmmac_hw_setup()
3365 netdev_warn(priv->dev, in stmmac_hw_setup()
3371 if (ret == -EOPNOTSUPP) in stmmac_hw_setup()
3372 netdev_info(priv->dev, "PTP not supported by HW\n"); in stmmac_hw_setup()
3374 netdev_warn(priv->dev, "PTP init failed\n"); in stmmac_hw_setup()
3378 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; in stmmac_hw_setup()
3381 if (!priv->tx_lpi_timer) in stmmac_hw_setup()
3382 priv->tx_lpi_timer = eee_timer * 1000; in stmmac_hw_setup()
3384 if (priv->use_riwt) { in stmmac_hw_setup()
3388 if (!priv->rx_riwt[queue]) in stmmac_hw_setup()
3389 priv->rx_riwt[queue] = DEF_DMA_RIWT; in stmmac_hw_setup()
3391 stmmac_rx_watchdog(priv, priv->ioaddr, in stmmac_hw_setup()
3392 priv->rx_riwt[queue], queue); in stmmac_hw_setup()
3396 if (priv->hw->pcs) in stmmac_hw_setup()
3397 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); in stmmac_hw_setup()
3403 if (priv->tso) { in stmmac_hw_setup()
3405 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; in stmmac_hw_setup()
3407 /* TSO and TBS cannot co-exist */ in stmmac_hw_setup()
3408 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_hw_setup()
3411 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); in stmmac_hw_setup()
3416 sph_en = (priv->hw->rx_csum > 0) && priv->sph; in stmmac_hw_setup()
3418 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); in stmmac_hw_setup()
3422 if (priv->dma_cap.vlins) in stmmac_hw_setup()
3423 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); in stmmac_hw_setup()
3427 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; in stmmac_hw_setup()
3428 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; in stmmac_hw_setup()
3430 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); in stmmac_hw_setup()
3434 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); in stmmac_hw_setup()
3435 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); in stmmac_hw_setup()
3440 if (priv->dma_cap.fpesel) { in stmmac_hw_setup()
3443 if (priv->plat->fpe_cfg->enable) in stmmac_hw_setup()
3454 clk_disable_unprepare(priv->plat->clk_ptp_ref); in stmmac_hw_teardown()
3465 irq_idx = priv->plat->tx_queues_to_use; in stmmac_free_irq()
3468 for (j = irq_idx - 1; j >= 0; j--) { in stmmac_free_irq()
3469 if (priv->tx_irq[j] > 0) { in stmmac_free_irq()
3470 irq_set_affinity_hint(priv->tx_irq[j], NULL); in stmmac_free_irq()
3471 free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); in stmmac_free_irq()
3474 irq_idx = priv->plat->rx_queues_to_use; in stmmac_free_irq()
3477 for (j = irq_idx - 1; j >= 0; j--) { in stmmac_free_irq()
3478 if (priv->rx_irq[j] > 0) { in stmmac_free_irq()
3479 irq_set_affinity_hint(priv->rx_irq[j], NULL); in stmmac_free_irq()
3480 free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); in stmmac_free_irq()
3484 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) in stmmac_free_irq()
3485 free_irq(priv->sfty_ue_irq, dev); in stmmac_free_irq()
3488 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) in stmmac_free_irq()
3489 free_irq(priv->sfty_ce_irq, dev); in stmmac_free_irq()
3492 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) in stmmac_free_irq()
3493 free_irq(priv->lpi_irq, dev); in stmmac_free_irq()
3496 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) in stmmac_free_irq()
3497 free_irq(priv->wol_irq, dev); in stmmac_free_irq()
3500 free_irq(dev->irq, dev); in stmmac_free_irq()
3520 int_name = priv->int_name_mac; in stmmac_request_irq_multi_msi()
3521 sprintf(int_name, "%s:%s", dev->name, "mac"); in stmmac_request_irq_multi_msi()
3522 ret = request_irq(dev->irq, stmmac_mac_interrupt, in stmmac_request_irq_multi_msi()
3525 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3527 __func__, dev->irq, ret); in stmmac_request_irq_multi_msi()
3535 priv->wol_irq_disabled = true; in stmmac_request_irq_multi_msi()
3536 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3537 int_name = priv->int_name_wol; in stmmac_request_irq_multi_msi()
3538 sprintf(int_name, "%s:%s", dev->name, "wol"); in stmmac_request_irq_multi_msi()
3539 ret = request_irq(priv->wol_irq, in stmmac_request_irq_multi_msi()
3543 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3545 __func__, priv->wol_irq, ret); in stmmac_request_irq_multi_msi()
3554 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3555 int_name = priv->int_name_lpi; in stmmac_request_irq_multi_msi()
3556 sprintf(int_name, "%s:%s", dev->name, "lpi"); in stmmac_request_irq_multi_msi()
3557 ret = request_irq(priv->lpi_irq, in stmmac_request_irq_multi_msi()
3561 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3563 __func__, priv->lpi_irq, ret); in stmmac_request_irq_multi_msi()
3572 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3573 int_name = priv->int_name_sfty_ce; in stmmac_request_irq_multi_msi()
3574 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); in stmmac_request_irq_multi_msi()
3575 ret = request_irq(priv->sfty_ce_irq, in stmmac_request_irq_multi_msi()
3579 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3581 __func__, priv->sfty_ce_irq, ret); in stmmac_request_irq_multi_msi()
3590 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3591 int_name = priv->int_name_sfty_ue; in stmmac_request_irq_multi_msi()
3592 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); in stmmac_request_irq_multi_msi()
3593 ret = request_irq(priv->sfty_ue_irq, in stmmac_request_irq_multi_msi()
3597 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3599 __func__, priv->sfty_ue_irq, ret); in stmmac_request_irq_multi_msi()
3606 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { in stmmac_request_irq_multi_msi()
3609 if (priv->rx_irq[i] == 0) in stmmac_request_irq_multi_msi()
3612 int_name = priv->int_name_rx_irq[i]; in stmmac_request_irq_multi_msi()
3613 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); in stmmac_request_irq_multi_msi()
3614 ret = request_irq(priv->rx_irq[i], in stmmac_request_irq_multi_msi()
3616 0, int_name, &priv->dma_conf.rx_queue[i]); in stmmac_request_irq_multi_msi()
3618 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3619 "%s: alloc rx-%d MSI %d (error: %d)\n", in stmmac_request_irq_multi_msi()
3620 __func__, i, priv->rx_irq[i], ret); in stmmac_request_irq_multi_msi()
3627 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); in stmmac_request_irq_multi_msi()
3631 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { in stmmac_request_irq_multi_msi()
3634 if (priv->tx_irq[i] == 0) in stmmac_request_irq_multi_msi()
3637 int_name = priv->int_name_tx_irq[i]; in stmmac_request_irq_multi_msi()
3638 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); in stmmac_request_irq_multi_msi()
3639 ret = request_irq(priv->tx_irq[i], in stmmac_request_irq_multi_msi()
3641 0, int_name, &priv->dma_conf.tx_queue[i]); in stmmac_request_irq_multi_msi()
3643 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3644 "%s: alloc tx-%d MSI %d (error: %d)\n", in stmmac_request_irq_multi_msi()
3645 __func__, i, priv->tx_irq[i], ret); in stmmac_request_irq_multi_msi()
3652 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); in stmmac_request_irq_multi_msi()
3668 ret = request_irq(dev->irq, stmmac_interrupt, in stmmac_request_irq_single()
3669 IRQF_SHARED, dev->name, dev); in stmmac_request_irq_single()
3671 netdev_err(priv->dev, in stmmac_request_irq_single()
3673 __func__, dev->irq, ret); in stmmac_request_irq_single()
3681 priv->wol_irq_disabled = true; in stmmac_request_irq_single()
3682 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { in stmmac_request_irq_single()
3683 ret = request_irq(priv->wol_irq, stmmac_interrupt, in stmmac_request_irq_single()
3684 IRQF_SHARED, dev->name, dev); in stmmac_request_irq_single()
3686 netdev_err(priv->dev, in stmmac_request_irq_single()
3688 __func__, priv->wol_irq, ret); in stmmac_request_irq_single()
3695 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { in stmmac_request_irq_single()
3696 ret = request_irq(priv->lpi_irq, stmmac_interrupt, in stmmac_request_irq_single()
3697 IRQF_SHARED, dev->name, dev); in stmmac_request_irq_single()
3699 netdev_err(priv->dev, in stmmac_request_irq_single()
3701 __func__, priv->lpi_irq, ret); in stmmac_request_irq_single()
3720 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) in stmmac_request_irq()
3729 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue
3745 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", in stmmac_setup_dma_desc()
3747 return ERR_PTR(-ENOMEM); in stmmac_setup_dma_desc()
3757 dma_conf->dma_buf_sz = bfsize; in stmmac_setup_dma_desc()
3761 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; in stmmac_setup_dma_desc()
3762 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; in stmmac_setup_dma_desc()
3764 if (!dma_conf->dma_tx_size) in stmmac_setup_dma_desc()
3765 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; in stmmac_setup_dma_desc()
3766 if (!dma_conf->dma_rx_size) in stmmac_setup_dma_desc()
3767 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; in stmmac_setup_dma_desc()
3770 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { in stmmac_setup_dma_desc()
3771 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; in stmmac_setup_dma_desc()
3772 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; in stmmac_setup_dma_desc()
3774 /* Setup per-TXQ tbs flag before TX descriptor alloc */ in stmmac_setup_dma_desc()
3775 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; in stmmac_setup_dma_desc()
3780 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", in stmmac_setup_dma_desc()
3785 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); in stmmac_setup_dma_desc()
3787 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", in stmmac_setup_dma_desc()
3802 * __stmmac_open - open entry point of the driver
3808 * 0 on success and an appropriate (-)ve integer as defined in errno.h
3815 int mode = priv->plat->phy_interface; in __stmmac_open()
3819 ret = pm_runtime_resume_and_get(priv->device); in __stmmac_open()
3823 if (priv->hw->pcs != STMMAC_PCS_TBI && in __stmmac_open()
3824 priv->hw->pcs != STMMAC_PCS_RTBI && in __stmmac_open()
3825 (!priv->hw->xpcs || in __stmmac_open()
3826 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73) && in __stmmac_open()
3827 !priv->hw->lynx_pcs) { in __stmmac_open()
3830 netdev_err(priv->dev, in __stmmac_open()
3837 priv->rx_copybreak = STMMAC_RX_COPYBREAK; in __stmmac_open()
3839 buf_sz = dma_conf->dma_buf_sz; in __stmmac_open()
3841 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) in __stmmac_open()
3842 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; in __stmmac_open()
3843 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); in __stmmac_open()
3847 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && in __stmmac_open()
3848 priv->plat->serdes_powerup) { in __stmmac_open()
3849 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); in __stmmac_open()
3851 netdev_err(priv->dev, "%s: Serdes powerup failed\n", in __stmmac_open()
3859 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); in __stmmac_open()
3865 phylink_start(priv->phylink); in __stmmac_open()
3867 phylink_speed_up(priv->phylink); in __stmmac_open()
3874 netif_tx_start_all_queues(priv->dev); in __stmmac_open()
3880 phylink_stop(priv->phylink); in __stmmac_open()
3882 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in __stmmac_open()
3883 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); in __stmmac_open()
3887 phylink_disconnect_phy(priv->phylink); in __stmmac_open()
3889 pm_runtime_put(priv->device); in __stmmac_open()
3899 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); in stmmac_open()
3913 set_bit(__FPE_REMOVING, &priv->fpe_task_state); in stmmac_fpe_stop_wq()
3915 if (priv->fpe_wq) { in stmmac_fpe_stop_wq()
3916 destroy_workqueue(priv->fpe_wq); in stmmac_fpe_stop_wq()
3917 priv->fpe_wq = NULL; in stmmac_fpe_stop_wq()
3920 netdev_info(priv->dev, "FPE workqueue stop"); in stmmac_fpe_stop_wq()
3924 * stmmac_release - close entry point of the driver
3934 if (device_may_wakeup(priv->device)) in stmmac_release()
3935 phylink_speed_down(priv->phylink, false); in stmmac_release()
3937 phylink_stop(priv->phylink); in stmmac_release()
3938 phylink_disconnect_phy(priv->phylink); in stmmac_release()
3942 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_release()
3943 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); in stmmac_release()
3950 if (priv->eee_enabled) { in stmmac_release()
3951 priv->tx_path_in_lpi_mode = false; in stmmac_release()
3952 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_release()
3959 free_dma_desc_resources(priv, &priv->dma_conf); in stmmac_release()
3962 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_release()
3965 if (priv->plat->serdes_powerdown) in stmmac_release()
3966 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); in stmmac_release()
3972 pm_runtime_put(priv->device); in stmmac_release()
3974 if (priv->dma_cap.fpesel) in stmmac_release()
3987 if (!priv->dma_cap.vlins) in stmmac_vlan_insert()
3991 if (skb->vlan_proto == htons(ETH_P_8021AD)) { in stmmac_vlan_insert()
3998 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_vlan_insert()
3999 p = &tx_q->dma_entx[tx_q->cur_tx].basic; in stmmac_vlan_insert()
4001 p = &tx_q->dma_tx[tx_q->cur_tx]; in stmmac_vlan_insert()
4007 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); in stmmac_vlan_insert()
4012 * stmmac_tso_allocator - close entry point of the driver
4025 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_tso_allocator()
4035 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, in stmmac_tso_allocator()
4036 priv->dma_conf.dma_tx_size); in stmmac_tso_allocator()
4037 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); in stmmac_tso_allocator()
4039 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tso_allocator()
4040 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; in stmmac_tso_allocator()
4042 desc = &tx_q->dma_tx[tx_q->cur_tx]; in stmmac_tso_allocator()
4044 curr_addr = des + (total_len - tmp_len); in stmmac_tso_allocator()
4045 if (priv->dma_cap.addr64 <= 32) in stmmac_tso_allocator()
4046 desc->des0 = cpu_to_le32(curr_addr); in stmmac_tso_allocator()
4058 tmp_len -= TSO_MAX_BUFF_SIZE; in stmmac_tso_allocator()
4064 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_flush_tx_descriptors()
4067 if (likely(priv->extend_desc)) in stmmac_flush_tx_descriptors()
4069 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_flush_tx_descriptors()
4080 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); in stmmac_flush_tx_descriptors()
4081 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); in stmmac_flush_tx_descriptors()
4085 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
4093 * --------
4094 * | DES0 |---> buffer1 = L2/L3/L4 header
4095 * | DES1 |---> TCP Payload (can continue on next descr...)
4096 * | DES2 |---> buffer 1 and 2 len
4097 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
4098 * --------
4102 * --------
4103 * | DES0 | --| Split TCP Payload on Buffers 1 and 2
4104 * | DES1 | --|
4105 * | DES2 | --> buffer 1 and 2 len
4107 * --------
4115 int nfrags = skb_shinfo(skb)->nr_frags; in stmmac_tso_xmit()
4127 tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_tso_xmit()
4128 txq_stats = &priv->xstats.txq_stats[queue]; in stmmac_tso_xmit()
4129 first_tx = tx_q->cur_tx; in stmmac_tso_xmit()
4132 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { in stmmac_tso_xmit()
4142 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { in stmmac_tso_xmit()
4144 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, in stmmac_tso_xmit()
4147 netdev_err(priv->dev, in stmmac_tso_xmit()
4154 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ in stmmac_tso_xmit()
4156 mss = skb_shinfo(skb)->gso_size; in stmmac_tso_xmit()
4159 if (mss != tx_q->mss) { in stmmac_tso_xmit()
4160 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tso_xmit()
4161 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; in stmmac_tso_xmit()
4163 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; in stmmac_tso_xmit()
4166 tx_q->mss = mss; in stmmac_tso_xmit()
4167 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, in stmmac_tso_xmit()
4168 priv->dma_conf.dma_tx_size); in stmmac_tso_xmit()
4169 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); in stmmac_tso_xmit()
4175 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, in stmmac_tso_xmit()
4176 skb->data_len); in stmmac_tso_xmit()
4182 first_entry = tx_q->cur_tx; in stmmac_tso_xmit()
4183 WARN_ON(tx_q->tx_skbuff[first_entry]); in stmmac_tso_xmit()
4185 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tso_xmit()
4186 desc = &tx_q->dma_entx[first_entry].basic; in stmmac_tso_xmit()
4188 desc = &tx_q->dma_tx[first_entry]; in stmmac_tso_xmit()
4195 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), in stmmac_tso_xmit()
4197 if (dma_mapping_error(priv->device, des)) in stmmac_tso_xmit()
4200 if (priv->dma_cap.addr64 <= 32) { in stmmac_tso_xmit()
4201 first->des0 = cpu_to_le32(des); in stmmac_tso_xmit()
4205 first->des1 = cpu_to_le32(des + proto_hdr_len); in stmmac_tso_xmit()
4208 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; in stmmac_tso_xmit()
4220 * non-paged SKB data, the DMA buffer address should be saved to in stmmac_tso_xmit()
4221 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, in stmmac_tso_xmit()
4222 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee in stmmac_tso_xmit()
4226 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf in stmmac_tso_xmit()
4231 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; in stmmac_tso_xmit()
4232 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb); in stmmac_tso_xmit()
4233 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false; in stmmac_tso_xmit()
4234 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_tso_xmit()
4238 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in stmmac_tso_xmit()
4240 des = skb_frag_dma_map(priv->device, frag, 0, in stmmac_tso_xmit()
4243 if (dma_mapping_error(priv->device, des)) in stmmac_tso_xmit()
4247 (i == nfrags - 1), queue); in stmmac_tso_xmit()
4249 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; in stmmac_tso_xmit()
4250 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); in stmmac_tso_xmit()
4251 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; in stmmac_tso_xmit()
4252 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_tso_xmit()
4255 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; in stmmac_tso_xmit()
4258 tx_q->tx_skbuff[tx_q->cur_tx] = skb; in stmmac_tso_xmit()
4259 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_tso_xmit()
4262 tx_packets = (tx_q->cur_tx + 1) - first_tx; in stmmac_tso_xmit()
4263 tx_q->tx_count_frames += tx_packets; in stmmac_tso_xmit()
4265 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) in stmmac_tso_xmit()
4267 else if (!priv->tx_coal_frames[queue]) in stmmac_tso_xmit()
4269 else if (tx_packets > priv->tx_coal_frames[queue]) in stmmac_tso_xmit()
4271 else if ((tx_q->tx_count_frames % in stmmac_tso_xmit()
4272 priv->tx_coal_frames[queue]) < tx_packets) in stmmac_tso_xmit()
4278 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tso_xmit()
4279 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; in stmmac_tso_xmit()
4281 desc = &tx_q->dma_tx[tx_q->cur_tx]; in stmmac_tso_xmit()
4283 tx_q->tx_count_frames = 0; in stmmac_tso_xmit()
4292 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); in stmmac_tso_xmit()
4295 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", in stmmac_tso_xmit()
4297 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tso_xmit()
4300 u64_stats_update_begin(&txq_stats->q_syncp); in stmmac_tso_xmit()
4301 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); in stmmac_tso_xmit()
4302 u64_stats_inc(&txq_stats->q.tx_tso_frames); in stmmac_tso_xmit()
4303 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); in stmmac_tso_xmit()
4305 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); in stmmac_tso_xmit()
4306 u64_stats_update_end(&txq_stats->q_syncp); in stmmac_tso_xmit()
4308 if (priv->sarc_type) in stmmac_tso_xmit()
4309 stmmac_set_desc_sarc(priv, first, priv->sarc_type); in stmmac_tso_xmit()
4313 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && in stmmac_tso_xmit()
4314 priv->hwts_tx_en)) { in stmmac_tso_xmit()
4316 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; in stmmac_tso_xmit()
4324 1, tx_q->tx_skbuff_dma[first_entry].last_segment, in stmmac_tso_xmit()
4325 hdr / 4, (skb->len - proto_hdr_len)); in stmmac_tso_xmit()
4340 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, in stmmac_tso_xmit()
4341 tx_q->cur_tx, first, nfrags); in stmmac_tso_xmit()
4343 print_pkt(skb->data, skb_headlen(skb)); in stmmac_tso_xmit()
4346 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); in stmmac_tso_xmit()
4354 dev_err(priv->device, "Tx dma map failed\n"); in stmmac_tso_xmit()
4356 priv->xstats.tx_dropped++; in stmmac_tso_xmit()
4361 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype
4383 * stmmac_xmit - Tx entry point of the driver
4397 int nfrags = skb_shinfo(skb)->nr_frags; in stmmac_xmit()
4398 int gso = skb_shinfo(skb)->gso_type; in stmmac_xmit()
4407 tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_xmit()
4408 txq_stats = &priv->xstats.txq_stats[queue]; in stmmac_xmit()
4409 first_tx = tx_q->cur_tx; in stmmac_xmit()
4411 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) in stmmac_xmit()
4415 if (skb_is_gso(skb) && priv->tso) { in stmmac_xmit()
4418 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) in stmmac_xmit()
4424 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, in stmmac_xmit()
4427 netdev_err(priv->dev, in stmmac_xmit()
4437 entry = tx_q->cur_tx; in stmmac_xmit()
4439 WARN_ON(tx_q->tx_skbuff[first_entry]); in stmmac_xmit()
4441 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); in stmmac_xmit()
4446 * Packets that won't trigger the COE e.g. most DSA-tagged packets will in stmmac_xmit()
4450 (priv->plat->tx_queues_cfg[queue].coe_unsupported || in stmmac_xmit()
4457 if (likely(priv->extend_desc)) in stmmac_xmit()
4458 desc = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_xmit()
4459 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xmit()
4460 desc = &tx_q->dma_entx[entry].basic; in stmmac_xmit()
4462 desc = tx_q->dma_tx + entry; in stmmac_xmit()
4469 enh_desc = priv->plat->enh_desc; in stmmac_xmit()
4472 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); in stmmac_xmit()
4476 if (unlikely(entry < 0) && (entry != -EINVAL)) in stmmac_xmit()
4481 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in stmmac_xmit()
4483 bool last_segment = (i == (nfrags - 1)); in stmmac_xmit()
4485 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); in stmmac_xmit()
4486 WARN_ON(tx_q->tx_skbuff[entry]); in stmmac_xmit()
4488 if (likely(priv->extend_desc)) in stmmac_xmit()
4489 desc = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_xmit()
4490 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xmit()
4491 desc = &tx_q->dma_entx[entry].basic; in stmmac_xmit()
4493 desc = tx_q->dma_tx + entry; in stmmac_xmit()
4495 des = skb_frag_dma_map(priv->device, frag, 0, len, in stmmac_xmit()
4497 if (dma_mapping_error(priv->device, des)) in stmmac_xmit()
4500 tx_q->tx_skbuff_dma[entry].buf = des; in stmmac_xmit()
4504 tx_q->tx_skbuff_dma[entry].map_as_page = true; in stmmac_xmit()
4505 tx_q->tx_skbuff_dma[entry].len = len; in stmmac_xmit()
4506 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; in stmmac_xmit()
4507 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_xmit()
4511 priv->mode, 1, last_segment, skb->len); in stmmac_xmit()
4515 tx_q->tx_skbuff[entry] = skb; in stmmac_xmit()
4516 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_xmit()
4519 * segment is reset and the timer re-started to clean the tx status. in stmmac_xmit()
4523 tx_packets = (entry + 1) - first_tx; in stmmac_xmit()
4524 tx_q->tx_count_frames += tx_packets; in stmmac_xmit()
4526 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) in stmmac_xmit()
4528 else if (!priv->tx_coal_frames[queue]) in stmmac_xmit()
4530 else if (tx_packets > priv->tx_coal_frames[queue]) in stmmac_xmit()
4532 else if ((tx_q->tx_count_frames % in stmmac_xmit()
4533 priv->tx_coal_frames[queue]) < tx_packets) in stmmac_xmit()
4539 if (likely(priv->extend_desc)) in stmmac_xmit()
4540 desc = &tx_q->dma_etx[entry].basic; in stmmac_xmit()
4541 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xmit()
4542 desc = &tx_q->dma_entx[entry].basic; in stmmac_xmit()
4544 desc = &tx_q->dma_tx[entry]; in stmmac_xmit()
4546 tx_q->tx_count_frames = 0; in stmmac_xmit()
4555 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); in stmmac_xmit()
4556 tx_q->cur_tx = entry; in stmmac_xmit()
4559 netdev_dbg(priv->dev, in stmmac_xmit()
4561 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, in stmmac_xmit()
4564 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); in stmmac_xmit()
4565 print_pkt(skb->data, skb->len); in stmmac_xmit()
4569 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", in stmmac_xmit()
4571 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_xmit()
4574 u64_stats_update_begin(&txq_stats->q_syncp); in stmmac_xmit()
4575 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); in stmmac_xmit()
4577 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); in stmmac_xmit()
4578 u64_stats_update_end(&txq_stats->q_syncp); in stmmac_xmit()
4580 if (priv->sarc_type) in stmmac_xmit()
4581 stmmac_set_desc_sarc(priv, first, priv->sarc_type); in stmmac_xmit()
4592 des = dma_map_single(priv->device, skb->data, in stmmac_xmit()
4594 if (dma_mapping_error(priv->device, des)) in stmmac_xmit()
4597 tx_q->tx_skbuff_dma[first_entry].buf = des; in stmmac_xmit()
4598 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_xmit()
4599 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; in stmmac_xmit()
4603 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; in stmmac_xmit()
4604 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; in stmmac_xmit()
4606 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && in stmmac_xmit()
4607 priv->hwts_tx_en)) { in stmmac_xmit()
4609 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; in stmmac_xmit()
4615 csum_insertion, priv->mode, 0, last_segment, in stmmac_xmit()
4616 skb->len); in stmmac_xmit()
4619 if (tx_q->tbs & STMMAC_TBS_EN) { in stmmac_xmit()
4620 struct timespec64 ts = ns_to_timespec64(skb->tstamp); in stmmac_xmit()
4622 tbs_desc = &tx_q->dma_entx[first_entry]; in stmmac_xmit()
4628 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); in stmmac_xmit()
4630 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xmit()
4638 netdev_err(priv->dev, "Tx DMA map failed\n"); in stmmac_xmit()
4640 priv->xstats.tx_dropped++; in stmmac_xmit()
4647 __be16 vlan_proto = veth->h_vlan_proto; in stmmac_rx_vlan()
4651 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || in stmmac_rx_vlan()
4653 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { in stmmac_rx_vlan()
4655 vlanid = ntohs(veth->h_vlan_TCI); in stmmac_rx_vlan()
4656 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); in stmmac_rx_vlan()
4663 * stmmac_rx_refill - refill used skb preallocated buffers
4667 * that is based on zero-copy.
4671 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_rx_refill()
4673 unsigned int entry = rx_q->dirty_rx; in stmmac_rx_refill()
4676 if (priv->dma_cap.host_dma_width <= 32) in stmmac_rx_refill()
4679 while (dirty-- > 0) { in stmmac_rx_refill()
4680 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; in stmmac_rx_refill()
4684 if (priv->extend_desc) in stmmac_rx_refill()
4685 p = (struct dma_desc *)(rx_q->dma_erx + entry); in stmmac_rx_refill()
4687 p = rx_q->dma_rx + entry; in stmmac_rx_refill()
4689 if (!buf->page) { in stmmac_rx_refill()
4690 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); in stmmac_rx_refill()
4691 if (!buf->page) in stmmac_rx_refill()
4695 if (priv->sph && !buf->sec_page) { in stmmac_rx_refill()
4696 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); in stmmac_rx_refill()
4697 if (!buf->sec_page) in stmmac_rx_refill()
4700 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); in stmmac_rx_refill()
4703 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; in stmmac_rx_refill()
4705 stmmac_set_desc_addr(priv, p, buf->addr); in stmmac_rx_refill()
4706 if (priv->sph) in stmmac_rx_refill()
4707 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); in stmmac_rx_refill()
4709 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); in stmmac_rx_refill()
4712 rx_q->rx_count_frames++; in stmmac_rx_refill()
4713 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; in stmmac_rx_refill()
4714 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) in stmmac_rx_refill()
4715 rx_q->rx_count_frames = 0; in stmmac_rx_refill()
4717 use_rx_wd = !priv->rx_coal_frames[queue]; in stmmac_rx_refill()
4718 use_rx_wd |= rx_q->rx_count_frames > 0; in stmmac_rx_refill()
4719 if (!priv->use_riwt) in stmmac_rx_refill()
4725 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); in stmmac_rx_refill()
4727 rx_q->dirty_rx = entry; in stmmac_rx_refill()
4728 rx_q->rx_tail_addr = rx_q->dma_rx_phy + in stmmac_rx_refill()
4729 (rx_q->dirty_rx * sizeof(struct dma_desc)); in stmmac_rx_refill()
4730 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); in stmmac_rx_refill()
4738 int coe = priv->hw->rx_csum; in stmmac_rx_buf1_len()
4741 if (priv->sph && len) in stmmac_rx_buf1_len()
4746 if (priv->sph && hlen) { in stmmac_rx_buf1_len()
4747 priv->xstats.rx_split_hdr_pkt_n++; in stmmac_rx_buf1_len()
4753 return priv->dma_conf.dma_buf_sz; in stmmac_rx_buf1_len()
4758 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); in stmmac_rx_buf1_len()
4765 int coe = priv->hw->rx_csum; in stmmac_rx_buf2_len()
4769 if (!priv->sph) in stmmac_rx_buf2_len()
4774 return priv->dma_conf.dma_buf_sz; in stmmac_rx_buf2_len()
4779 return plen - len; in stmmac_rx_buf2_len()
4785 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; in stmmac_xdp_xmit_xdpf()
4786 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_xdp_xmit_xdpf()
4787 unsigned int entry = tx_q->cur_tx; in stmmac_xdp_xmit_xdpf()
4795 if (likely(priv->extend_desc)) in stmmac_xdp_xmit_xdpf()
4796 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_xdp_xmit_xdpf()
4797 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xdp_xmit_xdpf()
4798 tx_desc = &tx_q->dma_entx[entry].basic; in stmmac_xdp_xmit_xdpf()
4800 tx_desc = tx_q->dma_tx + entry; in stmmac_xdp_xmit_xdpf()
4803 dma_addr = dma_map_single(priv->device, xdpf->data, in stmmac_xdp_xmit_xdpf()
4804 xdpf->len, DMA_TO_DEVICE); in stmmac_xdp_xmit_xdpf()
4805 if (dma_mapping_error(priv->device, dma_addr)) in stmmac_xdp_xmit_xdpf()
4808 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; in stmmac_xdp_xmit_xdpf()
4810 struct page *page = virt_to_page(xdpf->data); in stmmac_xdp_xmit_xdpf()
4813 xdpf->headroom; in stmmac_xdp_xmit_xdpf()
4814 dma_sync_single_for_device(priv->device, dma_addr, in stmmac_xdp_xmit_xdpf()
4815 xdpf->len, DMA_BIDIRECTIONAL); in stmmac_xdp_xmit_xdpf()
4817 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; in stmmac_xdp_xmit_xdpf()
4820 tx_q->tx_skbuff_dma[entry].buf = dma_addr; in stmmac_xdp_xmit_xdpf()
4821 tx_q->tx_skbuff_dma[entry].map_as_page = false; in stmmac_xdp_xmit_xdpf()
4822 tx_q->tx_skbuff_dma[entry].len = xdpf->len; in stmmac_xdp_xmit_xdpf()
4823 tx_q->tx_skbuff_dma[entry].last_segment = true; in stmmac_xdp_xmit_xdpf()
4824 tx_q->tx_skbuff_dma[entry].is_jumbo = false; in stmmac_xdp_xmit_xdpf()
4826 tx_q->xdpf[entry] = xdpf; in stmmac_xdp_xmit_xdpf()
4830 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, in stmmac_xdp_xmit_xdpf()
4831 true, priv->mode, true, true, in stmmac_xdp_xmit_xdpf()
4832 xdpf->len); in stmmac_xdp_xmit_xdpf()
4834 tx_q->tx_count_frames++; in stmmac_xdp_xmit_xdpf()
4836 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) in stmmac_xdp_xmit_xdpf()
4842 tx_q->tx_count_frames = 0; in stmmac_xdp_xmit_xdpf()
4844 u64_stats_update_begin(&txq_stats->q_syncp); in stmmac_xdp_xmit_xdpf()
4845 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); in stmmac_xdp_xmit_xdpf()
4846 u64_stats_update_end(&txq_stats->q_syncp); in stmmac_xdp_xmit_xdpf()
4849 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xdp_xmit_xdpf()
4851 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); in stmmac_xdp_xmit_xdpf()
4852 tx_q->cur_tx = entry; in stmmac_xdp_xmit_xdpf()
4865 while (index >= priv->plat->tx_queues_to_use) in stmmac_xdp_get_tx_queue()
4866 index -= priv->plat->tx_queues_to_use; in stmmac_xdp_get_tx_queue()
4884 nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit_back()
4887 /* Avoids TX time-out as we are sharing with slow path */ in stmmac_xdp_xmit_back()
4915 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) in __stmmac_xdp_run_prog()
4921 bpf_warn_invalid_xdp_action(priv->dev, prog, act); in __stmmac_xdp_run_prog()
4924 trace_xdp_exception(priv->dev, prog, act); in __stmmac_xdp_run_prog()
4940 prog = READ_ONCE(priv->xdp_prog); in stmmac_xdp_run_prog()
4948 return ERR_PTR(-res); in stmmac_xdp_run_prog()
4969 unsigned int metasize = xdp->data - xdp->data_meta; in stmmac_construct_skb_zc()
4970 unsigned int datasize = xdp->data_end - xdp->data; in stmmac_construct_skb_zc()
4973 skb = __napi_alloc_skb(&ch->rxtx_napi, in stmmac_construct_skb_zc()
4974 xdp->data_end - xdp->data_hard_start, in stmmac_construct_skb_zc()
4979 skb_reserve(skb, xdp->data - xdp->data_hard_start); in stmmac_construct_skb_zc()
4980 memcpy(__skb_put(skb, datasize), xdp->data, datasize); in stmmac_construct_skb_zc()
4991 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; in stmmac_dispatch_skb_zc()
4992 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_dispatch_skb_zc()
4993 unsigned int len = xdp->data_end - xdp->data; in stmmac_dispatch_skb_zc()
4995 int coe = priv->hw->rx_csum; in stmmac_dispatch_skb_zc()
5001 priv->xstats.rx_dropped++; in stmmac_dispatch_skb_zc()
5006 stmmac_rx_vlan(priv->dev, skb); in stmmac_dispatch_skb_zc()
5007 skb->protocol = eth_type_trans(skb, priv->dev); in stmmac_dispatch_skb_zc()
5012 skb->ip_summed = CHECKSUM_UNNECESSARY; in stmmac_dispatch_skb_zc()
5018 napi_gro_receive(&ch->rxtx_napi, skb); in stmmac_dispatch_skb_zc()
5020 u64_stats_update_begin(&rxq_stats->napi_syncp); in stmmac_dispatch_skb_zc()
5021 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); in stmmac_dispatch_skb_zc()
5022 u64_stats_add(&rxq_stats->napi.rx_bytes, len); in stmmac_dispatch_skb_zc()
5023 u64_stats_update_end(&rxq_stats->napi_syncp); in stmmac_dispatch_skb_zc()
5028 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_rx_refill_zc()
5029 unsigned int entry = rx_q->dirty_rx; in stmmac_rx_refill_zc()
5035 while (budget-- > 0 && entry != rx_q->cur_rx) { in stmmac_rx_refill_zc()
5036 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; in stmmac_rx_refill_zc()
5040 if (!buf->xdp) { in stmmac_rx_refill_zc()
5041 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); in stmmac_rx_refill_zc()
5042 if (!buf->xdp) { in stmmac_rx_refill_zc()
5048 if (priv->extend_desc) in stmmac_rx_refill_zc()
5049 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); in stmmac_rx_refill_zc()
5051 rx_desc = rx_q->dma_rx + entry; in stmmac_rx_refill_zc()
5053 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); in stmmac_rx_refill_zc()
5058 rx_q->rx_count_frames++; in stmmac_rx_refill_zc()
5059 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; in stmmac_rx_refill_zc()
5060 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) in stmmac_rx_refill_zc()
5061 rx_q->rx_count_frames = 0; in stmmac_rx_refill_zc()
5063 use_rx_wd = !priv->rx_coal_frames[queue]; in stmmac_rx_refill_zc()
5064 use_rx_wd |= rx_q->rx_count_frames > 0; in stmmac_rx_refill_zc()
5065 if (!priv->use_riwt) in stmmac_rx_refill_zc()
5071 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); in stmmac_rx_refill_zc()
5075 rx_q->dirty_rx = entry; in stmmac_rx_refill_zc()
5076 rx_q->rx_tail_addr = rx_q->dma_rx_phy + in stmmac_rx_refill_zc()
5077 (rx_q->dirty_rx * sizeof(struct dma_desc)); in stmmac_rx_refill_zc()
5078 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); in stmmac_rx_refill_zc()
5096 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; in stmmac_rx_zc()
5097 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_rx_zc()
5100 unsigned int next_entry = rx_q->cur_rx; in stmmac_rx_zc()
5111 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); in stmmac_rx_zc()
5112 if (priv->extend_desc) { in stmmac_rx_zc()
5113 rx_head = (void *)rx_q->dma_erx; in stmmac_rx_zc()
5116 rx_head = (void *)rx_q->dma_rx; in stmmac_rx_zc()
5120 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, in stmmac_rx_zc()
5121 rx_q->dma_rx_phy, desc_size); in stmmac_rx_zc()
5131 if (!count && rx_q->state_saved) { in stmmac_rx_zc()
5132 error = rx_q->state.error; in stmmac_rx_zc()
5133 len = rx_q->state.len; in stmmac_rx_zc()
5135 rx_q->state_saved = false; in stmmac_rx_zc()
5146 buf = &rx_q->buf_pool[entry]; in stmmac_rx_zc()
5154 if (priv->extend_desc) in stmmac_rx_zc()
5155 p = (struct dma_desc *)(rx_q->dma_erx + entry); in stmmac_rx_zc()
5157 p = rx_q->dma_rx + entry; in stmmac_rx_zc()
5160 status = stmmac_rx_status(priv, &priv->xstats, p); in stmmac_rx_zc()
5166 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, in stmmac_rx_zc()
5167 priv->dma_conf.dma_rx_size); in stmmac_rx_zc()
5168 next_entry = rx_q->cur_rx; in stmmac_rx_zc()
5170 if (priv->extend_desc) in stmmac_rx_zc()
5171 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); in stmmac_rx_zc()
5173 np = rx_q->dma_rx + next_entry; in stmmac_rx_zc()
5178 if (!buf->xdp) in stmmac_rx_zc()
5181 if (priv->extend_desc) in stmmac_rx_zc()
5182 stmmac_rx_extended_status(priv, &priv->xstats, in stmmac_rx_zc()
5183 rx_q->dma_erx + entry); in stmmac_rx_zc()
5185 xsk_buff_free(buf->xdp); in stmmac_rx_zc()
5186 buf->xdp = NULL; in stmmac_rx_zc()
5189 if (!priv->hwts_rx_en) in stmmac_rx_zc()
5202 xsk_buff_free(buf->xdp); in stmmac_rx_zc()
5203 buf->xdp = NULL; in stmmac_rx_zc()
5209 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); in stmmac_rx_zc()
5210 ctx->priv = priv; in stmmac_rx_zc()
5211 ctx->desc = p; in stmmac_rx_zc()
5212 ctx->ndesc = np; in stmmac_rx_zc()
5220 buf1_len -= ETH_FCS_LEN; in stmmac_rx_zc()
5221 len -= ETH_FCS_LEN; in stmmac_rx_zc()
5225 buf->xdp->data_end = buf->xdp->data + buf1_len; in stmmac_rx_zc()
5226 xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); in stmmac_rx_zc()
5228 prog = READ_ONCE(priv->xdp_prog); in stmmac_rx_zc()
5229 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); in stmmac_rx_zc()
5233 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); in stmmac_rx_zc()
5234 xsk_buff_free(buf->xdp); in stmmac_rx_zc()
5237 xsk_buff_free(buf->xdp); in stmmac_rx_zc()
5246 buf->xdp = NULL; in stmmac_rx_zc()
5252 rx_q->state_saved = true; in stmmac_rx_zc()
5253 rx_q->state.error = error; in stmmac_rx_zc()
5254 rx_q->state.len = len; in stmmac_rx_zc()
5259 u64_stats_update_begin(&rxq_stats->napi_syncp); in stmmac_rx_zc()
5260 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); in stmmac_rx_zc()
5261 u64_stats_update_end(&rxq_stats->napi_syncp); in stmmac_rx_zc()
5263 priv->xstats.rx_dropped += rx_dropped; in stmmac_rx_zc()
5264 priv->xstats.rx_errors += rx_errors; in stmmac_rx_zc()
5266 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { in stmmac_rx_zc()
5268 xsk_set_rx_need_wakeup(rx_q->xsk_pool); in stmmac_rx_zc()
5270 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); in stmmac_rx_zc()
5279 * stmmac_rx - manage the receive process
5289 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; in stmmac_rx()
5290 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_rx()
5291 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_rx()
5293 int status = 0, coe = priv->hw->rx_csum; in stmmac_rx()
5294 unsigned int next_entry = rx_q->cur_rx; in stmmac_rx()
5302 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); in stmmac_rx()
5303 buf_sz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; in stmmac_rx()
5304 limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); in stmmac_rx()
5309 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); in stmmac_rx()
5310 if (priv->extend_desc) { in stmmac_rx()
5311 rx_head = (void *)rx_q->dma_erx; in stmmac_rx()
5314 rx_head = (void *)rx_q->dma_rx; in stmmac_rx()
5318 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, in stmmac_rx()
5319 rx_q->dma_rx_phy, desc_size); in stmmac_rx()
5329 if (!count && rx_q->state_saved) { in stmmac_rx()
5330 skb = rx_q->state.skb; in stmmac_rx()
5331 error = rx_q->state.error; in stmmac_rx()
5332 len = rx_q->state.len; in stmmac_rx()
5334 rx_q->state_saved = false; in stmmac_rx()
5347 buf = &rx_q->buf_pool[entry]; in stmmac_rx()
5349 if (priv->extend_desc) in stmmac_rx()
5350 p = (struct dma_desc *)(rx_q->dma_erx + entry); in stmmac_rx()
5352 p = rx_q->dma_rx + entry; in stmmac_rx()
5355 status = stmmac_rx_status(priv, &priv->xstats, p); in stmmac_rx()
5360 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, in stmmac_rx()
5361 priv->dma_conf.dma_rx_size); in stmmac_rx()
5362 next_entry = rx_q->cur_rx; in stmmac_rx()
5364 if (priv->extend_desc) in stmmac_rx()
5365 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); in stmmac_rx()
5367 np = rx_q->dma_rx + next_entry; in stmmac_rx()
5371 if (priv->extend_desc) in stmmac_rx()
5372 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); in stmmac_rx()
5374 page_pool_recycle_direct(rx_q->page_pool, buf->page); in stmmac_rx()
5375 buf->page = NULL; in stmmac_rx()
5377 if (!priv->hwts_rx_en) in stmmac_rx()
5392 prefetch(page_address(buf->page) + buf->page_offset); in stmmac_rx()
5393 if (buf->sec_page) in stmmac_rx()
5394 prefetch(page_address(buf->sec_page)); in stmmac_rx()
5404 buf2_len -= ETH_FCS_LEN; in stmmac_rx()
5405 len -= ETH_FCS_LEN; in stmmac_rx()
5407 buf1_len -= ETH_FCS_LEN; in stmmac_rx()
5408 len -= ETH_FCS_LEN; in stmmac_rx()
5415 dma_sync_single_for_cpu(priv->device, buf->addr, in stmmac_rx()
5418 xdp_init_buff(&ctx.xdp, buf_sz, &rx_q->xdp_rxq); in stmmac_rx()
5419 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), in stmmac_rx()
5420 buf->page_offset, buf1_len, true); in stmmac_rx()
5422 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - in stmmac_rx()
5423 buf->page_offset; in stmmac_rx()
5433 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - in stmmac_rx()
5434 buf->page_offset; in stmmac_rx()
5439 unsigned int xdp_res = -PTR_ERR(skb); in stmmac_rx()
5442 page_pool_put_page(rx_q->page_pool, in stmmac_rx()
5445 buf->page = NULL; in stmmac_rx()
5461 buf->page = NULL; in stmmac_rx()
5471 buf1_len = ctx.xdp.data_end - ctx.xdp.data; in stmmac_rx()
5473 skb = napi_alloc_skb(&ch->rx_napi, buf1_len); in stmmac_rx()
5485 page_pool_recycle_direct(rx_q->page_pool, buf->page); in stmmac_rx()
5486 buf->page = NULL; in stmmac_rx()
5488 dma_sync_single_for_cpu(priv->device, buf->addr, in stmmac_rx()
5490 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, in stmmac_rx()
5491 buf->page, buf->page_offset, buf1_len, in stmmac_rx()
5492 priv->dma_conf.dma_buf_sz); in stmmac_rx()
5496 buf->page = NULL; in stmmac_rx()
5500 dma_sync_single_for_cpu(priv->device, buf->sec_addr, in stmmac_rx()
5502 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, in stmmac_rx()
5503 buf->sec_page, 0, buf2_len, in stmmac_rx()
5504 priv->dma_conf.dma_buf_sz); in stmmac_rx()
5508 buf->sec_page = NULL; in stmmac_rx()
5520 stmmac_rx_vlan(priv->dev, skb); in stmmac_rx()
5521 skb->protocol = eth_type_trans(skb, priv->dev); in stmmac_rx()
5526 skb->ip_summed = CHECKSUM_UNNECESSARY; in stmmac_rx()
5532 napi_gro_receive(&ch->rx_napi, skb); in stmmac_rx()
5541 rx_q->state_saved = true; in stmmac_rx()
5542 rx_q->state.skb = skb; in stmmac_rx()
5543 rx_q->state.error = error; in stmmac_rx()
5544 rx_q->state.len = len; in stmmac_rx()
5551 u64_stats_update_begin(&rxq_stats->napi_syncp); in stmmac_rx()
5552 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); in stmmac_rx()
5553 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); in stmmac_rx()
5554 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); in stmmac_rx()
5555 u64_stats_update_end(&rxq_stats->napi_syncp); in stmmac_rx()
5557 priv->xstats.rx_dropped += rx_dropped; in stmmac_rx()
5558 priv->xstats.rx_errors += rx_errors; in stmmac_rx()
5567 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_rx()
5569 u32 chan = ch->index; in stmmac_napi_poll_rx()
5572 rxq_stats = &priv->xstats.rxq_stats[chan]; in stmmac_napi_poll_rx()
5573 u64_stats_update_begin(&rxq_stats->napi_syncp); in stmmac_napi_poll_rx()
5574 u64_stats_inc(&rxq_stats->napi.poll); in stmmac_napi_poll_rx()
5575 u64_stats_update_end(&rxq_stats->napi_syncp); in stmmac_napi_poll_rx()
5581 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_poll_rx()
5582 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); in stmmac_napi_poll_rx()
5583 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_poll_rx()
5593 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_tx()
5595 u32 chan = ch->index; in stmmac_napi_poll_tx()
5598 txq_stats = &priv->xstats.txq_stats[chan]; in stmmac_napi_poll_tx()
5599 u64_stats_update_begin(&txq_stats->napi_syncp); in stmmac_napi_poll_tx()
5600 u64_stats_inc(&txq_stats->napi.poll); in stmmac_napi_poll_tx()
5601 u64_stats_update_end(&txq_stats->napi_syncp); in stmmac_napi_poll_tx()
5609 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_poll_tx()
5610 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); in stmmac_napi_poll_tx()
5611 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_poll_tx()
5621 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_rxtx()
5625 u32 chan = ch->index; in stmmac_napi_poll_rxtx()
5627 rxq_stats = &priv->xstats.rxq_stats[chan]; in stmmac_napi_poll_rxtx()
5628 u64_stats_update_begin(&rxq_stats->napi_syncp); in stmmac_napi_poll_rxtx()
5629 u64_stats_inc(&rxq_stats->napi.poll); in stmmac_napi_poll_rxtx()
5630 u64_stats_update_end(&rxq_stats->napi_syncp); in stmmac_napi_poll_rxtx()
5632 txq_stats = &priv->xstats.txq_stats[chan]; in stmmac_napi_poll_rxtx()
5633 u64_stats_update_begin(&txq_stats->napi_syncp); in stmmac_napi_poll_rxtx()
5634 u64_stats_inc(&txq_stats->napi.poll); in stmmac_napi_poll_rxtx()
5635 u64_stats_update_end(&txq_stats->napi_syncp); in stmmac_napi_poll_rxtx()
5654 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_poll_rxtx()
5658 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_napi_poll_rxtx()
5659 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_poll_rxtx()
5662 return min(rxtx_done, budget - 1); in stmmac_napi_poll_rxtx()
5682 * stmmac_set_rx_mode - entry point for multicast addressing
5694 stmmac_set_filter(priv, priv->hw, dev); in stmmac_set_rx_mode()
5698 * stmmac_change_mtu - entry point to change MTU size for the device.
5705 * 0 on success and an appropriate (-)ve integer as defined in errno.h
5711 int txfifosz = priv->plat->tx_fifo_size; in stmmac_change_mtu()
5717 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_change_mtu()
5719 txfifosz /= priv->plat->tx_queues_to_use; in stmmac_change_mtu()
5722 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); in stmmac_change_mtu()
5723 return -EINVAL; in stmmac_change_mtu()
5730 return -EINVAL; in stmmac_change_mtu()
5733 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); in stmmac_change_mtu()
5737 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", in stmmac_change_mtu()
5748 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); in stmmac_change_mtu()
5757 dev->mtu = mtu; in stmmac_change_mtu()
5768 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) in stmmac_fix_features()
5771 if (!priv->plat->tx_coe) in stmmac_fix_features()
5779 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) in stmmac_fix_features()
5783 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { in stmmac_fix_features()
5785 priv->tso = true; in stmmac_fix_features()
5787 priv->tso = false; in stmmac_fix_features()
5800 priv->hw->rx_csum = priv->plat->rx_coe; in stmmac_set_features()
5802 priv->hw->rx_csum = 0; in stmmac_set_features()
5806 stmmac_rx_ipc(priv, priv->hw); in stmmac_set_features()
5808 if (priv->sph_cap) { in stmmac_set_features()
5809 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; in stmmac_set_features()
5812 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) in stmmac_set_features()
5813 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); in stmmac_set_features()
5821 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_event_status()
5822 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; in stmmac_fpe_event_status()
5823 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; in stmmac_fpe_event_status()
5824 bool *hs_enable = &fpe_cfg->hs_enable; in stmmac_fpe_event_status()
5836 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_event_status()
5855 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && in stmmac_fpe_event_status()
5856 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && in stmmac_fpe_event_status()
5857 priv->fpe_wq) { in stmmac_fpe_event_status()
5858 queue_work(priv->fpe_wq, &priv->fpe_task); in stmmac_fpe_event_status()
5864 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_common_interrupt()
5865 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_common_interrupt()
5870 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_common_interrupt()
5873 if (priv->irq_wake) in stmmac_common_interrupt()
5874 pm_wakeup_event(priv->device, 0); in stmmac_common_interrupt()
5876 if (priv->dma_cap.estsel) in stmmac_common_interrupt()
5877 stmmac_est_irq_status(priv, priv->ioaddr, priv->dev, in stmmac_common_interrupt()
5878 &priv->xstats, tx_cnt); in stmmac_common_interrupt()
5880 if (priv->dma_cap.fpesel) { in stmmac_common_interrupt()
5881 int status = stmmac_fpe_irq_status(priv, priv->ioaddr, in stmmac_common_interrupt()
5882 priv->dev); in stmmac_common_interrupt()
5888 if ((priv->plat->has_gmac) || xmac) { in stmmac_common_interrupt()
5889 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); in stmmac_common_interrupt()
5894 priv->tx_path_in_lpi_mode = true; in stmmac_common_interrupt()
5896 priv->tx_path_in_lpi_mode = false; in stmmac_common_interrupt()
5900 status = stmmac_host_mtl_irq_status(priv, priv->hw, in stmmac_common_interrupt()
5905 if (priv->hw->pcs && in stmmac_common_interrupt()
5906 !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { in stmmac_common_interrupt()
5907 if (priv->xstats.pcs_link) in stmmac_common_interrupt()
5908 netif_carrier_on(priv->dev); in stmmac_common_interrupt()
5910 netif_carrier_off(priv->dev); in stmmac_common_interrupt()
5918 * stmmac_interrupt - main ISR
5925 * o Core interrupts to manage: remote wake-up, management counter, LPI
5934 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_interrupt()
5956 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_mac_interrupt()
5971 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_safety_interrupt()
5984 int chan = tx_q->queue_index; in stmmac_msi_intr_tx()
5992 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_msi_intr_tx()
6011 int chan = rx_q->queue_index; in stmmac_msi_intr_rx()
6018 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_msi_intr_rx()
6027 * stmmac_ioctl - Entry point for the Ioctl
6038 int ret = -EOPNOTSUPP; in stmmac_ioctl()
6041 return -EINVAL; in stmmac_ioctl()
6047 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); in stmmac_ioctl()
6066 int ret = -EOPNOTSUPP; in stmmac_setup_tc_block_cb()
6068 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) in stmmac_setup_tc_block_cb()
6110 return -EOPNOTSUPP; in stmmac_setup_tc()
6117 int gso = skb_shinfo(skb)->gso_type; in stmmac_select_queue()
6129 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; in stmmac_select_queue()
6137 ret = pm_runtime_resume_and_get(priv->device); in stmmac_set_mac_address()
6145 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); in stmmac_set_mac_address()
6148 pm_runtime_put(priv->device); in stmmac_set_mac_address()
6169 le32_to_cpu(ep->basic.des0), in sysfs_display_ring()
6170 le32_to_cpu(ep->basic.des1), in sysfs_display_ring()
6171 le32_to_cpu(ep->basic.des2), in sysfs_display_ring()
6172 le32_to_cpu(ep->basic.des3)); in sysfs_display_ring()
6178 le32_to_cpu(p->des0), le32_to_cpu(p->des1), in sysfs_display_ring()
6179 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); in sysfs_display_ring()
6188 struct net_device *dev = seq->private; in stmmac_rings_status_show()
6190 u32 rx_count = priv->plat->rx_queues_to_use; in stmmac_rings_status_show()
6191 u32 tx_count = priv->plat->tx_queues_to_use; in stmmac_rings_status_show()
6194 if ((dev->flags & IFF_UP) == 0) in stmmac_rings_status_show()
6198 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_rings_status_show()
6202 if (priv->extend_desc) { in stmmac_rings_status_show()
6204 sysfs_display_ring((void *)rx_q->dma_erx, in stmmac_rings_status_show()
6205 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); in stmmac_rings_status_show()
6208 sysfs_display_ring((void *)rx_q->dma_rx, in stmmac_rings_status_show()
6209 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); in stmmac_rings_status_show()
6214 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_rings_status_show()
6218 if (priv->extend_desc) { in stmmac_rings_status_show()
6220 sysfs_display_ring((void *)tx_q->dma_etx, in stmmac_rings_status_show()
6221 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); in stmmac_rings_status_show()
6222 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { in stmmac_rings_status_show()
6224 sysfs_display_ring((void *)tx_q->dma_tx, in stmmac_rings_status_show()
6225 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); in stmmac_rings_status_show()
6251 struct net_device *dev = seq->private; in stmmac_dma_cap_show()
6254 if (!priv->hw_cap_support) { in stmmac_dma_cap_show()
6264 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); in stmmac_dma_cap_show()
6266 (priv->dma_cap.mbps_1000) ? "Y" : "N"); in stmmac_dma_cap_show()
6268 (priv->dma_cap.half_duplex) ? "Y" : "N"); in stmmac_dma_cap_show()
6269 if (priv->plat->has_xgmac) { in stmmac_dma_cap_show()
6272 priv->dma_cap.multi_addr); in stmmac_dma_cap_show()
6275 (priv->dma_cap.hash_filter) ? "Y" : "N"); in stmmac_dma_cap_show()
6277 (priv->dma_cap.multi_addr) ? "Y" : "N"); in stmmac_dma_cap_show()
6280 (priv->dma_cap.pcs) ? "Y" : "N"); in stmmac_dma_cap_show()
6282 (priv->dma_cap.sma_mdio) ? "Y" : "N"); in stmmac_dma_cap_show()
6284 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); in stmmac_dma_cap_show()
6286 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); in stmmac_dma_cap_show()
6288 (priv->dma_cap.rmon) ? "Y" : "N"); in stmmac_dma_cap_show()
6289 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", in stmmac_dma_cap_show()
6290 (priv->dma_cap.time_stamp) ? "Y" : "N"); in stmmac_dma_cap_show()
6291 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", in stmmac_dma_cap_show()
6292 (priv->dma_cap.atime_stamp) ? "Y" : "N"); in stmmac_dma_cap_show()
6293 if (priv->plat->has_xgmac) in stmmac_dma_cap_show()
6295 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); in stmmac_dma_cap_show()
6296 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", in stmmac_dma_cap_show()
6297 (priv->dma_cap.eee) ? "Y" : "N"); in stmmac_dma_cap_show()
6298 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); in stmmac_dma_cap_show()
6300 (priv->dma_cap.tx_coe) ? "Y" : "N"); in stmmac_dma_cap_show()
6301 if (priv->synopsys_id >= DWMAC_CORE_4_00 || in stmmac_dma_cap_show()
6302 priv->plat->has_xgmac) { in stmmac_dma_cap_show()
6304 (priv->dma_cap.rx_coe) ? "Y" : "N"); in stmmac_dma_cap_show()
6307 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); in stmmac_dma_cap_show()
6309 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); in stmmac_dma_cap_show()
6311 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); in stmmac_dma_cap_show()
6314 priv->dma_cap.number_rx_channel); in stmmac_dma_cap_show()
6316 priv->dma_cap.number_tx_channel); in stmmac_dma_cap_show()
6318 priv->dma_cap.number_rx_queues); in stmmac_dma_cap_show()
6320 priv->dma_cap.number_tx_queues); in stmmac_dma_cap_show()
6322 (priv->dma_cap.enh_desc) ? "Y" : "N"); in stmmac_dma_cap_show()
6323 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); in stmmac_dma_cap_show()
6324 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); in stmmac_dma_cap_show()
6325 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? in stmmac_dma_cap_show()
6326 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); in stmmac_dma_cap_show()
6327 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); in stmmac_dma_cap_show()
6329 priv->dma_cap.pps_out_num); in stmmac_dma_cap_show()
6331 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); in stmmac_dma_cap_show()
6333 priv->dma_cap.frpsel ? "Y" : "N"); in stmmac_dma_cap_show()
6335 priv->dma_cap.host_dma_width); in stmmac_dma_cap_show()
6337 priv->dma_cap.rssen ? "Y" : "N"); in stmmac_dma_cap_show()
6339 priv->dma_cap.vlhash ? "Y" : "N"); in stmmac_dma_cap_show()
6341 priv->dma_cap.sphen ? "Y" : "N"); in stmmac_dma_cap_show()
6343 priv->dma_cap.vlins ? "Y" : "N"); in stmmac_dma_cap_show()
6345 priv->dma_cap.dvlan ? "Y" : "N"); in stmmac_dma_cap_show()
6347 priv->dma_cap.l3l4fnum); in stmmac_dma_cap_show()
6349 priv->dma_cap.arpoffsel ? "Y" : "N"); in stmmac_dma_cap_show()
6351 priv->dma_cap.estsel ? "Y" : "N"); in stmmac_dma_cap_show()
6353 priv->dma_cap.fpesel ? "Y" : "N"); in stmmac_dma_cap_show()
6354 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", in stmmac_dma_cap_show()
6355 priv->dma_cap.tbssel ? "Y" : "N"); in stmmac_dma_cap_show()
6357 priv->dma_cap.tbs_ch_num); in stmmac_dma_cap_show()
6358 seq_printf(seq, "\tPer-Stream Filtering: %s\n", in stmmac_dma_cap_show()
6359 priv->dma_cap.sgfsel ? "Y" : "N"); in stmmac_dma_cap_show()
6361 BIT(priv->dma_cap.ttsfd) >> 1); in stmmac_dma_cap_show()
6363 priv->dma_cap.numtc); in stmmac_dma_cap_show()
6365 priv->dma_cap.dcben ? "Y" : "N"); in stmmac_dma_cap_show()
6367 priv->dma_cap.advthword ? "Y" : "N"); in stmmac_dma_cap_show()
6369 priv->dma_cap.ptoen ? "Y" : "N"); in stmmac_dma_cap_show()
6370 seq_printf(seq, "\tOne-Step Timestamping: %s\n", in stmmac_dma_cap_show()
6371 priv->dma_cap.osten ? "Y" : "N"); in stmmac_dma_cap_show()
6372 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", in stmmac_dma_cap_show()
6373 priv->dma_cap.pfcen ? "Y" : "N"); in stmmac_dma_cap_show()
6375 BIT(priv->dma_cap.frpes) << 6); in stmmac_dma_cap_show()
6377 BIT(priv->dma_cap.frpbs) << 6); in stmmac_dma_cap_show()
6379 priv->dma_cap.frppipe_num); in stmmac_dma_cap_show()
6381 priv->dma_cap.nrvf_num ? in stmmac_dma_cap_show()
6382 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); in stmmac_dma_cap_show()
6384 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); in stmmac_dma_cap_show()
6386 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); in stmmac_dma_cap_show()
6387 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", in stmmac_dma_cap_show()
6388 priv->dma_cap.cbtisel ? "Y" : "N"); in stmmac_dma_cap_show()
6390 priv->dma_cap.aux_snapshot_n); in stmmac_dma_cap_show()
6391 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", in stmmac_dma_cap_show()
6392 priv->dma_cap.pou_ost_en ? "Y" : "N"); in stmmac_dma_cap_show()
6394 priv->dma_cap.edma ? "Y" : "N"); in stmmac_dma_cap_show()
6396 priv->dma_cap.ediffc ? "Y" : "N"); in stmmac_dma_cap_show()
6398 priv->dma_cap.vxn ? "Y" : "N"); in stmmac_dma_cap_show()
6400 priv->dma_cap.dbgmem ? "Y" : "N"); in stmmac_dma_cap_show()
6402 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); in stmmac_dma_cap_show()
6415 if (dev->netdev_ops != &stmmac_netdev_ops) in stmmac_device_event()
6420 if (priv->dbgfs_dir) in stmmac_device_event()
6421 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, in stmmac_device_event()
6422 priv->dbgfs_dir, in stmmac_device_event()
6424 dev->name); in stmmac_device_event()
6442 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); in stmmac_init_fs()
6445 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, in stmmac_init_fs()
6449 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, in stmmac_init_fs()
6459 debugfs_remove_recursive(priv->dbgfs_dir); in stmmac_exit_fs()
6494 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { in stmmac_vlan_update()
6501 if (!priv->dma_cap.vlhash) { in stmmac_vlan_update()
6503 return -EOPNOTSUPP; in stmmac_vlan_update()
6509 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); in stmmac_vlan_update()
6518 ret = pm_runtime_resume_and_get(priv->device); in stmmac_vlan_rx_add_vid()
6525 set_bit(vid, priv->active_vlans); in stmmac_vlan_rx_add_vid()
6528 clear_bit(vid, priv->active_vlans); in stmmac_vlan_rx_add_vid()
6532 if (priv->hw->num_vlan) { in stmmac_vlan_rx_add_vid()
6533 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); in stmmac_vlan_rx_add_vid()
6538 pm_runtime_put(priv->device); in stmmac_vlan_rx_add_vid()
6549 ret = pm_runtime_resume_and_get(priv->device); in stmmac_vlan_rx_kill_vid()
6556 clear_bit(vid, priv->active_vlans); in stmmac_vlan_rx_kill_vid()
6558 if (priv->hw->num_vlan) { in stmmac_vlan_rx_kill_vid()
6559 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); in stmmac_vlan_rx_kill_vid()
6567 pm_runtime_put(priv->device); in stmmac_vlan_rx_kill_vid()
6576 switch (bpf->command) { in stmmac_bpf()
6578 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); in stmmac_bpf()
6580 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, in stmmac_bpf()
6581 bpf->xsk.queue_id); in stmmac_bpf()
6583 return -EOPNOTSUPP; in stmmac_bpf()
6596 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) in stmmac_xdp_xmit()
6597 return -ENETDOWN; in stmmac_xdp_xmit()
6600 return -EINVAL; in stmmac_xdp_xmit()
6603 nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit()
6606 /* Avoids TX time-out as we are sharing with slow path */ in stmmac_xdp_xmit()
6631 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_disable_rx_queue()
6634 spin_lock_irqsave(&ch->lock, flags); in stmmac_disable_rx_queue()
6635 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); in stmmac_disable_rx_queue()
6636 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_disable_rx_queue()
6639 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); in stmmac_disable_rx_queue()
6644 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_enable_rx_queue()
6645 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_rx_queue()
6650 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); in stmmac_enable_rx_queue()
6652 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); in stmmac_enable_rx_queue()
6656 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); in stmmac_enable_rx_queue()
6658 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); in stmmac_enable_rx_queue()
6659 netdev_err(priv->dev, "Failed to init RX desc.\n"); in stmmac_enable_rx_queue()
6664 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); in stmmac_enable_rx_queue()
6666 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_enable_rx_queue()
6667 rx_q->dma_rx_phy, rx_q->queue_index); in stmmac_enable_rx_queue()
6669 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * in stmmac_enable_rx_queue()
6671 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6672 rx_q->rx_tail_addr, rx_q->queue_index); in stmmac_enable_rx_queue()
6674 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { in stmmac_enable_rx_queue()
6675 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); in stmmac_enable_rx_queue()
6676 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6678 rx_q->queue_index); in stmmac_enable_rx_queue()
6680 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6681 priv->dma_conf.dma_buf_sz, in stmmac_enable_rx_queue()
6682 rx_q->queue_index); in stmmac_enable_rx_queue()
6687 spin_lock_irqsave(&ch->lock, flags); in stmmac_enable_rx_queue()
6688 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); in stmmac_enable_rx_queue()
6689 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_enable_rx_queue()
6694 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_disable_tx_queue()
6697 spin_lock_irqsave(&ch->lock, flags); in stmmac_disable_tx_queue()
6698 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); in stmmac_disable_tx_queue()
6699 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_disable_tx_queue()
6702 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); in stmmac_disable_tx_queue()
6707 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_enable_tx_queue()
6708 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_tx_queue()
6712 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); in stmmac_enable_tx_queue()
6714 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); in stmmac_enable_tx_queue()
6718 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); in stmmac_enable_tx_queue()
6720 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); in stmmac_enable_tx_queue()
6721 netdev_err(priv->dev, "Failed to init TX desc.\n"); in stmmac_enable_tx_queue()
6726 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); in stmmac_enable_tx_queue()
6728 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_enable_tx_queue()
6729 tx_q->dma_tx_phy, tx_q->queue_index); in stmmac_enable_tx_queue()
6731 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_enable_tx_queue()
6732 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); in stmmac_enable_tx_queue()
6734 tx_q->tx_tail_addr = tx_q->dma_tx_phy; in stmmac_enable_tx_queue()
6735 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, in stmmac_enable_tx_queue()
6736 tx_q->tx_tail_addr, tx_q->queue_index); in stmmac_enable_tx_queue()
6740 spin_lock_irqsave(&ch->lock, flags); in stmmac_enable_tx_queue()
6741 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); in stmmac_enable_tx_queue()
6742 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_enable_tx_queue()
6756 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_xdp_release()
6757 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); in stmmac_xdp_release()
6766 free_dma_desc_resources(priv, &priv->dma_conf); in stmmac_xdp_release()
6769 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_xdp_release()
6781 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_xdp_open()
6782 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_xdp_open()
6791 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); in stmmac_xdp_open()
6798 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); in stmmac_xdp_open()
6809 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); in stmmac_xdp_open()
6810 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_xdp_open()
6814 sph_en = (priv->hw->rx_csum > 0) && priv->sph; in stmmac_xdp_open()
6818 rx_q = &priv->dma_conf.rx_queue[chan]; in stmmac_xdp_open()
6820 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_xdp_open()
6821 rx_q->dma_rx_phy, chan); in stmmac_xdp_open()
6823 rx_q->rx_tail_addr = rx_q->dma_rx_phy + in stmmac_xdp_open()
6824 (rx_q->buf_alloc_num * in stmmac_xdp_open()
6826 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, in stmmac_xdp_open()
6827 rx_q->rx_tail_addr, chan); in stmmac_xdp_open()
6829 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { in stmmac_xdp_open()
6830 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); in stmmac_xdp_open()
6831 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_xdp_open()
6833 rx_q->queue_index); in stmmac_xdp_open()
6835 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_xdp_open()
6836 priv->dma_conf.dma_buf_sz, in stmmac_xdp_open()
6837 rx_q->queue_index); in stmmac_xdp_open()
6840 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); in stmmac_xdp_open()
6845 tx_q = &priv->dma_conf.tx_queue[chan]; in stmmac_xdp_open()
6847 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_xdp_open()
6848 tx_q->dma_tx_phy, chan); in stmmac_xdp_open()
6850 tx_q->tx_tail_addr = tx_q->dma_tx_phy; in stmmac_xdp_open()
6851 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, in stmmac_xdp_open()
6852 tx_q->tx_tail_addr, chan); in stmmac_xdp_open()
6854 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in stmmac_xdp_open()
6855 tx_q->txtimer.function = stmmac_tx_timer; in stmmac_xdp_open()
6859 stmmac_mac_set(priv, priv->ioaddr, true); in stmmac_xdp_open()
6877 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_xdp_open()
6878 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); in stmmac_xdp_open()
6882 free_dma_desc_resources(priv, &priv->dma_conf); in stmmac_xdp_open()
6894 if (test_bit(STMMAC_DOWN, &priv->state) || in stmmac_xsk_wakeup()
6895 !netif_carrier_ok(priv->dev)) in stmmac_xsk_wakeup()
6896 return -ENETDOWN; in stmmac_xsk_wakeup()
6899 return -EINVAL; in stmmac_xsk_wakeup()
6901 if (queue >= priv->plat->rx_queues_to_use || in stmmac_xsk_wakeup()
6902 queue >= priv->plat->tx_queues_to_use) in stmmac_xsk_wakeup()
6903 return -EINVAL; in stmmac_xsk_wakeup()
6905 rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_xsk_wakeup()
6906 tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_xsk_wakeup()
6907 ch = &priv->channel[queue]; in stmmac_xsk_wakeup()
6909 if (!rx_q->xsk_pool && !tx_q->xsk_pool) in stmmac_xsk_wakeup()
6910 return -EINVAL; in stmmac_xsk_wakeup()
6912 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { in stmmac_xsk_wakeup()
6913 /* EQoS does not have per-DMA channel SW interrupt, in stmmac_xsk_wakeup()
6914 * so we schedule RX Napi straight-away. in stmmac_xsk_wakeup()
6916 if (likely(napi_schedule_prep(&ch->rxtx_napi))) in stmmac_xsk_wakeup()
6917 __napi_schedule(&ch->rxtx_napi); in stmmac_xsk_wakeup()
6926 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_get_stats64()
6927 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_get_stats64()
6932 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; in stmmac_get_stats64()
6937 start = u64_stats_fetch_begin(&txq_stats->q_syncp); in stmmac_get_stats64()
6938 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); in stmmac_get_stats64()
6939 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); in stmmac_get_stats64()
6941 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); in stmmac_get_stats64()
6942 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); in stmmac_get_stats64()
6943 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); in stmmac_get_stats64()
6945 stats->tx_packets += tx_packets; in stmmac_get_stats64()
6946 stats->tx_bytes += tx_bytes; in stmmac_get_stats64()
6950 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; in stmmac_get_stats64()
6955 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); in stmmac_get_stats64()
6956 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); in stmmac_get_stats64()
6957 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); in stmmac_get_stats64()
6958 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); in stmmac_get_stats64()
6960 stats->rx_packets += rx_packets; in stmmac_get_stats64()
6961 stats->rx_bytes += rx_bytes; in stmmac_get_stats64()
6964 stats->rx_dropped = priv->xstats.rx_dropped; in stmmac_get_stats64()
6965 stats->rx_errors = priv->xstats.rx_errors; in stmmac_get_stats64()
6966 stats->tx_dropped = priv->xstats.tx_dropped; in stmmac_get_stats64()
6967 stats->tx_errors = priv->xstats.tx_errors; in stmmac_get_stats64()
6968 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; in stmmac_get_stats64()
6969 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; in stmmac_get_stats64()
6970 stats->rx_length_errors = priv->xstats.rx_length; in stmmac_get_stats64()
6971 stats->rx_crc_errors = priv->xstats.rx_crc_errors; in stmmac_get_stats64()
6972 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; in stmmac_get_stats64()
6973 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; in stmmac_get_stats64()
6999 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) in stmmac_reset_subtask()
7001 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_reset_subtask()
7004 netdev_err(priv->dev, "Reset adapter.\n"); in stmmac_reset_subtask()
7007 netif_trans_update(priv->dev); in stmmac_reset_subtask()
7008 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) in stmmac_reset_subtask()
7011 set_bit(STMMAC_DOWN, &priv->state); in stmmac_reset_subtask()
7012 dev_close(priv->dev); in stmmac_reset_subtask()
7013 dev_open(priv->dev, NULL); in stmmac_reset_subtask()
7014 clear_bit(STMMAC_DOWN, &priv->state); in stmmac_reset_subtask()
7015 clear_bit(STMMAC_RESETING, &priv->state); in stmmac_reset_subtask()
7025 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); in stmmac_service_task()
7029 * stmmac_hw_init - Init the MAC device
7040 /* dwmac-sun8i only work in chain mode */ in stmmac_hw_init()
7041 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) in stmmac_hw_init()
7043 priv->chain_mode = chain_mode; in stmmac_hw_init()
7051 priv->hw_cap_support = stmmac_get_hw_features(priv); in stmmac_hw_init()
7052 if (priv->hw_cap_support) { in stmmac_hw_init()
7053 dev_info(priv->device, "DMA HW capability register supported\n"); in stmmac_hw_init()
7060 priv->plat->enh_desc = priv->dma_cap.enh_desc; in stmmac_hw_init()
7061 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && in stmmac_hw_init()
7062 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); in stmmac_hw_init()
7063 priv->hw->pmt = priv->plat->pmt; in stmmac_hw_init()
7064 if (priv->dma_cap.hash_tb_sz) { in stmmac_hw_init()
7065 priv->hw->multicast_filter_bins = in stmmac_hw_init()
7066 (BIT(priv->dma_cap.hash_tb_sz) << 5); in stmmac_hw_init()
7067 priv->hw->mcast_bits_log2 = in stmmac_hw_init()
7068 ilog2(priv->hw->multicast_filter_bins); in stmmac_hw_init()
7072 if (priv->plat->force_thresh_dma_mode) in stmmac_hw_init()
7073 priv->plat->tx_coe = 0; in stmmac_hw_init()
7075 priv->plat->tx_coe = priv->dma_cap.tx_coe; in stmmac_hw_init()
7078 priv->plat->rx_coe = priv->dma_cap.rx_coe; in stmmac_hw_init()
7080 if (priv->dma_cap.rx_coe_type2) in stmmac_hw_init()
7081 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; in stmmac_hw_init()
7082 else if (priv->dma_cap.rx_coe_type1) in stmmac_hw_init()
7083 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; in stmmac_hw_init()
7086 dev_info(priv->device, "No HW DMA feature register supported\n"); in stmmac_hw_init()
7089 if (priv->plat->rx_coe) { in stmmac_hw_init()
7090 priv->hw->rx_csum = priv->plat->rx_coe; in stmmac_hw_init()
7091 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); in stmmac_hw_init()
7092 if (priv->synopsys_id < DWMAC_CORE_4_00) in stmmac_hw_init()
7093 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); in stmmac_hw_init()
7095 if (priv->plat->tx_coe) in stmmac_hw_init()
7096 dev_info(priv->device, "TX Checksum insertion supported\n"); in stmmac_hw_init()
7098 if (priv->plat->pmt) { in stmmac_hw_init()
7099 dev_info(priv->device, "Wake-Up On Lan supported\n"); in stmmac_hw_init()
7100 device_set_wakeup_capable(priv->device, 1); in stmmac_hw_init()
7103 if (priv->dma_cap.tsoen) in stmmac_hw_init()
7104 dev_info(priv->device, "TSO supported\n"); in stmmac_hw_init()
7106 priv->hw->vlan_fail_q_en = in stmmac_hw_init()
7107 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); in stmmac_hw_init()
7108 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; in stmmac_hw_init()
7111 if (priv->hwif_quirks) { in stmmac_hw_init()
7112 ret = priv->hwif_quirks(priv); in stmmac_hw_init()
7122 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || in stmmac_hw_init()
7123 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { in stmmac_hw_init()
7124 priv->use_riwt = 1; in stmmac_hw_init()
7125 dev_info(priv->device, in stmmac_hw_init()
7137 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); in stmmac_napi_add()
7140 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_napi_add()
7142 ch->priv_data = priv; in stmmac_napi_add()
7143 ch->index = queue; in stmmac_napi_add()
7144 spin_lock_init(&ch->lock); in stmmac_napi_add()
7146 if (queue < priv->plat->rx_queues_to_use) { in stmmac_napi_add()
7147 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); in stmmac_napi_add()
7149 if (queue < priv->plat->tx_queues_to_use) { in stmmac_napi_add()
7150 netif_napi_add_tx(dev, &ch->tx_napi, in stmmac_napi_add()
7153 if (queue < priv->plat->rx_queues_to_use && in stmmac_napi_add()
7154 queue < priv->plat->tx_queues_to_use) { in stmmac_napi_add()
7155 netif_napi_add(dev, &ch->rxtx_napi, in stmmac_napi_add()
7166 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); in stmmac_napi_del()
7169 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_napi_del()
7171 if (queue < priv->plat->rx_queues_to_use) in stmmac_napi_del()
7172 netif_napi_del(&ch->rx_napi); in stmmac_napi_del()
7173 if (queue < priv->plat->tx_queues_to_use) in stmmac_napi_del()
7174 netif_napi_del(&ch->tx_napi); in stmmac_napi_del()
7175 if (queue < priv->plat->rx_queues_to_use && in stmmac_napi_del()
7176 queue < priv->plat->tx_queues_to_use) { in stmmac_napi_del()
7177 netif_napi_del(&ch->rxtx_napi); in stmmac_napi_del()
7193 priv->plat->rx_queues_to_use = rx_cnt; in stmmac_reinit_queues()
7194 priv->plat->tx_queues_to_use = tx_cnt; in stmmac_reinit_queues()
7196 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) in stmmac_reinit_queues()
7197 priv->rss.table[i] = ethtool_rxfh_indir_default(i, in stmmac_reinit_queues()
7202 priv->phylink_config.mac_capabilities = priv->hw->link.caps; in stmmac_reinit_queues()
7204 max_speed = priv->plat->max_speed; in stmmac_reinit_queues()
7206 phylink_limit_mac_speed(&priv->phylink_config, max_speed); in stmmac_reinit_queues()
7224 priv->dma_conf.dma_rx_size = rx_size; in stmmac_reinit_ringparam()
7225 priv->dma_conf.dma_tx_size = tx_size; in stmmac_reinit_ringparam()
7238 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_lp_task()
7239 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; in stmmac_fpe_lp_task()
7240 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; in stmmac_fpe_lp_task()
7241 bool *hs_enable = &fpe_cfg->hs_enable; in stmmac_fpe_lp_task()
7242 bool *enable = &fpe_cfg->enable; in stmmac_fpe_lp_task()
7245 while (retries-- > 0) { in stmmac_fpe_lp_task()
7252 stmmac_fpe_configure(priv, priv->ioaddr, in stmmac_fpe_lp_task()
7254 priv->plat->tx_queues_to_use, in stmmac_fpe_lp_task()
7255 priv->plat->rx_queues_to_use, in stmmac_fpe_lp_task()
7258 netdev_info(priv->dev, "configured FPE\n"); in stmmac_fpe_lp_task()
7262 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); in stmmac_fpe_lp_task()
7269 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, in stmmac_fpe_lp_task()
7271 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_lp_task()
7279 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); in stmmac_fpe_lp_task()
7284 if (priv->plat->fpe_cfg->hs_enable != enable) { in stmmac_fpe_handshake()
7286 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_handshake()
7287 priv->plat->fpe_cfg, in stmmac_fpe_handshake()
7290 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; in stmmac_fpe_handshake()
7291 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; in stmmac_fpe_handshake()
7294 priv->plat->fpe_cfg->hs_enable = enable; in stmmac_fpe_handshake()
7301 struct dma_desc *desc_contains_ts = ctx->desc; in stmmac_xdp_rx_timestamp()
7302 struct stmmac_priv *priv = ctx->priv; in stmmac_xdp_rx_timestamp()
7303 struct dma_desc *ndesc = ctx->ndesc; in stmmac_xdp_rx_timestamp()
7304 struct dma_desc *desc = ctx->desc; in stmmac_xdp_rx_timestamp()
7307 if (!priv->hwts_rx_en) in stmmac_xdp_rx_timestamp()
7308 return -ENODATA; in stmmac_xdp_rx_timestamp()
7311 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) in stmmac_xdp_rx_timestamp()
7315 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { in stmmac_xdp_rx_timestamp()
7316 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); in stmmac_xdp_rx_timestamp()
7317 ns -= priv->plat->cdc_error_adj; in stmmac_xdp_rx_timestamp()
7322 return -ENODATA; in stmmac_xdp_rx_timestamp()
7351 return -ENOMEM; in stmmac_dvr_probe()
7356 priv->device = device; in stmmac_dvr_probe()
7357 priv->dev = ndev; in stmmac_dvr_probe()
7360 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); in stmmac_dvr_probe()
7362 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); in stmmac_dvr_probe()
7363 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); in stmmac_dvr_probe()
7366 priv->xstats.pcpu_stats = in stmmac_dvr_probe()
7368 if (!priv->xstats.pcpu_stats) in stmmac_dvr_probe()
7369 return -ENOMEM; in stmmac_dvr_probe()
7372 priv->pause = pause; in stmmac_dvr_probe()
7373 priv->plat = plat_dat; in stmmac_dvr_probe()
7374 priv->ioaddr = res->addr; in stmmac_dvr_probe()
7375 priv->dev->base_addr = (unsigned long)res->addr; in stmmac_dvr_probe()
7376 priv->plat->dma_cfg->multi_msi_en = in stmmac_dvr_probe()
7377 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); in stmmac_dvr_probe()
7379 priv->dev->irq = res->irq; in stmmac_dvr_probe()
7380 priv->wol_irq = res->wol_irq; in stmmac_dvr_probe()
7381 priv->lpi_irq = res->lpi_irq; in stmmac_dvr_probe()
7382 priv->sfty_ce_irq = res->sfty_ce_irq; in stmmac_dvr_probe()
7383 priv->sfty_ue_irq = res->sfty_ue_irq; in stmmac_dvr_probe()
7385 priv->rx_irq[i] = res->rx_irq[i]; in stmmac_dvr_probe()
7387 priv->tx_irq[i] = res->tx_irq[i]; in stmmac_dvr_probe()
7389 if (!is_zero_ether_addr(res->mac)) in stmmac_dvr_probe()
7390 eth_hw_addr_set(priv->dev, res->mac); in stmmac_dvr_probe()
7392 dev_set_drvdata(device, priv->dev); in stmmac_dvr_probe()
7397 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); in stmmac_dvr_probe()
7398 if (!priv->af_xdp_zc_qps) in stmmac_dvr_probe()
7399 return -ENOMEM; in stmmac_dvr_probe()
7402 priv->wq = create_singlethread_workqueue("stmmac_wq"); in stmmac_dvr_probe()
7403 if (!priv->wq) { in stmmac_dvr_probe()
7404 dev_err(priv->device, "failed to create workqueue\n"); in stmmac_dvr_probe()
7405 ret = -ENOMEM; in stmmac_dvr_probe()
7409 INIT_WORK(&priv->service_task, stmmac_service_task); in stmmac_dvr_probe()
7412 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); in stmmac_dvr_probe()
7418 priv->plat->phy_addr = phyaddr; in stmmac_dvr_probe()
7420 if (priv->plat->stmmac_rst) { in stmmac_dvr_probe()
7421 ret = reset_control_assert(priv->plat->stmmac_rst); in stmmac_dvr_probe()
7422 reset_control_deassert(priv->plat->stmmac_rst); in stmmac_dvr_probe()
7426 if (ret == -ENOTSUPP) in stmmac_dvr_probe()
7427 reset_control_reset(priv->plat->stmmac_rst); in stmmac_dvr_probe()
7430 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); in stmmac_dvr_probe()
7431 if (ret == -ENOTSUPP) in stmmac_dvr_probe()
7432 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", in stmmac_dvr_probe()
7445 if (priv->synopsys_id < DWMAC_CORE_5_20) in stmmac_dvr_probe()
7446 priv->plat->dma_cfg->dche = false; in stmmac_dvr_probe()
7450 ndev->netdev_ops = &stmmac_netdev_ops; in stmmac_dvr_probe()
7452 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; in stmmac_dvr_probe()
7454 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | in stmmac_dvr_probe()
7456 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | in stmmac_dvr_probe()
7461 ndev->hw_features |= NETIF_F_HW_TC; in stmmac_dvr_probe()
7464 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { in stmmac_dvr_probe()
7465 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; in stmmac_dvr_probe()
7466 if (priv->plat->has_gmac4) in stmmac_dvr_probe()
7467 ndev->hw_features |= NETIF_F_GSO_UDP_L4; in stmmac_dvr_probe()
7468 priv->tso = true; in stmmac_dvr_probe()
7469 dev_info(priv->device, "TSO feature enabled\n"); in stmmac_dvr_probe()
7472 if (priv->dma_cap.sphen && in stmmac_dvr_probe()
7473 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { in stmmac_dvr_probe()
7474 ndev->hw_features |= NETIF_F_GRO; in stmmac_dvr_probe()
7475 priv->sph_cap = true; in stmmac_dvr_probe()
7476 priv->sph = priv->sph_cap; in stmmac_dvr_probe()
7477 dev_info(priv->device, "SPH feature enabled\n"); in stmmac_dvr_probe()
7485 if (priv->plat->host_dma_width) in stmmac_dvr_probe()
7486 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; in stmmac_dvr_probe()
7488 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; in stmmac_dvr_probe()
7490 if (priv->dma_cap.host_dma_width) { in stmmac_dvr_probe()
7492 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); in stmmac_dvr_probe()
7494 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", in stmmac_dvr_probe()
7495 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); in stmmac_dvr_probe()
7502 priv->plat->dma_cfg->eame = true; in stmmac_dvr_probe()
7506 dev_err(priv->device, "Failed to set DMA Mask\n"); in stmmac_dvr_probe()
7510 priv->dma_cap.host_dma_width = 32; in stmmac_dvr_probe()
7514 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; in stmmac_dvr_probe()
7515 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); in stmmac_dvr_probe()
7518 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; in stmmac_dvr_probe()
7519 if (priv->dma_cap.vlhash) { in stmmac_dvr_probe()
7520 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; in stmmac_dvr_probe()
7521 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; in stmmac_dvr_probe()
7523 if (priv->dma_cap.vlins) { in stmmac_dvr_probe()
7524 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; in stmmac_dvr_probe()
7525 if (priv->dma_cap.dvlan) in stmmac_dvr_probe()
7526 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; in stmmac_dvr_probe()
7529 priv->msg_enable = netif_msg_init(debug, default_msg_level); in stmmac_dvr_probe()
7531 priv->xstats.threshold = tc; in stmmac_dvr_probe()
7534 rxq = priv->plat->rx_queues_to_use; in stmmac_dvr_probe()
7535 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); in stmmac_dvr_probe()
7536 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) in stmmac_dvr_probe()
7537 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); in stmmac_dvr_probe()
7539 if (priv->dma_cap.rssen && priv->plat->rss_en) in stmmac_dvr_probe()
7540 ndev->features |= NETIF_F_RXHASH; in stmmac_dvr_probe()
7542 ndev->vlan_features |= ndev->features; in stmmac_dvr_probe()
7544 ndev->vlan_features &= ~NETIF_F_TSO; in stmmac_dvr_probe()
7546 /* MTU range: 46 - hw-specific max */ in stmmac_dvr_probe()
7547 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; in stmmac_dvr_probe()
7548 if (priv->plat->has_xgmac) in stmmac_dvr_probe()
7549 ndev->max_mtu = XGMAC_JUMBO_LEN; in stmmac_dvr_probe()
7550 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) in stmmac_dvr_probe()
7551 ndev->max_mtu = JUMBO_LEN; in stmmac_dvr_probe()
7553 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); in stmmac_dvr_probe()
7554 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu in stmmac_dvr_probe()
7555 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. in stmmac_dvr_probe()
7557 if ((priv->plat->maxmtu < ndev->max_mtu) && in stmmac_dvr_probe()
7558 (priv->plat->maxmtu >= ndev->min_mtu)) in stmmac_dvr_probe()
7559 ndev->max_mtu = priv->plat->maxmtu; in stmmac_dvr_probe()
7560 else if (priv->plat->maxmtu < ndev->min_mtu) in stmmac_dvr_probe()
7561 dev_warn(priv->device, in stmmac_dvr_probe()
7563 __func__, priv->plat->maxmtu); in stmmac_dvr_probe()
7566 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ in stmmac_dvr_probe()
7568 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; in stmmac_dvr_probe()
7573 mutex_init(&priv->lock); in stmmac_dvr_probe()
7577 * changed at run-time and it is fixed. Viceversa the driver'll try to in stmmac_dvr_probe()
7581 if (priv->plat->clk_csr >= 0) in stmmac_dvr_probe()
7582 priv->clk_csr = priv->plat->clk_csr; in stmmac_dvr_probe()
7593 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_probe()
7594 priv->hw->pcs != STMMAC_PCS_RTBI) { in stmmac_dvr_probe()
7598 dev_err_probe(priv->device, ret, in stmmac_dvr_probe()
7600 __func__, priv->plat->bus_id); in stmmac_dvr_probe()
7605 if (priv->plat->speed_mode_2500) in stmmac_dvr_probe()
7606 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); in stmmac_dvr_probe()
7608 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { in stmmac_dvr_probe()
7609 ret = stmmac_xpcs_setup(priv->mii); in stmmac_dvr_probe()
7622 dev_err(priv->device, "%s: ERROR %i registering the device\n", in stmmac_dvr_probe()
7631 if (priv->plat->dump_debug_regs) in stmmac_dvr_probe()
7632 priv->plat->dump_debug_regs(priv->plat->bsp_priv); in stmmac_dvr_probe()
7642 phylink_destroy(priv->phylink); in stmmac_dvr_probe()
7645 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_probe()
7646 priv->hw->pcs != STMMAC_PCS_RTBI) in stmmac_dvr_probe()
7651 destroy_workqueue(priv->wq); in stmmac_dvr_probe()
7653 bitmap_free(priv->af_xdp_zc_qps); in stmmac_dvr_probe()
7670 netdev_info(priv->dev, "%s: removing driver", __func__); in stmmac_dvr_remove()
7675 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_dvr_remove()
7682 phylink_destroy(priv->phylink); in stmmac_dvr_remove()
7683 if (priv->plat->stmmac_rst) in stmmac_dvr_remove()
7684 reset_control_assert(priv->plat->stmmac_rst); in stmmac_dvr_remove()
7685 reset_control_assert(priv->plat->stmmac_ahb_rst); in stmmac_dvr_remove()
7686 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_remove()
7687 priv->hw->pcs != STMMAC_PCS_RTBI) in stmmac_dvr_remove()
7689 destroy_workqueue(priv->wq); in stmmac_dvr_remove()
7690 mutex_destroy(&priv->lock); in stmmac_dvr_remove()
7691 bitmap_free(priv->af_xdp_zc_qps); in stmmac_dvr_remove()
7699 * stmmac_suspend - suspend callback
7714 mutex_lock(&priv->lock); in stmmac_suspend()
7720 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_suspend()
7721 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); in stmmac_suspend()
7723 if (priv->eee_enabled) { in stmmac_suspend()
7724 priv->tx_path_in_lpi_mode = false; in stmmac_suspend()
7725 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_suspend()
7731 if (priv->plat->serdes_powerdown) in stmmac_suspend()
7732 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); in stmmac_suspend()
7735 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_suspend()
7736 stmmac_pmt(priv, priv->hw, priv->wolopts); in stmmac_suspend()
7737 priv->irq_wake = 1; in stmmac_suspend()
7739 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_suspend()
7740 pinctrl_pm_select_sleep_state(priv->device); in stmmac_suspend()
7743 mutex_unlock(&priv->lock); in stmmac_suspend()
7746 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_suspend()
7747 phylink_suspend(priv->phylink, true); in stmmac_suspend()
7749 if (device_may_wakeup(priv->device)) in stmmac_suspend()
7750 phylink_speed_down(priv->phylink, false); in stmmac_suspend()
7751 phylink_suspend(priv->phylink, false); in stmmac_suspend()
7755 if (priv->dma_cap.fpesel) { in stmmac_suspend()
7757 stmmac_fpe_configure(priv, priv->ioaddr, in stmmac_suspend()
7758 priv->plat->fpe_cfg, in stmmac_suspend()
7759 priv->plat->tx_queues_to_use, in stmmac_suspend()
7760 priv->plat->rx_queues_to_use, false); in stmmac_suspend()
7766 priv->speed = SPEED_UNKNOWN; in stmmac_suspend()
7773 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; in stmmac_reset_rx_queue()
7775 rx_q->cur_rx = 0; in stmmac_reset_rx_queue()
7776 rx_q->dirty_rx = 0; in stmmac_reset_rx_queue()
7781 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; in stmmac_reset_tx_queue()
7783 tx_q->cur_tx = 0; in stmmac_reset_tx_queue()
7784 tx_q->dirty_tx = 0; in stmmac_reset_tx_queue()
7785 tx_q->mss = 0; in stmmac_reset_tx_queue()
7787 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_reset_tx_queue()
7791 * stmmac_reset_queues_param - reset queue parameters
7796 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_reset_queues_param()
7797 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_reset_queues_param()
7808 * stmmac_resume - resume callback
7823 * automatically as soon as a magic packet or a Wake-up frame in stmmac_resume()
7828 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_resume()
7829 mutex_lock(&priv->lock); in stmmac_resume()
7830 stmmac_pmt(priv, priv->hw, 0); in stmmac_resume()
7831 mutex_unlock(&priv->lock); in stmmac_resume()
7832 priv->irq_wake = 0; in stmmac_resume()
7834 pinctrl_pm_select_default_state(priv->device); in stmmac_resume()
7836 if (priv->mii) in stmmac_resume()
7837 stmmac_mdio_reset(priv->mii); in stmmac_resume()
7840 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && in stmmac_resume()
7841 priv->plat->serdes_powerup) { in stmmac_resume()
7842 ret = priv->plat->serdes_powerup(ndev, in stmmac_resume()
7843 priv->plat->bsp_priv); in stmmac_resume()
7850 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_resume()
7851 phylink_resume(priv->phylink); in stmmac_resume()
7853 phylink_resume(priv->phylink); in stmmac_resume()
7854 if (device_may_wakeup(priv->device)) in stmmac_resume()
7855 phylink_speed_up(priv->phylink); in stmmac_resume()
7860 mutex_lock(&priv->lock); in stmmac_resume()
7865 stmmac_clear_descriptors(priv, &priv->dma_conf); in stmmac_resume()
7871 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); in stmmac_resume()
7876 mutex_unlock(&priv->lock); in stmmac_resume()