1 // SPDX-License-Identifier: GPL-2.0-only
2 /*******************************************************************************
3   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
4   ST Ethernet IPs are built around a Synopsys IP Core.
5 
6 	Copyright(C) 2007-2011 STMicroelectronics Ltd
7 
8 
9   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
10 
11   Documentation available at:
12 	http://www.stlinux.com
13   Support available at:
14 	https://bugzilla.stlinux.com/
15 *******************************************************************************/
16 
17 #include <linux/clk.h>
18 #include <linux/kernel.h>
19 #include <linux/interrupt.h>
20 #include <linux/ip.h>
21 #include <linux/tcp.h>
22 #include <linux/skbuff.h>
23 #include <linux/ethtool.h>
24 #include <linux/if_ether.h>
25 #include <linux/crc32.h>
26 #include <linux/mii.h>
27 #include <linux/if.h>
28 #include <linux/if_vlan.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/slab.h>
31 #include <linux/pm_runtime.h>
32 #include <linux/prefetch.h>
33 #include <linux/pinctrl/consumer.h>
34 #ifdef CONFIG_DEBUG_FS
35 #include <linux/debugfs.h>
36 #include <linux/seq_file.h>
37 #endif /* CONFIG_DEBUG_FS */
38 #include <linux/net_tstamp.h>
39 #include <linux/phylink.h>
40 #include <linux/udp.h>
41 #include <linux/bpf_trace.h>
42 #include <net/pkt_cls.h>
43 #include <net/xdp_sock_drv.h>
44 #include "stmmac_ptp.h"
45 #include "stmmac.h"
46 #include "stmmac_xdp.h"
47 #include <linux/reset.h>
48 #include <linux/of_mdio.h>
49 #include "dwmac1000.h"
50 #include "dwxgmac2.h"
51 #include "hwif.h"
52 
53 /* As long as the interface is active, we keep the timestamping counter enabled
54  * with fine resolution and binary rollover. This avoid non-monotonic behavior
55  * (clock jumps) when changing timestamping settings at runtime.
56  */
57 #define STMMAC_HWTS_ACTIVE	(PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \
58 				 PTP_TCR_TSCTRLSSR)
59 
60 #define	STMMAC_ALIGN(x)		ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
61 #define	TSO_MAX_BUFF_SIZE	(SZ_16K - 1)
62 
63 /* Module parameters */
64 #define TX_TIMEO	5000
65 static int watchdog = TX_TIMEO;
66 module_param(watchdog, int, 0644);
67 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
68 
69 static int debug = -1;
70 module_param(debug, int, 0644);
71 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
72 
73 static int phyaddr = -1;
74 module_param(phyaddr, int, 0444);
75 MODULE_PARM_DESC(phyaddr, "Physical device address");
76 
77 #define STMMAC_TX_THRESH(x)	((x)->dma_tx_size / 4)
78 #define STMMAC_RX_THRESH(x)	((x)->dma_rx_size / 4)
79 
80 /* Limit to make sure XDP TX and slow path can coexist */
81 #define STMMAC_XSK_TX_BUDGET_MAX	256
82 #define STMMAC_TX_XSK_AVAIL		16
83 #define STMMAC_RX_FILL_BATCH		16
84 
85 #define STMMAC_XDP_PASS		0
86 #define STMMAC_XDP_CONSUMED	BIT(0)
87 #define STMMAC_XDP_TX		BIT(1)
88 #define STMMAC_XDP_REDIRECT	BIT(2)
89 
90 static int flow_ctrl = FLOW_AUTO;
91 module_param(flow_ctrl, int, 0644);
92 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
93 
94 static int pause = PAUSE_TIME;
95 module_param(pause, int, 0644);
96 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
97 
98 #define TC_DEFAULT 64
99 static int tc = TC_DEFAULT;
100 module_param(tc, int, 0644);
101 MODULE_PARM_DESC(tc, "DMA threshold control value");
102 
103 #define	DEFAULT_BUFSIZE	1536
104 static int buf_sz = DEFAULT_BUFSIZE;
105 module_param(buf_sz, int, 0644);
106 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
107 
108 #define	STMMAC_RX_COPYBREAK	256
109 
110 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
111 				      NETIF_MSG_LINK | NETIF_MSG_IFUP |
112 				      NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
113 
114 #define STMMAC_DEFAULT_LPI_TIMER	1000
115 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
116 module_param(eee_timer, int, 0644);
117 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
118 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x))
119 
120 /* By default the driver will use the ring mode to manage tx and rx descriptors,
121  * but allow user to force to use the chain instead of the ring
122  */
123 static unsigned int chain_mode;
124 module_param(chain_mode, int, 0444);
125 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
126 
127 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
128 /* For MSI interrupts handling */
129 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id);
130 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id);
131 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data);
132 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data);
133 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
134 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
135 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
136 					  u32 rxmode, u32 chan);
137 
138 #ifdef CONFIG_DEBUG_FS
139 static const struct net_device_ops stmmac_netdev_ops;
140 static void stmmac_init_fs(struct net_device *dev);
141 static void stmmac_exit_fs(struct net_device *dev);
142 #endif
143 
144 #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC))
145 
146 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled)
147 {
148 	int ret = 0;
149 
150 	if (enabled) {
151 		ret = clk_prepare_enable(priv->plat->stmmac_clk);
152 		if (ret)
153 			return ret;
154 		ret = clk_prepare_enable(priv->plat->pclk);
155 		if (ret) {
156 			clk_disable_unprepare(priv->plat->stmmac_clk);
157 			return ret;
158 		}
159 		if (priv->plat->clks_config) {
160 			ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled);
161 			if (ret) {
162 				clk_disable_unprepare(priv->plat->stmmac_clk);
163 				clk_disable_unprepare(priv->plat->pclk);
164 				return ret;
165 			}
166 		}
167 	} else {
168 		clk_disable_unprepare(priv->plat->stmmac_clk);
169 		clk_disable_unprepare(priv->plat->pclk);
170 		if (priv->plat->clks_config)
171 			priv->plat->clks_config(priv->plat->bsp_priv, enabled);
172 	}
173 
174 	return ret;
175 }
176 EXPORT_SYMBOL_GPL(stmmac_bus_clks_config);
177 
178 /**
179  * stmmac_verify_args - verify the driver parameters.
180  * Description: it checks the driver parameters and set a default in case of
181  * errors.
182  */
183 static void stmmac_verify_args(void)
184 {
185 	if (unlikely(watchdog < 0))
186 		watchdog = TX_TIMEO;
187 	if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
188 		buf_sz = DEFAULT_BUFSIZE;
189 	if (unlikely(flow_ctrl > 1))
190 		flow_ctrl = FLOW_AUTO;
191 	else if (likely(flow_ctrl < 0))
192 		flow_ctrl = FLOW_OFF;
193 	if (unlikely((pause < 0) || (pause > 0xffff)))
194 		pause = PAUSE_TIME;
195 	if (eee_timer < 0)
196 		eee_timer = STMMAC_DEFAULT_LPI_TIMER;
197 }
198 
199 static void __stmmac_disable_all_queues(struct stmmac_priv *priv)
200 {
201 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
202 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
203 	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
204 	u32 queue;
205 
206 	for (queue = 0; queue < maxq; queue++) {
207 		struct stmmac_channel *ch = &priv->channel[queue];
208 
209 		if (stmmac_xdp_is_enabled(priv) &&
210 		    test_bit(queue, priv->af_xdp_zc_qps)) {
211 			napi_disable(&ch->rxtx_napi);
212 			continue;
213 		}
214 
215 		if (queue < rx_queues_cnt)
216 			napi_disable(&ch->rx_napi);
217 		if (queue < tx_queues_cnt)
218 			napi_disable(&ch->tx_napi);
219 	}
220 }
221 
222 /**
223  * stmmac_disable_all_queues - Disable all queues
224  * @priv: driver private structure
225  */
226 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
227 {
228 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
229 	struct stmmac_rx_queue *rx_q;
230 	u32 queue;
231 
232 	/* synchronize_rcu() needed for pending XDP buffers to drain */
233 	for (queue = 0; queue < rx_queues_cnt; queue++) {
234 		rx_q = &priv->rx_queue[queue];
235 		if (rx_q->xsk_pool) {
236 			synchronize_rcu();
237 			break;
238 		}
239 	}
240 
241 	__stmmac_disable_all_queues(priv);
242 }
243 
244 /**
245  * stmmac_enable_all_queues - Enable all queues
246  * @priv: driver private structure
247  */
248 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
249 {
250 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
251 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
252 	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
253 	u32 queue;
254 
255 	for (queue = 0; queue < maxq; queue++) {
256 		struct stmmac_channel *ch = &priv->channel[queue];
257 
258 		if (stmmac_xdp_is_enabled(priv) &&
259 		    test_bit(queue, priv->af_xdp_zc_qps)) {
260 			napi_enable(&ch->rxtx_napi);
261 			continue;
262 		}
263 
264 		if (queue < rx_queues_cnt)
265 			napi_enable(&ch->rx_napi);
266 		if (queue < tx_queues_cnt)
267 			napi_enable(&ch->tx_napi);
268 	}
269 }
270 
271 static void stmmac_service_event_schedule(struct stmmac_priv *priv)
272 {
273 	if (!test_bit(STMMAC_DOWN, &priv->state) &&
274 	    !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
275 		queue_work(priv->wq, &priv->service_task);
276 }
277 
278 static void stmmac_global_err(struct stmmac_priv *priv)
279 {
280 	netif_carrier_off(priv->dev);
281 	set_bit(STMMAC_RESET_REQUESTED, &priv->state);
282 	stmmac_service_event_schedule(priv);
283 }
284 
285 /**
286  * stmmac_clk_csr_set - dynamically set the MDC clock
287  * @priv: driver private structure
288  * Description: this is to dynamically set the MDC clock according to the csr
289  * clock input.
290  * Note:
291  *	If a specific clk_csr value is passed from the platform
292  *	this means that the CSR Clock Range selection cannot be
293  *	changed at run-time and it is fixed (as reported in the driver
294  *	documentation). Viceversa the driver will try to set the MDC
295  *	clock dynamically according to the actual clock input.
296  */
297 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
298 {
299 	u32 clk_rate;
300 
301 	clk_rate = clk_get_rate(priv->plat->stmmac_clk);
302 
303 	/* Platform provided default clk_csr would be assumed valid
304 	 * for all other cases except for the below mentioned ones.
305 	 * For values higher than the IEEE 802.3 specified frequency
306 	 * we can not estimate the proper divider as it is not known
307 	 * the frequency of clk_csr_i. So we do not change the default
308 	 * divider.
309 	 */
310 	if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
311 		if (clk_rate < CSR_F_35M)
312 			priv->clk_csr = STMMAC_CSR_20_35M;
313 		else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
314 			priv->clk_csr = STMMAC_CSR_35_60M;
315 		else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
316 			priv->clk_csr = STMMAC_CSR_60_100M;
317 		else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
318 			priv->clk_csr = STMMAC_CSR_100_150M;
319 		else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
320 			priv->clk_csr = STMMAC_CSR_150_250M;
321 		else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M))
322 			priv->clk_csr = STMMAC_CSR_250_300M;
323 	}
324 
325 	if (priv->plat->has_sun8i) {
326 		if (clk_rate > 160000000)
327 			priv->clk_csr = 0x03;
328 		else if (clk_rate > 80000000)
329 			priv->clk_csr = 0x02;
330 		else if (clk_rate > 40000000)
331 			priv->clk_csr = 0x01;
332 		else
333 			priv->clk_csr = 0;
334 	}
335 
336 	if (priv->plat->has_xgmac) {
337 		if (clk_rate > 400000000)
338 			priv->clk_csr = 0x5;
339 		else if (clk_rate > 350000000)
340 			priv->clk_csr = 0x4;
341 		else if (clk_rate > 300000000)
342 			priv->clk_csr = 0x3;
343 		else if (clk_rate > 250000000)
344 			priv->clk_csr = 0x2;
345 		else if (clk_rate > 150000000)
346 			priv->clk_csr = 0x1;
347 		else
348 			priv->clk_csr = 0x0;
349 	}
350 }
351 
352 static void print_pkt(unsigned char *buf, int len)
353 {
354 	pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
355 	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
356 }
357 
358 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
359 {
360 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
361 	u32 avail;
362 
363 	if (tx_q->dirty_tx > tx_q->cur_tx)
364 		avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
365 	else
366 		avail = priv->dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;
367 
368 	return avail;
369 }
370 
371 /**
372  * stmmac_rx_dirty - Get RX queue dirty
373  * @priv: driver private structure
374  * @queue: RX queue index
375  */
376 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
377 {
378 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
379 	u32 dirty;
380 
381 	if (rx_q->dirty_rx <= rx_q->cur_rx)
382 		dirty = rx_q->cur_rx - rx_q->dirty_rx;
383 	else
384 		dirty = priv->dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;
385 
386 	return dirty;
387 }
388 
389 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en)
390 {
391 	int tx_lpi_timer;
392 
393 	/* Clear/set the SW EEE timer flag based on LPI ET enablement */
394 	priv->eee_sw_timer_en = en ? 0 : 1;
395 	tx_lpi_timer  = en ? priv->tx_lpi_timer : 0;
396 	stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer);
397 }
398 
399 /**
400  * stmmac_enable_eee_mode - check and enter in LPI mode
401  * @priv: driver private structure
402  * Description: this function is to verify and enter in LPI mode in case of
403  * EEE.
404  */
405 static int stmmac_enable_eee_mode(struct stmmac_priv *priv)
406 {
407 	u32 tx_cnt = priv->plat->tx_queues_to_use;
408 	u32 queue;
409 
410 	/* check if all TX queues have the work finished */
411 	for (queue = 0; queue < tx_cnt; queue++) {
412 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
413 
414 		if (tx_q->dirty_tx != tx_q->cur_tx)
415 			return -EBUSY; /* still unfinished work */
416 	}
417 
418 	/* Check and enter in LPI mode */
419 	if (!priv->tx_path_in_lpi_mode)
420 		stmmac_set_eee_mode(priv, priv->hw,
421 				priv->plat->en_tx_lpi_clockgating);
422 	return 0;
423 }
424 
425 /**
426  * stmmac_disable_eee_mode - disable and exit from LPI mode
427  * @priv: driver private structure
428  * Description: this function is to exit and disable EEE in case of
429  * LPI state is true. This is called by the xmit.
430  */
431 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
432 {
433 	if (!priv->eee_sw_timer_en) {
434 		stmmac_lpi_entry_timer_config(priv, 0);
435 		return;
436 	}
437 
438 	stmmac_reset_eee_mode(priv, priv->hw);
439 	del_timer_sync(&priv->eee_ctrl_timer);
440 	priv->tx_path_in_lpi_mode = false;
441 }
442 
443 /**
444  * stmmac_eee_ctrl_timer - EEE TX SW timer.
445  * @t:  timer_list struct containing private info
446  * Description:
447  *  if there is no data transfer and if we are not in LPI state,
448  *  then MAC Transmitter can be moved to LPI state.
449  */
450 static void stmmac_eee_ctrl_timer(struct timer_list *t)
451 {
452 	struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
453 
454 	if (stmmac_enable_eee_mode(priv))
455 		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
456 }
457 
458 /**
459  * stmmac_eee_init - init EEE
460  * @priv: driver private structure
461  * Description:
462  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
463  *  can also manage EEE, this function enable the LPI state and start related
464  *  timer.
465  */
466 bool stmmac_eee_init(struct stmmac_priv *priv)
467 {
468 	int eee_tw_timer = priv->eee_tw_timer;
469 
470 	/* Using PCS we cannot dial with the phy registers at this stage
471 	 * so we do not support extra feature like EEE.
472 	 */
473 	if (priv->hw->pcs == STMMAC_PCS_TBI ||
474 	    priv->hw->pcs == STMMAC_PCS_RTBI)
475 		return false;
476 
477 	/* Check if MAC core supports the EEE feature. */
478 	if (!priv->dma_cap.eee)
479 		return false;
480 
481 	mutex_lock(&priv->lock);
482 
483 	/* Check if it needs to be deactivated */
484 	if (!priv->eee_active) {
485 		if (priv->eee_enabled) {
486 			netdev_dbg(priv->dev, "disable EEE\n");
487 			stmmac_lpi_entry_timer_config(priv, 0);
488 			del_timer_sync(&priv->eee_ctrl_timer);
489 			stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer);
490 			if (priv->hw->xpcs)
491 				xpcs_config_eee(priv->hw->xpcs,
492 						priv->plat->mult_fact_100ns,
493 						false);
494 		}
495 		mutex_unlock(&priv->lock);
496 		return false;
497 	}
498 
499 	if (priv->eee_active && !priv->eee_enabled) {
500 		timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
501 		stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
502 				     eee_tw_timer);
503 		if (priv->hw->xpcs)
504 			xpcs_config_eee(priv->hw->xpcs,
505 					priv->plat->mult_fact_100ns,
506 					true);
507 	}
508 
509 	if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) {
510 		del_timer_sync(&priv->eee_ctrl_timer);
511 		priv->tx_path_in_lpi_mode = false;
512 		stmmac_lpi_entry_timer_config(priv, 1);
513 	} else {
514 		stmmac_lpi_entry_timer_config(priv, 0);
515 		mod_timer(&priv->eee_ctrl_timer,
516 			  STMMAC_LPI_T(priv->tx_lpi_timer));
517 	}
518 
519 	mutex_unlock(&priv->lock);
520 	netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
521 	return true;
522 }
523 
524 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
525  * @priv: driver private structure
526  * @p : descriptor pointer
527  * @skb : the socket buffer
528  * Description :
529  * This function will read timestamp from the descriptor & pass it to stack.
530  * and also perform some sanity checks.
531  */
532 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
533 				   struct dma_desc *p, struct sk_buff *skb)
534 {
535 	struct skb_shared_hwtstamps shhwtstamp;
536 	bool found = false;
537 	u64 ns = 0;
538 
539 	if (!priv->hwts_tx_en)
540 		return;
541 
542 	/* exit if skb doesn't support hw tstamp */
543 	if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
544 		return;
545 
546 	/* check tx tstamp status */
547 	if (stmmac_get_tx_timestamp_status(priv, p)) {
548 		stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
549 		found = true;
550 	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
551 		found = true;
552 	}
553 
554 	if (found) {
555 		ns -= priv->plat->cdc_error_adj;
556 
557 		memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
558 		shhwtstamp.hwtstamp = ns_to_ktime(ns);
559 
560 		netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
561 		/* pass tstamp to stack */
562 		skb_tstamp_tx(skb, &shhwtstamp);
563 	}
564 }
565 
566 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
567  * @priv: driver private structure
568  * @p : descriptor pointer
569  * @np : next descriptor pointer
570  * @skb : the socket buffer
571  * Description :
572  * This function will read received packet's timestamp from the descriptor
573  * and pass it to stack. It also perform some sanity checks.
574  */
575 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
576 				   struct dma_desc *np, struct sk_buff *skb)
577 {
578 	struct skb_shared_hwtstamps *shhwtstamp = NULL;
579 	struct dma_desc *desc = p;
580 	u64 ns = 0;
581 
582 	if (!priv->hwts_rx_en)
583 		return;
584 	/* For GMAC4, the valid timestamp is from CTX next desc. */
585 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
586 		desc = np;
587 
588 	/* Check if timestamp is available */
589 	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
590 		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
591 
592 		ns -= priv->plat->cdc_error_adj;
593 
594 		netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
595 		shhwtstamp = skb_hwtstamps(skb);
596 		memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
597 		shhwtstamp->hwtstamp = ns_to_ktime(ns);
598 	} else  {
599 		netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
600 	}
601 }
602 
603 /**
604  *  stmmac_hwtstamp_set - control hardware timestamping.
605  *  @dev: device pointer.
606  *  @ifr: An IOCTL specific structure, that can contain a pointer to
607  *  a proprietary structure used to pass information to the driver.
608  *  Description:
609  *  This function configures the MAC to enable/disable both outgoing(TX)
610  *  and incoming(RX) packets time stamping based on user input.
611  *  Return Value:
612  *  0 on success and an appropriate -ve integer on failure.
613  */
614 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
615 {
616 	struct stmmac_priv *priv = netdev_priv(dev);
617 	struct hwtstamp_config config;
618 	u32 ptp_v2 = 0;
619 	u32 tstamp_all = 0;
620 	u32 ptp_over_ipv4_udp = 0;
621 	u32 ptp_over_ipv6_udp = 0;
622 	u32 ptp_over_ethernet = 0;
623 	u32 snap_type_sel = 0;
624 	u32 ts_master_en = 0;
625 	u32 ts_event_en = 0;
626 
627 	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
628 		netdev_alert(priv->dev, "No support for HW time stamping\n");
629 		priv->hwts_tx_en = 0;
630 		priv->hwts_rx_en = 0;
631 
632 		return -EOPNOTSUPP;
633 	}
634 
635 	if (copy_from_user(&config, ifr->ifr_data,
636 			   sizeof(config)))
637 		return -EFAULT;
638 
639 	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
640 		   __func__, config.flags, config.tx_type, config.rx_filter);
641 
642 	if (config.tx_type != HWTSTAMP_TX_OFF &&
643 	    config.tx_type != HWTSTAMP_TX_ON)
644 		return -ERANGE;
645 
646 	if (priv->adv_ts) {
647 		switch (config.rx_filter) {
648 		case HWTSTAMP_FILTER_NONE:
649 			/* time stamp no incoming packet at all */
650 			config.rx_filter = HWTSTAMP_FILTER_NONE;
651 			break;
652 
653 		case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
654 			/* PTP v1, UDP, any kind of event packet */
655 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
656 			/* 'xmac' hardware can support Sync, Pdelay_Req and
657 			 * Pdelay_resp by setting bit14 and bits17/16 to 01
658 			 * This leaves Delay_Req timestamps out.
659 			 * Enable all events *and* general purpose message
660 			 * timestamping
661 			 */
662 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
663 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
664 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
665 			break;
666 
667 		case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
668 			/* PTP v1, UDP, Sync packet */
669 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
670 			/* take time stamp for SYNC messages only */
671 			ts_event_en = PTP_TCR_TSEVNTENA;
672 
673 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
674 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
675 			break;
676 
677 		case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
678 			/* PTP v1, UDP, Delay_req packet */
679 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
680 			/* take time stamp for Delay_Req messages only */
681 			ts_master_en = PTP_TCR_TSMSTRENA;
682 			ts_event_en = PTP_TCR_TSEVNTENA;
683 
684 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
685 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
686 			break;
687 
688 		case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
689 			/* PTP v2, UDP, any kind of event packet */
690 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
691 			ptp_v2 = PTP_TCR_TSVER2ENA;
692 			/* take time stamp for all event messages */
693 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
694 
695 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
696 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
697 			break;
698 
699 		case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
700 			/* PTP v2, UDP, Sync packet */
701 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
702 			ptp_v2 = PTP_TCR_TSVER2ENA;
703 			/* take time stamp for SYNC messages only */
704 			ts_event_en = PTP_TCR_TSEVNTENA;
705 
706 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
707 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
708 			break;
709 
710 		case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
711 			/* PTP v2, UDP, Delay_req packet */
712 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
713 			ptp_v2 = PTP_TCR_TSVER2ENA;
714 			/* take time stamp for Delay_Req messages only */
715 			ts_master_en = PTP_TCR_TSMSTRENA;
716 			ts_event_en = PTP_TCR_TSEVNTENA;
717 
718 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
719 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
720 			break;
721 
722 		case HWTSTAMP_FILTER_PTP_V2_EVENT:
723 			/* PTP v2/802.AS1 any layer, any kind of event packet */
724 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
725 			ptp_v2 = PTP_TCR_TSVER2ENA;
726 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
727 			if (priv->synopsys_id < DWMAC_CORE_4_10)
728 				ts_event_en = PTP_TCR_TSEVNTENA;
729 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
730 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
731 			ptp_over_ethernet = PTP_TCR_TSIPENA;
732 			break;
733 
734 		case HWTSTAMP_FILTER_PTP_V2_SYNC:
735 			/* PTP v2/802.AS1, any layer, Sync packet */
736 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
737 			ptp_v2 = PTP_TCR_TSVER2ENA;
738 			/* take time stamp for SYNC messages only */
739 			ts_event_en = PTP_TCR_TSEVNTENA;
740 
741 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
742 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
743 			ptp_over_ethernet = PTP_TCR_TSIPENA;
744 			break;
745 
746 		case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
747 			/* PTP v2/802.AS1, any layer, Delay_req packet */
748 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
749 			ptp_v2 = PTP_TCR_TSVER2ENA;
750 			/* take time stamp for Delay_Req messages only */
751 			ts_master_en = PTP_TCR_TSMSTRENA;
752 			ts_event_en = PTP_TCR_TSEVNTENA;
753 
754 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
755 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
756 			ptp_over_ethernet = PTP_TCR_TSIPENA;
757 			break;
758 
759 		case HWTSTAMP_FILTER_NTP_ALL:
760 		case HWTSTAMP_FILTER_ALL:
761 			/* time stamp any incoming packet */
762 			config.rx_filter = HWTSTAMP_FILTER_ALL;
763 			tstamp_all = PTP_TCR_TSENALL;
764 			break;
765 
766 		default:
767 			return -ERANGE;
768 		}
769 	} else {
770 		switch (config.rx_filter) {
771 		case HWTSTAMP_FILTER_NONE:
772 			config.rx_filter = HWTSTAMP_FILTER_NONE;
773 			break;
774 		default:
775 			/* PTP v1, UDP, any kind of event packet */
776 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
777 			break;
778 		}
779 	}
780 	priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
781 	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
782 
783 	priv->systime_flags = STMMAC_HWTS_ACTIVE;
784 
785 	if (priv->hwts_tx_en || priv->hwts_rx_en) {
786 		priv->systime_flags |= tstamp_all | ptp_v2 |
787 				       ptp_over_ethernet | ptp_over_ipv6_udp |
788 				       ptp_over_ipv4_udp | ts_event_en |
789 				       ts_master_en | snap_type_sel;
790 	}
791 
792 	stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags);
793 
794 	memcpy(&priv->tstamp_config, &config, sizeof(config));
795 
796 	return copy_to_user(ifr->ifr_data, &config,
797 			    sizeof(config)) ? -EFAULT : 0;
798 }
799 
800 /**
801  *  stmmac_hwtstamp_get - read hardware timestamping.
802  *  @dev: device pointer.
803  *  @ifr: An IOCTL specific structure, that can contain a pointer to
804  *  a proprietary structure used to pass information to the driver.
805  *  Description:
806  *  This function obtain the current hardware timestamping settings
807  *  as requested.
808  */
809 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
810 {
811 	struct stmmac_priv *priv = netdev_priv(dev);
812 	struct hwtstamp_config *config = &priv->tstamp_config;
813 
814 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
815 		return -EOPNOTSUPP;
816 
817 	return copy_to_user(ifr->ifr_data, config,
818 			    sizeof(*config)) ? -EFAULT : 0;
819 }
820 
821 /**
822  * stmmac_init_tstamp_counter - init hardware timestamping counter
823  * @priv: driver private structure
824  * @systime_flags: timestamping flags
825  * Description:
826  * Initialize hardware counter for packet timestamping.
827  * This is valid as long as the interface is open and not suspended.
828  * Will be rerun after resuming from suspend, case in which the timestamping
829  * flags updated by stmmac_hwtstamp_set() also need to be restored.
830  */
831 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags)
832 {
833 	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
834 	struct timespec64 now;
835 	u32 sec_inc = 0;
836 	u64 temp = 0;
837 	int ret;
838 
839 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
840 		return -EOPNOTSUPP;
841 
842 	ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
843 	if (ret < 0) {
844 		netdev_warn(priv->dev,
845 			    "failed to enable PTP reference clock: %pe\n",
846 			    ERR_PTR(ret));
847 		return ret;
848 	}
849 
850 	stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags);
851 	priv->systime_flags = systime_flags;
852 
853 	/* program Sub Second Increment reg */
854 	stmmac_config_sub_second_increment(priv, priv->ptpaddr,
855 					   priv->plat->clk_ptp_rate,
856 					   xmac, &sec_inc);
857 	temp = div_u64(1000000000ULL, sec_inc);
858 
859 	/* Store sub second increment for later use */
860 	priv->sub_second_inc = sec_inc;
861 
862 	/* calculate default added value:
863 	 * formula is :
864 	 * addend = (2^32)/freq_div_ratio;
865 	 * where, freq_div_ratio = 1e9ns/sec_inc
866 	 */
867 	temp = (u64)(temp << 32);
868 	priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
869 	stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
870 
871 	/* initialize system time */
872 	ktime_get_real_ts64(&now);
873 
874 	/* lower 32 bits of tv_sec are safe until y2106 */
875 	stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec);
876 
877 	return 0;
878 }
879 EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter);
880 
881 /**
882  * stmmac_init_ptp - init PTP
883  * @priv: driver private structure
884  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
885  * This is done by looking at the HW cap. register.
886  * This function also registers the ptp driver.
887  */
888 static int stmmac_init_ptp(struct stmmac_priv *priv)
889 {
890 	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
891 	int ret;
892 
893 	if (priv->plat->ptp_clk_freq_config)
894 		priv->plat->ptp_clk_freq_config(priv);
895 
896 	ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE);
897 	if (ret)
898 		return ret;
899 
900 	priv->adv_ts = 0;
901 	/* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
902 	if (xmac && priv->dma_cap.atime_stamp)
903 		priv->adv_ts = 1;
904 	/* Dwmac 3.x core with extend_desc can support adv_ts */
905 	else if (priv->extend_desc && priv->dma_cap.atime_stamp)
906 		priv->adv_ts = 1;
907 
908 	if (priv->dma_cap.time_stamp)
909 		netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
910 
911 	if (priv->adv_ts)
912 		netdev_info(priv->dev,
913 			    "IEEE 1588-2008 Advanced Timestamp supported\n");
914 
915 	priv->hwts_tx_en = 0;
916 	priv->hwts_rx_en = 0;
917 
918 	return 0;
919 }
920 
921 static void stmmac_release_ptp(struct stmmac_priv *priv)
922 {
923 	clk_disable_unprepare(priv->plat->clk_ptp_ref);
924 	stmmac_ptp_unregister(priv);
925 }
926 
927 /**
928  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
929  *  @priv: driver private structure
930  *  @duplex: duplex passed to the next function
931  *  Description: It is used for configuring the flow control in all queues
932  */
933 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
934 {
935 	u32 tx_cnt = priv->plat->tx_queues_to_use;
936 
937 	stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
938 			priv->pause, tx_cnt);
939 }
940 
941 static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config,
942 						 phy_interface_t interface)
943 {
944 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
945 
946 	if (!priv->hw->xpcs)
947 		return NULL;
948 
949 	return &priv->hw->xpcs->pcs;
950 }
951 
952 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
953 			      const struct phylink_link_state *state)
954 {
955 	/* Nothing to do, xpcs_config() handles everything */
956 }
957 
958 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up)
959 {
960 	struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg;
961 	enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state;
962 	enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state;
963 	bool *hs_enable = &fpe_cfg->hs_enable;
964 
965 	if (is_up && *hs_enable) {
966 		stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY);
967 	} else {
968 		*lo_state = FPE_STATE_OFF;
969 		*lp_state = FPE_STATE_OFF;
970 	}
971 }
972 
973 static void stmmac_mac_link_down(struct phylink_config *config,
974 				 unsigned int mode, phy_interface_t interface)
975 {
976 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
977 
978 	stmmac_mac_set(priv, priv->ioaddr, false);
979 	priv->eee_active = false;
980 	priv->tx_lpi_enabled = false;
981 	priv->eee_enabled = stmmac_eee_init(priv);
982 	stmmac_set_eee_pls(priv, priv->hw, false);
983 
984 	if (priv->dma_cap.fpesel)
985 		stmmac_fpe_link_state_handle(priv, false);
986 }
987 
988 static void stmmac_mac_link_up(struct phylink_config *config,
989 			       struct phy_device *phy,
990 			       unsigned int mode, phy_interface_t interface,
991 			       int speed, int duplex,
992 			       bool tx_pause, bool rx_pause)
993 {
994 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
995 	u32 ctrl;
996 
997 	ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
998 	ctrl &= ~priv->hw->link.speed_mask;
999 
1000 	if (interface == PHY_INTERFACE_MODE_USXGMII) {
1001 		switch (speed) {
1002 		case SPEED_10000:
1003 			ctrl |= priv->hw->link.xgmii.speed10000;
1004 			break;
1005 		case SPEED_5000:
1006 			ctrl |= priv->hw->link.xgmii.speed5000;
1007 			break;
1008 		case SPEED_2500:
1009 			ctrl |= priv->hw->link.xgmii.speed2500;
1010 			break;
1011 		default:
1012 			return;
1013 		}
1014 	} else if (interface == PHY_INTERFACE_MODE_XLGMII) {
1015 		switch (speed) {
1016 		case SPEED_100000:
1017 			ctrl |= priv->hw->link.xlgmii.speed100000;
1018 			break;
1019 		case SPEED_50000:
1020 			ctrl |= priv->hw->link.xlgmii.speed50000;
1021 			break;
1022 		case SPEED_40000:
1023 			ctrl |= priv->hw->link.xlgmii.speed40000;
1024 			break;
1025 		case SPEED_25000:
1026 			ctrl |= priv->hw->link.xlgmii.speed25000;
1027 			break;
1028 		case SPEED_10000:
1029 			ctrl |= priv->hw->link.xgmii.speed10000;
1030 			break;
1031 		case SPEED_2500:
1032 			ctrl |= priv->hw->link.speed2500;
1033 			break;
1034 		case SPEED_1000:
1035 			ctrl |= priv->hw->link.speed1000;
1036 			break;
1037 		default:
1038 			return;
1039 		}
1040 	} else {
1041 		switch (speed) {
1042 		case SPEED_2500:
1043 			ctrl |= priv->hw->link.speed2500;
1044 			break;
1045 		case SPEED_1000:
1046 			ctrl |= priv->hw->link.speed1000;
1047 			break;
1048 		case SPEED_100:
1049 			ctrl |= priv->hw->link.speed100;
1050 			break;
1051 		case SPEED_10:
1052 			ctrl |= priv->hw->link.speed10;
1053 			break;
1054 		default:
1055 			return;
1056 		}
1057 	}
1058 
1059 	priv->speed = speed;
1060 
1061 	if (priv->plat->fix_mac_speed)
1062 		priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed);
1063 
1064 	if (!duplex)
1065 		ctrl &= ~priv->hw->link.duplex;
1066 	else
1067 		ctrl |= priv->hw->link.duplex;
1068 
1069 	/* Flow Control operation */
1070 	if (tx_pause && rx_pause)
1071 		stmmac_mac_flow_ctrl(priv, duplex);
1072 
1073 	writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
1074 
1075 	stmmac_mac_set(priv, priv->ioaddr, true);
1076 	if (phy && priv->dma_cap.eee) {
1077 		priv->eee_active = phy_init_eee(phy, 1) >= 0;
1078 		priv->eee_enabled = stmmac_eee_init(priv);
1079 		priv->tx_lpi_enabled = priv->eee_enabled;
1080 		stmmac_set_eee_pls(priv, priv->hw, true);
1081 	}
1082 
1083 	if (priv->dma_cap.fpesel)
1084 		stmmac_fpe_link_state_handle(priv, true);
1085 }
1086 
1087 static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
1088 	.validate = phylink_generic_validate,
1089 	.mac_select_pcs = stmmac_mac_select_pcs,
1090 	.mac_config = stmmac_mac_config,
1091 	.mac_link_down = stmmac_mac_link_down,
1092 	.mac_link_up = stmmac_mac_link_up,
1093 };
1094 
1095 /**
1096  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
1097  * @priv: driver private structure
1098  * Description: this is to verify if the HW supports the PCS.
1099  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
1100  * configured for the TBI, RTBI, or SGMII PHY interface.
1101  */
1102 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
1103 {
1104 	int interface = priv->plat->interface;
1105 
1106 	if (priv->dma_cap.pcs) {
1107 		if ((interface == PHY_INTERFACE_MODE_RGMII) ||
1108 		    (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
1109 		    (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
1110 		    (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
1111 			netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
1112 			priv->hw->pcs = STMMAC_PCS_RGMII;
1113 		} else if (interface == PHY_INTERFACE_MODE_SGMII) {
1114 			netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
1115 			priv->hw->pcs = STMMAC_PCS_SGMII;
1116 		}
1117 	}
1118 }
1119 
1120 /**
1121  * stmmac_init_phy - PHY initialization
1122  * @dev: net device structure
1123  * Description: it initializes the driver's PHY state, and attaches the PHY
1124  * to the mac driver.
1125  *  Return value:
1126  *  0 on success
1127  */
1128 static int stmmac_init_phy(struct net_device *dev)
1129 {
1130 	struct stmmac_priv *priv = netdev_priv(dev);
1131 	struct device_node *node;
1132 	int ret;
1133 
1134 	node = priv->plat->phylink_node;
1135 
1136 	if (node)
1137 		ret = phylink_of_phy_connect(priv->phylink, node, 0);
1138 
1139 	/* Some DT bindings do not set-up the PHY handle. Let's try to
1140 	 * manually parse it
1141 	 */
1142 	if (!node || ret) {
1143 		int addr = priv->plat->phy_addr;
1144 		struct phy_device *phydev;
1145 
1146 		phydev = mdiobus_get_phy(priv->mii, addr);
1147 		if (!phydev) {
1148 			netdev_err(priv->dev, "no phy at addr %d\n", addr);
1149 			return -ENODEV;
1150 		}
1151 
1152 		ret = phylink_connect_phy(priv->phylink, phydev);
1153 	}
1154 
1155 	if (!priv->plat->pmt) {
1156 		struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
1157 
1158 		phylink_ethtool_get_wol(priv->phylink, &wol);
1159 		device_set_wakeup_capable(priv->device, !!wol.supported);
1160 	}
1161 
1162 	return ret;
1163 }
1164 
1165 static int stmmac_phy_setup(struct stmmac_priv *priv)
1166 {
1167 	struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data;
1168 	struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node);
1169 	int max_speed = priv->plat->max_speed;
1170 	int mode = priv->plat->phy_interface;
1171 	struct phylink *phylink;
1172 
1173 	priv->phylink_config.dev = &priv->dev->dev;
1174 	priv->phylink_config.type = PHYLINK_NETDEV;
1175 	if (priv->plat->mdio_bus_data)
1176 		priv->phylink_config.ovr_an_inband =
1177 			mdio_bus_data->xpcs_an_inband;
1178 
1179 	if (!fwnode)
1180 		fwnode = dev_fwnode(priv->device);
1181 
1182 	/* Set the platform/firmware specified interface mode */
1183 	__set_bit(mode, priv->phylink_config.supported_interfaces);
1184 
1185 	/* If we have an xpcs, it defines which PHY interfaces are supported. */
1186 	if (priv->hw->xpcs)
1187 		xpcs_get_interfaces(priv->hw->xpcs,
1188 				    priv->phylink_config.supported_interfaces);
1189 
1190 	priv->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
1191 		MAC_10 | MAC_100;
1192 
1193 	if (!max_speed || max_speed >= 1000)
1194 		priv->phylink_config.mac_capabilities |= MAC_1000;
1195 
1196 	if (priv->plat->has_gmac4) {
1197 		if (!max_speed || max_speed >= 2500)
1198 			priv->phylink_config.mac_capabilities |= MAC_2500FD;
1199 	} else if (priv->plat->has_xgmac) {
1200 		if (!max_speed || max_speed >= 2500)
1201 			priv->phylink_config.mac_capabilities |= MAC_2500FD;
1202 		if (!max_speed || max_speed >= 5000)
1203 			priv->phylink_config.mac_capabilities |= MAC_5000FD;
1204 		if (!max_speed || max_speed >= 10000)
1205 			priv->phylink_config.mac_capabilities |= MAC_10000FD;
1206 		if (!max_speed || max_speed >= 25000)
1207 			priv->phylink_config.mac_capabilities |= MAC_25000FD;
1208 		if (!max_speed || max_speed >= 40000)
1209 			priv->phylink_config.mac_capabilities |= MAC_40000FD;
1210 		if (!max_speed || max_speed >= 50000)
1211 			priv->phylink_config.mac_capabilities |= MAC_50000FD;
1212 		if (!max_speed || max_speed >= 100000)
1213 			priv->phylink_config.mac_capabilities |= MAC_100000FD;
1214 	}
1215 
1216 	/* Half-Duplex can only work with single queue */
1217 	if (priv->plat->tx_queues_to_use > 1)
1218 		priv->phylink_config.mac_capabilities &=
1219 			~(MAC_10HD | MAC_100HD | MAC_1000HD);
1220 
1221 	phylink = phylink_create(&priv->phylink_config, fwnode,
1222 				 mode, &stmmac_phylink_mac_ops);
1223 	if (IS_ERR(phylink))
1224 		return PTR_ERR(phylink);
1225 
1226 	priv->phylink = phylink;
1227 	return 0;
1228 }
1229 
1230 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
1231 {
1232 	u32 rx_cnt = priv->plat->rx_queues_to_use;
1233 	unsigned int desc_size;
1234 	void *head_rx;
1235 	u32 queue;
1236 
1237 	/* Display RX rings */
1238 	for (queue = 0; queue < rx_cnt; queue++) {
1239 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1240 
1241 		pr_info("\tRX Queue %u rings\n", queue);
1242 
1243 		if (priv->extend_desc) {
1244 			head_rx = (void *)rx_q->dma_erx;
1245 			desc_size = sizeof(struct dma_extended_desc);
1246 		} else {
1247 			head_rx = (void *)rx_q->dma_rx;
1248 			desc_size = sizeof(struct dma_desc);
1249 		}
1250 
1251 		/* Display RX ring */
1252 		stmmac_display_ring(priv, head_rx, priv->dma_rx_size, true,
1253 				    rx_q->dma_rx_phy, desc_size);
1254 	}
1255 }
1256 
1257 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1258 {
1259 	u32 tx_cnt = priv->plat->tx_queues_to_use;
1260 	unsigned int desc_size;
1261 	void *head_tx;
1262 	u32 queue;
1263 
1264 	/* Display TX rings */
1265 	for (queue = 0; queue < tx_cnt; queue++) {
1266 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1267 
1268 		pr_info("\tTX Queue %d rings\n", queue);
1269 
1270 		if (priv->extend_desc) {
1271 			head_tx = (void *)tx_q->dma_etx;
1272 			desc_size = sizeof(struct dma_extended_desc);
1273 		} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
1274 			head_tx = (void *)tx_q->dma_entx;
1275 			desc_size = sizeof(struct dma_edesc);
1276 		} else {
1277 			head_tx = (void *)tx_q->dma_tx;
1278 			desc_size = sizeof(struct dma_desc);
1279 		}
1280 
1281 		stmmac_display_ring(priv, head_tx, priv->dma_tx_size, false,
1282 				    tx_q->dma_tx_phy, desc_size);
1283 	}
1284 }
1285 
1286 static void stmmac_display_rings(struct stmmac_priv *priv)
1287 {
1288 	/* Display RX ring */
1289 	stmmac_display_rx_rings(priv);
1290 
1291 	/* Display TX ring */
1292 	stmmac_display_tx_rings(priv);
1293 }
1294 
1295 static int stmmac_set_bfsize(int mtu, int bufsize)
1296 {
1297 	int ret = bufsize;
1298 
1299 	if (mtu >= BUF_SIZE_8KiB)
1300 		ret = BUF_SIZE_16KiB;
1301 	else if (mtu >= BUF_SIZE_4KiB)
1302 		ret = BUF_SIZE_8KiB;
1303 	else if (mtu >= BUF_SIZE_2KiB)
1304 		ret = BUF_SIZE_4KiB;
1305 	else if (mtu > DEFAULT_BUFSIZE)
1306 		ret = BUF_SIZE_2KiB;
1307 	else
1308 		ret = DEFAULT_BUFSIZE;
1309 
1310 	return ret;
1311 }
1312 
1313 /**
1314  * stmmac_clear_rx_descriptors - clear RX descriptors
1315  * @priv: driver private structure
1316  * @queue: RX queue index
1317  * Description: this function is called to clear the RX descriptors
1318  * in case of both basic and extended descriptors are used.
1319  */
1320 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1321 {
1322 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1323 	int i;
1324 
1325 	/* Clear the RX descriptors */
1326 	for (i = 0; i < priv->dma_rx_size; i++)
1327 		if (priv->extend_desc)
1328 			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1329 					priv->use_riwt, priv->mode,
1330 					(i == priv->dma_rx_size - 1),
1331 					priv->dma_buf_sz);
1332 		else
1333 			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1334 					priv->use_riwt, priv->mode,
1335 					(i == priv->dma_rx_size - 1),
1336 					priv->dma_buf_sz);
1337 }
1338 
1339 /**
1340  * stmmac_clear_tx_descriptors - clear tx descriptors
1341  * @priv: driver private structure
1342  * @queue: TX queue index.
1343  * Description: this function is called to clear the TX descriptors
1344  * in case of both basic and extended descriptors are used.
1345  */
1346 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1347 {
1348 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1349 	int i;
1350 
1351 	/* Clear the TX descriptors */
1352 	for (i = 0; i < priv->dma_tx_size; i++) {
1353 		int last = (i == (priv->dma_tx_size - 1));
1354 		struct dma_desc *p;
1355 
1356 		if (priv->extend_desc)
1357 			p = &tx_q->dma_etx[i].basic;
1358 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1359 			p = &tx_q->dma_entx[i].basic;
1360 		else
1361 			p = &tx_q->dma_tx[i];
1362 
1363 		stmmac_init_tx_desc(priv, p, priv->mode, last);
1364 	}
1365 }
1366 
1367 /**
1368  * stmmac_clear_descriptors - clear descriptors
1369  * @priv: driver private structure
1370  * Description: this function is called to clear the TX and RX descriptors
1371  * in case of both basic and extended descriptors are used.
1372  */
1373 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1374 {
1375 	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1376 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1377 	u32 queue;
1378 
1379 	/* Clear the RX descriptors */
1380 	for (queue = 0; queue < rx_queue_cnt; queue++)
1381 		stmmac_clear_rx_descriptors(priv, queue);
1382 
1383 	/* Clear the TX descriptors */
1384 	for (queue = 0; queue < tx_queue_cnt; queue++)
1385 		stmmac_clear_tx_descriptors(priv, queue);
1386 }
1387 
1388 /**
1389  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1390  * @priv: driver private structure
1391  * @p: descriptor pointer
1392  * @i: descriptor index
1393  * @flags: gfp flag
1394  * @queue: RX queue index
1395  * Description: this function is called to allocate a receive buffer, perform
1396  * the DMA mapping and init the descriptor.
1397  */
1398 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1399 				  int i, gfp_t flags, u32 queue)
1400 {
1401 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1402 	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1403 	gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
1404 
1405 	if (priv->dma_cap.addr64 <= 32)
1406 		gfp |= GFP_DMA32;
1407 
1408 	if (!buf->page) {
1409 		buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp);
1410 		if (!buf->page)
1411 			return -ENOMEM;
1412 		buf->page_offset = stmmac_rx_offset(priv);
1413 	}
1414 
1415 	if (priv->sph && !buf->sec_page) {
1416 		buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp);
1417 		if (!buf->sec_page)
1418 			return -ENOMEM;
1419 
1420 		buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
1421 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
1422 	} else {
1423 		buf->sec_page = NULL;
1424 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
1425 	}
1426 
1427 	buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset;
1428 
1429 	stmmac_set_desc_addr(priv, p, buf->addr);
1430 	if (priv->dma_buf_sz == BUF_SIZE_16KiB)
1431 		stmmac_init_desc3(priv, p);
1432 
1433 	return 0;
1434 }
1435 
1436 /**
1437  * stmmac_free_rx_buffer - free RX dma buffers
1438  * @priv: private structure
1439  * @queue: RX queue index
1440  * @i: buffer index.
1441  */
1442 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1443 {
1444 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1445 	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1446 
1447 	if (buf->page)
1448 		page_pool_put_full_page(rx_q->page_pool, buf->page, false);
1449 	buf->page = NULL;
1450 
1451 	if (buf->sec_page)
1452 		page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false);
1453 	buf->sec_page = NULL;
1454 }
1455 
1456 /**
1457  * stmmac_free_tx_buffer - free RX dma buffers
1458  * @priv: private structure
1459  * @queue: RX queue index
1460  * @i: buffer index.
1461  */
1462 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1463 {
1464 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1465 
1466 	if (tx_q->tx_skbuff_dma[i].buf &&
1467 	    tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) {
1468 		if (tx_q->tx_skbuff_dma[i].map_as_page)
1469 			dma_unmap_page(priv->device,
1470 				       tx_q->tx_skbuff_dma[i].buf,
1471 				       tx_q->tx_skbuff_dma[i].len,
1472 				       DMA_TO_DEVICE);
1473 		else
1474 			dma_unmap_single(priv->device,
1475 					 tx_q->tx_skbuff_dma[i].buf,
1476 					 tx_q->tx_skbuff_dma[i].len,
1477 					 DMA_TO_DEVICE);
1478 	}
1479 
1480 	if (tx_q->xdpf[i] &&
1481 	    (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX ||
1482 	     tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) {
1483 		xdp_return_frame(tx_q->xdpf[i]);
1484 		tx_q->xdpf[i] = NULL;
1485 	}
1486 
1487 	if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX)
1488 		tx_q->xsk_frames_done++;
1489 
1490 	if (tx_q->tx_skbuff[i] &&
1491 	    tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) {
1492 		dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1493 		tx_q->tx_skbuff[i] = NULL;
1494 	}
1495 
1496 	tx_q->tx_skbuff_dma[i].buf = 0;
1497 	tx_q->tx_skbuff_dma[i].map_as_page = false;
1498 }
1499 
1500 /**
1501  * dma_free_rx_skbufs - free RX dma buffers
1502  * @priv: private structure
1503  * @queue: RX queue index
1504  */
1505 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1506 {
1507 	int i;
1508 
1509 	for (i = 0; i < priv->dma_rx_size; i++)
1510 		stmmac_free_rx_buffer(priv, queue, i);
1511 }
1512 
1513 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, u32 queue,
1514 				   gfp_t flags)
1515 {
1516 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1517 	int i;
1518 
1519 	for (i = 0; i < priv->dma_rx_size; i++) {
1520 		struct dma_desc *p;
1521 		int ret;
1522 
1523 		if (priv->extend_desc)
1524 			p = &((rx_q->dma_erx + i)->basic);
1525 		else
1526 			p = rx_q->dma_rx + i;
1527 
1528 		ret = stmmac_init_rx_buffers(priv, p, i, flags,
1529 					     queue);
1530 		if (ret)
1531 			return ret;
1532 
1533 		rx_q->buf_alloc_num++;
1534 	}
1535 
1536 	return 0;
1537 }
1538 
1539 /**
1540  * dma_free_rx_xskbufs - free RX dma buffers from XSK pool
1541  * @priv: private structure
1542  * @queue: RX queue index
1543  */
1544 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, u32 queue)
1545 {
1546 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1547 	int i;
1548 
1549 	for (i = 0; i < priv->dma_rx_size; i++) {
1550 		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1551 
1552 		if (!buf->xdp)
1553 			continue;
1554 
1555 		xsk_buff_free(buf->xdp);
1556 		buf->xdp = NULL;
1557 	}
1558 }
1559 
1560 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, u32 queue)
1561 {
1562 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1563 	int i;
1564 
1565 	for (i = 0; i < priv->dma_rx_size; i++) {
1566 		struct stmmac_rx_buffer *buf;
1567 		dma_addr_t dma_addr;
1568 		struct dma_desc *p;
1569 
1570 		if (priv->extend_desc)
1571 			p = (struct dma_desc *)(rx_q->dma_erx + i);
1572 		else
1573 			p = rx_q->dma_rx + i;
1574 
1575 		buf = &rx_q->buf_pool[i];
1576 
1577 		buf->xdp = xsk_buff_alloc(rx_q->xsk_pool);
1578 		if (!buf->xdp)
1579 			return -ENOMEM;
1580 
1581 		dma_addr = xsk_buff_xdp_get_dma(buf->xdp);
1582 		stmmac_set_desc_addr(priv, p, dma_addr);
1583 		rx_q->buf_alloc_num++;
1584 	}
1585 
1586 	return 0;
1587 }
1588 
1589 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue)
1590 {
1591 	if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps))
1592 		return NULL;
1593 
1594 	return xsk_get_pool_from_qid(priv->dev, queue);
1595 }
1596 
1597 /**
1598  * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue)
1599  * @priv: driver private structure
1600  * @queue: RX queue index
1601  * @flags: gfp flag.
1602  * Description: this function initializes the DMA RX descriptors
1603  * and allocates the socket buffers. It supports the chained and ring
1604  * modes.
1605  */
1606 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, u32 queue, gfp_t flags)
1607 {
1608 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1609 	int ret;
1610 
1611 	netif_dbg(priv, probe, priv->dev,
1612 		  "(%s) dma_rx_phy=0x%08x\n", __func__,
1613 		  (u32)rx_q->dma_rx_phy);
1614 
1615 	stmmac_clear_rx_descriptors(priv, queue);
1616 
1617 	xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq);
1618 
1619 	rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
1620 
1621 	if (rx_q->xsk_pool) {
1622 		WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
1623 						   MEM_TYPE_XSK_BUFF_POOL,
1624 						   NULL));
1625 		netdev_info(priv->dev,
1626 			    "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n",
1627 			    rx_q->queue_index);
1628 		xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq);
1629 	} else {
1630 		WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
1631 						   MEM_TYPE_PAGE_POOL,
1632 						   rx_q->page_pool));
1633 		netdev_info(priv->dev,
1634 			    "Register MEM_TYPE_PAGE_POOL RxQ-%d\n",
1635 			    rx_q->queue_index);
1636 	}
1637 
1638 	if (rx_q->xsk_pool) {
1639 		/* RX XDP ZC buffer pool may not be populated, e.g.
1640 		 * xdpsock TX-only.
1641 		 */
1642 		stmmac_alloc_rx_buffers_zc(priv, queue);
1643 	} else {
1644 		ret = stmmac_alloc_rx_buffers(priv, queue, flags);
1645 		if (ret < 0)
1646 			return -ENOMEM;
1647 	}
1648 
1649 	rx_q->cur_rx = 0;
1650 	rx_q->dirty_rx = 0;
1651 
1652 	/* Setup the chained descriptor addresses */
1653 	if (priv->mode == STMMAC_CHAIN_MODE) {
1654 		if (priv->extend_desc)
1655 			stmmac_mode_init(priv, rx_q->dma_erx,
1656 					 rx_q->dma_rx_phy,
1657 					 priv->dma_rx_size, 1);
1658 		else
1659 			stmmac_mode_init(priv, rx_q->dma_rx,
1660 					 rx_q->dma_rx_phy,
1661 					 priv->dma_rx_size, 0);
1662 	}
1663 
1664 	return 0;
1665 }
1666 
1667 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1668 {
1669 	struct stmmac_priv *priv = netdev_priv(dev);
1670 	u32 rx_count = priv->plat->rx_queues_to_use;
1671 	int queue;
1672 	int ret;
1673 
1674 	/* RX INITIALIZATION */
1675 	netif_dbg(priv, probe, priv->dev,
1676 		  "SKB addresses:\nskb\t\tskb data\tdma data\n");
1677 
1678 	for (queue = 0; queue < rx_count; queue++) {
1679 		ret = __init_dma_rx_desc_rings(priv, queue, flags);
1680 		if (ret)
1681 			goto err_init_rx_buffers;
1682 	}
1683 
1684 	return 0;
1685 
1686 err_init_rx_buffers:
1687 	while (queue >= 0) {
1688 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1689 
1690 		if (rx_q->xsk_pool)
1691 			dma_free_rx_xskbufs(priv, queue);
1692 		else
1693 			dma_free_rx_skbufs(priv, queue);
1694 
1695 		rx_q->buf_alloc_num = 0;
1696 		rx_q->xsk_pool = NULL;
1697 
1698 		queue--;
1699 	}
1700 
1701 	return ret;
1702 }
1703 
1704 /**
1705  * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue)
1706  * @priv: driver private structure
1707  * @queue : TX queue index
1708  * Description: this function initializes the DMA TX descriptors
1709  * and allocates the socket buffers. It supports the chained and ring
1710  * modes.
1711  */
1712 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, u32 queue)
1713 {
1714 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1715 	int i;
1716 
1717 	netif_dbg(priv, probe, priv->dev,
1718 		  "(%s) dma_tx_phy=0x%08x\n", __func__,
1719 		  (u32)tx_q->dma_tx_phy);
1720 
1721 	/* Setup the chained descriptor addresses */
1722 	if (priv->mode == STMMAC_CHAIN_MODE) {
1723 		if (priv->extend_desc)
1724 			stmmac_mode_init(priv, tx_q->dma_etx,
1725 					 tx_q->dma_tx_phy,
1726 					 priv->dma_tx_size, 1);
1727 		else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
1728 			stmmac_mode_init(priv, tx_q->dma_tx,
1729 					 tx_q->dma_tx_phy,
1730 					 priv->dma_tx_size, 0);
1731 	}
1732 
1733 	tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
1734 
1735 	for (i = 0; i < priv->dma_tx_size; i++) {
1736 		struct dma_desc *p;
1737 
1738 		if (priv->extend_desc)
1739 			p = &((tx_q->dma_etx + i)->basic);
1740 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1741 			p = &((tx_q->dma_entx + i)->basic);
1742 		else
1743 			p = tx_q->dma_tx + i;
1744 
1745 		stmmac_clear_desc(priv, p);
1746 
1747 		tx_q->tx_skbuff_dma[i].buf = 0;
1748 		tx_q->tx_skbuff_dma[i].map_as_page = false;
1749 		tx_q->tx_skbuff_dma[i].len = 0;
1750 		tx_q->tx_skbuff_dma[i].last_segment = false;
1751 		tx_q->tx_skbuff[i] = NULL;
1752 	}
1753 
1754 	tx_q->dirty_tx = 0;
1755 	tx_q->cur_tx = 0;
1756 	tx_q->mss = 0;
1757 
1758 	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1759 
1760 	return 0;
1761 }
1762 
1763 static int init_dma_tx_desc_rings(struct net_device *dev)
1764 {
1765 	struct stmmac_priv *priv = netdev_priv(dev);
1766 	u32 tx_queue_cnt;
1767 	u32 queue;
1768 
1769 	tx_queue_cnt = priv->plat->tx_queues_to_use;
1770 
1771 	for (queue = 0; queue < tx_queue_cnt; queue++)
1772 		__init_dma_tx_desc_rings(priv, queue);
1773 
1774 	return 0;
1775 }
1776 
1777 /**
1778  * init_dma_desc_rings - init the RX/TX descriptor rings
1779  * @dev: net device structure
1780  * @flags: gfp flag.
1781  * Description: this function initializes the DMA RX/TX descriptors
1782  * and allocates the socket buffers. It supports the chained and ring
1783  * modes.
1784  */
1785 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1786 {
1787 	struct stmmac_priv *priv = netdev_priv(dev);
1788 	int ret;
1789 
1790 	ret = init_dma_rx_desc_rings(dev, flags);
1791 	if (ret)
1792 		return ret;
1793 
1794 	ret = init_dma_tx_desc_rings(dev);
1795 
1796 	stmmac_clear_descriptors(priv);
1797 
1798 	if (netif_msg_hw(priv))
1799 		stmmac_display_rings(priv);
1800 
1801 	return ret;
1802 }
1803 
1804 /**
1805  * dma_free_tx_skbufs - free TX dma buffers
1806  * @priv: private structure
1807  * @queue: TX queue index
1808  */
1809 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1810 {
1811 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1812 	int i;
1813 
1814 	tx_q->xsk_frames_done = 0;
1815 
1816 	for (i = 0; i < priv->dma_tx_size; i++)
1817 		stmmac_free_tx_buffer(priv, queue, i);
1818 
1819 	if (tx_q->xsk_pool && tx_q->xsk_frames_done) {
1820 		xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done);
1821 		tx_q->xsk_frames_done = 0;
1822 		tx_q->xsk_pool = NULL;
1823 	}
1824 }
1825 
1826 /**
1827  * stmmac_free_tx_skbufs - free TX skb buffers
1828  * @priv: private structure
1829  */
1830 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)
1831 {
1832 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1833 	u32 queue;
1834 
1835 	for (queue = 0; queue < tx_queue_cnt; queue++)
1836 		dma_free_tx_skbufs(priv, queue);
1837 }
1838 
1839 /**
1840  * __free_dma_rx_desc_resources - free RX dma desc resources (per queue)
1841  * @priv: private structure
1842  * @queue: RX queue index
1843  */
1844 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue)
1845 {
1846 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1847 
1848 	/* Release the DMA RX socket buffers */
1849 	if (rx_q->xsk_pool)
1850 		dma_free_rx_xskbufs(priv, queue);
1851 	else
1852 		dma_free_rx_skbufs(priv, queue);
1853 
1854 	rx_q->buf_alloc_num = 0;
1855 	rx_q->xsk_pool = NULL;
1856 
1857 	/* Free DMA regions of consistent memory previously allocated */
1858 	if (!priv->extend_desc)
1859 		dma_free_coherent(priv->device, priv->dma_rx_size *
1860 				  sizeof(struct dma_desc),
1861 				  rx_q->dma_rx, rx_q->dma_rx_phy);
1862 	else
1863 		dma_free_coherent(priv->device, priv->dma_rx_size *
1864 				  sizeof(struct dma_extended_desc),
1865 				  rx_q->dma_erx, rx_q->dma_rx_phy);
1866 
1867 	if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq))
1868 		xdp_rxq_info_unreg(&rx_q->xdp_rxq);
1869 
1870 	kfree(rx_q->buf_pool);
1871 	if (rx_q->page_pool)
1872 		page_pool_destroy(rx_q->page_pool);
1873 }
1874 
1875 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1876 {
1877 	u32 rx_count = priv->plat->rx_queues_to_use;
1878 	u32 queue;
1879 
1880 	/* Free RX queue resources */
1881 	for (queue = 0; queue < rx_count; queue++)
1882 		__free_dma_rx_desc_resources(priv, queue);
1883 }
1884 
1885 /**
1886  * __free_dma_tx_desc_resources - free TX dma desc resources (per queue)
1887  * @priv: private structure
1888  * @queue: TX queue index
1889  */
1890 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue)
1891 {
1892 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1893 	size_t size;
1894 	void *addr;
1895 
1896 	/* Release the DMA TX socket buffers */
1897 	dma_free_tx_skbufs(priv, queue);
1898 
1899 	if (priv->extend_desc) {
1900 		size = sizeof(struct dma_extended_desc);
1901 		addr = tx_q->dma_etx;
1902 	} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
1903 		size = sizeof(struct dma_edesc);
1904 		addr = tx_q->dma_entx;
1905 	} else {
1906 		size = sizeof(struct dma_desc);
1907 		addr = tx_q->dma_tx;
1908 	}
1909 
1910 	size *= priv->dma_tx_size;
1911 
1912 	dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);
1913 
1914 	kfree(tx_q->tx_skbuff_dma);
1915 	kfree(tx_q->tx_skbuff);
1916 }
1917 
1918 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1919 {
1920 	u32 tx_count = priv->plat->tx_queues_to_use;
1921 	u32 queue;
1922 
1923 	/* Free TX queue resources */
1924 	for (queue = 0; queue < tx_count; queue++)
1925 		__free_dma_tx_desc_resources(priv, queue);
1926 }
1927 
1928 /**
1929  * __alloc_dma_rx_desc_resources - alloc RX resources (per queue).
1930  * @priv: private structure
1931  * @queue: RX queue index
1932  * Description: according to which descriptor can be used (extend or basic)
1933  * this function allocates the resources for TX and RX paths. In case of
1934  * reception, for example, it pre-allocated the RX socket buffer in order to
1935  * allow zero-copy mechanism.
1936  */
1937 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue)
1938 {
1939 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1940 	struct stmmac_channel *ch = &priv->channel[queue];
1941 	bool xdp_prog = stmmac_xdp_is_enabled(priv);
1942 	struct page_pool_params pp_params = { 0 };
1943 	unsigned int num_pages;
1944 	unsigned int napi_id;
1945 	int ret;
1946 
1947 	rx_q->queue_index = queue;
1948 	rx_q->priv_data = priv;
1949 
1950 	pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
1951 	pp_params.pool_size = priv->dma_rx_size;
1952 	num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE);
1953 	pp_params.order = ilog2(num_pages);
1954 	pp_params.nid = dev_to_node(priv->device);
1955 	pp_params.dev = priv->device;
1956 	pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
1957 	pp_params.offset = stmmac_rx_offset(priv);
1958 	pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages);
1959 
1960 	rx_q->page_pool = page_pool_create(&pp_params);
1961 	if (IS_ERR(rx_q->page_pool)) {
1962 		ret = PTR_ERR(rx_q->page_pool);
1963 		rx_q->page_pool = NULL;
1964 		return ret;
1965 	}
1966 
1967 	rx_q->buf_pool = kcalloc(priv->dma_rx_size,
1968 				 sizeof(*rx_q->buf_pool),
1969 				 GFP_KERNEL);
1970 	if (!rx_q->buf_pool)
1971 		return -ENOMEM;
1972 
1973 	if (priv->extend_desc) {
1974 		rx_q->dma_erx = dma_alloc_coherent(priv->device,
1975 						   priv->dma_rx_size *
1976 						   sizeof(struct dma_extended_desc),
1977 						   &rx_q->dma_rx_phy,
1978 						   GFP_KERNEL);
1979 		if (!rx_q->dma_erx)
1980 			return -ENOMEM;
1981 
1982 	} else {
1983 		rx_q->dma_rx = dma_alloc_coherent(priv->device,
1984 						  priv->dma_rx_size *
1985 						  sizeof(struct dma_desc),
1986 						  &rx_q->dma_rx_phy,
1987 						  GFP_KERNEL);
1988 		if (!rx_q->dma_rx)
1989 			return -ENOMEM;
1990 	}
1991 
1992 	if (stmmac_xdp_is_enabled(priv) &&
1993 	    test_bit(queue, priv->af_xdp_zc_qps))
1994 		napi_id = ch->rxtx_napi.napi_id;
1995 	else
1996 		napi_id = ch->rx_napi.napi_id;
1997 
1998 	ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev,
1999 			       rx_q->queue_index,
2000 			       napi_id);
2001 	if (ret) {
2002 		netdev_err(priv->dev, "Failed to register xdp rxq info\n");
2003 		return -EINVAL;
2004 	}
2005 
2006 	return 0;
2007 }
2008 
2009 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
2010 {
2011 	u32 rx_count = priv->plat->rx_queues_to_use;
2012 	u32 queue;
2013 	int ret;
2014 
2015 	/* RX queues buffers and DMA */
2016 	for (queue = 0; queue < rx_count; queue++) {
2017 		ret = __alloc_dma_rx_desc_resources(priv, queue);
2018 		if (ret)
2019 			goto err_dma;
2020 	}
2021 
2022 	return 0;
2023 
2024 err_dma:
2025 	free_dma_rx_desc_resources(priv);
2026 
2027 	return ret;
2028 }
2029 
2030 /**
2031  * __alloc_dma_tx_desc_resources - alloc TX resources (per queue).
2032  * @priv: private structure
2033  * @queue: TX queue index
2034  * Description: according to which descriptor can be used (extend or basic)
2035  * this function allocates the resources for TX and RX paths. In case of
2036  * reception, for example, it pre-allocated the RX socket buffer in order to
2037  * allow zero-copy mechanism.
2038  */
2039 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue)
2040 {
2041 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2042 	size_t size;
2043 	void *addr;
2044 
2045 	tx_q->queue_index = queue;
2046 	tx_q->priv_data = priv;
2047 
2048 	tx_q->tx_skbuff_dma = kcalloc(priv->dma_tx_size,
2049 				      sizeof(*tx_q->tx_skbuff_dma),
2050 				      GFP_KERNEL);
2051 	if (!tx_q->tx_skbuff_dma)
2052 		return -ENOMEM;
2053 
2054 	tx_q->tx_skbuff = kcalloc(priv->dma_tx_size,
2055 				  sizeof(struct sk_buff *),
2056 				  GFP_KERNEL);
2057 	if (!tx_q->tx_skbuff)
2058 		return -ENOMEM;
2059 
2060 	if (priv->extend_desc)
2061 		size = sizeof(struct dma_extended_desc);
2062 	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2063 		size = sizeof(struct dma_edesc);
2064 	else
2065 		size = sizeof(struct dma_desc);
2066 
2067 	size *= priv->dma_tx_size;
2068 
2069 	addr = dma_alloc_coherent(priv->device, size,
2070 				  &tx_q->dma_tx_phy, GFP_KERNEL);
2071 	if (!addr)
2072 		return -ENOMEM;
2073 
2074 	if (priv->extend_desc)
2075 		tx_q->dma_etx = addr;
2076 	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2077 		tx_q->dma_entx = addr;
2078 	else
2079 		tx_q->dma_tx = addr;
2080 
2081 	return 0;
2082 }
2083 
2084 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
2085 {
2086 	u32 tx_count = priv->plat->tx_queues_to_use;
2087 	u32 queue;
2088 	int ret;
2089 
2090 	/* TX queues buffers and DMA */
2091 	for (queue = 0; queue < tx_count; queue++) {
2092 		ret = __alloc_dma_tx_desc_resources(priv, queue);
2093 		if (ret)
2094 			goto err_dma;
2095 	}
2096 
2097 	return 0;
2098 
2099 err_dma:
2100 	free_dma_tx_desc_resources(priv);
2101 	return ret;
2102 }
2103 
2104 /**
2105  * alloc_dma_desc_resources - alloc TX/RX resources.
2106  * @priv: private structure
2107  * Description: according to which descriptor can be used (extend or basic)
2108  * this function allocates the resources for TX and RX paths. In case of
2109  * reception, for example, it pre-allocated the RX socket buffer in order to
2110  * allow zero-copy mechanism.
2111  */
2112 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
2113 {
2114 	/* RX Allocation */
2115 	int ret = alloc_dma_rx_desc_resources(priv);
2116 
2117 	if (ret)
2118 		return ret;
2119 
2120 	ret = alloc_dma_tx_desc_resources(priv);
2121 
2122 	return ret;
2123 }
2124 
2125 /**
2126  * free_dma_desc_resources - free dma desc resources
2127  * @priv: private structure
2128  */
2129 static void free_dma_desc_resources(struct stmmac_priv *priv)
2130 {
2131 	/* Release the DMA TX socket buffers */
2132 	free_dma_tx_desc_resources(priv);
2133 
2134 	/* Release the DMA RX socket buffers later
2135 	 * to ensure all pending XDP_TX buffers are returned.
2136 	 */
2137 	free_dma_rx_desc_resources(priv);
2138 }
2139 
2140 /**
2141  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
2142  *  @priv: driver private structure
2143  *  Description: It is used for enabling the rx queues in the MAC
2144  */
2145 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
2146 {
2147 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2148 	int queue;
2149 	u8 mode;
2150 
2151 	for (queue = 0; queue < rx_queues_count; queue++) {
2152 		mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
2153 		stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
2154 	}
2155 }
2156 
2157 /**
2158  * stmmac_start_rx_dma - start RX DMA channel
2159  * @priv: driver private structure
2160  * @chan: RX channel index
2161  * Description:
2162  * This starts a RX DMA channel
2163  */
2164 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
2165 {
2166 	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
2167 	stmmac_start_rx(priv, priv->ioaddr, chan);
2168 }
2169 
2170 /**
2171  * stmmac_start_tx_dma - start TX DMA channel
2172  * @priv: driver private structure
2173  * @chan: TX channel index
2174  * Description:
2175  * This starts a TX DMA channel
2176  */
2177 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
2178 {
2179 	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
2180 	stmmac_start_tx(priv, priv->ioaddr, chan);
2181 }
2182 
2183 /**
2184  * stmmac_stop_rx_dma - stop RX DMA channel
2185  * @priv: driver private structure
2186  * @chan: RX channel index
2187  * Description:
2188  * This stops a RX DMA channel
2189  */
2190 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
2191 {
2192 	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
2193 	stmmac_stop_rx(priv, priv->ioaddr, chan);
2194 }
2195 
2196 /**
2197  * stmmac_stop_tx_dma - stop TX DMA channel
2198  * @priv: driver private structure
2199  * @chan: TX channel index
2200  * Description:
2201  * This stops a TX DMA channel
2202  */
2203 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
2204 {
2205 	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
2206 	stmmac_stop_tx(priv, priv->ioaddr, chan);
2207 }
2208 
2209 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)
2210 {
2211 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2212 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2213 	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2214 	u32 chan;
2215 
2216 	for (chan = 0; chan < dma_csr_ch; chan++) {
2217 		struct stmmac_channel *ch = &priv->channel[chan];
2218 		unsigned long flags;
2219 
2220 		spin_lock_irqsave(&ch->lock, flags);
2221 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
2222 		spin_unlock_irqrestore(&ch->lock, flags);
2223 	}
2224 }
2225 
2226 /**
2227  * stmmac_start_all_dma - start all RX and TX DMA channels
2228  * @priv: driver private structure
2229  * Description:
2230  * This starts all the RX and TX DMA channels
2231  */
2232 static void stmmac_start_all_dma(struct stmmac_priv *priv)
2233 {
2234 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2235 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2236 	u32 chan = 0;
2237 
2238 	for (chan = 0; chan < rx_channels_count; chan++)
2239 		stmmac_start_rx_dma(priv, chan);
2240 
2241 	for (chan = 0; chan < tx_channels_count; chan++)
2242 		stmmac_start_tx_dma(priv, chan);
2243 }
2244 
2245 /**
2246  * stmmac_stop_all_dma - stop all RX and TX DMA channels
2247  * @priv: driver private structure
2248  * Description:
2249  * This stops the RX and TX DMA channels
2250  */
2251 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
2252 {
2253 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2254 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2255 	u32 chan = 0;
2256 
2257 	for (chan = 0; chan < rx_channels_count; chan++)
2258 		stmmac_stop_rx_dma(priv, chan);
2259 
2260 	for (chan = 0; chan < tx_channels_count; chan++)
2261 		stmmac_stop_tx_dma(priv, chan);
2262 }
2263 
2264 /**
2265  *  stmmac_dma_operation_mode - HW DMA operation mode
2266  *  @priv: driver private structure
2267  *  Description: it is used for configuring the DMA operation mode register in
2268  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
2269  */
2270 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
2271 {
2272 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2273 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2274 	int rxfifosz = priv->plat->rx_fifo_size;
2275 	int txfifosz = priv->plat->tx_fifo_size;
2276 	u32 txmode = 0;
2277 	u32 rxmode = 0;
2278 	u32 chan = 0;
2279 	u8 qmode = 0;
2280 
2281 	if (rxfifosz == 0)
2282 		rxfifosz = priv->dma_cap.rx_fifo_size;
2283 	if (txfifosz == 0)
2284 		txfifosz = priv->dma_cap.tx_fifo_size;
2285 
2286 	/* Adjust for real per queue fifo size */
2287 	rxfifosz /= rx_channels_count;
2288 	txfifosz /= tx_channels_count;
2289 
2290 	if (priv->plat->force_thresh_dma_mode) {
2291 		txmode = tc;
2292 		rxmode = tc;
2293 	} else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
2294 		/*
2295 		 * In case of GMAC, SF mode can be enabled
2296 		 * to perform the TX COE in HW. This depends on:
2297 		 * 1) TX COE if actually supported
2298 		 * 2) There is no bugged Jumbo frame support
2299 		 *    that needs to not insert csum in the TDES.
2300 		 */
2301 		txmode = SF_DMA_MODE;
2302 		rxmode = SF_DMA_MODE;
2303 		priv->xstats.threshold = SF_DMA_MODE;
2304 	} else {
2305 		txmode = tc;
2306 		rxmode = SF_DMA_MODE;
2307 	}
2308 
2309 	/* configure all channels */
2310 	for (chan = 0; chan < rx_channels_count; chan++) {
2311 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
2312 		u32 buf_size;
2313 
2314 		qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2315 
2316 		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
2317 				rxfifosz, qmode);
2318 
2319 		if (rx_q->xsk_pool) {
2320 			buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
2321 			stmmac_set_dma_bfsize(priv, priv->ioaddr,
2322 					      buf_size,
2323 					      chan);
2324 		} else {
2325 			stmmac_set_dma_bfsize(priv, priv->ioaddr,
2326 					      priv->dma_buf_sz,
2327 					      chan);
2328 		}
2329 	}
2330 
2331 	for (chan = 0; chan < tx_channels_count; chan++) {
2332 		qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2333 
2334 		stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
2335 				txfifosz, qmode);
2336 	}
2337 }
2338 
2339 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
2340 {
2341 	struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue);
2342 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2343 	struct xsk_buff_pool *pool = tx_q->xsk_pool;
2344 	unsigned int entry = tx_q->cur_tx;
2345 	struct dma_desc *tx_desc = NULL;
2346 	struct xdp_desc xdp_desc;
2347 	bool work_done = true;
2348 
2349 	/* Avoids TX time-out as we are sharing with slow path */
2350 	txq_trans_cond_update(nq);
2351 
2352 	budget = min(budget, stmmac_tx_avail(priv, queue));
2353 
2354 	while (budget-- > 0) {
2355 		dma_addr_t dma_addr;
2356 		bool set_ic;
2357 
2358 		/* We are sharing with slow path and stop XSK TX desc submission when
2359 		 * available TX ring is less than threshold.
2360 		 */
2361 		if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) ||
2362 		    !netif_carrier_ok(priv->dev)) {
2363 			work_done = false;
2364 			break;
2365 		}
2366 
2367 		if (!xsk_tx_peek_desc(pool, &xdp_desc))
2368 			break;
2369 
2370 		if (likely(priv->extend_desc))
2371 			tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
2372 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2373 			tx_desc = &tx_q->dma_entx[entry].basic;
2374 		else
2375 			tx_desc = tx_q->dma_tx + entry;
2376 
2377 		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
2378 		xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len);
2379 
2380 		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX;
2381 
2382 		/* To return XDP buffer to XSK pool, we simple call
2383 		 * xsk_tx_completed(), so we don't need to fill up
2384 		 * 'buf' and 'xdpf'.
2385 		 */
2386 		tx_q->tx_skbuff_dma[entry].buf = 0;
2387 		tx_q->xdpf[entry] = NULL;
2388 
2389 		tx_q->tx_skbuff_dma[entry].map_as_page = false;
2390 		tx_q->tx_skbuff_dma[entry].len = xdp_desc.len;
2391 		tx_q->tx_skbuff_dma[entry].last_segment = true;
2392 		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
2393 
2394 		stmmac_set_desc_addr(priv, tx_desc, dma_addr);
2395 
2396 		tx_q->tx_count_frames++;
2397 
2398 		if (!priv->tx_coal_frames[queue])
2399 			set_ic = false;
2400 		else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)
2401 			set_ic = true;
2402 		else
2403 			set_ic = false;
2404 
2405 		if (set_ic) {
2406 			tx_q->tx_count_frames = 0;
2407 			stmmac_set_tx_ic(priv, tx_desc);
2408 			priv->xstats.tx_set_ic_bit++;
2409 		}
2410 
2411 		stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len,
2412 				       true, priv->mode, true, true,
2413 				       xdp_desc.len);
2414 
2415 		stmmac_enable_dma_transmission(priv, priv->ioaddr);
2416 
2417 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
2418 		entry = tx_q->cur_tx;
2419 	}
2420 
2421 	if (tx_desc) {
2422 		stmmac_flush_tx_descriptors(priv, queue);
2423 		xsk_tx_release(pool);
2424 	}
2425 
2426 	/* Return true if all of the 3 conditions are met
2427 	 *  a) TX Budget is still available
2428 	 *  b) work_done = true when XSK TX desc peek is empty (no more
2429 	 *     pending XSK TX for transmission)
2430 	 */
2431 	return !!budget && work_done;
2432 }
2433 
2434 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan)
2435 {
2436 	if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) {
2437 		tc += 64;
2438 
2439 		if (priv->plat->force_thresh_dma_mode)
2440 			stmmac_set_dma_operation_mode(priv, tc, tc, chan);
2441 		else
2442 			stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE,
2443 						      chan);
2444 
2445 		priv->xstats.threshold = tc;
2446 	}
2447 }
2448 
2449 /**
2450  * stmmac_tx_clean - to manage the transmission completion
2451  * @priv: driver private structure
2452  * @budget: napi budget limiting this functions packet handling
2453  * @queue: TX queue index
2454  * Description: it reclaims the transmit resources after transmission completes.
2455  */
2456 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
2457 {
2458 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2459 	unsigned int bytes_compl = 0, pkts_compl = 0;
2460 	unsigned int entry, xmits = 0, count = 0;
2461 
2462 	__netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
2463 
2464 	priv->xstats.tx_clean++;
2465 
2466 	tx_q->xsk_frames_done = 0;
2467 
2468 	entry = tx_q->dirty_tx;
2469 
2470 	/* Try to clean all TX complete frame in 1 shot */
2471 	while ((entry != tx_q->cur_tx) && count < priv->dma_tx_size) {
2472 		struct xdp_frame *xdpf;
2473 		struct sk_buff *skb;
2474 		struct dma_desc *p;
2475 		int status;
2476 
2477 		if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX ||
2478 		    tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
2479 			xdpf = tx_q->xdpf[entry];
2480 			skb = NULL;
2481 		} else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
2482 			xdpf = NULL;
2483 			skb = tx_q->tx_skbuff[entry];
2484 		} else {
2485 			xdpf = NULL;
2486 			skb = NULL;
2487 		}
2488 
2489 		if (priv->extend_desc)
2490 			p = (struct dma_desc *)(tx_q->dma_etx + entry);
2491 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2492 			p = &tx_q->dma_entx[entry].basic;
2493 		else
2494 			p = tx_q->dma_tx + entry;
2495 
2496 		status = stmmac_tx_status(priv, &priv->dev->stats,
2497 				&priv->xstats, p, priv->ioaddr);
2498 		/* Check if the descriptor is owned by the DMA */
2499 		if (unlikely(status & tx_dma_own))
2500 			break;
2501 
2502 		count++;
2503 
2504 		/* Make sure descriptor fields are read after reading
2505 		 * the own bit.
2506 		 */
2507 		dma_rmb();
2508 
2509 		/* Just consider the last segment and ...*/
2510 		if (likely(!(status & tx_not_ls))) {
2511 			/* ... verify the status error condition */
2512 			if (unlikely(status & tx_err)) {
2513 				priv->dev->stats.tx_errors++;
2514 				if (unlikely(status & tx_err_bump_tc))
2515 					stmmac_bump_dma_threshold(priv, queue);
2516 			} else {
2517 				priv->dev->stats.tx_packets++;
2518 				priv->xstats.tx_pkt_n++;
2519 				priv->xstats.txq_stats[queue].tx_pkt_n++;
2520 			}
2521 			if (skb)
2522 				stmmac_get_tx_hwtstamp(priv, p, skb);
2523 		}
2524 
2525 		if (likely(tx_q->tx_skbuff_dma[entry].buf &&
2526 			   tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) {
2527 			if (tx_q->tx_skbuff_dma[entry].map_as_page)
2528 				dma_unmap_page(priv->device,
2529 					       tx_q->tx_skbuff_dma[entry].buf,
2530 					       tx_q->tx_skbuff_dma[entry].len,
2531 					       DMA_TO_DEVICE);
2532 			else
2533 				dma_unmap_single(priv->device,
2534 						 tx_q->tx_skbuff_dma[entry].buf,
2535 						 tx_q->tx_skbuff_dma[entry].len,
2536 						 DMA_TO_DEVICE);
2537 			tx_q->tx_skbuff_dma[entry].buf = 0;
2538 			tx_q->tx_skbuff_dma[entry].len = 0;
2539 			tx_q->tx_skbuff_dma[entry].map_as_page = false;
2540 		}
2541 
2542 		stmmac_clean_desc3(priv, tx_q, p);
2543 
2544 		tx_q->tx_skbuff_dma[entry].last_segment = false;
2545 		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
2546 
2547 		if (xdpf &&
2548 		    tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) {
2549 			xdp_return_frame_rx_napi(xdpf);
2550 			tx_q->xdpf[entry] = NULL;
2551 		}
2552 
2553 		if (xdpf &&
2554 		    tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
2555 			xdp_return_frame(xdpf);
2556 			tx_q->xdpf[entry] = NULL;
2557 		}
2558 
2559 		if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX)
2560 			tx_q->xsk_frames_done++;
2561 
2562 		if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
2563 			if (likely(skb)) {
2564 				pkts_compl++;
2565 				bytes_compl += skb->len;
2566 				dev_consume_skb_any(skb);
2567 				tx_q->tx_skbuff[entry] = NULL;
2568 			}
2569 		}
2570 
2571 		stmmac_release_tx_desc(priv, p, priv->mode);
2572 
2573 		entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
2574 	}
2575 	tx_q->dirty_tx = entry;
2576 
2577 	netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
2578 				  pkts_compl, bytes_compl);
2579 
2580 	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
2581 								queue))) &&
2582 	    stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) {
2583 
2584 		netif_dbg(priv, tx_done, priv->dev,
2585 			  "%s: restart transmit\n", __func__);
2586 		netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
2587 	}
2588 
2589 	if (tx_q->xsk_pool) {
2590 		bool work_done;
2591 
2592 		if (tx_q->xsk_frames_done)
2593 			xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done);
2594 
2595 		if (xsk_uses_need_wakeup(tx_q->xsk_pool))
2596 			xsk_set_tx_need_wakeup(tx_q->xsk_pool);
2597 
2598 		/* For XSK TX, we try to send as many as possible.
2599 		 * If XSK work done (XSK TX desc empty and budget still
2600 		 * available), return "budget - 1" to reenable TX IRQ.
2601 		 * Else, return "budget" to make NAPI continue polling.
2602 		 */
2603 		work_done = stmmac_xdp_xmit_zc(priv, queue,
2604 					       STMMAC_XSK_TX_BUDGET_MAX);
2605 		if (work_done)
2606 			xmits = budget - 1;
2607 		else
2608 			xmits = budget;
2609 	}
2610 
2611 	if (priv->eee_enabled && !priv->tx_path_in_lpi_mode &&
2612 	    priv->eee_sw_timer_en) {
2613 		if (stmmac_enable_eee_mode(priv))
2614 			mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
2615 	}
2616 
2617 	/* We still have pending packets, let's call for a new scheduling */
2618 	if (tx_q->dirty_tx != tx_q->cur_tx)
2619 		hrtimer_start(&tx_q->txtimer,
2620 			      STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]),
2621 			      HRTIMER_MODE_REL);
2622 
2623 	__netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
2624 
2625 	/* Combine decisions from TX clean and XSK TX */
2626 	return max(count, xmits);
2627 }
2628 
2629 /**
2630  * stmmac_tx_err - to manage the tx error
2631  * @priv: driver private structure
2632  * @chan: channel index
2633  * Description: it cleans the descriptors and restarts the transmission
2634  * in case of transmission errors.
2635  */
2636 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
2637 {
2638 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2639 
2640 	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
2641 
2642 	stmmac_stop_tx_dma(priv, chan);
2643 	dma_free_tx_skbufs(priv, chan);
2644 	stmmac_clear_tx_descriptors(priv, chan);
2645 	tx_q->dirty_tx = 0;
2646 	tx_q->cur_tx = 0;
2647 	tx_q->mss = 0;
2648 	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
2649 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2650 			    tx_q->dma_tx_phy, chan);
2651 	stmmac_start_tx_dma(priv, chan);
2652 
2653 	priv->dev->stats.tx_errors++;
2654 	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
2655 }
2656 
2657 /**
2658  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2659  *  @priv: driver private structure
2660  *  @txmode: TX operating mode
2661  *  @rxmode: RX operating mode
2662  *  @chan: channel index
2663  *  Description: it is used for configuring of the DMA operation mode in
2664  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2665  *  mode.
2666  */
2667 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
2668 					  u32 rxmode, u32 chan)
2669 {
2670 	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2671 	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2672 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2673 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2674 	int rxfifosz = priv->plat->rx_fifo_size;
2675 	int txfifosz = priv->plat->tx_fifo_size;
2676 
2677 	if (rxfifosz == 0)
2678 		rxfifosz = priv->dma_cap.rx_fifo_size;
2679 	if (txfifosz == 0)
2680 		txfifosz = priv->dma_cap.tx_fifo_size;
2681 
2682 	/* Adjust for real per queue fifo size */
2683 	rxfifosz /= rx_channels_count;
2684 	txfifosz /= tx_channels_count;
2685 
2686 	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2687 	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
2688 }
2689 
2690 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2691 {
2692 	int ret;
2693 
2694 	ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2695 			priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2696 	if (ret && (ret != -EINVAL)) {
2697 		stmmac_global_err(priv);
2698 		return true;
2699 	}
2700 
2701 	return false;
2702 }
2703 
2704 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir)
2705 {
2706 	int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2707 						 &priv->xstats, chan, dir);
2708 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
2709 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2710 	struct stmmac_channel *ch = &priv->channel[chan];
2711 	struct napi_struct *rx_napi;
2712 	struct napi_struct *tx_napi;
2713 	unsigned long flags;
2714 
2715 	rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi;
2716 	tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
2717 
2718 	if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
2719 		if (napi_schedule_prep(rx_napi)) {
2720 			spin_lock_irqsave(&ch->lock, flags);
2721 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
2722 			spin_unlock_irqrestore(&ch->lock, flags);
2723 			__napi_schedule(rx_napi);
2724 		}
2725 	}
2726 
2727 	if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {
2728 		if (napi_schedule_prep(tx_napi)) {
2729 			spin_lock_irqsave(&ch->lock, flags);
2730 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
2731 			spin_unlock_irqrestore(&ch->lock, flags);
2732 			__napi_schedule(tx_napi);
2733 		}
2734 	}
2735 
2736 	return status;
2737 }
2738 
2739 /**
2740  * stmmac_dma_interrupt - DMA ISR
2741  * @priv: driver private structure
2742  * Description: this is the DMA ISR. It is called by the main ISR.
2743  * It calls the dwmac dma routine and schedule poll method in case of some
2744  * work can be done.
2745  */
2746 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2747 {
2748 	u32 tx_channel_count = priv->plat->tx_queues_to_use;
2749 	u32 rx_channel_count = priv->plat->rx_queues_to_use;
2750 	u32 channels_to_check = tx_channel_count > rx_channel_count ?
2751 				tx_channel_count : rx_channel_count;
2752 	u32 chan;
2753 	int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
2754 
2755 	/* Make sure we never check beyond our status buffer. */
2756 	if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
2757 		channels_to_check = ARRAY_SIZE(status);
2758 
2759 	for (chan = 0; chan < channels_to_check; chan++)
2760 		status[chan] = stmmac_napi_check(priv, chan,
2761 						 DMA_DIR_RXTX);
2762 
2763 	for (chan = 0; chan < tx_channel_count; chan++) {
2764 		if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
2765 			/* Try to bump up the dma threshold on this failure */
2766 			stmmac_bump_dma_threshold(priv, chan);
2767 		} else if (unlikely(status[chan] == tx_hard_error)) {
2768 			stmmac_tx_err(priv, chan);
2769 		}
2770 	}
2771 }
2772 
2773 /**
2774  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2775  * @priv: driver private structure
2776  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2777  */
2778 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2779 {
2780 	unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2781 			    MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2782 
2783 	stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
2784 
2785 	if (priv->dma_cap.rmon) {
2786 		stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
2787 		memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2788 	} else
2789 		netdev_info(priv->dev, "No MAC Management Counters available\n");
2790 }
2791 
2792 /**
2793  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2794  * @priv: driver private structure
2795  * Description:
2796  *  new GMAC chip generations have a new register to indicate the
2797  *  presence of the optional feature/functions.
2798  *  This can be also used to override the value passed through the
2799  *  platform and necessary for old MAC10/100 and GMAC chips.
2800  */
2801 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2802 {
2803 	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
2804 }
2805 
2806 /**
2807  * stmmac_check_ether_addr - check if the MAC addr is valid
2808  * @priv: driver private structure
2809  * Description:
2810  * it is to verify if the MAC address is valid, in case of failures it
2811  * generates a random MAC address
2812  */
2813 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2814 {
2815 	u8 addr[ETH_ALEN];
2816 
2817 	if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2818 		stmmac_get_umac_addr(priv, priv->hw, addr, 0);
2819 		if (is_valid_ether_addr(addr))
2820 			eth_hw_addr_set(priv->dev, addr);
2821 		else
2822 			eth_hw_addr_random(priv->dev);
2823 		dev_info(priv->device, "device MAC address %pM\n",
2824 			 priv->dev->dev_addr);
2825 	}
2826 }
2827 
2828 /**
2829  * stmmac_init_dma_engine - DMA init.
2830  * @priv: driver private structure
2831  * Description:
2832  * It inits the DMA invoking the specific MAC/GMAC callback.
2833  * Some DMA parameters can be passed from the platform;
2834  * in case of these are not passed a default is kept for the MAC or GMAC.
2835  */
2836 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2837 {
2838 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2839 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2840 	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2841 	struct stmmac_rx_queue *rx_q;
2842 	struct stmmac_tx_queue *tx_q;
2843 	u32 chan = 0;
2844 	int atds = 0;
2845 	int ret = 0;
2846 
2847 	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2848 		dev_err(priv->device, "Invalid DMA configuration\n");
2849 		return -EINVAL;
2850 	}
2851 
2852 	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2853 		atds = 1;
2854 
2855 	ret = stmmac_reset(priv, priv->ioaddr);
2856 	if (ret) {
2857 		dev_err(priv->device, "Failed to reset the dma\n");
2858 		return ret;
2859 	}
2860 
2861 	/* DMA Configuration */
2862 	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
2863 
2864 	if (priv->plat->axi)
2865 		stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2866 
2867 	/* DMA CSR Channel configuration */
2868 	for (chan = 0; chan < dma_csr_ch; chan++) {
2869 		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
2870 		stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
2871 	}
2872 
2873 	/* DMA RX Channel Configuration */
2874 	for (chan = 0; chan < rx_channels_count; chan++) {
2875 		rx_q = &priv->rx_queue[chan];
2876 
2877 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2878 				    rx_q->dma_rx_phy, chan);
2879 
2880 		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2881 				     (rx_q->buf_alloc_num *
2882 				      sizeof(struct dma_desc));
2883 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
2884 				       rx_q->rx_tail_addr, chan);
2885 	}
2886 
2887 	/* DMA TX Channel Configuration */
2888 	for (chan = 0; chan < tx_channels_count; chan++) {
2889 		tx_q = &priv->tx_queue[chan];
2890 
2891 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2892 				    tx_q->dma_tx_phy, chan);
2893 
2894 		tx_q->tx_tail_addr = tx_q->dma_tx_phy;
2895 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
2896 				       tx_q->tx_tail_addr, chan);
2897 	}
2898 
2899 	return ret;
2900 }
2901 
2902 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2903 {
2904 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2905 
2906 	hrtimer_start(&tx_q->txtimer,
2907 		      STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]),
2908 		      HRTIMER_MODE_REL);
2909 }
2910 
2911 /**
2912  * stmmac_tx_timer - mitigation sw timer for tx.
2913  * @t: data pointer
2914  * Description:
2915  * This is the timer handler to directly invoke the stmmac_tx_clean.
2916  */
2917 static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t)
2918 {
2919 	struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer);
2920 	struct stmmac_priv *priv = tx_q->priv_data;
2921 	struct stmmac_channel *ch;
2922 	struct napi_struct *napi;
2923 
2924 	ch = &priv->channel[tx_q->queue_index];
2925 	napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
2926 
2927 	if (likely(napi_schedule_prep(napi))) {
2928 		unsigned long flags;
2929 
2930 		spin_lock_irqsave(&ch->lock, flags);
2931 		stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);
2932 		spin_unlock_irqrestore(&ch->lock, flags);
2933 		__napi_schedule(napi);
2934 	}
2935 
2936 	return HRTIMER_NORESTART;
2937 }
2938 
2939 /**
2940  * stmmac_init_coalesce - init mitigation options.
2941  * @priv: driver private structure
2942  * Description:
2943  * This inits the coalesce parameters: i.e. timer rate,
2944  * timer handler and default threshold used for enabling the
2945  * interrupt on completion bit.
2946  */
2947 static void stmmac_init_coalesce(struct stmmac_priv *priv)
2948 {
2949 	u32 tx_channel_count = priv->plat->tx_queues_to_use;
2950 	u32 rx_channel_count = priv->plat->rx_queues_to_use;
2951 	u32 chan;
2952 
2953 	for (chan = 0; chan < tx_channel_count; chan++) {
2954 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2955 
2956 		priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES;
2957 		priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER;
2958 
2959 		hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2960 		tx_q->txtimer.function = stmmac_tx_timer;
2961 	}
2962 
2963 	for (chan = 0; chan < rx_channel_count; chan++)
2964 		priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES;
2965 }
2966 
2967 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2968 {
2969 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2970 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2971 	u32 chan;
2972 
2973 	/* set TX ring length */
2974 	for (chan = 0; chan < tx_channels_count; chan++)
2975 		stmmac_set_tx_ring_len(priv, priv->ioaddr,
2976 				       (priv->dma_tx_size - 1), chan);
2977 
2978 	/* set RX ring length */
2979 	for (chan = 0; chan < rx_channels_count; chan++)
2980 		stmmac_set_rx_ring_len(priv, priv->ioaddr,
2981 				       (priv->dma_rx_size - 1), chan);
2982 }
2983 
2984 /**
2985  *  stmmac_set_tx_queue_weight - Set TX queue weight
2986  *  @priv: driver private structure
2987  *  Description: It is used for setting TX queues weight
2988  */
2989 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2990 {
2991 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2992 	u32 weight;
2993 	u32 queue;
2994 
2995 	for (queue = 0; queue < tx_queues_count; queue++) {
2996 		weight = priv->plat->tx_queues_cfg[queue].weight;
2997 		stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
2998 	}
2999 }
3000 
3001 /**
3002  *  stmmac_configure_cbs - Configure CBS in TX queue
3003  *  @priv: driver private structure
3004  *  Description: It is used for configuring CBS in AVB TX queues
3005  */
3006 static void stmmac_configure_cbs(struct stmmac_priv *priv)
3007 {
3008 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
3009 	u32 mode_to_use;
3010 	u32 queue;
3011 
3012 	/* queue 0 is reserved for legacy traffic */
3013 	for (queue = 1; queue < tx_queues_count; queue++) {
3014 		mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
3015 		if (mode_to_use == MTL_QUEUE_DCB)
3016 			continue;
3017 
3018 		stmmac_config_cbs(priv, priv->hw,
3019 				priv->plat->tx_queues_cfg[queue].send_slope,
3020 				priv->plat->tx_queues_cfg[queue].idle_slope,
3021 				priv->plat->tx_queues_cfg[queue].high_credit,
3022 				priv->plat->tx_queues_cfg[queue].low_credit,
3023 				queue);
3024 	}
3025 }
3026 
3027 /**
3028  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
3029  *  @priv: driver private structure
3030  *  Description: It is used for mapping RX queues to RX dma channels
3031  */
3032 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
3033 {
3034 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
3035 	u32 queue;
3036 	u32 chan;
3037 
3038 	for (queue = 0; queue < rx_queues_count; queue++) {
3039 		chan = priv->plat->rx_queues_cfg[queue].chan;
3040 		stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
3041 	}
3042 }
3043 
3044 /**
3045  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
3046  *  @priv: driver private structure
3047  *  Description: It is used for configuring the RX Queue Priority
3048  */
3049 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
3050 {
3051 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
3052 	u32 queue;
3053 	u32 prio;
3054 
3055 	for (queue = 0; queue < rx_queues_count; queue++) {
3056 		if (!priv->plat->rx_queues_cfg[queue].use_prio)
3057 			continue;
3058 
3059 		prio = priv->plat->rx_queues_cfg[queue].prio;
3060 		stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
3061 	}
3062 }
3063 
3064 /**
3065  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
3066  *  @priv: driver private structure
3067  *  Description: It is used for configuring the TX Queue Priority
3068  */
3069 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
3070 {
3071 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
3072 	u32 queue;
3073 	u32 prio;
3074 
3075 	for (queue = 0; queue < tx_queues_count; queue++) {
3076 		if (!priv->plat->tx_queues_cfg[queue].use_prio)
3077 			continue;
3078 
3079 		prio = priv->plat->tx_queues_cfg[queue].prio;
3080 		stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
3081 	}
3082 }
3083 
3084 /**
3085  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
3086  *  @priv: driver private structure
3087  *  Description: It is used for configuring the RX queue routing
3088  */
3089 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
3090 {
3091 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
3092 	u32 queue;
3093 	u8 packet;
3094 
3095 	for (queue = 0; queue < rx_queues_count; queue++) {
3096 		/* no specific packet type routing specified for the queue */
3097 		if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
3098 			continue;
3099 
3100 		packet = priv->plat->rx_queues_cfg[queue].pkt_route;
3101 		stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
3102 	}
3103 }
3104 
3105 static void stmmac_mac_config_rss(struct stmmac_priv *priv)
3106 {
3107 	if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
3108 		priv->rss.enable = false;
3109 		return;
3110 	}
3111 
3112 	if (priv->dev->features & NETIF_F_RXHASH)
3113 		priv->rss.enable = true;
3114 	else
3115 		priv->rss.enable = false;
3116 
3117 	stmmac_rss_configure(priv, priv->hw, &priv->rss,
3118 			     priv->plat->rx_queues_to_use);
3119 }
3120 
3121 /**
3122  *  stmmac_mtl_configuration - Configure MTL
3123  *  @priv: driver private structure
3124  *  Description: It is used for configurring MTL
3125  */
3126 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
3127 {
3128 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
3129 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
3130 
3131 	if (tx_queues_count > 1)
3132 		stmmac_set_tx_queue_weight(priv);
3133 
3134 	/* Configure MTL RX algorithms */
3135 	if (rx_queues_count > 1)
3136 		stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
3137 				priv->plat->rx_sched_algorithm);
3138 
3139 	/* Configure MTL TX algorithms */
3140 	if (tx_queues_count > 1)
3141 		stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
3142 				priv->plat->tx_sched_algorithm);
3143 
3144 	/* Configure CBS in AVB TX queues */
3145 	if (tx_queues_count > 1)
3146 		stmmac_configure_cbs(priv);
3147 
3148 	/* Map RX MTL to DMA channels */
3149 	stmmac_rx_queue_dma_chan_map(priv);
3150 
3151 	/* Enable MAC RX Queues */
3152 	stmmac_mac_enable_rx_queues(priv);
3153 
3154 	/* Set RX priorities */
3155 	if (rx_queues_count > 1)
3156 		stmmac_mac_config_rx_queues_prio(priv);
3157 
3158 	/* Set TX priorities */
3159 	if (tx_queues_count > 1)
3160 		stmmac_mac_config_tx_queues_prio(priv);
3161 
3162 	/* Set RX routing */
3163 	if (rx_queues_count > 1)
3164 		stmmac_mac_config_rx_queues_routing(priv);
3165 
3166 	/* Receive Side Scaling */
3167 	if (rx_queues_count > 1)
3168 		stmmac_mac_config_rss(priv);
3169 }
3170 
3171 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
3172 {
3173 	if (priv->dma_cap.asp) {
3174 		netdev_info(priv->dev, "Enabling Safety Features\n");
3175 		stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp,
3176 					  priv->plat->safety_feat_cfg);
3177 	} else {
3178 		netdev_info(priv->dev, "No Safety Features support found\n");
3179 	}
3180 }
3181 
3182 static int stmmac_fpe_start_wq(struct stmmac_priv *priv)
3183 {
3184 	char *name;
3185 
3186 	clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state);
3187 	clear_bit(__FPE_REMOVING,  &priv->fpe_task_state);
3188 
3189 	name = priv->wq_name;
3190 	sprintf(name, "%s-fpe", priv->dev->name);
3191 
3192 	priv->fpe_wq = create_singlethread_workqueue(name);
3193 	if (!priv->fpe_wq) {
3194 		netdev_err(priv->dev, "%s: Failed to create workqueue\n", name);
3195 
3196 		return -ENOMEM;
3197 	}
3198 	netdev_info(priv->dev, "FPE workqueue start");
3199 
3200 	return 0;
3201 }
3202 
3203 /**
3204  * stmmac_hw_setup - setup mac in a usable state.
3205  *  @dev : pointer to the device structure.
3206  *  @ptp_register: register PTP if set
3207  *  Description:
3208  *  this is the main function to setup the HW in a usable state because the
3209  *  dma engine is reset, the core registers are configured (e.g. AXI,
3210  *  Checksum features, timers). The DMA is ready to start receiving and
3211  *  transmitting.
3212  *  Return value:
3213  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3214  *  file on failure.
3215  */
3216 static int stmmac_hw_setup(struct net_device *dev, bool ptp_register)
3217 {
3218 	struct stmmac_priv *priv = netdev_priv(dev);
3219 	u32 rx_cnt = priv->plat->rx_queues_to_use;
3220 	u32 tx_cnt = priv->plat->tx_queues_to_use;
3221 	bool sph_en;
3222 	u32 chan;
3223 	int ret;
3224 
3225 	/* DMA initialization and SW reset */
3226 	ret = stmmac_init_dma_engine(priv);
3227 	if (ret < 0) {
3228 		netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
3229 			   __func__);
3230 		return ret;
3231 	}
3232 
3233 	/* Copy the MAC addr into the HW  */
3234 	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
3235 
3236 	/* PS and related bits will be programmed according to the speed */
3237 	if (priv->hw->pcs) {
3238 		int speed = priv->plat->mac_port_sel_speed;
3239 
3240 		if ((speed == SPEED_10) || (speed == SPEED_100) ||
3241 		    (speed == SPEED_1000)) {
3242 			priv->hw->ps = speed;
3243 		} else {
3244 			dev_warn(priv->device, "invalid port speed\n");
3245 			priv->hw->ps = 0;
3246 		}
3247 	}
3248 
3249 	/* Initialize the MAC Core */
3250 	stmmac_core_init(priv, priv->hw, dev);
3251 
3252 	/* Initialize MTL*/
3253 	stmmac_mtl_configuration(priv);
3254 
3255 	/* Initialize Safety Features */
3256 	stmmac_safety_feat_configuration(priv);
3257 
3258 	ret = stmmac_rx_ipc(priv, priv->hw);
3259 	if (!ret) {
3260 		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
3261 		priv->plat->rx_coe = STMMAC_RX_COE_NONE;
3262 		priv->hw->rx_csum = 0;
3263 	}
3264 
3265 	/* Enable the MAC Rx/Tx */
3266 	stmmac_mac_set(priv, priv->ioaddr, true);
3267 
3268 	/* Set the HW DMA mode and the COE */
3269 	stmmac_dma_operation_mode(priv);
3270 
3271 	stmmac_mmc_setup(priv);
3272 
3273 	ret = stmmac_init_ptp(priv);
3274 	if (ret == -EOPNOTSUPP)
3275 		netdev_info(priv->dev, "PTP not supported by HW\n");
3276 	else if (ret)
3277 		netdev_warn(priv->dev, "PTP init failed\n");
3278 	else if (ptp_register)
3279 		stmmac_ptp_register(priv);
3280 
3281 	priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS;
3282 
3283 	/* Convert the timer from msec to usec */
3284 	if (!priv->tx_lpi_timer)
3285 		priv->tx_lpi_timer = eee_timer * 1000;
3286 
3287 	if (priv->use_riwt) {
3288 		u32 queue;
3289 
3290 		for (queue = 0; queue < rx_cnt; queue++) {
3291 			if (!priv->rx_riwt[queue])
3292 				priv->rx_riwt[queue] = DEF_DMA_RIWT;
3293 
3294 			stmmac_rx_watchdog(priv, priv->ioaddr,
3295 					   priv->rx_riwt[queue], queue);
3296 		}
3297 	}
3298 
3299 	if (priv->hw->pcs)
3300 		stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
3301 
3302 	/* set TX and RX rings length */
3303 	stmmac_set_rings_length(priv);
3304 
3305 	/* Enable TSO */
3306 	if (priv->tso) {
3307 		for (chan = 0; chan < tx_cnt; chan++) {
3308 			struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
3309 
3310 			/* TSO and TBS cannot co-exist */
3311 			if (tx_q->tbs & STMMAC_TBS_AVAIL)
3312 				continue;
3313 
3314 			stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
3315 		}
3316 	}
3317 
3318 	/* Enable Split Header */
3319 	sph_en = (priv->hw->rx_csum > 0) && priv->sph;
3320 	for (chan = 0; chan < rx_cnt; chan++)
3321 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
3322 
3323 
3324 	/* VLAN Tag Insertion */
3325 	if (priv->dma_cap.vlins)
3326 		stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
3327 
3328 	/* TBS */
3329 	for (chan = 0; chan < tx_cnt; chan++) {
3330 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
3331 		int enable = tx_q->tbs & STMMAC_TBS_AVAIL;
3332 
3333 		stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);
3334 	}
3335 
3336 	/* Configure real RX and TX queues */
3337 	netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use);
3338 	netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use);
3339 
3340 	/* Start the ball rolling... */
3341 	stmmac_start_all_dma(priv);
3342 
3343 	if (priv->dma_cap.fpesel) {
3344 		stmmac_fpe_start_wq(priv);
3345 
3346 		if (priv->plat->fpe_cfg->enable)
3347 			stmmac_fpe_handshake(priv, true);
3348 	}
3349 
3350 	return 0;
3351 }
3352 
3353 static void stmmac_hw_teardown(struct net_device *dev)
3354 {
3355 	struct stmmac_priv *priv = netdev_priv(dev);
3356 
3357 	clk_disable_unprepare(priv->plat->clk_ptp_ref);
3358 }
3359 
3360 static void stmmac_free_irq(struct net_device *dev,
3361 			    enum request_irq_err irq_err, int irq_idx)
3362 {
3363 	struct stmmac_priv *priv = netdev_priv(dev);
3364 	int j;
3365 
3366 	switch (irq_err) {
3367 	case REQ_IRQ_ERR_ALL:
3368 		irq_idx = priv->plat->tx_queues_to_use;
3369 		fallthrough;
3370 	case REQ_IRQ_ERR_TX:
3371 		for (j = irq_idx - 1; j >= 0; j--) {
3372 			if (priv->tx_irq[j] > 0) {
3373 				irq_set_affinity_hint(priv->tx_irq[j], NULL);
3374 				free_irq(priv->tx_irq[j], &priv->tx_queue[j]);
3375 			}
3376 		}
3377 		irq_idx = priv->plat->rx_queues_to_use;
3378 		fallthrough;
3379 	case REQ_IRQ_ERR_RX:
3380 		for (j = irq_idx - 1; j >= 0; j--) {
3381 			if (priv->rx_irq[j] > 0) {
3382 				irq_set_affinity_hint(priv->rx_irq[j], NULL);
3383 				free_irq(priv->rx_irq[j], &priv->rx_queue[j]);
3384 			}
3385 		}
3386 
3387 		if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq)
3388 			free_irq(priv->sfty_ue_irq, dev);
3389 		fallthrough;
3390 	case REQ_IRQ_ERR_SFTY_UE:
3391 		if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq)
3392 			free_irq(priv->sfty_ce_irq, dev);
3393 		fallthrough;
3394 	case REQ_IRQ_ERR_SFTY_CE:
3395 		if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq)
3396 			free_irq(priv->lpi_irq, dev);
3397 		fallthrough;
3398 	case REQ_IRQ_ERR_LPI:
3399 		if (priv->wol_irq > 0 && priv->wol_irq != dev->irq)
3400 			free_irq(priv->wol_irq, dev);
3401 		fallthrough;
3402 	case REQ_IRQ_ERR_WOL:
3403 		free_irq(dev->irq, dev);
3404 		fallthrough;
3405 	case REQ_IRQ_ERR_MAC:
3406 	case REQ_IRQ_ERR_NO:
3407 		/* If MAC IRQ request error, no more IRQ to free */
3408 		break;
3409 	}
3410 }
3411 
3412 static int stmmac_request_irq_multi_msi(struct net_device *dev)
3413 {
3414 	struct stmmac_priv *priv = netdev_priv(dev);
3415 	enum request_irq_err irq_err;
3416 	cpumask_t cpu_mask;
3417 	int irq_idx = 0;
3418 	char *int_name;
3419 	int ret;
3420 	int i;
3421 
3422 	/* For common interrupt */
3423 	int_name = priv->int_name_mac;
3424 	sprintf(int_name, "%s:%s", dev->name, "mac");
3425 	ret = request_irq(dev->irq, stmmac_mac_interrupt,
3426 			  0, int_name, dev);
3427 	if (unlikely(ret < 0)) {
3428 		netdev_err(priv->dev,
3429 			   "%s: alloc mac MSI %d (error: %d)\n",
3430 			   __func__, dev->irq, ret);
3431 		irq_err = REQ_IRQ_ERR_MAC;
3432 		goto irq_error;
3433 	}
3434 
3435 	/* Request the Wake IRQ in case of another line
3436 	 * is used for WoL
3437 	 */
3438 	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
3439 		int_name = priv->int_name_wol;
3440 		sprintf(int_name, "%s:%s", dev->name, "wol");
3441 		ret = request_irq(priv->wol_irq,
3442 				  stmmac_mac_interrupt,
3443 				  0, int_name, dev);
3444 		if (unlikely(ret < 0)) {
3445 			netdev_err(priv->dev,
3446 				   "%s: alloc wol MSI %d (error: %d)\n",
3447 				   __func__, priv->wol_irq, ret);
3448 			irq_err = REQ_IRQ_ERR_WOL;
3449 			goto irq_error;
3450 		}
3451 	}
3452 
3453 	/* Request the LPI IRQ in case of another line
3454 	 * is used for LPI
3455 	 */
3456 	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {
3457 		int_name = priv->int_name_lpi;
3458 		sprintf(int_name, "%s:%s", dev->name, "lpi");
3459 		ret = request_irq(priv->lpi_irq,
3460 				  stmmac_mac_interrupt,
3461 				  0, int_name, dev);
3462 		if (unlikely(ret < 0)) {
3463 			netdev_err(priv->dev,
3464 				   "%s: alloc lpi MSI %d (error: %d)\n",
3465 				   __func__, priv->lpi_irq, ret);
3466 			irq_err = REQ_IRQ_ERR_LPI;
3467 			goto irq_error;
3468 		}
3469 	}
3470 
3471 	/* Request the Safety Feature Correctible Error line in
3472 	 * case of another line is used
3473 	 */
3474 	if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) {
3475 		int_name = priv->int_name_sfty_ce;
3476 		sprintf(int_name, "%s:%s", dev->name, "safety-ce");
3477 		ret = request_irq(priv->sfty_ce_irq,
3478 				  stmmac_safety_interrupt,
3479 				  0, int_name, dev);
3480 		if (unlikely(ret < 0)) {
3481 			netdev_err(priv->dev,
3482 				   "%s: alloc sfty ce MSI %d (error: %d)\n",
3483 				   __func__, priv->sfty_ce_irq, ret);
3484 			irq_err = REQ_IRQ_ERR_SFTY_CE;
3485 			goto irq_error;
3486 		}
3487 	}
3488 
3489 	/* Request the Safety Feature Uncorrectible Error line in
3490 	 * case of another line is used
3491 	 */
3492 	if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) {
3493 		int_name = priv->int_name_sfty_ue;
3494 		sprintf(int_name, "%s:%s", dev->name, "safety-ue");
3495 		ret = request_irq(priv->sfty_ue_irq,
3496 				  stmmac_safety_interrupt,
3497 				  0, int_name, dev);
3498 		if (unlikely(ret < 0)) {
3499 			netdev_err(priv->dev,
3500 				   "%s: alloc sfty ue MSI %d (error: %d)\n",
3501 				   __func__, priv->sfty_ue_irq, ret);
3502 			irq_err = REQ_IRQ_ERR_SFTY_UE;
3503 			goto irq_error;
3504 		}
3505 	}
3506 
3507 	/* Request Rx MSI irq */
3508 	for (i = 0; i < priv->plat->rx_queues_to_use; i++) {
3509 		if (i >= MTL_MAX_RX_QUEUES)
3510 			break;
3511 		if (priv->rx_irq[i] == 0)
3512 			continue;
3513 
3514 		int_name = priv->int_name_rx_irq[i];
3515 		sprintf(int_name, "%s:%s-%d", dev->name, "rx", i);
3516 		ret = request_irq(priv->rx_irq[i],
3517 				  stmmac_msi_intr_rx,
3518 				  0, int_name, &priv->rx_queue[i]);
3519 		if (unlikely(ret < 0)) {
3520 			netdev_err(priv->dev,
3521 				   "%s: alloc rx-%d  MSI %d (error: %d)\n",
3522 				   __func__, i, priv->rx_irq[i], ret);
3523 			irq_err = REQ_IRQ_ERR_RX;
3524 			irq_idx = i;
3525 			goto irq_error;
3526 		}
3527 		cpumask_clear(&cpu_mask);
3528 		cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
3529 		irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask);
3530 	}
3531 
3532 	/* Request Tx MSI irq */
3533 	for (i = 0; i < priv->plat->tx_queues_to_use; i++) {
3534 		if (i >= MTL_MAX_TX_QUEUES)
3535 			break;
3536 		if (priv->tx_irq[i] == 0)
3537 			continue;
3538 
3539 		int_name = priv->int_name_tx_irq[i];
3540 		sprintf(int_name, "%s:%s-%d", dev->name, "tx", i);
3541 		ret = request_irq(priv->tx_irq[i],
3542 				  stmmac_msi_intr_tx,
3543 				  0, int_name, &priv->tx_queue[i]);
3544 		if (unlikely(ret < 0)) {
3545 			netdev_err(priv->dev,
3546 				   "%s: alloc tx-%d  MSI %d (error: %d)\n",
3547 				   __func__, i, priv->tx_irq[i], ret);
3548 			irq_err = REQ_IRQ_ERR_TX;
3549 			irq_idx = i;
3550 			goto irq_error;
3551 		}
3552 		cpumask_clear(&cpu_mask);
3553 		cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
3554 		irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask);
3555 	}
3556 
3557 	return 0;
3558 
3559 irq_error:
3560 	stmmac_free_irq(dev, irq_err, irq_idx);
3561 	return ret;
3562 }
3563 
3564 static int stmmac_request_irq_single(struct net_device *dev)
3565 {
3566 	struct stmmac_priv *priv = netdev_priv(dev);
3567 	enum request_irq_err irq_err;
3568 	int ret;
3569 
3570 	ret = request_irq(dev->irq, stmmac_interrupt,
3571 			  IRQF_SHARED, dev->name, dev);
3572 	if (unlikely(ret < 0)) {
3573 		netdev_err(priv->dev,
3574 			   "%s: ERROR: allocating the IRQ %d (error: %d)\n",
3575 			   __func__, dev->irq, ret);
3576 		irq_err = REQ_IRQ_ERR_MAC;
3577 		goto irq_error;
3578 	}
3579 
3580 	/* Request the Wake IRQ in case of another line
3581 	 * is used for WoL
3582 	 */
3583 	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
3584 		ret = request_irq(priv->wol_irq, stmmac_interrupt,
3585 				  IRQF_SHARED, dev->name, dev);
3586 		if (unlikely(ret < 0)) {
3587 			netdev_err(priv->dev,
3588 				   "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
3589 				   __func__, priv->wol_irq, ret);
3590 			irq_err = REQ_IRQ_ERR_WOL;
3591 			goto irq_error;
3592 		}
3593 	}
3594 
3595 	/* Request the IRQ lines */
3596 	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {
3597 		ret = request_irq(priv->lpi_irq, stmmac_interrupt,
3598 				  IRQF_SHARED, dev->name, dev);
3599 		if (unlikely(ret < 0)) {
3600 			netdev_err(priv->dev,
3601 				   "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
3602 				   __func__, priv->lpi_irq, ret);
3603 			irq_err = REQ_IRQ_ERR_LPI;
3604 			goto irq_error;
3605 		}
3606 	}
3607 
3608 	return 0;
3609 
3610 irq_error:
3611 	stmmac_free_irq(dev, irq_err, 0);
3612 	return ret;
3613 }
3614 
3615 static int stmmac_request_irq(struct net_device *dev)
3616 {
3617 	struct stmmac_priv *priv = netdev_priv(dev);
3618 	int ret;
3619 
3620 	/* Request the IRQ lines */
3621 	if (priv->plat->multi_msi_en)
3622 		ret = stmmac_request_irq_multi_msi(dev);
3623 	else
3624 		ret = stmmac_request_irq_single(dev);
3625 
3626 	return ret;
3627 }
3628 
3629 /**
3630  *  stmmac_open - open entry point of the driver
3631  *  @dev : pointer to the device structure.
3632  *  Description:
3633  *  This function is the open entry point of the driver.
3634  *  Return value:
3635  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3636  *  file on failure.
3637  */
3638 static int stmmac_open(struct net_device *dev)
3639 {
3640 	struct stmmac_priv *priv = netdev_priv(dev);
3641 	int mode = priv->plat->phy_interface;
3642 	int bfsize = 0;
3643 	u32 chan;
3644 	int ret;
3645 
3646 	ret = pm_runtime_get_sync(priv->device);
3647 	if (ret < 0) {
3648 		pm_runtime_put_noidle(priv->device);
3649 		return ret;
3650 	}
3651 
3652 	if (priv->hw->pcs != STMMAC_PCS_TBI &&
3653 	    priv->hw->pcs != STMMAC_PCS_RTBI &&
3654 	    (!priv->hw->xpcs ||
3655 	     xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) {
3656 		ret = stmmac_init_phy(dev);
3657 		if (ret) {
3658 			netdev_err(priv->dev,
3659 				   "%s: Cannot attach to PHY (error: %d)\n",
3660 				   __func__, ret);
3661 			goto init_phy_error;
3662 		}
3663 	}
3664 
3665 	/* Extra statistics */
3666 	memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
3667 	priv->xstats.threshold = tc;
3668 
3669 	bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
3670 	if (bfsize < 0)
3671 		bfsize = 0;
3672 
3673 	if (bfsize < BUF_SIZE_16KiB)
3674 		bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
3675 
3676 	priv->dma_buf_sz = bfsize;
3677 	buf_sz = bfsize;
3678 
3679 	priv->rx_copybreak = STMMAC_RX_COPYBREAK;
3680 
3681 	if (!priv->dma_tx_size)
3682 		priv->dma_tx_size = DMA_DEFAULT_TX_SIZE;
3683 	if (!priv->dma_rx_size)
3684 		priv->dma_rx_size = DMA_DEFAULT_RX_SIZE;
3685 
3686 	/* Earlier check for TBS */
3687 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {
3688 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
3689 		int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;
3690 
3691 		/* Setup per-TXQ tbs flag before TX descriptor alloc */
3692 		tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0;
3693 	}
3694 
3695 	ret = alloc_dma_desc_resources(priv);
3696 	if (ret < 0) {
3697 		netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
3698 			   __func__);
3699 		goto dma_desc_error;
3700 	}
3701 
3702 	ret = init_dma_desc_rings(dev, GFP_KERNEL);
3703 	if (ret < 0) {
3704 		netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
3705 			   __func__);
3706 		goto init_error;
3707 	}
3708 
3709 	ret = stmmac_hw_setup(dev, true);
3710 	if (ret < 0) {
3711 		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
3712 		goto init_error;
3713 	}
3714 
3715 	stmmac_init_coalesce(priv);
3716 
3717 	phylink_start(priv->phylink);
3718 	/* We may have called phylink_speed_down before */
3719 	phylink_speed_up(priv->phylink);
3720 
3721 	ret = stmmac_request_irq(dev);
3722 	if (ret)
3723 		goto irq_error;
3724 
3725 	stmmac_enable_all_queues(priv);
3726 	netif_tx_start_all_queues(priv->dev);
3727 	stmmac_enable_all_dma_irq(priv);
3728 
3729 	return 0;
3730 
3731 irq_error:
3732 	phylink_stop(priv->phylink);
3733 
3734 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
3735 		hrtimer_cancel(&priv->tx_queue[chan].txtimer);
3736 
3737 	stmmac_hw_teardown(dev);
3738 init_error:
3739 	free_dma_desc_resources(priv);
3740 dma_desc_error:
3741 	phylink_disconnect_phy(priv->phylink);
3742 init_phy_error:
3743 	pm_runtime_put(priv->device);
3744 	return ret;
3745 }
3746 
3747 static void stmmac_fpe_stop_wq(struct stmmac_priv *priv)
3748 {
3749 	set_bit(__FPE_REMOVING, &priv->fpe_task_state);
3750 
3751 	if (priv->fpe_wq)
3752 		destroy_workqueue(priv->fpe_wq);
3753 
3754 	netdev_info(priv->dev, "FPE workqueue stop");
3755 }
3756 
3757 /**
3758  *  stmmac_release - close entry point of the driver
3759  *  @dev : device pointer.
3760  *  Description:
3761  *  This is the stop entry point of the driver.
3762  */
3763 static int stmmac_release(struct net_device *dev)
3764 {
3765 	struct stmmac_priv *priv = netdev_priv(dev);
3766 	u32 chan;
3767 
3768 	netif_tx_disable(dev);
3769 
3770 	if (device_may_wakeup(priv->device))
3771 		phylink_speed_down(priv->phylink, false);
3772 	/* Stop and disconnect the PHY */
3773 	phylink_stop(priv->phylink);
3774 	phylink_disconnect_phy(priv->phylink);
3775 
3776 	stmmac_disable_all_queues(priv);
3777 
3778 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
3779 		hrtimer_cancel(&priv->tx_queue[chan].txtimer);
3780 
3781 	/* Free the IRQ lines */
3782 	stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0);
3783 
3784 	if (priv->eee_enabled) {
3785 		priv->tx_path_in_lpi_mode = false;
3786 		del_timer_sync(&priv->eee_ctrl_timer);
3787 	}
3788 
3789 	/* Stop TX/RX DMA and clear the descriptors */
3790 	stmmac_stop_all_dma(priv);
3791 
3792 	/* Release and free the Rx/Tx resources */
3793 	free_dma_desc_resources(priv);
3794 
3795 	/* Disable the MAC Rx/Tx */
3796 	stmmac_mac_set(priv, priv->ioaddr, false);
3797 
3798 	netif_carrier_off(dev);
3799 
3800 	stmmac_release_ptp(priv);
3801 
3802 	pm_runtime_put(priv->device);
3803 
3804 	if (priv->dma_cap.fpesel)
3805 		stmmac_fpe_stop_wq(priv);
3806 
3807 	return 0;
3808 }
3809 
3810 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
3811 			       struct stmmac_tx_queue *tx_q)
3812 {
3813 	u16 tag = 0x0, inner_tag = 0x0;
3814 	u32 inner_type = 0x0;
3815 	struct dma_desc *p;
3816 
3817 	if (!priv->dma_cap.vlins)
3818 		return false;
3819 	if (!skb_vlan_tag_present(skb))
3820 		return false;
3821 	if (skb->vlan_proto == htons(ETH_P_8021AD)) {
3822 		inner_tag = skb_vlan_tag_get(skb);
3823 		inner_type = STMMAC_VLAN_INSERT;
3824 	}
3825 
3826 	tag = skb_vlan_tag_get(skb);
3827 
3828 	if (tx_q->tbs & STMMAC_TBS_AVAIL)
3829 		p = &tx_q->dma_entx[tx_q->cur_tx].basic;
3830 	else
3831 		p = &tx_q->dma_tx[tx_q->cur_tx];
3832 
3833 	if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))
3834 		return false;
3835 
3836 	stmmac_set_tx_owner(priv, p);
3837 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
3838 	return true;
3839 }
3840 
3841 /**
3842  *  stmmac_tso_allocator - close entry point of the driver
3843  *  @priv: driver private structure
3844  *  @des: buffer start address
3845  *  @total_len: total length to fill in descriptors
3846  *  @last_segment: condition for the last descriptor
3847  *  @queue: TX queue index
3848  *  Description:
3849  *  This function fills descriptor and request new descriptors according to
3850  *  buffer length to fill
3851  */
3852 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
3853 				 int total_len, bool last_segment, u32 queue)
3854 {
3855 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3856 	struct dma_desc *desc;
3857 	u32 buff_size;
3858 	int tmp_len;
3859 
3860 	tmp_len = total_len;
3861 
3862 	while (tmp_len > 0) {
3863 		dma_addr_t curr_addr;
3864 
3865 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
3866 						priv->dma_tx_size);
3867 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
3868 
3869 		if (tx_q->tbs & STMMAC_TBS_AVAIL)
3870 			desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3871 		else
3872 			desc = &tx_q->dma_tx[tx_q->cur_tx];
3873 
3874 		curr_addr = des + (total_len - tmp_len);
3875 		if (priv->dma_cap.addr64 <= 32)
3876 			desc->des0 = cpu_to_le32(curr_addr);
3877 		else
3878 			stmmac_set_desc_addr(priv, desc, curr_addr);
3879 
3880 		buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
3881 			    TSO_MAX_BUFF_SIZE : tmp_len;
3882 
3883 		stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
3884 				0, 1,
3885 				(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
3886 				0, 0);
3887 
3888 		tmp_len -= TSO_MAX_BUFF_SIZE;
3889 	}
3890 }
3891 
3892 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)
3893 {
3894 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3895 	int desc_size;
3896 
3897 	if (likely(priv->extend_desc))
3898 		desc_size = sizeof(struct dma_extended_desc);
3899 	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3900 		desc_size = sizeof(struct dma_edesc);
3901 	else
3902 		desc_size = sizeof(struct dma_desc);
3903 
3904 	/* The own bit must be the latest setting done when prepare the
3905 	 * descriptor and then barrier is needed to make sure that
3906 	 * all is coherent before granting the DMA engine.
3907 	 */
3908 	wmb();
3909 
3910 	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
3911 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3912 }
3913 
3914 /**
3915  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
3916  *  @skb : the socket buffer
3917  *  @dev : device pointer
3918  *  Description: this is the transmit function that is called on TSO frames
3919  *  (support available on GMAC4 and newer chips).
3920  *  Diagram below show the ring programming in case of TSO frames:
3921  *
3922  *  First Descriptor
3923  *   --------
3924  *   | DES0 |---> buffer1 = L2/L3/L4 header
3925  *   | DES1 |---> TCP Payload (can continue on next descr...)
3926  *   | DES2 |---> buffer 1 and 2 len
3927  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
3928  *   --------
3929  *	|
3930  *     ...
3931  *	|
3932  *   --------
3933  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
3934  *   | DES1 | --|
3935  *   | DES2 | --> buffer 1 and 2 len
3936  *   | DES3 |
3937  *   --------
3938  *
3939  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
3940  */
3941 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
3942 {
3943 	struct dma_desc *desc, *first, *mss_desc = NULL;
3944 	struct stmmac_priv *priv = netdev_priv(dev);
3945 	int nfrags = skb_shinfo(skb)->nr_frags;
3946 	u32 queue = skb_get_queue_mapping(skb);
3947 	unsigned int first_entry, tx_packets;
3948 	int tmp_pay_len = 0, first_tx;
3949 	struct stmmac_tx_queue *tx_q;
3950 	bool has_vlan, set_ic;
3951 	u8 proto_hdr_len, hdr;
3952 	u32 pay_len, mss;
3953 	dma_addr_t des;
3954 	int i;
3955 
3956 	tx_q = &priv->tx_queue[queue];
3957 	first_tx = tx_q->cur_tx;
3958 
3959 	/* Compute header lengths */
3960 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
3961 		proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
3962 		hdr = sizeof(struct udphdr);
3963 	} else {
3964 		proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3965 		hdr = tcp_hdrlen(skb);
3966 	}
3967 
3968 	/* Desc availability based on threshold should be enough safe */
3969 	if (unlikely(stmmac_tx_avail(priv, queue) <
3970 		(((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
3971 		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3972 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3973 								queue));
3974 			/* This is a hard error, log it. */
3975 			netdev_err(priv->dev,
3976 				   "%s: Tx Ring full when queue awake\n",
3977 				   __func__);
3978 		}
3979 		return NETDEV_TX_BUSY;
3980 	}
3981 
3982 	pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
3983 
3984 	mss = skb_shinfo(skb)->gso_size;
3985 
3986 	/* set new MSS value if needed */
3987 	if (mss != tx_q->mss) {
3988 		if (tx_q->tbs & STMMAC_TBS_AVAIL)
3989 			mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3990 		else
3991 			mss_desc = &tx_q->dma_tx[tx_q->cur_tx];
3992 
3993 		stmmac_set_mss(priv, mss_desc, mss);
3994 		tx_q->mss = mss;
3995 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
3996 						priv->dma_tx_size);
3997 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
3998 	}
3999 
4000 	if (netif_msg_tx_queued(priv)) {
4001 		pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
4002 			__func__, hdr, proto_hdr_len, pay_len, mss);
4003 		pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
4004 			skb->data_len);
4005 	}
4006 
4007 	/* Check if VLAN can be inserted by HW */
4008 	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
4009 
4010 	first_entry = tx_q->cur_tx;
4011 	WARN_ON(tx_q->tx_skbuff[first_entry]);
4012 
4013 	if (tx_q->tbs & STMMAC_TBS_AVAIL)
4014 		desc = &tx_q->dma_entx[first_entry].basic;
4015 	else
4016 		desc = &tx_q->dma_tx[first_entry];
4017 	first = desc;
4018 
4019 	if (has_vlan)
4020 		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
4021 
4022 	/* first descriptor: fill Headers on Buf1 */
4023 	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
4024 			     DMA_TO_DEVICE);
4025 	if (dma_mapping_error(priv->device, des))
4026 		goto dma_map_err;
4027 
4028 	tx_q->tx_skbuff_dma[first_entry].buf = des;
4029 	tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
4030 	tx_q->tx_skbuff_dma[first_entry].map_as_page = false;
4031 	tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB;
4032 
4033 	if (priv->dma_cap.addr64 <= 32) {
4034 		first->des0 = cpu_to_le32(des);
4035 
4036 		/* Fill start of payload in buff2 of first descriptor */
4037 		if (pay_len)
4038 			first->des1 = cpu_to_le32(des + proto_hdr_len);
4039 
4040 		/* If needed take extra descriptors to fill the remaining payload */
4041 		tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
4042 	} else {
4043 		stmmac_set_desc_addr(priv, first, des);
4044 		tmp_pay_len = pay_len;
4045 		des += proto_hdr_len;
4046 		pay_len = 0;
4047 	}
4048 
4049 	stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
4050 
4051 	/* Prepare fragments */
4052 	for (i = 0; i < nfrags; i++) {
4053 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4054 
4055 		des = skb_frag_dma_map(priv->device, frag, 0,
4056 				       skb_frag_size(frag),
4057 				       DMA_TO_DEVICE);
4058 		if (dma_mapping_error(priv->device, des))
4059 			goto dma_map_err;
4060 
4061 		stmmac_tso_allocator(priv, des, skb_frag_size(frag),
4062 				     (i == nfrags - 1), queue);
4063 
4064 		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
4065 		tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
4066 		tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
4067 		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
4068 	}
4069 
4070 	tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
4071 
4072 	/* Only the last descriptor gets to point to the skb. */
4073 	tx_q->tx_skbuff[tx_q->cur_tx] = skb;
4074 	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
4075 
4076 	/* Manage tx mitigation */
4077 	tx_packets = (tx_q->cur_tx + 1) - first_tx;
4078 	tx_q->tx_count_frames += tx_packets;
4079 
4080 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
4081 		set_ic = true;
4082 	else if (!priv->tx_coal_frames[queue])
4083 		set_ic = false;
4084 	else if (tx_packets > priv->tx_coal_frames[queue])
4085 		set_ic = true;
4086 	else if ((tx_q->tx_count_frames %
4087 		  priv->tx_coal_frames[queue]) < tx_packets)
4088 		set_ic = true;
4089 	else
4090 		set_ic = false;
4091 
4092 	if (set_ic) {
4093 		if (tx_q->tbs & STMMAC_TBS_AVAIL)
4094 			desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
4095 		else
4096 			desc = &tx_q->dma_tx[tx_q->cur_tx];
4097 
4098 		tx_q->tx_count_frames = 0;
4099 		stmmac_set_tx_ic(priv, desc);
4100 		priv->xstats.tx_set_ic_bit++;
4101 	}
4102 
4103 	/* We've used all descriptors we need for this skb, however,
4104 	 * advance cur_tx so that it references a fresh descriptor.
4105 	 * ndo_start_xmit will fill this descriptor the next time it's
4106 	 * called and stmmac_tx_clean may clean up to this descriptor.
4107 	 */
4108 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
4109 
4110 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
4111 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
4112 			  __func__);
4113 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
4114 	}
4115 
4116 	dev->stats.tx_bytes += skb->len;
4117 	priv->xstats.tx_tso_frames++;
4118 	priv->xstats.tx_tso_nfrags += nfrags;
4119 
4120 	if (priv->sarc_type)
4121 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
4122 
4123 	skb_tx_timestamp(skb);
4124 
4125 	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
4126 		     priv->hwts_tx_en)) {
4127 		/* declare that device is doing timestamping */
4128 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4129 		stmmac_enable_tx_timestamp(priv, first);
4130 	}
4131 
4132 	/* Complete the first descriptor before granting the DMA */
4133 	stmmac_prepare_tso_tx_desc(priv, first, 1,
4134 			proto_hdr_len,
4135 			pay_len,
4136 			1, tx_q->tx_skbuff_dma[first_entry].last_segment,
4137 			hdr / 4, (skb->len - proto_hdr_len));
4138 
4139 	/* If context desc is used to change MSS */
4140 	if (mss_desc) {
4141 		/* Make sure that first descriptor has been completely
4142 		 * written, including its own bit. This is because MSS is
4143 		 * actually before first descriptor, so we need to make
4144 		 * sure that MSS's own bit is the last thing written.
4145 		 */
4146 		dma_wmb();
4147 		stmmac_set_tx_owner(priv, mss_desc);
4148 	}
4149 
4150 	if (netif_msg_pktdata(priv)) {
4151 		pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
4152 			__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
4153 			tx_q->cur_tx, first, nfrags);
4154 		pr_info(">>> frame to be transmitted: ");
4155 		print_pkt(skb->data, skb_headlen(skb));
4156 	}
4157 
4158 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
4159 
4160 	stmmac_flush_tx_descriptors(priv, queue);
4161 	stmmac_tx_timer_arm(priv, queue);
4162 
4163 	return NETDEV_TX_OK;
4164 
4165 dma_map_err:
4166 	dev_err(priv->device, "Tx dma map failed\n");
4167 	dev_kfree_skb(skb);
4168 	priv->dev->stats.tx_dropped++;
4169 	return NETDEV_TX_OK;
4170 }
4171 
4172 /**
4173  *  stmmac_xmit - Tx entry point of the driver
4174  *  @skb : the socket buffer
4175  *  @dev : device pointer
4176  *  Description : this is the tx entry point of the driver.
4177  *  It programs the chain or the ring and supports oversized frames
4178  *  and SG feature.
4179  */
4180 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
4181 {
4182 	unsigned int first_entry, tx_packets, enh_desc;
4183 	struct stmmac_priv *priv = netdev_priv(dev);
4184 	unsigned int nopaged_len = skb_headlen(skb);
4185 	int i, csum_insertion = 0, is_jumbo = 0;
4186 	u32 queue = skb_get_queue_mapping(skb);
4187 	int nfrags = skb_shinfo(skb)->nr_frags;
4188 	int gso = skb_shinfo(skb)->gso_type;
4189 	struct dma_edesc *tbs_desc = NULL;
4190 	struct dma_desc *desc, *first;
4191 	struct stmmac_tx_queue *tx_q;
4192 	bool has_vlan, set_ic;
4193 	int entry, first_tx;
4194 	dma_addr_t des;
4195 
4196 	tx_q = &priv->tx_queue[queue];
4197 	first_tx = tx_q->cur_tx;
4198 
4199 	if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en)
4200 		stmmac_disable_eee_mode(priv);
4201 
4202 	/* Manage oversized TCP frames for GMAC4 device */
4203 	if (skb_is_gso(skb) && priv->tso) {
4204 		if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
4205 			return stmmac_tso_xmit(skb, dev);
4206 		if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
4207 			return stmmac_tso_xmit(skb, dev);
4208 	}
4209 
4210 	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
4211 		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
4212 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
4213 								queue));
4214 			/* This is a hard error, log it. */
4215 			netdev_err(priv->dev,
4216 				   "%s: Tx Ring full when queue awake\n",
4217 				   __func__);
4218 		}
4219 		return NETDEV_TX_BUSY;
4220 	}
4221 
4222 	/* Check if VLAN can be inserted by HW */
4223 	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
4224 
4225 	entry = tx_q->cur_tx;
4226 	first_entry = entry;
4227 	WARN_ON(tx_q->tx_skbuff[first_entry]);
4228 
4229 	csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
4230 
4231 	if (likely(priv->extend_desc))
4232 		desc = (struct dma_desc *)(tx_q->dma_etx + entry);
4233 	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4234 		desc = &tx_q->dma_entx[entry].basic;
4235 	else
4236 		desc = tx_q->dma_tx + entry;
4237 
4238 	first = desc;
4239 
4240 	if (has_vlan)
4241 		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
4242 
4243 	enh_desc = priv->plat->enh_desc;
4244 	/* To program the descriptors according to the size of the frame */
4245 	if (enh_desc)
4246 		is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
4247 
4248 	if (unlikely(is_jumbo)) {
4249 		entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
4250 		if (unlikely(entry < 0) && (entry != -EINVAL))
4251 			goto dma_map_err;
4252 	}
4253 
4254 	for (i = 0; i < nfrags; i++) {
4255 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4256 		int len = skb_frag_size(frag);
4257 		bool last_segment = (i == (nfrags - 1));
4258 
4259 		entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
4260 		WARN_ON(tx_q->tx_skbuff[entry]);
4261 
4262 		if (likely(priv->extend_desc))
4263 			desc = (struct dma_desc *)(tx_q->dma_etx + entry);
4264 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4265 			desc = &tx_q->dma_entx[entry].basic;
4266 		else
4267 			desc = tx_q->dma_tx + entry;
4268 
4269 		des = skb_frag_dma_map(priv->device, frag, 0, len,
4270 				       DMA_TO_DEVICE);
4271 		if (dma_mapping_error(priv->device, des))
4272 			goto dma_map_err; /* should reuse desc w/o issues */
4273 
4274 		tx_q->tx_skbuff_dma[entry].buf = des;
4275 
4276 		stmmac_set_desc_addr(priv, desc, des);
4277 
4278 		tx_q->tx_skbuff_dma[entry].map_as_page = true;
4279 		tx_q->tx_skbuff_dma[entry].len = len;
4280 		tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
4281 		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
4282 
4283 		/* Prepare the descriptor and set the own bit too */
4284 		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
4285 				priv->mode, 1, last_segment, skb->len);
4286 	}
4287 
4288 	/* Only the last descriptor gets to point to the skb. */
4289 	tx_q->tx_skbuff[entry] = skb;
4290 	tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
4291 
4292 	/* According to the coalesce parameter the IC bit for the latest
4293 	 * segment is reset and the timer re-started to clean the tx status.
4294 	 * This approach takes care about the fragments: desc is the first
4295 	 * element in case of no SG.
4296 	 */
4297 	tx_packets = (entry + 1) - first_tx;
4298 	tx_q->tx_count_frames += tx_packets;
4299 
4300 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
4301 		set_ic = true;
4302 	else if (!priv->tx_coal_frames[queue])
4303 		set_ic = false;
4304 	else if (tx_packets > priv->tx_coal_frames[queue])
4305 		set_ic = true;
4306 	else if ((tx_q->tx_count_frames %
4307 		  priv->tx_coal_frames[queue]) < tx_packets)
4308 		set_ic = true;
4309 	else
4310 		set_ic = false;
4311 
4312 	if (set_ic) {
4313 		if (likely(priv->extend_desc))
4314 			desc = &tx_q->dma_etx[entry].basic;
4315 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4316 			desc = &tx_q->dma_entx[entry].basic;
4317 		else
4318 			desc = &tx_q->dma_tx[entry];
4319 
4320 		tx_q->tx_count_frames = 0;
4321 		stmmac_set_tx_ic(priv, desc);
4322 		priv->xstats.tx_set_ic_bit++;
4323 	}
4324 
4325 	/* We've used all descriptors we need for this skb, however,
4326 	 * advance cur_tx so that it references a fresh descriptor.
4327 	 * ndo_start_xmit will fill this descriptor the next time it's
4328 	 * called and stmmac_tx_clean may clean up to this descriptor.
4329 	 */
4330 	entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
4331 	tx_q->cur_tx = entry;
4332 
4333 	if (netif_msg_pktdata(priv)) {
4334 		netdev_dbg(priv->dev,
4335 			   "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
4336 			   __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
4337 			   entry, first, nfrags);
4338 
4339 		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
4340 		print_pkt(skb->data, skb->len);
4341 	}
4342 
4343 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
4344 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
4345 			  __func__);
4346 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
4347 	}
4348 
4349 	dev->stats.tx_bytes += skb->len;
4350 
4351 	if (priv->sarc_type)
4352 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
4353 
4354 	skb_tx_timestamp(skb);
4355 
4356 	/* Ready to fill the first descriptor and set the OWN bit w/o any
4357 	 * problems because all the descriptors are actually ready to be
4358 	 * passed to the DMA engine.
4359 	 */
4360 	if (likely(!is_jumbo)) {
4361 		bool last_segment = (nfrags == 0);
4362 
4363 		des = dma_map_single(priv->device, skb->data,
4364 				     nopaged_len, DMA_TO_DEVICE);
4365 		if (dma_mapping_error(priv->device, des))
4366 			goto dma_map_err;
4367 
4368 		tx_q->tx_skbuff_dma[first_entry].buf = des;
4369 		tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB;
4370 		tx_q->tx_skbuff_dma[first_entry].map_as_page = false;
4371 
4372 		stmmac_set_desc_addr(priv, first, des);
4373 
4374 		tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
4375 		tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
4376 
4377 		if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
4378 			     priv->hwts_tx_en)) {
4379 			/* declare that device is doing timestamping */
4380 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4381 			stmmac_enable_tx_timestamp(priv, first);
4382 		}
4383 
4384 		/* Prepare the first descriptor setting the OWN bit too */
4385 		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
4386 				csum_insertion, priv->mode, 0, last_segment,
4387 				skb->len);
4388 	}
4389 
4390 	if (tx_q->tbs & STMMAC_TBS_EN) {
4391 		struct timespec64 ts = ns_to_timespec64(skb->tstamp);
4392 
4393 		tbs_desc = &tx_q->dma_entx[first_entry];
4394 		stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
4395 	}
4396 
4397 	stmmac_set_tx_owner(priv, first);
4398 
4399 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
4400 
4401 	stmmac_enable_dma_transmission(priv, priv->ioaddr);
4402 
4403 	stmmac_flush_tx_descriptors(priv, queue);
4404 	stmmac_tx_timer_arm(priv, queue);
4405 
4406 	return NETDEV_TX_OK;
4407 
4408 dma_map_err:
4409 	netdev_err(priv->dev, "Tx DMA map failed\n");
4410 	dev_kfree_skb(skb);
4411 	priv->dev->stats.tx_dropped++;
4412 	return NETDEV_TX_OK;
4413 }
4414 
4415 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
4416 {
4417 	struct vlan_ethhdr *veth;
4418 	__be16 vlan_proto;
4419 	u16 vlanid;
4420 
4421 	veth = (struct vlan_ethhdr *)skb->data;
4422 	vlan_proto = veth->h_vlan_proto;
4423 
4424 	if ((vlan_proto == htons(ETH_P_8021Q) &&
4425 	     dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
4426 	    (vlan_proto == htons(ETH_P_8021AD) &&
4427 	     dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
4428 		/* pop the vlan tag */
4429 		vlanid = ntohs(veth->h_vlan_TCI);
4430 		memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
4431 		skb_pull(skb, VLAN_HLEN);
4432 		__vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
4433 	}
4434 }
4435 
4436 /**
4437  * stmmac_rx_refill - refill used skb preallocated buffers
4438  * @priv: driver private structure
4439  * @queue: RX queue index
4440  * Description : this is to reallocate the skb for the reception process
4441  * that is based on zero-copy.
4442  */
4443 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
4444 {
4445 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4446 	int dirty = stmmac_rx_dirty(priv, queue);
4447 	unsigned int entry = rx_q->dirty_rx;
4448 	gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
4449 
4450 	if (priv->dma_cap.addr64 <= 32)
4451 		gfp |= GFP_DMA32;
4452 
4453 	while (dirty-- > 0) {
4454 		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
4455 		struct dma_desc *p;
4456 		bool use_rx_wd;
4457 
4458 		if (priv->extend_desc)
4459 			p = (struct dma_desc *)(rx_q->dma_erx + entry);
4460 		else
4461 			p = rx_q->dma_rx + entry;
4462 
4463 		if (!buf->page) {
4464 			buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp);
4465 			if (!buf->page)
4466 				break;
4467 		}
4468 
4469 		if (priv->sph && !buf->sec_page) {
4470 			buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp);
4471 			if (!buf->sec_page)
4472 				break;
4473 
4474 			buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
4475 		}
4476 
4477 		buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset;
4478 
4479 		stmmac_set_desc_addr(priv, p, buf->addr);
4480 		if (priv->sph)
4481 			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
4482 		else
4483 			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
4484 		stmmac_refill_desc3(priv, rx_q, p);
4485 
4486 		rx_q->rx_count_frames++;
4487 		rx_q->rx_count_frames += priv->rx_coal_frames[queue];
4488 		if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])
4489 			rx_q->rx_count_frames = 0;
4490 
4491 		use_rx_wd = !priv->rx_coal_frames[queue];
4492 		use_rx_wd |= rx_q->rx_count_frames > 0;
4493 		if (!priv->use_riwt)
4494 			use_rx_wd = false;
4495 
4496 		dma_wmb();
4497 		stmmac_set_rx_owner(priv, p, use_rx_wd);
4498 
4499 		entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size);
4500 	}
4501 	rx_q->dirty_rx = entry;
4502 	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
4503 			    (rx_q->dirty_rx * sizeof(struct dma_desc));
4504 	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
4505 }
4506 
4507 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
4508 				       struct dma_desc *p,
4509 				       int status, unsigned int len)
4510 {
4511 	unsigned int plen = 0, hlen = 0;
4512 	int coe = priv->hw->rx_csum;
4513 
4514 	/* Not first descriptor, buffer is always zero */
4515 	if (priv->sph && len)
4516 		return 0;
4517 
4518 	/* First descriptor, get split header length */
4519 	stmmac_get_rx_header_len(priv, p, &hlen);
4520 	if (priv->sph && hlen) {
4521 		priv->xstats.rx_split_hdr_pkt_n++;
4522 		return hlen;
4523 	}
4524 
4525 	/* First descriptor, not last descriptor and not split header */
4526 	if (status & rx_not_ls)
4527 		return priv->dma_buf_sz;
4528 
4529 	plen = stmmac_get_rx_frame_len(priv, p, coe);
4530 
4531 	/* First descriptor and last descriptor and not split header */
4532 	return min_t(unsigned int, priv->dma_buf_sz, plen);
4533 }
4534 
4535 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
4536 				       struct dma_desc *p,
4537 				       int status, unsigned int len)
4538 {
4539 	int coe = priv->hw->rx_csum;
4540 	unsigned int plen = 0;
4541 
4542 	/* Not split header, buffer is not available */
4543 	if (!priv->sph)
4544 		return 0;
4545 
4546 	/* Not last descriptor */
4547 	if (status & rx_not_ls)
4548 		return priv->dma_buf_sz;
4549 
4550 	plen = stmmac_get_rx_frame_len(priv, p, coe);
4551 
4552 	/* Last descriptor */
4553 	return plen - len;
4554 }
4555 
4556 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
4557 				struct xdp_frame *xdpf, bool dma_map)
4558 {
4559 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4560 	unsigned int entry = tx_q->cur_tx;
4561 	struct dma_desc *tx_desc;
4562 	dma_addr_t dma_addr;
4563 	bool set_ic;
4564 
4565 	if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv))
4566 		return STMMAC_XDP_CONSUMED;
4567 
4568 	if (likely(priv->extend_desc))
4569 		tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
4570 	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4571 		tx_desc = &tx_q->dma_entx[entry].basic;
4572 	else
4573 		tx_desc = tx_q->dma_tx + entry;
4574 
4575 	if (dma_map) {
4576 		dma_addr = dma_map_single(priv->device, xdpf->data,
4577 					  xdpf->len, DMA_TO_DEVICE);
4578 		if (dma_mapping_error(priv->device, dma_addr))
4579 			return STMMAC_XDP_CONSUMED;
4580 
4581 		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO;
4582 	} else {
4583 		struct page *page = virt_to_page(xdpf->data);
4584 
4585 		dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) +
4586 			   xdpf->headroom;
4587 		dma_sync_single_for_device(priv->device, dma_addr,
4588 					   xdpf->len, DMA_BIDIRECTIONAL);
4589 
4590 		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX;
4591 	}
4592 
4593 	tx_q->tx_skbuff_dma[entry].buf = dma_addr;
4594 	tx_q->tx_skbuff_dma[entry].map_as_page = false;
4595 	tx_q->tx_skbuff_dma[entry].len = xdpf->len;
4596 	tx_q->tx_skbuff_dma[entry].last_segment = true;
4597 	tx_q->tx_skbuff_dma[entry].is_jumbo = false;
4598 
4599 	tx_q->xdpf[entry] = xdpf;
4600 
4601 	stmmac_set_desc_addr(priv, tx_desc, dma_addr);
4602 
4603 	stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len,
4604 			       true, priv->mode, true, true,
4605 			       xdpf->len);
4606 
4607 	tx_q->tx_count_frames++;
4608 
4609 	if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)
4610 		set_ic = true;
4611 	else
4612 		set_ic = false;
4613 
4614 	if (set_ic) {
4615 		tx_q->tx_count_frames = 0;
4616 		stmmac_set_tx_ic(priv, tx_desc);
4617 		priv->xstats.tx_set_ic_bit++;
4618 	}
4619 
4620 	stmmac_enable_dma_transmission(priv, priv->ioaddr);
4621 
4622 	entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
4623 	tx_q->cur_tx = entry;
4624 
4625 	return STMMAC_XDP_TX;
4626 }
4627 
4628 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv,
4629 				   int cpu)
4630 {
4631 	int index = cpu;
4632 
4633 	if (unlikely(index < 0))
4634 		index = 0;
4635 
4636 	while (index >= priv->plat->tx_queues_to_use)
4637 		index -= priv->plat->tx_queues_to_use;
4638 
4639 	return index;
4640 }
4641 
4642 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv,
4643 				struct xdp_buff *xdp)
4644 {
4645 	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
4646 	int cpu = smp_processor_id();
4647 	struct netdev_queue *nq;
4648 	int queue;
4649 	int res;
4650 
4651 	if (unlikely(!xdpf))
4652 		return STMMAC_XDP_CONSUMED;
4653 
4654 	queue = stmmac_xdp_get_tx_queue(priv, cpu);
4655 	nq = netdev_get_tx_queue(priv->dev, queue);
4656 
4657 	__netif_tx_lock(nq, cpu);
4658 	/* Avoids TX time-out as we are sharing with slow path */
4659 	txq_trans_cond_update(nq);
4660 
4661 	res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false);
4662 	if (res == STMMAC_XDP_TX)
4663 		stmmac_flush_tx_descriptors(priv, queue);
4664 
4665 	__netif_tx_unlock(nq);
4666 
4667 	return res;
4668 }
4669 
4670 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv,
4671 				 struct bpf_prog *prog,
4672 				 struct xdp_buff *xdp)
4673 {
4674 	u32 act;
4675 	int res;
4676 
4677 	act = bpf_prog_run_xdp(prog, xdp);
4678 	switch (act) {
4679 	case XDP_PASS:
4680 		res = STMMAC_XDP_PASS;
4681 		break;
4682 	case XDP_TX:
4683 		res = stmmac_xdp_xmit_back(priv, xdp);
4684 		break;
4685 	case XDP_REDIRECT:
4686 		if (xdp_do_redirect(priv->dev, xdp, prog) < 0)
4687 			res = STMMAC_XDP_CONSUMED;
4688 		else
4689 			res = STMMAC_XDP_REDIRECT;
4690 		break;
4691 	default:
4692 		bpf_warn_invalid_xdp_action(priv->dev, prog, act);
4693 		fallthrough;
4694 	case XDP_ABORTED:
4695 		trace_xdp_exception(priv->dev, prog, act);
4696 		fallthrough;
4697 	case XDP_DROP:
4698 		res = STMMAC_XDP_CONSUMED;
4699 		break;
4700 	}
4701 
4702 	return res;
4703 }
4704 
4705 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv,
4706 					   struct xdp_buff *xdp)
4707 {
4708 	struct bpf_prog *prog;
4709 	int res;
4710 
4711 	prog = READ_ONCE(priv->xdp_prog);
4712 	if (!prog) {
4713 		res = STMMAC_XDP_PASS;
4714 		goto out;
4715 	}
4716 
4717 	res = __stmmac_xdp_run_prog(priv, prog, xdp);
4718 out:
4719 	return ERR_PTR(-res);
4720 }
4721 
4722 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv,
4723 				   int xdp_status)
4724 {
4725 	int cpu = smp_processor_id();
4726 	int queue;
4727 
4728 	queue = stmmac_xdp_get_tx_queue(priv, cpu);
4729 
4730 	if (xdp_status & STMMAC_XDP_TX)
4731 		stmmac_tx_timer_arm(priv, queue);
4732 
4733 	if (xdp_status & STMMAC_XDP_REDIRECT)
4734 		xdp_do_flush();
4735 }
4736 
4737 static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch,
4738 					       struct xdp_buff *xdp)
4739 {
4740 	unsigned int metasize = xdp->data - xdp->data_meta;
4741 	unsigned int datasize = xdp->data_end - xdp->data;
4742 	struct sk_buff *skb;
4743 
4744 	skb = __napi_alloc_skb(&ch->rxtx_napi,
4745 			       xdp->data_end - xdp->data_hard_start,
4746 			       GFP_ATOMIC | __GFP_NOWARN);
4747 	if (unlikely(!skb))
4748 		return NULL;
4749 
4750 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
4751 	memcpy(__skb_put(skb, datasize), xdp->data, datasize);
4752 	if (metasize)
4753 		skb_metadata_set(skb, metasize);
4754 
4755 	return skb;
4756 }
4757 
4758 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,
4759 				   struct dma_desc *p, struct dma_desc *np,
4760 				   struct xdp_buff *xdp)
4761 {
4762 	struct stmmac_channel *ch = &priv->channel[queue];
4763 	unsigned int len = xdp->data_end - xdp->data;
4764 	enum pkt_hash_types hash_type;
4765 	int coe = priv->hw->rx_csum;
4766 	struct sk_buff *skb;
4767 	u32 hash;
4768 
4769 	skb = stmmac_construct_skb_zc(ch, xdp);
4770 	if (!skb) {
4771 		priv->dev->stats.rx_dropped++;
4772 		return;
4773 	}
4774 
4775 	stmmac_get_rx_hwtstamp(priv, p, np, skb);
4776 	stmmac_rx_vlan(priv->dev, skb);
4777 	skb->protocol = eth_type_trans(skb, priv->dev);
4778 
4779 	if (unlikely(!coe))
4780 		skb_checksum_none_assert(skb);
4781 	else
4782 		skb->ip_summed = CHECKSUM_UNNECESSARY;
4783 
4784 	if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
4785 		skb_set_hash(skb, hash, hash_type);
4786 
4787 	skb_record_rx_queue(skb, queue);
4788 	napi_gro_receive(&ch->rxtx_napi, skb);
4789 
4790 	priv->dev->stats.rx_packets++;
4791 	priv->dev->stats.rx_bytes += len;
4792 }
4793 
4794 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
4795 {
4796 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4797 	unsigned int entry = rx_q->dirty_rx;
4798 	struct dma_desc *rx_desc = NULL;
4799 	bool ret = true;
4800 
4801 	budget = min(budget, stmmac_rx_dirty(priv, queue));
4802 
4803 	while (budget-- > 0 && entry != rx_q->cur_rx) {
4804 		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
4805 		dma_addr_t dma_addr;
4806 		bool use_rx_wd;
4807 
4808 		if (!buf->xdp) {
4809 			buf->xdp = xsk_buff_alloc(rx_q->xsk_pool);
4810 			if (!buf->xdp) {
4811 				ret = false;
4812 				break;
4813 			}
4814 		}
4815 
4816 		if (priv->extend_desc)
4817 			rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry);
4818 		else
4819 			rx_desc = rx_q->dma_rx + entry;
4820 
4821 		dma_addr = xsk_buff_xdp_get_dma(buf->xdp);
4822 		stmmac_set_desc_addr(priv, rx_desc, dma_addr);
4823 		stmmac_set_desc_sec_addr(priv, rx_desc, 0, false);
4824 		stmmac_refill_desc3(priv, rx_q, rx_desc);
4825 
4826 		rx_q->rx_count_frames++;
4827 		rx_q->rx_count_frames += priv->rx_coal_frames[queue];
4828 		if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])
4829 			rx_q->rx_count_frames = 0;
4830 
4831 		use_rx_wd = !priv->rx_coal_frames[queue];
4832 		use_rx_wd |= rx_q->rx_count_frames > 0;
4833 		if (!priv->use_riwt)
4834 			use_rx_wd = false;
4835 
4836 		dma_wmb();
4837 		stmmac_set_rx_owner(priv, rx_desc, use_rx_wd);
4838 
4839 		entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size);
4840 	}
4841 
4842 	if (rx_desc) {
4843 		rx_q->dirty_rx = entry;
4844 		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
4845 				     (rx_q->dirty_rx * sizeof(struct dma_desc));
4846 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
4847 	}
4848 
4849 	return ret;
4850 }
4851 
4852 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
4853 {
4854 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4855 	unsigned int count = 0, error = 0, len = 0;
4856 	int dirty = stmmac_rx_dirty(priv, queue);
4857 	unsigned int next_entry = rx_q->cur_rx;
4858 	unsigned int desc_size;
4859 	struct bpf_prog *prog;
4860 	bool failure = false;
4861 	int xdp_status = 0;
4862 	int status = 0;
4863 
4864 	if (netif_msg_rx_status(priv)) {
4865 		void *rx_head;
4866 
4867 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
4868 		if (priv->extend_desc) {
4869 			rx_head = (void *)rx_q->dma_erx;
4870 			desc_size = sizeof(struct dma_extended_desc);
4871 		} else {
4872 			rx_head = (void *)rx_q->dma_rx;
4873 			desc_size = sizeof(struct dma_desc);
4874 		}
4875 
4876 		stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true,
4877 				    rx_q->dma_rx_phy, desc_size);
4878 	}
4879 	while (count < limit) {
4880 		struct stmmac_rx_buffer *buf;
4881 		unsigned int buf1_len = 0;
4882 		struct dma_desc *np, *p;
4883 		int entry;
4884 		int res;
4885 
4886 		if (!count && rx_q->state_saved) {
4887 			error = rx_q->state.error;
4888 			len = rx_q->state.len;
4889 		} else {
4890 			rx_q->state_saved = false;
4891 			error = 0;
4892 			len = 0;
4893 		}
4894 
4895 		if (count >= limit)
4896 			break;
4897 
4898 read_again:
4899 		buf1_len = 0;
4900 		entry = next_entry;
4901 		buf = &rx_q->buf_pool[entry];
4902 
4903 		if (dirty >= STMMAC_RX_FILL_BATCH) {
4904 			failure = failure ||
4905 				  !stmmac_rx_refill_zc(priv, queue, dirty);
4906 			dirty = 0;
4907 		}
4908 
4909 		if (priv->extend_desc)
4910 			p = (struct dma_desc *)(rx_q->dma_erx + entry);
4911 		else
4912 			p = rx_q->dma_rx + entry;
4913 
4914 		/* read the status of the incoming frame */
4915 		status = stmmac_rx_status(priv, &priv->dev->stats,
4916 					  &priv->xstats, p);
4917 		/* check if managed by the DMA otherwise go ahead */
4918 		if (unlikely(status & dma_own))
4919 			break;
4920 
4921 		/* Prefetch the next RX descriptor */
4922 		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
4923 						priv->dma_rx_size);
4924 		next_entry = rx_q->cur_rx;
4925 
4926 		if (priv->extend_desc)
4927 			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
4928 		else
4929 			np = rx_q->dma_rx + next_entry;
4930 
4931 		prefetch(np);
4932 
4933 		/* Ensure a valid XSK buffer before proceed */
4934 		if (!buf->xdp)
4935 			break;
4936 
4937 		if (priv->extend_desc)
4938 			stmmac_rx_extended_status(priv, &priv->dev->stats,
4939 						  &priv->xstats,
4940 						  rx_q->dma_erx + entry);
4941 		if (unlikely(status == discard_frame)) {
4942 			xsk_buff_free(buf->xdp);
4943 			buf->xdp = NULL;
4944 			dirty++;
4945 			error = 1;
4946 			if (!priv->hwts_rx_en)
4947 				priv->dev->stats.rx_errors++;
4948 		}
4949 
4950 		if (unlikely(error && (status & rx_not_ls)))
4951 			goto read_again;
4952 		if (unlikely(error)) {
4953 			count++;
4954 			continue;
4955 		}
4956 
4957 		/* XSK pool expects RX frame 1:1 mapped to XSK buffer */
4958 		if (likely(status & rx_not_ls)) {
4959 			xsk_buff_free(buf->xdp);
4960 			buf->xdp = NULL;
4961 			dirty++;
4962 			count++;
4963 			goto read_again;
4964 		}
4965 
4966 		/* XDP ZC Frame only support primary buffers for now */
4967 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
4968 		len += buf1_len;
4969 
4970 		/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
4971 		 * Type frames (LLC/LLC-SNAP)
4972 		 *
4973 		 * llc_snap is never checked in GMAC >= 4, so this ACS
4974 		 * feature is always disabled and packets need to be
4975 		 * stripped manually.
4976 		 */
4977 		if (likely(!(status & rx_not_ls)) &&
4978 		    (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
4979 		     unlikely(status != llc_snap))) {
4980 			buf1_len -= ETH_FCS_LEN;
4981 			len -= ETH_FCS_LEN;
4982 		}
4983 
4984 		/* RX buffer is good and fit into a XSK pool buffer */
4985 		buf->xdp->data_end = buf->xdp->data + buf1_len;
4986 		xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool);
4987 
4988 		prog = READ_ONCE(priv->xdp_prog);
4989 		res = __stmmac_xdp_run_prog(priv, prog, buf->xdp);
4990 
4991 		switch (res) {
4992 		case STMMAC_XDP_PASS:
4993 			stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp);
4994 			xsk_buff_free(buf->xdp);
4995 			break;
4996 		case STMMAC_XDP_CONSUMED:
4997 			xsk_buff_free(buf->xdp);
4998 			priv->dev->stats.rx_dropped++;
4999 			break;
5000 		case STMMAC_XDP_TX:
5001 		case STMMAC_XDP_REDIRECT:
5002 			xdp_status |= res;
5003 			break;
5004 		}
5005 
5006 		buf->xdp = NULL;
5007 		dirty++;
5008 		count++;
5009 	}
5010 
5011 	if (status & rx_not_ls) {
5012 		rx_q->state_saved = true;
5013 		rx_q->state.error = error;
5014 		rx_q->state.len = len;
5015 	}
5016 
5017 	stmmac_finalize_xdp_rx(priv, xdp_status);
5018 
5019 	priv->xstats.rx_pkt_n += count;
5020 	priv->xstats.rxq_stats[queue].rx_pkt_n += count;
5021 
5022 	if (xsk_uses_need_wakeup(rx_q->xsk_pool)) {
5023 		if (failure || stmmac_rx_dirty(priv, queue) > 0)
5024 			xsk_set_rx_need_wakeup(rx_q->xsk_pool);
5025 		else
5026 			xsk_clear_rx_need_wakeup(rx_q->xsk_pool);
5027 
5028 		return (int)count;
5029 	}
5030 
5031 	return failure ? limit : (int)count;
5032 }
5033 
5034 /**
5035  * stmmac_rx - manage the receive process
5036  * @priv: driver private structure
5037  * @limit: napi bugget
5038  * @queue: RX queue index.
5039  * Description :  this the function called by the napi poll method.
5040  * It gets all the frames inside the ring.
5041  */
5042 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
5043 {
5044 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
5045 	struct stmmac_channel *ch = &priv->channel[queue];
5046 	unsigned int count = 0, error = 0, len = 0;
5047 	int status = 0, coe = priv->hw->rx_csum;
5048 	unsigned int next_entry = rx_q->cur_rx;
5049 	enum dma_data_direction dma_dir;
5050 	unsigned int desc_size;
5051 	struct sk_buff *skb = NULL;
5052 	struct xdp_buff xdp;
5053 	int xdp_status = 0;
5054 	int buf_sz;
5055 
5056 	dma_dir = page_pool_get_dma_dir(rx_q->page_pool);
5057 	buf_sz = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
5058 
5059 	if (netif_msg_rx_status(priv)) {
5060 		void *rx_head;
5061 
5062 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
5063 		if (priv->extend_desc) {
5064 			rx_head = (void *)rx_q->dma_erx;
5065 			desc_size = sizeof(struct dma_extended_desc);
5066 		} else {
5067 			rx_head = (void *)rx_q->dma_rx;
5068 			desc_size = sizeof(struct dma_desc);
5069 		}
5070 
5071 		stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true,
5072 				    rx_q->dma_rx_phy, desc_size);
5073 	}
5074 	while (count < limit) {
5075 		unsigned int buf1_len = 0, buf2_len = 0;
5076 		enum pkt_hash_types hash_type;
5077 		struct stmmac_rx_buffer *buf;
5078 		struct dma_desc *np, *p;
5079 		int entry;
5080 		u32 hash;
5081 
5082 		if (!count && rx_q->state_saved) {
5083 			skb = rx_q->state.skb;
5084 			error = rx_q->state.error;
5085 			len = rx_q->state.len;
5086 		} else {
5087 			rx_q->state_saved = false;
5088 			skb = NULL;
5089 			error = 0;
5090 			len = 0;
5091 		}
5092 
5093 		if (count >= limit)
5094 			break;
5095 
5096 read_again:
5097 		buf1_len = 0;
5098 		buf2_len = 0;
5099 		entry = next_entry;
5100 		buf = &rx_q->buf_pool[entry];
5101 
5102 		if (priv->extend_desc)
5103 			p = (struct dma_desc *)(rx_q->dma_erx + entry);
5104 		else
5105 			p = rx_q->dma_rx + entry;
5106 
5107 		/* read the status of the incoming frame */
5108 		status = stmmac_rx_status(priv, &priv->dev->stats,
5109 				&priv->xstats, p);
5110 		/* check if managed by the DMA otherwise go ahead */
5111 		if (unlikely(status & dma_own))
5112 			break;
5113 
5114 		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
5115 						priv->dma_rx_size);
5116 		next_entry = rx_q->cur_rx;
5117 
5118 		if (priv->extend_desc)
5119 			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
5120 		else
5121 			np = rx_q->dma_rx + next_entry;
5122 
5123 		prefetch(np);
5124 
5125 		if (priv->extend_desc)
5126 			stmmac_rx_extended_status(priv, &priv->dev->stats,
5127 					&priv->xstats, rx_q->dma_erx + entry);
5128 		if (unlikely(status == discard_frame)) {
5129 			page_pool_recycle_direct(rx_q->page_pool, buf->page);
5130 			buf->page = NULL;
5131 			error = 1;
5132 			if (!priv->hwts_rx_en)
5133 				priv->dev->stats.rx_errors++;
5134 		}
5135 
5136 		if (unlikely(error && (status & rx_not_ls)))
5137 			goto read_again;
5138 		if (unlikely(error)) {
5139 			dev_kfree_skb(skb);
5140 			skb = NULL;
5141 			count++;
5142 			continue;
5143 		}
5144 
5145 		/* Buffer is good. Go on. */
5146 
5147 		prefetch(page_address(buf->page) + buf->page_offset);
5148 		if (buf->sec_page)
5149 			prefetch(page_address(buf->sec_page));
5150 
5151 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
5152 		len += buf1_len;
5153 		buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
5154 		len += buf2_len;
5155 
5156 		/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
5157 		 * Type frames (LLC/LLC-SNAP)
5158 		 *
5159 		 * llc_snap is never checked in GMAC >= 4, so this ACS
5160 		 * feature is always disabled and packets need to be
5161 		 * stripped manually.
5162 		 */
5163 		if (likely(!(status & rx_not_ls)) &&
5164 		    (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
5165 		     unlikely(status != llc_snap))) {
5166 			if (buf2_len) {
5167 				buf2_len -= ETH_FCS_LEN;
5168 				len -= ETH_FCS_LEN;
5169 			} else if (buf1_len) {
5170 				buf1_len -= ETH_FCS_LEN;
5171 				len -= ETH_FCS_LEN;
5172 			}
5173 		}
5174 
5175 		if (!skb) {
5176 			unsigned int pre_len, sync_len;
5177 
5178 			dma_sync_single_for_cpu(priv->device, buf->addr,
5179 						buf1_len, dma_dir);
5180 
5181 			xdp_init_buff(&xdp, buf_sz, &rx_q->xdp_rxq);
5182 			xdp_prepare_buff(&xdp, page_address(buf->page),
5183 					 buf->page_offset, buf1_len, false);
5184 
5185 			pre_len = xdp.data_end - xdp.data_hard_start -
5186 				  buf->page_offset;
5187 			skb = stmmac_xdp_run_prog(priv, &xdp);
5188 			/* Due xdp_adjust_tail: DMA sync for_device
5189 			 * cover max len CPU touch
5190 			 */
5191 			sync_len = xdp.data_end - xdp.data_hard_start -
5192 				   buf->page_offset;
5193 			sync_len = max(sync_len, pre_len);
5194 
5195 			/* For Not XDP_PASS verdict */
5196 			if (IS_ERR(skb)) {
5197 				unsigned int xdp_res = -PTR_ERR(skb);
5198 
5199 				if (xdp_res & STMMAC_XDP_CONSUMED) {
5200 					page_pool_put_page(rx_q->page_pool,
5201 							   virt_to_head_page(xdp.data),
5202 							   sync_len, true);
5203 					buf->page = NULL;
5204 					priv->dev->stats.rx_dropped++;
5205 
5206 					/* Clear skb as it was set as
5207 					 * status by XDP program.
5208 					 */
5209 					skb = NULL;
5210 
5211 					if (unlikely((status & rx_not_ls)))
5212 						goto read_again;
5213 
5214 					count++;
5215 					continue;
5216 				} else if (xdp_res & (STMMAC_XDP_TX |
5217 						      STMMAC_XDP_REDIRECT)) {
5218 					xdp_status |= xdp_res;
5219 					buf->page = NULL;
5220 					skb = NULL;
5221 					count++;
5222 					continue;
5223 				}
5224 			}
5225 		}
5226 
5227 		if (!skb) {
5228 			/* XDP program may expand or reduce tail */
5229 			buf1_len = xdp.data_end - xdp.data;
5230 
5231 			skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
5232 			if (!skb) {
5233 				priv->dev->stats.rx_dropped++;
5234 				count++;
5235 				goto drain_data;
5236 			}
5237 
5238 			/* XDP program may adjust header */
5239 			skb_copy_to_linear_data(skb, xdp.data, buf1_len);
5240 			skb_put(skb, buf1_len);
5241 
5242 			/* Data payload copied into SKB, page ready for recycle */
5243 			page_pool_recycle_direct(rx_q->page_pool, buf->page);
5244 			buf->page = NULL;
5245 		} else if (buf1_len) {
5246 			dma_sync_single_for_cpu(priv->device, buf->addr,
5247 						buf1_len, dma_dir);
5248 			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
5249 					buf->page, buf->page_offset, buf1_len,
5250 					priv->dma_buf_sz);
5251 
5252 			/* Data payload appended into SKB */
5253 			page_pool_release_page(rx_q->page_pool, buf->page);
5254 			buf->page = NULL;
5255 		}
5256 
5257 		if (buf2_len) {
5258 			dma_sync_single_for_cpu(priv->device, buf->sec_addr,
5259 						buf2_len, dma_dir);
5260 			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
5261 					buf->sec_page, 0, buf2_len,
5262 					priv->dma_buf_sz);
5263 
5264 			/* Data payload appended into SKB */
5265 			page_pool_release_page(rx_q->page_pool, buf->sec_page);
5266 			buf->sec_page = NULL;
5267 		}
5268 
5269 drain_data:
5270 		if (likely(status & rx_not_ls))
5271 			goto read_again;
5272 		if (!skb)
5273 			continue;
5274 
5275 		/* Got entire packet into SKB. Finish it. */
5276 
5277 		stmmac_get_rx_hwtstamp(priv, p, np, skb);
5278 		stmmac_rx_vlan(priv->dev, skb);
5279 		skb->protocol = eth_type_trans(skb, priv->dev);
5280 
5281 		if (unlikely(!coe))
5282 			skb_checksum_none_assert(skb);
5283 		else
5284 			skb->ip_summed = CHECKSUM_UNNECESSARY;
5285 
5286 		if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
5287 			skb_set_hash(skb, hash, hash_type);
5288 
5289 		skb_record_rx_queue(skb, queue);
5290 		napi_gro_receive(&ch->rx_napi, skb);
5291 		skb = NULL;
5292 
5293 		priv->dev->stats.rx_packets++;
5294 		priv->dev->stats.rx_bytes += len;
5295 		count++;
5296 	}
5297 
5298 	if (status & rx_not_ls || skb) {
5299 		rx_q->state_saved = true;
5300 		rx_q->state.skb = skb;
5301 		rx_q->state.error = error;
5302 		rx_q->state.len = len;
5303 	}
5304 
5305 	stmmac_finalize_xdp_rx(priv, xdp_status);
5306 
5307 	stmmac_rx_refill(priv, queue);
5308 
5309 	priv->xstats.rx_pkt_n += count;
5310 	priv->xstats.rxq_stats[queue].rx_pkt_n += count;
5311 
5312 	return count;
5313 }
5314 
5315 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
5316 {
5317 	struct stmmac_channel *ch =
5318 		container_of(napi, struct stmmac_channel, rx_napi);
5319 	struct stmmac_priv *priv = ch->priv_data;
5320 	u32 chan = ch->index;
5321 	int work_done;
5322 
5323 	priv->xstats.napi_poll++;
5324 
5325 	work_done = stmmac_rx(priv, budget, chan);
5326 	if (work_done < budget && napi_complete_done(napi, work_done)) {
5327 		unsigned long flags;
5328 
5329 		spin_lock_irqsave(&ch->lock, flags);
5330 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
5331 		spin_unlock_irqrestore(&ch->lock, flags);
5332 	}
5333 
5334 	return work_done;
5335 }
5336 
5337 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
5338 {
5339 	struct stmmac_channel *ch =
5340 		container_of(napi, struct stmmac_channel, tx_napi);
5341 	struct stmmac_priv *priv = ch->priv_data;
5342 	u32 chan = ch->index;
5343 	int work_done;
5344 
5345 	priv->xstats.napi_poll++;
5346 
5347 	work_done = stmmac_tx_clean(priv, budget, chan);
5348 	work_done = min(work_done, budget);
5349 
5350 	if (work_done < budget && napi_complete_done(napi, work_done)) {
5351 		unsigned long flags;
5352 
5353 		spin_lock_irqsave(&ch->lock, flags);
5354 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
5355 		spin_unlock_irqrestore(&ch->lock, flags);
5356 	}
5357 
5358 	return work_done;
5359 }
5360 
5361 static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget)
5362 {
5363 	struct stmmac_channel *ch =
5364 		container_of(napi, struct stmmac_channel, rxtx_napi);
5365 	struct stmmac_priv *priv = ch->priv_data;
5366 	int rx_done, tx_done, rxtx_done;
5367 	u32 chan = ch->index;
5368 
5369 	priv->xstats.napi_poll++;
5370 
5371 	tx_done = stmmac_tx_clean(priv, budget, chan);
5372 	tx_done = min(tx_done, budget);
5373 
5374 	rx_done = stmmac_rx_zc(priv, budget, chan);
5375 
5376 	rxtx_done = max(tx_done, rx_done);
5377 
5378 	/* If either TX or RX work is not complete, return budget
5379 	 * and keep pooling
5380 	 */
5381 	if (rxtx_done >= budget)
5382 		return budget;
5383 
5384 	/* all work done, exit the polling mode */
5385 	if (napi_complete_done(napi, rxtx_done)) {
5386 		unsigned long flags;
5387 
5388 		spin_lock_irqsave(&ch->lock, flags);
5389 		/* Both RX and TX work done are compelte,
5390 		 * so enable both RX & TX IRQs.
5391 		 */
5392 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
5393 		spin_unlock_irqrestore(&ch->lock, flags);
5394 	}
5395 
5396 	return min(rxtx_done, budget - 1);
5397 }
5398 
5399 /**
5400  *  stmmac_tx_timeout
5401  *  @dev : Pointer to net device structure
5402  *  @txqueue: the index of the hanging transmit queue
5403  *  Description: this function is called when a packet transmission fails to
5404  *   complete within a reasonable time. The driver will mark the error in the
5405  *   netdev structure and arrange for the device to be reset to a sane state
5406  *   in order to transmit a new packet.
5407  */
5408 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue)
5409 {
5410 	struct stmmac_priv *priv = netdev_priv(dev);
5411 
5412 	stmmac_global_err(priv);
5413 }
5414 
5415 /**
5416  *  stmmac_set_rx_mode - entry point for multicast addressing
5417  *  @dev : pointer to the device structure
5418  *  Description:
5419  *  This function is a driver entry point which gets called by the kernel
5420  *  whenever multicast addresses must be enabled/disabled.
5421  *  Return value:
5422  *  void.
5423  */
5424 static void stmmac_set_rx_mode(struct net_device *dev)
5425 {
5426 	struct stmmac_priv *priv = netdev_priv(dev);
5427 
5428 	stmmac_set_filter(priv, priv->hw, dev);
5429 }
5430 
5431 /**
5432  *  stmmac_change_mtu - entry point to change MTU size for the device.
5433  *  @dev : device pointer.
5434  *  @new_mtu : the new MTU size for the device.
5435  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
5436  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
5437  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
5438  *  Return value:
5439  *  0 on success and an appropriate (-)ve integer as defined in errno.h
5440  *  file on failure.
5441  */
5442 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
5443 {
5444 	struct stmmac_priv *priv = netdev_priv(dev);
5445 	int txfifosz = priv->plat->tx_fifo_size;
5446 	const int mtu = new_mtu;
5447 
5448 	if (txfifosz == 0)
5449 		txfifosz = priv->dma_cap.tx_fifo_size;
5450 
5451 	txfifosz /= priv->plat->tx_queues_to_use;
5452 
5453 	if (netif_running(dev)) {
5454 		netdev_err(priv->dev, "must be stopped to change its MTU\n");
5455 		return -EBUSY;
5456 	}
5457 
5458 	if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) {
5459 		netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n");
5460 		return -EINVAL;
5461 	}
5462 
5463 	new_mtu = STMMAC_ALIGN(new_mtu);
5464 
5465 	/* If condition true, FIFO is too small or MTU too large */
5466 	if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
5467 		return -EINVAL;
5468 
5469 	dev->mtu = mtu;
5470 
5471 	netdev_update_features(dev);
5472 
5473 	return 0;
5474 }
5475 
5476 static netdev_features_t stmmac_fix_features(struct net_device *dev,
5477 					     netdev_features_t features)
5478 {
5479 	struct stmmac_priv *priv = netdev_priv(dev);
5480 
5481 	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
5482 		features &= ~NETIF_F_RXCSUM;
5483 
5484 	if (!priv->plat->tx_coe)
5485 		features &= ~NETIF_F_CSUM_MASK;
5486 
5487 	/* Some GMAC devices have a bugged Jumbo frame support that
5488 	 * needs to have the Tx COE disabled for oversized frames
5489 	 * (due to limited buffer sizes). In this case we disable
5490 	 * the TX csum insertion in the TDES and not use SF.
5491 	 */
5492 	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
5493 		features &= ~NETIF_F_CSUM_MASK;
5494 
5495 	/* Disable tso if asked by ethtool */
5496 	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
5497 		if (features & NETIF_F_TSO)
5498 			priv->tso = true;
5499 		else
5500 			priv->tso = false;
5501 	}
5502 
5503 	return features;
5504 }
5505 
5506 static int stmmac_set_features(struct net_device *netdev,
5507 			       netdev_features_t features)
5508 {
5509 	struct stmmac_priv *priv = netdev_priv(netdev);
5510 
5511 	/* Keep the COE Type in case of csum is supporting */
5512 	if (features & NETIF_F_RXCSUM)
5513 		priv->hw->rx_csum = priv->plat->rx_coe;
5514 	else
5515 		priv->hw->rx_csum = 0;
5516 	/* No check needed because rx_coe has been set before and it will be
5517 	 * fixed in case of issue.
5518 	 */
5519 	stmmac_rx_ipc(priv, priv->hw);
5520 
5521 	if (priv->sph_cap) {
5522 		bool sph_en = (priv->hw->rx_csum > 0) && priv->sph;
5523 		u32 chan;
5524 
5525 		for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
5526 			stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
5527 	}
5528 
5529 	return 0;
5530 }
5531 
5532 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status)
5533 {
5534 	struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg;
5535 	enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state;
5536 	enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state;
5537 	bool *hs_enable = &fpe_cfg->hs_enable;
5538 
5539 	if (status == FPE_EVENT_UNKNOWN || !*hs_enable)
5540 		return;
5541 
5542 	/* If LP has sent verify mPacket, LP is FPE capable */
5543 	if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER) {
5544 		if (*lp_state < FPE_STATE_CAPABLE)
5545 			*lp_state = FPE_STATE_CAPABLE;
5546 
5547 		/* If user has requested FPE enable, quickly response */
5548 		if (*hs_enable)
5549 			stmmac_fpe_send_mpacket(priv, priv->ioaddr,
5550 						MPACKET_RESPONSE);
5551 	}
5552 
5553 	/* If Local has sent verify mPacket, Local is FPE capable */
5554 	if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER) {
5555 		if (*lo_state < FPE_STATE_CAPABLE)
5556 			*lo_state = FPE_STATE_CAPABLE;
5557 	}
5558 
5559 	/* If LP has sent response mPacket, LP is entering FPE ON */
5560 	if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP)
5561 		*lp_state = FPE_STATE_ENTERING_ON;
5562 
5563 	/* If Local has sent response mPacket, Local is entering FPE ON */
5564 	if ((status & FPE_EVENT_TRSP) == FPE_EVENT_TRSP)
5565 		*lo_state = FPE_STATE_ENTERING_ON;
5566 
5567 	if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) &&
5568 	    !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) &&
5569 	    priv->fpe_wq) {
5570 		queue_work(priv->fpe_wq, &priv->fpe_task);
5571 	}
5572 }
5573 
5574 static void stmmac_common_interrupt(struct stmmac_priv *priv)
5575 {
5576 	u32 rx_cnt = priv->plat->rx_queues_to_use;
5577 	u32 tx_cnt = priv->plat->tx_queues_to_use;
5578 	u32 queues_count;
5579 	u32 queue;
5580 	bool xmac;
5581 
5582 	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
5583 	queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
5584 
5585 	if (priv->irq_wake)
5586 		pm_wakeup_event(priv->device, 0);
5587 
5588 	if (priv->dma_cap.estsel)
5589 		stmmac_est_irq_status(priv, priv->ioaddr, priv->dev,
5590 				      &priv->xstats, tx_cnt);
5591 
5592 	if (priv->dma_cap.fpesel) {
5593 		int status = stmmac_fpe_irq_status(priv, priv->ioaddr,
5594 						   priv->dev);
5595 
5596 		stmmac_fpe_event_status(priv, status);
5597 	}
5598 
5599 	/* To handle GMAC own interrupts */
5600 	if ((priv->plat->has_gmac) || xmac) {
5601 		int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
5602 
5603 		if (unlikely(status)) {
5604 			/* For LPI we need to save the tx status */
5605 			if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
5606 				priv->tx_path_in_lpi_mode = true;
5607 			if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
5608 				priv->tx_path_in_lpi_mode = false;
5609 		}
5610 
5611 		for (queue = 0; queue < queues_count; queue++) {
5612 			status = stmmac_host_mtl_irq_status(priv, priv->hw,
5613 							    queue);
5614 		}
5615 
5616 		/* PCS link status */
5617 		if (priv->hw->pcs) {
5618 			if (priv->xstats.pcs_link)
5619 				netif_carrier_on(priv->dev);
5620 			else
5621 				netif_carrier_off(priv->dev);
5622 		}
5623 
5624 		stmmac_timestamp_interrupt(priv, priv);
5625 	}
5626 }
5627 
5628 /**
5629  *  stmmac_interrupt - main ISR
5630  *  @irq: interrupt number.
5631  *  @dev_id: to pass the net device pointer.
5632  *  Description: this is the main driver interrupt service routine.
5633  *  It can call:
5634  *  o DMA service routine (to manage incoming frame reception and transmission
5635  *    status)
5636  *  o Core interrupts to manage: remote wake-up, management counter, LPI
5637  *    interrupts.
5638  */
5639 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
5640 {
5641 	struct net_device *dev = (struct net_device *)dev_id;
5642 	struct stmmac_priv *priv = netdev_priv(dev);
5643 
5644 	/* Check if adapter is up */
5645 	if (test_bit(STMMAC_DOWN, &priv->state))
5646 		return IRQ_HANDLED;
5647 
5648 	/* Check if a fatal error happened */
5649 	if (stmmac_safety_feat_interrupt(priv))
5650 		return IRQ_HANDLED;
5651 
5652 	/* To handle Common interrupts */
5653 	stmmac_common_interrupt(priv);
5654 
5655 	/* To handle DMA interrupts */
5656 	stmmac_dma_interrupt(priv);
5657 
5658 	return IRQ_HANDLED;
5659 }
5660 
5661 static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id)
5662 {
5663 	struct net_device *dev = (struct net_device *)dev_id;
5664 	struct stmmac_priv *priv = netdev_priv(dev);
5665 
5666 	if (unlikely(!dev)) {
5667 		netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
5668 		return IRQ_NONE;
5669 	}
5670 
5671 	/* Check if adapter is up */
5672 	if (test_bit(STMMAC_DOWN, &priv->state))
5673 		return IRQ_HANDLED;
5674 
5675 	/* To handle Common interrupts */
5676 	stmmac_common_interrupt(priv);
5677 
5678 	return IRQ_HANDLED;
5679 }
5680 
5681 static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id)
5682 {
5683 	struct net_device *dev = (struct net_device *)dev_id;
5684 	struct stmmac_priv *priv = netdev_priv(dev);
5685 
5686 	if (unlikely(!dev)) {
5687 		netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
5688 		return IRQ_NONE;
5689 	}
5690 
5691 	/* Check if adapter is up */
5692 	if (test_bit(STMMAC_DOWN, &priv->state))
5693 		return IRQ_HANDLED;
5694 
5695 	/* Check if a fatal error happened */
5696 	stmmac_safety_feat_interrupt(priv);
5697 
5698 	return IRQ_HANDLED;
5699 }
5700 
5701 static irqreturn_t stmmac_msi_intr_tx(int irq, void *data)
5702 {
5703 	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data;
5704 	int chan = tx_q->queue_index;
5705 	struct stmmac_priv *priv;
5706 	int status;
5707 
5708 	priv = container_of(tx_q, struct stmmac_priv, tx_queue[chan]);
5709 
5710 	if (unlikely(!data)) {
5711 		netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
5712 		return IRQ_NONE;
5713 	}
5714 
5715 	/* Check if adapter is up */
5716 	if (test_bit(STMMAC_DOWN, &priv->state))
5717 		return IRQ_HANDLED;
5718 
5719 	status = stmmac_napi_check(priv, chan, DMA_DIR_TX);
5720 
5721 	if (unlikely(status & tx_hard_error_bump_tc)) {
5722 		/* Try to bump up the dma threshold on this failure */
5723 		stmmac_bump_dma_threshold(priv, chan);
5724 	} else if (unlikely(status == tx_hard_error)) {
5725 		stmmac_tx_err(priv, chan);
5726 	}
5727 
5728 	return IRQ_HANDLED;
5729 }
5730 
5731 static irqreturn_t stmmac_msi_intr_rx(int irq, void *data)
5732 {
5733 	struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data;
5734 	int chan = rx_q->queue_index;
5735 	struct stmmac_priv *priv;
5736 
5737 	priv = container_of(rx_q, struct stmmac_priv, rx_queue[chan]);
5738 
5739 	if (unlikely(!data)) {
5740 		netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
5741 		return IRQ_NONE;
5742 	}
5743 
5744 	/* Check if adapter is up */
5745 	if (test_bit(STMMAC_DOWN, &priv->state))
5746 		return IRQ_HANDLED;
5747 
5748 	stmmac_napi_check(priv, chan, DMA_DIR_RX);
5749 
5750 	return IRQ_HANDLED;
5751 }
5752 
5753 #ifdef CONFIG_NET_POLL_CONTROLLER
5754 /* Polling receive - used by NETCONSOLE and other diagnostic tools
5755  * to allow network I/O with interrupts disabled.
5756  */
5757 static void stmmac_poll_controller(struct net_device *dev)
5758 {
5759 	struct stmmac_priv *priv = netdev_priv(dev);
5760 	int i;
5761 
5762 	/* If adapter is down, do nothing */
5763 	if (test_bit(STMMAC_DOWN, &priv->state))
5764 		return;
5765 
5766 	if (priv->plat->multi_msi_en) {
5767 		for (i = 0; i < priv->plat->rx_queues_to_use; i++)
5768 			stmmac_msi_intr_rx(0, &priv->rx_queue[i]);
5769 
5770 		for (i = 0; i < priv->plat->tx_queues_to_use; i++)
5771 			stmmac_msi_intr_tx(0, &priv->tx_queue[i]);
5772 	} else {
5773 		disable_irq(dev->irq);
5774 		stmmac_interrupt(dev->irq, dev);
5775 		enable_irq(dev->irq);
5776 	}
5777 }
5778 #endif
5779 
5780 /**
5781  *  stmmac_ioctl - Entry point for the Ioctl
5782  *  @dev: Device pointer.
5783  *  @rq: An IOCTL specefic structure, that can contain a pointer to
5784  *  a proprietary structure used to pass information to the driver.
5785  *  @cmd: IOCTL command
5786  *  Description:
5787  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
5788  */
5789 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
5790 {
5791 	struct stmmac_priv *priv = netdev_priv (dev);
5792 	int ret = -EOPNOTSUPP;
5793 
5794 	if (!netif_running(dev))
5795 		return -EINVAL;
5796 
5797 	switch (cmd) {
5798 	case SIOCGMIIPHY:
5799 	case SIOCGMIIREG:
5800 	case SIOCSMIIREG:
5801 		ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
5802 		break;
5803 	case SIOCSHWTSTAMP:
5804 		ret = stmmac_hwtstamp_set(dev, rq);
5805 		break;
5806 	case SIOCGHWTSTAMP:
5807 		ret = stmmac_hwtstamp_get(dev, rq);
5808 		break;
5809 	default:
5810 		break;
5811 	}
5812 
5813 	return ret;
5814 }
5815 
5816 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
5817 				    void *cb_priv)
5818 {
5819 	struct stmmac_priv *priv = cb_priv;
5820 	int ret = -EOPNOTSUPP;
5821 
5822 	if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
5823 		return ret;
5824 
5825 	__stmmac_disable_all_queues(priv);
5826 
5827 	switch (type) {
5828 	case TC_SETUP_CLSU32:
5829 		ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
5830 		break;
5831 	case TC_SETUP_CLSFLOWER:
5832 		ret = stmmac_tc_setup_cls(priv, priv, type_data);
5833 		break;
5834 	default:
5835 		break;
5836 	}
5837 
5838 	stmmac_enable_all_queues(priv);
5839 	return ret;
5840 }
5841 
5842 static LIST_HEAD(stmmac_block_cb_list);
5843 
5844 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
5845 			   void *type_data)
5846 {
5847 	struct stmmac_priv *priv = netdev_priv(ndev);
5848 
5849 	switch (type) {
5850 	case TC_SETUP_BLOCK:
5851 		return flow_block_cb_setup_simple(type_data,
5852 						  &stmmac_block_cb_list,
5853 						  stmmac_setup_tc_block_cb,
5854 						  priv, priv, true);
5855 	case TC_SETUP_QDISC_CBS:
5856 		return stmmac_tc_setup_cbs(priv, priv, type_data);
5857 	case TC_SETUP_QDISC_TAPRIO:
5858 		return stmmac_tc_setup_taprio(priv, priv, type_data);
5859 	case TC_SETUP_QDISC_ETF:
5860 		return stmmac_tc_setup_etf(priv, priv, type_data);
5861 	default:
5862 		return -EOPNOTSUPP;
5863 	}
5864 }
5865 
5866 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
5867 			       struct net_device *sb_dev)
5868 {
5869 	int gso = skb_shinfo(skb)->gso_type;
5870 
5871 	if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
5872 		/*
5873 		 * There is no way to determine the number of TSO/USO
5874 		 * capable Queues. Let's use always the Queue 0
5875 		 * because if TSO/USO is supported then at least this
5876 		 * one will be capable.
5877 		 */
5878 		return 0;
5879 	}
5880 
5881 	return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
5882 }
5883 
5884 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
5885 {
5886 	struct stmmac_priv *priv = netdev_priv(ndev);
5887 	int ret = 0;
5888 
5889 	ret = pm_runtime_get_sync(priv->device);
5890 	if (ret < 0) {
5891 		pm_runtime_put_noidle(priv->device);
5892 		return ret;
5893 	}
5894 
5895 	ret = eth_mac_addr(ndev, addr);
5896 	if (ret)
5897 		goto set_mac_error;
5898 
5899 	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
5900 
5901 set_mac_error:
5902 	pm_runtime_put(priv->device);
5903 
5904 	return ret;
5905 }
5906 
5907 #ifdef CONFIG_DEBUG_FS
5908 static struct dentry *stmmac_fs_dir;
5909 
5910 static void sysfs_display_ring(void *head, int size, int extend_desc,
5911 			       struct seq_file *seq, dma_addr_t dma_phy_addr)
5912 {
5913 	int i;
5914 	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
5915 	struct dma_desc *p = (struct dma_desc *)head;
5916 	dma_addr_t dma_addr;
5917 
5918 	for (i = 0; i < size; i++) {
5919 		if (extend_desc) {
5920 			dma_addr = dma_phy_addr + i * sizeof(*ep);
5921 			seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
5922 				   i, &dma_addr,
5923 				   le32_to_cpu(ep->basic.des0),
5924 				   le32_to_cpu(ep->basic.des1),
5925 				   le32_to_cpu(ep->basic.des2),
5926 				   le32_to_cpu(ep->basic.des3));
5927 			ep++;
5928 		} else {
5929 			dma_addr = dma_phy_addr + i * sizeof(*p);
5930 			seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
5931 				   i, &dma_addr,
5932 				   le32_to_cpu(p->des0), le32_to_cpu(p->des1),
5933 				   le32_to_cpu(p->des2), le32_to_cpu(p->des3));
5934 			p++;
5935 		}
5936 		seq_printf(seq, "\n");
5937 	}
5938 }
5939 
5940 static int stmmac_rings_status_show(struct seq_file *seq, void *v)
5941 {
5942 	struct net_device *dev = seq->private;
5943 	struct stmmac_priv *priv = netdev_priv(dev);
5944 	u32 rx_count = priv->plat->rx_queues_to_use;
5945 	u32 tx_count = priv->plat->tx_queues_to_use;
5946 	u32 queue;
5947 
5948 	if ((dev->flags & IFF_UP) == 0)
5949 		return 0;
5950 
5951 	for (queue = 0; queue < rx_count; queue++) {
5952 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
5953 
5954 		seq_printf(seq, "RX Queue %d:\n", queue);
5955 
5956 		if (priv->extend_desc) {
5957 			seq_printf(seq, "Extended descriptor ring:\n");
5958 			sysfs_display_ring((void *)rx_q->dma_erx,
5959 					   priv->dma_rx_size, 1, seq, rx_q->dma_rx_phy);
5960 		} else {
5961 			seq_printf(seq, "Descriptor ring:\n");
5962 			sysfs_display_ring((void *)rx_q->dma_rx,
5963 					   priv->dma_rx_size, 0, seq, rx_q->dma_rx_phy);
5964 		}
5965 	}
5966 
5967 	for (queue = 0; queue < tx_count; queue++) {
5968 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
5969 
5970 		seq_printf(seq, "TX Queue %d:\n", queue);
5971 
5972 		if (priv->extend_desc) {
5973 			seq_printf(seq, "Extended descriptor ring:\n");
5974 			sysfs_display_ring((void *)tx_q->dma_etx,
5975 					   priv->dma_tx_size, 1, seq, tx_q->dma_tx_phy);
5976 		} else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) {
5977 			seq_printf(seq, "Descriptor ring:\n");
5978 			sysfs_display_ring((void *)tx_q->dma_tx,
5979 					   priv->dma_tx_size, 0, seq, tx_q->dma_tx_phy);
5980 		}
5981 	}
5982 
5983 	return 0;
5984 }
5985 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
5986 
5987 static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
5988 {
5989 	struct net_device *dev = seq->private;
5990 	struct stmmac_priv *priv = netdev_priv(dev);
5991 
5992 	if (!priv->hw_cap_support) {
5993 		seq_printf(seq, "DMA HW features not supported\n");
5994 		return 0;
5995 	}
5996 
5997 	seq_printf(seq, "==============================\n");
5998 	seq_printf(seq, "\tDMA HW features\n");
5999 	seq_printf(seq, "==============================\n");
6000 
6001 	seq_printf(seq, "\t10/100 Mbps: %s\n",
6002 		   (priv->dma_cap.mbps_10_100) ? "Y" : "N");
6003 	seq_printf(seq, "\t1000 Mbps: %s\n",
6004 		   (priv->dma_cap.mbps_1000) ? "Y" : "N");
6005 	seq_printf(seq, "\tHalf duplex: %s\n",
6006 		   (priv->dma_cap.half_duplex) ? "Y" : "N");
6007 	seq_printf(seq, "\tHash Filter: %s\n",
6008 		   (priv->dma_cap.hash_filter) ? "Y" : "N");
6009 	seq_printf(seq, "\tMultiple MAC address registers: %s\n",
6010 		   (priv->dma_cap.multi_addr) ? "Y" : "N");
6011 	seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
6012 		   (priv->dma_cap.pcs) ? "Y" : "N");
6013 	seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
6014 		   (priv->dma_cap.sma_mdio) ? "Y" : "N");
6015 	seq_printf(seq, "\tPMT Remote wake up: %s\n",
6016 		   (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
6017 	seq_printf(seq, "\tPMT Magic Frame: %s\n",
6018 		   (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
6019 	seq_printf(seq, "\tRMON module: %s\n",
6020 		   (priv->dma_cap.rmon) ? "Y" : "N");
6021 	seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
6022 		   (priv->dma_cap.time_stamp) ? "Y" : "N");
6023 	seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
6024 		   (priv->dma_cap.atime_stamp) ? "Y" : "N");
6025 	seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
6026 		   (priv->dma_cap.eee) ? "Y" : "N");
6027 	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
6028 	seq_printf(seq, "\tChecksum Offload in TX: %s\n",
6029 		   (priv->dma_cap.tx_coe) ? "Y" : "N");
6030 	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
6031 		seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
6032 			   (priv->dma_cap.rx_coe) ? "Y" : "N");
6033 	} else {
6034 		seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
6035 			   (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
6036 		seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
6037 			   (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
6038 	}
6039 	seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
6040 		   (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
6041 	seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
6042 		   priv->dma_cap.number_rx_channel);
6043 	seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
6044 		   priv->dma_cap.number_tx_channel);
6045 	seq_printf(seq, "\tNumber of Additional RX queues: %d\n",
6046 		   priv->dma_cap.number_rx_queues);
6047 	seq_printf(seq, "\tNumber of Additional TX queues: %d\n",
6048 		   priv->dma_cap.number_tx_queues);
6049 	seq_printf(seq, "\tEnhanced descriptors: %s\n",
6050 		   (priv->dma_cap.enh_desc) ? "Y" : "N");
6051 	seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);
6052 	seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);
6053 	seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz);
6054 	seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");
6055 	seq_printf(seq, "\tNumber of PPS Outputs: %d\n",
6056 		   priv->dma_cap.pps_out_num);
6057 	seq_printf(seq, "\tSafety Features: %s\n",
6058 		   priv->dma_cap.asp ? "Y" : "N");
6059 	seq_printf(seq, "\tFlexible RX Parser: %s\n",
6060 		   priv->dma_cap.frpsel ? "Y" : "N");
6061 	seq_printf(seq, "\tEnhanced Addressing: %d\n",
6062 		   priv->dma_cap.addr64);
6063 	seq_printf(seq, "\tReceive Side Scaling: %s\n",
6064 		   priv->dma_cap.rssen ? "Y" : "N");
6065 	seq_printf(seq, "\tVLAN Hash Filtering: %s\n",
6066 		   priv->dma_cap.vlhash ? "Y" : "N");
6067 	seq_printf(seq, "\tSplit Header: %s\n",
6068 		   priv->dma_cap.sphen ? "Y" : "N");
6069 	seq_printf(seq, "\tVLAN TX Insertion: %s\n",
6070 		   priv->dma_cap.vlins ? "Y" : "N");
6071 	seq_printf(seq, "\tDouble VLAN: %s\n",
6072 		   priv->dma_cap.dvlan ? "Y" : "N");
6073 	seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n",
6074 		   priv->dma_cap.l3l4fnum);
6075 	seq_printf(seq, "\tARP Offloading: %s\n",
6076 		   priv->dma_cap.arpoffsel ? "Y" : "N");
6077 	seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n",
6078 		   priv->dma_cap.estsel ? "Y" : "N");
6079 	seq_printf(seq, "\tFrame Preemption (FPE): %s\n",
6080 		   priv->dma_cap.fpesel ? "Y" : "N");
6081 	seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n",
6082 		   priv->dma_cap.tbssel ? "Y" : "N");
6083 	return 0;
6084 }
6085 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
6086 
6087 /* Use network device events to rename debugfs file entries.
6088  */
6089 static int stmmac_device_event(struct notifier_block *unused,
6090 			       unsigned long event, void *ptr)
6091 {
6092 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
6093 	struct stmmac_priv *priv = netdev_priv(dev);
6094 
6095 	if (dev->netdev_ops != &stmmac_netdev_ops)
6096 		goto done;
6097 
6098 	switch (event) {
6099 	case NETDEV_CHANGENAME:
6100 		if (priv->dbgfs_dir)
6101 			priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,
6102 							 priv->dbgfs_dir,
6103 							 stmmac_fs_dir,
6104 							 dev->name);
6105 		break;
6106 	}
6107 done:
6108 	return NOTIFY_DONE;
6109 }
6110 
6111 static struct notifier_block stmmac_notifier = {
6112 	.notifier_call = stmmac_device_event,
6113 };
6114 
6115 static void stmmac_init_fs(struct net_device *dev)
6116 {
6117 	struct stmmac_priv *priv = netdev_priv(dev);
6118 
6119 	rtnl_lock();
6120 
6121 	/* Create per netdev entries */
6122 	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
6123 
6124 	/* Entry to report DMA RX/TX rings */
6125 	debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
6126 			    &stmmac_rings_status_fops);
6127 
6128 	/* Entry to report the DMA HW features */
6129 	debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
6130 			    &stmmac_dma_cap_fops);
6131 
6132 	rtnl_unlock();
6133 }
6134 
6135 static void stmmac_exit_fs(struct net_device *dev)
6136 {
6137 	struct stmmac_priv *priv = netdev_priv(dev);
6138 
6139 	debugfs_remove_recursive(priv->dbgfs_dir);
6140 }
6141 #endif /* CONFIG_DEBUG_FS */
6142 
6143 static u32 stmmac_vid_crc32_le(__le16 vid_le)
6144 {
6145 	unsigned char *data = (unsigned char *)&vid_le;
6146 	unsigned char data_byte = 0;
6147 	u32 crc = ~0x0;
6148 	u32 temp = 0;
6149 	int i, bits;
6150 
6151 	bits = get_bitmask_order(VLAN_VID_MASK);
6152 	for (i = 0; i < bits; i++) {
6153 		if ((i % 8) == 0)
6154 			data_byte = data[i / 8];
6155 
6156 		temp = ((crc & 1) ^ data_byte) & 1;
6157 		crc >>= 1;
6158 		data_byte >>= 1;
6159 
6160 		if (temp)
6161 			crc ^= 0xedb88320;
6162 	}
6163 
6164 	return crc;
6165 }
6166 
6167 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
6168 {
6169 	u32 crc, hash = 0;
6170 	__le16 pmatch = 0;
6171 	int count = 0;
6172 	u16 vid = 0;
6173 
6174 	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
6175 		__le16 vid_le = cpu_to_le16(vid);
6176 		crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
6177 		hash |= (1 << crc);
6178 		count++;
6179 	}
6180 
6181 	if (!priv->dma_cap.vlhash) {
6182 		if (count > 2) /* VID = 0 always passes filter */
6183 			return -EOPNOTSUPP;
6184 
6185 		pmatch = cpu_to_le16(vid);
6186 		hash = 0;
6187 	}
6188 
6189 	return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
6190 }
6191 
6192 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
6193 {
6194 	struct stmmac_priv *priv = netdev_priv(ndev);
6195 	bool is_double = false;
6196 	int ret;
6197 
6198 	if (be16_to_cpu(proto) == ETH_P_8021AD)
6199 		is_double = true;
6200 
6201 	set_bit(vid, priv->active_vlans);
6202 	ret = stmmac_vlan_update(priv, is_double);
6203 	if (ret) {
6204 		clear_bit(vid, priv->active_vlans);
6205 		return ret;
6206 	}
6207 
6208 	if (priv->hw->num_vlan) {
6209 		ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
6210 		if (ret)
6211 			return ret;
6212 	}
6213 
6214 	return 0;
6215 }
6216 
6217 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
6218 {
6219 	struct stmmac_priv *priv = netdev_priv(ndev);
6220 	bool is_double = false;
6221 	int ret;
6222 
6223 	ret = pm_runtime_get_sync(priv->device);
6224 	if (ret < 0) {
6225 		pm_runtime_put_noidle(priv->device);
6226 		return ret;
6227 	}
6228 
6229 	if (be16_to_cpu(proto) == ETH_P_8021AD)
6230 		is_double = true;
6231 
6232 	clear_bit(vid, priv->active_vlans);
6233 
6234 	if (priv->hw->num_vlan) {
6235 		ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
6236 		if (ret)
6237 			goto del_vlan_error;
6238 	}
6239 
6240 	ret = stmmac_vlan_update(priv, is_double);
6241 
6242 del_vlan_error:
6243 	pm_runtime_put(priv->device);
6244 
6245 	return ret;
6246 }
6247 
6248 static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf)
6249 {
6250 	struct stmmac_priv *priv = netdev_priv(dev);
6251 
6252 	switch (bpf->command) {
6253 	case XDP_SETUP_PROG:
6254 		return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack);
6255 	case XDP_SETUP_XSK_POOL:
6256 		return stmmac_xdp_setup_pool(priv, bpf->xsk.pool,
6257 					     bpf->xsk.queue_id);
6258 	default:
6259 		return -EOPNOTSUPP;
6260 	}
6261 }
6262 
6263 static int stmmac_xdp_xmit(struct net_device *dev, int num_frames,
6264 			   struct xdp_frame **frames, u32 flags)
6265 {
6266 	struct stmmac_priv *priv = netdev_priv(dev);
6267 	int cpu = smp_processor_id();
6268 	struct netdev_queue *nq;
6269 	int i, nxmit = 0;
6270 	int queue;
6271 
6272 	if (unlikely(test_bit(STMMAC_DOWN, &priv->state)))
6273 		return -ENETDOWN;
6274 
6275 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
6276 		return -EINVAL;
6277 
6278 	queue = stmmac_xdp_get_tx_queue(priv, cpu);
6279 	nq = netdev_get_tx_queue(priv->dev, queue);
6280 
6281 	__netif_tx_lock(nq, cpu);
6282 	/* Avoids TX time-out as we are sharing with slow path */
6283 	txq_trans_cond_update(nq);
6284 
6285 	for (i = 0; i < num_frames; i++) {
6286 		int res;
6287 
6288 		res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true);
6289 		if (res == STMMAC_XDP_CONSUMED)
6290 			break;
6291 
6292 		nxmit++;
6293 	}
6294 
6295 	if (flags & XDP_XMIT_FLUSH) {
6296 		stmmac_flush_tx_descriptors(priv, queue);
6297 		stmmac_tx_timer_arm(priv, queue);
6298 	}
6299 
6300 	__netif_tx_unlock(nq);
6301 
6302 	return nxmit;
6303 }
6304 
6305 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue)
6306 {
6307 	struct stmmac_channel *ch = &priv->channel[queue];
6308 	unsigned long flags;
6309 
6310 	spin_lock_irqsave(&ch->lock, flags);
6311 	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
6312 	spin_unlock_irqrestore(&ch->lock, flags);
6313 
6314 	stmmac_stop_rx_dma(priv, queue);
6315 	__free_dma_rx_desc_resources(priv, queue);
6316 }
6317 
6318 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)
6319 {
6320 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
6321 	struct stmmac_channel *ch = &priv->channel[queue];
6322 	unsigned long flags;
6323 	u32 buf_size;
6324 	int ret;
6325 
6326 	ret = __alloc_dma_rx_desc_resources(priv, queue);
6327 	if (ret) {
6328 		netdev_err(priv->dev, "Failed to alloc RX desc.\n");
6329 		return;
6330 	}
6331 
6332 	ret = __init_dma_rx_desc_rings(priv, queue, GFP_KERNEL);
6333 	if (ret) {
6334 		__free_dma_rx_desc_resources(priv, queue);
6335 		netdev_err(priv->dev, "Failed to init RX desc.\n");
6336 		return;
6337 	}
6338 
6339 	stmmac_clear_rx_descriptors(priv, queue);
6340 
6341 	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
6342 			    rx_q->dma_rx_phy, rx_q->queue_index);
6343 
6344 	rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num *
6345 			     sizeof(struct dma_desc));
6346 	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
6347 			       rx_q->rx_tail_addr, rx_q->queue_index);
6348 
6349 	if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
6350 		buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
6351 		stmmac_set_dma_bfsize(priv, priv->ioaddr,
6352 				      buf_size,
6353 				      rx_q->queue_index);
6354 	} else {
6355 		stmmac_set_dma_bfsize(priv, priv->ioaddr,
6356 				      priv->dma_buf_sz,
6357 				      rx_q->queue_index);
6358 	}
6359 
6360 	stmmac_start_rx_dma(priv, queue);
6361 
6362 	spin_lock_irqsave(&ch->lock, flags);
6363 	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
6364 	spin_unlock_irqrestore(&ch->lock, flags);
6365 }
6366 
6367 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue)
6368 {
6369 	struct stmmac_channel *ch = &priv->channel[queue];
6370 	unsigned long flags;
6371 
6372 	spin_lock_irqsave(&ch->lock, flags);
6373 	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
6374 	spin_unlock_irqrestore(&ch->lock, flags);
6375 
6376 	stmmac_stop_tx_dma(priv, queue);
6377 	__free_dma_tx_desc_resources(priv, queue);
6378 }
6379 
6380 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)
6381 {
6382 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
6383 	struct stmmac_channel *ch = &priv->channel[queue];
6384 	unsigned long flags;
6385 	int ret;
6386 
6387 	ret = __alloc_dma_tx_desc_resources(priv, queue);
6388 	if (ret) {
6389 		netdev_err(priv->dev, "Failed to alloc TX desc.\n");
6390 		return;
6391 	}
6392 
6393 	ret = __init_dma_tx_desc_rings(priv, queue);
6394 	if (ret) {
6395 		__free_dma_tx_desc_resources(priv, queue);
6396 		netdev_err(priv->dev, "Failed to init TX desc.\n");
6397 		return;
6398 	}
6399 
6400 	stmmac_clear_tx_descriptors(priv, queue);
6401 
6402 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
6403 			    tx_q->dma_tx_phy, tx_q->queue_index);
6404 
6405 	if (tx_q->tbs & STMMAC_TBS_AVAIL)
6406 		stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index);
6407 
6408 	tx_q->tx_tail_addr = tx_q->dma_tx_phy;
6409 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
6410 			       tx_q->tx_tail_addr, tx_q->queue_index);
6411 
6412 	stmmac_start_tx_dma(priv, queue);
6413 
6414 	spin_lock_irqsave(&ch->lock, flags);
6415 	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
6416 	spin_unlock_irqrestore(&ch->lock, flags);
6417 }
6418 
6419 void stmmac_xdp_release(struct net_device *dev)
6420 {
6421 	struct stmmac_priv *priv = netdev_priv(dev);
6422 	u32 chan;
6423 
6424 	/* Disable NAPI process */
6425 	stmmac_disable_all_queues(priv);
6426 
6427 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
6428 		hrtimer_cancel(&priv->tx_queue[chan].txtimer);
6429 
6430 	/* Free the IRQ lines */
6431 	stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0);
6432 
6433 	/* Stop TX/RX DMA channels */
6434 	stmmac_stop_all_dma(priv);
6435 
6436 	/* Release and free the Rx/Tx resources */
6437 	free_dma_desc_resources(priv);
6438 
6439 	/* Disable the MAC Rx/Tx */
6440 	stmmac_mac_set(priv, priv->ioaddr, false);
6441 
6442 	/* set trans_start so we don't get spurious
6443 	 * watchdogs during reset
6444 	 */
6445 	netif_trans_update(dev);
6446 	netif_carrier_off(dev);
6447 }
6448 
6449 int stmmac_xdp_open(struct net_device *dev)
6450 {
6451 	struct stmmac_priv *priv = netdev_priv(dev);
6452 	u32 rx_cnt = priv->plat->rx_queues_to_use;
6453 	u32 tx_cnt = priv->plat->tx_queues_to_use;
6454 	u32 dma_csr_ch = max(rx_cnt, tx_cnt);
6455 	struct stmmac_rx_queue *rx_q;
6456 	struct stmmac_tx_queue *tx_q;
6457 	u32 buf_size;
6458 	bool sph_en;
6459 	u32 chan;
6460 	int ret;
6461 
6462 	ret = alloc_dma_desc_resources(priv);
6463 	if (ret < 0) {
6464 		netdev_err(dev, "%s: DMA descriptors allocation failed\n",
6465 			   __func__);
6466 		goto dma_desc_error;
6467 	}
6468 
6469 	ret = init_dma_desc_rings(dev, GFP_KERNEL);
6470 	if (ret < 0) {
6471 		netdev_err(dev, "%s: DMA descriptors initialization failed\n",
6472 			   __func__);
6473 		goto init_error;
6474 	}
6475 
6476 	/* DMA CSR Channel configuration */
6477 	for (chan = 0; chan < dma_csr_ch; chan++) {
6478 		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
6479 		stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
6480 	}
6481 
6482 	/* Adjust Split header */
6483 	sph_en = (priv->hw->rx_csum > 0) && priv->sph;
6484 
6485 	/* DMA RX Channel Configuration */
6486 	for (chan = 0; chan < rx_cnt; chan++) {
6487 		rx_q = &priv->rx_queue[chan];
6488 
6489 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
6490 				    rx_q->dma_rx_phy, chan);
6491 
6492 		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
6493 				     (rx_q->buf_alloc_num *
6494 				      sizeof(struct dma_desc));
6495 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
6496 				       rx_q->rx_tail_addr, chan);
6497 
6498 		if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
6499 			buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
6500 			stmmac_set_dma_bfsize(priv, priv->ioaddr,
6501 					      buf_size,
6502 					      rx_q->queue_index);
6503 		} else {
6504 			stmmac_set_dma_bfsize(priv, priv->ioaddr,
6505 					      priv->dma_buf_sz,
6506 					      rx_q->queue_index);
6507 		}
6508 
6509 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
6510 	}
6511 
6512 	/* DMA TX Channel Configuration */
6513 	for (chan = 0; chan < tx_cnt; chan++) {
6514 		tx_q = &priv->tx_queue[chan];
6515 
6516 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
6517 				    tx_q->dma_tx_phy, chan);
6518 
6519 		tx_q->tx_tail_addr = tx_q->dma_tx_phy;
6520 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
6521 				       tx_q->tx_tail_addr, chan);
6522 
6523 		hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
6524 		tx_q->txtimer.function = stmmac_tx_timer;
6525 	}
6526 
6527 	/* Enable the MAC Rx/Tx */
6528 	stmmac_mac_set(priv, priv->ioaddr, true);
6529 
6530 	/* Start Rx & Tx DMA Channels */
6531 	stmmac_start_all_dma(priv);
6532 
6533 	ret = stmmac_request_irq(dev);
6534 	if (ret)
6535 		goto irq_error;
6536 
6537 	/* Enable NAPI process*/
6538 	stmmac_enable_all_queues(priv);
6539 	netif_carrier_on(dev);
6540 	netif_tx_start_all_queues(dev);
6541 	stmmac_enable_all_dma_irq(priv);
6542 
6543 	return 0;
6544 
6545 irq_error:
6546 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
6547 		hrtimer_cancel(&priv->tx_queue[chan].txtimer);
6548 
6549 	stmmac_hw_teardown(dev);
6550 init_error:
6551 	free_dma_desc_resources(priv);
6552 dma_desc_error:
6553 	return ret;
6554 }
6555 
6556 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags)
6557 {
6558 	struct stmmac_priv *priv = netdev_priv(dev);
6559 	struct stmmac_rx_queue *rx_q;
6560 	struct stmmac_tx_queue *tx_q;
6561 	struct stmmac_channel *ch;
6562 
6563 	if (test_bit(STMMAC_DOWN, &priv->state) ||
6564 	    !netif_carrier_ok(priv->dev))
6565 		return -ENETDOWN;
6566 
6567 	if (!stmmac_xdp_is_enabled(priv))
6568 		return -ENXIO;
6569 
6570 	if (queue >= priv->plat->rx_queues_to_use ||
6571 	    queue >= priv->plat->tx_queues_to_use)
6572 		return -EINVAL;
6573 
6574 	rx_q = &priv->rx_queue[queue];
6575 	tx_q = &priv->tx_queue[queue];
6576 	ch = &priv->channel[queue];
6577 
6578 	if (!rx_q->xsk_pool && !tx_q->xsk_pool)
6579 		return -ENXIO;
6580 
6581 	if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) {
6582 		/* EQoS does not have per-DMA channel SW interrupt,
6583 		 * so we schedule RX Napi straight-away.
6584 		 */
6585 		if (likely(napi_schedule_prep(&ch->rxtx_napi)))
6586 			__napi_schedule(&ch->rxtx_napi);
6587 	}
6588 
6589 	return 0;
6590 }
6591 
6592 static const struct net_device_ops stmmac_netdev_ops = {
6593 	.ndo_open = stmmac_open,
6594 	.ndo_start_xmit = stmmac_xmit,
6595 	.ndo_stop = stmmac_release,
6596 	.ndo_change_mtu = stmmac_change_mtu,
6597 	.ndo_fix_features = stmmac_fix_features,
6598 	.ndo_set_features = stmmac_set_features,
6599 	.ndo_set_rx_mode = stmmac_set_rx_mode,
6600 	.ndo_tx_timeout = stmmac_tx_timeout,
6601 	.ndo_eth_ioctl = stmmac_ioctl,
6602 	.ndo_setup_tc = stmmac_setup_tc,
6603 	.ndo_select_queue = stmmac_select_queue,
6604 #ifdef CONFIG_NET_POLL_CONTROLLER
6605 	.ndo_poll_controller = stmmac_poll_controller,
6606 #endif
6607 	.ndo_set_mac_address = stmmac_set_mac_address,
6608 	.ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
6609 	.ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
6610 	.ndo_bpf = stmmac_bpf,
6611 	.ndo_xdp_xmit = stmmac_xdp_xmit,
6612 	.ndo_xsk_wakeup = stmmac_xsk_wakeup,
6613 };
6614 
6615 static void stmmac_reset_subtask(struct stmmac_priv *priv)
6616 {
6617 	if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
6618 		return;
6619 	if (test_bit(STMMAC_DOWN, &priv->state))
6620 		return;
6621 
6622 	netdev_err(priv->dev, "Reset adapter.\n");
6623 
6624 	rtnl_lock();
6625 	netif_trans_update(priv->dev);
6626 	while (test_and_set_bit(STMMAC_RESETING, &priv->state))
6627 		usleep_range(1000, 2000);
6628 
6629 	set_bit(STMMAC_DOWN, &priv->state);
6630 	dev_close(priv->dev);
6631 	dev_open(priv->dev, NULL);
6632 	clear_bit(STMMAC_DOWN, &priv->state);
6633 	clear_bit(STMMAC_RESETING, &priv->state);
6634 	rtnl_unlock();
6635 }
6636 
6637 static void stmmac_service_task(struct work_struct *work)
6638 {
6639 	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
6640 			service_task);
6641 
6642 	stmmac_reset_subtask(priv);
6643 	clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
6644 }
6645 
6646 /**
6647  *  stmmac_hw_init - Init the MAC device
6648  *  @priv: driver private structure
6649  *  Description: this function is to configure the MAC device according to
6650  *  some platform parameters or the HW capability register. It prepares the
6651  *  driver to use either ring or chain modes and to setup either enhanced or
6652  *  normal descriptors.
6653  */
6654 static int stmmac_hw_init(struct stmmac_priv *priv)
6655 {
6656 	int ret;
6657 
6658 	/* dwmac-sun8i only work in chain mode */
6659 	if (priv->plat->has_sun8i)
6660 		chain_mode = 1;
6661 	priv->chain_mode = chain_mode;
6662 
6663 	/* Initialize HW Interface */
6664 	ret = stmmac_hwif_init(priv);
6665 	if (ret)
6666 		return ret;
6667 
6668 	/* Get the HW capability (new GMAC newer than 3.50a) */
6669 	priv->hw_cap_support = stmmac_get_hw_features(priv);
6670 	if (priv->hw_cap_support) {
6671 		dev_info(priv->device, "DMA HW capability register supported\n");
6672 
6673 		/* We can override some gmac/dma configuration fields: e.g.
6674 		 * enh_desc, tx_coe (e.g. that are passed through the
6675 		 * platform) with the values from the HW capability
6676 		 * register (if supported).
6677 		 */
6678 		priv->plat->enh_desc = priv->dma_cap.enh_desc;
6679 		priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up &&
6680 				!priv->plat->use_phy_wol;
6681 		priv->hw->pmt = priv->plat->pmt;
6682 		if (priv->dma_cap.hash_tb_sz) {
6683 			priv->hw->multicast_filter_bins =
6684 					(BIT(priv->dma_cap.hash_tb_sz) << 5);
6685 			priv->hw->mcast_bits_log2 =
6686 					ilog2(priv->hw->multicast_filter_bins);
6687 		}
6688 
6689 		/* TXCOE doesn't work in thresh DMA mode */
6690 		if (priv->plat->force_thresh_dma_mode)
6691 			priv->plat->tx_coe = 0;
6692 		else
6693 			priv->plat->tx_coe = priv->dma_cap.tx_coe;
6694 
6695 		/* In case of GMAC4 rx_coe is from HW cap register. */
6696 		priv->plat->rx_coe = priv->dma_cap.rx_coe;
6697 
6698 		if (priv->dma_cap.rx_coe_type2)
6699 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
6700 		else if (priv->dma_cap.rx_coe_type1)
6701 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
6702 
6703 	} else {
6704 		dev_info(priv->device, "No HW DMA feature register supported\n");
6705 	}
6706 
6707 	if (priv->plat->rx_coe) {
6708 		priv->hw->rx_csum = priv->plat->rx_coe;
6709 		dev_info(priv->device, "RX Checksum Offload Engine supported\n");
6710 		if (priv->synopsys_id < DWMAC_CORE_4_00)
6711 			dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
6712 	}
6713 	if (priv->plat->tx_coe)
6714 		dev_info(priv->device, "TX Checksum insertion supported\n");
6715 
6716 	if (priv->plat->pmt) {
6717 		dev_info(priv->device, "Wake-Up On Lan supported\n");
6718 		device_set_wakeup_capable(priv->device, 1);
6719 	}
6720 
6721 	if (priv->dma_cap.tsoen)
6722 		dev_info(priv->device, "TSO supported\n");
6723 
6724 	priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en;
6725 	priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;
6726 
6727 	/* Run HW quirks, if any */
6728 	if (priv->hwif_quirks) {
6729 		ret = priv->hwif_quirks(priv);
6730 		if (ret)
6731 			return ret;
6732 	}
6733 
6734 	/* Rx Watchdog is available in the COREs newer than the 3.40.
6735 	 * In some case, for example on bugged HW this feature
6736 	 * has to be disable and this can be done by passing the
6737 	 * riwt_off field from the platform.
6738 	 */
6739 	if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
6740 	    (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
6741 		priv->use_riwt = 1;
6742 		dev_info(priv->device,
6743 			 "Enable RX Mitigation via HW Watchdog Timer\n");
6744 	}
6745 
6746 	return 0;
6747 }
6748 
6749 static void stmmac_napi_add(struct net_device *dev)
6750 {
6751 	struct stmmac_priv *priv = netdev_priv(dev);
6752 	u32 queue, maxq;
6753 
6754 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
6755 
6756 	for (queue = 0; queue < maxq; queue++) {
6757 		struct stmmac_channel *ch = &priv->channel[queue];
6758 
6759 		ch->priv_data = priv;
6760 		ch->index = queue;
6761 		spin_lock_init(&ch->lock);
6762 
6763 		if (queue < priv->plat->rx_queues_to_use) {
6764 			netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx,
6765 				       NAPI_POLL_WEIGHT);
6766 		}
6767 		if (queue < priv->plat->tx_queues_to_use) {
6768 			netif_tx_napi_add(dev, &ch->tx_napi,
6769 					  stmmac_napi_poll_tx,
6770 					  NAPI_POLL_WEIGHT);
6771 		}
6772 		if (queue < priv->plat->rx_queues_to_use &&
6773 		    queue < priv->plat->tx_queues_to_use) {
6774 			netif_napi_add(dev, &ch->rxtx_napi,
6775 				       stmmac_napi_poll_rxtx,
6776 				       NAPI_POLL_WEIGHT);
6777 		}
6778 	}
6779 }
6780 
6781 static void stmmac_napi_del(struct net_device *dev)
6782 {
6783 	struct stmmac_priv *priv = netdev_priv(dev);
6784 	u32 queue, maxq;
6785 
6786 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
6787 
6788 	for (queue = 0; queue < maxq; queue++) {
6789 		struct stmmac_channel *ch = &priv->channel[queue];
6790 
6791 		if (queue < priv->plat->rx_queues_to_use)
6792 			netif_napi_del(&ch->rx_napi);
6793 		if (queue < priv->plat->tx_queues_to_use)
6794 			netif_napi_del(&ch->tx_napi);
6795 		if (queue < priv->plat->rx_queues_to_use &&
6796 		    queue < priv->plat->tx_queues_to_use) {
6797 			netif_napi_del(&ch->rxtx_napi);
6798 		}
6799 	}
6800 }
6801 
6802 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt)
6803 {
6804 	struct stmmac_priv *priv = netdev_priv(dev);
6805 	int ret = 0;
6806 
6807 	if (netif_running(dev))
6808 		stmmac_release(dev);
6809 
6810 	stmmac_napi_del(dev);
6811 
6812 	priv->plat->rx_queues_to_use = rx_cnt;
6813 	priv->plat->tx_queues_to_use = tx_cnt;
6814 
6815 	stmmac_napi_add(dev);
6816 
6817 	if (netif_running(dev))
6818 		ret = stmmac_open(dev);
6819 
6820 	return ret;
6821 }
6822 
6823 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size)
6824 {
6825 	struct stmmac_priv *priv = netdev_priv(dev);
6826 	int ret = 0;
6827 
6828 	if (netif_running(dev))
6829 		stmmac_release(dev);
6830 
6831 	priv->dma_rx_size = rx_size;
6832 	priv->dma_tx_size = tx_size;
6833 
6834 	if (netif_running(dev))
6835 		ret = stmmac_open(dev);
6836 
6837 	return ret;
6838 }
6839 
6840 #define SEND_VERIFY_MPAKCET_FMT "Send Verify mPacket lo_state=%d lp_state=%d\n"
6841 static void stmmac_fpe_lp_task(struct work_struct *work)
6842 {
6843 	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
6844 						fpe_task);
6845 	struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg;
6846 	enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state;
6847 	enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state;
6848 	bool *hs_enable = &fpe_cfg->hs_enable;
6849 	bool *enable = &fpe_cfg->enable;
6850 	int retries = 20;
6851 
6852 	while (retries-- > 0) {
6853 		/* Bail out immediately if FPE handshake is OFF */
6854 		if (*lo_state == FPE_STATE_OFF || !*hs_enable)
6855 			break;
6856 
6857 		if (*lo_state == FPE_STATE_ENTERING_ON &&
6858 		    *lp_state == FPE_STATE_ENTERING_ON) {
6859 			stmmac_fpe_configure(priv, priv->ioaddr,
6860 					     priv->plat->tx_queues_to_use,
6861 					     priv->plat->rx_queues_to_use,
6862 					     *enable);
6863 
6864 			netdev_info(priv->dev, "configured FPE\n");
6865 
6866 			*lo_state = FPE_STATE_ON;
6867 			*lp_state = FPE_STATE_ON;
6868 			netdev_info(priv->dev, "!!! BOTH FPE stations ON\n");
6869 			break;
6870 		}
6871 
6872 		if ((*lo_state == FPE_STATE_CAPABLE ||
6873 		     *lo_state == FPE_STATE_ENTERING_ON) &&
6874 		     *lp_state != FPE_STATE_ON) {
6875 			netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT,
6876 				    *lo_state, *lp_state);
6877 			stmmac_fpe_send_mpacket(priv, priv->ioaddr,
6878 						MPACKET_VERIFY);
6879 		}
6880 		/* Sleep then retry */
6881 		msleep(500);
6882 	}
6883 
6884 	clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state);
6885 }
6886 
6887 void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable)
6888 {
6889 	if (priv->plat->fpe_cfg->hs_enable != enable) {
6890 		if (enable) {
6891 			stmmac_fpe_send_mpacket(priv, priv->ioaddr,
6892 						MPACKET_VERIFY);
6893 		} else {
6894 			priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF;
6895 			priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF;
6896 		}
6897 
6898 		priv->plat->fpe_cfg->hs_enable = enable;
6899 	}
6900 }
6901 
6902 /**
6903  * stmmac_dvr_probe
6904  * @device: device pointer
6905  * @plat_dat: platform data pointer
6906  * @res: stmmac resource pointer
6907  * Description: this is the main probe function used to
6908  * call the alloc_etherdev, allocate the priv structure.
6909  * Return:
6910  * returns 0 on success, otherwise errno.
6911  */
6912 int stmmac_dvr_probe(struct device *device,
6913 		     struct plat_stmmacenet_data *plat_dat,
6914 		     struct stmmac_resources *res)
6915 {
6916 	struct net_device *ndev = NULL;
6917 	struct stmmac_priv *priv;
6918 	u32 rxq;
6919 	int i, ret = 0;
6920 
6921 	ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
6922 				       MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
6923 	if (!ndev)
6924 		return -ENOMEM;
6925 
6926 	SET_NETDEV_DEV(ndev, device);
6927 
6928 	priv = netdev_priv(ndev);
6929 	priv->device = device;
6930 	priv->dev = ndev;
6931 
6932 	stmmac_set_ethtool_ops(ndev);
6933 	priv->pause = pause;
6934 	priv->plat = plat_dat;
6935 	priv->ioaddr = res->addr;
6936 	priv->dev->base_addr = (unsigned long)res->addr;
6937 	priv->plat->dma_cfg->multi_msi_en = priv->plat->multi_msi_en;
6938 
6939 	priv->dev->irq = res->irq;
6940 	priv->wol_irq = res->wol_irq;
6941 	priv->lpi_irq = res->lpi_irq;
6942 	priv->sfty_ce_irq = res->sfty_ce_irq;
6943 	priv->sfty_ue_irq = res->sfty_ue_irq;
6944 	for (i = 0; i < MTL_MAX_RX_QUEUES; i++)
6945 		priv->rx_irq[i] = res->rx_irq[i];
6946 	for (i = 0; i < MTL_MAX_TX_QUEUES; i++)
6947 		priv->tx_irq[i] = res->tx_irq[i];
6948 
6949 	if (!is_zero_ether_addr(res->mac))
6950 		eth_hw_addr_set(priv->dev, res->mac);
6951 
6952 	dev_set_drvdata(device, priv->dev);
6953 
6954 	/* Verify driver arguments */
6955 	stmmac_verify_args();
6956 
6957 	priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL);
6958 	if (!priv->af_xdp_zc_qps)
6959 		return -ENOMEM;
6960 
6961 	/* Allocate workqueue */
6962 	priv->wq = create_singlethread_workqueue("stmmac_wq");
6963 	if (!priv->wq) {
6964 		dev_err(priv->device, "failed to create workqueue\n");
6965 		return -ENOMEM;
6966 	}
6967 
6968 	INIT_WORK(&priv->service_task, stmmac_service_task);
6969 
6970 	/* Initialize Link Partner FPE workqueue */
6971 	INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task);
6972 
6973 	/* Override with kernel parameters if supplied XXX CRS XXX
6974 	 * this needs to have multiple instances
6975 	 */
6976 	if ((phyaddr >= 0) && (phyaddr <= 31))
6977 		priv->plat->phy_addr = phyaddr;
6978 
6979 	if (priv->plat->stmmac_rst) {
6980 		ret = reset_control_assert(priv->plat->stmmac_rst);
6981 		reset_control_deassert(priv->plat->stmmac_rst);
6982 		/* Some reset controllers have only reset callback instead of
6983 		 * assert + deassert callbacks pair.
6984 		 */
6985 		if (ret == -ENOTSUPP)
6986 			reset_control_reset(priv->plat->stmmac_rst);
6987 	}
6988 
6989 	ret = reset_control_deassert(priv->plat->stmmac_ahb_rst);
6990 	if (ret == -ENOTSUPP)
6991 		dev_err(priv->device, "unable to bring out of ahb reset: %pe\n",
6992 			ERR_PTR(ret));
6993 
6994 	/* Init MAC and get the capabilities */
6995 	ret = stmmac_hw_init(priv);
6996 	if (ret)
6997 		goto error_hw_init;
6998 
6999 	/* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch.
7000 	 */
7001 	if (priv->synopsys_id < DWMAC_CORE_5_20)
7002 		priv->plat->dma_cfg->dche = false;
7003 
7004 	stmmac_check_ether_addr(priv);
7005 
7006 	ndev->netdev_ops = &stmmac_netdev_ops;
7007 
7008 	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
7009 			    NETIF_F_RXCSUM;
7010 
7011 	ret = stmmac_tc_init(priv, priv);
7012 	if (!ret) {
7013 		ndev->hw_features |= NETIF_F_HW_TC;
7014 	}
7015 
7016 	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
7017 		ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
7018 		if (priv->plat->has_gmac4)
7019 			ndev->hw_features |= NETIF_F_GSO_UDP_L4;
7020 		priv->tso = true;
7021 		dev_info(priv->device, "TSO feature enabled\n");
7022 	}
7023 
7024 	if (priv->dma_cap.sphen) {
7025 		ndev->hw_features |= NETIF_F_GRO;
7026 		priv->sph_cap = true;
7027 		priv->sph = priv->sph_cap;
7028 		dev_info(priv->device, "SPH feature enabled\n");
7029 	}
7030 
7031 	/* The current IP register MAC_HW_Feature1[ADDR64] only define
7032 	 * 32/40/64 bit width, but some SOC support others like i.MX8MP
7033 	 * support 34 bits but it map to 40 bits width in MAC_HW_Feature1[ADDR64].
7034 	 * So overwrite dma_cap.addr64 according to HW real design.
7035 	 */
7036 	if (priv->plat->addr64)
7037 		priv->dma_cap.addr64 = priv->plat->addr64;
7038 
7039 	if (priv->dma_cap.addr64) {
7040 		ret = dma_set_mask_and_coherent(device,
7041 				DMA_BIT_MASK(priv->dma_cap.addr64));
7042 		if (!ret) {
7043 			dev_info(priv->device, "Using %d bits DMA width\n",
7044 				 priv->dma_cap.addr64);
7045 
7046 			/*
7047 			 * If more than 32 bits can be addressed, make sure to
7048 			 * enable enhanced addressing mode.
7049 			 */
7050 			if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
7051 				priv->plat->dma_cfg->eame = true;
7052 		} else {
7053 			ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
7054 			if (ret) {
7055 				dev_err(priv->device, "Failed to set DMA Mask\n");
7056 				goto error_hw_init;
7057 			}
7058 
7059 			priv->dma_cap.addr64 = 32;
7060 		}
7061 	}
7062 
7063 	ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
7064 	ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
7065 #ifdef STMMAC_VLAN_TAG_USED
7066 	/* Both mac100 and gmac support receive VLAN tag detection */
7067 	ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
7068 	if (priv->dma_cap.vlhash) {
7069 		ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
7070 		ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
7071 	}
7072 	if (priv->dma_cap.vlins) {
7073 		ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
7074 		if (priv->dma_cap.dvlan)
7075 			ndev->features |= NETIF_F_HW_VLAN_STAG_TX;
7076 	}
7077 #endif
7078 	priv->msg_enable = netif_msg_init(debug, default_msg_level);
7079 
7080 	/* Initialize RSS */
7081 	rxq = priv->plat->rx_queues_to_use;
7082 	netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
7083 	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
7084 		priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
7085 
7086 	if (priv->dma_cap.rssen && priv->plat->rss_en)
7087 		ndev->features |= NETIF_F_RXHASH;
7088 
7089 	/* MTU range: 46 - hw-specific max */
7090 	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
7091 	if (priv->plat->has_xgmac)
7092 		ndev->max_mtu = XGMAC_JUMBO_LEN;
7093 	else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
7094 		ndev->max_mtu = JUMBO_LEN;
7095 	else
7096 		ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
7097 	/* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
7098 	 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
7099 	 */
7100 	if ((priv->plat->maxmtu < ndev->max_mtu) &&
7101 	    (priv->plat->maxmtu >= ndev->min_mtu))
7102 		ndev->max_mtu = priv->plat->maxmtu;
7103 	else if (priv->plat->maxmtu < ndev->min_mtu)
7104 		dev_warn(priv->device,
7105 			 "%s: warning: maxmtu having invalid value (%d)\n",
7106 			 __func__, priv->plat->maxmtu);
7107 
7108 	if (flow_ctrl)
7109 		priv->flow_ctrl = FLOW_AUTO;	/* RX/TX pause on */
7110 
7111 	/* Setup channels NAPI */
7112 	stmmac_napi_add(ndev);
7113 
7114 	mutex_init(&priv->lock);
7115 
7116 	/* If a specific clk_csr value is passed from the platform
7117 	 * this means that the CSR Clock Range selection cannot be
7118 	 * changed at run-time and it is fixed. Viceversa the driver'll try to
7119 	 * set the MDC clock dynamically according to the csr actual
7120 	 * clock input.
7121 	 */
7122 	if (priv->plat->clk_csr >= 0)
7123 		priv->clk_csr = priv->plat->clk_csr;
7124 	else
7125 		stmmac_clk_csr_set(priv);
7126 
7127 	stmmac_check_pcs_mode(priv);
7128 
7129 	pm_runtime_get_noresume(device);
7130 	pm_runtime_set_active(device);
7131 	if (!pm_runtime_enabled(device))
7132 		pm_runtime_enable(device);
7133 
7134 	if (priv->hw->pcs != STMMAC_PCS_TBI &&
7135 	    priv->hw->pcs != STMMAC_PCS_RTBI) {
7136 		/* MDIO bus Registration */
7137 		ret = stmmac_mdio_register(ndev);
7138 		if (ret < 0) {
7139 			dev_err(priv->device,
7140 				"%s: MDIO bus (id: %d) registration failed",
7141 				__func__, priv->plat->bus_id);
7142 			goto error_mdio_register;
7143 		}
7144 	}
7145 
7146 	if (priv->plat->speed_mode_2500)
7147 		priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv);
7148 
7149 	if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) {
7150 		ret = stmmac_xpcs_setup(priv->mii);
7151 		if (ret)
7152 			goto error_xpcs_setup;
7153 	}
7154 
7155 	ret = stmmac_phy_setup(priv);
7156 	if (ret) {
7157 		netdev_err(ndev, "failed to setup phy (%d)\n", ret);
7158 		goto error_phy_setup;
7159 	}
7160 
7161 	ret = register_netdev(ndev);
7162 	if (ret) {
7163 		dev_err(priv->device, "%s: ERROR %i registering the device\n",
7164 			__func__, ret);
7165 		goto error_netdev_register;
7166 	}
7167 
7168 	if (priv->plat->serdes_powerup) {
7169 		ret = priv->plat->serdes_powerup(ndev,
7170 						 priv->plat->bsp_priv);
7171 
7172 		if (ret < 0)
7173 			goto error_serdes_powerup;
7174 	}
7175 
7176 #ifdef CONFIG_DEBUG_FS
7177 	stmmac_init_fs(ndev);
7178 #endif
7179 
7180 	if (priv->plat->dump_debug_regs)
7181 		priv->plat->dump_debug_regs(priv->plat->bsp_priv);
7182 
7183 	/* Let pm_runtime_put() disable the clocks.
7184 	 * If CONFIG_PM is not enabled, the clocks will stay powered.
7185 	 */
7186 	pm_runtime_put(device);
7187 
7188 	return ret;
7189 
7190 error_serdes_powerup:
7191 	unregister_netdev(ndev);
7192 error_netdev_register:
7193 	phylink_destroy(priv->phylink);
7194 error_xpcs_setup:
7195 error_phy_setup:
7196 	if (priv->hw->pcs != STMMAC_PCS_TBI &&
7197 	    priv->hw->pcs != STMMAC_PCS_RTBI)
7198 		stmmac_mdio_unregister(ndev);
7199 error_mdio_register:
7200 	stmmac_napi_del(ndev);
7201 error_hw_init:
7202 	destroy_workqueue(priv->wq);
7203 	bitmap_free(priv->af_xdp_zc_qps);
7204 
7205 	return ret;
7206 }
7207 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
7208 
7209 /**
7210  * stmmac_dvr_remove
7211  * @dev: device pointer
7212  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
7213  * changes the link status, releases the DMA descriptor rings.
7214  */
7215 int stmmac_dvr_remove(struct device *dev)
7216 {
7217 	struct net_device *ndev = dev_get_drvdata(dev);
7218 	struct stmmac_priv *priv = netdev_priv(ndev);
7219 
7220 	netdev_info(priv->dev, "%s: removing driver", __func__);
7221 
7222 	pm_runtime_get_sync(dev);
7223 	pm_runtime_disable(dev);
7224 	pm_runtime_put_noidle(dev);
7225 
7226 	stmmac_stop_all_dma(priv);
7227 	stmmac_mac_set(priv, priv->ioaddr, false);
7228 	netif_carrier_off(ndev);
7229 	unregister_netdev(ndev);
7230 
7231 	/* Serdes power down needs to happen after VLAN filter
7232 	 * is deleted that is triggered by unregister_netdev().
7233 	 */
7234 	if (priv->plat->serdes_powerdown)
7235 		priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
7236 
7237 #ifdef CONFIG_DEBUG_FS
7238 	stmmac_exit_fs(ndev);
7239 #endif
7240 	phylink_destroy(priv->phylink);
7241 	if (priv->plat->stmmac_rst)
7242 		reset_control_assert(priv->plat->stmmac_rst);
7243 	reset_control_assert(priv->plat->stmmac_ahb_rst);
7244 	if (priv->hw->pcs != STMMAC_PCS_TBI &&
7245 	    priv->hw->pcs != STMMAC_PCS_RTBI)
7246 		stmmac_mdio_unregister(ndev);
7247 	destroy_workqueue(priv->wq);
7248 	mutex_destroy(&priv->lock);
7249 	bitmap_free(priv->af_xdp_zc_qps);
7250 
7251 	return 0;
7252 }
7253 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
7254 
7255 /**
7256  * stmmac_suspend - suspend callback
7257  * @dev: device pointer
7258  * Description: this is the function to suspend the device and it is called
7259  * by the platform driver to stop the network queue, release the resources,
7260  * program the PMT register (for WoL), clean and release driver resources.
7261  */
7262 int stmmac_suspend(struct device *dev)
7263 {
7264 	struct net_device *ndev = dev_get_drvdata(dev);
7265 	struct stmmac_priv *priv = netdev_priv(ndev);
7266 	u32 chan;
7267 
7268 	if (!ndev || !netif_running(ndev))
7269 		return 0;
7270 
7271 	mutex_lock(&priv->lock);
7272 
7273 	netif_device_detach(ndev);
7274 
7275 	stmmac_disable_all_queues(priv);
7276 
7277 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
7278 		hrtimer_cancel(&priv->tx_queue[chan].txtimer);
7279 
7280 	if (priv->eee_enabled) {
7281 		priv->tx_path_in_lpi_mode = false;
7282 		del_timer_sync(&priv->eee_ctrl_timer);
7283 	}
7284 
7285 	/* Stop TX/RX DMA */
7286 	stmmac_stop_all_dma(priv);
7287 
7288 	if (priv->plat->serdes_powerdown)
7289 		priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
7290 
7291 	/* Enable Power down mode by programming the PMT regs */
7292 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {
7293 		stmmac_pmt(priv, priv->hw, priv->wolopts);
7294 		priv->irq_wake = 1;
7295 	} else {
7296 		stmmac_mac_set(priv, priv->ioaddr, false);
7297 		pinctrl_pm_select_sleep_state(priv->device);
7298 	}
7299 
7300 	mutex_unlock(&priv->lock);
7301 
7302 	rtnl_lock();
7303 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {
7304 		phylink_suspend(priv->phylink, true);
7305 	} else {
7306 		if (device_may_wakeup(priv->device))
7307 			phylink_speed_down(priv->phylink, false);
7308 		phylink_suspend(priv->phylink, false);
7309 	}
7310 	rtnl_unlock();
7311 
7312 	if (priv->dma_cap.fpesel) {
7313 		/* Disable FPE */
7314 		stmmac_fpe_configure(priv, priv->ioaddr,
7315 				     priv->plat->tx_queues_to_use,
7316 				     priv->plat->rx_queues_to_use, false);
7317 
7318 		stmmac_fpe_handshake(priv, false);
7319 		stmmac_fpe_stop_wq(priv);
7320 	}
7321 
7322 	priv->speed = SPEED_UNKNOWN;
7323 	return 0;
7324 }
7325 EXPORT_SYMBOL_GPL(stmmac_suspend);
7326 
7327 /**
7328  * stmmac_reset_queues_param - reset queue parameters
7329  * @priv: device pointer
7330  */
7331 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
7332 {
7333 	u32 rx_cnt = priv->plat->rx_queues_to_use;
7334 	u32 tx_cnt = priv->plat->tx_queues_to_use;
7335 	u32 queue;
7336 
7337 	for (queue = 0; queue < rx_cnt; queue++) {
7338 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
7339 
7340 		rx_q->cur_rx = 0;
7341 		rx_q->dirty_rx = 0;
7342 	}
7343 
7344 	for (queue = 0; queue < tx_cnt; queue++) {
7345 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
7346 
7347 		tx_q->cur_tx = 0;
7348 		tx_q->dirty_tx = 0;
7349 		tx_q->mss = 0;
7350 
7351 		netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
7352 	}
7353 }
7354 
7355 /**
7356  * stmmac_resume - resume callback
7357  * @dev: device pointer
7358  * Description: when resume this function is invoked to setup the DMA and CORE
7359  * in a usable state.
7360  */
7361 int stmmac_resume(struct device *dev)
7362 {
7363 	struct net_device *ndev = dev_get_drvdata(dev);
7364 	struct stmmac_priv *priv = netdev_priv(ndev);
7365 	int ret;
7366 
7367 	if (!netif_running(ndev))
7368 		return 0;
7369 
7370 	/* Power Down bit, into the PM register, is cleared
7371 	 * automatically as soon as a magic packet or a Wake-up frame
7372 	 * is received. Anyway, it's better to manually clear
7373 	 * this bit because it can generate problems while resuming
7374 	 * from another devices (e.g. serial console).
7375 	 */
7376 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {
7377 		mutex_lock(&priv->lock);
7378 		stmmac_pmt(priv, priv->hw, 0);
7379 		mutex_unlock(&priv->lock);
7380 		priv->irq_wake = 0;
7381 	} else {
7382 		pinctrl_pm_select_default_state(priv->device);
7383 		/* reset the phy so that it's ready */
7384 		if (priv->mii)
7385 			stmmac_mdio_reset(priv->mii);
7386 	}
7387 
7388 	if (priv->plat->serdes_powerup) {
7389 		ret = priv->plat->serdes_powerup(ndev,
7390 						 priv->plat->bsp_priv);
7391 
7392 		if (ret < 0)
7393 			return ret;
7394 	}
7395 
7396 	rtnl_lock();
7397 	if (device_may_wakeup(priv->device) && priv->plat->pmt) {
7398 		phylink_resume(priv->phylink);
7399 	} else {
7400 		phylink_resume(priv->phylink);
7401 		if (device_may_wakeup(priv->device))
7402 			phylink_speed_up(priv->phylink);
7403 	}
7404 	rtnl_unlock();
7405 
7406 	rtnl_lock();
7407 	mutex_lock(&priv->lock);
7408 
7409 	stmmac_reset_queues_param(priv);
7410 
7411 	stmmac_free_tx_skbufs(priv);
7412 	stmmac_clear_descriptors(priv);
7413 
7414 	stmmac_hw_setup(ndev, false);
7415 	stmmac_init_coalesce(priv);
7416 	stmmac_set_rx_mode(ndev);
7417 
7418 	stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);
7419 
7420 	stmmac_enable_all_queues(priv);
7421 	stmmac_enable_all_dma_irq(priv);
7422 
7423 	mutex_unlock(&priv->lock);
7424 	rtnl_unlock();
7425 
7426 	netif_device_attach(ndev);
7427 
7428 	return 0;
7429 }
7430 EXPORT_SYMBOL_GPL(stmmac_resume);
7431 
7432 #ifndef MODULE
7433 static int __init stmmac_cmdline_opt(char *str)
7434 {
7435 	char *opt;
7436 
7437 	if (!str || !*str)
7438 		return 1;
7439 	while ((opt = strsep(&str, ",")) != NULL) {
7440 		if (!strncmp(opt, "debug:", 6)) {
7441 			if (kstrtoint(opt + 6, 0, &debug))
7442 				goto err;
7443 		} else if (!strncmp(opt, "phyaddr:", 8)) {
7444 			if (kstrtoint(opt + 8, 0, &phyaddr))
7445 				goto err;
7446 		} else if (!strncmp(opt, "buf_sz:", 7)) {
7447 			if (kstrtoint(opt + 7, 0, &buf_sz))
7448 				goto err;
7449 		} else if (!strncmp(opt, "tc:", 3)) {
7450 			if (kstrtoint(opt + 3, 0, &tc))
7451 				goto err;
7452 		} else if (!strncmp(opt, "watchdog:", 9)) {
7453 			if (kstrtoint(opt + 9, 0, &watchdog))
7454 				goto err;
7455 		} else if (!strncmp(opt, "flow_ctrl:", 10)) {
7456 			if (kstrtoint(opt + 10, 0, &flow_ctrl))
7457 				goto err;
7458 		} else if (!strncmp(opt, "pause:", 6)) {
7459 			if (kstrtoint(opt + 6, 0, &pause))
7460 				goto err;
7461 		} else if (!strncmp(opt, "eee_timer:", 10)) {
7462 			if (kstrtoint(opt + 10, 0, &eee_timer))
7463 				goto err;
7464 		} else if (!strncmp(opt, "chain_mode:", 11)) {
7465 			if (kstrtoint(opt + 11, 0, &chain_mode))
7466 				goto err;
7467 		}
7468 	}
7469 	return 1;
7470 
7471 err:
7472 	pr_err("%s: ERROR broken module parameter conversion", __func__);
7473 	return 1;
7474 }
7475 
7476 __setup("stmmaceth=", stmmac_cmdline_opt);
7477 #endif /* MODULE */
7478 
7479 static int __init stmmac_init(void)
7480 {
7481 #ifdef CONFIG_DEBUG_FS
7482 	/* Create debugfs main directory if it doesn't exist yet */
7483 	if (!stmmac_fs_dir)
7484 		stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
7485 	register_netdevice_notifier(&stmmac_notifier);
7486 #endif
7487 
7488 	return 0;
7489 }
7490 
7491 static void __exit stmmac_exit(void)
7492 {
7493 #ifdef CONFIG_DEBUG_FS
7494 	unregister_netdevice_notifier(&stmmac_notifier);
7495 	debugfs_remove_recursive(stmmac_fs_dir);
7496 #endif
7497 }
7498 
7499 module_init(stmmac_init)
7500 module_exit(stmmac_exit)
7501 
7502 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
7503 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
7504 MODULE_LICENSE("GPL");
7505