1 // SPDX-License-Identifier: GPL-2.0-only
2 /*******************************************************************************
3   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
4   ST Ethernet IPs are built around a Synopsys IP Core.
5 
6 	Copyright(C) 2007-2011 STMicroelectronics Ltd
7 
8 
9   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
10 
11   Documentation available at:
12 	http://www.stlinux.com
13   Support available at:
14 	https://bugzilla.stlinux.com/
15 *******************************************************************************/
16 
17 #include <linux/clk.h>
18 #include <linux/kernel.h>
19 #include <linux/interrupt.h>
20 #include <linux/ip.h>
21 #include <linux/tcp.h>
22 #include <linux/skbuff.h>
23 #include <linux/ethtool.h>
24 #include <linux/if_ether.h>
25 #include <linux/crc32.h>
26 #include <linux/mii.h>
27 #include <linux/if.h>
28 #include <linux/if_vlan.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/slab.h>
31 #include <linux/prefetch.h>
32 #include <linux/pinctrl/consumer.h>
33 #ifdef CONFIG_DEBUG_FS
34 #include <linux/debugfs.h>
35 #include <linux/seq_file.h>
36 #endif /* CONFIG_DEBUG_FS */
37 #include <linux/net_tstamp.h>
38 #include <linux/phylink.h>
39 #include <linux/udp.h>
40 #include <net/pkt_cls.h>
41 #include "stmmac_ptp.h"
42 #include "stmmac.h"
43 #include <linux/reset.h>
44 #include <linux/of_mdio.h>
45 #include "dwmac1000.h"
46 #include "dwxgmac2.h"
47 #include "hwif.h"
48 
49 #define	STMMAC_ALIGN(x)		ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
50 #define	TSO_MAX_BUFF_SIZE	(SZ_16K - 1)
51 
52 /* Module parameters */
53 #define TX_TIMEO	5000
54 static int watchdog = TX_TIMEO;
55 module_param(watchdog, int, 0644);
56 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
57 
58 static int debug = -1;
59 module_param(debug, int, 0644);
60 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
61 
62 static int phyaddr = -1;
63 module_param(phyaddr, int, 0444);
64 MODULE_PARM_DESC(phyaddr, "Physical device address");
65 
66 #define STMMAC_TX_THRESH	(DMA_TX_SIZE / 4)
67 #define STMMAC_RX_THRESH	(DMA_RX_SIZE / 4)
68 
69 static int flow_ctrl = FLOW_AUTO;
70 module_param(flow_ctrl, int, 0644);
71 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
72 
73 static int pause = PAUSE_TIME;
74 module_param(pause, int, 0644);
75 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
76 
77 #define TC_DEFAULT 64
78 static int tc = TC_DEFAULT;
79 module_param(tc, int, 0644);
80 MODULE_PARM_DESC(tc, "DMA threshold control value");
81 
82 #define	DEFAULT_BUFSIZE	1536
83 static int buf_sz = DEFAULT_BUFSIZE;
84 module_param(buf_sz, int, 0644);
85 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
86 
87 #define	STMMAC_RX_COPYBREAK	256
88 
89 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
90 				      NETIF_MSG_LINK | NETIF_MSG_IFUP |
91 				      NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
92 
93 #define STMMAC_DEFAULT_LPI_TIMER	1000
94 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
95 module_param(eee_timer, int, 0644);
96 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
97 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
98 
99 /* By default the driver will use the ring mode to manage tx and rx descriptors,
100  * but allow user to force to use the chain instead of the ring
101  */
102 static unsigned int chain_mode;
103 module_param(chain_mode, int, 0444);
104 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
105 
106 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
107 
108 #ifdef CONFIG_DEBUG_FS
109 static const struct net_device_ops stmmac_netdev_ops;
110 static void stmmac_init_fs(struct net_device *dev);
111 static void stmmac_exit_fs(struct net_device *dev);
112 #endif
113 
114 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
115 
116 /**
117  * stmmac_verify_args - verify the driver parameters.
118  * Description: it checks the driver parameters and set a default in case of
119  * errors.
120  */
121 static void stmmac_verify_args(void)
122 {
123 	if (unlikely(watchdog < 0))
124 		watchdog = TX_TIMEO;
125 	if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
126 		buf_sz = DEFAULT_BUFSIZE;
127 	if (unlikely(flow_ctrl > 1))
128 		flow_ctrl = FLOW_AUTO;
129 	else if (likely(flow_ctrl < 0))
130 		flow_ctrl = FLOW_OFF;
131 	if (unlikely((pause < 0) || (pause > 0xffff)))
132 		pause = PAUSE_TIME;
133 	if (eee_timer < 0)
134 		eee_timer = STMMAC_DEFAULT_LPI_TIMER;
135 }
136 
137 /**
138  * stmmac_disable_all_queues - Disable all queues
139  * @priv: driver private structure
140  */
141 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
142 {
143 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
144 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
145 	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
146 	u32 queue;
147 
148 	for (queue = 0; queue < maxq; queue++) {
149 		struct stmmac_channel *ch = &priv->channel[queue];
150 
151 		if (queue < rx_queues_cnt)
152 			napi_disable(&ch->rx_napi);
153 		if (queue < tx_queues_cnt)
154 			napi_disable(&ch->tx_napi);
155 	}
156 }
157 
158 /**
159  * stmmac_enable_all_queues - Enable all queues
160  * @priv: driver private structure
161  */
162 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
163 {
164 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
165 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
166 	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
167 	u32 queue;
168 
169 	for (queue = 0; queue < maxq; queue++) {
170 		struct stmmac_channel *ch = &priv->channel[queue];
171 
172 		if (queue < rx_queues_cnt)
173 			napi_enable(&ch->rx_napi);
174 		if (queue < tx_queues_cnt)
175 			napi_enable(&ch->tx_napi);
176 	}
177 }
178 
179 /**
180  * stmmac_stop_all_queues - Stop all queues
181  * @priv: driver private structure
182  */
183 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
184 {
185 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
186 	u32 queue;
187 
188 	for (queue = 0; queue < tx_queues_cnt; queue++)
189 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
190 }
191 
192 /**
193  * stmmac_start_all_queues - Start all queues
194  * @priv: driver private structure
195  */
196 static void stmmac_start_all_queues(struct stmmac_priv *priv)
197 {
198 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
199 	u32 queue;
200 
201 	for (queue = 0; queue < tx_queues_cnt; queue++)
202 		netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
203 }
204 
205 static void stmmac_service_event_schedule(struct stmmac_priv *priv)
206 {
207 	if (!test_bit(STMMAC_DOWN, &priv->state) &&
208 	    !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
209 		queue_work(priv->wq, &priv->service_task);
210 }
211 
212 static void stmmac_global_err(struct stmmac_priv *priv)
213 {
214 	netif_carrier_off(priv->dev);
215 	set_bit(STMMAC_RESET_REQUESTED, &priv->state);
216 	stmmac_service_event_schedule(priv);
217 }
218 
219 /**
220  * stmmac_clk_csr_set - dynamically set the MDC clock
221  * @priv: driver private structure
222  * Description: this is to dynamically set the MDC clock according to the csr
223  * clock input.
224  * Note:
225  *	If a specific clk_csr value is passed from the platform
226  *	this means that the CSR Clock Range selection cannot be
227  *	changed at run-time and it is fixed (as reported in the driver
228  *	documentation). Viceversa the driver will try to set the MDC
229  *	clock dynamically according to the actual clock input.
230  */
231 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
232 {
233 	u32 clk_rate;
234 
235 	clk_rate = clk_get_rate(priv->plat->stmmac_clk);
236 
237 	/* Platform provided default clk_csr would be assumed valid
238 	 * for all other cases except for the below mentioned ones.
239 	 * For values higher than the IEEE 802.3 specified frequency
240 	 * we can not estimate the proper divider as it is not known
241 	 * the frequency of clk_csr_i. So we do not change the default
242 	 * divider.
243 	 */
244 	if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
245 		if (clk_rate < CSR_F_35M)
246 			priv->clk_csr = STMMAC_CSR_20_35M;
247 		else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
248 			priv->clk_csr = STMMAC_CSR_35_60M;
249 		else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
250 			priv->clk_csr = STMMAC_CSR_60_100M;
251 		else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
252 			priv->clk_csr = STMMAC_CSR_100_150M;
253 		else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
254 			priv->clk_csr = STMMAC_CSR_150_250M;
255 		else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
256 			priv->clk_csr = STMMAC_CSR_250_300M;
257 	}
258 
259 	if (priv->plat->has_sun8i) {
260 		if (clk_rate > 160000000)
261 			priv->clk_csr = 0x03;
262 		else if (clk_rate > 80000000)
263 			priv->clk_csr = 0x02;
264 		else if (clk_rate > 40000000)
265 			priv->clk_csr = 0x01;
266 		else
267 			priv->clk_csr = 0;
268 	}
269 
270 	if (priv->plat->has_xgmac) {
271 		if (clk_rate > 400000000)
272 			priv->clk_csr = 0x5;
273 		else if (clk_rate > 350000000)
274 			priv->clk_csr = 0x4;
275 		else if (clk_rate > 300000000)
276 			priv->clk_csr = 0x3;
277 		else if (clk_rate > 250000000)
278 			priv->clk_csr = 0x2;
279 		else if (clk_rate > 150000000)
280 			priv->clk_csr = 0x1;
281 		else
282 			priv->clk_csr = 0x0;
283 	}
284 }
285 
286 static void print_pkt(unsigned char *buf, int len)
287 {
288 	pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
289 	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
290 }
291 
292 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
293 {
294 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
295 	u32 avail;
296 
297 	if (tx_q->dirty_tx > tx_q->cur_tx)
298 		avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
299 	else
300 		avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
301 
302 	return avail;
303 }
304 
305 /**
306  * stmmac_rx_dirty - Get RX queue dirty
307  * @priv: driver private structure
308  * @queue: RX queue index
309  */
310 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
311 {
312 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
313 	u32 dirty;
314 
315 	if (rx_q->dirty_rx <= rx_q->cur_rx)
316 		dirty = rx_q->cur_rx - rx_q->dirty_rx;
317 	else
318 		dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
319 
320 	return dirty;
321 }
322 
323 /**
324  * stmmac_enable_eee_mode - check and enter in LPI mode
325  * @priv: driver private structure
326  * Description: this function is to verify and enter in LPI mode in case of
327  * EEE.
328  */
329 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
330 {
331 	u32 tx_cnt = priv->plat->tx_queues_to_use;
332 	u32 queue;
333 
334 	/* check if all TX queues have the work finished */
335 	for (queue = 0; queue < tx_cnt; queue++) {
336 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
337 
338 		if (tx_q->dirty_tx != tx_q->cur_tx)
339 			return; /* still unfinished work */
340 	}
341 
342 	/* Check and enter in LPI mode */
343 	if (!priv->tx_path_in_lpi_mode)
344 		stmmac_set_eee_mode(priv, priv->hw,
345 				priv->plat->en_tx_lpi_clockgating);
346 }
347 
348 /**
349  * stmmac_disable_eee_mode - disable and exit from LPI mode
350  * @priv: driver private structure
351  * Description: this function is to exit and disable EEE in case of
352  * LPI state is true. This is called by the xmit.
353  */
354 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
355 {
356 	stmmac_reset_eee_mode(priv, priv->hw);
357 	del_timer_sync(&priv->eee_ctrl_timer);
358 	priv->tx_path_in_lpi_mode = false;
359 }
360 
361 /**
362  * stmmac_eee_ctrl_timer - EEE TX SW timer.
363  * @arg : data hook
364  * Description:
365  *  if there is no data transfer and if we are not in LPI state,
366  *  then MAC Transmitter can be moved to LPI state.
367  */
368 static void stmmac_eee_ctrl_timer(struct timer_list *t)
369 {
370 	struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
371 
372 	stmmac_enable_eee_mode(priv);
373 	mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
374 }
375 
376 /**
377  * stmmac_eee_init - init EEE
378  * @priv: driver private structure
379  * Description:
380  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
381  *  can also manage EEE, this function enable the LPI state and start related
382  *  timer.
383  */
384 bool stmmac_eee_init(struct stmmac_priv *priv)
385 {
386 	int tx_lpi_timer = priv->tx_lpi_timer;
387 
388 	/* Using PCS we cannot dial with the phy registers at this stage
389 	 * so we do not support extra feature like EEE.
390 	 */
391 	if (priv->hw->pcs == STMMAC_PCS_TBI ||
392 	    priv->hw->pcs == STMMAC_PCS_RTBI)
393 		return false;
394 
395 	/* Check if MAC core supports the EEE feature. */
396 	if (!priv->dma_cap.eee)
397 		return false;
398 
399 	mutex_lock(&priv->lock);
400 
401 	/* Check if it needs to be deactivated */
402 	if (!priv->eee_active) {
403 		if (priv->eee_enabled) {
404 			netdev_dbg(priv->dev, "disable EEE\n");
405 			del_timer_sync(&priv->eee_ctrl_timer);
406 			stmmac_set_eee_timer(priv, priv->hw, 0, tx_lpi_timer);
407 		}
408 		mutex_unlock(&priv->lock);
409 		return false;
410 	}
411 
412 	if (priv->eee_active && !priv->eee_enabled) {
413 		timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
414 		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
415 		stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
416 				     tx_lpi_timer);
417 	}
418 
419 	mutex_unlock(&priv->lock);
420 	netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
421 	return true;
422 }
423 
424 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
425  * @priv: driver private structure
426  * @p : descriptor pointer
427  * @skb : the socket buffer
428  * Description :
429  * This function will read timestamp from the descriptor & pass it to stack.
430  * and also perform some sanity checks.
431  */
432 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
433 				   struct dma_desc *p, struct sk_buff *skb)
434 {
435 	struct skb_shared_hwtstamps shhwtstamp;
436 	bool found = false;
437 	u64 ns = 0;
438 
439 	if (!priv->hwts_tx_en)
440 		return;
441 
442 	/* exit if skb doesn't support hw tstamp */
443 	if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
444 		return;
445 
446 	/* check tx tstamp status */
447 	if (stmmac_get_tx_timestamp_status(priv, p)) {
448 		stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
449 		found = true;
450 	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
451 		found = true;
452 	}
453 
454 	if (found) {
455 		memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
456 		shhwtstamp.hwtstamp = ns_to_ktime(ns);
457 
458 		netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
459 		/* pass tstamp to stack */
460 		skb_tstamp_tx(skb, &shhwtstamp);
461 	}
462 }
463 
464 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
465  * @priv: driver private structure
466  * @p : descriptor pointer
467  * @np : next descriptor pointer
468  * @skb : the socket buffer
469  * Description :
470  * This function will read received packet's timestamp from the descriptor
471  * and pass it to stack. It also perform some sanity checks.
472  */
473 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
474 				   struct dma_desc *np, struct sk_buff *skb)
475 {
476 	struct skb_shared_hwtstamps *shhwtstamp = NULL;
477 	struct dma_desc *desc = p;
478 	u64 ns = 0;
479 
480 	if (!priv->hwts_rx_en)
481 		return;
482 	/* For GMAC4, the valid timestamp is from CTX next desc. */
483 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
484 		desc = np;
485 
486 	/* Check if timestamp is available */
487 	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
488 		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
489 		netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
490 		shhwtstamp = skb_hwtstamps(skb);
491 		memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
492 		shhwtstamp->hwtstamp = ns_to_ktime(ns);
493 	} else  {
494 		netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
495 	}
496 }
497 
498 /**
499  *  stmmac_hwtstamp_set - control hardware timestamping.
500  *  @dev: device pointer.
501  *  @ifr: An IOCTL specific structure, that can contain a pointer to
502  *  a proprietary structure used to pass information to the driver.
503  *  Description:
504  *  This function configures the MAC to enable/disable both outgoing(TX)
505  *  and incoming(RX) packets time stamping based on user input.
506  *  Return Value:
507  *  0 on success and an appropriate -ve integer on failure.
508  */
509 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
510 {
511 	struct stmmac_priv *priv = netdev_priv(dev);
512 	struct hwtstamp_config config;
513 	struct timespec64 now;
514 	u64 temp = 0;
515 	u32 ptp_v2 = 0;
516 	u32 tstamp_all = 0;
517 	u32 ptp_over_ipv4_udp = 0;
518 	u32 ptp_over_ipv6_udp = 0;
519 	u32 ptp_over_ethernet = 0;
520 	u32 snap_type_sel = 0;
521 	u32 ts_master_en = 0;
522 	u32 ts_event_en = 0;
523 	u32 sec_inc = 0;
524 	u32 value = 0;
525 	bool xmac;
526 
527 	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
528 
529 	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
530 		netdev_alert(priv->dev, "No support for HW time stamping\n");
531 		priv->hwts_tx_en = 0;
532 		priv->hwts_rx_en = 0;
533 
534 		return -EOPNOTSUPP;
535 	}
536 
537 	if (copy_from_user(&config, ifr->ifr_data,
538 			   sizeof(config)))
539 		return -EFAULT;
540 
541 	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
542 		   __func__, config.flags, config.tx_type, config.rx_filter);
543 
544 	/* reserved for future extensions */
545 	if (config.flags)
546 		return -EINVAL;
547 
548 	if (config.tx_type != HWTSTAMP_TX_OFF &&
549 	    config.tx_type != HWTSTAMP_TX_ON)
550 		return -ERANGE;
551 
552 	if (priv->adv_ts) {
553 		switch (config.rx_filter) {
554 		case HWTSTAMP_FILTER_NONE:
555 			/* time stamp no incoming packet at all */
556 			config.rx_filter = HWTSTAMP_FILTER_NONE;
557 			break;
558 
559 		case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
560 			/* PTP v1, UDP, any kind of event packet */
561 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
562 			/* 'xmac' hardware can support Sync, Pdelay_Req and
563 			 * Pdelay_resp by setting bit14 and bits17/16 to 01
564 			 * This leaves Delay_Req timestamps out.
565 			 * Enable all events *and* general purpose message
566 			 * timestamping
567 			 */
568 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
569 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
570 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
571 			break;
572 
573 		case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
574 			/* PTP v1, UDP, Sync packet */
575 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
576 			/* take time stamp for SYNC messages only */
577 			ts_event_en = PTP_TCR_TSEVNTENA;
578 
579 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
580 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
581 			break;
582 
583 		case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
584 			/* PTP v1, UDP, Delay_req packet */
585 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
586 			/* take time stamp for Delay_Req messages only */
587 			ts_master_en = PTP_TCR_TSMSTRENA;
588 			ts_event_en = PTP_TCR_TSEVNTENA;
589 
590 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
591 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
592 			break;
593 
594 		case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
595 			/* PTP v2, UDP, any kind of event packet */
596 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
597 			ptp_v2 = PTP_TCR_TSVER2ENA;
598 			/* take time stamp for all event messages */
599 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
600 
601 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
602 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
603 			break;
604 
605 		case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
606 			/* PTP v2, UDP, Sync packet */
607 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
608 			ptp_v2 = PTP_TCR_TSVER2ENA;
609 			/* take time stamp for SYNC messages only */
610 			ts_event_en = PTP_TCR_TSEVNTENA;
611 
612 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
613 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
614 			break;
615 
616 		case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
617 			/* PTP v2, UDP, Delay_req packet */
618 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
619 			ptp_v2 = PTP_TCR_TSVER2ENA;
620 			/* take time stamp for Delay_Req messages only */
621 			ts_master_en = PTP_TCR_TSMSTRENA;
622 			ts_event_en = PTP_TCR_TSEVNTENA;
623 
624 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
625 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
626 			break;
627 
628 		case HWTSTAMP_FILTER_PTP_V2_EVENT:
629 			/* PTP v2/802.AS1 any layer, any kind of event packet */
630 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
631 			ptp_v2 = PTP_TCR_TSVER2ENA;
632 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
633 			ts_event_en = PTP_TCR_TSEVNTENA;
634 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
635 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
636 			ptp_over_ethernet = PTP_TCR_TSIPENA;
637 			break;
638 
639 		case HWTSTAMP_FILTER_PTP_V2_SYNC:
640 			/* PTP v2/802.AS1, any layer, Sync packet */
641 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
642 			ptp_v2 = PTP_TCR_TSVER2ENA;
643 			/* take time stamp for SYNC messages only */
644 			ts_event_en = PTP_TCR_TSEVNTENA;
645 
646 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
647 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
648 			ptp_over_ethernet = PTP_TCR_TSIPENA;
649 			break;
650 
651 		case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
652 			/* PTP v2/802.AS1, any layer, Delay_req packet */
653 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
654 			ptp_v2 = PTP_TCR_TSVER2ENA;
655 			/* take time stamp for Delay_Req messages only */
656 			ts_master_en = PTP_TCR_TSMSTRENA;
657 			ts_event_en = PTP_TCR_TSEVNTENA;
658 
659 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
660 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
661 			ptp_over_ethernet = PTP_TCR_TSIPENA;
662 			break;
663 
664 		case HWTSTAMP_FILTER_NTP_ALL:
665 		case HWTSTAMP_FILTER_ALL:
666 			/* time stamp any incoming packet */
667 			config.rx_filter = HWTSTAMP_FILTER_ALL;
668 			tstamp_all = PTP_TCR_TSENALL;
669 			break;
670 
671 		default:
672 			return -ERANGE;
673 		}
674 	} else {
675 		switch (config.rx_filter) {
676 		case HWTSTAMP_FILTER_NONE:
677 			config.rx_filter = HWTSTAMP_FILTER_NONE;
678 			break;
679 		default:
680 			/* PTP v1, UDP, any kind of event packet */
681 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
682 			break;
683 		}
684 	}
685 	priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
686 	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
687 
688 	if (!priv->hwts_tx_en && !priv->hwts_rx_en)
689 		stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
690 	else {
691 		value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
692 			 tstamp_all | ptp_v2 | ptp_over_ethernet |
693 			 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
694 			 ts_master_en | snap_type_sel);
695 		stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
696 
697 		/* program Sub Second Increment reg */
698 		stmmac_config_sub_second_increment(priv,
699 				priv->ptpaddr, priv->plat->clk_ptp_rate,
700 				xmac, &sec_inc);
701 		temp = div_u64(1000000000ULL, sec_inc);
702 
703 		/* Store sub second increment and flags for later use */
704 		priv->sub_second_inc = sec_inc;
705 		priv->systime_flags = value;
706 
707 		/* calculate default added value:
708 		 * formula is :
709 		 * addend = (2^32)/freq_div_ratio;
710 		 * where, freq_div_ratio = 1e9ns/sec_inc
711 		 */
712 		temp = (u64)(temp << 32);
713 		priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
714 		stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
715 
716 		/* initialize system time */
717 		ktime_get_real_ts64(&now);
718 
719 		/* lower 32 bits of tv_sec are safe until y2106 */
720 		stmmac_init_systime(priv, priv->ptpaddr,
721 				(u32)now.tv_sec, now.tv_nsec);
722 	}
723 
724 	memcpy(&priv->tstamp_config, &config, sizeof(config));
725 
726 	return copy_to_user(ifr->ifr_data, &config,
727 			    sizeof(config)) ? -EFAULT : 0;
728 }
729 
730 /**
731  *  stmmac_hwtstamp_get - read hardware timestamping.
732  *  @dev: device pointer.
733  *  @ifr: An IOCTL specific structure, that can contain a pointer to
734  *  a proprietary structure used to pass information to the driver.
735  *  Description:
736  *  This function obtain the current hardware timestamping settings
737     as requested.
738  */
739 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
740 {
741 	struct stmmac_priv *priv = netdev_priv(dev);
742 	struct hwtstamp_config *config = &priv->tstamp_config;
743 
744 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
745 		return -EOPNOTSUPP;
746 
747 	return copy_to_user(ifr->ifr_data, config,
748 			    sizeof(*config)) ? -EFAULT : 0;
749 }
750 
751 /**
752  * stmmac_init_ptp - init PTP
753  * @priv: driver private structure
754  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
755  * This is done by looking at the HW cap. register.
756  * This function also registers the ptp driver.
757  */
758 static int stmmac_init_ptp(struct stmmac_priv *priv)
759 {
760 	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
761 
762 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
763 		return -EOPNOTSUPP;
764 
765 	priv->adv_ts = 0;
766 	/* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
767 	if (xmac && priv->dma_cap.atime_stamp)
768 		priv->adv_ts = 1;
769 	/* Dwmac 3.x core with extend_desc can support adv_ts */
770 	else if (priv->extend_desc && priv->dma_cap.atime_stamp)
771 		priv->adv_ts = 1;
772 
773 	if (priv->dma_cap.time_stamp)
774 		netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
775 
776 	if (priv->adv_ts)
777 		netdev_info(priv->dev,
778 			    "IEEE 1588-2008 Advanced Timestamp supported\n");
779 
780 	priv->hwts_tx_en = 0;
781 	priv->hwts_rx_en = 0;
782 
783 	stmmac_ptp_register(priv);
784 
785 	return 0;
786 }
787 
788 static void stmmac_release_ptp(struct stmmac_priv *priv)
789 {
790 	if (priv->plat->clk_ptp_ref)
791 		clk_disable_unprepare(priv->plat->clk_ptp_ref);
792 	stmmac_ptp_unregister(priv);
793 }
794 
795 /**
796  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
797  *  @priv: driver private structure
798  *  Description: It is used for configuring the flow control in all queues
799  */
800 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
801 {
802 	u32 tx_cnt = priv->plat->tx_queues_to_use;
803 
804 	stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
805 			priv->pause, tx_cnt);
806 }
807 
808 static void stmmac_validate(struct phylink_config *config,
809 			    unsigned long *supported,
810 			    struct phylink_link_state *state)
811 {
812 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
813 	__ETHTOOL_DECLARE_LINK_MODE_MASK(mac_supported) = { 0, };
814 	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
815 	int tx_cnt = priv->plat->tx_queues_to_use;
816 	int max_speed = priv->plat->max_speed;
817 
818 	phylink_set(mac_supported, 10baseT_Half);
819 	phylink_set(mac_supported, 10baseT_Full);
820 	phylink_set(mac_supported, 100baseT_Half);
821 	phylink_set(mac_supported, 100baseT_Full);
822 	phylink_set(mac_supported, 1000baseT_Half);
823 	phylink_set(mac_supported, 1000baseT_Full);
824 	phylink_set(mac_supported, 1000baseKX_Full);
825 
826 	phylink_set(mac_supported, Autoneg);
827 	phylink_set(mac_supported, Pause);
828 	phylink_set(mac_supported, Asym_Pause);
829 	phylink_set_port_modes(mac_supported);
830 
831 	/* Cut down 1G if asked to */
832 	if ((max_speed > 0) && (max_speed < 1000)) {
833 		phylink_set(mask, 1000baseT_Full);
834 		phylink_set(mask, 1000baseX_Full);
835 	} else if (priv->plat->has_xgmac) {
836 		if (!max_speed || (max_speed >= 2500)) {
837 			phylink_set(mac_supported, 2500baseT_Full);
838 			phylink_set(mac_supported, 2500baseX_Full);
839 		}
840 		if (!max_speed || (max_speed >= 5000)) {
841 			phylink_set(mac_supported, 5000baseT_Full);
842 		}
843 		if (!max_speed || (max_speed >= 10000)) {
844 			phylink_set(mac_supported, 10000baseSR_Full);
845 			phylink_set(mac_supported, 10000baseLR_Full);
846 			phylink_set(mac_supported, 10000baseER_Full);
847 			phylink_set(mac_supported, 10000baseLRM_Full);
848 			phylink_set(mac_supported, 10000baseT_Full);
849 			phylink_set(mac_supported, 10000baseKX4_Full);
850 			phylink_set(mac_supported, 10000baseKR_Full);
851 		}
852 	}
853 
854 	/* Half-Duplex can only work with single queue */
855 	if (tx_cnt > 1) {
856 		phylink_set(mask, 10baseT_Half);
857 		phylink_set(mask, 100baseT_Half);
858 		phylink_set(mask, 1000baseT_Half);
859 	}
860 
861 	bitmap_and(supported, supported, mac_supported,
862 		   __ETHTOOL_LINK_MODE_MASK_NBITS);
863 	bitmap_andnot(supported, supported, mask,
864 		      __ETHTOOL_LINK_MODE_MASK_NBITS);
865 	bitmap_and(state->advertising, state->advertising, mac_supported,
866 		   __ETHTOOL_LINK_MODE_MASK_NBITS);
867 	bitmap_andnot(state->advertising, state->advertising, mask,
868 		      __ETHTOOL_LINK_MODE_MASK_NBITS);
869 }
870 
871 static void stmmac_mac_pcs_get_state(struct phylink_config *config,
872 				     struct phylink_link_state *state)
873 {
874 	state->link = 0;
875 }
876 
877 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
878 			      const struct phylink_link_state *state)
879 {
880 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
881 	u32 ctrl;
882 
883 	ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
884 	ctrl &= ~priv->hw->link.speed_mask;
885 
886 	if (state->interface == PHY_INTERFACE_MODE_USXGMII) {
887 		switch (state->speed) {
888 		case SPEED_10000:
889 			ctrl |= priv->hw->link.xgmii.speed10000;
890 			break;
891 		case SPEED_5000:
892 			ctrl |= priv->hw->link.xgmii.speed5000;
893 			break;
894 		case SPEED_2500:
895 			ctrl |= priv->hw->link.xgmii.speed2500;
896 			break;
897 		default:
898 			return;
899 		}
900 	} else {
901 		switch (state->speed) {
902 		case SPEED_2500:
903 			ctrl |= priv->hw->link.speed2500;
904 			break;
905 		case SPEED_1000:
906 			ctrl |= priv->hw->link.speed1000;
907 			break;
908 		case SPEED_100:
909 			ctrl |= priv->hw->link.speed100;
910 			break;
911 		case SPEED_10:
912 			ctrl |= priv->hw->link.speed10;
913 			break;
914 		default:
915 			return;
916 		}
917 	}
918 
919 	priv->speed = state->speed;
920 
921 	if (priv->plat->fix_mac_speed)
922 		priv->plat->fix_mac_speed(priv->plat->bsp_priv, state->speed);
923 
924 	if (!state->duplex)
925 		ctrl &= ~priv->hw->link.duplex;
926 	else
927 		ctrl |= priv->hw->link.duplex;
928 
929 	/* Flow Control operation */
930 	if (state->pause)
931 		stmmac_mac_flow_ctrl(priv, state->duplex);
932 
933 	writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
934 }
935 
936 static void stmmac_mac_an_restart(struct phylink_config *config)
937 {
938 	/* Not Supported */
939 }
940 
941 static void stmmac_mac_link_down(struct phylink_config *config,
942 				 unsigned int mode, phy_interface_t interface)
943 {
944 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
945 
946 	stmmac_mac_set(priv, priv->ioaddr, false);
947 	priv->eee_active = false;
948 	stmmac_eee_init(priv);
949 	stmmac_set_eee_pls(priv, priv->hw, false);
950 }
951 
952 static void stmmac_mac_link_up(struct phylink_config *config,
953 			       unsigned int mode, phy_interface_t interface,
954 			       struct phy_device *phy)
955 {
956 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
957 
958 	stmmac_mac_set(priv, priv->ioaddr, true);
959 	if (phy && priv->dma_cap.eee) {
960 		priv->eee_active = phy_init_eee(phy, 1) >= 0;
961 		priv->eee_enabled = stmmac_eee_init(priv);
962 		stmmac_set_eee_pls(priv, priv->hw, true);
963 	}
964 }
965 
966 static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
967 	.validate = stmmac_validate,
968 	.mac_pcs_get_state = stmmac_mac_pcs_get_state,
969 	.mac_config = stmmac_mac_config,
970 	.mac_an_restart = stmmac_mac_an_restart,
971 	.mac_link_down = stmmac_mac_link_down,
972 	.mac_link_up = stmmac_mac_link_up,
973 };
974 
975 /**
976  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
977  * @priv: driver private structure
978  * Description: this is to verify if the HW supports the PCS.
979  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
980  * configured for the TBI, RTBI, or SGMII PHY interface.
981  */
982 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
983 {
984 	int interface = priv->plat->interface;
985 
986 	if (priv->dma_cap.pcs) {
987 		if ((interface == PHY_INTERFACE_MODE_RGMII) ||
988 		    (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
989 		    (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
990 		    (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
991 			netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
992 			priv->hw->pcs = STMMAC_PCS_RGMII;
993 		} else if (interface == PHY_INTERFACE_MODE_SGMII) {
994 			netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
995 			priv->hw->pcs = STMMAC_PCS_SGMII;
996 		}
997 	}
998 }
999 
1000 /**
1001  * stmmac_init_phy - PHY initialization
1002  * @dev: net device structure
1003  * Description: it initializes the driver's PHY state, and attaches the PHY
1004  * to the mac driver.
1005  *  Return value:
1006  *  0 on success
1007  */
1008 static int stmmac_init_phy(struct net_device *dev)
1009 {
1010 	struct stmmac_priv *priv = netdev_priv(dev);
1011 	struct device_node *node;
1012 	int ret;
1013 
1014 	node = priv->plat->phylink_node;
1015 
1016 	if (node)
1017 		ret = phylink_of_phy_connect(priv->phylink, node, 0);
1018 
1019 	/* Some DT bindings do not set-up the PHY handle. Let's try to
1020 	 * manually parse it
1021 	 */
1022 	if (!node || ret) {
1023 		int addr = priv->plat->phy_addr;
1024 		struct phy_device *phydev;
1025 
1026 		phydev = mdiobus_get_phy(priv->mii, addr);
1027 		if (!phydev) {
1028 			netdev_err(priv->dev, "no phy at addr %d\n", addr);
1029 			return -ENODEV;
1030 		}
1031 
1032 		ret = phylink_connect_phy(priv->phylink, phydev);
1033 	}
1034 
1035 	return ret;
1036 }
1037 
1038 static int stmmac_phy_setup(struct stmmac_priv *priv)
1039 {
1040 	struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node);
1041 	int mode = priv->plat->phy_interface;
1042 	struct phylink *phylink;
1043 
1044 	priv->phylink_config.dev = &priv->dev->dev;
1045 	priv->phylink_config.type = PHYLINK_NETDEV;
1046 
1047 	phylink = phylink_create(&priv->phylink_config, fwnode,
1048 				 mode, &stmmac_phylink_mac_ops);
1049 	if (IS_ERR(phylink))
1050 		return PTR_ERR(phylink);
1051 
1052 	priv->phylink = phylink;
1053 	return 0;
1054 }
1055 
1056 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
1057 {
1058 	u32 rx_cnt = priv->plat->rx_queues_to_use;
1059 	void *head_rx;
1060 	u32 queue;
1061 
1062 	/* Display RX rings */
1063 	for (queue = 0; queue < rx_cnt; queue++) {
1064 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1065 
1066 		pr_info("\tRX Queue %u rings\n", queue);
1067 
1068 		if (priv->extend_desc)
1069 			head_rx = (void *)rx_q->dma_erx;
1070 		else
1071 			head_rx = (void *)rx_q->dma_rx;
1072 
1073 		/* Display RX ring */
1074 		stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true);
1075 	}
1076 }
1077 
1078 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1079 {
1080 	u32 tx_cnt = priv->plat->tx_queues_to_use;
1081 	void *head_tx;
1082 	u32 queue;
1083 
1084 	/* Display TX rings */
1085 	for (queue = 0; queue < tx_cnt; queue++) {
1086 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1087 
1088 		pr_info("\tTX Queue %d rings\n", queue);
1089 
1090 		if (priv->extend_desc)
1091 			head_tx = (void *)tx_q->dma_etx;
1092 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1093 			head_tx = (void *)tx_q->dma_entx;
1094 		else
1095 			head_tx = (void *)tx_q->dma_tx;
1096 
1097 		stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false);
1098 	}
1099 }
1100 
1101 static void stmmac_display_rings(struct stmmac_priv *priv)
1102 {
1103 	/* Display RX ring */
1104 	stmmac_display_rx_rings(priv);
1105 
1106 	/* Display TX ring */
1107 	stmmac_display_tx_rings(priv);
1108 }
1109 
1110 static int stmmac_set_bfsize(int mtu, int bufsize)
1111 {
1112 	int ret = bufsize;
1113 
1114 	if (mtu >= BUF_SIZE_8KiB)
1115 		ret = BUF_SIZE_16KiB;
1116 	else if (mtu >= BUF_SIZE_4KiB)
1117 		ret = BUF_SIZE_8KiB;
1118 	else if (mtu >= BUF_SIZE_2KiB)
1119 		ret = BUF_SIZE_4KiB;
1120 	else if (mtu > DEFAULT_BUFSIZE)
1121 		ret = BUF_SIZE_2KiB;
1122 	else
1123 		ret = DEFAULT_BUFSIZE;
1124 
1125 	return ret;
1126 }
1127 
1128 /**
1129  * stmmac_clear_rx_descriptors - clear RX descriptors
1130  * @priv: driver private structure
1131  * @queue: RX queue index
1132  * Description: this function is called to clear the RX descriptors
1133  * in case of both basic and extended descriptors are used.
1134  */
1135 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1136 {
1137 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1138 	int i;
1139 
1140 	/* Clear the RX descriptors */
1141 	for (i = 0; i < DMA_RX_SIZE; i++)
1142 		if (priv->extend_desc)
1143 			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1144 					priv->use_riwt, priv->mode,
1145 					(i == DMA_RX_SIZE - 1),
1146 					priv->dma_buf_sz);
1147 		else
1148 			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1149 					priv->use_riwt, priv->mode,
1150 					(i == DMA_RX_SIZE - 1),
1151 					priv->dma_buf_sz);
1152 }
1153 
1154 /**
1155  * stmmac_clear_tx_descriptors - clear tx descriptors
1156  * @priv: driver private structure
1157  * @queue: TX queue index.
1158  * Description: this function is called to clear the TX descriptors
1159  * in case of both basic and extended descriptors are used.
1160  */
1161 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1162 {
1163 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1164 	int i;
1165 
1166 	/* Clear the TX descriptors */
1167 	for (i = 0; i < DMA_TX_SIZE; i++) {
1168 		int last = (i == (DMA_TX_SIZE - 1));
1169 		struct dma_desc *p;
1170 
1171 		if (priv->extend_desc)
1172 			p = &tx_q->dma_etx[i].basic;
1173 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1174 			p = &tx_q->dma_entx[i].basic;
1175 		else
1176 			p = &tx_q->dma_tx[i];
1177 
1178 		stmmac_init_tx_desc(priv, p, priv->mode, last);
1179 	}
1180 }
1181 
1182 /**
1183  * stmmac_clear_descriptors - clear descriptors
1184  * @priv: driver private structure
1185  * Description: this function is called to clear the TX and RX descriptors
1186  * in case of both basic and extended descriptors are used.
1187  */
1188 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1189 {
1190 	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1191 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1192 	u32 queue;
1193 
1194 	/* Clear the RX descriptors */
1195 	for (queue = 0; queue < rx_queue_cnt; queue++)
1196 		stmmac_clear_rx_descriptors(priv, queue);
1197 
1198 	/* Clear the TX descriptors */
1199 	for (queue = 0; queue < tx_queue_cnt; queue++)
1200 		stmmac_clear_tx_descriptors(priv, queue);
1201 }
1202 
1203 /**
1204  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1205  * @priv: driver private structure
1206  * @p: descriptor pointer
1207  * @i: descriptor index
1208  * @flags: gfp flag
1209  * @queue: RX queue index
1210  * Description: this function is called to allocate a receive buffer, perform
1211  * the DMA mapping and init the descriptor.
1212  */
1213 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1214 				  int i, gfp_t flags, u32 queue)
1215 {
1216 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1217 	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1218 
1219 	buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
1220 	if (!buf->page)
1221 		return -ENOMEM;
1222 
1223 	if (priv->sph) {
1224 		buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
1225 		if (!buf->sec_page)
1226 			return -ENOMEM;
1227 
1228 		buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
1229 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
1230 	} else {
1231 		buf->sec_page = NULL;
1232 	}
1233 
1234 	buf->addr = page_pool_get_dma_addr(buf->page);
1235 	stmmac_set_desc_addr(priv, p, buf->addr);
1236 	if (priv->dma_buf_sz == BUF_SIZE_16KiB)
1237 		stmmac_init_desc3(priv, p);
1238 
1239 	return 0;
1240 }
1241 
1242 /**
1243  * stmmac_free_rx_buffer - free RX dma buffers
1244  * @priv: private structure
1245  * @queue: RX queue index
1246  * @i: buffer index.
1247  */
1248 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1249 {
1250 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1251 	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1252 
1253 	if (buf->page)
1254 		page_pool_put_page(rx_q->page_pool, buf->page, false);
1255 	buf->page = NULL;
1256 
1257 	if (buf->sec_page)
1258 		page_pool_put_page(rx_q->page_pool, buf->sec_page, false);
1259 	buf->sec_page = NULL;
1260 }
1261 
1262 /**
1263  * stmmac_free_tx_buffer - free RX dma buffers
1264  * @priv: private structure
1265  * @queue: RX queue index
1266  * @i: buffer index.
1267  */
1268 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1269 {
1270 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1271 
1272 	if (tx_q->tx_skbuff_dma[i].buf) {
1273 		if (tx_q->tx_skbuff_dma[i].map_as_page)
1274 			dma_unmap_page(priv->device,
1275 				       tx_q->tx_skbuff_dma[i].buf,
1276 				       tx_q->tx_skbuff_dma[i].len,
1277 				       DMA_TO_DEVICE);
1278 		else
1279 			dma_unmap_single(priv->device,
1280 					 tx_q->tx_skbuff_dma[i].buf,
1281 					 tx_q->tx_skbuff_dma[i].len,
1282 					 DMA_TO_DEVICE);
1283 	}
1284 
1285 	if (tx_q->tx_skbuff[i]) {
1286 		dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1287 		tx_q->tx_skbuff[i] = NULL;
1288 		tx_q->tx_skbuff_dma[i].buf = 0;
1289 		tx_q->tx_skbuff_dma[i].map_as_page = false;
1290 	}
1291 }
1292 
1293 /**
1294  * init_dma_rx_desc_rings - init the RX descriptor rings
1295  * @dev: net device structure
1296  * @flags: gfp flag.
1297  * Description: this function initializes the DMA RX descriptors
1298  * and allocates the socket buffers. It supports the chained and ring
1299  * modes.
1300  */
1301 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1302 {
1303 	struct stmmac_priv *priv = netdev_priv(dev);
1304 	u32 rx_count = priv->plat->rx_queues_to_use;
1305 	int ret = -ENOMEM;
1306 	int queue;
1307 	int i;
1308 
1309 	/* RX INITIALIZATION */
1310 	netif_dbg(priv, probe, priv->dev,
1311 		  "SKB addresses:\nskb\t\tskb data\tdma data\n");
1312 
1313 	for (queue = 0; queue < rx_count; queue++) {
1314 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1315 
1316 		netif_dbg(priv, probe, priv->dev,
1317 			  "(%s) dma_rx_phy=0x%08x\n", __func__,
1318 			  (u32)rx_q->dma_rx_phy);
1319 
1320 		stmmac_clear_rx_descriptors(priv, queue);
1321 
1322 		for (i = 0; i < DMA_RX_SIZE; i++) {
1323 			struct dma_desc *p;
1324 
1325 			if (priv->extend_desc)
1326 				p = &((rx_q->dma_erx + i)->basic);
1327 			else
1328 				p = rx_q->dma_rx + i;
1329 
1330 			ret = stmmac_init_rx_buffers(priv, p, i, flags,
1331 						     queue);
1332 			if (ret)
1333 				goto err_init_rx_buffers;
1334 		}
1335 
1336 		rx_q->cur_rx = 0;
1337 		rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1338 
1339 		/* Setup the chained descriptor addresses */
1340 		if (priv->mode == STMMAC_CHAIN_MODE) {
1341 			if (priv->extend_desc)
1342 				stmmac_mode_init(priv, rx_q->dma_erx,
1343 						rx_q->dma_rx_phy, DMA_RX_SIZE, 1);
1344 			else
1345 				stmmac_mode_init(priv, rx_q->dma_rx,
1346 						rx_q->dma_rx_phy, DMA_RX_SIZE, 0);
1347 		}
1348 	}
1349 
1350 	return 0;
1351 
1352 err_init_rx_buffers:
1353 	while (queue >= 0) {
1354 		while (--i >= 0)
1355 			stmmac_free_rx_buffer(priv, queue, i);
1356 
1357 		if (queue == 0)
1358 			break;
1359 
1360 		i = DMA_RX_SIZE;
1361 		queue--;
1362 	}
1363 
1364 	return ret;
1365 }
1366 
1367 /**
1368  * init_dma_tx_desc_rings - init the TX descriptor rings
1369  * @dev: net device structure.
1370  * Description: this function initializes the DMA TX descriptors
1371  * and allocates the socket buffers. It supports the chained and ring
1372  * modes.
1373  */
1374 static int init_dma_tx_desc_rings(struct net_device *dev)
1375 {
1376 	struct stmmac_priv *priv = netdev_priv(dev);
1377 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1378 	u32 queue;
1379 	int i;
1380 
1381 	for (queue = 0; queue < tx_queue_cnt; queue++) {
1382 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1383 
1384 		netif_dbg(priv, probe, priv->dev,
1385 			  "(%s) dma_tx_phy=0x%08x\n", __func__,
1386 			 (u32)tx_q->dma_tx_phy);
1387 
1388 		/* Setup the chained descriptor addresses */
1389 		if (priv->mode == STMMAC_CHAIN_MODE) {
1390 			if (priv->extend_desc)
1391 				stmmac_mode_init(priv, tx_q->dma_etx,
1392 						tx_q->dma_tx_phy, DMA_TX_SIZE, 1);
1393 			else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
1394 				stmmac_mode_init(priv, tx_q->dma_tx,
1395 						tx_q->dma_tx_phy, DMA_TX_SIZE, 0);
1396 		}
1397 
1398 		for (i = 0; i < DMA_TX_SIZE; i++) {
1399 			struct dma_desc *p;
1400 			if (priv->extend_desc)
1401 				p = &((tx_q->dma_etx + i)->basic);
1402 			else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1403 				p = &((tx_q->dma_entx + i)->basic);
1404 			else
1405 				p = tx_q->dma_tx + i;
1406 
1407 			stmmac_clear_desc(priv, p);
1408 
1409 			tx_q->tx_skbuff_dma[i].buf = 0;
1410 			tx_q->tx_skbuff_dma[i].map_as_page = false;
1411 			tx_q->tx_skbuff_dma[i].len = 0;
1412 			tx_q->tx_skbuff_dma[i].last_segment = false;
1413 			tx_q->tx_skbuff[i] = NULL;
1414 		}
1415 
1416 		tx_q->dirty_tx = 0;
1417 		tx_q->cur_tx = 0;
1418 		tx_q->mss = 0;
1419 
1420 		netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1421 	}
1422 
1423 	return 0;
1424 }
1425 
1426 /**
1427  * init_dma_desc_rings - init the RX/TX descriptor rings
1428  * @dev: net device structure
1429  * @flags: gfp flag.
1430  * Description: this function initializes the DMA RX/TX descriptors
1431  * and allocates the socket buffers. It supports the chained and ring
1432  * modes.
1433  */
1434 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1435 {
1436 	struct stmmac_priv *priv = netdev_priv(dev);
1437 	int ret;
1438 
1439 	ret = init_dma_rx_desc_rings(dev, flags);
1440 	if (ret)
1441 		return ret;
1442 
1443 	ret = init_dma_tx_desc_rings(dev);
1444 
1445 	stmmac_clear_descriptors(priv);
1446 
1447 	if (netif_msg_hw(priv))
1448 		stmmac_display_rings(priv);
1449 
1450 	return ret;
1451 }
1452 
1453 /**
1454  * dma_free_rx_skbufs - free RX dma buffers
1455  * @priv: private structure
1456  * @queue: RX queue index
1457  */
1458 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1459 {
1460 	int i;
1461 
1462 	for (i = 0; i < DMA_RX_SIZE; i++)
1463 		stmmac_free_rx_buffer(priv, queue, i);
1464 }
1465 
1466 /**
1467  * dma_free_tx_skbufs - free TX dma buffers
1468  * @priv: private structure
1469  * @queue: TX queue index
1470  */
1471 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1472 {
1473 	int i;
1474 
1475 	for (i = 0; i < DMA_TX_SIZE; i++)
1476 		stmmac_free_tx_buffer(priv, queue, i);
1477 }
1478 
1479 /**
1480  * free_dma_rx_desc_resources - free RX dma desc resources
1481  * @priv: private structure
1482  */
1483 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1484 {
1485 	u32 rx_count = priv->plat->rx_queues_to_use;
1486 	u32 queue;
1487 
1488 	/* Free RX queue resources */
1489 	for (queue = 0; queue < rx_count; queue++) {
1490 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1491 
1492 		/* Release the DMA RX socket buffers */
1493 		dma_free_rx_skbufs(priv, queue);
1494 
1495 		/* Free DMA regions of consistent memory previously allocated */
1496 		if (!priv->extend_desc)
1497 			dma_free_coherent(priv->device,
1498 					  DMA_RX_SIZE * sizeof(struct dma_desc),
1499 					  rx_q->dma_rx, rx_q->dma_rx_phy);
1500 		else
1501 			dma_free_coherent(priv->device, DMA_RX_SIZE *
1502 					  sizeof(struct dma_extended_desc),
1503 					  rx_q->dma_erx, rx_q->dma_rx_phy);
1504 
1505 		kfree(rx_q->buf_pool);
1506 		if (rx_q->page_pool)
1507 			page_pool_destroy(rx_q->page_pool);
1508 	}
1509 }
1510 
1511 /**
1512  * free_dma_tx_desc_resources - free TX dma desc resources
1513  * @priv: private structure
1514  */
1515 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1516 {
1517 	u32 tx_count = priv->plat->tx_queues_to_use;
1518 	u32 queue;
1519 
1520 	/* Free TX queue resources */
1521 	for (queue = 0; queue < tx_count; queue++) {
1522 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1523 		size_t size;
1524 		void *addr;
1525 
1526 		/* Release the DMA TX socket buffers */
1527 		dma_free_tx_skbufs(priv, queue);
1528 
1529 		if (priv->extend_desc) {
1530 			size = sizeof(struct dma_extended_desc);
1531 			addr = tx_q->dma_etx;
1532 		} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
1533 			size = sizeof(struct dma_edesc);
1534 			addr = tx_q->dma_entx;
1535 		} else {
1536 			size = sizeof(struct dma_desc);
1537 			addr = tx_q->dma_tx;
1538 		}
1539 
1540 		size *= DMA_TX_SIZE;
1541 
1542 		dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);
1543 
1544 		kfree(tx_q->tx_skbuff_dma);
1545 		kfree(tx_q->tx_skbuff);
1546 	}
1547 }
1548 
1549 /**
1550  * alloc_dma_rx_desc_resources - alloc RX resources.
1551  * @priv: private structure
1552  * Description: according to which descriptor can be used (extend or basic)
1553  * this function allocates the resources for TX and RX paths. In case of
1554  * reception, for example, it pre-allocated the RX socket buffer in order to
1555  * allow zero-copy mechanism.
1556  */
1557 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1558 {
1559 	u32 rx_count = priv->plat->rx_queues_to_use;
1560 	int ret = -ENOMEM;
1561 	u32 queue;
1562 
1563 	/* RX queues buffers and DMA */
1564 	for (queue = 0; queue < rx_count; queue++) {
1565 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1566 		struct page_pool_params pp_params = { 0 };
1567 		unsigned int num_pages;
1568 
1569 		rx_q->queue_index = queue;
1570 		rx_q->priv_data = priv;
1571 
1572 		pp_params.flags = PP_FLAG_DMA_MAP;
1573 		pp_params.pool_size = DMA_RX_SIZE;
1574 		num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE);
1575 		pp_params.order = ilog2(num_pages);
1576 		pp_params.nid = dev_to_node(priv->device);
1577 		pp_params.dev = priv->device;
1578 		pp_params.dma_dir = DMA_FROM_DEVICE;
1579 
1580 		rx_q->page_pool = page_pool_create(&pp_params);
1581 		if (IS_ERR(rx_q->page_pool)) {
1582 			ret = PTR_ERR(rx_q->page_pool);
1583 			rx_q->page_pool = NULL;
1584 			goto err_dma;
1585 		}
1586 
1587 		rx_q->buf_pool = kcalloc(DMA_RX_SIZE, sizeof(*rx_q->buf_pool),
1588 					 GFP_KERNEL);
1589 		if (!rx_q->buf_pool)
1590 			goto err_dma;
1591 
1592 		if (priv->extend_desc) {
1593 			rx_q->dma_erx = dma_alloc_coherent(priv->device,
1594 							   DMA_RX_SIZE * sizeof(struct dma_extended_desc),
1595 							   &rx_q->dma_rx_phy,
1596 							   GFP_KERNEL);
1597 			if (!rx_q->dma_erx)
1598 				goto err_dma;
1599 
1600 		} else {
1601 			rx_q->dma_rx = dma_alloc_coherent(priv->device,
1602 							  DMA_RX_SIZE * sizeof(struct dma_desc),
1603 							  &rx_q->dma_rx_phy,
1604 							  GFP_KERNEL);
1605 			if (!rx_q->dma_rx)
1606 				goto err_dma;
1607 		}
1608 	}
1609 
1610 	return 0;
1611 
1612 err_dma:
1613 	free_dma_rx_desc_resources(priv);
1614 
1615 	return ret;
1616 }
1617 
1618 /**
1619  * alloc_dma_tx_desc_resources - alloc TX resources.
1620  * @priv: private structure
1621  * Description: according to which descriptor can be used (extend or basic)
1622  * this function allocates the resources for TX and RX paths. In case of
1623  * reception, for example, it pre-allocated the RX socket buffer in order to
1624  * allow zero-copy mechanism.
1625  */
1626 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1627 {
1628 	u32 tx_count = priv->plat->tx_queues_to_use;
1629 	int ret = -ENOMEM;
1630 	u32 queue;
1631 
1632 	/* TX queues buffers and DMA */
1633 	for (queue = 0; queue < tx_count; queue++) {
1634 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1635 		size_t size;
1636 		void *addr;
1637 
1638 		tx_q->queue_index = queue;
1639 		tx_q->priv_data = priv;
1640 
1641 		tx_q->tx_skbuff_dma = kcalloc(DMA_TX_SIZE,
1642 					      sizeof(*tx_q->tx_skbuff_dma),
1643 					      GFP_KERNEL);
1644 		if (!tx_q->tx_skbuff_dma)
1645 			goto err_dma;
1646 
1647 		tx_q->tx_skbuff = kcalloc(DMA_TX_SIZE,
1648 					  sizeof(struct sk_buff *),
1649 					  GFP_KERNEL);
1650 		if (!tx_q->tx_skbuff)
1651 			goto err_dma;
1652 
1653 		if (priv->extend_desc)
1654 			size = sizeof(struct dma_extended_desc);
1655 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1656 			size = sizeof(struct dma_edesc);
1657 		else
1658 			size = sizeof(struct dma_desc);
1659 
1660 		size *= DMA_TX_SIZE;
1661 
1662 		addr = dma_alloc_coherent(priv->device, size,
1663 					  &tx_q->dma_tx_phy, GFP_KERNEL);
1664 		if (!addr)
1665 			goto err_dma;
1666 
1667 		if (priv->extend_desc)
1668 			tx_q->dma_etx = addr;
1669 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1670 			tx_q->dma_entx = addr;
1671 		else
1672 			tx_q->dma_tx = addr;
1673 	}
1674 
1675 	return 0;
1676 
1677 err_dma:
1678 	free_dma_tx_desc_resources(priv);
1679 	return ret;
1680 }
1681 
1682 /**
1683  * alloc_dma_desc_resources - alloc TX/RX resources.
1684  * @priv: private structure
1685  * Description: according to which descriptor can be used (extend or basic)
1686  * this function allocates the resources for TX and RX paths. In case of
1687  * reception, for example, it pre-allocated the RX socket buffer in order to
1688  * allow zero-copy mechanism.
1689  */
1690 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1691 {
1692 	/* RX Allocation */
1693 	int ret = alloc_dma_rx_desc_resources(priv);
1694 
1695 	if (ret)
1696 		return ret;
1697 
1698 	ret = alloc_dma_tx_desc_resources(priv);
1699 
1700 	return ret;
1701 }
1702 
1703 /**
1704  * free_dma_desc_resources - free dma desc resources
1705  * @priv: private structure
1706  */
1707 static void free_dma_desc_resources(struct stmmac_priv *priv)
1708 {
1709 	/* Release the DMA RX socket buffers */
1710 	free_dma_rx_desc_resources(priv);
1711 
1712 	/* Release the DMA TX socket buffers */
1713 	free_dma_tx_desc_resources(priv);
1714 }
1715 
1716 /**
1717  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1718  *  @priv: driver private structure
1719  *  Description: It is used for enabling the rx queues in the MAC
1720  */
1721 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1722 {
1723 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
1724 	int queue;
1725 	u8 mode;
1726 
1727 	for (queue = 0; queue < rx_queues_count; queue++) {
1728 		mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1729 		stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
1730 	}
1731 }
1732 
1733 /**
1734  * stmmac_start_rx_dma - start RX DMA channel
1735  * @priv: driver private structure
1736  * @chan: RX channel index
1737  * Description:
1738  * This starts a RX DMA channel
1739  */
1740 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1741 {
1742 	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1743 	stmmac_start_rx(priv, priv->ioaddr, chan);
1744 }
1745 
1746 /**
1747  * stmmac_start_tx_dma - start TX DMA channel
1748  * @priv: driver private structure
1749  * @chan: TX channel index
1750  * Description:
1751  * This starts a TX DMA channel
1752  */
1753 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1754 {
1755 	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1756 	stmmac_start_tx(priv, priv->ioaddr, chan);
1757 }
1758 
1759 /**
1760  * stmmac_stop_rx_dma - stop RX DMA channel
1761  * @priv: driver private structure
1762  * @chan: RX channel index
1763  * Description:
1764  * This stops a RX DMA channel
1765  */
1766 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1767 {
1768 	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1769 	stmmac_stop_rx(priv, priv->ioaddr, chan);
1770 }
1771 
1772 /**
1773  * stmmac_stop_tx_dma - stop TX DMA channel
1774  * @priv: driver private structure
1775  * @chan: TX channel index
1776  * Description:
1777  * This stops a TX DMA channel
1778  */
1779 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1780 {
1781 	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1782 	stmmac_stop_tx(priv, priv->ioaddr, chan);
1783 }
1784 
1785 /**
1786  * stmmac_start_all_dma - start all RX and TX DMA channels
1787  * @priv: driver private structure
1788  * Description:
1789  * This starts all the RX and TX DMA channels
1790  */
1791 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1792 {
1793 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
1794 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
1795 	u32 chan = 0;
1796 
1797 	for (chan = 0; chan < rx_channels_count; chan++)
1798 		stmmac_start_rx_dma(priv, chan);
1799 
1800 	for (chan = 0; chan < tx_channels_count; chan++)
1801 		stmmac_start_tx_dma(priv, chan);
1802 }
1803 
1804 /**
1805  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1806  * @priv: driver private structure
1807  * Description:
1808  * This stops the RX and TX DMA channels
1809  */
1810 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1811 {
1812 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
1813 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
1814 	u32 chan = 0;
1815 
1816 	for (chan = 0; chan < rx_channels_count; chan++)
1817 		stmmac_stop_rx_dma(priv, chan);
1818 
1819 	for (chan = 0; chan < tx_channels_count; chan++)
1820 		stmmac_stop_tx_dma(priv, chan);
1821 }
1822 
1823 /**
1824  *  stmmac_dma_operation_mode - HW DMA operation mode
1825  *  @priv: driver private structure
1826  *  Description: it is used for configuring the DMA operation mode register in
1827  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1828  */
1829 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1830 {
1831 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
1832 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
1833 	int rxfifosz = priv->plat->rx_fifo_size;
1834 	int txfifosz = priv->plat->tx_fifo_size;
1835 	u32 txmode = 0;
1836 	u32 rxmode = 0;
1837 	u32 chan = 0;
1838 	u8 qmode = 0;
1839 
1840 	if (rxfifosz == 0)
1841 		rxfifosz = priv->dma_cap.rx_fifo_size;
1842 	if (txfifosz == 0)
1843 		txfifosz = priv->dma_cap.tx_fifo_size;
1844 
1845 	/* Adjust for real per queue fifo size */
1846 	rxfifosz /= rx_channels_count;
1847 	txfifosz /= tx_channels_count;
1848 
1849 	if (priv->plat->force_thresh_dma_mode) {
1850 		txmode = tc;
1851 		rxmode = tc;
1852 	} else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1853 		/*
1854 		 * In case of GMAC, SF mode can be enabled
1855 		 * to perform the TX COE in HW. This depends on:
1856 		 * 1) TX COE if actually supported
1857 		 * 2) There is no bugged Jumbo frame support
1858 		 *    that needs to not insert csum in the TDES.
1859 		 */
1860 		txmode = SF_DMA_MODE;
1861 		rxmode = SF_DMA_MODE;
1862 		priv->xstats.threshold = SF_DMA_MODE;
1863 	} else {
1864 		txmode = tc;
1865 		rxmode = SF_DMA_MODE;
1866 	}
1867 
1868 	/* configure all channels */
1869 	for (chan = 0; chan < rx_channels_count; chan++) {
1870 		qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1871 
1872 		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
1873 				rxfifosz, qmode);
1874 		stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz,
1875 				chan);
1876 	}
1877 
1878 	for (chan = 0; chan < tx_channels_count; chan++) {
1879 		qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1880 
1881 		stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
1882 				txfifosz, qmode);
1883 	}
1884 }
1885 
1886 /**
1887  * stmmac_tx_clean - to manage the transmission completion
1888  * @priv: driver private structure
1889  * @queue: TX queue index
1890  * Description: it reclaims the transmit resources after transmission completes.
1891  */
1892 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
1893 {
1894 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1895 	unsigned int bytes_compl = 0, pkts_compl = 0;
1896 	unsigned int entry, count = 0;
1897 
1898 	__netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
1899 
1900 	priv->xstats.tx_clean++;
1901 
1902 	entry = tx_q->dirty_tx;
1903 	while ((entry != tx_q->cur_tx) && (count < budget)) {
1904 		struct sk_buff *skb = tx_q->tx_skbuff[entry];
1905 		struct dma_desc *p;
1906 		int status;
1907 
1908 		if (priv->extend_desc)
1909 			p = (struct dma_desc *)(tx_q->dma_etx + entry);
1910 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1911 			p = &tx_q->dma_entx[entry].basic;
1912 		else
1913 			p = tx_q->dma_tx + entry;
1914 
1915 		status = stmmac_tx_status(priv, &priv->dev->stats,
1916 				&priv->xstats, p, priv->ioaddr);
1917 		/* Check if the descriptor is owned by the DMA */
1918 		if (unlikely(status & tx_dma_own))
1919 			break;
1920 
1921 		count++;
1922 
1923 		/* Make sure descriptor fields are read after reading
1924 		 * the own bit.
1925 		 */
1926 		dma_rmb();
1927 
1928 		/* Just consider the last segment and ...*/
1929 		if (likely(!(status & tx_not_ls))) {
1930 			/* ... verify the status error condition */
1931 			if (unlikely(status & tx_err)) {
1932 				priv->dev->stats.tx_errors++;
1933 			} else {
1934 				priv->dev->stats.tx_packets++;
1935 				priv->xstats.tx_pkt_n++;
1936 			}
1937 			stmmac_get_tx_hwtstamp(priv, p, skb);
1938 		}
1939 
1940 		if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1941 			if (tx_q->tx_skbuff_dma[entry].map_as_page)
1942 				dma_unmap_page(priv->device,
1943 					       tx_q->tx_skbuff_dma[entry].buf,
1944 					       tx_q->tx_skbuff_dma[entry].len,
1945 					       DMA_TO_DEVICE);
1946 			else
1947 				dma_unmap_single(priv->device,
1948 						 tx_q->tx_skbuff_dma[entry].buf,
1949 						 tx_q->tx_skbuff_dma[entry].len,
1950 						 DMA_TO_DEVICE);
1951 			tx_q->tx_skbuff_dma[entry].buf = 0;
1952 			tx_q->tx_skbuff_dma[entry].len = 0;
1953 			tx_q->tx_skbuff_dma[entry].map_as_page = false;
1954 		}
1955 
1956 		stmmac_clean_desc3(priv, tx_q, p);
1957 
1958 		tx_q->tx_skbuff_dma[entry].last_segment = false;
1959 		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1960 
1961 		if (likely(skb != NULL)) {
1962 			pkts_compl++;
1963 			bytes_compl += skb->len;
1964 			dev_consume_skb_any(skb);
1965 			tx_q->tx_skbuff[entry] = NULL;
1966 		}
1967 
1968 		stmmac_release_tx_desc(priv, p, priv->mode);
1969 
1970 		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1971 	}
1972 	tx_q->dirty_tx = entry;
1973 
1974 	netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1975 				  pkts_compl, bytes_compl);
1976 
1977 	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1978 								queue))) &&
1979 	    stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1980 
1981 		netif_dbg(priv, tx_done, priv->dev,
1982 			  "%s: restart transmit\n", __func__);
1983 		netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1984 	}
1985 
1986 	if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1987 		stmmac_enable_eee_mode(priv);
1988 		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1989 	}
1990 
1991 	/* We still have pending packets, let's call for a new scheduling */
1992 	if (tx_q->dirty_tx != tx_q->cur_tx)
1993 		mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
1994 
1995 	__netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
1996 
1997 	return count;
1998 }
1999 
2000 /**
2001  * stmmac_tx_err - to manage the tx error
2002  * @priv: driver private structure
2003  * @chan: channel index
2004  * Description: it cleans the descriptors and restarts the transmission
2005  * in case of transmission errors.
2006  */
2007 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
2008 {
2009 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2010 
2011 	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
2012 
2013 	stmmac_stop_tx_dma(priv, chan);
2014 	dma_free_tx_skbufs(priv, chan);
2015 	stmmac_clear_tx_descriptors(priv, chan);
2016 	tx_q->dirty_tx = 0;
2017 	tx_q->cur_tx = 0;
2018 	tx_q->mss = 0;
2019 	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
2020 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2021 			    tx_q->dma_tx_phy, chan);
2022 	stmmac_start_tx_dma(priv, chan);
2023 
2024 	priv->dev->stats.tx_errors++;
2025 	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
2026 }
2027 
2028 /**
2029  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2030  *  @priv: driver private structure
2031  *  @txmode: TX operating mode
2032  *  @rxmode: RX operating mode
2033  *  @chan: channel index
2034  *  Description: it is used for configuring of the DMA operation mode in
2035  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2036  *  mode.
2037  */
2038 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
2039 					  u32 rxmode, u32 chan)
2040 {
2041 	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2042 	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2043 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2044 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2045 	int rxfifosz = priv->plat->rx_fifo_size;
2046 	int txfifosz = priv->plat->tx_fifo_size;
2047 
2048 	if (rxfifosz == 0)
2049 		rxfifosz = priv->dma_cap.rx_fifo_size;
2050 	if (txfifosz == 0)
2051 		txfifosz = priv->dma_cap.tx_fifo_size;
2052 
2053 	/* Adjust for real per queue fifo size */
2054 	rxfifosz /= rx_channels_count;
2055 	txfifosz /= tx_channels_count;
2056 
2057 	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2058 	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
2059 }
2060 
2061 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2062 {
2063 	int ret;
2064 
2065 	ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2066 			priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2067 	if (ret && (ret != -EINVAL)) {
2068 		stmmac_global_err(priv);
2069 		return true;
2070 	}
2071 
2072 	return false;
2073 }
2074 
2075 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan)
2076 {
2077 	int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2078 						 &priv->xstats, chan);
2079 	struct stmmac_channel *ch = &priv->channel[chan];
2080 	unsigned long flags;
2081 
2082 	if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
2083 		if (napi_schedule_prep(&ch->rx_napi)) {
2084 			spin_lock_irqsave(&ch->lock, flags);
2085 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
2086 			spin_unlock_irqrestore(&ch->lock, flags);
2087 			__napi_schedule_irqoff(&ch->rx_napi);
2088 		}
2089 	}
2090 
2091 	if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {
2092 		if (napi_schedule_prep(&ch->tx_napi)) {
2093 			spin_lock_irqsave(&ch->lock, flags);
2094 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
2095 			spin_unlock_irqrestore(&ch->lock, flags);
2096 			__napi_schedule_irqoff(&ch->tx_napi);
2097 		}
2098 	}
2099 
2100 	return status;
2101 }
2102 
2103 /**
2104  * stmmac_dma_interrupt - DMA ISR
2105  * @priv: driver private structure
2106  * Description: this is the DMA ISR. It is called by the main ISR.
2107  * It calls the dwmac dma routine and schedule poll method in case of some
2108  * work can be done.
2109  */
2110 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2111 {
2112 	u32 tx_channel_count = priv->plat->tx_queues_to_use;
2113 	u32 rx_channel_count = priv->plat->rx_queues_to_use;
2114 	u32 channels_to_check = tx_channel_count > rx_channel_count ?
2115 				tx_channel_count : rx_channel_count;
2116 	u32 chan;
2117 	int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
2118 
2119 	/* Make sure we never check beyond our status buffer. */
2120 	if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
2121 		channels_to_check = ARRAY_SIZE(status);
2122 
2123 	for (chan = 0; chan < channels_to_check; chan++)
2124 		status[chan] = stmmac_napi_check(priv, chan);
2125 
2126 	for (chan = 0; chan < tx_channel_count; chan++) {
2127 		if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
2128 			/* Try to bump up the dma threshold on this failure */
2129 			if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2130 			    (tc <= 256)) {
2131 				tc += 64;
2132 				if (priv->plat->force_thresh_dma_mode)
2133 					stmmac_set_dma_operation_mode(priv,
2134 								      tc,
2135 								      tc,
2136 								      chan);
2137 				else
2138 					stmmac_set_dma_operation_mode(priv,
2139 								    tc,
2140 								    SF_DMA_MODE,
2141 								    chan);
2142 				priv->xstats.threshold = tc;
2143 			}
2144 		} else if (unlikely(status[chan] == tx_hard_error)) {
2145 			stmmac_tx_err(priv, chan);
2146 		}
2147 	}
2148 }
2149 
2150 /**
2151  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2152  * @priv: driver private structure
2153  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2154  */
2155 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2156 {
2157 	unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2158 			    MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2159 
2160 	stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
2161 
2162 	if (priv->dma_cap.rmon) {
2163 		stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
2164 		memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2165 	} else
2166 		netdev_info(priv->dev, "No MAC Management Counters available\n");
2167 }
2168 
2169 /**
2170  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2171  * @priv: driver private structure
2172  * Description:
2173  *  new GMAC chip generations have a new register to indicate the
2174  *  presence of the optional feature/functions.
2175  *  This can be also used to override the value passed through the
2176  *  platform and necessary for old MAC10/100 and GMAC chips.
2177  */
2178 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2179 {
2180 	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
2181 }
2182 
2183 /**
2184  * stmmac_check_ether_addr - check if the MAC addr is valid
2185  * @priv: driver private structure
2186  * Description:
2187  * it is to verify if the MAC address is valid, in case of failures it
2188  * generates a random MAC address
2189  */
2190 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2191 {
2192 	if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2193 		stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
2194 		if (!is_valid_ether_addr(priv->dev->dev_addr))
2195 			eth_hw_addr_random(priv->dev);
2196 		dev_info(priv->device, "device MAC address %pM\n",
2197 			 priv->dev->dev_addr);
2198 	}
2199 }
2200 
2201 /**
2202  * stmmac_init_dma_engine - DMA init.
2203  * @priv: driver private structure
2204  * Description:
2205  * It inits the DMA invoking the specific MAC/GMAC callback.
2206  * Some DMA parameters can be passed from the platform;
2207  * in case of these are not passed a default is kept for the MAC or GMAC.
2208  */
2209 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2210 {
2211 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2212 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2213 	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2214 	struct stmmac_rx_queue *rx_q;
2215 	struct stmmac_tx_queue *tx_q;
2216 	u32 chan = 0;
2217 	int atds = 0;
2218 	int ret = 0;
2219 
2220 	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2221 		dev_err(priv->device, "Invalid DMA configuration\n");
2222 		return -EINVAL;
2223 	}
2224 
2225 	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2226 		atds = 1;
2227 
2228 	ret = stmmac_reset(priv, priv->ioaddr);
2229 	if (ret) {
2230 		dev_err(priv->device, "Failed to reset the dma\n");
2231 		return ret;
2232 	}
2233 
2234 	/* DMA Configuration */
2235 	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
2236 
2237 	if (priv->plat->axi)
2238 		stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2239 
2240 	/* DMA CSR Channel configuration */
2241 	for (chan = 0; chan < dma_csr_ch; chan++)
2242 		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
2243 
2244 	/* DMA RX Channel Configuration */
2245 	for (chan = 0; chan < rx_channels_count; chan++) {
2246 		rx_q = &priv->rx_queue[chan];
2247 
2248 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2249 				    rx_q->dma_rx_phy, chan);
2250 
2251 		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2252 			    (DMA_RX_SIZE * sizeof(struct dma_desc));
2253 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
2254 				       rx_q->rx_tail_addr, chan);
2255 	}
2256 
2257 	/* DMA TX Channel Configuration */
2258 	for (chan = 0; chan < tx_channels_count; chan++) {
2259 		tx_q = &priv->tx_queue[chan];
2260 
2261 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2262 				    tx_q->dma_tx_phy, chan);
2263 
2264 		tx_q->tx_tail_addr = tx_q->dma_tx_phy;
2265 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
2266 				       tx_q->tx_tail_addr, chan);
2267 	}
2268 
2269 	return ret;
2270 }
2271 
2272 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2273 {
2274 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2275 
2276 	mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
2277 }
2278 
2279 /**
2280  * stmmac_tx_timer - mitigation sw timer for tx.
2281  * @data: data pointer
2282  * Description:
2283  * This is the timer handler to directly invoke the stmmac_tx_clean.
2284  */
2285 static void stmmac_tx_timer(struct timer_list *t)
2286 {
2287 	struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer);
2288 	struct stmmac_priv *priv = tx_q->priv_data;
2289 	struct stmmac_channel *ch;
2290 
2291 	ch = &priv->channel[tx_q->queue_index];
2292 
2293 	if (likely(napi_schedule_prep(&ch->tx_napi))) {
2294 		unsigned long flags;
2295 
2296 		spin_lock_irqsave(&ch->lock, flags);
2297 		stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);
2298 		spin_unlock_irqrestore(&ch->lock, flags);
2299 		__napi_schedule(&ch->tx_napi);
2300 	}
2301 }
2302 
2303 /**
2304  * stmmac_init_coalesce - init mitigation options.
2305  * @priv: driver private structure
2306  * Description:
2307  * This inits the coalesce parameters: i.e. timer rate,
2308  * timer handler and default threshold used for enabling the
2309  * interrupt on completion bit.
2310  */
2311 static void stmmac_init_coalesce(struct stmmac_priv *priv)
2312 {
2313 	u32 tx_channel_count = priv->plat->tx_queues_to_use;
2314 	u32 chan;
2315 
2316 	priv->tx_coal_frames = STMMAC_TX_FRAMES;
2317 	priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2318 	priv->rx_coal_frames = STMMAC_RX_FRAMES;
2319 
2320 	for (chan = 0; chan < tx_channel_count; chan++) {
2321 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2322 
2323 		timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0);
2324 	}
2325 }
2326 
2327 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2328 {
2329 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2330 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2331 	u32 chan;
2332 
2333 	/* set TX ring length */
2334 	for (chan = 0; chan < tx_channels_count; chan++)
2335 		stmmac_set_tx_ring_len(priv, priv->ioaddr,
2336 				(DMA_TX_SIZE - 1), chan);
2337 
2338 	/* set RX ring length */
2339 	for (chan = 0; chan < rx_channels_count; chan++)
2340 		stmmac_set_rx_ring_len(priv, priv->ioaddr,
2341 				(DMA_RX_SIZE - 1), chan);
2342 }
2343 
2344 /**
2345  *  stmmac_set_tx_queue_weight - Set TX queue weight
2346  *  @priv: driver private structure
2347  *  Description: It is used for setting TX queues weight
2348  */
2349 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2350 {
2351 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2352 	u32 weight;
2353 	u32 queue;
2354 
2355 	for (queue = 0; queue < tx_queues_count; queue++) {
2356 		weight = priv->plat->tx_queues_cfg[queue].weight;
2357 		stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
2358 	}
2359 }
2360 
2361 /**
2362  *  stmmac_configure_cbs - Configure CBS in TX queue
2363  *  @priv: driver private structure
2364  *  Description: It is used for configuring CBS in AVB TX queues
2365  */
2366 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2367 {
2368 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2369 	u32 mode_to_use;
2370 	u32 queue;
2371 
2372 	/* queue 0 is reserved for legacy traffic */
2373 	for (queue = 1; queue < tx_queues_count; queue++) {
2374 		mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2375 		if (mode_to_use == MTL_QUEUE_DCB)
2376 			continue;
2377 
2378 		stmmac_config_cbs(priv, priv->hw,
2379 				priv->plat->tx_queues_cfg[queue].send_slope,
2380 				priv->plat->tx_queues_cfg[queue].idle_slope,
2381 				priv->plat->tx_queues_cfg[queue].high_credit,
2382 				priv->plat->tx_queues_cfg[queue].low_credit,
2383 				queue);
2384 	}
2385 }
2386 
2387 /**
2388  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2389  *  @priv: driver private structure
2390  *  Description: It is used for mapping RX queues to RX dma channels
2391  */
2392 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2393 {
2394 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2395 	u32 queue;
2396 	u32 chan;
2397 
2398 	for (queue = 0; queue < rx_queues_count; queue++) {
2399 		chan = priv->plat->rx_queues_cfg[queue].chan;
2400 		stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
2401 	}
2402 }
2403 
2404 /**
2405  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2406  *  @priv: driver private structure
2407  *  Description: It is used for configuring the RX Queue Priority
2408  */
2409 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2410 {
2411 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2412 	u32 queue;
2413 	u32 prio;
2414 
2415 	for (queue = 0; queue < rx_queues_count; queue++) {
2416 		if (!priv->plat->rx_queues_cfg[queue].use_prio)
2417 			continue;
2418 
2419 		prio = priv->plat->rx_queues_cfg[queue].prio;
2420 		stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
2421 	}
2422 }
2423 
2424 /**
2425  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2426  *  @priv: driver private structure
2427  *  Description: It is used for configuring the TX Queue Priority
2428  */
2429 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2430 {
2431 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2432 	u32 queue;
2433 	u32 prio;
2434 
2435 	for (queue = 0; queue < tx_queues_count; queue++) {
2436 		if (!priv->plat->tx_queues_cfg[queue].use_prio)
2437 			continue;
2438 
2439 		prio = priv->plat->tx_queues_cfg[queue].prio;
2440 		stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
2441 	}
2442 }
2443 
2444 /**
2445  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2446  *  @priv: driver private structure
2447  *  Description: It is used for configuring the RX queue routing
2448  */
2449 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2450 {
2451 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2452 	u32 queue;
2453 	u8 packet;
2454 
2455 	for (queue = 0; queue < rx_queues_count; queue++) {
2456 		/* no specific packet type routing specified for the queue */
2457 		if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2458 			continue;
2459 
2460 		packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2461 		stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
2462 	}
2463 }
2464 
2465 static void stmmac_mac_config_rss(struct stmmac_priv *priv)
2466 {
2467 	if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
2468 		priv->rss.enable = false;
2469 		return;
2470 	}
2471 
2472 	if (priv->dev->features & NETIF_F_RXHASH)
2473 		priv->rss.enable = true;
2474 	else
2475 		priv->rss.enable = false;
2476 
2477 	stmmac_rss_configure(priv, priv->hw, &priv->rss,
2478 			     priv->plat->rx_queues_to_use);
2479 }
2480 
2481 /**
2482  *  stmmac_mtl_configuration - Configure MTL
2483  *  @priv: driver private structure
2484  *  Description: It is used for configurring MTL
2485  */
2486 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2487 {
2488 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2489 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2490 
2491 	if (tx_queues_count > 1)
2492 		stmmac_set_tx_queue_weight(priv);
2493 
2494 	/* Configure MTL RX algorithms */
2495 	if (rx_queues_count > 1)
2496 		stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
2497 				priv->plat->rx_sched_algorithm);
2498 
2499 	/* Configure MTL TX algorithms */
2500 	if (tx_queues_count > 1)
2501 		stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
2502 				priv->plat->tx_sched_algorithm);
2503 
2504 	/* Configure CBS in AVB TX queues */
2505 	if (tx_queues_count > 1)
2506 		stmmac_configure_cbs(priv);
2507 
2508 	/* Map RX MTL to DMA channels */
2509 	stmmac_rx_queue_dma_chan_map(priv);
2510 
2511 	/* Enable MAC RX Queues */
2512 	stmmac_mac_enable_rx_queues(priv);
2513 
2514 	/* Set RX priorities */
2515 	if (rx_queues_count > 1)
2516 		stmmac_mac_config_rx_queues_prio(priv);
2517 
2518 	/* Set TX priorities */
2519 	if (tx_queues_count > 1)
2520 		stmmac_mac_config_tx_queues_prio(priv);
2521 
2522 	/* Set RX routing */
2523 	if (rx_queues_count > 1)
2524 		stmmac_mac_config_rx_queues_routing(priv);
2525 
2526 	/* Receive Side Scaling */
2527 	if (rx_queues_count > 1)
2528 		stmmac_mac_config_rss(priv);
2529 }
2530 
2531 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
2532 {
2533 	if (priv->dma_cap.asp) {
2534 		netdev_info(priv->dev, "Enabling Safety Features\n");
2535 		stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
2536 	} else {
2537 		netdev_info(priv->dev, "No Safety Features support found\n");
2538 	}
2539 }
2540 
2541 /**
2542  * stmmac_hw_setup - setup mac in a usable state.
2543  *  @dev : pointer to the device structure.
2544  *  Description:
2545  *  this is the main function to setup the HW in a usable state because the
2546  *  dma engine is reset, the core registers are configured (e.g. AXI,
2547  *  Checksum features, timers). The DMA is ready to start receiving and
2548  *  transmitting.
2549  *  Return value:
2550  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2551  *  file on failure.
2552  */
2553 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2554 {
2555 	struct stmmac_priv *priv = netdev_priv(dev);
2556 	u32 rx_cnt = priv->plat->rx_queues_to_use;
2557 	u32 tx_cnt = priv->plat->tx_queues_to_use;
2558 	u32 chan;
2559 	int ret;
2560 
2561 	/* DMA initialization and SW reset */
2562 	ret = stmmac_init_dma_engine(priv);
2563 	if (ret < 0) {
2564 		netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2565 			   __func__);
2566 		return ret;
2567 	}
2568 
2569 	/* Copy the MAC addr into the HW  */
2570 	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
2571 
2572 	/* PS and related bits will be programmed according to the speed */
2573 	if (priv->hw->pcs) {
2574 		int speed = priv->plat->mac_port_sel_speed;
2575 
2576 		if ((speed == SPEED_10) || (speed == SPEED_100) ||
2577 		    (speed == SPEED_1000)) {
2578 			priv->hw->ps = speed;
2579 		} else {
2580 			dev_warn(priv->device, "invalid port speed\n");
2581 			priv->hw->ps = 0;
2582 		}
2583 	}
2584 
2585 	/* Initialize the MAC Core */
2586 	stmmac_core_init(priv, priv->hw, dev);
2587 
2588 	/* Initialize MTL*/
2589 	stmmac_mtl_configuration(priv);
2590 
2591 	/* Initialize Safety Features */
2592 	stmmac_safety_feat_configuration(priv);
2593 
2594 	ret = stmmac_rx_ipc(priv, priv->hw);
2595 	if (!ret) {
2596 		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2597 		priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2598 		priv->hw->rx_csum = 0;
2599 	}
2600 
2601 	/* Enable the MAC Rx/Tx */
2602 	stmmac_mac_set(priv, priv->ioaddr, true);
2603 
2604 	/* Set the HW DMA mode and the COE */
2605 	stmmac_dma_operation_mode(priv);
2606 
2607 	stmmac_mmc_setup(priv);
2608 
2609 	if (init_ptp) {
2610 		ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2611 		if (ret < 0)
2612 			netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2613 
2614 		ret = stmmac_init_ptp(priv);
2615 		if (ret == -EOPNOTSUPP)
2616 			netdev_warn(priv->dev, "PTP not supported by HW\n");
2617 		else if (ret)
2618 			netdev_warn(priv->dev, "PTP init failed\n");
2619 	}
2620 
2621 	priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2622 
2623 	if (priv->use_riwt) {
2624 		if (!priv->rx_riwt)
2625 			priv->rx_riwt = DEF_DMA_RIWT;
2626 
2627 		ret = stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt);
2628 	}
2629 
2630 	if (priv->hw->pcs)
2631 		stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
2632 
2633 	/* set TX and RX rings length */
2634 	stmmac_set_rings_length(priv);
2635 
2636 	/* Enable TSO */
2637 	if (priv->tso) {
2638 		for (chan = 0; chan < tx_cnt; chan++)
2639 			stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
2640 	}
2641 
2642 	/* Enable Split Header */
2643 	if (priv->sph && priv->hw->rx_csum) {
2644 		for (chan = 0; chan < rx_cnt; chan++)
2645 			stmmac_enable_sph(priv, priv->ioaddr, 1, chan);
2646 	}
2647 
2648 	/* VLAN Tag Insertion */
2649 	if (priv->dma_cap.vlins)
2650 		stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
2651 
2652 	/* TBS */
2653 	for (chan = 0; chan < tx_cnt; chan++) {
2654 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2655 		int enable = tx_q->tbs & STMMAC_TBS_AVAIL;
2656 
2657 		stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);
2658 	}
2659 
2660 	/* Start the ball rolling... */
2661 	stmmac_start_all_dma(priv);
2662 
2663 	return 0;
2664 }
2665 
2666 static void stmmac_hw_teardown(struct net_device *dev)
2667 {
2668 	struct stmmac_priv *priv = netdev_priv(dev);
2669 
2670 	clk_disable_unprepare(priv->plat->clk_ptp_ref);
2671 }
2672 
2673 /**
2674  *  stmmac_open - open entry point of the driver
2675  *  @dev : pointer to the device structure.
2676  *  Description:
2677  *  This function is the open entry point of the driver.
2678  *  Return value:
2679  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2680  *  file on failure.
2681  */
2682 static int stmmac_open(struct net_device *dev)
2683 {
2684 	struct stmmac_priv *priv = netdev_priv(dev);
2685 	int bfsize = 0;
2686 	u32 chan;
2687 	int ret;
2688 
2689 	if (priv->hw->pcs != STMMAC_PCS_TBI &&
2690 	    priv->hw->pcs != STMMAC_PCS_RTBI) {
2691 		ret = stmmac_init_phy(dev);
2692 		if (ret) {
2693 			netdev_err(priv->dev,
2694 				   "%s: Cannot attach to PHY (error: %d)\n",
2695 				   __func__, ret);
2696 			return ret;
2697 		}
2698 	}
2699 
2700 	/* Extra statistics */
2701 	memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2702 	priv->xstats.threshold = tc;
2703 
2704 	bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
2705 	if (bfsize < 0)
2706 		bfsize = 0;
2707 
2708 	if (bfsize < BUF_SIZE_16KiB)
2709 		bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
2710 
2711 	priv->dma_buf_sz = bfsize;
2712 	buf_sz = bfsize;
2713 
2714 	priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2715 
2716 	/* Earlier check for TBS */
2717 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {
2718 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2719 		int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;
2720 
2721 		tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0;
2722 		if (stmmac_enable_tbs(priv, priv->ioaddr, tbs_en, chan))
2723 			tx_q->tbs &= ~STMMAC_TBS_AVAIL;
2724 	}
2725 
2726 	ret = alloc_dma_desc_resources(priv);
2727 	if (ret < 0) {
2728 		netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2729 			   __func__);
2730 		goto dma_desc_error;
2731 	}
2732 
2733 	ret = init_dma_desc_rings(dev, GFP_KERNEL);
2734 	if (ret < 0) {
2735 		netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2736 			   __func__);
2737 		goto init_error;
2738 	}
2739 
2740 	ret = stmmac_hw_setup(dev, true);
2741 	if (ret < 0) {
2742 		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2743 		goto init_error;
2744 	}
2745 
2746 	stmmac_init_coalesce(priv);
2747 
2748 	phylink_start(priv->phylink);
2749 
2750 	/* Request the IRQ lines */
2751 	ret = request_irq(dev->irq, stmmac_interrupt,
2752 			  IRQF_SHARED, dev->name, dev);
2753 	if (unlikely(ret < 0)) {
2754 		netdev_err(priv->dev,
2755 			   "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2756 			   __func__, dev->irq, ret);
2757 		goto irq_error;
2758 	}
2759 
2760 	/* Request the Wake IRQ in case of another line is used for WoL */
2761 	if (priv->wol_irq != dev->irq) {
2762 		ret = request_irq(priv->wol_irq, stmmac_interrupt,
2763 				  IRQF_SHARED, dev->name, dev);
2764 		if (unlikely(ret < 0)) {
2765 			netdev_err(priv->dev,
2766 				   "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2767 				   __func__, priv->wol_irq, ret);
2768 			goto wolirq_error;
2769 		}
2770 	}
2771 
2772 	/* Request the IRQ lines */
2773 	if (priv->lpi_irq > 0) {
2774 		ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2775 				  dev->name, dev);
2776 		if (unlikely(ret < 0)) {
2777 			netdev_err(priv->dev,
2778 				   "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2779 				   __func__, priv->lpi_irq, ret);
2780 			goto lpiirq_error;
2781 		}
2782 	}
2783 
2784 	stmmac_enable_all_queues(priv);
2785 	stmmac_start_all_queues(priv);
2786 
2787 	return 0;
2788 
2789 lpiirq_error:
2790 	if (priv->wol_irq != dev->irq)
2791 		free_irq(priv->wol_irq, dev);
2792 wolirq_error:
2793 	free_irq(dev->irq, dev);
2794 irq_error:
2795 	phylink_stop(priv->phylink);
2796 
2797 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2798 		del_timer_sync(&priv->tx_queue[chan].txtimer);
2799 
2800 	stmmac_hw_teardown(dev);
2801 init_error:
2802 	free_dma_desc_resources(priv);
2803 dma_desc_error:
2804 	phylink_disconnect_phy(priv->phylink);
2805 	return ret;
2806 }
2807 
2808 /**
2809  *  stmmac_release - close entry point of the driver
2810  *  @dev : device pointer.
2811  *  Description:
2812  *  This is the stop entry point of the driver.
2813  */
2814 static int stmmac_release(struct net_device *dev)
2815 {
2816 	struct stmmac_priv *priv = netdev_priv(dev);
2817 	u32 chan;
2818 
2819 	if (priv->eee_enabled)
2820 		del_timer_sync(&priv->eee_ctrl_timer);
2821 
2822 	/* Stop and disconnect the PHY */
2823 	phylink_stop(priv->phylink);
2824 	phylink_disconnect_phy(priv->phylink);
2825 
2826 	stmmac_stop_all_queues(priv);
2827 
2828 	stmmac_disable_all_queues(priv);
2829 
2830 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2831 		del_timer_sync(&priv->tx_queue[chan].txtimer);
2832 
2833 	/* Free the IRQ lines */
2834 	free_irq(dev->irq, dev);
2835 	if (priv->wol_irq != dev->irq)
2836 		free_irq(priv->wol_irq, dev);
2837 	if (priv->lpi_irq > 0)
2838 		free_irq(priv->lpi_irq, dev);
2839 
2840 	/* Stop TX/RX DMA and clear the descriptors */
2841 	stmmac_stop_all_dma(priv);
2842 
2843 	/* Release and free the Rx/Tx resources */
2844 	free_dma_desc_resources(priv);
2845 
2846 	/* Disable the MAC Rx/Tx */
2847 	stmmac_mac_set(priv, priv->ioaddr, false);
2848 
2849 	netif_carrier_off(dev);
2850 
2851 	stmmac_release_ptp(priv);
2852 
2853 	return 0;
2854 }
2855 
2856 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
2857 			       struct stmmac_tx_queue *tx_q)
2858 {
2859 	u16 tag = 0x0, inner_tag = 0x0;
2860 	u32 inner_type = 0x0;
2861 	struct dma_desc *p;
2862 
2863 	if (!priv->dma_cap.vlins)
2864 		return false;
2865 	if (!skb_vlan_tag_present(skb))
2866 		return false;
2867 	if (skb->vlan_proto == htons(ETH_P_8021AD)) {
2868 		inner_tag = skb_vlan_tag_get(skb);
2869 		inner_type = STMMAC_VLAN_INSERT;
2870 	}
2871 
2872 	tag = skb_vlan_tag_get(skb);
2873 
2874 	if (tx_q->tbs & STMMAC_TBS_AVAIL)
2875 		p = &tx_q->dma_entx[tx_q->cur_tx].basic;
2876 	else
2877 		p = &tx_q->dma_tx[tx_q->cur_tx];
2878 
2879 	if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))
2880 		return false;
2881 
2882 	stmmac_set_tx_owner(priv, p);
2883 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2884 	return true;
2885 }
2886 
2887 /**
2888  *  stmmac_tso_allocator - close entry point of the driver
2889  *  @priv: driver private structure
2890  *  @des: buffer start address
2891  *  @total_len: total length to fill in descriptors
2892  *  @last_segmant: condition for the last descriptor
2893  *  @queue: TX queue index
2894  *  Description:
2895  *  This function fills descriptor and request new descriptors according to
2896  *  buffer length to fill
2897  */
2898 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
2899 				 int total_len, bool last_segment, u32 queue)
2900 {
2901 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2902 	struct dma_desc *desc;
2903 	u32 buff_size;
2904 	int tmp_len;
2905 
2906 	tmp_len = total_len;
2907 
2908 	while (tmp_len > 0) {
2909 		dma_addr_t curr_addr;
2910 
2911 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2912 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
2913 
2914 		if (tx_q->tbs & STMMAC_TBS_AVAIL)
2915 			desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
2916 		else
2917 			desc = &tx_q->dma_tx[tx_q->cur_tx];
2918 
2919 		curr_addr = des + (total_len - tmp_len);
2920 		if (priv->dma_cap.addr64 <= 32)
2921 			desc->des0 = cpu_to_le32(curr_addr);
2922 		else
2923 			stmmac_set_desc_addr(priv, desc, curr_addr);
2924 
2925 		buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2926 			    TSO_MAX_BUFF_SIZE : tmp_len;
2927 
2928 		stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
2929 				0, 1,
2930 				(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
2931 				0, 0);
2932 
2933 		tmp_len -= TSO_MAX_BUFF_SIZE;
2934 	}
2935 }
2936 
2937 /**
2938  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2939  *  @skb : the socket buffer
2940  *  @dev : device pointer
2941  *  Description: this is the transmit function that is called on TSO frames
2942  *  (support available on GMAC4 and newer chips).
2943  *  Diagram below show the ring programming in case of TSO frames:
2944  *
2945  *  First Descriptor
2946  *   --------
2947  *   | DES0 |---> buffer1 = L2/L3/L4 header
2948  *   | DES1 |---> TCP Payload (can continue on next descr...)
2949  *   | DES2 |---> buffer 1 and 2 len
2950  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2951  *   --------
2952  *	|
2953  *     ...
2954  *	|
2955  *   --------
2956  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2957  *   | DES1 | --|
2958  *   | DES2 | --> buffer 1 and 2 len
2959  *   | DES3 |
2960  *   --------
2961  *
2962  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2963  */
2964 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2965 {
2966 	struct dma_desc *desc, *first, *mss_desc = NULL;
2967 	struct stmmac_priv *priv = netdev_priv(dev);
2968 	int desc_size, tmp_pay_len = 0, first_tx;
2969 	int nfrags = skb_shinfo(skb)->nr_frags;
2970 	u32 queue = skb_get_queue_mapping(skb);
2971 	unsigned int first_entry, tx_packets;
2972 	struct stmmac_tx_queue *tx_q;
2973 	bool has_vlan, set_ic;
2974 	u8 proto_hdr_len, hdr;
2975 	u32 pay_len, mss;
2976 	dma_addr_t des;
2977 	int i;
2978 
2979 	tx_q = &priv->tx_queue[queue];
2980 	first_tx = tx_q->cur_tx;
2981 
2982 	/* Compute header lengths */
2983 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
2984 		proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
2985 		hdr = sizeof(struct udphdr);
2986 	} else {
2987 		proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2988 		hdr = tcp_hdrlen(skb);
2989 	}
2990 
2991 	/* Desc availability based on threshold should be enough safe */
2992 	if (unlikely(stmmac_tx_avail(priv, queue) <
2993 		(((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2994 		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2995 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2996 								queue));
2997 			/* This is a hard error, log it. */
2998 			netdev_err(priv->dev,
2999 				   "%s: Tx Ring full when queue awake\n",
3000 				   __func__);
3001 		}
3002 		return NETDEV_TX_BUSY;
3003 	}
3004 
3005 	pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
3006 
3007 	mss = skb_shinfo(skb)->gso_size;
3008 
3009 	/* set new MSS value if needed */
3010 	if (mss != tx_q->mss) {
3011 		if (tx_q->tbs & STMMAC_TBS_AVAIL)
3012 			mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3013 		else
3014 			mss_desc = &tx_q->dma_tx[tx_q->cur_tx];
3015 
3016 		stmmac_set_mss(priv, mss_desc, mss);
3017 		tx_q->mss = mss;
3018 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
3019 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
3020 	}
3021 
3022 	if (netif_msg_tx_queued(priv)) {
3023 		pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
3024 			__func__, hdr, proto_hdr_len, pay_len, mss);
3025 		pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
3026 			skb->data_len);
3027 	}
3028 
3029 	/* Check if VLAN can be inserted by HW */
3030 	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
3031 
3032 	first_entry = tx_q->cur_tx;
3033 	WARN_ON(tx_q->tx_skbuff[first_entry]);
3034 
3035 	if (tx_q->tbs & STMMAC_TBS_AVAIL)
3036 		desc = &tx_q->dma_entx[first_entry].basic;
3037 	else
3038 		desc = &tx_q->dma_tx[first_entry];
3039 	first = desc;
3040 
3041 	if (has_vlan)
3042 		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
3043 
3044 	/* first descriptor: fill Headers on Buf1 */
3045 	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
3046 			     DMA_TO_DEVICE);
3047 	if (dma_mapping_error(priv->device, des))
3048 		goto dma_map_err;
3049 
3050 	tx_q->tx_skbuff_dma[first_entry].buf = des;
3051 	tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
3052 
3053 	if (priv->dma_cap.addr64 <= 32) {
3054 		first->des0 = cpu_to_le32(des);
3055 
3056 		/* Fill start of payload in buff2 of first descriptor */
3057 		if (pay_len)
3058 			first->des1 = cpu_to_le32(des + proto_hdr_len);
3059 
3060 		/* If needed take extra descriptors to fill the remaining payload */
3061 		tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
3062 	} else {
3063 		stmmac_set_desc_addr(priv, first, des);
3064 		tmp_pay_len = pay_len;
3065 		des += proto_hdr_len;
3066 		pay_len = 0;
3067 	}
3068 
3069 	stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
3070 
3071 	/* Prepare fragments */
3072 	for (i = 0; i < nfrags; i++) {
3073 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3074 
3075 		des = skb_frag_dma_map(priv->device, frag, 0,
3076 				       skb_frag_size(frag),
3077 				       DMA_TO_DEVICE);
3078 		if (dma_mapping_error(priv->device, des))
3079 			goto dma_map_err;
3080 
3081 		stmmac_tso_allocator(priv, des, skb_frag_size(frag),
3082 				     (i == nfrags - 1), queue);
3083 
3084 		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
3085 		tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
3086 		tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
3087 	}
3088 
3089 	tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
3090 
3091 	/* Only the last descriptor gets to point to the skb. */
3092 	tx_q->tx_skbuff[tx_q->cur_tx] = skb;
3093 
3094 	/* Manage tx mitigation */
3095 	tx_packets = (tx_q->cur_tx + 1) - first_tx;
3096 	tx_q->tx_count_frames += tx_packets;
3097 
3098 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
3099 		set_ic = true;
3100 	else if (!priv->tx_coal_frames)
3101 		set_ic = false;
3102 	else if (tx_packets > priv->tx_coal_frames)
3103 		set_ic = true;
3104 	else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
3105 		set_ic = true;
3106 	else
3107 		set_ic = false;
3108 
3109 	if (set_ic) {
3110 		if (tx_q->tbs & STMMAC_TBS_AVAIL)
3111 			desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3112 		else
3113 			desc = &tx_q->dma_tx[tx_q->cur_tx];
3114 
3115 		tx_q->tx_count_frames = 0;
3116 		stmmac_set_tx_ic(priv, desc);
3117 		priv->xstats.tx_set_ic_bit++;
3118 	}
3119 
3120 	/* We've used all descriptors we need for this skb, however,
3121 	 * advance cur_tx so that it references a fresh descriptor.
3122 	 * ndo_start_xmit will fill this descriptor the next time it's
3123 	 * called and stmmac_tx_clean may clean up to this descriptor.
3124 	 */
3125 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
3126 
3127 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3128 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3129 			  __func__);
3130 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3131 	}
3132 
3133 	dev->stats.tx_bytes += skb->len;
3134 	priv->xstats.tx_tso_frames++;
3135 	priv->xstats.tx_tso_nfrags += nfrags;
3136 
3137 	if (priv->sarc_type)
3138 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3139 
3140 	skb_tx_timestamp(skb);
3141 
3142 	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3143 		     priv->hwts_tx_en)) {
3144 		/* declare that device is doing timestamping */
3145 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3146 		stmmac_enable_tx_timestamp(priv, first);
3147 	}
3148 
3149 	/* Complete the first descriptor before granting the DMA */
3150 	stmmac_prepare_tso_tx_desc(priv, first, 1,
3151 			proto_hdr_len,
3152 			pay_len,
3153 			1, tx_q->tx_skbuff_dma[first_entry].last_segment,
3154 			hdr / 4, (skb->len - proto_hdr_len));
3155 
3156 	/* If context desc is used to change MSS */
3157 	if (mss_desc) {
3158 		/* Make sure that first descriptor has been completely
3159 		 * written, including its own bit. This is because MSS is
3160 		 * actually before first descriptor, so we need to make
3161 		 * sure that MSS's own bit is the last thing written.
3162 		 */
3163 		dma_wmb();
3164 		stmmac_set_tx_owner(priv, mss_desc);
3165 	}
3166 
3167 	/* The own bit must be the latest setting done when prepare the
3168 	 * descriptor and then barrier is needed to make sure that
3169 	 * all is coherent before granting the DMA engine.
3170 	 */
3171 	wmb();
3172 
3173 	if (netif_msg_pktdata(priv)) {
3174 		pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
3175 			__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3176 			tx_q->cur_tx, first, nfrags);
3177 		pr_info(">>> frame to be transmitted: ");
3178 		print_pkt(skb->data, skb_headlen(skb));
3179 	}
3180 
3181 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3182 
3183 	if (tx_q->tbs & STMMAC_TBS_AVAIL)
3184 		desc_size = sizeof(struct dma_edesc);
3185 	else
3186 		desc_size = sizeof(struct dma_desc);
3187 
3188 	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
3189 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3190 	stmmac_tx_timer_arm(priv, queue);
3191 
3192 	return NETDEV_TX_OK;
3193 
3194 dma_map_err:
3195 	dev_err(priv->device, "Tx dma map failed\n");
3196 	dev_kfree_skb(skb);
3197 	priv->dev->stats.tx_dropped++;
3198 	return NETDEV_TX_OK;
3199 }
3200 
3201 /**
3202  *  stmmac_xmit - Tx entry point of the driver
3203  *  @skb : the socket buffer
3204  *  @dev : device pointer
3205  *  Description : this is the tx entry point of the driver.
3206  *  It programs the chain or the ring and supports oversized frames
3207  *  and SG feature.
3208  */
3209 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
3210 {
3211 	unsigned int first_entry, tx_packets, enh_desc;
3212 	struct stmmac_priv *priv = netdev_priv(dev);
3213 	unsigned int nopaged_len = skb_headlen(skb);
3214 	int i, csum_insertion = 0, is_jumbo = 0;
3215 	u32 queue = skb_get_queue_mapping(skb);
3216 	int nfrags = skb_shinfo(skb)->nr_frags;
3217 	int gso = skb_shinfo(skb)->gso_type;
3218 	struct dma_edesc *tbs_desc = NULL;
3219 	int entry, desc_size, first_tx;
3220 	struct dma_desc *desc, *first;
3221 	struct stmmac_tx_queue *tx_q;
3222 	bool has_vlan, set_ic;
3223 	dma_addr_t des;
3224 
3225 	tx_q = &priv->tx_queue[queue];
3226 	first_tx = tx_q->cur_tx;
3227 
3228 	if (priv->tx_path_in_lpi_mode)
3229 		stmmac_disable_eee_mode(priv);
3230 
3231 	/* Manage oversized TCP frames for GMAC4 device */
3232 	if (skb_is_gso(skb) && priv->tso) {
3233 		if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
3234 			return stmmac_tso_xmit(skb, dev);
3235 		if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
3236 			return stmmac_tso_xmit(skb, dev);
3237 	}
3238 
3239 	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
3240 		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3241 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3242 								queue));
3243 			/* This is a hard error, log it. */
3244 			netdev_err(priv->dev,
3245 				   "%s: Tx Ring full when queue awake\n",
3246 				   __func__);
3247 		}
3248 		return NETDEV_TX_BUSY;
3249 	}
3250 
3251 	/* Check if VLAN can be inserted by HW */
3252 	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
3253 
3254 	entry = tx_q->cur_tx;
3255 	first_entry = entry;
3256 	WARN_ON(tx_q->tx_skbuff[first_entry]);
3257 
3258 	csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
3259 
3260 	if (likely(priv->extend_desc))
3261 		desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3262 	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3263 		desc = &tx_q->dma_entx[entry].basic;
3264 	else
3265 		desc = tx_q->dma_tx + entry;
3266 
3267 	first = desc;
3268 
3269 	if (has_vlan)
3270 		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
3271 
3272 	enh_desc = priv->plat->enh_desc;
3273 	/* To program the descriptors according to the size of the frame */
3274 	if (enh_desc)
3275 		is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
3276 
3277 	if (unlikely(is_jumbo)) {
3278 		entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
3279 		if (unlikely(entry < 0) && (entry != -EINVAL))
3280 			goto dma_map_err;
3281 	}
3282 
3283 	for (i = 0; i < nfrags; i++) {
3284 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3285 		int len = skb_frag_size(frag);
3286 		bool last_segment = (i == (nfrags - 1));
3287 
3288 		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3289 		WARN_ON(tx_q->tx_skbuff[entry]);
3290 
3291 		if (likely(priv->extend_desc))
3292 			desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3293 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3294 			desc = &tx_q->dma_entx[entry].basic;
3295 		else
3296 			desc = tx_q->dma_tx + entry;
3297 
3298 		des = skb_frag_dma_map(priv->device, frag, 0, len,
3299 				       DMA_TO_DEVICE);
3300 		if (dma_mapping_error(priv->device, des))
3301 			goto dma_map_err; /* should reuse desc w/o issues */
3302 
3303 		tx_q->tx_skbuff_dma[entry].buf = des;
3304 
3305 		stmmac_set_desc_addr(priv, desc, des);
3306 
3307 		tx_q->tx_skbuff_dma[entry].map_as_page = true;
3308 		tx_q->tx_skbuff_dma[entry].len = len;
3309 		tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3310 
3311 		/* Prepare the descriptor and set the own bit too */
3312 		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
3313 				priv->mode, 1, last_segment, skb->len);
3314 	}
3315 
3316 	/* Only the last descriptor gets to point to the skb. */
3317 	tx_q->tx_skbuff[entry] = skb;
3318 
3319 	/* According to the coalesce parameter the IC bit for the latest
3320 	 * segment is reset and the timer re-started to clean the tx status.
3321 	 * This approach takes care about the fragments: desc is the first
3322 	 * element in case of no SG.
3323 	 */
3324 	tx_packets = (entry + 1) - first_tx;
3325 	tx_q->tx_count_frames += tx_packets;
3326 
3327 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
3328 		set_ic = true;
3329 	else if (!priv->tx_coal_frames)
3330 		set_ic = false;
3331 	else if (tx_packets > priv->tx_coal_frames)
3332 		set_ic = true;
3333 	else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
3334 		set_ic = true;
3335 	else
3336 		set_ic = false;
3337 
3338 	if (set_ic) {
3339 		if (likely(priv->extend_desc))
3340 			desc = &tx_q->dma_etx[entry].basic;
3341 		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3342 			desc = &tx_q->dma_entx[entry].basic;
3343 		else
3344 			desc = &tx_q->dma_tx[entry];
3345 
3346 		tx_q->tx_count_frames = 0;
3347 		stmmac_set_tx_ic(priv, desc);
3348 		priv->xstats.tx_set_ic_bit++;
3349 	}
3350 
3351 	/* We've used all descriptors we need for this skb, however,
3352 	 * advance cur_tx so that it references a fresh descriptor.
3353 	 * ndo_start_xmit will fill this descriptor the next time it's
3354 	 * called and stmmac_tx_clean may clean up to this descriptor.
3355 	 */
3356 	entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3357 	tx_q->cur_tx = entry;
3358 
3359 	if (netif_msg_pktdata(priv)) {
3360 		netdev_dbg(priv->dev,
3361 			   "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3362 			   __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3363 			   entry, first, nfrags);
3364 
3365 		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3366 		print_pkt(skb->data, skb->len);
3367 	}
3368 
3369 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3370 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3371 			  __func__);
3372 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3373 	}
3374 
3375 	dev->stats.tx_bytes += skb->len;
3376 
3377 	if (priv->sarc_type)
3378 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3379 
3380 	skb_tx_timestamp(skb);
3381 
3382 	/* Ready to fill the first descriptor and set the OWN bit w/o any
3383 	 * problems because all the descriptors are actually ready to be
3384 	 * passed to the DMA engine.
3385 	 */
3386 	if (likely(!is_jumbo)) {
3387 		bool last_segment = (nfrags == 0);
3388 
3389 		des = dma_map_single(priv->device, skb->data,
3390 				     nopaged_len, DMA_TO_DEVICE);
3391 		if (dma_mapping_error(priv->device, des))
3392 			goto dma_map_err;
3393 
3394 		tx_q->tx_skbuff_dma[first_entry].buf = des;
3395 
3396 		stmmac_set_desc_addr(priv, first, des);
3397 
3398 		tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3399 		tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3400 
3401 		if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3402 			     priv->hwts_tx_en)) {
3403 			/* declare that device is doing timestamping */
3404 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3405 			stmmac_enable_tx_timestamp(priv, first);
3406 		}
3407 
3408 		/* Prepare the first descriptor setting the OWN bit too */
3409 		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
3410 				csum_insertion, priv->mode, 0, last_segment,
3411 				skb->len);
3412 	}
3413 
3414 	if (tx_q->tbs & STMMAC_TBS_EN) {
3415 		struct timespec64 ts = ns_to_timespec64(skb->tstamp);
3416 
3417 		tbs_desc = &tx_q->dma_entx[first_entry];
3418 		stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
3419 	}
3420 
3421 	stmmac_set_tx_owner(priv, first);
3422 
3423 	/* The own bit must be the latest setting done when prepare the
3424 	 * descriptor and then barrier is needed to make sure that
3425 	 * all is coherent before granting the DMA engine.
3426 	 */
3427 	wmb();
3428 
3429 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3430 
3431 	stmmac_enable_dma_transmission(priv, priv->ioaddr);
3432 
3433 	if (likely(priv->extend_desc))
3434 		desc_size = sizeof(struct dma_extended_desc);
3435 	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3436 		desc_size = sizeof(struct dma_edesc);
3437 	else
3438 		desc_size = sizeof(struct dma_desc);
3439 
3440 	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
3441 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3442 	stmmac_tx_timer_arm(priv, queue);
3443 
3444 	return NETDEV_TX_OK;
3445 
3446 dma_map_err:
3447 	netdev_err(priv->dev, "Tx DMA map failed\n");
3448 	dev_kfree_skb(skb);
3449 	priv->dev->stats.tx_dropped++;
3450 	return NETDEV_TX_OK;
3451 }
3452 
3453 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3454 {
3455 	struct vlan_ethhdr *veth;
3456 	__be16 vlan_proto;
3457 	u16 vlanid;
3458 
3459 	veth = (struct vlan_ethhdr *)skb->data;
3460 	vlan_proto = veth->h_vlan_proto;
3461 
3462 	if ((vlan_proto == htons(ETH_P_8021Q) &&
3463 	     dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
3464 	    (vlan_proto == htons(ETH_P_8021AD) &&
3465 	     dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
3466 		/* pop the vlan tag */
3467 		vlanid = ntohs(veth->h_vlan_TCI);
3468 		memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
3469 		skb_pull(skb, VLAN_HLEN);
3470 		__vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
3471 	}
3472 }
3473 
3474 
3475 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3476 {
3477 	if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3478 		return 0;
3479 
3480 	return 1;
3481 }
3482 
3483 /**
3484  * stmmac_rx_refill - refill used skb preallocated buffers
3485  * @priv: driver private structure
3486  * @queue: RX queue index
3487  * Description : this is to reallocate the skb for the reception process
3488  * that is based on zero-copy.
3489  */
3490 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3491 {
3492 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3493 	int len, dirty = stmmac_rx_dirty(priv, queue);
3494 	unsigned int entry = rx_q->dirty_rx;
3495 
3496 	len = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
3497 
3498 	while (dirty-- > 0) {
3499 		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
3500 		struct dma_desc *p;
3501 		bool use_rx_wd;
3502 
3503 		if (priv->extend_desc)
3504 			p = (struct dma_desc *)(rx_q->dma_erx + entry);
3505 		else
3506 			p = rx_q->dma_rx + entry;
3507 
3508 		if (!buf->page) {
3509 			buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
3510 			if (!buf->page)
3511 				break;
3512 		}
3513 
3514 		if (priv->sph && !buf->sec_page) {
3515 			buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
3516 			if (!buf->sec_page)
3517 				break;
3518 
3519 			buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
3520 
3521 			dma_sync_single_for_device(priv->device, buf->sec_addr,
3522 						   len, DMA_FROM_DEVICE);
3523 		}
3524 
3525 		buf->addr = page_pool_get_dma_addr(buf->page);
3526 
3527 		/* Sync whole allocation to device. This will invalidate old
3528 		 * data.
3529 		 */
3530 		dma_sync_single_for_device(priv->device, buf->addr, len,
3531 					   DMA_FROM_DEVICE);
3532 
3533 		stmmac_set_desc_addr(priv, p, buf->addr);
3534 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
3535 		stmmac_refill_desc3(priv, rx_q, p);
3536 
3537 		rx_q->rx_count_frames++;
3538 		rx_q->rx_count_frames += priv->rx_coal_frames;
3539 		if (rx_q->rx_count_frames > priv->rx_coal_frames)
3540 			rx_q->rx_count_frames = 0;
3541 
3542 		use_rx_wd = !priv->rx_coal_frames;
3543 		use_rx_wd |= rx_q->rx_count_frames > 0;
3544 		if (!priv->use_riwt)
3545 			use_rx_wd = false;
3546 
3547 		dma_wmb();
3548 		stmmac_set_rx_owner(priv, p, use_rx_wd);
3549 
3550 		entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3551 	}
3552 	rx_q->dirty_rx = entry;
3553 	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
3554 			    (rx_q->dirty_rx * sizeof(struct dma_desc));
3555 	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
3556 }
3557 
3558 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
3559 				       struct dma_desc *p,
3560 				       int status, unsigned int len)
3561 {
3562 	int ret, coe = priv->hw->rx_csum;
3563 	unsigned int plen = 0, hlen = 0;
3564 
3565 	/* Not first descriptor, buffer is always zero */
3566 	if (priv->sph && len)
3567 		return 0;
3568 
3569 	/* First descriptor, get split header length */
3570 	ret = stmmac_get_rx_header_len(priv, p, &hlen);
3571 	if (priv->sph && hlen) {
3572 		priv->xstats.rx_split_hdr_pkt_n++;
3573 		return hlen;
3574 	}
3575 
3576 	/* First descriptor, not last descriptor and not split header */
3577 	if (status & rx_not_ls)
3578 		return priv->dma_buf_sz;
3579 
3580 	plen = stmmac_get_rx_frame_len(priv, p, coe);
3581 
3582 	/* First descriptor and last descriptor and not split header */
3583 	return min_t(unsigned int, priv->dma_buf_sz, plen);
3584 }
3585 
3586 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
3587 				       struct dma_desc *p,
3588 				       int status, unsigned int len)
3589 {
3590 	int coe = priv->hw->rx_csum;
3591 	unsigned int plen = 0;
3592 
3593 	/* Not split header, buffer is not available */
3594 	if (!priv->sph)
3595 		return 0;
3596 
3597 	/* Not last descriptor */
3598 	if (status & rx_not_ls)
3599 		return priv->dma_buf_sz;
3600 
3601 	plen = stmmac_get_rx_frame_len(priv, p, coe);
3602 
3603 	/* Last descriptor */
3604 	return plen - len;
3605 }
3606 
3607 /**
3608  * stmmac_rx - manage the receive process
3609  * @priv: driver private structure
3610  * @limit: napi bugget
3611  * @queue: RX queue index.
3612  * Description :  this the function called by the napi poll method.
3613  * It gets all the frames inside the ring.
3614  */
3615 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3616 {
3617 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3618 	struct stmmac_channel *ch = &priv->channel[queue];
3619 	unsigned int count = 0, error = 0, len = 0;
3620 	int status = 0, coe = priv->hw->rx_csum;
3621 	unsigned int next_entry = rx_q->cur_rx;
3622 	struct sk_buff *skb = NULL;
3623 
3624 	if (netif_msg_rx_status(priv)) {
3625 		void *rx_head;
3626 
3627 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3628 		if (priv->extend_desc)
3629 			rx_head = (void *)rx_q->dma_erx;
3630 		else
3631 			rx_head = (void *)rx_q->dma_rx;
3632 
3633 		stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true);
3634 	}
3635 	while (count < limit) {
3636 		unsigned int buf1_len = 0, buf2_len = 0;
3637 		enum pkt_hash_types hash_type;
3638 		struct stmmac_rx_buffer *buf;
3639 		struct dma_desc *np, *p;
3640 		int entry;
3641 		u32 hash;
3642 
3643 		if (!count && rx_q->state_saved) {
3644 			skb = rx_q->state.skb;
3645 			error = rx_q->state.error;
3646 			len = rx_q->state.len;
3647 		} else {
3648 			rx_q->state_saved = false;
3649 			skb = NULL;
3650 			error = 0;
3651 			len = 0;
3652 		}
3653 
3654 		if (count >= limit)
3655 			break;
3656 
3657 read_again:
3658 		buf1_len = 0;
3659 		buf2_len = 0;
3660 		entry = next_entry;
3661 		buf = &rx_q->buf_pool[entry];
3662 
3663 		if (priv->extend_desc)
3664 			p = (struct dma_desc *)(rx_q->dma_erx + entry);
3665 		else
3666 			p = rx_q->dma_rx + entry;
3667 
3668 		/* read the status of the incoming frame */
3669 		status = stmmac_rx_status(priv, &priv->dev->stats,
3670 				&priv->xstats, p);
3671 		/* check if managed by the DMA otherwise go ahead */
3672 		if (unlikely(status & dma_own))
3673 			break;
3674 
3675 		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3676 		next_entry = rx_q->cur_rx;
3677 
3678 		if (priv->extend_desc)
3679 			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3680 		else
3681 			np = rx_q->dma_rx + next_entry;
3682 
3683 		prefetch(np);
3684 
3685 		if (priv->extend_desc)
3686 			stmmac_rx_extended_status(priv, &priv->dev->stats,
3687 					&priv->xstats, rx_q->dma_erx + entry);
3688 		if (unlikely(status == discard_frame)) {
3689 			page_pool_recycle_direct(rx_q->page_pool, buf->page);
3690 			buf->page = NULL;
3691 			error = 1;
3692 			if (!priv->hwts_rx_en)
3693 				priv->dev->stats.rx_errors++;
3694 		}
3695 
3696 		if (unlikely(error && (status & rx_not_ls)))
3697 			goto read_again;
3698 		if (unlikely(error)) {
3699 			dev_kfree_skb(skb);
3700 			skb = NULL;
3701 			count++;
3702 			continue;
3703 		}
3704 
3705 		/* Buffer is good. Go on. */
3706 
3707 		prefetch(page_address(buf->page));
3708 		if (buf->sec_page)
3709 			prefetch(page_address(buf->sec_page));
3710 
3711 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
3712 		len += buf1_len;
3713 		buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
3714 		len += buf2_len;
3715 
3716 		/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3717 		 * Type frames (LLC/LLC-SNAP)
3718 		 *
3719 		 * llc_snap is never checked in GMAC >= 4, so this ACS
3720 		 * feature is always disabled and packets need to be
3721 		 * stripped manually.
3722 		 */
3723 		if (likely(!(status & rx_not_ls)) &&
3724 		    (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
3725 		     unlikely(status != llc_snap))) {
3726 			if (buf2_len)
3727 				buf2_len -= ETH_FCS_LEN;
3728 			else
3729 				buf1_len -= ETH_FCS_LEN;
3730 
3731 			len -= ETH_FCS_LEN;
3732 		}
3733 
3734 		if (!skb) {
3735 			skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
3736 			if (!skb) {
3737 				priv->dev->stats.rx_dropped++;
3738 				count++;
3739 				goto drain_data;
3740 			}
3741 
3742 			dma_sync_single_for_cpu(priv->device, buf->addr,
3743 						buf1_len, DMA_FROM_DEVICE);
3744 			skb_copy_to_linear_data(skb, page_address(buf->page),
3745 						buf1_len);
3746 			skb_put(skb, buf1_len);
3747 
3748 			/* Data payload copied into SKB, page ready for recycle */
3749 			page_pool_recycle_direct(rx_q->page_pool, buf->page);
3750 			buf->page = NULL;
3751 		} else if (buf1_len) {
3752 			dma_sync_single_for_cpu(priv->device, buf->addr,
3753 						buf1_len, DMA_FROM_DEVICE);
3754 			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
3755 					buf->page, 0, buf1_len,
3756 					priv->dma_buf_sz);
3757 
3758 			/* Data payload appended into SKB */
3759 			page_pool_release_page(rx_q->page_pool, buf->page);
3760 			buf->page = NULL;
3761 		}
3762 
3763 		if (buf2_len) {
3764 			dma_sync_single_for_cpu(priv->device, buf->sec_addr,
3765 						buf2_len, DMA_FROM_DEVICE);
3766 			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
3767 					buf->sec_page, 0, buf2_len,
3768 					priv->dma_buf_sz);
3769 
3770 			/* Data payload appended into SKB */
3771 			page_pool_release_page(rx_q->page_pool, buf->sec_page);
3772 			buf->sec_page = NULL;
3773 		}
3774 
3775 drain_data:
3776 		if (likely(status & rx_not_ls))
3777 			goto read_again;
3778 		if (!skb)
3779 			continue;
3780 
3781 		/* Got entire packet into SKB. Finish it. */
3782 
3783 		stmmac_get_rx_hwtstamp(priv, p, np, skb);
3784 		stmmac_rx_vlan(priv->dev, skb);
3785 		skb->protocol = eth_type_trans(skb, priv->dev);
3786 
3787 		if (unlikely(!coe))
3788 			skb_checksum_none_assert(skb);
3789 		else
3790 			skb->ip_summed = CHECKSUM_UNNECESSARY;
3791 
3792 		if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
3793 			skb_set_hash(skb, hash, hash_type);
3794 
3795 		skb_record_rx_queue(skb, queue);
3796 		napi_gro_receive(&ch->rx_napi, skb);
3797 		skb = NULL;
3798 
3799 		priv->dev->stats.rx_packets++;
3800 		priv->dev->stats.rx_bytes += len;
3801 		count++;
3802 	}
3803 
3804 	if (status & rx_not_ls || skb) {
3805 		rx_q->state_saved = true;
3806 		rx_q->state.skb = skb;
3807 		rx_q->state.error = error;
3808 		rx_q->state.len = len;
3809 	}
3810 
3811 	stmmac_rx_refill(priv, queue);
3812 
3813 	priv->xstats.rx_pkt_n += count;
3814 
3815 	return count;
3816 }
3817 
3818 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
3819 {
3820 	struct stmmac_channel *ch =
3821 		container_of(napi, struct stmmac_channel, rx_napi);
3822 	struct stmmac_priv *priv = ch->priv_data;
3823 	u32 chan = ch->index;
3824 	int work_done;
3825 
3826 	priv->xstats.napi_poll++;
3827 
3828 	work_done = stmmac_rx(priv, budget, chan);
3829 	if (work_done < budget && napi_complete_done(napi, work_done)) {
3830 		unsigned long flags;
3831 
3832 		spin_lock_irqsave(&ch->lock, flags);
3833 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
3834 		spin_unlock_irqrestore(&ch->lock, flags);
3835 	}
3836 
3837 	return work_done;
3838 }
3839 
3840 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
3841 {
3842 	struct stmmac_channel *ch =
3843 		container_of(napi, struct stmmac_channel, tx_napi);
3844 	struct stmmac_priv *priv = ch->priv_data;
3845 	u32 chan = ch->index;
3846 	int work_done;
3847 
3848 	priv->xstats.napi_poll++;
3849 
3850 	work_done = stmmac_tx_clean(priv, DMA_TX_SIZE, chan);
3851 	work_done = min(work_done, budget);
3852 
3853 	if (work_done < budget && napi_complete_done(napi, work_done)) {
3854 		unsigned long flags;
3855 
3856 		spin_lock_irqsave(&ch->lock, flags);
3857 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
3858 		spin_unlock_irqrestore(&ch->lock, flags);
3859 	}
3860 
3861 	return work_done;
3862 }
3863 
3864 /**
3865  *  stmmac_tx_timeout
3866  *  @dev : Pointer to net device structure
3867  *  Description: this function is called when a packet transmission fails to
3868  *   complete within a reasonable time. The driver will mark the error in the
3869  *   netdev structure and arrange for the device to be reset to a sane state
3870  *   in order to transmit a new packet.
3871  */
3872 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue)
3873 {
3874 	struct stmmac_priv *priv = netdev_priv(dev);
3875 
3876 	stmmac_global_err(priv);
3877 }
3878 
3879 /**
3880  *  stmmac_set_rx_mode - entry point for multicast addressing
3881  *  @dev : pointer to the device structure
3882  *  Description:
3883  *  This function is a driver entry point which gets called by the kernel
3884  *  whenever multicast addresses must be enabled/disabled.
3885  *  Return value:
3886  *  void.
3887  */
3888 static void stmmac_set_rx_mode(struct net_device *dev)
3889 {
3890 	struct stmmac_priv *priv = netdev_priv(dev);
3891 
3892 	stmmac_set_filter(priv, priv->hw, dev);
3893 }
3894 
3895 /**
3896  *  stmmac_change_mtu - entry point to change MTU size for the device.
3897  *  @dev : device pointer.
3898  *  @new_mtu : the new MTU size for the device.
3899  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3900  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3901  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3902  *  Return value:
3903  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3904  *  file on failure.
3905  */
3906 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3907 {
3908 	struct stmmac_priv *priv = netdev_priv(dev);
3909 	int txfifosz = priv->plat->tx_fifo_size;
3910 
3911 	if (txfifosz == 0)
3912 		txfifosz = priv->dma_cap.tx_fifo_size;
3913 
3914 	txfifosz /= priv->plat->tx_queues_to_use;
3915 
3916 	if (netif_running(dev)) {
3917 		netdev_err(priv->dev, "must be stopped to change its MTU\n");
3918 		return -EBUSY;
3919 	}
3920 
3921 	new_mtu = STMMAC_ALIGN(new_mtu);
3922 
3923 	/* If condition true, FIFO is too small or MTU too large */
3924 	if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
3925 		return -EINVAL;
3926 
3927 	dev->mtu = new_mtu;
3928 
3929 	netdev_update_features(dev);
3930 
3931 	return 0;
3932 }
3933 
3934 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3935 					     netdev_features_t features)
3936 {
3937 	struct stmmac_priv *priv = netdev_priv(dev);
3938 
3939 	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3940 		features &= ~NETIF_F_RXCSUM;
3941 
3942 	if (!priv->plat->tx_coe)
3943 		features &= ~NETIF_F_CSUM_MASK;
3944 
3945 	/* Some GMAC devices have a bugged Jumbo frame support that
3946 	 * needs to have the Tx COE disabled for oversized frames
3947 	 * (due to limited buffer sizes). In this case we disable
3948 	 * the TX csum insertion in the TDES and not use SF.
3949 	 */
3950 	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3951 		features &= ~NETIF_F_CSUM_MASK;
3952 
3953 	/* Disable tso if asked by ethtool */
3954 	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3955 		if (features & NETIF_F_TSO)
3956 			priv->tso = true;
3957 		else
3958 			priv->tso = false;
3959 	}
3960 
3961 	return features;
3962 }
3963 
3964 static int stmmac_set_features(struct net_device *netdev,
3965 			       netdev_features_t features)
3966 {
3967 	struct stmmac_priv *priv = netdev_priv(netdev);
3968 	bool sph_en;
3969 	u32 chan;
3970 
3971 	/* Keep the COE Type in case of csum is supporting */
3972 	if (features & NETIF_F_RXCSUM)
3973 		priv->hw->rx_csum = priv->plat->rx_coe;
3974 	else
3975 		priv->hw->rx_csum = 0;
3976 	/* No check needed because rx_coe has been set before and it will be
3977 	 * fixed in case of issue.
3978 	 */
3979 	stmmac_rx_ipc(priv, priv->hw);
3980 
3981 	sph_en = (priv->hw->rx_csum > 0) && priv->sph;
3982 	for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
3983 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
3984 
3985 	return 0;
3986 }
3987 
3988 /**
3989  *  stmmac_interrupt - main ISR
3990  *  @irq: interrupt number.
3991  *  @dev_id: to pass the net device pointer.
3992  *  Description: this is the main driver interrupt service routine.
3993  *  It can call:
3994  *  o DMA service routine (to manage incoming frame reception and transmission
3995  *    status)
3996  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3997  *    interrupts.
3998  */
3999 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
4000 {
4001 	struct net_device *dev = (struct net_device *)dev_id;
4002 	struct stmmac_priv *priv = netdev_priv(dev);
4003 	u32 rx_cnt = priv->plat->rx_queues_to_use;
4004 	u32 tx_cnt = priv->plat->tx_queues_to_use;
4005 	u32 queues_count;
4006 	u32 queue;
4007 	bool xmac;
4008 
4009 	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
4010 	queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
4011 
4012 	if (priv->irq_wake)
4013 		pm_wakeup_event(priv->device, 0);
4014 
4015 	if (unlikely(!dev)) {
4016 		netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
4017 		return IRQ_NONE;
4018 	}
4019 
4020 	/* Check if adapter is up */
4021 	if (test_bit(STMMAC_DOWN, &priv->state))
4022 		return IRQ_HANDLED;
4023 	/* Check if a fatal error happened */
4024 	if (stmmac_safety_feat_interrupt(priv))
4025 		return IRQ_HANDLED;
4026 
4027 	/* To handle GMAC own interrupts */
4028 	if ((priv->plat->has_gmac) || xmac) {
4029 		int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
4030 		int mtl_status;
4031 
4032 		if (unlikely(status)) {
4033 			/* For LPI we need to save the tx status */
4034 			if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
4035 				priv->tx_path_in_lpi_mode = true;
4036 			if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
4037 				priv->tx_path_in_lpi_mode = false;
4038 		}
4039 
4040 		for (queue = 0; queue < queues_count; queue++) {
4041 			struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4042 
4043 			mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw,
4044 								queue);
4045 			if (mtl_status != -EINVAL)
4046 				status |= mtl_status;
4047 
4048 			if (status & CORE_IRQ_MTL_RX_OVERFLOW)
4049 				stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
4050 						       rx_q->rx_tail_addr,
4051 						       queue);
4052 		}
4053 
4054 		/* PCS link status */
4055 		if (priv->hw->pcs) {
4056 			if (priv->xstats.pcs_link)
4057 				netif_carrier_on(dev);
4058 			else
4059 				netif_carrier_off(dev);
4060 		}
4061 	}
4062 
4063 	/* To handle DMA interrupts */
4064 	stmmac_dma_interrupt(priv);
4065 
4066 	return IRQ_HANDLED;
4067 }
4068 
4069 #ifdef CONFIG_NET_POLL_CONTROLLER
4070 /* Polling receive - used by NETCONSOLE and other diagnostic tools
4071  * to allow network I/O with interrupts disabled.
4072  */
4073 static void stmmac_poll_controller(struct net_device *dev)
4074 {
4075 	disable_irq(dev->irq);
4076 	stmmac_interrupt(dev->irq, dev);
4077 	enable_irq(dev->irq);
4078 }
4079 #endif
4080 
4081 /**
4082  *  stmmac_ioctl - Entry point for the Ioctl
4083  *  @dev: Device pointer.
4084  *  @rq: An IOCTL specefic structure, that can contain a pointer to
4085  *  a proprietary structure used to pass information to the driver.
4086  *  @cmd: IOCTL command
4087  *  Description:
4088  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
4089  */
4090 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
4091 {
4092 	struct stmmac_priv *priv = netdev_priv (dev);
4093 	int ret = -EOPNOTSUPP;
4094 
4095 	if (!netif_running(dev))
4096 		return -EINVAL;
4097 
4098 	switch (cmd) {
4099 	case SIOCGMIIPHY:
4100 	case SIOCGMIIREG:
4101 	case SIOCSMIIREG:
4102 		ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
4103 		break;
4104 	case SIOCSHWTSTAMP:
4105 		ret = stmmac_hwtstamp_set(dev, rq);
4106 		break;
4107 	case SIOCGHWTSTAMP:
4108 		ret = stmmac_hwtstamp_get(dev, rq);
4109 		break;
4110 	default:
4111 		break;
4112 	}
4113 
4114 	return ret;
4115 }
4116 
4117 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4118 				    void *cb_priv)
4119 {
4120 	struct stmmac_priv *priv = cb_priv;
4121 	int ret = -EOPNOTSUPP;
4122 
4123 	if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
4124 		return ret;
4125 
4126 	stmmac_disable_all_queues(priv);
4127 
4128 	switch (type) {
4129 	case TC_SETUP_CLSU32:
4130 		ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
4131 		break;
4132 	case TC_SETUP_CLSFLOWER:
4133 		ret = stmmac_tc_setup_cls(priv, priv, type_data);
4134 		break;
4135 	default:
4136 		break;
4137 	}
4138 
4139 	stmmac_enable_all_queues(priv);
4140 	return ret;
4141 }
4142 
4143 static LIST_HEAD(stmmac_block_cb_list);
4144 
4145 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
4146 			   void *type_data)
4147 {
4148 	struct stmmac_priv *priv = netdev_priv(ndev);
4149 
4150 	switch (type) {
4151 	case TC_SETUP_BLOCK:
4152 		return flow_block_cb_setup_simple(type_data,
4153 						  &stmmac_block_cb_list,
4154 						  stmmac_setup_tc_block_cb,
4155 						  priv, priv, true);
4156 	case TC_SETUP_QDISC_CBS:
4157 		return stmmac_tc_setup_cbs(priv, priv, type_data);
4158 	case TC_SETUP_QDISC_TAPRIO:
4159 		return stmmac_tc_setup_taprio(priv, priv, type_data);
4160 	case TC_SETUP_QDISC_ETF:
4161 		return stmmac_tc_setup_etf(priv, priv, type_data);
4162 	default:
4163 		return -EOPNOTSUPP;
4164 	}
4165 }
4166 
4167 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
4168 			       struct net_device *sb_dev)
4169 {
4170 	int gso = skb_shinfo(skb)->gso_type;
4171 
4172 	if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
4173 		/*
4174 		 * There is no way to determine the number of TSO/USO
4175 		 * capable Queues. Let's use always the Queue 0
4176 		 * because if TSO/USO is supported then at least this
4177 		 * one will be capable.
4178 		 */
4179 		return 0;
4180 	}
4181 
4182 	return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
4183 }
4184 
4185 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
4186 {
4187 	struct stmmac_priv *priv = netdev_priv(ndev);
4188 	int ret = 0;
4189 
4190 	ret = eth_mac_addr(ndev, addr);
4191 	if (ret)
4192 		return ret;
4193 
4194 	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
4195 
4196 	return ret;
4197 }
4198 
4199 #ifdef CONFIG_DEBUG_FS
4200 static struct dentry *stmmac_fs_dir;
4201 
4202 static void sysfs_display_ring(void *head, int size, int extend_desc,
4203 			       struct seq_file *seq)
4204 {
4205 	int i;
4206 	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
4207 	struct dma_desc *p = (struct dma_desc *)head;
4208 
4209 	for (i = 0; i < size; i++) {
4210 		if (extend_desc) {
4211 			seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
4212 				   i, (unsigned int)virt_to_phys(ep),
4213 				   le32_to_cpu(ep->basic.des0),
4214 				   le32_to_cpu(ep->basic.des1),
4215 				   le32_to_cpu(ep->basic.des2),
4216 				   le32_to_cpu(ep->basic.des3));
4217 			ep++;
4218 		} else {
4219 			seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
4220 				   i, (unsigned int)virt_to_phys(p),
4221 				   le32_to_cpu(p->des0), le32_to_cpu(p->des1),
4222 				   le32_to_cpu(p->des2), le32_to_cpu(p->des3));
4223 			p++;
4224 		}
4225 		seq_printf(seq, "\n");
4226 	}
4227 }
4228 
4229 static int stmmac_rings_status_show(struct seq_file *seq, void *v)
4230 {
4231 	struct net_device *dev = seq->private;
4232 	struct stmmac_priv *priv = netdev_priv(dev);
4233 	u32 rx_count = priv->plat->rx_queues_to_use;
4234 	u32 tx_count = priv->plat->tx_queues_to_use;
4235 	u32 queue;
4236 
4237 	if ((dev->flags & IFF_UP) == 0)
4238 		return 0;
4239 
4240 	for (queue = 0; queue < rx_count; queue++) {
4241 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4242 
4243 		seq_printf(seq, "RX Queue %d:\n", queue);
4244 
4245 		if (priv->extend_desc) {
4246 			seq_printf(seq, "Extended descriptor ring:\n");
4247 			sysfs_display_ring((void *)rx_q->dma_erx,
4248 					   DMA_RX_SIZE, 1, seq);
4249 		} else {
4250 			seq_printf(seq, "Descriptor ring:\n");
4251 			sysfs_display_ring((void *)rx_q->dma_rx,
4252 					   DMA_RX_SIZE, 0, seq);
4253 		}
4254 	}
4255 
4256 	for (queue = 0; queue < tx_count; queue++) {
4257 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4258 
4259 		seq_printf(seq, "TX Queue %d:\n", queue);
4260 
4261 		if (priv->extend_desc) {
4262 			seq_printf(seq, "Extended descriptor ring:\n");
4263 			sysfs_display_ring((void *)tx_q->dma_etx,
4264 					   DMA_TX_SIZE, 1, seq);
4265 		} else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) {
4266 			seq_printf(seq, "Descriptor ring:\n");
4267 			sysfs_display_ring((void *)tx_q->dma_tx,
4268 					   DMA_TX_SIZE, 0, seq);
4269 		}
4270 	}
4271 
4272 	return 0;
4273 }
4274 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
4275 
4276 static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
4277 {
4278 	struct net_device *dev = seq->private;
4279 	struct stmmac_priv *priv = netdev_priv(dev);
4280 
4281 	if (!priv->hw_cap_support) {
4282 		seq_printf(seq, "DMA HW features not supported\n");
4283 		return 0;
4284 	}
4285 
4286 	seq_printf(seq, "==============================\n");
4287 	seq_printf(seq, "\tDMA HW features\n");
4288 	seq_printf(seq, "==============================\n");
4289 
4290 	seq_printf(seq, "\t10/100 Mbps: %s\n",
4291 		   (priv->dma_cap.mbps_10_100) ? "Y" : "N");
4292 	seq_printf(seq, "\t1000 Mbps: %s\n",
4293 		   (priv->dma_cap.mbps_1000) ? "Y" : "N");
4294 	seq_printf(seq, "\tHalf duplex: %s\n",
4295 		   (priv->dma_cap.half_duplex) ? "Y" : "N");
4296 	seq_printf(seq, "\tHash Filter: %s\n",
4297 		   (priv->dma_cap.hash_filter) ? "Y" : "N");
4298 	seq_printf(seq, "\tMultiple MAC address registers: %s\n",
4299 		   (priv->dma_cap.multi_addr) ? "Y" : "N");
4300 	seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
4301 		   (priv->dma_cap.pcs) ? "Y" : "N");
4302 	seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
4303 		   (priv->dma_cap.sma_mdio) ? "Y" : "N");
4304 	seq_printf(seq, "\tPMT Remote wake up: %s\n",
4305 		   (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
4306 	seq_printf(seq, "\tPMT Magic Frame: %s\n",
4307 		   (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
4308 	seq_printf(seq, "\tRMON module: %s\n",
4309 		   (priv->dma_cap.rmon) ? "Y" : "N");
4310 	seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
4311 		   (priv->dma_cap.time_stamp) ? "Y" : "N");
4312 	seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
4313 		   (priv->dma_cap.atime_stamp) ? "Y" : "N");
4314 	seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
4315 		   (priv->dma_cap.eee) ? "Y" : "N");
4316 	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
4317 	seq_printf(seq, "\tChecksum Offload in TX: %s\n",
4318 		   (priv->dma_cap.tx_coe) ? "Y" : "N");
4319 	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
4320 		seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
4321 			   (priv->dma_cap.rx_coe) ? "Y" : "N");
4322 	} else {
4323 		seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
4324 			   (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
4325 		seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
4326 			   (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
4327 	}
4328 	seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
4329 		   (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
4330 	seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
4331 		   priv->dma_cap.number_rx_channel);
4332 	seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
4333 		   priv->dma_cap.number_tx_channel);
4334 	seq_printf(seq, "\tNumber of Additional RX queues: %d\n",
4335 		   priv->dma_cap.number_rx_queues);
4336 	seq_printf(seq, "\tNumber of Additional TX queues: %d\n",
4337 		   priv->dma_cap.number_tx_queues);
4338 	seq_printf(seq, "\tEnhanced descriptors: %s\n",
4339 		   (priv->dma_cap.enh_desc) ? "Y" : "N");
4340 	seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);
4341 	seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);
4342 	seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz);
4343 	seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");
4344 	seq_printf(seq, "\tNumber of PPS Outputs: %d\n",
4345 		   priv->dma_cap.pps_out_num);
4346 	seq_printf(seq, "\tSafety Features: %s\n",
4347 		   priv->dma_cap.asp ? "Y" : "N");
4348 	seq_printf(seq, "\tFlexible RX Parser: %s\n",
4349 		   priv->dma_cap.frpsel ? "Y" : "N");
4350 	seq_printf(seq, "\tEnhanced Addressing: %d\n",
4351 		   priv->dma_cap.addr64);
4352 	seq_printf(seq, "\tReceive Side Scaling: %s\n",
4353 		   priv->dma_cap.rssen ? "Y" : "N");
4354 	seq_printf(seq, "\tVLAN Hash Filtering: %s\n",
4355 		   priv->dma_cap.vlhash ? "Y" : "N");
4356 	seq_printf(seq, "\tSplit Header: %s\n",
4357 		   priv->dma_cap.sphen ? "Y" : "N");
4358 	seq_printf(seq, "\tVLAN TX Insertion: %s\n",
4359 		   priv->dma_cap.vlins ? "Y" : "N");
4360 	seq_printf(seq, "\tDouble VLAN: %s\n",
4361 		   priv->dma_cap.dvlan ? "Y" : "N");
4362 	seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n",
4363 		   priv->dma_cap.l3l4fnum);
4364 	seq_printf(seq, "\tARP Offloading: %s\n",
4365 		   priv->dma_cap.arpoffsel ? "Y" : "N");
4366 	seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n",
4367 		   priv->dma_cap.estsel ? "Y" : "N");
4368 	seq_printf(seq, "\tFrame Preemption (FPE): %s\n",
4369 		   priv->dma_cap.fpesel ? "Y" : "N");
4370 	seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n",
4371 		   priv->dma_cap.tbssel ? "Y" : "N");
4372 	return 0;
4373 }
4374 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
4375 
4376 /* Use network device events to rename debugfs file entries.
4377  */
4378 static int stmmac_device_event(struct notifier_block *unused,
4379 			       unsigned long event, void *ptr)
4380 {
4381 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4382 	struct stmmac_priv *priv = netdev_priv(dev);
4383 
4384 	if (dev->netdev_ops != &stmmac_netdev_ops)
4385 		goto done;
4386 
4387 	switch (event) {
4388 	case NETDEV_CHANGENAME:
4389 		if (priv->dbgfs_dir)
4390 			priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,
4391 							 priv->dbgfs_dir,
4392 							 stmmac_fs_dir,
4393 							 dev->name);
4394 		break;
4395 	}
4396 done:
4397 	return NOTIFY_DONE;
4398 }
4399 
4400 static struct notifier_block stmmac_notifier = {
4401 	.notifier_call = stmmac_device_event,
4402 };
4403 
4404 static void stmmac_init_fs(struct net_device *dev)
4405 {
4406 	struct stmmac_priv *priv = netdev_priv(dev);
4407 
4408 	rtnl_lock();
4409 
4410 	/* Create per netdev entries */
4411 	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
4412 
4413 	/* Entry to report DMA RX/TX rings */
4414 	debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
4415 			    &stmmac_rings_status_fops);
4416 
4417 	/* Entry to report the DMA HW features */
4418 	debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
4419 			    &stmmac_dma_cap_fops);
4420 
4421 	rtnl_unlock();
4422 }
4423 
4424 static void stmmac_exit_fs(struct net_device *dev)
4425 {
4426 	struct stmmac_priv *priv = netdev_priv(dev);
4427 
4428 	debugfs_remove_recursive(priv->dbgfs_dir);
4429 }
4430 #endif /* CONFIG_DEBUG_FS */
4431 
4432 static u32 stmmac_vid_crc32_le(__le16 vid_le)
4433 {
4434 	unsigned char *data = (unsigned char *)&vid_le;
4435 	unsigned char data_byte = 0;
4436 	u32 crc = ~0x0;
4437 	u32 temp = 0;
4438 	int i, bits;
4439 
4440 	bits = get_bitmask_order(VLAN_VID_MASK);
4441 	for (i = 0; i < bits; i++) {
4442 		if ((i % 8) == 0)
4443 			data_byte = data[i / 8];
4444 
4445 		temp = ((crc & 1) ^ data_byte) & 1;
4446 		crc >>= 1;
4447 		data_byte >>= 1;
4448 
4449 		if (temp)
4450 			crc ^= 0xedb88320;
4451 	}
4452 
4453 	return crc;
4454 }
4455 
4456 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
4457 {
4458 	u32 crc, hash = 0;
4459 	__le16 pmatch = 0;
4460 	int count = 0;
4461 	u16 vid = 0;
4462 
4463 	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
4464 		__le16 vid_le = cpu_to_le16(vid);
4465 		crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
4466 		hash |= (1 << crc);
4467 		count++;
4468 	}
4469 
4470 	if (!priv->dma_cap.vlhash) {
4471 		if (count > 2) /* VID = 0 always passes filter */
4472 			return -EOPNOTSUPP;
4473 
4474 		pmatch = cpu_to_le16(vid);
4475 		hash = 0;
4476 	}
4477 
4478 	return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
4479 }
4480 
4481 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
4482 {
4483 	struct stmmac_priv *priv = netdev_priv(ndev);
4484 	bool is_double = false;
4485 	int ret;
4486 
4487 	if (be16_to_cpu(proto) == ETH_P_8021AD)
4488 		is_double = true;
4489 
4490 	set_bit(vid, priv->active_vlans);
4491 	ret = stmmac_vlan_update(priv, is_double);
4492 	if (ret) {
4493 		clear_bit(vid, priv->active_vlans);
4494 		return ret;
4495 	}
4496 
4497 	return ret;
4498 }
4499 
4500 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
4501 {
4502 	struct stmmac_priv *priv = netdev_priv(ndev);
4503 	bool is_double = false;
4504 
4505 	if (be16_to_cpu(proto) == ETH_P_8021AD)
4506 		is_double = true;
4507 
4508 	clear_bit(vid, priv->active_vlans);
4509 	return stmmac_vlan_update(priv, is_double);
4510 }
4511 
4512 static const struct net_device_ops stmmac_netdev_ops = {
4513 	.ndo_open = stmmac_open,
4514 	.ndo_start_xmit = stmmac_xmit,
4515 	.ndo_stop = stmmac_release,
4516 	.ndo_change_mtu = stmmac_change_mtu,
4517 	.ndo_fix_features = stmmac_fix_features,
4518 	.ndo_set_features = stmmac_set_features,
4519 	.ndo_set_rx_mode = stmmac_set_rx_mode,
4520 	.ndo_tx_timeout = stmmac_tx_timeout,
4521 	.ndo_do_ioctl = stmmac_ioctl,
4522 	.ndo_setup_tc = stmmac_setup_tc,
4523 	.ndo_select_queue = stmmac_select_queue,
4524 #ifdef CONFIG_NET_POLL_CONTROLLER
4525 	.ndo_poll_controller = stmmac_poll_controller,
4526 #endif
4527 	.ndo_set_mac_address = stmmac_set_mac_address,
4528 	.ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
4529 	.ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
4530 };
4531 
4532 static void stmmac_reset_subtask(struct stmmac_priv *priv)
4533 {
4534 	if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
4535 		return;
4536 	if (test_bit(STMMAC_DOWN, &priv->state))
4537 		return;
4538 
4539 	netdev_err(priv->dev, "Reset adapter.\n");
4540 
4541 	rtnl_lock();
4542 	netif_trans_update(priv->dev);
4543 	while (test_and_set_bit(STMMAC_RESETING, &priv->state))
4544 		usleep_range(1000, 2000);
4545 
4546 	set_bit(STMMAC_DOWN, &priv->state);
4547 	dev_close(priv->dev);
4548 	dev_open(priv->dev, NULL);
4549 	clear_bit(STMMAC_DOWN, &priv->state);
4550 	clear_bit(STMMAC_RESETING, &priv->state);
4551 	rtnl_unlock();
4552 }
4553 
4554 static void stmmac_service_task(struct work_struct *work)
4555 {
4556 	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
4557 			service_task);
4558 
4559 	stmmac_reset_subtask(priv);
4560 	clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
4561 }
4562 
4563 /**
4564  *  stmmac_hw_init - Init the MAC device
4565  *  @priv: driver private structure
4566  *  Description: this function is to configure the MAC device according to
4567  *  some platform parameters or the HW capability register. It prepares the
4568  *  driver to use either ring or chain modes and to setup either enhanced or
4569  *  normal descriptors.
4570  */
4571 static int stmmac_hw_init(struct stmmac_priv *priv)
4572 {
4573 	int ret;
4574 
4575 	/* dwmac-sun8i only work in chain mode */
4576 	if (priv->plat->has_sun8i)
4577 		chain_mode = 1;
4578 	priv->chain_mode = chain_mode;
4579 
4580 	/* Initialize HW Interface */
4581 	ret = stmmac_hwif_init(priv);
4582 	if (ret)
4583 		return ret;
4584 
4585 	/* Get the HW capability (new GMAC newer than 3.50a) */
4586 	priv->hw_cap_support = stmmac_get_hw_features(priv);
4587 	if (priv->hw_cap_support) {
4588 		dev_info(priv->device, "DMA HW capability register supported\n");
4589 
4590 		/* We can override some gmac/dma configuration fields: e.g.
4591 		 * enh_desc, tx_coe (e.g. that are passed through the
4592 		 * platform) with the values from the HW capability
4593 		 * register (if supported).
4594 		 */
4595 		priv->plat->enh_desc = priv->dma_cap.enh_desc;
4596 		priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4597 		priv->hw->pmt = priv->plat->pmt;
4598 		if (priv->dma_cap.hash_tb_sz) {
4599 			priv->hw->multicast_filter_bins =
4600 					(BIT(priv->dma_cap.hash_tb_sz) << 5);
4601 			priv->hw->mcast_bits_log2 =
4602 					ilog2(priv->hw->multicast_filter_bins);
4603 		}
4604 
4605 		/* TXCOE doesn't work in thresh DMA mode */
4606 		if (priv->plat->force_thresh_dma_mode)
4607 			priv->plat->tx_coe = 0;
4608 		else
4609 			priv->plat->tx_coe = priv->dma_cap.tx_coe;
4610 
4611 		/* In case of GMAC4 rx_coe is from HW cap register. */
4612 		priv->plat->rx_coe = priv->dma_cap.rx_coe;
4613 
4614 		if (priv->dma_cap.rx_coe_type2)
4615 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4616 		else if (priv->dma_cap.rx_coe_type1)
4617 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4618 
4619 	} else {
4620 		dev_info(priv->device, "No HW DMA feature register supported\n");
4621 	}
4622 
4623 	if (priv->plat->rx_coe) {
4624 		priv->hw->rx_csum = priv->plat->rx_coe;
4625 		dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4626 		if (priv->synopsys_id < DWMAC_CORE_4_00)
4627 			dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4628 	}
4629 	if (priv->plat->tx_coe)
4630 		dev_info(priv->device, "TX Checksum insertion supported\n");
4631 
4632 	if (priv->plat->pmt) {
4633 		dev_info(priv->device, "Wake-Up On Lan supported\n");
4634 		device_set_wakeup_capable(priv->device, 1);
4635 	}
4636 
4637 	if (priv->dma_cap.tsoen)
4638 		dev_info(priv->device, "TSO supported\n");
4639 
4640 	/* Run HW quirks, if any */
4641 	if (priv->hwif_quirks) {
4642 		ret = priv->hwif_quirks(priv);
4643 		if (ret)
4644 			return ret;
4645 	}
4646 
4647 	/* Rx Watchdog is available in the COREs newer than the 3.40.
4648 	 * In some case, for example on bugged HW this feature
4649 	 * has to be disable and this can be done by passing the
4650 	 * riwt_off field from the platform.
4651 	 */
4652 	if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
4653 	    (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
4654 		priv->use_riwt = 1;
4655 		dev_info(priv->device,
4656 			 "Enable RX Mitigation via HW Watchdog Timer\n");
4657 	}
4658 
4659 	return 0;
4660 }
4661 
4662 /**
4663  * stmmac_dvr_probe
4664  * @device: device pointer
4665  * @plat_dat: platform data pointer
4666  * @res: stmmac resource pointer
4667  * Description: this is the main probe function used to
4668  * call the alloc_etherdev, allocate the priv structure.
4669  * Return:
4670  * returns 0 on success, otherwise errno.
4671  */
4672 int stmmac_dvr_probe(struct device *device,
4673 		     struct plat_stmmacenet_data *plat_dat,
4674 		     struct stmmac_resources *res)
4675 {
4676 	struct net_device *ndev = NULL;
4677 	struct stmmac_priv *priv;
4678 	u32 queue, rxq, maxq;
4679 	int i, ret = 0;
4680 
4681 	ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
4682 				       MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
4683 	if (!ndev)
4684 		return -ENOMEM;
4685 
4686 	SET_NETDEV_DEV(ndev, device);
4687 
4688 	priv = netdev_priv(ndev);
4689 	priv->device = device;
4690 	priv->dev = ndev;
4691 
4692 	stmmac_set_ethtool_ops(ndev);
4693 	priv->pause = pause;
4694 	priv->plat = plat_dat;
4695 	priv->ioaddr = res->addr;
4696 	priv->dev->base_addr = (unsigned long)res->addr;
4697 
4698 	priv->dev->irq = res->irq;
4699 	priv->wol_irq = res->wol_irq;
4700 	priv->lpi_irq = res->lpi_irq;
4701 
4702 	if (!IS_ERR_OR_NULL(res->mac))
4703 		memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4704 
4705 	dev_set_drvdata(device, priv->dev);
4706 
4707 	/* Verify driver arguments */
4708 	stmmac_verify_args();
4709 
4710 	/* Allocate workqueue */
4711 	priv->wq = create_singlethread_workqueue("stmmac_wq");
4712 	if (!priv->wq) {
4713 		dev_err(priv->device, "failed to create workqueue\n");
4714 		return -ENOMEM;
4715 	}
4716 
4717 	INIT_WORK(&priv->service_task, stmmac_service_task);
4718 
4719 	/* Override with kernel parameters if supplied XXX CRS XXX
4720 	 * this needs to have multiple instances
4721 	 */
4722 	if ((phyaddr >= 0) && (phyaddr <= 31))
4723 		priv->plat->phy_addr = phyaddr;
4724 
4725 	if (priv->plat->stmmac_rst) {
4726 		ret = reset_control_assert(priv->plat->stmmac_rst);
4727 		reset_control_deassert(priv->plat->stmmac_rst);
4728 		/* Some reset controllers have only reset callback instead of
4729 		 * assert + deassert callbacks pair.
4730 		 */
4731 		if (ret == -ENOTSUPP)
4732 			reset_control_reset(priv->plat->stmmac_rst);
4733 	}
4734 
4735 	/* Init MAC and get the capabilities */
4736 	ret = stmmac_hw_init(priv);
4737 	if (ret)
4738 		goto error_hw_init;
4739 
4740 	stmmac_check_ether_addr(priv);
4741 
4742 	/* Configure real RX and TX queues */
4743 	netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4744 	netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4745 
4746 	ndev->netdev_ops = &stmmac_netdev_ops;
4747 
4748 	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4749 			    NETIF_F_RXCSUM;
4750 
4751 	ret = stmmac_tc_init(priv, priv);
4752 	if (!ret) {
4753 		ndev->hw_features |= NETIF_F_HW_TC;
4754 	}
4755 
4756 	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4757 		ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
4758 		if (priv->plat->has_gmac4)
4759 			ndev->hw_features |= NETIF_F_GSO_UDP_L4;
4760 		priv->tso = true;
4761 		dev_info(priv->device, "TSO feature enabled\n");
4762 	}
4763 
4764 	if (priv->dma_cap.sphen) {
4765 		ndev->hw_features |= NETIF_F_GRO;
4766 		priv->sph = true;
4767 		dev_info(priv->device, "SPH feature enabled\n");
4768 	}
4769 
4770 	if (priv->dma_cap.addr64) {
4771 		ret = dma_set_mask_and_coherent(device,
4772 				DMA_BIT_MASK(priv->dma_cap.addr64));
4773 		if (!ret) {
4774 			dev_info(priv->device, "Using %d bits DMA width\n",
4775 				 priv->dma_cap.addr64);
4776 
4777 			/*
4778 			 * If more than 32 bits can be addressed, make sure to
4779 			 * enable enhanced addressing mode.
4780 			 */
4781 			if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
4782 				priv->plat->dma_cfg->eame = true;
4783 		} else {
4784 			ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
4785 			if (ret) {
4786 				dev_err(priv->device, "Failed to set DMA Mask\n");
4787 				goto error_hw_init;
4788 			}
4789 
4790 			priv->dma_cap.addr64 = 32;
4791 		}
4792 	}
4793 
4794 	ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4795 	ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4796 #ifdef STMMAC_VLAN_TAG_USED
4797 	/* Both mac100 and gmac support receive VLAN tag detection */
4798 	ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
4799 	if (priv->dma_cap.vlhash) {
4800 		ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4801 		ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
4802 	}
4803 	if (priv->dma_cap.vlins) {
4804 		ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
4805 		if (priv->dma_cap.dvlan)
4806 			ndev->features |= NETIF_F_HW_VLAN_STAG_TX;
4807 	}
4808 #endif
4809 	priv->msg_enable = netif_msg_init(debug, default_msg_level);
4810 
4811 	/* Initialize RSS */
4812 	rxq = priv->plat->rx_queues_to_use;
4813 	netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
4814 	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
4815 		priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
4816 
4817 	if (priv->dma_cap.rssen && priv->plat->rss_en)
4818 		ndev->features |= NETIF_F_RXHASH;
4819 
4820 	/* MTU range: 46 - hw-specific max */
4821 	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4822 	if (priv->plat->has_xgmac)
4823 		ndev->max_mtu = XGMAC_JUMBO_LEN;
4824 	else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4825 		ndev->max_mtu = JUMBO_LEN;
4826 	else
4827 		ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4828 	/* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4829 	 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4830 	 */
4831 	if ((priv->plat->maxmtu < ndev->max_mtu) &&
4832 	    (priv->plat->maxmtu >= ndev->min_mtu))
4833 		ndev->max_mtu = priv->plat->maxmtu;
4834 	else if (priv->plat->maxmtu < ndev->min_mtu)
4835 		dev_warn(priv->device,
4836 			 "%s: warning: maxmtu having invalid value (%d)\n",
4837 			 __func__, priv->plat->maxmtu);
4838 
4839 	if (flow_ctrl)
4840 		priv->flow_ctrl = FLOW_AUTO;	/* RX/TX pause on */
4841 
4842 	/* Setup channels NAPI */
4843 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
4844 
4845 	for (queue = 0; queue < maxq; queue++) {
4846 		struct stmmac_channel *ch = &priv->channel[queue];
4847 
4848 		spin_lock_init(&ch->lock);
4849 		ch->priv_data = priv;
4850 		ch->index = queue;
4851 
4852 		if (queue < priv->plat->rx_queues_to_use) {
4853 			netif_napi_add(ndev, &ch->rx_napi, stmmac_napi_poll_rx,
4854 				       NAPI_POLL_WEIGHT);
4855 		}
4856 		if (queue < priv->plat->tx_queues_to_use) {
4857 			netif_tx_napi_add(ndev, &ch->tx_napi,
4858 					  stmmac_napi_poll_tx,
4859 					  NAPI_POLL_WEIGHT);
4860 		}
4861 	}
4862 
4863 	mutex_init(&priv->lock);
4864 
4865 	/* If a specific clk_csr value is passed from the platform
4866 	 * this means that the CSR Clock Range selection cannot be
4867 	 * changed at run-time and it is fixed. Viceversa the driver'll try to
4868 	 * set the MDC clock dynamically according to the csr actual
4869 	 * clock input.
4870 	 */
4871 	if (priv->plat->clk_csr >= 0)
4872 		priv->clk_csr = priv->plat->clk_csr;
4873 	else
4874 		stmmac_clk_csr_set(priv);
4875 
4876 	stmmac_check_pcs_mode(priv);
4877 
4878 	if (priv->hw->pcs != STMMAC_PCS_TBI &&
4879 	    priv->hw->pcs != STMMAC_PCS_RTBI) {
4880 		/* MDIO bus Registration */
4881 		ret = stmmac_mdio_register(ndev);
4882 		if (ret < 0) {
4883 			dev_err(priv->device,
4884 				"%s: MDIO bus (id: %d) registration failed",
4885 				__func__, priv->plat->bus_id);
4886 			goto error_mdio_register;
4887 		}
4888 	}
4889 
4890 	ret = stmmac_phy_setup(priv);
4891 	if (ret) {
4892 		netdev_err(ndev, "failed to setup phy (%d)\n", ret);
4893 		goto error_phy_setup;
4894 	}
4895 
4896 	ret = register_netdev(ndev);
4897 	if (ret) {
4898 		dev_err(priv->device, "%s: ERROR %i registering the device\n",
4899 			__func__, ret);
4900 		goto error_netdev_register;
4901 	}
4902 
4903 #ifdef CONFIG_DEBUG_FS
4904 	stmmac_init_fs(ndev);
4905 #endif
4906 
4907 	return ret;
4908 
4909 error_netdev_register:
4910 	phylink_destroy(priv->phylink);
4911 error_phy_setup:
4912 	if (priv->hw->pcs != STMMAC_PCS_TBI &&
4913 	    priv->hw->pcs != STMMAC_PCS_RTBI)
4914 		stmmac_mdio_unregister(ndev);
4915 error_mdio_register:
4916 	for (queue = 0; queue < maxq; queue++) {
4917 		struct stmmac_channel *ch = &priv->channel[queue];
4918 
4919 		if (queue < priv->plat->rx_queues_to_use)
4920 			netif_napi_del(&ch->rx_napi);
4921 		if (queue < priv->plat->tx_queues_to_use)
4922 			netif_napi_del(&ch->tx_napi);
4923 	}
4924 error_hw_init:
4925 	destroy_workqueue(priv->wq);
4926 
4927 	return ret;
4928 }
4929 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4930 
4931 /**
4932  * stmmac_dvr_remove
4933  * @dev: device pointer
4934  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4935  * changes the link status, releases the DMA descriptor rings.
4936  */
4937 int stmmac_dvr_remove(struct device *dev)
4938 {
4939 	struct net_device *ndev = dev_get_drvdata(dev);
4940 	struct stmmac_priv *priv = netdev_priv(ndev);
4941 
4942 	netdev_info(priv->dev, "%s: removing driver", __func__);
4943 
4944 	stmmac_stop_all_dma(priv);
4945 
4946 	stmmac_mac_set(priv, priv->ioaddr, false);
4947 	netif_carrier_off(ndev);
4948 	unregister_netdev(ndev);
4949 #ifdef CONFIG_DEBUG_FS
4950 	stmmac_exit_fs(ndev);
4951 #endif
4952 	phylink_destroy(priv->phylink);
4953 	if (priv->plat->stmmac_rst)
4954 		reset_control_assert(priv->plat->stmmac_rst);
4955 	clk_disable_unprepare(priv->plat->pclk);
4956 	clk_disable_unprepare(priv->plat->stmmac_clk);
4957 	if (priv->hw->pcs != STMMAC_PCS_TBI &&
4958 	    priv->hw->pcs != STMMAC_PCS_RTBI)
4959 		stmmac_mdio_unregister(ndev);
4960 	destroy_workqueue(priv->wq);
4961 	mutex_destroy(&priv->lock);
4962 
4963 	return 0;
4964 }
4965 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4966 
4967 /**
4968  * stmmac_suspend - suspend callback
4969  * @dev: device pointer
4970  * Description: this is the function to suspend the device and it is called
4971  * by the platform driver to stop the network queue, release the resources,
4972  * program the PMT register (for WoL), clean and release driver resources.
4973  */
4974 int stmmac_suspend(struct device *dev)
4975 {
4976 	struct net_device *ndev = dev_get_drvdata(dev);
4977 	struct stmmac_priv *priv = netdev_priv(ndev);
4978 	u32 chan;
4979 
4980 	if (!ndev || !netif_running(ndev))
4981 		return 0;
4982 
4983 	phylink_mac_change(priv->phylink, false);
4984 
4985 	mutex_lock(&priv->lock);
4986 
4987 	netif_device_detach(ndev);
4988 	stmmac_stop_all_queues(priv);
4989 
4990 	stmmac_disable_all_queues(priv);
4991 
4992 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
4993 		del_timer_sync(&priv->tx_queue[chan].txtimer);
4994 
4995 	/* Stop TX/RX DMA */
4996 	stmmac_stop_all_dma(priv);
4997 
4998 	/* Enable Power down mode by programming the PMT regs */
4999 	if (device_may_wakeup(priv->device)) {
5000 		stmmac_pmt(priv, priv->hw, priv->wolopts);
5001 		priv->irq_wake = 1;
5002 	} else {
5003 		mutex_unlock(&priv->lock);
5004 		rtnl_lock();
5005 		phylink_stop(priv->phylink);
5006 		rtnl_unlock();
5007 		mutex_lock(&priv->lock);
5008 
5009 		stmmac_mac_set(priv, priv->ioaddr, false);
5010 		pinctrl_pm_select_sleep_state(priv->device);
5011 		/* Disable clock in case of PWM is off */
5012 		if (priv->plat->clk_ptp_ref)
5013 			clk_disable_unprepare(priv->plat->clk_ptp_ref);
5014 		clk_disable_unprepare(priv->plat->pclk);
5015 		clk_disable_unprepare(priv->plat->stmmac_clk);
5016 	}
5017 	mutex_unlock(&priv->lock);
5018 
5019 	priv->speed = SPEED_UNKNOWN;
5020 	return 0;
5021 }
5022 EXPORT_SYMBOL_GPL(stmmac_suspend);
5023 
5024 /**
5025  * stmmac_reset_queues_param - reset queue parameters
5026  * @dev: device pointer
5027  */
5028 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
5029 {
5030 	u32 rx_cnt = priv->plat->rx_queues_to_use;
5031 	u32 tx_cnt = priv->plat->tx_queues_to_use;
5032 	u32 queue;
5033 
5034 	for (queue = 0; queue < rx_cnt; queue++) {
5035 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
5036 
5037 		rx_q->cur_rx = 0;
5038 		rx_q->dirty_rx = 0;
5039 	}
5040 
5041 	for (queue = 0; queue < tx_cnt; queue++) {
5042 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
5043 
5044 		tx_q->cur_tx = 0;
5045 		tx_q->dirty_tx = 0;
5046 		tx_q->mss = 0;
5047 	}
5048 }
5049 
5050 /**
5051  * stmmac_resume - resume callback
5052  * @dev: device pointer
5053  * Description: when resume this function is invoked to setup the DMA and CORE
5054  * in a usable state.
5055  */
5056 int stmmac_resume(struct device *dev)
5057 {
5058 	struct net_device *ndev = dev_get_drvdata(dev);
5059 	struct stmmac_priv *priv = netdev_priv(ndev);
5060 
5061 	if (!netif_running(ndev))
5062 		return 0;
5063 
5064 	/* Power Down bit, into the PM register, is cleared
5065 	 * automatically as soon as a magic packet or a Wake-up frame
5066 	 * is received. Anyway, it's better to manually clear
5067 	 * this bit because it can generate problems while resuming
5068 	 * from another devices (e.g. serial console).
5069 	 */
5070 	if (device_may_wakeup(priv->device)) {
5071 		mutex_lock(&priv->lock);
5072 		stmmac_pmt(priv, priv->hw, 0);
5073 		mutex_unlock(&priv->lock);
5074 		priv->irq_wake = 0;
5075 	} else {
5076 		pinctrl_pm_select_default_state(priv->device);
5077 		/* enable the clk previously disabled */
5078 		clk_prepare_enable(priv->plat->stmmac_clk);
5079 		clk_prepare_enable(priv->plat->pclk);
5080 		if (priv->plat->clk_ptp_ref)
5081 			clk_prepare_enable(priv->plat->clk_ptp_ref);
5082 		/* reset the phy so that it's ready */
5083 		if (priv->mii)
5084 			stmmac_mdio_reset(priv->mii);
5085 	}
5086 
5087 	netif_device_attach(ndev);
5088 
5089 	mutex_lock(&priv->lock);
5090 
5091 	stmmac_reset_queues_param(priv);
5092 
5093 	stmmac_clear_descriptors(priv);
5094 
5095 	stmmac_hw_setup(ndev, false);
5096 	stmmac_init_coalesce(priv);
5097 	stmmac_set_rx_mode(ndev);
5098 
5099 	stmmac_enable_all_queues(priv);
5100 
5101 	stmmac_start_all_queues(priv);
5102 
5103 	mutex_unlock(&priv->lock);
5104 
5105 	if (!device_may_wakeup(priv->device)) {
5106 		rtnl_lock();
5107 		phylink_start(priv->phylink);
5108 		rtnl_unlock();
5109 	}
5110 
5111 	phylink_mac_change(priv->phylink, true);
5112 
5113 	return 0;
5114 }
5115 EXPORT_SYMBOL_GPL(stmmac_resume);
5116 
5117 #ifndef MODULE
5118 static int __init stmmac_cmdline_opt(char *str)
5119 {
5120 	char *opt;
5121 
5122 	if (!str || !*str)
5123 		return -EINVAL;
5124 	while ((opt = strsep(&str, ",")) != NULL) {
5125 		if (!strncmp(opt, "debug:", 6)) {
5126 			if (kstrtoint(opt + 6, 0, &debug))
5127 				goto err;
5128 		} else if (!strncmp(opt, "phyaddr:", 8)) {
5129 			if (kstrtoint(opt + 8, 0, &phyaddr))
5130 				goto err;
5131 		} else if (!strncmp(opt, "buf_sz:", 7)) {
5132 			if (kstrtoint(opt + 7, 0, &buf_sz))
5133 				goto err;
5134 		} else if (!strncmp(opt, "tc:", 3)) {
5135 			if (kstrtoint(opt + 3, 0, &tc))
5136 				goto err;
5137 		} else if (!strncmp(opt, "watchdog:", 9)) {
5138 			if (kstrtoint(opt + 9, 0, &watchdog))
5139 				goto err;
5140 		} else if (!strncmp(opt, "flow_ctrl:", 10)) {
5141 			if (kstrtoint(opt + 10, 0, &flow_ctrl))
5142 				goto err;
5143 		} else if (!strncmp(opt, "pause:", 6)) {
5144 			if (kstrtoint(opt + 6, 0, &pause))
5145 				goto err;
5146 		} else if (!strncmp(opt, "eee_timer:", 10)) {
5147 			if (kstrtoint(opt + 10, 0, &eee_timer))
5148 				goto err;
5149 		} else if (!strncmp(opt, "chain_mode:", 11)) {
5150 			if (kstrtoint(opt + 11, 0, &chain_mode))
5151 				goto err;
5152 		}
5153 	}
5154 	return 0;
5155 
5156 err:
5157 	pr_err("%s: ERROR broken module parameter conversion", __func__);
5158 	return -EINVAL;
5159 }
5160 
5161 __setup("stmmaceth=", stmmac_cmdline_opt);
5162 #endif /* MODULE */
5163 
5164 static int __init stmmac_init(void)
5165 {
5166 #ifdef CONFIG_DEBUG_FS
5167 	/* Create debugfs main directory if it doesn't exist yet */
5168 	if (!stmmac_fs_dir)
5169 		stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
5170 	register_netdevice_notifier(&stmmac_notifier);
5171 #endif
5172 
5173 	return 0;
5174 }
5175 
5176 static void __exit stmmac_exit(void)
5177 {
5178 #ifdef CONFIG_DEBUG_FS
5179 	unregister_netdevice_notifier(&stmmac_notifier);
5180 	debugfs_remove_recursive(stmmac_fs_dir);
5181 #endif
5182 }
5183 
5184 module_init(stmmac_init)
5185 module_exit(stmmac_exit)
5186 
5187 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
5188 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
5189 MODULE_LICENSE("GPL");
5190