1 // SPDX-License-Identifier: GPL-2.0-only
2 /*******************************************************************************
3   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
4   ST Ethernet IPs are built around a Synopsys IP Core.
5 
6 	Copyright(C) 2007-2011 STMicroelectronics Ltd
7 
8 
9   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
10 
11   Documentation available at:
12 	http://www.stlinux.com
13   Support available at:
14 	https://bugzilla.stlinux.com/
15 *******************************************************************************/
16 
17 #include <linux/clk.h>
18 #include <linux/kernel.h>
19 #include <linux/interrupt.h>
20 #include <linux/ip.h>
21 #include <linux/tcp.h>
22 #include <linux/skbuff.h>
23 #include <linux/ethtool.h>
24 #include <linux/if_ether.h>
25 #include <linux/crc32.h>
26 #include <linux/mii.h>
27 #include <linux/if.h>
28 #include <linux/if_vlan.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/slab.h>
31 #include <linux/prefetch.h>
32 #include <linux/pinctrl/consumer.h>
33 #ifdef CONFIG_DEBUG_FS
34 #include <linux/debugfs.h>
35 #include <linux/seq_file.h>
36 #endif /* CONFIG_DEBUG_FS */
37 #include <linux/net_tstamp.h>
38 #include <linux/phylink.h>
39 #include <linux/udp.h>
40 #include <net/pkt_cls.h>
41 #include "stmmac_ptp.h"
42 #include "stmmac.h"
43 #include <linux/reset.h>
44 #include <linux/of_mdio.h>
45 #include "dwmac1000.h"
46 #include "dwxgmac2.h"
47 #include "hwif.h"
48 
49 #define	STMMAC_ALIGN(x)		ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
50 #define	TSO_MAX_BUFF_SIZE	(SZ_16K - 1)
51 
52 /* Module parameters */
53 #define TX_TIMEO	5000
54 static int watchdog = TX_TIMEO;
55 module_param(watchdog, int, 0644);
56 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
57 
58 static int debug = -1;
59 module_param(debug, int, 0644);
60 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
61 
62 static int phyaddr = -1;
63 module_param(phyaddr, int, 0444);
64 MODULE_PARM_DESC(phyaddr, "Physical device address");
65 
66 #define STMMAC_TX_THRESH	(DMA_TX_SIZE / 4)
67 #define STMMAC_RX_THRESH	(DMA_RX_SIZE / 4)
68 
69 static int flow_ctrl = FLOW_AUTO;
70 module_param(flow_ctrl, int, 0644);
71 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
72 
73 static int pause = PAUSE_TIME;
74 module_param(pause, int, 0644);
75 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
76 
77 #define TC_DEFAULT 64
78 static int tc = TC_DEFAULT;
79 module_param(tc, int, 0644);
80 MODULE_PARM_DESC(tc, "DMA threshold control value");
81 
82 #define	DEFAULT_BUFSIZE	1536
83 static int buf_sz = DEFAULT_BUFSIZE;
84 module_param(buf_sz, int, 0644);
85 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
86 
87 #define	STMMAC_RX_COPYBREAK	256
88 
89 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
90 				      NETIF_MSG_LINK | NETIF_MSG_IFUP |
91 				      NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
92 
93 #define STMMAC_DEFAULT_LPI_TIMER	1000
94 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
95 module_param(eee_timer, int, 0644);
96 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
97 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
98 
99 /* By default the driver will use the ring mode to manage tx and rx descriptors,
100  * but allow user to force to use the chain instead of the ring
101  */
102 static unsigned int chain_mode;
103 module_param(chain_mode, int, 0444);
104 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
105 
106 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
107 
108 #ifdef CONFIG_DEBUG_FS
109 static const struct net_device_ops stmmac_netdev_ops;
110 static void stmmac_init_fs(struct net_device *dev);
111 static void stmmac_exit_fs(struct net_device *dev);
112 #endif
113 
114 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
115 
116 /**
117  * stmmac_verify_args - verify the driver parameters.
118  * Description: it checks the driver parameters and set a default in case of
119  * errors.
120  */
121 static void stmmac_verify_args(void)
122 {
123 	if (unlikely(watchdog < 0))
124 		watchdog = TX_TIMEO;
125 	if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
126 		buf_sz = DEFAULT_BUFSIZE;
127 	if (unlikely(flow_ctrl > 1))
128 		flow_ctrl = FLOW_AUTO;
129 	else if (likely(flow_ctrl < 0))
130 		flow_ctrl = FLOW_OFF;
131 	if (unlikely((pause < 0) || (pause > 0xffff)))
132 		pause = PAUSE_TIME;
133 	if (eee_timer < 0)
134 		eee_timer = STMMAC_DEFAULT_LPI_TIMER;
135 }
136 
137 /**
138  * stmmac_disable_all_queues - Disable all queues
139  * @priv: driver private structure
140  */
141 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
142 {
143 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
144 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
145 	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
146 	u32 queue;
147 
148 	for (queue = 0; queue < maxq; queue++) {
149 		struct stmmac_channel *ch = &priv->channel[queue];
150 
151 		if (queue < rx_queues_cnt)
152 			napi_disable(&ch->rx_napi);
153 		if (queue < tx_queues_cnt)
154 			napi_disable(&ch->tx_napi);
155 	}
156 }
157 
158 /**
159  * stmmac_enable_all_queues - Enable all queues
160  * @priv: driver private structure
161  */
162 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
163 {
164 	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
165 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
166 	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
167 	u32 queue;
168 
169 	for (queue = 0; queue < maxq; queue++) {
170 		struct stmmac_channel *ch = &priv->channel[queue];
171 
172 		if (queue < rx_queues_cnt)
173 			napi_enable(&ch->rx_napi);
174 		if (queue < tx_queues_cnt)
175 			napi_enable(&ch->tx_napi);
176 	}
177 }
178 
179 /**
180  * stmmac_stop_all_queues - Stop all queues
181  * @priv: driver private structure
182  */
183 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
184 {
185 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
186 	u32 queue;
187 
188 	for (queue = 0; queue < tx_queues_cnt; queue++)
189 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
190 }
191 
192 /**
193  * stmmac_start_all_queues - Start all queues
194  * @priv: driver private structure
195  */
196 static void stmmac_start_all_queues(struct stmmac_priv *priv)
197 {
198 	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
199 	u32 queue;
200 
201 	for (queue = 0; queue < tx_queues_cnt; queue++)
202 		netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
203 }
204 
205 static void stmmac_service_event_schedule(struct stmmac_priv *priv)
206 {
207 	if (!test_bit(STMMAC_DOWN, &priv->state) &&
208 	    !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
209 		queue_work(priv->wq, &priv->service_task);
210 }
211 
212 static void stmmac_global_err(struct stmmac_priv *priv)
213 {
214 	netif_carrier_off(priv->dev);
215 	set_bit(STMMAC_RESET_REQUESTED, &priv->state);
216 	stmmac_service_event_schedule(priv);
217 }
218 
219 /**
220  * stmmac_clk_csr_set - dynamically set the MDC clock
221  * @priv: driver private structure
222  * Description: this is to dynamically set the MDC clock according to the csr
223  * clock input.
224  * Note:
225  *	If a specific clk_csr value is passed from the platform
226  *	this means that the CSR Clock Range selection cannot be
227  *	changed at run-time and it is fixed (as reported in the driver
228  *	documentation). Viceversa the driver will try to set the MDC
229  *	clock dynamically according to the actual clock input.
230  */
231 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
232 {
233 	u32 clk_rate;
234 
235 	clk_rate = clk_get_rate(priv->plat->stmmac_clk);
236 
237 	/* Platform provided default clk_csr would be assumed valid
238 	 * for all other cases except for the below mentioned ones.
239 	 * For values higher than the IEEE 802.3 specified frequency
240 	 * we can not estimate the proper divider as it is not known
241 	 * the frequency of clk_csr_i. So we do not change the default
242 	 * divider.
243 	 */
244 	if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
245 		if (clk_rate < CSR_F_35M)
246 			priv->clk_csr = STMMAC_CSR_20_35M;
247 		else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
248 			priv->clk_csr = STMMAC_CSR_35_60M;
249 		else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
250 			priv->clk_csr = STMMAC_CSR_60_100M;
251 		else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
252 			priv->clk_csr = STMMAC_CSR_100_150M;
253 		else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
254 			priv->clk_csr = STMMAC_CSR_150_250M;
255 		else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
256 			priv->clk_csr = STMMAC_CSR_250_300M;
257 	}
258 
259 	if (priv->plat->has_sun8i) {
260 		if (clk_rate > 160000000)
261 			priv->clk_csr = 0x03;
262 		else if (clk_rate > 80000000)
263 			priv->clk_csr = 0x02;
264 		else if (clk_rate > 40000000)
265 			priv->clk_csr = 0x01;
266 		else
267 			priv->clk_csr = 0;
268 	}
269 
270 	if (priv->plat->has_xgmac) {
271 		if (clk_rate > 400000000)
272 			priv->clk_csr = 0x5;
273 		else if (clk_rate > 350000000)
274 			priv->clk_csr = 0x4;
275 		else if (clk_rate > 300000000)
276 			priv->clk_csr = 0x3;
277 		else if (clk_rate > 250000000)
278 			priv->clk_csr = 0x2;
279 		else if (clk_rate > 150000000)
280 			priv->clk_csr = 0x1;
281 		else
282 			priv->clk_csr = 0x0;
283 	}
284 }
285 
286 static void print_pkt(unsigned char *buf, int len)
287 {
288 	pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
289 	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
290 }
291 
292 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
293 {
294 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
295 	u32 avail;
296 
297 	if (tx_q->dirty_tx > tx_q->cur_tx)
298 		avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
299 	else
300 		avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
301 
302 	return avail;
303 }
304 
305 /**
306  * stmmac_rx_dirty - Get RX queue dirty
307  * @priv: driver private structure
308  * @queue: RX queue index
309  */
310 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
311 {
312 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
313 	u32 dirty;
314 
315 	if (rx_q->dirty_rx <= rx_q->cur_rx)
316 		dirty = rx_q->cur_rx - rx_q->dirty_rx;
317 	else
318 		dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
319 
320 	return dirty;
321 }
322 
323 /**
324  * stmmac_enable_eee_mode - check and enter in LPI mode
325  * @priv: driver private structure
326  * Description: this function is to verify and enter in LPI mode in case of
327  * EEE.
328  */
329 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
330 {
331 	u32 tx_cnt = priv->plat->tx_queues_to_use;
332 	u32 queue;
333 
334 	/* check if all TX queues have the work finished */
335 	for (queue = 0; queue < tx_cnt; queue++) {
336 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
337 
338 		if (tx_q->dirty_tx != tx_q->cur_tx)
339 			return; /* still unfinished work */
340 	}
341 
342 	/* Check and enter in LPI mode */
343 	if (!priv->tx_path_in_lpi_mode)
344 		stmmac_set_eee_mode(priv, priv->hw,
345 				priv->plat->en_tx_lpi_clockgating);
346 }
347 
348 /**
349  * stmmac_disable_eee_mode - disable and exit from LPI mode
350  * @priv: driver private structure
351  * Description: this function is to exit and disable EEE in case of
352  * LPI state is true. This is called by the xmit.
353  */
354 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
355 {
356 	stmmac_reset_eee_mode(priv, priv->hw);
357 	del_timer_sync(&priv->eee_ctrl_timer);
358 	priv->tx_path_in_lpi_mode = false;
359 }
360 
361 /**
362  * stmmac_eee_ctrl_timer - EEE TX SW timer.
363  * @arg : data hook
364  * Description:
365  *  if there is no data transfer and if we are not in LPI state,
366  *  then MAC Transmitter can be moved to LPI state.
367  */
368 static void stmmac_eee_ctrl_timer(struct timer_list *t)
369 {
370 	struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
371 
372 	stmmac_enable_eee_mode(priv);
373 	mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
374 }
375 
376 /**
377  * stmmac_eee_init - init EEE
378  * @priv: driver private structure
379  * Description:
380  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
381  *  can also manage EEE, this function enable the LPI state and start related
382  *  timer.
383  */
384 bool stmmac_eee_init(struct stmmac_priv *priv)
385 {
386 	int tx_lpi_timer = priv->tx_lpi_timer;
387 
388 	/* Using PCS we cannot dial with the phy registers at this stage
389 	 * so we do not support extra feature like EEE.
390 	 */
391 	if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
392 	    (priv->hw->pcs == STMMAC_PCS_TBI) ||
393 	    (priv->hw->pcs == STMMAC_PCS_RTBI))
394 		return false;
395 
396 	/* Check if MAC core supports the EEE feature. */
397 	if (!priv->dma_cap.eee)
398 		return false;
399 
400 	mutex_lock(&priv->lock);
401 
402 	/* Check if it needs to be deactivated */
403 	if (!priv->eee_active) {
404 		if (priv->eee_enabled) {
405 			netdev_dbg(priv->dev, "disable EEE\n");
406 			del_timer_sync(&priv->eee_ctrl_timer);
407 			stmmac_set_eee_timer(priv, priv->hw, 0, tx_lpi_timer);
408 		}
409 		mutex_unlock(&priv->lock);
410 		return false;
411 	}
412 
413 	if (priv->eee_active && !priv->eee_enabled) {
414 		timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
415 		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
416 		stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
417 				     tx_lpi_timer);
418 	}
419 
420 	mutex_unlock(&priv->lock);
421 	netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
422 	return true;
423 }
424 
425 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
426  * @priv: driver private structure
427  * @p : descriptor pointer
428  * @skb : the socket buffer
429  * Description :
430  * This function will read timestamp from the descriptor & pass it to stack.
431  * and also perform some sanity checks.
432  */
433 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
434 				   struct dma_desc *p, struct sk_buff *skb)
435 {
436 	struct skb_shared_hwtstamps shhwtstamp;
437 	bool found = false;
438 	u64 ns = 0;
439 
440 	if (!priv->hwts_tx_en)
441 		return;
442 
443 	/* exit if skb doesn't support hw tstamp */
444 	if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
445 		return;
446 
447 	/* check tx tstamp status */
448 	if (stmmac_get_tx_timestamp_status(priv, p)) {
449 		stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
450 		found = true;
451 	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
452 		found = true;
453 	}
454 
455 	if (found) {
456 		memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
457 		shhwtstamp.hwtstamp = ns_to_ktime(ns);
458 
459 		netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
460 		/* pass tstamp to stack */
461 		skb_tstamp_tx(skb, &shhwtstamp);
462 	}
463 }
464 
465 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
466  * @priv: driver private structure
467  * @p : descriptor pointer
468  * @np : next descriptor pointer
469  * @skb : the socket buffer
470  * Description :
471  * This function will read received packet's timestamp from the descriptor
472  * and pass it to stack. It also perform some sanity checks.
473  */
474 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
475 				   struct dma_desc *np, struct sk_buff *skb)
476 {
477 	struct skb_shared_hwtstamps *shhwtstamp = NULL;
478 	struct dma_desc *desc = p;
479 	u64 ns = 0;
480 
481 	if (!priv->hwts_rx_en)
482 		return;
483 	/* For GMAC4, the valid timestamp is from CTX next desc. */
484 	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
485 		desc = np;
486 
487 	/* Check if timestamp is available */
488 	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
489 		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
490 		netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
491 		shhwtstamp = skb_hwtstamps(skb);
492 		memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
493 		shhwtstamp->hwtstamp = ns_to_ktime(ns);
494 	} else  {
495 		netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
496 	}
497 }
498 
499 /**
500  *  stmmac_hwtstamp_set - control hardware timestamping.
501  *  @dev: device pointer.
502  *  @ifr: An IOCTL specific structure, that can contain a pointer to
503  *  a proprietary structure used to pass information to the driver.
504  *  Description:
505  *  This function configures the MAC to enable/disable both outgoing(TX)
506  *  and incoming(RX) packets time stamping based on user input.
507  *  Return Value:
508  *  0 on success and an appropriate -ve integer on failure.
509  */
510 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
511 {
512 	struct stmmac_priv *priv = netdev_priv(dev);
513 	struct hwtstamp_config config;
514 	struct timespec64 now;
515 	u64 temp = 0;
516 	u32 ptp_v2 = 0;
517 	u32 tstamp_all = 0;
518 	u32 ptp_over_ipv4_udp = 0;
519 	u32 ptp_over_ipv6_udp = 0;
520 	u32 ptp_over_ethernet = 0;
521 	u32 snap_type_sel = 0;
522 	u32 ts_master_en = 0;
523 	u32 ts_event_en = 0;
524 	u32 sec_inc = 0;
525 	u32 value = 0;
526 	bool xmac;
527 
528 	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
529 
530 	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
531 		netdev_alert(priv->dev, "No support for HW time stamping\n");
532 		priv->hwts_tx_en = 0;
533 		priv->hwts_rx_en = 0;
534 
535 		return -EOPNOTSUPP;
536 	}
537 
538 	if (copy_from_user(&config, ifr->ifr_data,
539 			   sizeof(config)))
540 		return -EFAULT;
541 
542 	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
543 		   __func__, config.flags, config.tx_type, config.rx_filter);
544 
545 	/* reserved for future extensions */
546 	if (config.flags)
547 		return -EINVAL;
548 
549 	if (config.tx_type != HWTSTAMP_TX_OFF &&
550 	    config.tx_type != HWTSTAMP_TX_ON)
551 		return -ERANGE;
552 
553 	if (priv->adv_ts) {
554 		switch (config.rx_filter) {
555 		case HWTSTAMP_FILTER_NONE:
556 			/* time stamp no incoming packet at all */
557 			config.rx_filter = HWTSTAMP_FILTER_NONE;
558 			break;
559 
560 		case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
561 			/* PTP v1, UDP, any kind of event packet */
562 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
563 			/* 'xmac' hardware can support Sync, Pdelay_Req and
564 			 * Pdelay_resp by setting bit14 and bits17/16 to 01
565 			 * This leaves Delay_Req timestamps out.
566 			 * Enable all events *and* general purpose message
567 			 * timestamping
568 			 */
569 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
570 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
571 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
572 			break;
573 
574 		case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
575 			/* PTP v1, UDP, Sync packet */
576 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
577 			/* take time stamp for SYNC messages only */
578 			ts_event_en = PTP_TCR_TSEVNTENA;
579 
580 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
581 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
582 			break;
583 
584 		case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
585 			/* PTP v1, UDP, Delay_req packet */
586 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
587 			/* take time stamp for Delay_Req messages only */
588 			ts_master_en = PTP_TCR_TSMSTRENA;
589 			ts_event_en = PTP_TCR_TSEVNTENA;
590 
591 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
592 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
593 			break;
594 
595 		case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
596 			/* PTP v2, UDP, any kind of event packet */
597 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
598 			ptp_v2 = PTP_TCR_TSVER2ENA;
599 			/* take time stamp for all event messages */
600 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
601 
602 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
603 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
604 			break;
605 
606 		case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
607 			/* PTP v2, UDP, Sync packet */
608 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
609 			ptp_v2 = PTP_TCR_TSVER2ENA;
610 			/* take time stamp for SYNC messages only */
611 			ts_event_en = PTP_TCR_TSEVNTENA;
612 
613 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
614 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
615 			break;
616 
617 		case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
618 			/* PTP v2, UDP, Delay_req packet */
619 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
620 			ptp_v2 = PTP_TCR_TSVER2ENA;
621 			/* take time stamp for Delay_Req messages only */
622 			ts_master_en = PTP_TCR_TSMSTRENA;
623 			ts_event_en = PTP_TCR_TSEVNTENA;
624 
625 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
626 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
627 			break;
628 
629 		case HWTSTAMP_FILTER_PTP_V2_EVENT:
630 			/* PTP v2/802.AS1 any layer, any kind of event packet */
631 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
632 			ptp_v2 = PTP_TCR_TSVER2ENA;
633 			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
634 			ts_event_en = PTP_TCR_TSEVNTENA;
635 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
636 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
637 			ptp_over_ethernet = PTP_TCR_TSIPENA;
638 			break;
639 
640 		case HWTSTAMP_FILTER_PTP_V2_SYNC:
641 			/* PTP v2/802.AS1, any layer, Sync packet */
642 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
643 			ptp_v2 = PTP_TCR_TSVER2ENA;
644 			/* take time stamp for SYNC messages only */
645 			ts_event_en = PTP_TCR_TSEVNTENA;
646 
647 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
648 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
649 			ptp_over_ethernet = PTP_TCR_TSIPENA;
650 			break;
651 
652 		case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
653 			/* PTP v2/802.AS1, any layer, Delay_req packet */
654 			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
655 			ptp_v2 = PTP_TCR_TSVER2ENA;
656 			/* take time stamp for Delay_Req messages only */
657 			ts_master_en = PTP_TCR_TSMSTRENA;
658 			ts_event_en = PTP_TCR_TSEVNTENA;
659 
660 			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
661 			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
662 			ptp_over_ethernet = PTP_TCR_TSIPENA;
663 			break;
664 
665 		case HWTSTAMP_FILTER_NTP_ALL:
666 		case HWTSTAMP_FILTER_ALL:
667 			/* time stamp any incoming packet */
668 			config.rx_filter = HWTSTAMP_FILTER_ALL;
669 			tstamp_all = PTP_TCR_TSENALL;
670 			break;
671 
672 		default:
673 			return -ERANGE;
674 		}
675 	} else {
676 		switch (config.rx_filter) {
677 		case HWTSTAMP_FILTER_NONE:
678 			config.rx_filter = HWTSTAMP_FILTER_NONE;
679 			break;
680 		default:
681 			/* PTP v1, UDP, any kind of event packet */
682 			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
683 			break;
684 		}
685 	}
686 	priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
687 	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
688 
689 	if (!priv->hwts_tx_en && !priv->hwts_rx_en)
690 		stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
691 	else {
692 		value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
693 			 tstamp_all | ptp_v2 | ptp_over_ethernet |
694 			 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
695 			 ts_master_en | snap_type_sel);
696 		stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
697 
698 		/* program Sub Second Increment reg */
699 		stmmac_config_sub_second_increment(priv,
700 				priv->ptpaddr, priv->plat->clk_ptp_rate,
701 				xmac, &sec_inc);
702 		temp = div_u64(1000000000ULL, sec_inc);
703 
704 		/* Store sub second increment and flags for later use */
705 		priv->sub_second_inc = sec_inc;
706 		priv->systime_flags = value;
707 
708 		/* calculate default added value:
709 		 * formula is :
710 		 * addend = (2^32)/freq_div_ratio;
711 		 * where, freq_div_ratio = 1e9ns/sec_inc
712 		 */
713 		temp = (u64)(temp << 32);
714 		priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
715 		stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
716 
717 		/* initialize system time */
718 		ktime_get_real_ts64(&now);
719 
720 		/* lower 32 bits of tv_sec are safe until y2106 */
721 		stmmac_init_systime(priv, priv->ptpaddr,
722 				(u32)now.tv_sec, now.tv_nsec);
723 	}
724 
725 	memcpy(&priv->tstamp_config, &config, sizeof(config));
726 
727 	return copy_to_user(ifr->ifr_data, &config,
728 			    sizeof(config)) ? -EFAULT : 0;
729 }
730 
731 /**
732  *  stmmac_hwtstamp_get - read hardware timestamping.
733  *  @dev: device pointer.
734  *  @ifr: An IOCTL specific structure, that can contain a pointer to
735  *  a proprietary structure used to pass information to the driver.
736  *  Description:
737  *  This function obtain the current hardware timestamping settings
738     as requested.
739  */
740 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
741 {
742 	struct stmmac_priv *priv = netdev_priv(dev);
743 	struct hwtstamp_config *config = &priv->tstamp_config;
744 
745 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
746 		return -EOPNOTSUPP;
747 
748 	return copy_to_user(ifr->ifr_data, config,
749 			    sizeof(*config)) ? -EFAULT : 0;
750 }
751 
752 /**
753  * stmmac_init_ptp - init PTP
754  * @priv: driver private structure
755  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
756  * This is done by looking at the HW cap. register.
757  * This function also registers the ptp driver.
758  */
759 static int stmmac_init_ptp(struct stmmac_priv *priv)
760 {
761 	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
762 
763 	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
764 		return -EOPNOTSUPP;
765 
766 	priv->adv_ts = 0;
767 	/* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
768 	if (xmac && priv->dma_cap.atime_stamp)
769 		priv->adv_ts = 1;
770 	/* Dwmac 3.x core with extend_desc can support adv_ts */
771 	else if (priv->extend_desc && priv->dma_cap.atime_stamp)
772 		priv->adv_ts = 1;
773 
774 	if (priv->dma_cap.time_stamp)
775 		netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
776 
777 	if (priv->adv_ts)
778 		netdev_info(priv->dev,
779 			    "IEEE 1588-2008 Advanced Timestamp supported\n");
780 
781 	priv->hwts_tx_en = 0;
782 	priv->hwts_rx_en = 0;
783 
784 	stmmac_ptp_register(priv);
785 
786 	return 0;
787 }
788 
789 static void stmmac_release_ptp(struct stmmac_priv *priv)
790 {
791 	if (priv->plat->clk_ptp_ref)
792 		clk_disable_unprepare(priv->plat->clk_ptp_ref);
793 	stmmac_ptp_unregister(priv);
794 }
795 
796 /**
797  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
798  *  @priv: driver private structure
799  *  Description: It is used for configuring the flow control in all queues
800  */
801 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
802 {
803 	u32 tx_cnt = priv->plat->tx_queues_to_use;
804 
805 	stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
806 			priv->pause, tx_cnt);
807 }
808 
809 static void stmmac_validate(struct phylink_config *config,
810 			    unsigned long *supported,
811 			    struct phylink_link_state *state)
812 {
813 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
814 	__ETHTOOL_DECLARE_LINK_MODE_MASK(mac_supported) = { 0, };
815 	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
816 	int tx_cnt = priv->plat->tx_queues_to_use;
817 	int max_speed = priv->plat->max_speed;
818 
819 	phylink_set(mac_supported, 10baseT_Half);
820 	phylink_set(mac_supported, 10baseT_Full);
821 	phylink_set(mac_supported, 100baseT_Half);
822 	phylink_set(mac_supported, 100baseT_Full);
823 	phylink_set(mac_supported, 1000baseT_Half);
824 	phylink_set(mac_supported, 1000baseT_Full);
825 	phylink_set(mac_supported, 1000baseKX_Full);
826 
827 	phylink_set(mac_supported, Autoneg);
828 	phylink_set(mac_supported, Pause);
829 	phylink_set(mac_supported, Asym_Pause);
830 	phylink_set_port_modes(mac_supported);
831 
832 	/* Cut down 1G if asked to */
833 	if ((max_speed > 0) && (max_speed < 1000)) {
834 		phylink_set(mask, 1000baseT_Full);
835 		phylink_set(mask, 1000baseX_Full);
836 	} else if (priv->plat->has_xgmac) {
837 		if (!max_speed || (max_speed >= 2500)) {
838 			phylink_set(mac_supported, 2500baseT_Full);
839 			phylink_set(mac_supported, 2500baseX_Full);
840 		}
841 		if (!max_speed || (max_speed >= 5000)) {
842 			phylink_set(mac_supported, 5000baseT_Full);
843 		}
844 		if (!max_speed || (max_speed >= 10000)) {
845 			phylink_set(mac_supported, 10000baseSR_Full);
846 			phylink_set(mac_supported, 10000baseLR_Full);
847 			phylink_set(mac_supported, 10000baseER_Full);
848 			phylink_set(mac_supported, 10000baseLRM_Full);
849 			phylink_set(mac_supported, 10000baseT_Full);
850 			phylink_set(mac_supported, 10000baseKX4_Full);
851 			phylink_set(mac_supported, 10000baseKR_Full);
852 		}
853 	}
854 
855 	/* Half-Duplex can only work with single queue */
856 	if (tx_cnt > 1) {
857 		phylink_set(mask, 10baseT_Half);
858 		phylink_set(mask, 100baseT_Half);
859 		phylink_set(mask, 1000baseT_Half);
860 	}
861 
862 	bitmap_and(supported, supported, mac_supported,
863 		   __ETHTOOL_LINK_MODE_MASK_NBITS);
864 	bitmap_andnot(supported, supported, mask,
865 		      __ETHTOOL_LINK_MODE_MASK_NBITS);
866 	bitmap_and(state->advertising, state->advertising, mac_supported,
867 		   __ETHTOOL_LINK_MODE_MASK_NBITS);
868 	bitmap_andnot(state->advertising, state->advertising, mask,
869 		      __ETHTOOL_LINK_MODE_MASK_NBITS);
870 }
871 
872 static void stmmac_mac_pcs_get_state(struct phylink_config *config,
873 				     struct phylink_link_state *state)
874 {
875 	state->link = 0;
876 }
877 
878 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
879 			      const struct phylink_link_state *state)
880 {
881 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
882 	u32 ctrl;
883 
884 	ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
885 	ctrl &= ~priv->hw->link.speed_mask;
886 
887 	if (state->interface == PHY_INTERFACE_MODE_USXGMII) {
888 		switch (state->speed) {
889 		case SPEED_10000:
890 			ctrl |= priv->hw->link.xgmii.speed10000;
891 			break;
892 		case SPEED_5000:
893 			ctrl |= priv->hw->link.xgmii.speed5000;
894 			break;
895 		case SPEED_2500:
896 			ctrl |= priv->hw->link.xgmii.speed2500;
897 			break;
898 		default:
899 			return;
900 		}
901 	} else {
902 		switch (state->speed) {
903 		case SPEED_2500:
904 			ctrl |= priv->hw->link.speed2500;
905 			break;
906 		case SPEED_1000:
907 			ctrl |= priv->hw->link.speed1000;
908 			break;
909 		case SPEED_100:
910 			ctrl |= priv->hw->link.speed100;
911 			break;
912 		case SPEED_10:
913 			ctrl |= priv->hw->link.speed10;
914 			break;
915 		default:
916 			return;
917 		}
918 	}
919 
920 	priv->speed = state->speed;
921 
922 	if (priv->plat->fix_mac_speed)
923 		priv->plat->fix_mac_speed(priv->plat->bsp_priv, state->speed);
924 
925 	if (!state->duplex)
926 		ctrl &= ~priv->hw->link.duplex;
927 	else
928 		ctrl |= priv->hw->link.duplex;
929 
930 	/* Flow Control operation */
931 	if (state->pause)
932 		stmmac_mac_flow_ctrl(priv, state->duplex);
933 
934 	writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
935 }
936 
937 static void stmmac_mac_an_restart(struct phylink_config *config)
938 {
939 	/* Not Supported */
940 }
941 
942 static void stmmac_mac_link_down(struct phylink_config *config,
943 				 unsigned int mode, phy_interface_t interface)
944 {
945 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
946 
947 	stmmac_mac_set(priv, priv->ioaddr, false);
948 	priv->eee_active = false;
949 	stmmac_eee_init(priv);
950 	stmmac_set_eee_pls(priv, priv->hw, false);
951 }
952 
953 static void stmmac_mac_link_up(struct phylink_config *config,
954 			       unsigned int mode, phy_interface_t interface,
955 			       struct phy_device *phy)
956 {
957 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
958 
959 	stmmac_mac_set(priv, priv->ioaddr, true);
960 	if (phy && priv->dma_cap.eee) {
961 		priv->eee_active = phy_init_eee(phy, 1) >= 0;
962 		priv->eee_enabled = stmmac_eee_init(priv);
963 		stmmac_set_eee_pls(priv, priv->hw, true);
964 	}
965 }
966 
967 static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
968 	.validate = stmmac_validate,
969 	.mac_pcs_get_state = stmmac_mac_pcs_get_state,
970 	.mac_config = stmmac_mac_config,
971 	.mac_an_restart = stmmac_mac_an_restart,
972 	.mac_link_down = stmmac_mac_link_down,
973 	.mac_link_up = stmmac_mac_link_up,
974 };
975 
976 /**
977  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
978  * @priv: driver private structure
979  * Description: this is to verify if the HW supports the PCS.
980  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
981  * configured for the TBI, RTBI, or SGMII PHY interface.
982  */
983 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
984 {
985 	int interface = priv->plat->interface;
986 
987 	if (priv->dma_cap.pcs) {
988 		if ((interface == PHY_INTERFACE_MODE_RGMII) ||
989 		    (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
990 		    (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
991 		    (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
992 			netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
993 			priv->hw->pcs = STMMAC_PCS_RGMII;
994 		} else if (interface == PHY_INTERFACE_MODE_SGMII) {
995 			netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
996 			priv->hw->pcs = STMMAC_PCS_SGMII;
997 		}
998 	}
999 }
1000 
1001 /**
1002  * stmmac_init_phy - PHY initialization
1003  * @dev: net device structure
1004  * Description: it initializes the driver's PHY state, and attaches the PHY
1005  * to the mac driver.
1006  *  Return value:
1007  *  0 on success
1008  */
1009 static int stmmac_init_phy(struct net_device *dev)
1010 {
1011 	struct stmmac_priv *priv = netdev_priv(dev);
1012 	struct device_node *node;
1013 	int ret;
1014 
1015 	node = priv->plat->phylink_node;
1016 
1017 	if (node)
1018 		ret = phylink_of_phy_connect(priv->phylink, node, 0);
1019 
1020 	/* Some DT bindings do not set-up the PHY handle. Let's try to
1021 	 * manually parse it
1022 	 */
1023 	if (!node || ret) {
1024 		int addr = priv->plat->phy_addr;
1025 		struct phy_device *phydev;
1026 
1027 		phydev = mdiobus_get_phy(priv->mii, addr);
1028 		if (!phydev) {
1029 			netdev_err(priv->dev, "no phy at addr %d\n", addr);
1030 			return -ENODEV;
1031 		}
1032 
1033 		ret = phylink_connect_phy(priv->phylink, phydev);
1034 	}
1035 
1036 	return ret;
1037 }
1038 
1039 static int stmmac_phy_setup(struct stmmac_priv *priv)
1040 {
1041 	struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node);
1042 	int mode = priv->plat->phy_interface;
1043 	struct phylink *phylink;
1044 
1045 	priv->phylink_config.dev = &priv->dev->dev;
1046 	priv->phylink_config.type = PHYLINK_NETDEV;
1047 
1048 	phylink = phylink_create(&priv->phylink_config, fwnode,
1049 				 mode, &stmmac_phylink_mac_ops);
1050 	if (IS_ERR(phylink))
1051 		return PTR_ERR(phylink);
1052 
1053 	priv->phylink = phylink;
1054 	return 0;
1055 }
1056 
1057 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
1058 {
1059 	u32 rx_cnt = priv->plat->rx_queues_to_use;
1060 	void *head_rx;
1061 	u32 queue;
1062 
1063 	/* Display RX rings */
1064 	for (queue = 0; queue < rx_cnt; queue++) {
1065 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1066 
1067 		pr_info("\tRX Queue %u rings\n", queue);
1068 
1069 		if (priv->extend_desc)
1070 			head_rx = (void *)rx_q->dma_erx;
1071 		else
1072 			head_rx = (void *)rx_q->dma_rx;
1073 
1074 		/* Display RX ring */
1075 		stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true);
1076 	}
1077 }
1078 
1079 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1080 {
1081 	u32 tx_cnt = priv->plat->tx_queues_to_use;
1082 	void *head_tx;
1083 	u32 queue;
1084 
1085 	/* Display TX rings */
1086 	for (queue = 0; queue < tx_cnt; queue++) {
1087 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1088 
1089 		pr_info("\tTX Queue %d rings\n", queue);
1090 
1091 		if (priv->extend_desc)
1092 			head_tx = (void *)tx_q->dma_etx;
1093 		else
1094 			head_tx = (void *)tx_q->dma_tx;
1095 
1096 		stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false);
1097 	}
1098 }
1099 
1100 static void stmmac_display_rings(struct stmmac_priv *priv)
1101 {
1102 	/* Display RX ring */
1103 	stmmac_display_rx_rings(priv);
1104 
1105 	/* Display TX ring */
1106 	stmmac_display_tx_rings(priv);
1107 }
1108 
1109 static int stmmac_set_bfsize(int mtu, int bufsize)
1110 {
1111 	int ret = bufsize;
1112 
1113 	if (mtu >= BUF_SIZE_8KiB)
1114 		ret = BUF_SIZE_16KiB;
1115 	else if (mtu >= BUF_SIZE_4KiB)
1116 		ret = BUF_SIZE_8KiB;
1117 	else if (mtu >= BUF_SIZE_2KiB)
1118 		ret = BUF_SIZE_4KiB;
1119 	else if (mtu > DEFAULT_BUFSIZE)
1120 		ret = BUF_SIZE_2KiB;
1121 	else
1122 		ret = DEFAULT_BUFSIZE;
1123 
1124 	return ret;
1125 }
1126 
1127 /**
1128  * stmmac_clear_rx_descriptors - clear RX descriptors
1129  * @priv: driver private structure
1130  * @queue: RX queue index
1131  * Description: this function is called to clear the RX descriptors
1132  * in case of both basic and extended descriptors are used.
1133  */
1134 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1135 {
1136 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1137 	int i;
1138 
1139 	/* Clear the RX descriptors */
1140 	for (i = 0; i < DMA_RX_SIZE; i++)
1141 		if (priv->extend_desc)
1142 			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1143 					priv->use_riwt, priv->mode,
1144 					(i == DMA_RX_SIZE - 1),
1145 					priv->dma_buf_sz);
1146 		else
1147 			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1148 					priv->use_riwt, priv->mode,
1149 					(i == DMA_RX_SIZE - 1),
1150 					priv->dma_buf_sz);
1151 }
1152 
1153 /**
1154  * stmmac_clear_tx_descriptors - clear tx descriptors
1155  * @priv: driver private structure
1156  * @queue: TX queue index.
1157  * Description: this function is called to clear the TX descriptors
1158  * in case of both basic and extended descriptors are used.
1159  */
1160 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1161 {
1162 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1163 	int i;
1164 
1165 	/* Clear the TX descriptors */
1166 	for (i = 0; i < DMA_TX_SIZE; i++)
1167 		if (priv->extend_desc)
1168 			stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
1169 					priv->mode, (i == DMA_TX_SIZE - 1));
1170 		else
1171 			stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
1172 					priv->mode, (i == DMA_TX_SIZE - 1));
1173 }
1174 
1175 /**
1176  * stmmac_clear_descriptors - clear descriptors
1177  * @priv: driver private structure
1178  * Description: this function is called to clear the TX and RX descriptors
1179  * in case of both basic and extended descriptors are used.
1180  */
1181 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1182 {
1183 	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1184 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1185 	u32 queue;
1186 
1187 	/* Clear the RX descriptors */
1188 	for (queue = 0; queue < rx_queue_cnt; queue++)
1189 		stmmac_clear_rx_descriptors(priv, queue);
1190 
1191 	/* Clear the TX descriptors */
1192 	for (queue = 0; queue < tx_queue_cnt; queue++)
1193 		stmmac_clear_tx_descriptors(priv, queue);
1194 }
1195 
1196 /**
1197  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1198  * @priv: driver private structure
1199  * @p: descriptor pointer
1200  * @i: descriptor index
1201  * @flags: gfp flag
1202  * @queue: RX queue index
1203  * Description: this function is called to allocate a receive buffer, perform
1204  * the DMA mapping and init the descriptor.
1205  */
1206 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1207 				  int i, gfp_t flags, u32 queue)
1208 {
1209 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1210 	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1211 
1212 	buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
1213 	if (!buf->page)
1214 		return -ENOMEM;
1215 
1216 	if (priv->sph) {
1217 		buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
1218 		if (!buf->sec_page)
1219 			return -ENOMEM;
1220 
1221 		buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
1222 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
1223 	} else {
1224 		buf->sec_page = NULL;
1225 	}
1226 
1227 	buf->addr = page_pool_get_dma_addr(buf->page);
1228 	stmmac_set_desc_addr(priv, p, buf->addr);
1229 	if (priv->dma_buf_sz == BUF_SIZE_16KiB)
1230 		stmmac_init_desc3(priv, p);
1231 
1232 	return 0;
1233 }
1234 
1235 /**
1236  * stmmac_free_rx_buffer - free RX dma buffers
1237  * @priv: private structure
1238  * @queue: RX queue index
1239  * @i: buffer index.
1240  */
1241 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1242 {
1243 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1244 	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1245 
1246 	if (buf->page)
1247 		page_pool_put_page(rx_q->page_pool, buf->page, false);
1248 	buf->page = NULL;
1249 
1250 	if (buf->sec_page)
1251 		page_pool_put_page(rx_q->page_pool, buf->sec_page, false);
1252 	buf->sec_page = NULL;
1253 }
1254 
1255 /**
1256  * stmmac_free_tx_buffer - free RX dma buffers
1257  * @priv: private structure
1258  * @queue: RX queue index
1259  * @i: buffer index.
1260  */
1261 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1262 {
1263 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1264 
1265 	if (tx_q->tx_skbuff_dma[i].buf) {
1266 		if (tx_q->tx_skbuff_dma[i].map_as_page)
1267 			dma_unmap_page(priv->device,
1268 				       tx_q->tx_skbuff_dma[i].buf,
1269 				       tx_q->tx_skbuff_dma[i].len,
1270 				       DMA_TO_DEVICE);
1271 		else
1272 			dma_unmap_single(priv->device,
1273 					 tx_q->tx_skbuff_dma[i].buf,
1274 					 tx_q->tx_skbuff_dma[i].len,
1275 					 DMA_TO_DEVICE);
1276 	}
1277 
1278 	if (tx_q->tx_skbuff[i]) {
1279 		dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1280 		tx_q->tx_skbuff[i] = NULL;
1281 		tx_q->tx_skbuff_dma[i].buf = 0;
1282 		tx_q->tx_skbuff_dma[i].map_as_page = false;
1283 	}
1284 }
1285 
1286 /**
1287  * init_dma_rx_desc_rings - init the RX descriptor rings
1288  * @dev: net device structure
1289  * @flags: gfp flag.
1290  * Description: this function initializes the DMA RX descriptors
1291  * and allocates the socket buffers. It supports the chained and ring
1292  * modes.
1293  */
1294 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1295 {
1296 	struct stmmac_priv *priv = netdev_priv(dev);
1297 	u32 rx_count = priv->plat->rx_queues_to_use;
1298 	int ret = -ENOMEM;
1299 	int queue;
1300 	int i;
1301 
1302 	/* RX INITIALIZATION */
1303 	netif_dbg(priv, probe, priv->dev,
1304 		  "SKB addresses:\nskb\t\tskb data\tdma data\n");
1305 
1306 	for (queue = 0; queue < rx_count; queue++) {
1307 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1308 
1309 		netif_dbg(priv, probe, priv->dev,
1310 			  "(%s) dma_rx_phy=0x%08x\n", __func__,
1311 			  (u32)rx_q->dma_rx_phy);
1312 
1313 		stmmac_clear_rx_descriptors(priv, queue);
1314 
1315 		for (i = 0; i < DMA_RX_SIZE; i++) {
1316 			struct dma_desc *p;
1317 
1318 			if (priv->extend_desc)
1319 				p = &((rx_q->dma_erx + i)->basic);
1320 			else
1321 				p = rx_q->dma_rx + i;
1322 
1323 			ret = stmmac_init_rx_buffers(priv, p, i, flags,
1324 						     queue);
1325 			if (ret)
1326 				goto err_init_rx_buffers;
1327 		}
1328 
1329 		rx_q->cur_rx = 0;
1330 		rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1331 
1332 		/* Setup the chained descriptor addresses */
1333 		if (priv->mode == STMMAC_CHAIN_MODE) {
1334 			if (priv->extend_desc)
1335 				stmmac_mode_init(priv, rx_q->dma_erx,
1336 						rx_q->dma_rx_phy, DMA_RX_SIZE, 1);
1337 			else
1338 				stmmac_mode_init(priv, rx_q->dma_rx,
1339 						rx_q->dma_rx_phy, DMA_RX_SIZE, 0);
1340 		}
1341 	}
1342 
1343 	return 0;
1344 
1345 err_init_rx_buffers:
1346 	while (queue >= 0) {
1347 		while (--i >= 0)
1348 			stmmac_free_rx_buffer(priv, queue, i);
1349 
1350 		if (queue == 0)
1351 			break;
1352 
1353 		i = DMA_RX_SIZE;
1354 		queue--;
1355 	}
1356 
1357 	return ret;
1358 }
1359 
1360 /**
1361  * init_dma_tx_desc_rings - init the TX descriptor rings
1362  * @dev: net device structure.
1363  * Description: this function initializes the DMA TX descriptors
1364  * and allocates the socket buffers. It supports the chained and ring
1365  * modes.
1366  */
1367 static int init_dma_tx_desc_rings(struct net_device *dev)
1368 {
1369 	struct stmmac_priv *priv = netdev_priv(dev);
1370 	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1371 	u32 queue;
1372 	int i;
1373 
1374 	for (queue = 0; queue < tx_queue_cnt; queue++) {
1375 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1376 
1377 		netif_dbg(priv, probe, priv->dev,
1378 			  "(%s) dma_tx_phy=0x%08x\n", __func__,
1379 			 (u32)tx_q->dma_tx_phy);
1380 
1381 		/* Setup the chained descriptor addresses */
1382 		if (priv->mode == STMMAC_CHAIN_MODE) {
1383 			if (priv->extend_desc)
1384 				stmmac_mode_init(priv, tx_q->dma_etx,
1385 						tx_q->dma_tx_phy, DMA_TX_SIZE, 1);
1386 			else
1387 				stmmac_mode_init(priv, tx_q->dma_tx,
1388 						tx_q->dma_tx_phy, DMA_TX_SIZE, 0);
1389 		}
1390 
1391 		for (i = 0; i < DMA_TX_SIZE; i++) {
1392 			struct dma_desc *p;
1393 			if (priv->extend_desc)
1394 				p = &((tx_q->dma_etx + i)->basic);
1395 			else
1396 				p = tx_q->dma_tx + i;
1397 
1398 			stmmac_clear_desc(priv, p);
1399 
1400 			tx_q->tx_skbuff_dma[i].buf = 0;
1401 			tx_q->tx_skbuff_dma[i].map_as_page = false;
1402 			tx_q->tx_skbuff_dma[i].len = 0;
1403 			tx_q->tx_skbuff_dma[i].last_segment = false;
1404 			tx_q->tx_skbuff[i] = NULL;
1405 		}
1406 
1407 		tx_q->dirty_tx = 0;
1408 		tx_q->cur_tx = 0;
1409 		tx_q->mss = 0;
1410 
1411 		netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1412 	}
1413 
1414 	return 0;
1415 }
1416 
1417 /**
1418  * init_dma_desc_rings - init the RX/TX descriptor rings
1419  * @dev: net device structure
1420  * @flags: gfp flag.
1421  * Description: this function initializes the DMA RX/TX descriptors
1422  * and allocates the socket buffers. It supports the chained and ring
1423  * modes.
1424  */
1425 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1426 {
1427 	struct stmmac_priv *priv = netdev_priv(dev);
1428 	int ret;
1429 
1430 	ret = init_dma_rx_desc_rings(dev, flags);
1431 	if (ret)
1432 		return ret;
1433 
1434 	ret = init_dma_tx_desc_rings(dev);
1435 
1436 	stmmac_clear_descriptors(priv);
1437 
1438 	if (netif_msg_hw(priv))
1439 		stmmac_display_rings(priv);
1440 
1441 	return ret;
1442 }
1443 
1444 /**
1445  * dma_free_rx_skbufs - free RX dma buffers
1446  * @priv: private structure
1447  * @queue: RX queue index
1448  */
1449 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1450 {
1451 	int i;
1452 
1453 	for (i = 0; i < DMA_RX_SIZE; i++)
1454 		stmmac_free_rx_buffer(priv, queue, i);
1455 }
1456 
1457 /**
1458  * dma_free_tx_skbufs - free TX dma buffers
1459  * @priv: private structure
1460  * @queue: TX queue index
1461  */
1462 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1463 {
1464 	int i;
1465 
1466 	for (i = 0; i < DMA_TX_SIZE; i++)
1467 		stmmac_free_tx_buffer(priv, queue, i);
1468 }
1469 
1470 /**
1471  * free_dma_rx_desc_resources - free RX dma desc resources
1472  * @priv: private structure
1473  */
1474 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1475 {
1476 	u32 rx_count = priv->plat->rx_queues_to_use;
1477 	u32 queue;
1478 
1479 	/* Free RX queue resources */
1480 	for (queue = 0; queue < rx_count; queue++) {
1481 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1482 
1483 		/* Release the DMA RX socket buffers */
1484 		dma_free_rx_skbufs(priv, queue);
1485 
1486 		/* Free DMA regions of consistent memory previously allocated */
1487 		if (!priv->extend_desc)
1488 			dma_free_coherent(priv->device,
1489 					  DMA_RX_SIZE * sizeof(struct dma_desc),
1490 					  rx_q->dma_rx, rx_q->dma_rx_phy);
1491 		else
1492 			dma_free_coherent(priv->device, DMA_RX_SIZE *
1493 					  sizeof(struct dma_extended_desc),
1494 					  rx_q->dma_erx, rx_q->dma_rx_phy);
1495 
1496 		kfree(rx_q->buf_pool);
1497 		if (rx_q->page_pool)
1498 			page_pool_destroy(rx_q->page_pool);
1499 	}
1500 }
1501 
1502 /**
1503  * free_dma_tx_desc_resources - free TX dma desc resources
1504  * @priv: private structure
1505  */
1506 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1507 {
1508 	u32 tx_count = priv->plat->tx_queues_to_use;
1509 	u32 queue;
1510 
1511 	/* Free TX queue resources */
1512 	for (queue = 0; queue < tx_count; queue++) {
1513 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1514 
1515 		/* Release the DMA TX socket buffers */
1516 		dma_free_tx_skbufs(priv, queue);
1517 
1518 		/* Free DMA regions of consistent memory previously allocated */
1519 		if (!priv->extend_desc)
1520 			dma_free_coherent(priv->device,
1521 					  DMA_TX_SIZE * sizeof(struct dma_desc),
1522 					  tx_q->dma_tx, tx_q->dma_tx_phy);
1523 		else
1524 			dma_free_coherent(priv->device, DMA_TX_SIZE *
1525 					  sizeof(struct dma_extended_desc),
1526 					  tx_q->dma_etx, tx_q->dma_tx_phy);
1527 
1528 		kfree(tx_q->tx_skbuff_dma);
1529 		kfree(tx_q->tx_skbuff);
1530 	}
1531 }
1532 
1533 /**
1534  * alloc_dma_rx_desc_resources - alloc RX resources.
1535  * @priv: private structure
1536  * Description: according to which descriptor can be used (extend or basic)
1537  * this function allocates the resources for TX and RX paths. In case of
1538  * reception, for example, it pre-allocated the RX socket buffer in order to
1539  * allow zero-copy mechanism.
1540  */
1541 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1542 {
1543 	u32 rx_count = priv->plat->rx_queues_to_use;
1544 	int ret = -ENOMEM;
1545 	u32 queue;
1546 
1547 	/* RX queues buffers and DMA */
1548 	for (queue = 0; queue < rx_count; queue++) {
1549 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1550 		struct page_pool_params pp_params = { 0 };
1551 		unsigned int num_pages;
1552 
1553 		rx_q->queue_index = queue;
1554 		rx_q->priv_data = priv;
1555 
1556 		pp_params.flags = PP_FLAG_DMA_MAP;
1557 		pp_params.pool_size = DMA_RX_SIZE;
1558 		num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE);
1559 		pp_params.order = ilog2(num_pages);
1560 		pp_params.nid = dev_to_node(priv->device);
1561 		pp_params.dev = priv->device;
1562 		pp_params.dma_dir = DMA_FROM_DEVICE;
1563 
1564 		rx_q->page_pool = page_pool_create(&pp_params);
1565 		if (IS_ERR(rx_q->page_pool)) {
1566 			ret = PTR_ERR(rx_q->page_pool);
1567 			rx_q->page_pool = NULL;
1568 			goto err_dma;
1569 		}
1570 
1571 		rx_q->buf_pool = kcalloc(DMA_RX_SIZE, sizeof(*rx_q->buf_pool),
1572 					 GFP_KERNEL);
1573 		if (!rx_q->buf_pool)
1574 			goto err_dma;
1575 
1576 		if (priv->extend_desc) {
1577 			rx_q->dma_erx = dma_alloc_coherent(priv->device,
1578 							   DMA_RX_SIZE * sizeof(struct dma_extended_desc),
1579 							   &rx_q->dma_rx_phy,
1580 							   GFP_KERNEL);
1581 			if (!rx_q->dma_erx)
1582 				goto err_dma;
1583 
1584 		} else {
1585 			rx_q->dma_rx = dma_alloc_coherent(priv->device,
1586 							  DMA_RX_SIZE * sizeof(struct dma_desc),
1587 							  &rx_q->dma_rx_phy,
1588 							  GFP_KERNEL);
1589 			if (!rx_q->dma_rx)
1590 				goto err_dma;
1591 		}
1592 	}
1593 
1594 	return 0;
1595 
1596 err_dma:
1597 	free_dma_rx_desc_resources(priv);
1598 
1599 	return ret;
1600 }
1601 
1602 /**
1603  * alloc_dma_tx_desc_resources - alloc TX resources.
1604  * @priv: private structure
1605  * Description: according to which descriptor can be used (extend or basic)
1606  * this function allocates the resources for TX and RX paths. In case of
1607  * reception, for example, it pre-allocated the RX socket buffer in order to
1608  * allow zero-copy mechanism.
1609  */
1610 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1611 {
1612 	u32 tx_count = priv->plat->tx_queues_to_use;
1613 	int ret = -ENOMEM;
1614 	u32 queue;
1615 
1616 	/* TX queues buffers and DMA */
1617 	for (queue = 0; queue < tx_count; queue++) {
1618 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1619 
1620 		tx_q->queue_index = queue;
1621 		tx_q->priv_data = priv;
1622 
1623 		tx_q->tx_skbuff_dma = kcalloc(DMA_TX_SIZE,
1624 					      sizeof(*tx_q->tx_skbuff_dma),
1625 					      GFP_KERNEL);
1626 		if (!tx_q->tx_skbuff_dma)
1627 			goto err_dma;
1628 
1629 		tx_q->tx_skbuff = kcalloc(DMA_TX_SIZE,
1630 					  sizeof(struct sk_buff *),
1631 					  GFP_KERNEL);
1632 		if (!tx_q->tx_skbuff)
1633 			goto err_dma;
1634 
1635 		if (priv->extend_desc) {
1636 			tx_q->dma_etx = dma_alloc_coherent(priv->device,
1637 							   DMA_TX_SIZE * sizeof(struct dma_extended_desc),
1638 							   &tx_q->dma_tx_phy,
1639 							   GFP_KERNEL);
1640 			if (!tx_q->dma_etx)
1641 				goto err_dma;
1642 		} else {
1643 			tx_q->dma_tx = dma_alloc_coherent(priv->device,
1644 							  DMA_TX_SIZE * sizeof(struct dma_desc),
1645 							  &tx_q->dma_tx_phy,
1646 							  GFP_KERNEL);
1647 			if (!tx_q->dma_tx)
1648 				goto err_dma;
1649 		}
1650 	}
1651 
1652 	return 0;
1653 
1654 err_dma:
1655 	free_dma_tx_desc_resources(priv);
1656 
1657 	return ret;
1658 }
1659 
1660 /**
1661  * alloc_dma_desc_resources - alloc TX/RX resources.
1662  * @priv: private structure
1663  * Description: according to which descriptor can be used (extend or basic)
1664  * this function allocates the resources for TX and RX paths. In case of
1665  * reception, for example, it pre-allocated the RX socket buffer in order to
1666  * allow zero-copy mechanism.
1667  */
1668 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1669 {
1670 	/* RX Allocation */
1671 	int ret = alloc_dma_rx_desc_resources(priv);
1672 
1673 	if (ret)
1674 		return ret;
1675 
1676 	ret = alloc_dma_tx_desc_resources(priv);
1677 
1678 	return ret;
1679 }
1680 
1681 /**
1682  * free_dma_desc_resources - free dma desc resources
1683  * @priv: private structure
1684  */
1685 static void free_dma_desc_resources(struct stmmac_priv *priv)
1686 {
1687 	/* Release the DMA RX socket buffers */
1688 	free_dma_rx_desc_resources(priv);
1689 
1690 	/* Release the DMA TX socket buffers */
1691 	free_dma_tx_desc_resources(priv);
1692 }
1693 
1694 /**
1695  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1696  *  @priv: driver private structure
1697  *  Description: It is used for enabling the rx queues in the MAC
1698  */
1699 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1700 {
1701 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
1702 	int queue;
1703 	u8 mode;
1704 
1705 	for (queue = 0; queue < rx_queues_count; queue++) {
1706 		mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1707 		stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
1708 	}
1709 }
1710 
1711 /**
1712  * stmmac_start_rx_dma - start RX DMA channel
1713  * @priv: driver private structure
1714  * @chan: RX channel index
1715  * Description:
1716  * This starts a RX DMA channel
1717  */
1718 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1719 {
1720 	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1721 	stmmac_start_rx(priv, priv->ioaddr, chan);
1722 }
1723 
1724 /**
1725  * stmmac_start_tx_dma - start TX DMA channel
1726  * @priv: driver private structure
1727  * @chan: TX channel index
1728  * Description:
1729  * This starts a TX DMA channel
1730  */
1731 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1732 {
1733 	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1734 	stmmac_start_tx(priv, priv->ioaddr, chan);
1735 }
1736 
1737 /**
1738  * stmmac_stop_rx_dma - stop RX DMA channel
1739  * @priv: driver private structure
1740  * @chan: RX channel index
1741  * Description:
1742  * This stops a RX DMA channel
1743  */
1744 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1745 {
1746 	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1747 	stmmac_stop_rx(priv, priv->ioaddr, chan);
1748 }
1749 
1750 /**
1751  * stmmac_stop_tx_dma - stop TX DMA channel
1752  * @priv: driver private structure
1753  * @chan: TX channel index
1754  * Description:
1755  * This stops a TX DMA channel
1756  */
1757 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1758 {
1759 	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1760 	stmmac_stop_tx(priv, priv->ioaddr, chan);
1761 }
1762 
1763 /**
1764  * stmmac_start_all_dma - start all RX and TX DMA channels
1765  * @priv: driver private structure
1766  * Description:
1767  * This starts all the RX and TX DMA channels
1768  */
1769 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1770 {
1771 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
1772 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
1773 	u32 chan = 0;
1774 
1775 	for (chan = 0; chan < rx_channels_count; chan++)
1776 		stmmac_start_rx_dma(priv, chan);
1777 
1778 	for (chan = 0; chan < tx_channels_count; chan++)
1779 		stmmac_start_tx_dma(priv, chan);
1780 }
1781 
1782 /**
1783  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1784  * @priv: driver private structure
1785  * Description:
1786  * This stops the RX and TX DMA channels
1787  */
1788 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1789 {
1790 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
1791 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
1792 	u32 chan = 0;
1793 
1794 	for (chan = 0; chan < rx_channels_count; chan++)
1795 		stmmac_stop_rx_dma(priv, chan);
1796 
1797 	for (chan = 0; chan < tx_channels_count; chan++)
1798 		stmmac_stop_tx_dma(priv, chan);
1799 }
1800 
1801 /**
1802  *  stmmac_dma_operation_mode - HW DMA operation mode
1803  *  @priv: driver private structure
1804  *  Description: it is used for configuring the DMA operation mode register in
1805  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1806  */
1807 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1808 {
1809 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
1810 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
1811 	int rxfifosz = priv->plat->rx_fifo_size;
1812 	int txfifosz = priv->plat->tx_fifo_size;
1813 	u32 txmode = 0;
1814 	u32 rxmode = 0;
1815 	u32 chan = 0;
1816 	u8 qmode = 0;
1817 
1818 	if (rxfifosz == 0)
1819 		rxfifosz = priv->dma_cap.rx_fifo_size;
1820 	if (txfifosz == 0)
1821 		txfifosz = priv->dma_cap.tx_fifo_size;
1822 
1823 	/* Adjust for real per queue fifo size */
1824 	rxfifosz /= rx_channels_count;
1825 	txfifosz /= tx_channels_count;
1826 
1827 	if (priv->plat->force_thresh_dma_mode) {
1828 		txmode = tc;
1829 		rxmode = tc;
1830 	} else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1831 		/*
1832 		 * In case of GMAC, SF mode can be enabled
1833 		 * to perform the TX COE in HW. This depends on:
1834 		 * 1) TX COE if actually supported
1835 		 * 2) There is no bugged Jumbo frame support
1836 		 *    that needs to not insert csum in the TDES.
1837 		 */
1838 		txmode = SF_DMA_MODE;
1839 		rxmode = SF_DMA_MODE;
1840 		priv->xstats.threshold = SF_DMA_MODE;
1841 	} else {
1842 		txmode = tc;
1843 		rxmode = SF_DMA_MODE;
1844 	}
1845 
1846 	/* configure all channels */
1847 	for (chan = 0; chan < rx_channels_count; chan++) {
1848 		qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1849 
1850 		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
1851 				rxfifosz, qmode);
1852 		stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz,
1853 				chan);
1854 	}
1855 
1856 	for (chan = 0; chan < tx_channels_count; chan++) {
1857 		qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1858 
1859 		stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
1860 				txfifosz, qmode);
1861 	}
1862 }
1863 
1864 /**
1865  * stmmac_tx_clean - to manage the transmission completion
1866  * @priv: driver private structure
1867  * @queue: TX queue index
1868  * Description: it reclaims the transmit resources after transmission completes.
1869  */
1870 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
1871 {
1872 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1873 	unsigned int bytes_compl = 0, pkts_compl = 0;
1874 	unsigned int entry, count = 0;
1875 
1876 	__netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
1877 
1878 	priv->xstats.tx_clean++;
1879 
1880 	entry = tx_q->dirty_tx;
1881 	while ((entry != tx_q->cur_tx) && (count < budget)) {
1882 		struct sk_buff *skb = tx_q->tx_skbuff[entry];
1883 		struct dma_desc *p;
1884 		int status;
1885 
1886 		if (priv->extend_desc)
1887 			p = (struct dma_desc *)(tx_q->dma_etx + entry);
1888 		else
1889 			p = tx_q->dma_tx + entry;
1890 
1891 		status = stmmac_tx_status(priv, &priv->dev->stats,
1892 				&priv->xstats, p, priv->ioaddr);
1893 		/* Check if the descriptor is owned by the DMA */
1894 		if (unlikely(status & tx_dma_own))
1895 			break;
1896 
1897 		count++;
1898 
1899 		/* Make sure descriptor fields are read after reading
1900 		 * the own bit.
1901 		 */
1902 		dma_rmb();
1903 
1904 		/* Just consider the last segment and ...*/
1905 		if (likely(!(status & tx_not_ls))) {
1906 			/* ... verify the status error condition */
1907 			if (unlikely(status & tx_err)) {
1908 				priv->dev->stats.tx_errors++;
1909 			} else {
1910 				priv->dev->stats.tx_packets++;
1911 				priv->xstats.tx_pkt_n++;
1912 			}
1913 			stmmac_get_tx_hwtstamp(priv, p, skb);
1914 		}
1915 
1916 		if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1917 			if (tx_q->tx_skbuff_dma[entry].map_as_page)
1918 				dma_unmap_page(priv->device,
1919 					       tx_q->tx_skbuff_dma[entry].buf,
1920 					       tx_q->tx_skbuff_dma[entry].len,
1921 					       DMA_TO_DEVICE);
1922 			else
1923 				dma_unmap_single(priv->device,
1924 						 tx_q->tx_skbuff_dma[entry].buf,
1925 						 tx_q->tx_skbuff_dma[entry].len,
1926 						 DMA_TO_DEVICE);
1927 			tx_q->tx_skbuff_dma[entry].buf = 0;
1928 			tx_q->tx_skbuff_dma[entry].len = 0;
1929 			tx_q->tx_skbuff_dma[entry].map_as_page = false;
1930 		}
1931 
1932 		stmmac_clean_desc3(priv, tx_q, p);
1933 
1934 		tx_q->tx_skbuff_dma[entry].last_segment = false;
1935 		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1936 
1937 		if (likely(skb != NULL)) {
1938 			pkts_compl++;
1939 			bytes_compl += skb->len;
1940 			dev_consume_skb_any(skb);
1941 			tx_q->tx_skbuff[entry] = NULL;
1942 		}
1943 
1944 		stmmac_release_tx_desc(priv, p, priv->mode);
1945 
1946 		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1947 	}
1948 	tx_q->dirty_tx = entry;
1949 
1950 	netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1951 				  pkts_compl, bytes_compl);
1952 
1953 	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1954 								queue))) &&
1955 	    stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1956 
1957 		netif_dbg(priv, tx_done, priv->dev,
1958 			  "%s: restart transmit\n", __func__);
1959 		netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1960 	}
1961 
1962 	if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1963 		stmmac_enable_eee_mode(priv);
1964 		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1965 	}
1966 
1967 	/* We still have pending packets, let's call for a new scheduling */
1968 	if (tx_q->dirty_tx != tx_q->cur_tx)
1969 		mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(10));
1970 
1971 	__netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
1972 
1973 	return count;
1974 }
1975 
1976 /**
1977  * stmmac_tx_err - to manage the tx error
1978  * @priv: driver private structure
1979  * @chan: channel index
1980  * Description: it cleans the descriptors and restarts the transmission
1981  * in case of transmission errors.
1982  */
1983 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1984 {
1985 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
1986 	int i;
1987 
1988 	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
1989 
1990 	stmmac_stop_tx_dma(priv, chan);
1991 	dma_free_tx_skbufs(priv, chan);
1992 	for (i = 0; i < DMA_TX_SIZE; i++)
1993 		if (priv->extend_desc)
1994 			stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
1995 					priv->mode, (i == DMA_TX_SIZE - 1));
1996 		else
1997 			stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
1998 					priv->mode, (i == DMA_TX_SIZE - 1));
1999 	tx_q->dirty_tx = 0;
2000 	tx_q->cur_tx = 0;
2001 	tx_q->mss = 0;
2002 	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
2003 	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2004 			    tx_q->dma_tx_phy, chan);
2005 	stmmac_start_tx_dma(priv, chan);
2006 
2007 	priv->dev->stats.tx_errors++;
2008 	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
2009 }
2010 
2011 /**
2012  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2013  *  @priv: driver private structure
2014  *  @txmode: TX operating mode
2015  *  @rxmode: RX operating mode
2016  *  @chan: channel index
2017  *  Description: it is used for configuring of the DMA operation mode in
2018  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2019  *  mode.
2020  */
2021 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
2022 					  u32 rxmode, u32 chan)
2023 {
2024 	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2025 	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2026 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2027 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2028 	int rxfifosz = priv->plat->rx_fifo_size;
2029 	int txfifosz = priv->plat->tx_fifo_size;
2030 
2031 	if (rxfifosz == 0)
2032 		rxfifosz = priv->dma_cap.rx_fifo_size;
2033 	if (txfifosz == 0)
2034 		txfifosz = priv->dma_cap.tx_fifo_size;
2035 
2036 	/* Adjust for real per queue fifo size */
2037 	rxfifosz /= rx_channels_count;
2038 	txfifosz /= tx_channels_count;
2039 
2040 	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2041 	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
2042 }
2043 
2044 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2045 {
2046 	int ret;
2047 
2048 	ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2049 			priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2050 	if (ret && (ret != -EINVAL)) {
2051 		stmmac_global_err(priv);
2052 		return true;
2053 	}
2054 
2055 	return false;
2056 }
2057 
2058 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan)
2059 {
2060 	int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2061 						 &priv->xstats, chan);
2062 	struct stmmac_channel *ch = &priv->channel[chan];
2063 
2064 	if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
2065 		if (napi_schedule_prep(&ch->rx_napi)) {
2066 			stmmac_disable_dma_irq(priv, priv->ioaddr, chan);
2067 			__napi_schedule_irqoff(&ch->rx_napi);
2068 			status |= handle_tx;
2069 		}
2070 	}
2071 
2072 	if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use))
2073 		napi_schedule_irqoff(&ch->tx_napi);
2074 
2075 	return status;
2076 }
2077 
2078 /**
2079  * stmmac_dma_interrupt - DMA ISR
2080  * @priv: driver private structure
2081  * Description: this is the DMA ISR. It is called by the main ISR.
2082  * It calls the dwmac dma routine and schedule poll method in case of some
2083  * work can be done.
2084  */
2085 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2086 {
2087 	u32 tx_channel_count = priv->plat->tx_queues_to_use;
2088 	u32 rx_channel_count = priv->plat->rx_queues_to_use;
2089 	u32 channels_to_check = tx_channel_count > rx_channel_count ?
2090 				tx_channel_count : rx_channel_count;
2091 	u32 chan;
2092 	int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
2093 
2094 	/* Make sure we never check beyond our status buffer. */
2095 	if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
2096 		channels_to_check = ARRAY_SIZE(status);
2097 
2098 	for (chan = 0; chan < channels_to_check; chan++)
2099 		status[chan] = stmmac_napi_check(priv, chan);
2100 
2101 	for (chan = 0; chan < tx_channel_count; chan++) {
2102 		if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
2103 			/* Try to bump up the dma threshold on this failure */
2104 			if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2105 			    (tc <= 256)) {
2106 				tc += 64;
2107 				if (priv->plat->force_thresh_dma_mode)
2108 					stmmac_set_dma_operation_mode(priv,
2109 								      tc,
2110 								      tc,
2111 								      chan);
2112 				else
2113 					stmmac_set_dma_operation_mode(priv,
2114 								    tc,
2115 								    SF_DMA_MODE,
2116 								    chan);
2117 				priv->xstats.threshold = tc;
2118 			}
2119 		} else if (unlikely(status[chan] == tx_hard_error)) {
2120 			stmmac_tx_err(priv, chan);
2121 		}
2122 	}
2123 }
2124 
2125 /**
2126  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2127  * @priv: driver private structure
2128  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2129  */
2130 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2131 {
2132 	unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2133 			    MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2134 
2135 	stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
2136 
2137 	if (priv->dma_cap.rmon) {
2138 		stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
2139 		memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2140 	} else
2141 		netdev_info(priv->dev, "No MAC Management Counters available\n");
2142 }
2143 
2144 /**
2145  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2146  * @priv: driver private structure
2147  * Description:
2148  *  new GMAC chip generations have a new register to indicate the
2149  *  presence of the optional feature/functions.
2150  *  This can be also used to override the value passed through the
2151  *  platform and necessary for old MAC10/100 and GMAC chips.
2152  */
2153 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2154 {
2155 	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
2156 }
2157 
2158 /**
2159  * stmmac_check_ether_addr - check if the MAC addr is valid
2160  * @priv: driver private structure
2161  * Description:
2162  * it is to verify if the MAC address is valid, in case of failures it
2163  * generates a random MAC address
2164  */
2165 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2166 {
2167 	if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2168 		stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
2169 		if (!is_valid_ether_addr(priv->dev->dev_addr))
2170 			eth_hw_addr_random(priv->dev);
2171 		dev_info(priv->device, "device MAC address %pM\n",
2172 			 priv->dev->dev_addr);
2173 	}
2174 }
2175 
2176 /**
2177  * stmmac_init_dma_engine - DMA init.
2178  * @priv: driver private structure
2179  * Description:
2180  * It inits the DMA invoking the specific MAC/GMAC callback.
2181  * Some DMA parameters can be passed from the platform;
2182  * in case of these are not passed a default is kept for the MAC or GMAC.
2183  */
2184 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2185 {
2186 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2187 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2188 	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2189 	struct stmmac_rx_queue *rx_q;
2190 	struct stmmac_tx_queue *tx_q;
2191 	u32 chan = 0;
2192 	int atds = 0;
2193 	int ret = 0;
2194 
2195 	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2196 		dev_err(priv->device, "Invalid DMA configuration\n");
2197 		return -EINVAL;
2198 	}
2199 
2200 	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2201 		atds = 1;
2202 
2203 	ret = stmmac_reset(priv, priv->ioaddr);
2204 	if (ret) {
2205 		dev_err(priv->device, "Failed to reset the dma\n");
2206 		return ret;
2207 	}
2208 
2209 	/* DMA Configuration */
2210 	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
2211 
2212 	if (priv->plat->axi)
2213 		stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2214 
2215 	/* DMA CSR Channel configuration */
2216 	for (chan = 0; chan < dma_csr_ch; chan++)
2217 		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
2218 
2219 	/* DMA RX Channel Configuration */
2220 	for (chan = 0; chan < rx_channels_count; chan++) {
2221 		rx_q = &priv->rx_queue[chan];
2222 
2223 		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2224 				    rx_q->dma_rx_phy, chan);
2225 
2226 		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2227 			    (DMA_RX_SIZE * sizeof(struct dma_desc));
2228 		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
2229 				       rx_q->rx_tail_addr, chan);
2230 	}
2231 
2232 	/* DMA TX Channel Configuration */
2233 	for (chan = 0; chan < tx_channels_count; chan++) {
2234 		tx_q = &priv->tx_queue[chan];
2235 
2236 		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2237 				    tx_q->dma_tx_phy, chan);
2238 
2239 		tx_q->tx_tail_addr = tx_q->dma_tx_phy;
2240 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
2241 				       tx_q->tx_tail_addr, chan);
2242 	}
2243 
2244 	return ret;
2245 }
2246 
2247 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2248 {
2249 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2250 
2251 	mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
2252 }
2253 
2254 /**
2255  * stmmac_tx_timer - mitigation sw timer for tx.
2256  * @data: data pointer
2257  * Description:
2258  * This is the timer handler to directly invoke the stmmac_tx_clean.
2259  */
2260 static void stmmac_tx_timer(struct timer_list *t)
2261 {
2262 	struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer);
2263 	struct stmmac_priv *priv = tx_q->priv_data;
2264 	struct stmmac_channel *ch;
2265 
2266 	ch = &priv->channel[tx_q->queue_index];
2267 
2268 	/*
2269 	 * If NAPI is already running we can miss some events. Let's rearm
2270 	 * the timer and try again.
2271 	 */
2272 	if (likely(napi_schedule_prep(&ch->tx_napi)))
2273 		__napi_schedule(&ch->tx_napi);
2274 	else
2275 		mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(10));
2276 }
2277 
2278 /**
2279  * stmmac_init_coalesce - init mitigation options.
2280  * @priv: driver private structure
2281  * Description:
2282  * This inits the coalesce parameters: i.e. timer rate,
2283  * timer handler and default threshold used for enabling the
2284  * interrupt on completion bit.
2285  */
2286 static void stmmac_init_coalesce(struct stmmac_priv *priv)
2287 {
2288 	u32 tx_channel_count = priv->plat->tx_queues_to_use;
2289 	u32 chan;
2290 
2291 	priv->tx_coal_frames = STMMAC_TX_FRAMES;
2292 	priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2293 	priv->rx_coal_frames = STMMAC_RX_FRAMES;
2294 
2295 	for (chan = 0; chan < tx_channel_count; chan++) {
2296 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2297 
2298 		timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0);
2299 	}
2300 }
2301 
2302 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2303 {
2304 	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2305 	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2306 	u32 chan;
2307 
2308 	/* set TX ring length */
2309 	for (chan = 0; chan < tx_channels_count; chan++)
2310 		stmmac_set_tx_ring_len(priv, priv->ioaddr,
2311 				(DMA_TX_SIZE - 1), chan);
2312 
2313 	/* set RX ring length */
2314 	for (chan = 0; chan < rx_channels_count; chan++)
2315 		stmmac_set_rx_ring_len(priv, priv->ioaddr,
2316 				(DMA_RX_SIZE - 1), chan);
2317 }
2318 
2319 /**
2320  *  stmmac_set_tx_queue_weight - Set TX queue weight
2321  *  @priv: driver private structure
2322  *  Description: It is used for setting TX queues weight
2323  */
2324 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2325 {
2326 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2327 	u32 weight;
2328 	u32 queue;
2329 
2330 	for (queue = 0; queue < tx_queues_count; queue++) {
2331 		weight = priv->plat->tx_queues_cfg[queue].weight;
2332 		stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
2333 	}
2334 }
2335 
2336 /**
2337  *  stmmac_configure_cbs - Configure CBS in TX queue
2338  *  @priv: driver private structure
2339  *  Description: It is used for configuring CBS in AVB TX queues
2340  */
2341 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2342 {
2343 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2344 	u32 mode_to_use;
2345 	u32 queue;
2346 
2347 	/* queue 0 is reserved for legacy traffic */
2348 	for (queue = 1; queue < tx_queues_count; queue++) {
2349 		mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2350 		if (mode_to_use == MTL_QUEUE_DCB)
2351 			continue;
2352 
2353 		stmmac_config_cbs(priv, priv->hw,
2354 				priv->plat->tx_queues_cfg[queue].send_slope,
2355 				priv->plat->tx_queues_cfg[queue].idle_slope,
2356 				priv->plat->tx_queues_cfg[queue].high_credit,
2357 				priv->plat->tx_queues_cfg[queue].low_credit,
2358 				queue);
2359 	}
2360 }
2361 
2362 /**
2363  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2364  *  @priv: driver private structure
2365  *  Description: It is used for mapping RX queues to RX dma channels
2366  */
2367 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2368 {
2369 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2370 	u32 queue;
2371 	u32 chan;
2372 
2373 	for (queue = 0; queue < rx_queues_count; queue++) {
2374 		chan = priv->plat->rx_queues_cfg[queue].chan;
2375 		stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
2376 	}
2377 }
2378 
2379 /**
2380  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2381  *  @priv: driver private structure
2382  *  Description: It is used for configuring the RX Queue Priority
2383  */
2384 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2385 {
2386 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2387 	u32 queue;
2388 	u32 prio;
2389 
2390 	for (queue = 0; queue < rx_queues_count; queue++) {
2391 		if (!priv->plat->rx_queues_cfg[queue].use_prio)
2392 			continue;
2393 
2394 		prio = priv->plat->rx_queues_cfg[queue].prio;
2395 		stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
2396 	}
2397 }
2398 
2399 /**
2400  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2401  *  @priv: driver private structure
2402  *  Description: It is used for configuring the TX Queue Priority
2403  */
2404 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2405 {
2406 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2407 	u32 queue;
2408 	u32 prio;
2409 
2410 	for (queue = 0; queue < tx_queues_count; queue++) {
2411 		if (!priv->plat->tx_queues_cfg[queue].use_prio)
2412 			continue;
2413 
2414 		prio = priv->plat->tx_queues_cfg[queue].prio;
2415 		stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
2416 	}
2417 }
2418 
2419 /**
2420  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2421  *  @priv: driver private structure
2422  *  Description: It is used for configuring the RX queue routing
2423  */
2424 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2425 {
2426 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2427 	u32 queue;
2428 	u8 packet;
2429 
2430 	for (queue = 0; queue < rx_queues_count; queue++) {
2431 		/* no specific packet type routing specified for the queue */
2432 		if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2433 			continue;
2434 
2435 		packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2436 		stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
2437 	}
2438 }
2439 
2440 static void stmmac_mac_config_rss(struct stmmac_priv *priv)
2441 {
2442 	if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
2443 		priv->rss.enable = false;
2444 		return;
2445 	}
2446 
2447 	if (priv->dev->features & NETIF_F_RXHASH)
2448 		priv->rss.enable = true;
2449 	else
2450 		priv->rss.enable = false;
2451 
2452 	stmmac_rss_configure(priv, priv->hw, &priv->rss,
2453 			     priv->plat->rx_queues_to_use);
2454 }
2455 
2456 /**
2457  *  stmmac_mtl_configuration - Configure MTL
2458  *  @priv: driver private structure
2459  *  Description: It is used for configurring MTL
2460  */
2461 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2462 {
2463 	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2464 	u32 tx_queues_count = priv->plat->tx_queues_to_use;
2465 
2466 	if (tx_queues_count > 1)
2467 		stmmac_set_tx_queue_weight(priv);
2468 
2469 	/* Configure MTL RX algorithms */
2470 	if (rx_queues_count > 1)
2471 		stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
2472 				priv->plat->rx_sched_algorithm);
2473 
2474 	/* Configure MTL TX algorithms */
2475 	if (tx_queues_count > 1)
2476 		stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
2477 				priv->plat->tx_sched_algorithm);
2478 
2479 	/* Configure CBS in AVB TX queues */
2480 	if (tx_queues_count > 1)
2481 		stmmac_configure_cbs(priv);
2482 
2483 	/* Map RX MTL to DMA channels */
2484 	stmmac_rx_queue_dma_chan_map(priv);
2485 
2486 	/* Enable MAC RX Queues */
2487 	stmmac_mac_enable_rx_queues(priv);
2488 
2489 	/* Set RX priorities */
2490 	if (rx_queues_count > 1)
2491 		stmmac_mac_config_rx_queues_prio(priv);
2492 
2493 	/* Set TX priorities */
2494 	if (tx_queues_count > 1)
2495 		stmmac_mac_config_tx_queues_prio(priv);
2496 
2497 	/* Set RX routing */
2498 	if (rx_queues_count > 1)
2499 		stmmac_mac_config_rx_queues_routing(priv);
2500 
2501 	/* Receive Side Scaling */
2502 	if (rx_queues_count > 1)
2503 		stmmac_mac_config_rss(priv);
2504 }
2505 
2506 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
2507 {
2508 	if (priv->dma_cap.asp) {
2509 		netdev_info(priv->dev, "Enabling Safety Features\n");
2510 		stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
2511 	} else {
2512 		netdev_info(priv->dev, "No Safety Features support found\n");
2513 	}
2514 }
2515 
2516 /**
2517  * stmmac_hw_setup - setup mac in a usable state.
2518  *  @dev : pointer to the device structure.
2519  *  Description:
2520  *  this is the main function to setup the HW in a usable state because the
2521  *  dma engine is reset, the core registers are configured (e.g. AXI,
2522  *  Checksum features, timers). The DMA is ready to start receiving and
2523  *  transmitting.
2524  *  Return value:
2525  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2526  *  file on failure.
2527  */
2528 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2529 {
2530 	struct stmmac_priv *priv = netdev_priv(dev);
2531 	u32 rx_cnt = priv->plat->rx_queues_to_use;
2532 	u32 tx_cnt = priv->plat->tx_queues_to_use;
2533 	u32 chan;
2534 	int ret;
2535 
2536 	/* DMA initialization and SW reset */
2537 	ret = stmmac_init_dma_engine(priv);
2538 	if (ret < 0) {
2539 		netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2540 			   __func__);
2541 		return ret;
2542 	}
2543 
2544 	/* Copy the MAC addr into the HW  */
2545 	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
2546 
2547 	/* PS and related bits will be programmed according to the speed */
2548 	if (priv->hw->pcs) {
2549 		int speed = priv->plat->mac_port_sel_speed;
2550 
2551 		if ((speed == SPEED_10) || (speed == SPEED_100) ||
2552 		    (speed == SPEED_1000)) {
2553 			priv->hw->ps = speed;
2554 		} else {
2555 			dev_warn(priv->device, "invalid port speed\n");
2556 			priv->hw->ps = 0;
2557 		}
2558 	}
2559 
2560 	/* Initialize the MAC Core */
2561 	stmmac_core_init(priv, priv->hw, dev);
2562 
2563 	/* Initialize MTL*/
2564 	stmmac_mtl_configuration(priv);
2565 
2566 	/* Initialize Safety Features */
2567 	stmmac_safety_feat_configuration(priv);
2568 
2569 	ret = stmmac_rx_ipc(priv, priv->hw);
2570 	if (!ret) {
2571 		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2572 		priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2573 		priv->hw->rx_csum = 0;
2574 	}
2575 
2576 	/* Enable the MAC Rx/Tx */
2577 	stmmac_mac_set(priv, priv->ioaddr, true);
2578 
2579 	/* Set the HW DMA mode and the COE */
2580 	stmmac_dma_operation_mode(priv);
2581 
2582 	stmmac_mmc_setup(priv);
2583 
2584 	if (init_ptp) {
2585 		ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2586 		if (ret < 0)
2587 			netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2588 
2589 		ret = stmmac_init_ptp(priv);
2590 		if (ret == -EOPNOTSUPP)
2591 			netdev_warn(priv->dev, "PTP not supported by HW\n");
2592 		else if (ret)
2593 			netdev_warn(priv->dev, "PTP init failed\n");
2594 	}
2595 
2596 	priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2597 
2598 	if (priv->use_riwt) {
2599 		if (!priv->rx_riwt)
2600 			priv->rx_riwt = DEF_DMA_RIWT;
2601 
2602 		ret = stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt);
2603 	}
2604 
2605 	if (priv->hw->pcs)
2606 		stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
2607 
2608 	/* set TX and RX rings length */
2609 	stmmac_set_rings_length(priv);
2610 
2611 	/* Enable TSO */
2612 	if (priv->tso) {
2613 		for (chan = 0; chan < tx_cnt; chan++)
2614 			stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
2615 	}
2616 
2617 	/* Enable Split Header */
2618 	if (priv->sph && priv->hw->rx_csum) {
2619 		for (chan = 0; chan < rx_cnt; chan++)
2620 			stmmac_enable_sph(priv, priv->ioaddr, 1, chan);
2621 	}
2622 
2623 	/* VLAN Tag Insertion */
2624 	if (priv->dma_cap.vlins)
2625 		stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
2626 
2627 	/* Start the ball rolling... */
2628 	stmmac_start_all_dma(priv);
2629 
2630 	return 0;
2631 }
2632 
2633 static void stmmac_hw_teardown(struct net_device *dev)
2634 {
2635 	struct stmmac_priv *priv = netdev_priv(dev);
2636 
2637 	clk_disable_unprepare(priv->plat->clk_ptp_ref);
2638 }
2639 
2640 /**
2641  *  stmmac_open - open entry point of the driver
2642  *  @dev : pointer to the device structure.
2643  *  Description:
2644  *  This function is the open entry point of the driver.
2645  *  Return value:
2646  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2647  *  file on failure.
2648  */
2649 static int stmmac_open(struct net_device *dev)
2650 {
2651 	struct stmmac_priv *priv = netdev_priv(dev);
2652 	int bfsize = 0;
2653 	u32 chan;
2654 	int ret;
2655 
2656 	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2657 	    priv->hw->pcs != STMMAC_PCS_TBI &&
2658 	    priv->hw->pcs != STMMAC_PCS_RTBI) {
2659 		ret = stmmac_init_phy(dev);
2660 		if (ret) {
2661 			netdev_err(priv->dev,
2662 				   "%s: Cannot attach to PHY (error: %d)\n",
2663 				   __func__, ret);
2664 			return ret;
2665 		}
2666 	}
2667 
2668 	/* Extra statistics */
2669 	memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2670 	priv->xstats.threshold = tc;
2671 
2672 	bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
2673 	if (bfsize < 0)
2674 		bfsize = 0;
2675 
2676 	if (bfsize < BUF_SIZE_16KiB)
2677 		bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
2678 
2679 	priv->dma_buf_sz = bfsize;
2680 	buf_sz = bfsize;
2681 
2682 	priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2683 
2684 	ret = alloc_dma_desc_resources(priv);
2685 	if (ret < 0) {
2686 		netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2687 			   __func__);
2688 		goto dma_desc_error;
2689 	}
2690 
2691 	ret = init_dma_desc_rings(dev, GFP_KERNEL);
2692 	if (ret < 0) {
2693 		netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2694 			   __func__);
2695 		goto init_error;
2696 	}
2697 
2698 	ret = stmmac_hw_setup(dev, true);
2699 	if (ret < 0) {
2700 		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2701 		goto init_error;
2702 	}
2703 
2704 	stmmac_init_coalesce(priv);
2705 
2706 	phylink_start(priv->phylink);
2707 
2708 	/* Request the IRQ lines */
2709 	ret = request_irq(dev->irq, stmmac_interrupt,
2710 			  IRQF_SHARED, dev->name, dev);
2711 	if (unlikely(ret < 0)) {
2712 		netdev_err(priv->dev,
2713 			   "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2714 			   __func__, dev->irq, ret);
2715 		goto irq_error;
2716 	}
2717 
2718 	/* Request the Wake IRQ in case of another line is used for WoL */
2719 	if (priv->wol_irq != dev->irq) {
2720 		ret = request_irq(priv->wol_irq, stmmac_interrupt,
2721 				  IRQF_SHARED, dev->name, dev);
2722 		if (unlikely(ret < 0)) {
2723 			netdev_err(priv->dev,
2724 				   "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2725 				   __func__, priv->wol_irq, ret);
2726 			goto wolirq_error;
2727 		}
2728 	}
2729 
2730 	/* Request the IRQ lines */
2731 	if (priv->lpi_irq > 0) {
2732 		ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2733 				  dev->name, dev);
2734 		if (unlikely(ret < 0)) {
2735 			netdev_err(priv->dev,
2736 				   "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2737 				   __func__, priv->lpi_irq, ret);
2738 			goto lpiirq_error;
2739 		}
2740 	}
2741 
2742 	stmmac_enable_all_queues(priv);
2743 	stmmac_start_all_queues(priv);
2744 
2745 	return 0;
2746 
2747 lpiirq_error:
2748 	if (priv->wol_irq != dev->irq)
2749 		free_irq(priv->wol_irq, dev);
2750 wolirq_error:
2751 	free_irq(dev->irq, dev);
2752 irq_error:
2753 	phylink_stop(priv->phylink);
2754 
2755 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2756 		del_timer_sync(&priv->tx_queue[chan].txtimer);
2757 
2758 	stmmac_hw_teardown(dev);
2759 init_error:
2760 	free_dma_desc_resources(priv);
2761 dma_desc_error:
2762 	phylink_disconnect_phy(priv->phylink);
2763 	return ret;
2764 }
2765 
2766 /**
2767  *  stmmac_release - close entry point of the driver
2768  *  @dev : device pointer.
2769  *  Description:
2770  *  This is the stop entry point of the driver.
2771  */
2772 static int stmmac_release(struct net_device *dev)
2773 {
2774 	struct stmmac_priv *priv = netdev_priv(dev);
2775 	u32 chan;
2776 
2777 	if (priv->eee_enabled)
2778 		del_timer_sync(&priv->eee_ctrl_timer);
2779 
2780 	/* Stop and disconnect the PHY */
2781 	phylink_stop(priv->phylink);
2782 	phylink_disconnect_phy(priv->phylink);
2783 
2784 	stmmac_stop_all_queues(priv);
2785 
2786 	stmmac_disable_all_queues(priv);
2787 
2788 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2789 		del_timer_sync(&priv->tx_queue[chan].txtimer);
2790 
2791 	/* Free the IRQ lines */
2792 	free_irq(dev->irq, dev);
2793 	if (priv->wol_irq != dev->irq)
2794 		free_irq(priv->wol_irq, dev);
2795 	if (priv->lpi_irq > 0)
2796 		free_irq(priv->lpi_irq, dev);
2797 
2798 	/* Stop TX/RX DMA and clear the descriptors */
2799 	stmmac_stop_all_dma(priv);
2800 
2801 	/* Release and free the Rx/Tx resources */
2802 	free_dma_desc_resources(priv);
2803 
2804 	/* Disable the MAC Rx/Tx */
2805 	stmmac_mac_set(priv, priv->ioaddr, false);
2806 
2807 	netif_carrier_off(dev);
2808 
2809 	stmmac_release_ptp(priv);
2810 
2811 	return 0;
2812 }
2813 
2814 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
2815 			       struct stmmac_tx_queue *tx_q)
2816 {
2817 	u16 tag = 0x0, inner_tag = 0x0;
2818 	u32 inner_type = 0x0;
2819 	struct dma_desc *p;
2820 
2821 	if (!priv->dma_cap.vlins)
2822 		return false;
2823 	if (!skb_vlan_tag_present(skb))
2824 		return false;
2825 	if (skb->vlan_proto == htons(ETH_P_8021AD)) {
2826 		inner_tag = skb_vlan_tag_get(skb);
2827 		inner_type = STMMAC_VLAN_INSERT;
2828 	}
2829 
2830 	tag = skb_vlan_tag_get(skb);
2831 
2832 	p = tx_q->dma_tx + tx_q->cur_tx;
2833 	if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))
2834 		return false;
2835 
2836 	stmmac_set_tx_owner(priv, p);
2837 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2838 	return true;
2839 }
2840 
2841 /**
2842  *  stmmac_tso_allocator - close entry point of the driver
2843  *  @priv: driver private structure
2844  *  @des: buffer start address
2845  *  @total_len: total length to fill in descriptors
2846  *  @last_segmant: condition for the last descriptor
2847  *  @queue: TX queue index
2848  *  Description:
2849  *  This function fills descriptor and request new descriptors according to
2850  *  buffer length to fill
2851  */
2852 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
2853 				 int total_len, bool last_segment, u32 queue)
2854 {
2855 	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2856 	struct dma_desc *desc;
2857 	u32 buff_size;
2858 	int tmp_len;
2859 
2860 	tmp_len = total_len;
2861 
2862 	while (tmp_len > 0) {
2863 		dma_addr_t curr_addr;
2864 
2865 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2866 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
2867 		desc = tx_q->dma_tx + tx_q->cur_tx;
2868 
2869 		curr_addr = des + (total_len - tmp_len);
2870 		if (priv->dma_cap.addr64 <= 32)
2871 			desc->des0 = cpu_to_le32(curr_addr);
2872 		else
2873 			stmmac_set_desc_addr(priv, desc, curr_addr);
2874 
2875 		buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2876 			    TSO_MAX_BUFF_SIZE : tmp_len;
2877 
2878 		stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
2879 				0, 1,
2880 				(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
2881 				0, 0);
2882 
2883 		tmp_len -= TSO_MAX_BUFF_SIZE;
2884 	}
2885 }
2886 
2887 /**
2888  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2889  *  @skb : the socket buffer
2890  *  @dev : device pointer
2891  *  Description: this is the transmit function that is called on TSO frames
2892  *  (support available on GMAC4 and newer chips).
2893  *  Diagram below show the ring programming in case of TSO frames:
2894  *
2895  *  First Descriptor
2896  *   --------
2897  *   | DES0 |---> buffer1 = L2/L3/L4 header
2898  *   | DES1 |---> TCP Payload (can continue on next descr...)
2899  *   | DES2 |---> buffer 1 and 2 len
2900  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2901  *   --------
2902  *	|
2903  *     ...
2904  *	|
2905  *   --------
2906  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2907  *   | DES1 | --|
2908  *   | DES2 | --> buffer 1 and 2 len
2909  *   | DES3 |
2910  *   --------
2911  *
2912  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2913  */
2914 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2915 {
2916 	struct dma_desc *desc, *first, *mss_desc = NULL;
2917 	struct stmmac_priv *priv = netdev_priv(dev);
2918 	int nfrags = skb_shinfo(skb)->nr_frags;
2919 	u32 queue = skb_get_queue_mapping(skb);
2920 	unsigned int first_entry, tx_packets;
2921 	int tmp_pay_len = 0, first_tx;
2922 	struct stmmac_tx_queue *tx_q;
2923 	u8 proto_hdr_len, hdr;
2924 	bool has_vlan, set_ic;
2925 	u32 pay_len, mss;
2926 	dma_addr_t des;
2927 	int i;
2928 
2929 	tx_q = &priv->tx_queue[queue];
2930 	first_tx = tx_q->cur_tx;
2931 
2932 	/* Compute header lengths */
2933 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
2934 		proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
2935 		hdr = sizeof(struct udphdr);
2936 	} else {
2937 		proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2938 		hdr = tcp_hdrlen(skb);
2939 	}
2940 
2941 	/* Desc availability based on threshold should be enough safe */
2942 	if (unlikely(stmmac_tx_avail(priv, queue) <
2943 		(((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2944 		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2945 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2946 								queue));
2947 			/* This is a hard error, log it. */
2948 			netdev_err(priv->dev,
2949 				   "%s: Tx Ring full when queue awake\n",
2950 				   __func__);
2951 		}
2952 		return NETDEV_TX_BUSY;
2953 	}
2954 
2955 	pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2956 
2957 	mss = skb_shinfo(skb)->gso_size;
2958 
2959 	/* set new MSS value if needed */
2960 	if (mss != tx_q->mss) {
2961 		mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2962 		stmmac_set_mss(priv, mss_desc, mss);
2963 		tx_q->mss = mss;
2964 		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2965 		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
2966 	}
2967 
2968 	if (netif_msg_tx_queued(priv)) {
2969 		pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2970 			__func__, hdr, proto_hdr_len, pay_len, mss);
2971 		pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2972 			skb->data_len);
2973 	}
2974 
2975 	/* Check if VLAN can be inserted by HW */
2976 	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
2977 
2978 	first_entry = tx_q->cur_tx;
2979 	WARN_ON(tx_q->tx_skbuff[first_entry]);
2980 
2981 	desc = tx_q->dma_tx + first_entry;
2982 	first = desc;
2983 
2984 	if (has_vlan)
2985 		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
2986 
2987 	/* first descriptor: fill Headers on Buf1 */
2988 	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2989 			     DMA_TO_DEVICE);
2990 	if (dma_mapping_error(priv->device, des))
2991 		goto dma_map_err;
2992 
2993 	tx_q->tx_skbuff_dma[first_entry].buf = des;
2994 	tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2995 
2996 	if (priv->dma_cap.addr64 <= 32) {
2997 		first->des0 = cpu_to_le32(des);
2998 
2999 		/* Fill start of payload in buff2 of first descriptor */
3000 		if (pay_len)
3001 			first->des1 = cpu_to_le32(des + proto_hdr_len);
3002 
3003 		/* If needed take extra descriptors to fill the remaining payload */
3004 		tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
3005 	} else {
3006 		stmmac_set_desc_addr(priv, first, des);
3007 		tmp_pay_len = pay_len;
3008 		des += proto_hdr_len;
3009 		pay_len = 0;
3010 	}
3011 
3012 	stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
3013 
3014 	/* Prepare fragments */
3015 	for (i = 0; i < nfrags; i++) {
3016 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3017 
3018 		des = skb_frag_dma_map(priv->device, frag, 0,
3019 				       skb_frag_size(frag),
3020 				       DMA_TO_DEVICE);
3021 		if (dma_mapping_error(priv->device, des))
3022 			goto dma_map_err;
3023 
3024 		stmmac_tso_allocator(priv, des, skb_frag_size(frag),
3025 				     (i == nfrags - 1), queue);
3026 
3027 		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
3028 		tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
3029 		tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
3030 	}
3031 
3032 	tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
3033 
3034 	/* Only the last descriptor gets to point to the skb. */
3035 	tx_q->tx_skbuff[tx_q->cur_tx] = skb;
3036 
3037 	/* Manage tx mitigation */
3038 	tx_packets = (tx_q->cur_tx + 1) - first_tx;
3039 	tx_q->tx_count_frames += tx_packets;
3040 
3041 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
3042 		set_ic = true;
3043 	else if (!priv->tx_coal_frames)
3044 		set_ic = false;
3045 	else if (tx_packets > priv->tx_coal_frames)
3046 		set_ic = true;
3047 	else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
3048 		set_ic = true;
3049 	else
3050 		set_ic = false;
3051 
3052 	if (set_ic) {
3053 		desc = &tx_q->dma_tx[tx_q->cur_tx];
3054 		tx_q->tx_count_frames = 0;
3055 		stmmac_set_tx_ic(priv, desc);
3056 		priv->xstats.tx_set_ic_bit++;
3057 	}
3058 
3059 	/* We've used all descriptors we need for this skb, however,
3060 	 * advance cur_tx so that it references a fresh descriptor.
3061 	 * ndo_start_xmit will fill this descriptor the next time it's
3062 	 * called and stmmac_tx_clean may clean up to this descriptor.
3063 	 */
3064 	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
3065 
3066 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3067 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3068 			  __func__);
3069 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3070 	}
3071 
3072 	dev->stats.tx_bytes += skb->len;
3073 	priv->xstats.tx_tso_frames++;
3074 	priv->xstats.tx_tso_nfrags += nfrags;
3075 
3076 	if (priv->sarc_type)
3077 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3078 
3079 	skb_tx_timestamp(skb);
3080 
3081 	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3082 		     priv->hwts_tx_en)) {
3083 		/* declare that device is doing timestamping */
3084 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3085 		stmmac_enable_tx_timestamp(priv, first);
3086 	}
3087 
3088 	/* Complete the first descriptor before granting the DMA */
3089 	stmmac_prepare_tso_tx_desc(priv, first, 1,
3090 			proto_hdr_len,
3091 			pay_len,
3092 			1, tx_q->tx_skbuff_dma[first_entry].last_segment,
3093 			hdr / 4, (skb->len - proto_hdr_len));
3094 
3095 	/* If context desc is used to change MSS */
3096 	if (mss_desc) {
3097 		/* Make sure that first descriptor has been completely
3098 		 * written, including its own bit. This is because MSS is
3099 		 * actually before first descriptor, so we need to make
3100 		 * sure that MSS's own bit is the last thing written.
3101 		 */
3102 		dma_wmb();
3103 		stmmac_set_tx_owner(priv, mss_desc);
3104 	}
3105 
3106 	/* The own bit must be the latest setting done when prepare the
3107 	 * descriptor and then barrier is needed to make sure that
3108 	 * all is coherent before granting the DMA engine.
3109 	 */
3110 	wmb();
3111 
3112 	if (netif_msg_pktdata(priv)) {
3113 		pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
3114 			__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3115 			tx_q->cur_tx, first, nfrags);
3116 
3117 		stmmac_display_ring(priv, (void *)tx_q->dma_tx, DMA_TX_SIZE, 0);
3118 
3119 		pr_info(">>> frame to be transmitted: ");
3120 		print_pkt(skb->data, skb_headlen(skb));
3121 	}
3122 
3123 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3124 
3125 	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc));
3126 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3127 	stmmac_tx_timer_arm(priv, queue);
3128 
3129 	return NETDEV_TX_OK;
3130 
3131 dma_map_err:
3132 	dev_err(priv->device, "Tx dma map failed\n");
3133 	dev_kfree_skb(skb);
3134 	priv->dev->stats.tx_dropped++;
3135 	return NETDEV_TX_OK;
3136 }
3137 
3138 /**
3139  *  stmmac_xmit - Tx entry point of the driver
3140  *  @skb : the socket buffer
3141  *  @dev : device pointer
3142  *  Description : this is the tx entry point of the driver.
3143  *  It programs the chain or the ring and supports oversized frames
3144  *  and SG feature.
3145  */
3146 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
3147 {
3148 	unsigned int first_entry, tx_packets, enh_desc;
3149 	struct stmmac_priv *priv = netdev_priv(dev);
3150 	unsigned int nopaged_len = skb_headlen(skb);
3151 	int i, csum_insertion = 0, is_jumbo = 0;
3152 	u32 queue = skb_get_queue_mapping(skb);
3153 	int nfrags = skb_shinfo(skb)->nr_frags;
3154 	int gso = skb_shinfo(skb)->gso_type;
3155 	struct dma_desc *desc, *first;
3156 	struct stmmac_tx_queue *tx_q;
3157 	bool has_vlan, set_ic;
3158 	int entry, first_tx;
3159 	dma_addr_t des;
3160 
3161 	tx_q = &priv->tx_queue[queue];
3162 	first_tx = tx_q->cur_tx;
3163 
3164 	if (priv->tx_path_in_lpi_mode)
3165 		stmmac_disable_eee_mode(priv);
3166 
3167 	/* Manage oversized TCP frames for GMAC4 device */
3168 	if (skb_is_gso(skb) && priv->tso) {
3169 		if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
3170 			return stmmac_tso_xmit(skb, dev);
3171 		if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
3172 			return stmmac_tso_xmit(skb, dev);
3173 	}
3174 
3175 	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
3176 		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3177 			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3178 								queue));
3179 			/* This is a hard error, log it. */
3180 			netdev_err(priv->dev,
3181 				   "%s: Tx Ring full when queue awake\n",
3182 				   __func__);
3183 		}
3184 		return NETDEV_TX_BUSY;
3185 	}
3186 
3187 	/* Check if VLAN can be inserted by HW */
3188 	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
3189 
3190 	entry = tx_q->cur_tx;
3191 	first_entry = entry;
3192 	WARN_ON(tx_q->tx_skbuff[first_entry]);
3193 
3194 	csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
3195 
3196 	if (likely(priv->extend_desc))
3197 		desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3198 	else
3199 		desc = tx_q->dma_tx + entry;
3200 
3201 	first = desc;
3202 
3203 	if (has_vlan)
3204 		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
3205 
3206 	enh_desc = priv->plat->enh_desc;
3207 	/* To program the descriptors according to the size of the frame */
3208 	if (enh_desc)
3209 		is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
3210 
3211 	if (unlikely(is_jumbo)) {
3212 		entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
3213 		if (unlikely(entry < 0) && (entry != -EINVAL))
3214 			goto dma_map_err;
3215 	}
3216 
3217 	for (i = 0; i < nfrags; i++) {
3218 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3219 		int len = skb_frag_size(frag);
3220 		bool last_segment = (i == (nfrags - 1));
3221 
3222 		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3223 		WARN_ON(tx_q->tx_skbuff[entry]);
3224 
3225 		if (likely(priv->extend_desc))
3226 			desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3227 		else
3228 			desc = tx_q->dma_tx + entry;
3229 
3230 		des = skb_frag_dma_map(priv->device, frag, 0, len,
3231 				       DMA_TO_DEVICE);
3232 		if (dma_mapping_error(priv->device, des))
3233 			goto dma_map_err; /* should reuse desc w/o issues */
3234 
3235 		tx_q->tx_skbuff_dma[entry].buf = des;
3236 
3237 		stmmac_set_desc_addr(priv, desc, des);
3238 
3239 		tx_q->tx_skbuff_dma[entry].map_as_page = true;
3240 		tx_q->tx_skbuff_dma[entry].len = len;
3241 		tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3242 
3243 		/* Prepare the descriptor and set the own bit too */
3244 		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
3245 				priv->mode, 1, last_segment, skb->len);
3246 	}
3247 
3248 	/* Only the last descriptor gets to point to the skb. */
3249 	tx_q->tx_skbuff[entry] = skb;
3250 
3251 	/* According to the coalesce parameter the IC bit for the latest
3252 	 * segment is reset and the timer re-started to clean the tx status.
3253 	 * This approach takes care about the fragments: desc is the first
3254 	 * element in case of no SG.
3255 	 */
3256 	tx_packets = (entry + 1) - first_tx;
3257 	tx_q->tx_count_frames += tx_packets;
3258 
3259 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
3260 		set_ic = true;
3261 	else if (!priv->tx_coal_frames)
3262 		set_ic = false;
3263 	else if (tx_packets > priv->tx_coal_frames)
3264 		set_ic = true;
3265 	else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
3266 		set_ic = true;
3267 	else
3268 		set_ic = false;
3269 
3270 	if (set_ic) {
3271 		if (likely(priv->extend_desc))
3272 			desc = &tx_q->dma_etx[entry].basic;
3273 		else
3274 			desc = &tx_q->dma_tx[entry];
3275 
3276 		tx_q->tx_count_frames = 0;
3277 		stmmac_set_tx_ic(priv, desc);
3278 		priv->xstats.tx_set_ic_bit++;
3279 	}
3280 
3281 	/* We've used all descriptors we need for this skb, however,
3282 	 * advance cur_tx so that it references a fresh descriptor.
3283 	 * ndo_start_xmit will fill this descriptor the next time it's
3284 	 * called and stmmac_tx_clean may clean up to this descriptor.
3285 	 */
3286 	entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3287 	tx_q->cur_tx = entry;
3288 
3289 	if (netif_msg_pktdata(priv)) {
3290 		void *tx_head;
3291 
3292 		netdev_dbg(priv->dev,
3293 			   "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3294 			   __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3295 			   entry, first, nfrags);
3296 
3297 		if (priv->extend_desc)
3298 			tx_head = (void *)tx_q->dma_etx;
3299 		else
3300 			tx_head = (void *)tx_q->dma_tx;
3301 
3302 		stmmac_display_ring(priv, tx_head, DMA_TX_SIZE, false);
3303 
3304 		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3305 		print_pkt(skb->data, skb->len);
3306 	}
3307 
3308 	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3309 		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3310 			  __func__);
3311 		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3312 	}
3313 
3314 	dev->stats.tx_bytes += skb->len;
3315 
3316 	if (priv->sarc_type)
3317 		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3318 
3319 	skb_tx_timestamp(skb);
3320 
3321 	/* Ready to fill the first descriptor and set the OWN bit w/o any
3322 	 * problems because all the descriptors are actually ready to be
3323 	 * passed to the DMA engine.
3324 	 */
3325 	if (likely(!is_jumbo)) {
3326 		bool last_segment = (nfrags == 0);
3327 
3328 		des = dma_map_single(priv->device, skb->data,
3329 				     nopaged_len, DMA_TO_DEVICE);
3330 		if (dma_mapping_error(priv->device, des))
3331 			goto dma_map_err;
3332 
3333 		tx_q->tx_skbuff_dma[first_entry].buf = des;
3334 
3335 		stmmac_set_desc_addr(priv, first, des);
3336 
3337 		tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3338 		tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3339 
3340 		if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3341 			     priv->hwts_tx_en)) {
3342 			/* declare that device is doing timestamping */
3343 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3344 			stmmac_enable_tx_timestamp(priv, first);
3345 		}
3346 
3347 		/* Prepare the first descriptor setting the OWN bit too */
3348 		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
3349 				csum_insertion, priv->mode, 1, last_segment,
3350 				skb->len);
3351 	} else {
3352 		stmmac_set_tx_owner(priv, first);
3353 	}
3354 
3355 	/* The own bit must be the latest setting done when prepare the
3356 	 * descriptor and then barrier is needed to make sure that
3357 	 * all is coherent before granting the DMA engine.
3358 	 */
3359 	wmb();
3360 
3361 	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3362 
3363 	stmmac_enable_dma_transmission(priv, priv->ioaddr);
3364 
3365 	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc));
3366 	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3367 	stmmac_tx_timer_arm(priv, queue);
3368 
3369 	return NETDEV_TX_OK;
3370 
3371 dma_map_err:
3372 	netdev_err(priv->dev, "Tx DMA map failed\n");
3373 	dev_kfree_skb(skb);
3374 	priv->dev->stats.tx_dropped++;
3375 	return NETDEV_TX_OK;
3376 }
3377 
3378 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3379 {
3380 	struct vlan_ethhdr *veth;
3381 	__be16 vlan_proto;
3382 	u16 vlanid;
3383 
3384 	veth = (struct vlan_ethhdr *)skb->data;
3385 	vlan_proto = veth->h_vlan_proto;
3386 
3387 	if ((vlan_proto == htons(ETH_P_8021Q) &&
3388 	     dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
3389 	    (vlan_proto == htons(ETH_P_8021AD) &&
3390 	     dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
3391 		/* pop the vlan tag */
3392 		vlanid = ntohs(veth->h_vlan_TCI);
3393 		memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
3394 		skb_pull(skb, VLAN_HLEN);
3395 		__vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
3396 	}
3397 }
3398 
3399 
3400 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3401 {
3402 	if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3403 		return 0;
3404 
3405 	return 1;
3406 }
3407 
3408 /**
3409  * stmmac_rx_refill - refill used skb preallocated buffers
3410  * @priv: driver private structure
3411  * @queue: RX queue index
3412  * Description : this is to reallocate the skb for the reception process
3413  * that is based on zero-copy.
3414  */
3415 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3416 {
3417 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3418 	int len, dirty = stmmac_rx_dirty(priv, queue);
3419 	unsigned int entry = rx_q->dirty_rx;
3420 
3421 	len = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
3422 
3423 	while (dirty-- > 0) {
3424 		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
3425 		struct dma_desc *p;
3426 		bool use_rx_wd;
3427 
3428 		if (priv->extend_desc)
3429 			p = (struct dma_desc *)(rx_q->dma_erx + entry);
3430 		else
3431 			p = rx_q->dma_rx + entry;
3432 
3433 		if (!buf->page) {
3434 			buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
3435 			if (!buf->page)
3436 				break;
3437 		}
3438 
3439 		if (priv->sph && !buf->sec_page) {
3440 			buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
3441 			if (!buf->sec_page)
3442 				break;
3443 
3444 			buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
3445 
3446 			dma_sync_single_for_device(priv->device, buf->sec_addr,
3447 						   len, DMA_FROM_DEVICE);
3448 		}
3449 
3450 		buf->addr = page_pool_get_dma_addr(buf->page);
3451 
3452 		/* Sync whole allocation to device. This will invalidate old
3453 		 * data.
3454 		 */
3455 		dma_sync_single_for_device(priv->device, buf->addr, len,
3456 					   DMA_FROM_DEVICE);
3457 
3458 		stmmac_set_desc_addr(priv, p, buf->addr);
3459 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
3460 		stmmac_refill_desc3(priv, rx_q, p);
3461 
3462 		rx_q->rx_count_frames++;
3463 		rx_q->rx_count_frames += priv->rx_coal_frames;
3464 		if (rx_q->rx_count_frames > priv->rx_coal_frames)
3465 			rx_q->rx_count_frames = 0;
3466 
3467 		use_rx_wd = !priv->rx_coal_frames;
3468 		use_rx_wd |= rx_q->rx_count_frames > 0;
3469 		if (!priv->use_riwt)
3470 			use_rx_wd = false;
3471 
3472 		dma_wmb();
3473 		stmmac_set_rx_owner(priv, p, use_rx_wd);
3474 
3475 		entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3476 	}
3477 	rx_q->dirty_rx = entry;
3478 	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
3479 			    (rx_q->dirty_rx * sizeof(struct dma_desc));
3480 	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
3481 }
3482 
3483 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
3484 				       struct dma_desc *p,
3485 				       int status, unsigned int len)
3486 {
3487 	int ret, coe = priv->hw->rx_csum;
3488 	unsigned int plen = 0, hlen = 0;
3489 
3490 	/* Not first descriptor, buffer is always zero */
3491 	if (priv->sph && len)
3492 		return 0;
3493 
3494 	/* First descriptor, get split header length */
3495 	ret = stmmac_get_rx_header_len(priv, p, &hlen);
3496 	if (priv->sph && hlen) {
3497 		priv->xstats.rx_split_hdr_pkt_n++;
3498 		return hlen;
3499 	}
3500 
3501 	/* First descriptor, not last descriptor and not split header */
3502 	if (status & rx_not_ls)
3503 		return priv->dma_buf_sz;
3504 
3505 	plen = stmmac_get_rx_frame_len(priv, p, coe);
3506 
3507 	/* First descriptor and last descriptor and not split header */
3508 	return min_t(unsigned int, priv->dma_buf_sz, plen);
3509 }
3510 
3511 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
3512 				       struct dma_desc *p,
3513 				       int status, unsigned int len)
3514 {
3515 	int coe = priv->hw->rx_csum;
3516 	unsigned int plen = 0;
3517 
3518 	/* Not split header, buffer is not available */
3519 	if (!priv->sph)
3520 		return 0;
3521 
3522 	/* Not last descriptor */
3523 	if (status & rx_not_ls)
3524 		return priv->dma_buf_sz;
3525 
3526 	plen = stmmac_get_rx_frame_len(priv, p, coe);
3527 
3528 	/* Last descriptor */
3529 	return plen - len;
3530 }
3531 
3532 /**
3533  * stmmac_rx - manage the receive process
3534  * @priv: driver private structure
3535  * @limit: napi bugget
3536  * @queue: RX queue index.
3537  * Description :  this the function called by the napi poll method.
3538  * It gets all the frames inside the ring.
3539  */
3540 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3541 {
3542 	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3543 	struct stmmac_channel *ch = &priv->channel[queue];
3544 	unsigned int count = 0, error = 0, len = 0;
3545 	int status = 0, coe = priv->hw->rx_csum;
3546 	unsigned int next_entry = rx_q->cur_rx;
3547 	struct sk_buff *skb = NULL;
3548 
3549 	if (netif_msg_rx_status(priv)) {
3550 		void *rx_head;
3551 
3552 		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3553 		if (priv->extend_desc)
3554 			rx_head = (void *)rx_q->dma_erx;
3555 		else
3556 			rx_head = (void *)rx_q->dma_rx;
3557 
3558 		stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true);
3559 	}
3560 	while (count < limit) {
3561 		unsigned int buf1_len = 0, buf2_len = 0;
3562 		enum pkt_hash_types hash_type;
3563 		struct stmmac_rx_buffer *buf;
3564 		struct dma_desc *np, *p;
3565 		int entry;
3566 		u32 hash;
3567 
3568 		if (!count && rx_q->state_saved) {
3569 			skb = rx_q->state.skb;
3570 			error = rx_q->state.error;
3571 			len = rx_q->state.len;
3572 		} else {
3573 			rx_q->state_saved = false;
3574 			skb = NULL;
3575 			error = 0;
3576 			len = 0;
3577 		}
3578 
3579 		if (count >= limit)
3580 			break;
3581 
3582 read_again:
3583 		buf1_len = 0;
3584 		buf2_len = 0;
3585 		entry = next_entry;
3586 		buf = &rx_q->buf_pool[entry];
3587 
3588 		if (priv->extend_desc)
3589 			p = (struct dma_desc *)(rx_q->dma_erx + entry);
3590 		else
3591 			p = rx_q->dma_rx + entry;
3592 
3593 		/* read the status of the incoming frame */
3594 		status = stmmac_rx_status(priv, &priv->dev->stats,
3595 				&priv->xstats, p);
3596 		/* check if managed by the DMA otherwise go ahead */
3597 		if (unlikely(status & dma_own))
3598 			break;
3599 
3600 		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3601 		next_entry = rx_q->cur_rx;
3602 
3603 		if (priv->extend_desc)
3604 			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3605 		else
3606 			np = rx_q->dma_rx + next_entry;
3607 
3608 		prefetch(np);
3609 
3610 		if (priv->extend_desc)
3611 			stmmac_rx_extended_status(priv, &priv->dev->stats,
3612 					&priv->xstats, rx_q->dma_erx + entry);
3613 		if (unlikely(status == discard_frame)) {
3614 			page_pool_recycle_direct(rx_q->page_pool, buf->page);
3615 			buf->page = NULL;
3616 			error = 1;
3617 			if (!priv->hwts_rx_en)
3618 				priv->dev->stats.rx_errors++;
3619 		}
3620 
3621 		if (unlikely(error && (status & rx_not_ls)))
3622 			goto read_again;
3623 		if (unlikely(error)) {
3624 			dev_kfree_skb(skb);
3625 			skb = NULL;
3626 			count++;
3627 			continue;
3628 		}
3629 
3630 		/* Buffer is good. Go on. */
3631 
3632 		prefetch(page_address(buf->page));
3633 		if (buf->sec_page)
3634 			prefetch(page_address(buf->sec_page));
3635 
3636 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
3637 		len += buf1_len;
3638 		buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
3639 		len += buf2_len;
3640 
3641 		/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3642 		 * Type frames (LLC/LLC-SNAP)
3643 		 *
3644 		 * llc_snap is never checked in GMAC >= 4, so this ACS
3645 		 * feature is always disabled and packets need to be
3646 		 * stripped manually.
3647 		 */
3648 		if (likely(!(status & rx_not_ls)) &&
3649 		    (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
3650 		     unlikely(status != llc_snap))) {
3651 			if (buf2_len)
3652 				buf2_len -= ETH_FCS_LEN;
3653 			else
3654 				buf1_len -= ETH_FCS_LEN;
3655 
3656 			len -= ETH_FCS_LEN;
3657 		}
3658 
3659 		if (!skb) {
3660 			skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
3661 			if (!skb) {
3662 				priv->dev->stats.rx_dropped++;
3663 				count++;
3664 				goto drain_data;
3665 			}
3666 
3667 			dma_sync_single_for_cpu(priv->device, buf->addr,
3668 						buf1_len, DMA_FROM_DEVICE);
3669 			skb_copy_to_linear_data(skb, page_address(buf->page),
3670 						buf1_len);
3671 			skb_put(skb, buf1_len);
3672 
3673 			/* Data payload copied into SKB, page ready for recycle */
3674 			page_pool_recycle_direct(rx_q->page_pool, buf->page);
3675 			buf->page = NULL;
3676 		} else if (buf1_len) {
3677 			dma_sync_single_for_cpu(priv->device, buf->addr,
3678 						buf1_len, DMA_FROM_DEVICE);
3679 			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
3680 					buf->page, 0, buf1_len,
3681 					priv->dma_buf_sz);
3682 
3683 			/* Data payload appended into SKB */
3684 			page_pool_release_page(rx_q->page_pool, buf->page);
3685 			buf->page = NULL;
3686 		}
3687 
3688 		if (buf2_len) {
3689 			dma_sync_single_for_cpu(priv->device, buf->sec_addr,
3690 						buf2_len, DMA_FROM_DEVICE);
3691 			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
3692 					buf->sec_page, 0, buf2_len,
3693 					priv->dma_buf_sz);
3694 
3695 			/* Data payload appended into SKB */
3696 			page_pool_release_page(rx_q->page_pool, buf->sec_page);
3697 			buf->sec_page = NULL;
3698 		}
3699 
3700 drain_data:
3701 		if (likely(status & rx_not_ls))
3702 			goto read_again;
3703 		if (!skb)
3704 			continue;
3705 
3706 		/* Got entire packet into SKB. Finish it. */
3707 
3708 		stmmac_get_rx_hwtstamp(priv, p, np, skb);
3709 		stmmac_rx_vlan(priv->dev, skb);
3710 		skb->protocol = eth_type_trans(skb, priv->dev);
3711 
3712 		if (unlikely(!coe))
3713 			skb_checksum_none_assert(skb);
3714 		else
3715 			skb->ip_summed = CHECKSUM_UNNECESSARY;
3716 
3717 		if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
3718 			skb_set_hash(skb, hash, hash_type);
3719 
3720 		skb_record_rx_queue(skb, queue);
3721 		napi_gro_receive(&ch->rx_napi, skb);
3722 		skb = NULL;
3723 
3724 		priv->dev->stats.rx_packets++;
3725 		priv->dev->stats.rx_bytes += len;
3726 		count++;
3727 	}
3728 
3729 	if (status & rx_not_ls || skb) {
3730 		rx_q->state_saved = true;
3731 		rx_q->state.skb = skb;
3732 		rx_q->state.error = error;
3733 		rx_q->state.len = len;
3734 	}
3735 
3736 	stmmac_rx_refill(priv, queue);
3737 
3738 	priv->xstats.rx_pkt_n += count;
3739 
3740 	return count;
3741 }
3742 
3743 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
3744 {
3745 	struct stmmac_channel *ch =
3746 		container_of(napi, struct stmmac_channel, rx_napi);
3747 	struct stmmac_priv *priv = ch->priv_data;
3748 	u32 chan = ch->index;
3749 	int work_done;
3750 
3751 	priv->xstats.napi_poll++;
3752 
3753 	work_done = stmmac_rx(priv, budget, chan);
3754 	if (work_done < budget && napi_complete_done(napi, work_done))
3755 		stmmac_enable_dma_irq(priv, priv->ioaddr, chan);
3756 	return work_done;
3757 }
3758 
3759 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
3760 {
3761 	struct stmmac_channel *ch =
3762 		container_of(napi, struct stmmac_channel, tx_napi);
3763 	struct stmmac_priv *priv = ch->priv_data;
3764 	struct stmmac_tx_queue *tx_q;
3765 	u32 chan = ch->index;
3766 	int work_done;
3767 
3768 	priv->xstats.napi_poll++;
3769 
3770 	work_done = stmmac_tx_clean(priv, DMA_TX_SIZE, chan);
3771 	work_done = min(work_done, budget);
3772 
3773 	if (work_done < budget)
3774 		napi_complete_done(napi, work_done);
3775 
3776 	/* Force transmission restart */
3777 	tx_q = &priv->tx_queue[chan];
3778 	if (tx_q->cur_tx != tx_q->dirty_tx) {
3779 		stmmac_enable_dma_transmission(priv, priv->ioaddr);
3780 		stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr,
3781 				       chan);
3782 	}
3783 
3784 	return work_done;
3785 }
3786 
3787 /**
3788  *  stmmac_tx_timeout
3789  *  @dev : Pointer to net device structure
3790  *  Description: this function is called when a packet transmission fails to
3791  *   complete within a reasonable time. The driver will mark the error in the
3792  *   netdev structure and arrange for the device to be reset to a sane state
3793  *   in order to transmit a new packet.
3794  */
3795 static void stmmac_tx_timeout(struct net_device *dev)
3796 {
3797 	struct stmmac_priv *priv = netdev_priv(dev);
3798 
3799 	stmmac_global_err(priv);
3800 }
3801 
3802 /**
3803  *  stmmac_set_rx_mode - entry point for multicast addressing
3804  *  @dev : pointer to the device structure
3805  *  Description:
3806  *  This function is a driver entry point which gets called by the kernel
3807  *  whenever multicast addresses must be enabled/disabled.
3808  *  Return value:
3809  *  void.
3810  */
3811 static void stmmac_set_rx_mode(struct net_device *dev)
3812 {
3813 	struct stmmac_priv *priv = netdev_priv(dev);
3814 
3815 	stmmac_set_filter(priv, priv->hw, dev);
3816 }
3817 
3818 /**
3819  *  stmmac_change_mtu - entry point to change MTU size for the device.
3820  *  @dev : device pointer.
3821  *  @new_mtu : the new MTU size for the device.
3822  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3823  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3824  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3825  *  Return value:
3826  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3827  *  file on failure.
3828  */
3829 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3830 {
3831 	struct stmmac_priv *priv = netdev_priv(dev);
3832 	int txfifosz = priv->plat->tx_fifo_size;
3833 
3834 	if (txfifosz == 0)
3835 		txfifosz = priv->dma_cap.tx_fifo_size;
3836 
3837 	txfifosz /= priv->plat->tx_queues_to_use;
3838 
3839 	if (netif_running(dev)) {
3840 		netdev_err(priv->dev, "must be stopped to change its MTU\n");
3841 		return -EBUSY;
3842 	}
3843 
3844 	new_mtu = STMMAC_ALIGN(new_mtu);
3845 
3846 	/* If condition true, FIFO is too small or MTU too large */
3847 	if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
3848 		return -EINVAL;
3849 
3850 	dev->mtu = new_mtu;
3851 
3852 	netdev_update_features(dev);
3853 
3854 	return 0;
3855 }
3856 
3857 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3858 					     netdev_features_t features)
3859 {
3860 	struct stmmac_priv *priv = netdev_priv(dev);
3861 
3862 	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3863 		features &= ~NETIF_F_RXCSUM;
3864 
3865 	if (!priv->plat->tx_coe)
3866 		features &= ~NETIF_F_CSUM_MASK;
3867 
3868 	/* Some GMAC devices have a bugged Jumbo frame support that
3869 	 * needs to have the Tx COE disabled for oversized frames
3870 	 * (due to limited buffer sizes). In this case we disable
3871 	 * the TX csum insertion in the TDES and not use SF.
3872 	 */
3873 	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3874 		features &= ~NETIF_F_CSUM_MASK;
3875 
3876 	/* Disable tso if asked by ethtool */
3877 	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3878 		if (features & NETIF_F_TSO)
3879 			priv->tso = true;
3880 		else
3881 			priv->tso = false;
3882 	}
3883 
3884 	return features;
3885 }
3886 
3887 static int stmmac_set_features(struct net_device *netdev,
3888 			       netdev_features_t features)
3889 {
3890 	struct stmmac_priv *priv = netdev_priv(netdev);
3891 	bool sph_en;
3892 	u32 chan;
3893 
3894 	/* Keep the COE Type in case of csum is supporting */
3895 	if (features & NETIF_F_RXCSUM)
3896 		priv->hw->rx_csum = priv->plat->rx_coe;
3897 	else
3898 		priv->hw->rx_csum = 0;
3899 	/* No check needed because rx_coe has been set before and it will be
3900 	 * fixed in case of issue.
3901 	 */
3902 	stmmac_rx_ipc(priv, priv->hw);
3903 
3904 	sph_en = (priv->hw->rx_csum > 0) && priv->sph;
3905 	for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
3906 		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
3907 
3908 	return 0;
3909 }
3910 
3911 /**
3912  *  stmmac_interrupt - main ISR
3913  *  @irq: interrupt number.
3914  *  @dev_id: to pass the net device pointer.
3915  *  Description: this is the main driver interrupt service routine.
3916  *  It can call:
3917  *  o DMA service routine (to manage incoming frame reception and transmission
3918  *    status)
3919  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3920  *    interrupts.
3921  */
3922 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3923 {
3924 	struct net_device *dev = (struct net_device *)dev_id;
3925 	struct stmmac_priv *priv = netdev_priv(dev);
3926 	u32 rx_cnt = priv->plat->rx_queues_to_use;
3927 	u32 tx_cnt = priv->plat->tx_queues_to_use;
3928 	u32 queues_count;
3929 	u32 queue;
3930 	bool xmac;
3931 
3932 	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
3933 	queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3934 
3935 	if (priv->irq_wake)
3936 		pm_wakeup_event(priv->device, 0);
3937 
3938 	if (unlikely(!dev)) {
3939 		netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3940 		return IRQ_NONE;
3941 	}
3942 
3943 	/* Check if adapter is up */
3944 	if (test_bit(STMMAC_DOWN, &priv->state))
3945 		return IRQ_HANDLED;
3946 	/* Check if a fatal error happened */
3947 	if (stmmac_safety_feat_interrupt(priv))
3948 		return IRQ_HANDLED;
3949 
3950 	/* To handle GMAC own interrupts */
3951 	if ((priv->plat->has_gmac) || xmac) {
3952 		int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
3953 		int mtl_status;
3954 
3955 		if (unlikely(status)) {
3956 			/* For LPI we need to save the tx status */
3957 			if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3958 				priv->tx_path_in_lpi_mode = true;
3959 			if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3960 				priv->tx_path_in_lpi_mode = false;
3961 		}
3962 
3963 		for (queue = 0; queue < queues_count; queue++) {
3964 			struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3965 
3966 			mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw,
3967 								queue);
3968 			if (mtl_status != -EINVAL)
3969 				status |= mtl_status;
3970 
3971 			if (status & CORE_IRQ_MTL_RX_OVERFLOW)
3972 				stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
3973 						       rx_q->rx_tail_addr,
3974 						       queue);
3975 		}
3976 
3977 		/* PCS link status */
3978 		if (priv->hw->pcs) {
3979 			if (priv->xstats.pcs_link)
3980 				netif_carrier_on(dev);
3981 			else
3982 				netif_carrier_off(dev);
3983 		}
3984 	}
3985 
3986 	/* To handle DMA interrupts */
3987 	stmmac_dma_interrupt(priv);
3988 
3989 	return IRQ_HANDLED;
3990 }
3991 
3992 #ifdef CONFIG_NET_POLL_CONTROLLER
3993 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3994  * to allow network I/O with interrupts disabled.
3995  */
3996 static void stmmac_poll_controller(struct net_device *dev)
3997 {
3998 	disable_irq(dev->irq);
3999 	stmmac_interrupt(dev->irq, dev);
4000 	enable_irq(dev->irq);
4001 }
4002 #endif
4003 
4004 /**
4005  *  stmmac_ioctl - Entry point for the Ioctl
4006  *  @dev: Device pointer.
4007  *  @rq: An IOCTL specefic structure, that can contain a pointer to
4008  *  a proprietary structure used to pass information to the driver.
4009  *  @cmd: IOCTL command
4010  *  Description:
4011  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
4012  */
4013 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
4014 {
4015 	struct stmmac_priv *priv = netdev_priv (dev);
4016 	int ret = -EOPNOTSUPP;
4017 
4018 	if (!netif_running(dev))
4019 		return -EINVAL;
4020 
4021 	switch (cmd) {
4022 	case SIOCGMIIPHY:
4023 	case SIOCGMIIREG:
4024 	case SIOCSMIIREG:
4025 		ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
4026 		break;
4027 	case SIOCSHWTSTAMP:
4028 		ret = stmmac_hwtstamp_set(dev, rq);
4029 		break;
4030 	case SIOCGHWTSTAMP:
4031 		ret = stmmac_hwtstamp_get(dev, rq);
4032 		break;
4033 	default:
4034 		break;
4035 	}
4036 
4037 	return ret;
4038 }
4039 
4040 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4041 				    void *cb_priv)
4042 {
4043 	struct stmmac_priv *priv = cb_priv;
4044 	int ret = -EOPNOTSUPP;
4045 
4046 	if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
4047 		return ret;
4048 
4049 	stmmac_disable_all_queues(priv);
4050 
4051 	switch (type) {
4052 	case TC_SETUP_CLSU32:
4053 		ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
4054 		break;
4055 	case TC_SETUP_CLSFLOWER:
4056 		ret = stmmac_tc_setup_cls(priv, priv, type_data);
4057 		break;
4058 	default:
4059 		break;
4060 	}
4061 
4062 	stmmac_enable_all_queues(priv);
4063 	return ret;
4064 }
4065 
4066 static LIST_HEAD(stmmac_block_cb_list);
4067 
4068 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
4069 			   void *type_data)
4070 {
4071 	struct stmmac_priv *priv = netdev_priv(ndev);
4072 
4073 	switch (type) {
4074 	case TC_SETUP_BLOCK:
4075 		return flow_block_cb_setup_simple(type_data,
4076 						  &stmmac_block_cb_list,
4077 						  stmmac_setup_tc_block_cb,
4078 						  priv, priv, true);
4079 	case TC_SETUP_QDISC_CBS:
4080 		return stmmac_tc_setup_cbs(priv, priv, type_data);
4081 	default:
4082 		return -EOPNOTSUPP;
4083 	}
4084 }
4085 
4086 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
4087 			       struct net_device *sb_dev)
4088 {
4089 	int gso = skb_shinfo(skb)->gso_type;
4090 
4091 	if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
4092 		/*
4093 		 * There is no way to determine the number of TSO/USO
4094 		 * capable Queues. Let's use always the Queue 0
4095 		 * because if TSO/USO is supported then at least this
4096 		 * one will be capable.
4097 		 */
4098 		return 0;
4099 	}
4100 
4101 	return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
4102 }
4103 
4104 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
4105 {
4106 	struct stmmac_priv *priv = netdev_priv(ndev);
4107 	int ret = 0;
4108 
4109 	ret = eth_mac_addr(ndev, addr);
4110 	if (ret)
4111 		return ret;
4112 
4113 	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
4114 
4115 	return ret;
4116 }
4117 
4118 #ifdef CONFIG_DEBUG_FS
4119 static struct dentry *stmmac_fs_dir;
4120 
4121 static void sysfs_display_ring(void *head, int size, int extend_desc,
4122 			       struct seq_file *seq)
4123 {
4124 	int i;
4125 	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
4126 	struct dma_desc *p = (struct dma_desc *)head;
4127 
4128 	for (i = 0; i < size; i++) {
4129 		if (extend_desc) {
4130 			seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
4131 				   i, (unsigned int)virt_to_phys(ep),
4132 				   le32_to_cpu(ep->basic.des0),
4133 				   le32_to_cpu(ep->basic.des1),
4134 				   le32_to_cpu(ep->basic.des2),
4135 				   le32_to_cpu(ep->basic.des3));
4136 			ep++;
4137 		} else {
4138 			seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
4139 				   i, (unsigned int)virt_to_phys(p),
4140 				   le32_to_cpu(p->des0), le32_to_cpu(p->des1),
4141 				   le32_to_cpu(p->des2), le32_to_cpu(p->des3));
4142 			p++;
4143 		}
4144 		seq_printf(seq, "\n");
4145 	}
4146 }
4147 
4148 static int stmmac_rings_status_show(struct seq_file *seq, void *v)
4149 {
4150 	struct net_device *dev = seq->private;
4151 	struct stmmac_priv *priv = netdev_priv(dev);
4152 	u32 rx_count = priv->plat->rx_queues_to_use;
4153 	u32 tx_count = priv->plat->tx_queues_to_use;
4154 	u32 queue;
4155 
4156 	if ((dev->flags & IFF_UP) == 0)
4157 		return 0;
4158 
4159 	for (queue = 0; queue < rx_count; queue++) {
4160 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4161 
4162 		seq_printf(seq, "RX Queue %d:\n", queue);
4163 
4164 		if (priv->extend_desc) {
4165 			seq_printf(seq, "Extended descriptor ring:\n");
4166 			sysfs_display_ring((void *)rx_q->dma_erx,
4167 					   DMA_RX_SIZE, 1, seq);
4168 		} else {
4169 			seq_printf(seq, "Descriptor ring:\n");
4170 			sysfs_display_ring((void *)rx_q->dma_rx,
4171 					   DMA_RX_SIZE, 0, seq);
4172 		}
4173 	}
4174 
4175 	for (queue = 0; queue < tx_count; queue++) {
4176 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4177 
4178 		seq_printf(seq, "TX Queue %d:\n", queue);
4179 
4180 		if (priv->extend_desc) {
4181 			seq_printf(seq, "Extended descriptor ring:\n");
4182 			sysfs_display_ring((void *)tx_q->dma_etx,
4183 					   DMA_TX_SIZE, 1, seq);
4184 		} else {
4185 			seq_printf(seq, "Descriptor ring:\n");
4186 			sysfs_display_ring((void *)tx_q->dma_tx,
4187 					   DMA_TX_SIZE, 0, seq);
4188 		}
4189 	}
4190 
4191 	return 0;
4192 }
4193 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
4194 
4195 static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
4196 {
4197 	struct net_device *dev = seq->private;
4198 	struct stmmac_priv *priv = netdev_priv(dev);
4199 
4200 	if (!priv->hw_cap_support) {
4201 		seq_printf(seq, "DMA HW features not supported\n");
4202 		return 0;
4203 	}
4204 
4205 	seq_printf(seq, "==============================\n");
4206 	seq_printf(seq, "\tDMA HW features\n");
4207 	seq_printf(seq, "==============================\n");
4208 
4209 	seq_printf(seq, "\t10/100 Mbps: %s\n",
4210 		   (priv->dma_cap.mbps_10_100) ? "Y" : "N");
4211 	seq_printf(seq, "\t1000 Mbps: %s\n",
4212 		   (priv->dma_cap.mbps_1000) ? "Y" : "N");
4213 	seq_printf(seq, "\tHalf duplex: %s\n",
4214 		   (priv->dma_cap.half_duplex) ? "Y" : "N");
4215 	seq_printf(seq, "\tHash Filter: %s\n",
4216 		   (priv->dma_cap.hash_filter) ? "Y" : "N");
4217 	seq_printf(seq, "\tMultiple MAC address registers: %s\n",
4218 		   (priv->dma_cap.multi_addr) ? "Y" : "N");
4219 	seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
4220 		   (priv->dma_cap.pcs) ? "Y" : "N");
4221 	seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
4222 		   (priv->dma_cap.sma_mdio) ? "Y" : "N");
4223 	seq_printf(seq, "\tPMT Remote wake up: %s\n",
4224 		   (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
4225 	seq_printf(seq, "\tPMT Magic Frame: %s\n",
4226 		   (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
4227 	seq_printf(seq, "\tRMON module: %s\n",
4228 		   (priv->dma_cap.rmon) ? "Y" : "N");
4229 	seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
4230 		   (priv->dma_cap.time_stamp) ? "Y" : "N");
4231 	seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
4232 		   (priv->dma_cap.atime_stamp) ? "Y" : "N");
4233 	seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
4234 		   (priv->dma_cap.eee) ? "Y" : "N");
4235 	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
4236 	seq_printf(seq, "\tChecksum Offload in TX: %s\n",
4237 		   (priv->dma_cap.tx_coe) ? "Y" : "N");
4238 	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
4239 		seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
4240 			   (priv->dma_cap.rx_coe) ? "Y" : "N");
4241 	} else {
4242 		seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
4243 			   (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
4244 		seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
4245 			   (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
4246 	}
4247 	seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
4248 		   (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
4249 	seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
4250 		   priv->dma_cap.number_rx_channel);
4251 	seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
4252 		   priv->dma_cap.number_tx_channel);
4253 	seq_printf(seq, "\tEnhanced descriptors: %s\n",
4254 		   (priv->dma_cap.enh_desc) ? "Y" : "N");
4255 
4256 	return 0;
4257 }
4258 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
4259 
4260 /* Use network device events to rename debugfs file entries.
4261  */
4262 static int stmmac_device_event(struct notifier_block *unused,
4263 			       unsigned long event, void *ptr)
4264 {
4265 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4266 	struct stmmac_priv *priv = netdev_priv(dev);
4267 
4268 	if (dev->netdev_ops != &stmmac_netdev_ops)
4269 		goto done;
4270 
4271 	switch (event) {
4272 	case NETDEV_CHANGENAME:
4273 		if (priv->dbgfs_dir)
4274 			priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,
4275 							 priv->dbgfs_dir,
4276 							 stmmac_fs_dir,
4277 							 dev->name);
4278 		break;
4279 	}
4280 done:
4281 	return NOTIFY_DONE;
4282 }
4283 
4284 static struct notifier_block stmmac_notifier = {
4285 	.notifier_call = stmmac_device_event,
4286 };
4287 
4288 static void stmmac_init_fs(struct net_device *dev)
4289 {
4290 	struct stmmac_priv *priv = netdev_priv(dev);
4291 
4292 	/* Create per netdev entries */
4293 	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
4294 
4295 	/* Entry to report DMA RX/TX rings */
4296 	debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
4297 			    &stmmac_rings_status_fops);
4298 
4299 	/* Entry to report the DMA HW features */
4300 	debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
4301 			    &stmmac_dma_cap_fops);
4302 
4303 	register_netdevice_notifier(&stmmac_notifier);
4304 }
4305 
4306 static void stmmac_exit_fs(struct net_device *dev)
4307 {
4308 	struct stmmac_priv *priv = netdev_priv(dev);
4309 
4310 	unregister_netdevice_notifier(&stmmac_notifier);
4311 	debugfs_remove_recursive(priv->dbgfs_dir);
4312 }
4313 #endif /* CONFIG_DEBUG_FS */
4314 
4315 static u32 stmmac_vid_crc32_le(__le16 vid_le)
4316 {
4317 	unsigned char *data = (unsigned char *)&vid_le;
4318 	unsigned char data_byte = 0;
4319 	u32 crc = ~0x0;
4320 	u32 temp = 0;
4321 	int i, bits;
4322 
4323 	bits = get_bitmask_order(VLAN_VID_MASK);
4324 	for (i = 0; i < bits; i++) {
4325 		if ((i % 8) == 0)
4326 			data_byte = data[i / 8];
4327 
4328 		temp = ((crc & 1) ^ data_byte) & 1;
4329 		crc >>= 1;
4330 		data_byte >>= 1;
4331 
4332 		if (temp)
4333 			crc ^= 0xedb88320;
4334 	}
4335 
4336 	return crc;
4337 }
4338 
4339 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
4340 {
4341 	u32 crc, hash = 0;
4342 	__le16 pmatch = 0;
4343 	int count = 0;
4344 	u16 vid = 0;
4345 
4346 	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
4347 		__le16 vid_le = cpu_to_le16(vid);
4348 		crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
4349 		hash |= (1 << crc);
4350 		count++;
4351 	}
4352 
4353 	if (!priv->dma_cap.vlhash) {
4354 		if (count > 2) /* VID = 0 always passes filter */
4355 			return -EOPNOTSUPP;
4356 
4357 		pmatch = cpu_to_le16(vid);
4358 		hash = 0;
4359 	}
4360 
4361 	return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
4362 }
4363 
4364 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
4365 {
4366 	struct stmmac_priv *priv = netdev_priv(ndev);
4367 	bool is_double = false;
4368 	int ret;
4369 
4370 	if (be16_to_cpu(proto) == ETH_P_8021AD)
4371 		is_double = true;
4372 
4373 	set_bit(vid, priv->active_vlans);
4374 	ret = stmmac_vlan_update(priv, is_double);
4375 	if (ret) {
4376 		clear_bit(vid, priv->active_vlans);
4377 		return ret;
4378 	}
4379 
4380 	return ret;
4381 }
4382 
4383 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
4384 {
4385 	struct stmmac_priv *priv = netdev_priv(ndev);
4386 	bool is_double = false;
4387 
4388 	if (be16_to_cpu(proto) == ETH_P_8021AD)
4389 		is_double = true;
4390 
4391 	clear_bit(vid, priv->active_vlans);
4392 	return stmmac_vlan_update(priv, is_double);
4393 }
4394 
4395 static const struct net_device_ops stmmac_netdev_ops = {
4396 	.ndo_open = stmmac_open,
4397 	.ndo_start_xmit = stmmac_xmit,
4398 	.ndo_stop = stmmac_release,
4399 	.ndo_change_mtu = stmmac_change_mtu,
4400 	.ndo_fix_features = stmmac_fix_features,
4401 	.ndo_set_features = stmmac_set_features,
4402 	.ndo_set_rx_mode = stmmac_set_rx_mode,
4403 	.ndo_tx_timeout = stmmac_tx_timeout,
4404 	.ndo_do_ioctl = stmmac_ioctl,
4405 	.ndo_setup_tc = stmmac_setup_tc,
4406 	.ndo_select_queue = stmmac_select_queue,
4407 #ifdef CONFIG_NET_POLL_CONTROLLER
4408 	.ndo_poll_controller = stmmac_poll_controller,
4409 #endif
4410 	.ndo_set_mac_address = stmmac_set_mac_address,
4411 	.ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
4412 	.ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
4413 };
4414 
4415 static void stmmac_reset_subtask(struct stmmac_priv *priv)
4416 {
4417 	if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
4418 		return;
4419 	if (test_bit(STMMAC_DOWN, &priv->state))
4420 		return;
4421 
4422 	netdev_err(priv->dev, "Reset adapter.\n");
4423 
4424 	rtnl_lock();
4425 	netif_trans_update(priv->dev);
4426 	while (test_and_set_bit(STMMAC_RESETING, &priv->state))
4427 		usleep_range(1000, 2000);
4428 
4429 	set_bit(STMMAC_DOWN, &priv->state);
4430 	dev_close(priv->dev);
4431 	dev_open(priv->dev, NULL);
4432 	clear_bit(STMMAC_DOWN, &priv->state);
4433 	clear_bit(STMMAC_RESETING, &priv->state);
4434 	rtnl_unlock();
4435 }
4436 
4437 static void stmmac_service_task(struct work_struct *work)
4438 {
4439 	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
4440 			service_task);
4441 
4442 	stmmac_reset_subtask(priv);
4443 	clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
4444 }
4445 
4446 /**
4447  *  stmmac_hw_init - Init the MAC device
4448  *  @priv: driver private structure
4449  *  Description: this function is to configure the MAC device according to
4450  *  some platform parameters or the HW capability register. It prepares the
4451  *  driver to use either ring or chain modes and to setup either enhanced or
4452  *  normal descriptors.
4453  */
4454 static int stmmac_hw_init(struct stmmac_priv *priv)
4455 {
4456 	int ret;
4457 
4458 	/* dwmac-sun8i only work in chain mode */
4459 	if (priv->plat->has_sun8i)
4460 		chain_mode = 1;
4461 	priv->chain_mode = chain_mode;
4462 
4463 	/* Initialize HW Interface */
4464 	ret = stmmac_hwif_init(priv);
4465 	if (ret)
4466 		return ret;
4467 
4468 	/* Get the HW capability (new GMAC newer than 3.50a) */
4469 	priv->hw_cap_support = stmmac_get_hw_features(priv);
4470 	if (priv->hw_cap_support) {
4471 		dev_info(priv->device, "DMA HW capability register supported\n");
4472 
4473 		/* We can override some gmac/dma configuration fields: e.g.
4474 		 * enh_desc, tx_coe (e.g. that are passed through the
4475 		 * platform) with the values from the HW capability
4476 		 * register (if supported).
4477 		 */
4478 		priv->plat->enh_desc = priv->dma_cap.enh_desc;
4479 		priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4480 		priv->hw->pmt = priv->plat->pmt;
4481 		if (priv->dma_cap.hash_tb_sz) {
4482 			priv->hw->multicast_filter_bins =
4483 					(BIT(priv->dma_cap.hash_tb_sz) << 5);
4484 			priv->hw->mcast_bits_log2 =
4485 					ilog2(priv->hw->multicast_filter_bins);
4486 		}
4487 
4488 		/* TXCOE doesn't work in thresh DMA mode */
4489 		if (priv->plat->force_thresh_dma_mode)
4490 			priv->plat->tx_coe = 0;
4491 		else
4492 			priv->plat->tx_coe = priv->dma_cap.tx_coe;
4493 
4494 		/* In case of GMAC4 rx_coe is from HW cap register. */
4495 		priv->plat->rx_coe = priv->dma_cap.rx_coe;
4496 
4497 		if (priv->dma_cap.rx_coe_type2)
4498 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4499 		else if (priv->dma_cap.rx_coe_type1)
4500 			priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4501 
4502 	} else {
4503 		dev_info(priv->device, "No HW DMA feature register supported\n");
4504 	}
4505 
4506 	if (priv->plat->rx_coe) {
4507 		priv->hw->rx_csum = priv->plat->rx_coe;
4508 		dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4509 		if (priv->synopsys_id < DWMAC_CORE_4_00)
4510 			dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4511 	}
4512 	if (priv->plat->tx_coe)
4513 		dev_info(priv->device, "TX Checksum insertion supported\n");
4514 
4515 	if (priv->plat->pmt) {
4516 		dev_info(priv->device, "Wake-Up On Lan supported\n");
4517 		device_set_wakeup_capable(priv->device, 1);
4518 	}
4519 
4520 	if (priv->dma_cap.tsoen)
4521 		dev_info(priv->device, "TSO supported\n");
4522 
4523 	/* Run HW quirks, if any */
4524 	if (priv->hwif_quirks) {
4525 		ret = priv->hwif_quirks(priv);
4526 		if (ret)
4527 			return ret;
4528 	}
4529 
4530 	/* Rx Watchdog is available in the COREs newer than the 3.40.
4531 	 * In some case, for example on bugged HW this feature
4532 	 * has to be disable and this can be done by passing the
4533 	 * riwt_off field from the platform.
4534 	 */
4535 	if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
4536 	    (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
4537 		priv->use_riwt = 1;
4538 		dev_info(priv->device,
4539 			 "Enable RX Mitigation via HW Watchdog Timer\n");
4540 	}
4541 
4542 	return 0;
4543 }
4544 
4545 /**
4546  * stmmac_dvr_probe
4547  * @device: device pointer
4548  * @plat_dat: platform data pointer
4549  * @res: stmmac resource pointer
4550  * Description: this is the main probe function used to
4551  * call the alloc_etherdev, allocate the priv structure.
4552  * Return:
4553  * returns 0 on success, otherwise errno.
4554  */
4555 int stmmac_dvr_probe(struct device *device,
4556 		     struct plat_stmmacenet_data *plat_dat,
4557 		     struct stmmac_resources *res)
4558 {
4559 	struct net_device *ndev = NULL;
4560 	struct stmmac_priv *priv;
4561 	u32 queue, rxq, maxq;
4562 	int i, ret = 0;
4563 
4564 	ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
4565 				       MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
4566 	if (!ndev)
4567 		return -ENOMEM;
4568 
4569 	SET_NETDEV_DEV(ndev, device);
4570 
4571 	priv = netdev_priv(ndev);
4572 	priv->device = device;
4573 	priv->dev = ndev;
4574 
4575 	stmmac_set_ethtool_ops(ndev);
4576 	priv->pause = pause;
4577 	priv->plat = plat_dat;
4578 	priv->ioaddr = res->addr;
4579 	priv->dev->base_addr = (unsigned long)res->addr;
4580 
4581 	priv->dev->irq = res->irq;
4582 	priv->wol_irq = res->wol_irq;
4583 	priv->lpi_irq = res->lpi_irq;
4584 
4585 	if (!IS_ERR_OR_NULL(res->mac))
4586 		memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4587 
4588 	dev_set_drvdata(device, priv->dev);
4589 
4590 	/* Verify driver arguments */
4591 	stmmac_verify_args();
4592 
4593 	/* Allocate workqueue */
4594 	priv->wq = create_singlethread_workqueue("stmmac_wq");
4595 	if (!priv->wq) {
4596 		dev_err(priv->device, "failed to create workqueue\n");
4597 		return -ENOMEM;
4598 	}
4599 
4600 	INIT_WORK(&priv->service_task, stmmac_service_task);
4601 
4602 	/* Override with kernel parameters if supplied XXX CRS XXX
4603 	 * this needs to have multiple instances
4604 	 */
4605 	if ((phyaddr >= 0) && (phyaddr <= 31))
4606 		priv->plat->phy_addr = phyaddr;
4607 
4608 	if (priv->plat->stmmac_rst) {
4609 		ret = reset_control_assert(priv->plat->stmmac_rst);
4610 		reset_control_deassert(priv->plat->stmmac_rst);
4611 		/* Some reset controllers have only reset callback instead of
4612 		 * assert + deassert callbacks pair.
4613 		 */
4614 		if (ret == -ENOTSUPP)
4615 			reset_control_reset(priv->plat->stmmac_rst);
4616 	}
4617 
4618 	/* Init MAC and get the capabilities */
4619 	ret = stmmac_hw_init(priv);
4620 	if (ret)
4621 		goto error_hw_init;
4622 
4623 	stmmac_check_ether_addr(priv);
4624 
4625 	/* Configure real RX and TX queues */
4626 	netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4627 	netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4628 
4629 	ndev->netdev_ops = &stmmac_netdev_ops;
4630 
4631 	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4632 			    NETIF_F_RXCSUM;
4633 
4634 	ret = stmmac_tc_init(priv, priv);
4635 	if (!ret) {
4636 		ndev->hw_features |= NETIF_F_HW_TC;
4637 	}
4638 
4639 	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4640 		ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
4641 		if (priv->plat->has_gmac4)
4642 			ndev->hw_features |= NETIF_F_GSO_UDP_L4;
4643 		priv->tso = true;
4644 		dev_info(priv->device, "TSO feature enabled\n");
4645 	}
4646 
4647 	if (priv->dma_cap.sphen) {
4648 		ndev->hw_features |= NETIF_F_GRO;
4649 		priv->sph = true;
4650 		dev_info(priv->device, "SPH feature enabled\n");
4651 	}
4652 
4653 	if (priv->dma_cap.addr64) {
4654 		ret = dma_set_mask_and_coherent(device,
4655 				DMA_BIT_MASK(priv->dma_cap.addr64));
4656 		if (!ret) {
4657 			dev_info(priv->device, "Using %d bits DMA width\n",
4658 				 priv->dma_cap.addr64);
4659 
4660 			/*
4661 			 * If more than 32 bits can be addressed, make sure to
4662 			 * enable enhanced addressing mode.
4663 			 */
4664 			if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
4665 				priv->plat->dma_cfg->eame = true;
4666 		} else {
4667 			ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
4668 			if (ret) {
4669 				dev_err(priv->device, "Failed to set DMA Mask\n");
4670 				goto error_hw_init;
4671 			}
4672 
4673 			priv->dma_cap.addr64 = 32;
4674 		}
4675 	}
4676 
4677 	ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4678 	ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4679 #ifdef STMMAC_VLAN_TAG_USED
4680 	/* Both mac100 and gmac support receive VLAN tag detection */
4681 	ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
4682 	if (priv->dma_cap.vlhash) {
4683 		ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4684 		ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
4685 	}
4686 	if (priv->dma_cap.vlins) {
4687 		ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
4688 		if (priv->dma_cap.dvlan)
4689 			ndev->features |= NETIF_F_HW_VLAN_STAG_TX;
4690 	}
4691 #endif
4692 	priv->msg_enable = netif_msg_init(debug, default_msg_level);
4693 
4694 	/* Initialize RSS */
4695 	rxq = priv->plat->rx_queues_to_use;
4696 	netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
4697 	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
4698 		priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
4699 
4700 	if (priv->dma_cap.rssen && priv->plat->rss_en)
4701 		ndev->features |= NETIF_F_RXHASH;
4702 
4703 	/* MTU range: 46 - hw-specific max */
4704 	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4705 	if (priv->plat->has_xgmac)
4706 		ndev->max_mtu = XGMAC_JUMBO_LEN;
4707 	else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4708 		ndev->max_mtu = JUMBO_LEN;
4709 	else
4710 		ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4711 	/* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4712 	 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4713 	 */
4714 	if ((priv->plat->maxmtu < ndev->max_mtu) &&
4715 	    (priv->plat->maxmtu >= ndev->min_mtu))
4716 		ndev->max_mtu = priv->plat->maxmtu;
4717 	else if (priv->plat->maxmtu < ndev->min_mtu)
4718 		dev_warn(priv->device,
4719 			 "%s: warning: maxmtu having invalid value (%d)\n",
4720 			 __func__, priv->plat->maxmtu);
4721 
4722 	if (flow_ctrl)
4723 		priv->flow_ctrl = FLOW_AUTO;	/* RX/TX pause on */
4724 
4725 	/* Setup channels NAPI */
4726 	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
4727 
4728 	for (queue = 0; queue < maxq; queue++) {
4729 		struct stmmac_channel *ch = &priv->channel[queue];
4730 
4731 		ch->priv_data = priv;
4732 		ch->index = queue;
4733 
4734 		if (queue < priv->plat->rx_queues_to_use) {
4735 			netif_napi_add(ndev, &ch->rx_napi, stmmac_napi_poll_rx,
4736 				       NAPI_POLL_WEIGHT);
4737 		}
4738 		if (queue < priv->plat->tx_queues_to_use) {
4739 			netif_tx_napi_add(ndev, &ch->tx_napi,
4740 					  stmmac_napi_poll_tx,
4741 					  NAPI_POLL_WEIGHT);
4742 		}
4743 	}
4744 
4745 	mutex_init(&priv->lock);
4746 
4747 	/* If a specific clk_csr value is passed from the platform
4748 	 * this means that the CSR Clock Range selection cannot be
4749 	 * changed at run-time and it is fixed. Viceversa the driver'll try to
4750 	 * set the MDC clock dynamically according to the csr actual
4751 	 * clock input.
4752 	 */
4753 	if (priv->plat->clk_csr >= 0)
4754 		priv->clk_csr = priv->plat->clk_csr;
4755 	else
4756 		stmmac_clk_csr_set(priv);
4757 
4758 	stmmac_check_pcs_mode(priv);
4759 
4760 	if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4761 	    priv->hw->pcs != STMMAC_PCS_TBI &&
4762 	    priv->hw->pcs != STMMAC_PCS_RTBI) {
4763 		/* MDIO bus Registration */
4764 		ret = stmmac_mdio_register(ndev);
4765 		if (ret < 0) {
4766 			dev_err(priv->device,
4767 				"%s: MDIO bus (id: %d) registration failed",
4768 				__func__, priv->plat->bus_id);
4769 			goto error_mdio_register;
4770 		}
4771 	}
4772 
4773 	ret = stmmac_phy_setup(priv);
4774 	if (ret) {
4775 		netdev_err(ndev, "failed to setup phy (%d)\n", ret);
4776 		goto error_phy_setup;
4777 	}
4778 
4779 	ret = register_netdev(ndev);
4780 	if (ret) {
4781 		dev_err(priv->device, "%s: ERROR %i registering the device\n",
4782 			__func__, ret);
4783 		goto error_netdev_register;
4784 	}
4785 
4786 #ifdef CONFIG_DEBUG_FS
4787 	stmmac_init_fs(ndev);
4788 #endif
4789 
4790 	return ret;
4791 
4792 error_netdev_register:
4793 	phylink_destroy(priv->phylink);
4794 error_phy_setup:
4795 	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4796 	    priv->hw->pcs != STMMAC_PCS_TBI &&
4797 	    priv->hw->pcs != STMMAC_PCS_RTBI)
4798 		stmmac_mdio_unregister(ndev);
4799 error_mdio_register:
4800 	for (queue = 0; queue < maxq; queue++) {
4801 		struct stmmac_channel *ch = &priv->channel[queue];
4802 
4803 		if (queue < priv->plat->rx_queues_to_use)
4804 			netif_napi_del(&ch->rx_napi);
4805 		if (queue < priv->plat->tx_queues_to_use)
4806 			netif_napi_del(&ch->tx_napi);
4807 	}
4808 error_hw_init:
4809 	destroy_workqueue(priv->wq);
4810 
4811 	return ret;
4812 }
4813 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4814 
4815 /**
4816  * stmmac_dvr_remove
4817  * @dev: device pointer
4818  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4819  * changes the link status, releases the DMA descriptor rings.
4820  */
4821 int stmmac_dvr_remove(struct device *dev)
4822 {
4823 	struct net_device *ndev = dev_get_drvdata(dev);
4824 	struct stmmac_priv *priv = netdev_priv(ndev);
4825 
4826 	netdev_info(priv->dev, "%s: removing driver", __func__);
4827 
4828 #ifdef CONFIG_DEBUG_FS
4829 	stmmac_exit_fs(ndev);
4830 #endif
4831 	stmmac_stop_all_dma(priv);
4832 
4833 	stmmac_mac_set(priv, priv->ioaddr, false);
4834 	netif_carrier_off(ndev);
4835 	unregister_netdev(ndev);
4836 	phylink_destroy(priv->phylink);
4837 	if (priv->plat->stmmac_rst)
4838 		reset_control_assert(priv->plat->stmmac_rst);
4839 	clk_disable_unprepare(priv->plat->pclk);
4840 	clk_disable_unprepare(priv->plat->stmmac_clk);
4841 	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4842 	    priv->hw->pcs != STMMAC_PCS_TBI &&
4843 	    priv->hw->pcs != STMMAC_PCS_RTBI)
4844 		stmmac_mdio_unregister(ndev);
4845 	destroy_workqueue(priv->wq);
4846 	mutex_destroy(&priv->lock);
4847 
4848 	return 0;
4849 }
4850 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4851 
4852 /**
4853  * stmmac_suspend - suspend callback
4854  * @dev: device pointer
4855  * Description: this is the function to suspend the device and it is called
4856  * by the platform driver to stop the network queue, release the resources,
4857  * program the PMT register (for WoL), clean and release driver resources.
4858  */
4859 int stmmac_suspend(struct device *dev)
4860 {
4861 	struct net_device *ndev = dev_get_drvdata(dev);
4862 	struct stmmac_priv *priv = netdev_priv(ndev);
4863 
4864 	if (!ndev || !netif_running(ndev))
4865 		return 0;
4866 
4867 	phylink_mac_change(priv->phylink, false);
4868 
4869 	mutex_lock(&priv->lock);
4870 
4871 	netif_device_detach(ndev);
4872 	stmmac_stop_all_queues(priv);
4873 
4874 	stmmac_disable_all_queues(priv);
4875 
4876 	/* Stop TX/RX DMA */
4877 	stmmac_stop_all_dma(priv);
4878 
4879 	/* Enable Power down mode by programming the PMT regs */
4880 	if (device_may_wakeup(priv->device)) {
4881 		stmmac_pmt(priv, priv->hw, priv->wolopts);
4882 		priv->irq_wake = 1;
4883 	} else {
4884 		mutex_unlock(&priv->lock);
4885 		rtnl_lock();
4886 		phylink_stop(priv->phylink);
4887 		rtnl_unlock();
4888 		mutex_lock(&priv->lock);
4889 
4890 		stmmac_mac_set(priv, priv->ioaddr, false);
4891 		pinctrl_pm_select_sleep_state(priv->device);
4892 		/* Disable clock in case of PWM is off */
4893 		if (priv->plat->clk_ptp_ref)
4894 			clk_disable_unprepare(priv->plat->clk_ptp_ref);
4895 		clk_disable_unprepare(priv->plat->pclk);
4896 		clk_disable_unprepare(priv->plat->stmmac_clk);
4897 	}
4898 	mutex_unlock(&priv->lock);
4899 
4900 	priv->speed = SPEED_UNKNOWN;
4901 	return 0;
4902 }
4903 EXPORT_SYMBOL_GPL(stmmac_suspend);
4904 
4905 /**
4906  * stmmac_reset_queues_param - reset queue parameters
4907  * @dev: device pointer
4908  */
4909 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4910 {
4911 	u32 rx_cnt = priv->plat->rx_queues_to_use;
4912 	u32 tx_cnt = priv->plat->tx_queues_to_use;
4913 	u32 queue;
4914 
4915 	for (queue = 0; queue < rx_cnt; queue++) {
4916 		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4917 
4918 		rx_q->cur_rx = 0;
4919 		rx_q->dirty_rx = 0;
4920 	}
4921 
4922 	for (queue = 0; queue < tx_cnt; queue++) {
4923 		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4924 
4925 		tx_q->cur_tx = 0;
4926 		tx_q->dirty_tx = 0;
4927 		tx_q->mss = 0;
4928 	}
4929 }
4930 
4931 /**
4932  * stmmac_resume - resume callback
4933  * @dev: device pointer
4934  * Description: when resume this function is invoked to setup the DMA and CORE
4935  * in a usable state.
4936  */
4937 int stmmac_resume(struct device *dev)
4938 {
4939 	struct net_device *ndev = dev_get_drvdata(dev);
4940 	struct stmmac_priv *priv = netdev_priv(ndev);
4941 
4942 	if (!netif_running(ndev))
4943 		return 0;
4944 
4945 	/* Power Down bit, into the PM register, is cleared
4946 	 * automatically as soon as a magic packet or a Wake-up frame
4947 	 * is received. Anyway, it's better to manually clear
4948 	 * this bit because it can generate problems while resuming
4949 	 * from another devices (e.g. serial console).
4950 	 */
4951 	if (device_may_wakeup(priv->device)) {
4952 		mutex_lock(&priv->lock);
4953 		stmmac_pmt(priv, priv->hw, 0);
4954 		mutex_unlock(&priv->lock);
4955 		priv->irq_wake = 0;
4956 	} else {
4957 		pinctrl_pm_select_default_state(priv->device);
4958 		/* enable the clk previously disabled */
4959 		clk_prepare_enable(priv->plat->stmmac_clk);
4960 		clk_prepare_enable(priv->plat->pclk);
4961 		if (priv->plat->clk_ptp_ref)
4962 			clk_prepare_enable(priv->plat->clk_ptp_ref);
4963 		/* reset the phy so that it's ready */
4964 		if (priv->mii)
4965 			stmmac_mdio_reset(priv->mii);
4966 	}
4967 
4968 	netif_device_attach(ndev);
4969 
4970 	mutex_lock(&priv->lock);
4971 
4972 	stmmac_reset_queues_param(priv);
4973 
4974 	stmmac_clear_descriptors(priv);
4975 
4976 	stmmac_hw_setup(ndev, false);
4977 	stmmac_init_coalesce(priv);
4978 	stmmac_set_rx_mode(ndev);
4979 
4980 	stmmac_enable_all_queues(priv);
4981 
4982 	stmmac_start_all_queues(priv);
4983 
4984 	mutex_unlock(&priv->lock);
4985 
4986 	if (!device_may_wakeup(priv->device)) {
4987 		rtnl_lock();
4988 		phylink_start(priv->phylink);
4989 		rtnl_unlock();
4990 	}
4991 
4992 	phylink_mac_change(priv->phylink, true);
4993 
4994 	return 0;
4995 }
4996 EXPORT_SYMBOL_GPL(stmmac_resume);
4997 
4998 #ifndef MODULE
4999 static int __init stmmac_cmdline_opt(char *str)
5000 {
5001 	char *opt;
5002 
5003 	if (!str || !*str)
5004 		return -EINVAL;
5005 	while ((opt = strsep(&str, ",")) != NULL) {
5006 		if (!strncmp(opt, "debug:", 6)) {
5007 			if (kstrtoint(opt + 6, 0, &debug))
5008 				goto err;
5009 		} else if (!strncmp(opt, "phyaddr:", 8)) {
5010 			if (kstrtoint(opt + 8, 0, &phyaddr))
5011 				goto err;
5012 		} else if (!strncmp(opt, "buf_sz:", 7)) {
5013 			if (kstrtoint(opt + 7, 0, &buf_sz))
5014 				goto err;
5015 		} else if (!strncmp(opt, "tc:", 3)) {
5016 			if (kstrtoint(opt + 3, 0, &tc))
5017 				goto err;
5018 		} else if (!strncmp(opt, "watchdog:", 9)) {
5019 			if (kstrtoint(opt + 9, 0, &watchdog))
5020 				goto err;
5021 		} else if (!strncmp(opt, "flow_ctrl:", 10)) {
5022 			if (kstrtoint(opt + 10, 0, &flow_ctrl))
5023 				goto err;
5024 		} else if (!strncmp(opt, "pause:", 6)) {
5025 			if (kstrtoint(opt + 6, 0, &pause))
5026 				goto err;
5027 		} else if (!strncmp(opt, "eee_timer:", 10)) {
5028 			if (kstrtoint(opt + 10, 0, &eee_timer))
5029 				goto err;
5030 		} else if (!strncmp(opt, "chain_mode:", 11)) {
5031 			if (kstrtoint(opt + 11, 0, &chain_mode))
5032 				goto err;
5033 		}
5034 	}
5035 	return 0;
5036 
5037 err:
5038 	pr_err("%s: ERROR broken module parameter conversion", __func__);
5039 	return -EINVAL;
5040 }
5041 
5042 __setup("stmmaceth=", stmmac_cmdline_opt);
5043 #endif /* MODULE */
5044 
5045 static int __init stmmac_init(void)
5046 {
5047 #ifdef CONFIG_DEBUG_FS
5048 	/* Create debugfs main directory if it doesn't exist yet */
5049 	if (!stmmac_fs_dir)
5050 		stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
5051 #endif
5052 
5053 	return 0;
5054 }
5055 
5056 static void __exit stmmac_exit(void)
5057 {
5058 #ifdef CONFIG_DEBUG_FS
5059 	debugfs_remove_recursive(stmmac_fs_dir);
5060 #endif
5061 }
5062 
5063 module_init(stmmac_init)
5064 module_exit(stmmac_exit)
5065 
5066 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
5067 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
5068 MODULE_LICENSE("GPL");
5069