xref: /openbmc/linux/drivers/net/ethernet/ti/cpsw.c (revision bdeeed09)
1 /*
2  * Texas Instruments Ethernet Switch Driver
3  *
4  * Copyright (C) 2012 Texas Instruments
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License as
8  * published by the Free Software Foundation version 2.
9  *
10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11  * kind, whether express or implied; without even the implied warranty
12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  */
15 
16 #include <linux/kernel.h>
17 #include <linux/io.h>
18 #include <linux/clk.h>
19 #include <linux/timer.h>
20 #include <linux/module.h>
21 #include <linux/platform_device.h>
22 #include <linux/irqreturn.h>
23 #include <linux/interrupt.h>
24 #include <linux/if_ether.h>
25 #include <linux/etherdevice.h>
26 #include <linux/netdevice.h>
27 #include <linux/net_tstamp.h>
28 #include <linux/phy.h>
29 #include <linux/workqueue.h>
30 #include <linux/delay.h>
31 #include <linux/pm_runtime.h>
32 #include <linux/gpio.h>
33 #include <linux/of.h>
34 #include <linux/of_mdio.h>
35 #include <linux/of_net.h>
36 #include <linux/of_device.h>
37 #include <linux/if_vlan.h>
38 #include <linux/kmemleak.h>
39 
40 #include <linux/pinctrl/consumer.h>
41 
42 #include "cpsw.h"
43 #include "cpsw_ale.h"
44 #include "cpts.h"
45 #include "davinci_cpdma.h"
46 
47 #define CPSW_DEBUG	(NETIF_MSG_HW		| NETIF_MSG_WOL		| \
48 			 NETIF_MSG_DRV		| NETIF_MSG_LINK	| \
49 			 NETIF_MSG_IFUP		| NETIF_MSG_INTR	| \
50 			 NETIF_MSG_PROBE	| NETIF_MSG_TIMER	| \
51 			 NETIF_MSG_IFDOWN	| NETIF_MSG_RX_ERR	| \
52 			 NETIF_MSG_TX_ERR	| NETIF_MSG_TX_DONE	| \
53 			 NETIF_MSG_PKTDATA	| NETIF_MSG_TX_QUEUED	| \
54 			 NETIF_MSG_RX_STATUS)
55 
56 #define cpsw_info(priv, type, format, ...)		\
57 do {								\
58 	if (netif_msg_##type(priv) && net_ratelimit())		\
59 		dev_info(priv->dev, format, ## __VA_ARGS__);	\
60 } while (0)
61 
62 #define cpsw_err(priv, type, format, ...)		\
63 do {								\
64 	if (netif_msg_##type(priv) && net_ratelimit())		\
65 		dev_err(priv->dev, format, ## __VA_ARGS__);	\
66 } while (0)
67 
68 #define cpsw_dbg(priv, type, format, ...)		\
69 do {								\
70 	if (netif_msg_##type(priv) && net_ratelimit())		\
71 		dev_dbg(priv->dev, format, ## __VA_ARGS__);	\
72 } while (0)
73 
74 #define cpsw_notice(priv, type, format, ...)		\
75 do {								\
76 	if (netif_msg_##type(priv) && net_ratelimit())		\
77 		dev_notice(priv->dev, format, ## __VA_ARGS__);	\
78 } while (0)
79 
80 #define ALE_ALL_PORTS		0x7
81 
82 #define CPSW_MAJOR_VERSION(reg)		(reg >> 8 & 0x7)
83 #define CPSW_MINOR_VERSION(reg)		(reg & 0xff)
84 #define CPSW_RTL_VERSION(reg)		((reg >> 11) & 0x1f)
85 
86 #define CPSW_VERSION_1		0x19010a
87 #define CPSW_VERSION_2		0x19010c
88 #define CPSW_VERSION_3		0x19010f
89 #define CPSW_VERSION_4		0x190112
90 
91 #define HOST_PORT_NUM		0
92 #define CPSW_ALE_PORTS_NUM	3
93 #define SLIVER_SIZE		0x40
94 
95 #define CPSW1_HOST_PORT_OFFSET	0x028
96 #define CPSW1_SLAVE_OFFSET	0x050
97 #define CPSW1_SLAVE_SIZE	0x040
98 #define CPSW1_CPDMA_OFFSET	0x100
99 #define CPSW1_STATERAM_OFFSET	0x200
100 #define CPSW1_HW_STATS		0x400
101 #define CPSW1_CPTS_OFFSET	0x500
102 #define CPSW1_ALE_OFFSET	0x600
103 #define CPSW1_SLIVER_OFFSET	0x700
104 
105 #define CPSW2_HOST_PORT_OFFSET	0x108
106 #define CPSW2_SLAVE_OFFSET	0x200
107 #define CPSW2_SLAVE_SIZE	0x100
108 #define CPSW2_CPDMA_OFFSET	0x800
109 #define CPSW2_HW_STATS		0x900
110 #define CPSW2_STATERAM_OFFSET	0xa00
111 #define CPSW2_CPTS_OFFSET	0xc00
112 #define CPSW2_ALE_OFFSET	0xd00
113 #define CPSW2_SLIVER_OFFSET	0xd80
114 #define CPSW2_BD_OFFSET		0x2000
115 
116 #define CPDMA_RXTHRESH		0x0c0
117 #define CPDMA_RXFREE		0x0e0
118 #define CPDMA_TXHDP		0x00
119 #define CPDMA_RXHDP		0x20
120 #define CPDMA_TXCP		0x40
121 #define CPDMA_RXCP		0x60
122 
123 #define CPSW_POLL_WEIGHT	64
124 #define CPSW_RX_VLAN_ENCAP_HDR_SIZE		4
125 #define CPSW_MIN_PACKET_SIZE	(VLAN_ETH_ZLEN)
126 #define CPSW_MAX_PACKET_SIZE	(VLAN_ETH_FRAME_LEN +\
127 				 ETH_FCS_LEN +\
128 				 CPSW_RX_VLAN_ENCAP_HDR_SIZE)
129 
130 #define RX_PRIORITY_MAPPING	0x76543210
131 #define TX_PRIORITY_MAPPING	0x33221100
132 #define CPDMA_TX_PRIORITY_MAP	0x76543210
133 
134 #define CPSW_VLAN_AWARE		BIT(1)
135 #define CPSW_RX_VLAN_ENCAP	BIT(2)
136 #define CPSW_ALE_VLAN_AWARE	1
137 
138 #define CPSW_FIFO_NORMAL_MODE		(0 << 16)
139 #define CPSW_FIFO_DUAL_MAC_MODE		(1 << 16)
140 #define CPSW_FIFO_RATE_LIMIT_MODE	(2 << 16)
141 
142 #define CPSW_INTPACEEN		(0x3f << 16)
143 #define CPSW_INTPRESCALE_MASK	(0x7FF << 0)
144 #define CPSW_CMINTMAX_CNT	63
145 #define CPSW_CMINTMIN_CNT	2
146 #define CPSW_CMINTMAX_INTVL	(1000 / CPSW_CMINTMIN_CNT)
147 #define CPSW_CMINTMIN_INTVL	((1000 / CPSW_CMINTMAX_CNT) + 1)
148 
149 #define cpsw_slave_index(cpsw, priv)				\
150 		((cpsw->data.dual_emac) ? priv->emac_port :	\
151 		cpsw->data.active_slave)
152 #define IRQ_NUM			2
153 #define CPSW_MAX_QUEUES		8
154 #define CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT 256
155 
156 #define CPSW_RX_VLAN_ENCAP_HDR_PRIO_SHIFT	29
157 #define CPSW_RX_VLAN_ENCAP_HDR_PRIO_MSK		GENMASK(2, 0)
158 #define CPSW_RX_VLAN_ENCAP_HDR_VID_SHIFT	16
159 #define CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_SHIFT	8
160 #define CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_MSK	GENMASK(1, 0)
161 enum {
162 	CPSW_RX_VLAN_ENCAP_HDR_PKT_VLAN_TAG = 0,
163 	CPSW_RX_VLAN_ENCAP_HDR_PKT_RESERV,
164 	CPSW_RX_VLAN_ENCAP_HDR_PKT_PRIO_TAG,
165 	CPSW_RX_VLAN_ENCAP_HDR_PKT_UNTAG,
166 };
167 
168 static int debug_level;
169 module_param(debug_level, int, 0);
170 MODULE_PARM_DESC(debug_level, "cpsw debug level (NETIF_MSG bits)");
171 
172 static int ale_ageout = 10;
173 module_param(ale_ageout, int, 0);
174 MODULE_PARM_DESC(ale_ageout, "cpsw ale ageout interval (seconds)");
175 
176 static int rx_packet_max = CPSW_MAX_PACKET_SIZE;
177 module_param(rx_packet_max, int, 0);
178 MODULE_PARM_DESC(rx_packet_max, "maximum receive packet size (bytes)");
179 
180 static int descs_pool_size = CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT;
181 module_param(descs_pool_size, int, 0444);
182 MODULE_PARM_DESC(descs_pool_size, "Number of CPDMA CPPI descriptors in pool");
183 
184 struct cpsw_wr_regs {
185 	u32	id_ver;
186 	u32	soft_reset;
187 	u32	control;
188 	u32	int_control;
189 	u32	rx_thresh_en;
190 	u32	rx_en;
191 	u32	tx_en;
192 	u32	misc_en;
193 	u32	mem_allign1[8];
194 	u32	rx_thresh_stat;
195 	u32	rx_stat;
196 	u32	tx_stat;
197 	u32	misc_stat;
198 	u32	mem_allign2[8];
199 	u32	rx_imax;
200 	u32	tx_imax;
201 
202 };
203 
204 struct cpsw_ss_regs {
205 	u32	id_ver;
206 	u32	control;
207 	u32	soft_reset;
208 	u32	stat_port_en;
209 	u32	ptype;
210 	u32	soft_idle;
211 	u32	thru_rate;
212 	u32	gap_thresh;
213 	u32	tx_start_wds;
214 	u32	flow_control;
215 	u32	vlan_ltype;
216 	u32	ts_ltype;
217 	u32	dlr_ltype;
218 };
219 
220 /* CPSW_PORT_V1 */
221 #define CPSW1_MAX_BLKS      0x00 /* Maximum FIFO Blocks */
222 #define CPSW1_BLK_CNT       0x04 /* FIFO Block Usage Count (Read Only) */
223 #define CPSW1_TX_IN_CTL     0x08 /* Transmit FIFO Control */
224 #define CPSW1_PORT_VLAN     0x0c /* VLAN Register */
225 #define CPSW1_TX_PRI_MAP    0x10 /* Tx Header Priority to Switch Pri Mapping */
226 #define CPSW1_TS_CTL        0x14 /* Time Sync Control */
227 #define CPSW1_TS_SEQ_LTYPE  0x18 /* Time Sync Sequence ID Offset and Msg Type */
228 #define CPSW1_TS_VLAN       0x1c /* Time Sync VLAN1 and VLAN2 */
229 
230 /* CPSW_PORT_V2 */
231 #define CPSW2_CONTROL       0x00 /* Control Register */
232 #define CPSW2_MAX_BLKS      0x08 /* Maximum FIFO Blocks */
233 #define CPSW2_BLK_CNT       0x0c /* FIFO Block Usage Count (Read Only) */
234 #define CPSW2_TX_IN_CTL     0x10 /* Transmit FIFO Control */
235 #define CPSW2_PORT_VLAN     0x14 /* VLAN Register */
236 #define CPSW2_TX_PRI_MAP    0x18 /* Tx Header Priority to Switch Pri Mapping */
237 #define CPSW2_TS_SEQ_MTYPE  0x1c /* Time Sync Sequence ID Offset and Msg Type */
238 
239 /* CPSW_PORT_V1 and V2 */
240 #define SA_LO               0x20 /* CPGMAC_SL Source Address Low */
241 #define SA_HI               0x24 /* CPGMAC_SL Source Address High */
242 #define SEND_PERCENT        0x28 /* Transmit Queue Send Percentages */
243 
244 /* CPSW_PORT_V2 only */
245 #define RX_DSCP_PRI_MAP0    0x30 /* Rx DSCP Priority to Rx Packet Mapping */
246 #define RX_DSCP_PRI_MAP1    0x34 /* Rx DSCP Priority to Rx Packet Mapping */
247 #define RX_DSCP_PRI_MAP2    0x38 /* Rx DSCP Priority to Rx Packet Mapping */
248 #define RX_DSCP_PRI_MAP3    0x3c /* Rx DSCP Priority to Rx Packet Mapping */
249 #define RX_DSCP_PRI_MAP4    0x40 /* Rx DSCP Priority to Rx Packet Mapping */
250 #define RX_DSCP_PRI_MAP5    0x44 /* Rx DSCP Priority to Rx Packet Mapping */
251 #define RX_DSCP_PRI_MAP6    0x48 /* Rx DSCP Priority to Rx Packet Mapping */
252 #define RX_DSCP_PRI_MAP7    0x4c /* Rx DSCP Priority to Rx Packet Mapping */
253 
254 /* Bit definitions for the CPSW2_CONTROL register */
255 #define PASS_PRI_TAGGED     (1<<24) /* Pass Priority Tagged */
256 #define VLAN_LTYPE2_EN      (1<<21) /* VLAN LTYPE 2 enable */
257 #define VLAN_LTYPE1_EN      (1<<20) /* VLAN LTYPE 1 enable */
258 #define DSCP_PRI_EN         (1<<16) /* DSCP Priority Enable */
259 #define TS_320              (1<<14) /* Time Sync Dest Port 320 enable */
260 #define TS_319              (1<<13) /* Time Sync Dest Port 319 enable */
261 #define TS_132              (1<<12) /* Time Sync Dest IP Addr 132 enable */
262 #define TS_131              (1<<11) /* Time Sync Dest IP Addr 131 enable */
263 #define TS_130              (1<<10) /* Time Sync Dest IP Addr 130 enable */
264 #define TS_129              (1<<9)  /* Time Sync Dest IP Addr 129 enable */
265 #define TS_TTL_NONZERO      (1<<8)  /* Time Sync Time To Live Non-zero enable */
266 #define TS_ANNEX_F_EN       (1<<6)  /* Time Sync Annex F enable */
267 #define TS_ANNEX_D_EN       (1<<4)  /* Time Sync Annex D enable */
268 #define TS_LTYPE2_EN        (1<<3)  /* Time Sync LTYPE 2 enable */
269 #define TS_LTYPE1_EN        (1<<2)  /* Time Sync LTYPE 1 enable */
270 #define TS_TX_EN            (1<<1)  /* Time Sync Transmit Enable */
271 #define TS_RX_EN            (1<<0)  /* Time Sync Receive Enable */
272 
273 #define CTRL_V2_TS_BITS \
274 	(TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
275 	 TS_TTL_NONZERO  | TS_ANNEX_D_EN | TS_LTYPE1_EN)
276 
277 #define CTRL_V2_ALL_TS_MASK (CTRL_V2_TS_BITS | TS_TX_EN | TS_RX_EN)
278 #define CTRL_V2_TX_TS_BITS  (CTRL_V2_TS_BITS | TS_TX_EN)
279 #define CTRL_V2_RX_TS_BITS  (CTRL_V2_TS_BITS | TS_RX_EN)
280 
281 
282 #define CTRL_V3_TS_BITS \
283 	(TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
284 	 TS_TTL_NONZERO | TS_ANNEX_F_EN | TS_ANNEX_D_EN |\
285 	 TS_LTYPE1_EN)
286 
287 #define CTRL_V3_ALL_TS_MASK (CTRL_V3_TS_BITS | TS_TX_EN | TS_RX_EN)
288 #define CTRL_V3_TX_TS_BITS  (CTRL_V3_TS_BITS | TS_TX_EN)
289 #define CTRL_V3_RX_TS_BITS  (CTRL_V3_TS_BITS | TS_RX_EN)
290 
291 /* Bit definitions for the CPSW2_TS_SEQ_MTYPE register */
292 #define TS_SEQ_ID_OFFSET_SHIFT   (16)    /* Time Sync Sequence ID Offset */
293 #define TS_SEQ_ID_OFFSET_MASK    (0x3f)
294 #define TS_MSG_TYPE_EN_SHIFT     (0)     /* Time Sync Message Type Enable */
295 #define TS_MSG_TYPE_EN_MASK      (0xffff)
296 
297 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
298 #define EVENT_MSG_BITS ((1<<0) | (1<<1) | (1<<2) | (1<<3))
299 
300 /* Bit definitions for the CPSW1_TS_CTL register */
301 #define CPSW_V1_TS_RX_EN		BIT(0)
302 #define CPSW_V1_TS_TX_EN		BIT(4)
303 #define CPSW_V1_MSG_TYPE_OFS		16
304 
305 /* Bit definitions for the CPSW1_TS_SEQ_LTYPE register */
306 #define CPSW_V1_SEQ_ID_OFS_SHIFT	16
307 
308 #define CPSW_MAX_BLKS_TX		15
309 #define CPSW_MAX_BLKS_TX_SHIFT		4
310 #define CPSW_MAX_BLKS_RX		5
311 
312 struct cpsw_host_regs {
313 	u32	max_blks;
314 	u32	blk_cnt;
315 	u32	tx_in_ctl;
316 	u32	port_vlan;
317 	u32	tx_pri_map;
318 	u32	cpdma_tx_pri_map;
319 	u32	cpdma_rx_chan_map;
320 };
321 
322 struct cpsw_sliver_regs {
323 	u32	id_ver;
324 	u32	mac_control;
325 	u32	mac_status;
326 	u32	soft_reset;
327 	u32	rx_maxlen;
328 	u32	__reserved_0;
329 	u32	rx_pause;
330 	u32	tx_pause;
331 	u32	__reserved_1;
332 	u32	rx_pri_map;
333 };
334 
335 struct cpsw_hw_stats {
336 	u32	rxgoodframes;
337 	u32	rxbroadcastframes;
338 	u32	rxmulticastframes;
339 	u32	rxpauseframes;
340 	u32	rxcrcerrors;
341 	u32	rxaligncodeerrors;
342 	u32	rxoversizedframes;
343 	u32	rxjabberframes;
344 	u32	rxundersizedframes;
345 	u32	rxfragments;
346 	u32	__pad_0[2];
347 	u32	rxoctets;
348 	u32	txgoodframes;
349 	u32	txbroadcastframes;
350 	u32	txmulticastframes;
351 	u32	txpauseframes;
352 	u32	txdeferredframes;
353 	u32	txcollisionframes;
354 	u32	txsinglecollframes;
355 	u32	txmultcollframes;
356 	u32	txexcessivecollisions;
357 	u32	txlatecollisions;
358 	u32	txunderrun;
359 	u32	txcarriersenseerrors;
360 	u32	txoctets;
361 	u32	octetframes64;
362 	u32	octetframes65t127;
363 	u32	octetframes128t255;
364 	u32	octetframes256t511;
365 	u32	octetframes512t1023;
366 	u32	octetframes1024tup;
367 	u32	netoctets;
368 	u32	rxsofoverruns;
369 	u32	rxmofoverruns;
370 	u32	rxdmaoverruns;
371 };
372 
373 struct cpsw_slave_data {
374 	struct device_node *phy_node;
375 	char		phy_id[MII_BUS_ID_SIZE];
376 	int		phy_if;
377 	u8		mac_addr[ETH_ALEN];
378 	u16		dual_emac_res_vlan;	/* Reserved VLAN for DualEMAC */
379 };
380 
381 struct cpsw_platform_data {
382 	struct cpsw_slave_data	*slave_data;
383 	u32	ss_reg_ofs;	/* Subsystem control register offset */
384 	u32	channels;	/* number of cpdma channels (symmetric) */
385 	u32	slaves;		/* number of slave cpgmac ports */
386 	u32	active_slave; /* time stamping, ethtool and SIOCGMIIPHY slave */
387 	u32	ale_entries;	/* ale table size */
388 	u32	bd_ram_size;  /*buffer descriptor ram size */
389 	u32	mac_control;	/* Mac control register */
390 	u16	default_vlan;	/* Def VLAN for ALE lookup in VLAN aware mode*/
391 	bool	dual_emac;	/* Enable Dual EMAC mode */
392 };
393 
394 struct cpsw_slave {
395 	void __iomem			*regs;
396 	struct cpsw_sliver_regs __iomem	*sliver;
397 	int				slave_num;
398 	u32				mac_control;
399 	struct cpsw_slave_data		*data;
400 	struct phy_device		*phy;
401 	struct net_device		*ndev;
402 	u32				port_vlan;
403 };
404 
405 static inline u32 slave_read(struct cpsw_slave *slave, u32 offset)
406 {
407 	return readl_relaxed(slave->regs + offset);
408 }
409 
410 static inline void slave_write(struct cpsw_slave *slave, u32 val, u32 offset)
411 {
412 	writel_relaxed(val, slave->regs + offset);
413 }
414 
415 struct cpsw_vector {
416 	struct cpdma_chan *ch;
417 	int budget;
418 };
419 
420 struct cpsw_common {
421 	struct device			*dev;
422 	struct cpsw_platform_data	data;
423 	struct napi_struct		napi_rx;
424 	struct napi_struct		napi_tx;
425 	struct cpsw_ss_regs __iomem	*regs;
426 	struct cpsw_wr_regs __iomem	*wr_regs;
427 	u8 __iomem			*hw_stats;
428 	struct cpsw_host_regs __iomem	*host_port_regs;
429 	u32				version;
430 	u32				coal_intvl;
431 	u32				bus_freq_mhz;
432 	int				rx_packet_max;
433 	struct cpsw_slave		*slaves;
434 	struct cpdma_ctlr		*dma;
435 	struct cpsw_vector		txv[CPSW_MAX_QUEUES];
436 	struct cpsw_vector		rxv[CPSW_MAX_QUEUES];
437 	struct cpsw_ale			*ale;
438 	bool				quirk_irq;
439 	bool				rx_irq_disabled;
440 	bool				tx_irq_disabled;
441 	u32 irqs_table[IRQ_NUM];
442 	struct cpts			*cpts;
443 	int				rx_ch_num, tx_ch_num;
444 	int				speed;
445 	int				usage_count;
446 };
447 
448 struct cpsw_priv {
449 	struct net_device		*ndev;
450 	struct device			*dev;
451 	u32				msg_enable;
452 	u8				mac_addr[ETH_ALEN];
453 	bool				rx_pause;
454 	bool				tx_pause;
455 	u32 emac_port;
456 	struct cpsw_common *cpsw;
457 };
458 
459 struct cpsw_stats {
460 	char stat_string[ETH_GSTRING_LEN];
461 	int type;
462 	int sizeof_stat;
463 	int stat_offset;
464 };
465 
466 enum {
467 	CPSW_STATS,
468 	CPDMA_RX_STATS,
469 	CPDMA_TX_STATS,
470 };
471 
472 #define CPSW_STAT(m)		CPSW_STATS,				\
473 				sizeof(((struct cpsw_hw_stats *)0)->m), \
474 				offsetof(struct cpsw_hw_stats, m)
475 #define CPDMA_RX_STAT(m)	CPDMA_RX_STATS,				   \
476 				sizeof(((struct cpdma_chan_stats *)0)->m), \
477 				offsetof(struct cpdma_chan_stats, m)
478 #define CPDMA_TX_STAT(m)	CPDMA_TX_STATS,				   \
479 				sizeof(((struct cpdma_chan_stats *)0)->m), \
480 				offsetof(struct cpdma_chan_stats, m)
481 
482 static const struct cpsw_stats cpsw_gstrings_stats[] = {
483 	{ "Good Rx Frames", CPSW_STAT(rxgoodframes) },
484 	{ "Broadcast Rx Frames", CPSW_STAT(rxbroadcastframes) },
485 	{ "Multicast Rx Frames", CPSW_STAT(rxmulticastframes) },
486 	{ "Pause Rx Frames", CPSW_STAT(rxpauseframes) },
487 	{ "Rx CRC Errors", CPSW_STAT(rxcrcerrors) },
488 	{ "Rx Align/Code Errors", CPSW_STAT(rxaligncodeerrors) },
489 	{ "Oversize Rx Frames", CPSW_STAT(rxoversizedframes) },
490 	{ "Rx Jabbers", CPSW_STAT(rxjabberframes) },
491 	{ "Undersize (Short) Rx Frames", CPSW_STAT(rxundersizedframes) },
492 	{ "Rx Fragments", CPSW_STAT(rxfragments) },
493 	{ "Rx Octets", CPSW_STAT(rxoctets) },
494 	{ "Good Tx Frames", CPSW_STAT(txgoodframes) },
495 	{ "Broadcast Tx Frames", CPSW_STAT(txbroadcastframes) },
496 	{ "Multicast Tx Frames", CPSW_STAT(txmulticastframes) },
497 	{ "Pause Tx Frames", CPSW_STAT(txpauseframes) },
498 	{ "Deferred Tx Frames", CPSW_STAT(txdeferredframes) },
499 	{ "Collisions", CPSW_STAT(txcollisionframes) },
500 	{ "Single Collision Tx Frames", CPSW_STAT(txsinglecollframes) },
501 	{ "Multiple Collision Tx Frames", CPSW_STAT(txmultcollframes) },
502 	{ "Excessive Collisions", CPSW_STAT(txexcessivecollisions) },
503 	{ "Late Collisions", CPSW_STAT(txlatecollisions) },
504 	{ "Tx Underrun", CPSW_STAT(txunderrun) },
505 	{ "Carrier Sense Errors", CPSW_STAT(txcarriersenseerrors) },
506 	{ "Tx Octets", CPSW_STAT(txoctets) },
507 	{ "Rx + Tx 64 Octet Frames", CPSW_STAT(octetframes64) },
508 	{ "Rx + Tx 65-127 Octet Frames", CPSW_STAT(octetframes65t127) },
509 	{ "Rx + Tx 128-255 Octet Frames", CPSW_STAT(octetframes128t255) },
510 	{ "Rx + Tx 256-511 Octet Frames", CPSW_STAT(octetframes256t511) },
511 	{ "Rx + Tx 512-1023 Octet Frames", CPSW_STAT(octetframes512t1023) },
512 	{ "Rx + Tx 1024-Up Octet Frames", CPSW_STAT(octetframes1024tup) },
513 	{ "Net Octets", CPSW_STAT(netoctets) },
514 	{ "Rx Start of Frame Overruns", CPSW_STAT(rxsofoverruns) },
515 	{ "Rx Middle of Frame Overruns", CPSW_STAT(rxmofoverruns) },
516 	{ "Rx DMA Overruns", CPSW_STAT(rxdmaoverruns) },
517 };
518 
519 static const struct cpsw_stats cpsw_gstrings_ch_stats[] = {
520 	{ "head_enqueue", CPDMA_RX_STAT(head_enqueue) },
521 	{ "tail_enqueue", CPDMA_RX_STAT(tail_enqueue) },
522 	{ "pad_enqueue", CPDMA_RX_STAT(pad_enqueue) },
523 	{ "misqueued", CPDMA_RX_STAT(misqueued) },
524 	{ "desc_alloc_fail", CPDMA_RX_STAT(desc_alloc_fail) },
525 	{ "pad_alloc_fail", CPDMA_RX_STAT(pad_alloc_fail) },
526 	{ "runt_receive_buf", CPDMA_RX_STAT(runt_receive_buff) },
527 	{ "runt_transmit_buf", CPDMA_RX_STAT(runt_transmit_buff) },
528 	{ "empty_dequeue", CPDMA_RX_STAT(empty_dequeue) },
529 	{ "busy_dequeue", CPDMA_RX_STAT(busy_dequeue) },
530 	{ "good_dequeue", CPDMA_RX_STAT(good_dequeue) },
531 	{ "requeue", CPDMA_RX_STAT(requeue) },
532 	{ "teardown_dequeue", CPDMA_RX_STAT(teardown_dequeue) },
533 };
534 
535 #define CPSW_STATS_COMMON_LEN	ARRAY_SIZE(cpsw_gstrings_stats)
536 #define CPSW_STATS_CH_LEN	ARRAY_SIZE(cpsw_gstrings_ch_stats)
537 
538 #define ndev_to_cpsw(ndev) (((struct cpsw_priv *)netdev_priv(ndev))->cpsw)
539 #define napi_to_cpsw(napi)	container_of(napi, struct cpsw_common, napi)
540 #define for_each_slave(priv, func, arg...)				\
541 	do {								\
542 		struct cpsw_slave *slave;				\
543 		struct cpsw_common *cpsw = (priv)->cpsw;		\
544 		int n;							\
545 		if (cpsw->data.dual_emac)				\
546 			(func)((cpsw)->slaves + priv->emac_port, ##arg);\
547 		else							\
548 			for (n = cpsw->data.slaves,			\
549 					slave = cpsw->slaves;		\
550 					n; n--)				\
551 				(func)(slave++, ##arg);			\
552 	} while (0)
553 
554 #define cpsw_dual_emac_src_port_detect(cpsw, status, ndev, skb)		\
555 	do {								\
556 		if (!cpsw->data.dual_emac)				\
557 			break;						\
558 		if (CPDMA_RX_SOURCE_PORT(status) == 1) {		\
559 			ndev = cpsw->slaves[0].ndev;			\
560 			skb->dev = ndev;				\
561 		} else if (CPDMA_RX_SOURCE_PORT(status) == 2) {		\
562 			ndev = cpsw->slaves[1].ndev;			\
563 			skb->dev = ndev;				\
564 		}							\
565 	} while (0)
566 #define cpsw_add_mcast(cpsw, priv, addr)				\
567 	do {								\
568 		if (cpsw->data.dual_emac) {				\
569 			struct cpsw_slave *slave = cpsw->slaves +	\
570 						priv->emac_port;	\
571 			int slave_port = cpsw_get_slave_port(		\
572 						slave->slave_num);	\
573 			cpsw_ale_add_mcast(cpsw->ale, addr,		\
574 				1 << slave_port | ALE_PORT_HOST,	\
575 				ALE_VLAN, slave->port_vlan, 0);		\
576 		} else {						\
577 			cpsw_ale_add_mcast(cpsw->ale, addr,		\
578 				ALE_ALL_PORTS,				\
579 				0, 0, 0);				\
580 		}							\
581 	} while (0)
582 
583 static inline int cpsw_get_slave_port(u32 slave_num)
584 {
585 	return slave_num + 1;
586 }
587 
588 static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
589 {
590 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
591 	struct cpsw_ale *ale = cpsw->ale;
592 	int i;
593 
594 	if (cpsw->data.dual_emac) {
595 		bool flag = false;
596 
597 		/* Enabling promiscuous mode for one interface will be
598 		 * common for both the interface as the interface shares
599 		 * the same hardware resource.
600 		 */
601 		for (i = 0; i < cpsw->data.slaves; i++)
602 			if (cpsw->slaves[i].ndev->flags & IFF_PROMISC)
603 				flag = true;
604 
605 		if (!enable && flag) {
606 			enable = true;
607 			dev_err(&ndev->dev, "promiscuity not disabled as the other interface is still in promiscuity mode\n");
608 		}
609 
610 		if (enable) {
611 			/* Enable Bypass */
612 			cpsw_ale_control_set(ale, 0, ALE_BYPASS, 1);
613 
614 			dev_dbg(&ndev->dev, "promiscuity enabled\n");
615 		} else {
616 			/* Disable Bypass */
617 			cpsw_ale_control_set(ale, 0, ALE_BYPASS, 0);
618 			dev_dbg(&ndev->dev, "promiscuity disabled\n");
619 		}
620 	} else {
621 		if (enable) {
622 			unsigned long timeout = jiffies + HZ;
623 
624 			/* Disable Learn for all ports (host is port 0 and slaves are port 1 and up */
625 			for (i = 0; i <= cpsw->data.slaves; i++) {
626 				cpsw_ale_control_set(ale, i,
627 						     ALE_PORT_NOLEARN, 1);
628 				cpsw_ale_control_set(ale, i,
629 						     ALE_PORT_NO_SA_UPDATE, 1);
630 			}
631 
632 			/* Clear All Untouched entries */
633 			cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
634 			do {
635 				cpu_relax();
636 				if (cpsw_ale_control_get(ale, 0, ALE_AGEOUT))
637 					break;
638 			} while (time_after(timeout, jiffies));
639 			cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
640 
641 			/* Clear all mcast from ALE */
642 			cpsw_ale_flush_multicast(ale, ALE_ALL_PORTS, -1);
643 
644 			/* Flood All Unicast Packets to Host port */
645 			cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 1);
646 			dev_dbg(&ndev->dev, "promiscuity enabled\n");
647 		} else {
648 			/* Don't Flood All Unicast Packets to Host port */
649 			cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 0);
650 
651 			/* Enable Learn for all ports (host is port 0 and slaves are port 1 and up */
652 			for (i = 0; i <= cpsw->data.slaves; i++) {
653 				cpsw_ale_control_set(ale, i,
654 						     ALE_PORT_NOLEARN, 0);
655 				cpsw_ale_control_set(ale, i,
656 						     ALE_PORT_NO_SA_UPDATE, 0);
657 			}
658 			dev_dbg(&ndev->dev, "promiscuity disabled\n");
659 		}
660 	}
661 }
662 
663 static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
664 {
665 	struct cpsw_priv *priv = netdev_priv(ndev);
666 	struct cpsw_common *cpsw = priv->cpsw;
667 	int vid;
668 
669 	if (cpsw->data.dual_emac)
670 		vid = cpsw->slaves[priv->emac_port].port_vlan;
671 	else
672 		vid = cpsw->data.default_vlan;
673 
674 	if (ndev->flags & IFF_PROMISC) {
675 		/* Enable promiscuous mode */
676 		cpsw_set_promiscious(ndev, true);
677 		cpsw_ale_set_allmulti(cpsw->ale, IFF_ALLMULTI);
678 		return;
679 	} else {
680 		/* Disable promiscuous mode */
681 		cpsw_set_promiscious(ndev, false);
682 	}
683 
684 	/* Restore allmulti on vlans if necessary */
685 	cpsw_ale_set_allmulti(cpsw->ale, priv->ndev->flags & IFF_ALLMULTI);
686 
687 	/* Clear all mcast from ALE */
688 	cpsw_ale_flush_multicast(cpsw->ale, ALE_ALL_PORTS, vid);
689 
690 	if (!netdev_mc_empty(ndev)) {
691 		struct netdev_hw_addr *ha;
692 
693 		/* program multicast address list into ALE register */
694 		netdev_for_each_mc_addr(ha, ndev) {
695 			cpsw_add_mcast(cpsw, priv, (u8 *)ha->addr);
696 		}
697 	}
698 }
699 
700 static void cpsw_intr_enable(struct cpsw_common *cpsw)
701 {
702 	writel_relaxed(0xFF, &cpsw->wr_regs->tx_en);
703 	writel_relaxed(0xFF, &cpsw->wr_regs->rx_en);
704 
705 	cpdma_ctlr_int_ctrl(cpsw->dma, true);
706 	return;
707 }
708 
709 static void cpsw_intr_disable(struct cpsw_common *cpsw)
710 {
711 	writel_relaxed(0, &cpsw->wr_regs->tx_en);
712 	writel_relaxed(0, &cpsw->wr_regs->rx_en);
713 
714 	cpdma_ctlr_int_ctrl(cpsw->dma, false);
715 	return;
716 }
717 
718 static void cpsw_tx_handler(void *token, int len, int status)
719 {
720 	struct netdev_queue	*txq;
721 	struct sk_buff		*skb = token;
722 	struct net_device	*ndev = skb->dev;
723 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
724 
725 	/* Check whether the queue is stopped due to stalled tx dma, if the
726 	 * queue is stopped then start the queue as we have free desc for tx
727 	 */
728 	txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
729 	if (unlikely(netif_tx_queue_stopped(txq)))
730 		netif_tx_wake_queue(txq);
731 
732 	cpts_tx_timestamp(cpsw->cpts, skb);
733 	ndev->stats.tx_packets++;
734 	ndev->stats.tx_bytes += len;
735 	dev_kfree_skb_any(skb);
736 }
737 
738 static void cpsw_rx_vlan_encap(struct sk_buff *skb)
739 {
740 	struct cpsw_priv *priv = netdev_priv(skb->dev);
741 	struct cpsw_common *cpsw = priv->cpsw;
742 	u32 rx_vlan_encap_hdr = *((u32 *)skb->data);
743 	u16 vtag, vid, prio, pkt_type;
744 
745 	/* Remove VLAN header encapsulation word */
746 	skb_pull(skb, CPSW_RX_VLAN_ENCAP_HDR_SIZE);
747 
748 	pkt_type = (rx_vlan_encap_hdr >>
749 		    CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_SHIFT) &
750 		    CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_MSK;
751 	/* Ignore unknown & Priority-tagged packets*/
752 	if (pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_RESERV ||
753 	    pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_PRIO_TAG)
754 		return;
755 
756 	vid = (rx_vlan_encap_hdr >>
757 	       CPSW_RX_VLAN_ENCAP_HDR_VID_SHIFT) &
758 	       VLAN_VID_MASK;
759 	/* Ignore vid 0 and pass packet as is */
760 	if (!vid)
761 		return;
762 	/* Ignore default vlans in dual mac mode */
763 	if (cpsw->data.dual_emac &&
764 	    vid == cpsw->slaves[priv->emac_port].port_vlan)
765 		return;
766 
767 	prio = (rx_vlan_encap_hdr >>
768 		CPSW_RX_VLAN_ENCAP_HDR_PRIO_SHIFT) &
769 		CPSW_RX_VLAN_ENCAP_HDR_PRIO_MSK;
770 
771 	vtag = (prio << VLAN_PRIO_SHIFT) | vid;
772 	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
773 
774 	/* strip vlan tag for VLAN-tagged packet */
775 	if (pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_VLAN_TAG) {
776 		memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
777 		skb_pull(skb, VLAN_HLEN);
778 	}
779 }
780 
781 static void cpsw_rx_handler(void *token, int len, int status)
782 {
783 	struct cpdma_chan	*ch;
784 	struct sk_buff		*skb = token;
785 	struct sk_buff		*new_skb;
786 	struct net_device	*ndev = skb->dev;
787 	int			ret = 0;
788 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
789 
790 	cpsw_dual_emac_src_port_detect(cpsw, status, ndev, skb);
791 
792 	if (unlikely(status < 0) || unlikely(!netif_running(ndev))) {
793 		/* In dual emac mode check for all interfaces */
794 		if (cpsw->data.dual_emac && cpsw->usage_count &&
795 		    (status >= 0)) {
796 			/* The packet received is for the interface which
797 			 * is already down and the other interface is up
798 			 * and running, instead of freeing which results
799 			 * in reducing of the number of rx descriptor in
800 			 * DMA engine, requeue skb back to cpdma.
801 			 */
802 			new_skb = skb;
803 			goto requeue;
804 		}
805 
806 		/* the interface is going down, skbs are purged */
807 		dev_kfree_skb_any(skb);
808 		return;
809 	}
810 
811 	new_skb = netdev_alloc_skb_ip_align(ndev, cpsw->rx_packet_max);
812 	if (new_skb) {
813 		skb_copy_queue_mapping(new_skb, skb);
814 		skb_put(skb, len);
815 		if (status & CPDMA_RX_VLAN_ENCAP)
816 			cpsw_rx_vlan_encap(skb);
817 		cpts_rx_timestamp(cpsw->cpts, skb);
818 		skb->protocol = eth_type_trans(skb, ndev);
819 		netif_receive_skb(skb);
820 		ndev->stats.rx_bytes += len;
821 		ndev->stats.rx_packets++;
822 		kmemleak_not_leak(new_skb);
823 	} else {
824 		ndev->stats.rx_dropped++;
825 		new_skb = skb;
826 	}
827 
828 requeue:
829 	if (netif_dormant(ndev)) {
830 		dev_kfree_skb_any(new_skb);
831 		return;
832 	}
833 
834 	ch = cpsw->rxv[skb_get_queue_mapping(new_skb)].ch;
835 	ret = cpdma_chan_submit(ch, new_skb, new_skb->data,
836 				skb_tailroom(new_skb), 0);
837 	if (WARN_ON(ret < 0))
838 		dev_kfree_skb_any(new_skb);
839 }
840 
841 static void cpsw_split_res(struct net_device *ndev)
842 {
843 	struct cpsw_priv *priv = netdev_priv(ndev);
844 	u32 consumed_rate = 0, bigest_rate = 0;
845 	struct cpsw_common *cpsw = priv->cpsw;
846 	struct cpsw_vector *txv = cpsw->txv;
847 	int i, ch_weight, rlim_ch_num = 0;
848 	int budget, bigest_rate_ch = 0;
849 	u32 ch_rate, max_rate;
850 	int ch_budget = 0;
851 
852 	for (i = 0; i < cpsw->tx_ch_num; i++) {
853 		ch_rate = cpdma_chan_get_rate(txv[i].ch);
854 		if (!ch_rate)
855 			continue;
856 
857 		rlim_ch_num++;
858 		consumed_rate += ch_rate;
859 	}
860 
861 	if (cpsw->tx_ch_num == rlim_ch_num) {
862 		max_rate = consumed_rate;
863 	} else if (!rlim_ch_num) {
864 		ch_budget = CPSW_POLL_WEIGHT / cpsw->tx_ch_num;
865 		bigest_rate = 0;
866 		max_rate = consumed_rate;
867 	} else {
868 		max_rate = cpsw->speed * 1000;
869 
870 		/* if max_rate is less then expected due to reduced link speed,
871 		 * split proportionally according next potential max speed
872 		 */
873 		if (max_rate < consumed_rate)
874 			max_rate *= 10;
875 
876 		if (max_rate < consumed_rate)
877 			max_rate *= 10;
878 
879 		ch_budget = (consumed_rate * CPSW_POLL_WEIGHT) / max_rate;
880 		ch_budget = (CPSW_POLL_WEIGHT - ch_budget) /
881 			    (cpsw->tx_ch_num - rlim_ch_num);
882 		bigest_rate = (max_rate - consumed_rate) /
883 			      (cpsw->tx_ch_num - rlim_ch_num);
884 	}
885 
886 	/* split tx weight/budget */
887 	budget = CPSW_POLL_WEIGHT;
888 	for (i = 0; i < cpsw->tx_ch_num; i++) {
889 		ch_rate = cpdma_chan_get_rate(txv[i].ch);
890 		if (ch_rate) {
891 			txv[i].budget = (ch_rate * CPSW_POLL_WEIGHT) / max_rate;
892 			if (!txv[i].budget)
893 				txv[i].budget++;
894 			if (ch_rate > bigest_rate) {
895 				bigest_rate_ch = i;
896 				bigest_rate = ch_rate;
897 			}
898 
899 			ch_weight = (ch_rate * 100) / max_rate;
900 			if (!ch_weight)
901 				ch_weight++;
902 			cpdma_chan_set_weight(cpsw->txv[i].ch, ch_weight);
903 		} else {
904 			txv[i].budget = ch_budget;
905 			if (!bigest_rate_ch)
906 				bigest_rate_ch = i;
907 			cpdma_chan_set_weight(cpsw->txv[i].ch, 0);
908 		}
909 
910 		budget -= txv[i].budget;
911 	}
912 
913 	if (budget)
914 		txv[bigest_rate_ch].budget += budget;
915 
916 	/* split rx budget */
917 	budget = CPSW_POLL_WEIGHT;
918 	ch_budget = budget / cpsw->rx_ch_num;
919 	for (i = 0; i < cpsw->rx_ch_num; i++) {
920 		cpsw->rxv[i].budget = ch_budget;
921 		budget -= ch_budget;
922 	}
923 
924 	if (budget)
925 		cpsw->rxv[0].budget += budget;
926 }
927 
928 static irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id)
929 {
930 	struct cpsw_common *cpsw = dev_id;
931 
932 	writel(0, &cpsw->wr_regs->tx_en);
933 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);
934 
935 	if (cpsw->quirk_irq) {
936 		disable_irq_nosync(cpsw->irqs_table[1]);
937 		cpsw->tx_irq_disabled = true;
938 	}
939 
940 	napi_schedule(&cpsw->napi_tx);
941 	return IRQ_HANDLED;
942 }
943 
944 static irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id)
945 {
946 	struct cpsw_common *cpsw = dev_id;
947 
948 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);
949 	writel(0, &cpsw->wr_regs->rx_en);
950 
951 	if (cpsw->quirk_irq) {
952 		disable_irq_nosync(cpsw->irqs_table[0]);
953 		cpsw->rx_irq_disabled = true;
954 	}
955 
956 	napi_schedule(&cpsw->napi_rx);
957 	return IRQ_HANDLED;
958 }
959 
960 static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
961 {
962 	u32			ch_map;
963 	int			num_tx, cur_budget, ch;
964 	struct cpsw_common	*cpsw = napi_to_cpsw(napi_tx);
965 	struct cpsw_vector	*txv;
966 
967 	/* process every unprocessed channel */
968 	ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
969 	for (ch = 0, num_tx = 0; ch_map; ch_map >>= 1, ch++) {
970 		if (!(ch_map & 0x01))
971 			continue;
972 
973 		txv = &cpsw->txv[ch];
974 		if (unlikely(txv->budget > budget - num_tx))
975 			cur_budget = budget - num_tx;
976 		else
977 			cur_budget = txv->budget;
978 
979 		num_tx += cpdma_chan_process(txv->ch, cur_budget);
980 		if (num_tx >= budget)
981 			break;
982 	}
983 
984 	if (num_tx < budget) {
985 		napi_complete(napi_tx);
986 		writel(0xff, &cpsw->wr_regs->tx_en);
987 		if (cpsw->quirk_irq && cpsw->tx_irq_disabled) {
988 			cpsw->tx_irq_disabled = false;
989 			enable_irq(cpsw->irqs_table[1]);
990 		}
991 	}
992 
993 	return num_tx;
994 }
995 
996 static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
997 {
998 	u32			ch_map;
999 	int			num_rx, cur_budget, ch;
1000 	struct cpsw_common	*cpsw = napi_to_cpsw(napi_rx);
1001 	struct cpsw_vector	*rxv;
1002 
1003 	/* process every unprocessed channel */
1004 	ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
1005 	for (ch = 0, num_rx = 0; ch_map; ch_map >>= 1, ch++) {
1006 		if (!(ch_map & 0x01))
1007 			continue;
1008 
1009 		rxv = &cpsw->rxv[ch];
1010 		if (unlikely(rxv->budget > budget - num_rx))
1011 			cur_budget = budget - num_rx;
1012 		else
1013 			cur_budget = rxv->budget;
1014 
1015 		num_rx += cpdma_chan_process(rxv->ch, cur_budget);
1016 		if (num_rx >= budget)
1017 			break;
1018 	}
1019 
1020 	if (num_rx < budget) {
1021 		napi_complete_done(napi_rx, num_rx);
1022 		writel(0xff, &cpsw->wr_regs->rx_en);
1023 		if (cpsw->quirk_irq && cpsw->rx_irq_disabled) {
1024 			cpsw->rx_irq_disabled = false;
1025 			enable_irq(cpsw->irqs_table[0]);
1026 		}
1027 	}
1028 
1029 	return num_rx;
1030 }
1031 
1032 static inline void soft_reset(const char *module, void __iomem *reg)
1033 {
1034 	unsigned long timeout = jiffies + HZ;
1035 
1036 	writel_relaxed(1, reg);
1037 	do {
1038 		cpu_relax();
1039 	} while ((readl_relaxed(reg) & 1) && time_after(timeout, jiffies));
1040 
1041 	WARN(readl_relaxed(reg) & 1, "failed to soft-reset %s\n", module);
1042 }
1043 
1044 static void cpsw_set_slave_mac(struct cpsw_slave *slave,
1045 			       struct cpsw_priv *priv)
1046 {
1047 	slave_write(slave, mac_hi(priv->mac_addr), SA_HI);
1048 	slave_write(slave, mac_lo(priv->mac_addr), SA_LO);
1049 }
1050 
1051 static void _cpsw_adjust_link(struct cpsw_slave *slave,
1052 			      struct cpsw_priv *priv, bool *link)
1053 {
1054 	struct phy_device	*phy = slave->phy;
1055 	u32			mac_control = 0;
1056 	u32			slave_port;
1057 	struct cpsw_common *cpsw = priv->cpsw;
1058 
1059 	if (!phy)
1060 		return;
1061 
1062 	slave_port = cpsw_get_slave_port(slave->slave_num);
1063 
1064 	if (phy->link) {
1065 		mac_control = cpsw->data.mac_control;
1066 
1067 		/* enable forwarding */
1068 		cpsw_ale_control_set(cpsw->ale, slave_port,
1069 				     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1070 
1071 		if (phy->speed == 1000)
1072 			mac_control |= BIT(7);	/* GIGABITEN	*/
1073 		if (phy->duplex)
1074 			mac_control |= BIT(0);	/* FULLDUPLEXEN	*/
1075 
1076 		/* set speed_in input in case RMII mode is used in 100Mbps */
1077 		if (phy->speed == 100)
1078 			mac_control |= BIT(15);
1079 		/* in band mode only works in 10Mbps RGMII mode */
1080 		else if ((phy->speed == 10) && phy_interface_is_rgmii(phy))
1081 			mac_control |= BIT(18); /* In Band mode */
1082 
1083 		if (priv->rx_pause)
1084 			mac_control |= BIT(3);
1085 
1086 		if (priv->tx_pause)
1087 			mac_control |= BIT(4);
1088 
1089 		*link = true;
1090 	} else {
1091 		mac_control = 0;
1092 		/* disable forwarding */
1093 		cpsw_ale_control_set(cpsw->ale, slave_port,
1094 				     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1095 	}
1096 
1097 	if (mac_control != slave->mac_control) {
1098 		phy_print_status(phy);
1099 		writel_relaxed(mac_control, &slave->sliver->mac_control);
1100 	}
1101 
1102 	slave->mac_control = mac_control;
1103 }
1104 
1105 static int cpsw_get_common_speed(struct cpsw_common *cpsw)
1106 {
1107 	int i, speed;
1108 
1109 	for (i = 0, speed = 0; i < cpsw->data.slaves; i++)
1110 		if (cpsw->slaves[i].phy && cpsw->slaves[i].phy->link)
1111 			speed += cpsw->slaves[i].phy->speed;
1112 
1113 	return speed;
1114 }
1115 
1116 static int cpsw_need_resplit(struct cpsw_common *cpsw)
1117 {
1118 	int i, rlim_ch_num;
1119 	int speed, ch_rate;
1120 
1121 	/* re-split resources only in case speed was changed */
1122 	speed = cpsw_get_common_speed(cpsw);
1123 	if (speed == cpsw->speed || !speed)
1124 		return 0;
1125 
1126 	cpsw->speed = speed;
1127 
1128 	for (i = 0, rlim_ch_num = 0; i < cpsw->tx_ch_num; i++) {
1129 		ch_rate = cpdma_chan_get_rate(cpsw->txv[i].ch);
1130 		if (!ch_rate)
1131 			break;
1132 
1133 		rlim_ch_num++;
1134 	}
1135 
1136 	/* cases not dependent on speed */
1137 	if (!rlim_ch_num || rlim_ch_num == cpsw->tx_ch_num)
1138 		return 0;
1139 
1140 	return 1;
1141 }
1142 
1143 static void cpsw_adjust_link(struct net_device *ndev)
1144 {
1145 	struct cpsw_priv	*priv = netdev_priv(ndev);
1146 	struct cpsw_common	*cpsw = priv->cpsw;
1147 	bool			link = false;
1148 
1149 	for_each_slave(priv, _cpsw_adjust_link, priv, &link);
1150 
1151 	if (link) {
1152 		if (cpsw_need_resplit(cpsw))
1153 			cpsw_split_res(ndev);
1154 
1155 		netif_carrier_on(ndev);
1156 		if (netif_running(ndev))
1157 			netif_tx_wake_all_queues(ndev);
1158 	} else {
1159 		netif_carrier_off(ndev);
1160 		netif_tx_stop_all_queues(ndev);
1161 	}
1162 }
1163 
1164 static int cpsw_get_coalesce(struct net_device *ndev,
1165 				struct ethtool_coalesce *coal)
1166 {
1167 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1168 
1169 	coal->rx_coalesce_usecs = cpsw->coal_intvl;
1170 	return 0;
1171 }
1172 
1173 static int cpsw_set_coalesce(struct net_device *ndev,
1174 				struct ethtool_coalesce *coal)
1175 {
1176 	struct cpsw_priv *priv = netdev_priv(ndev);
1177 	u32 int_ctrl;
1178 	u32 num_interrupts = 0;
1179 	u32 prescale = 0;
1180 	u32 addnl_dvdr = 1;
1181 	u32 coal_intvl = 0;
1182 	struct cpsw_common *cpsw = priv->cpsw;
1183 
1184 	coal_intvl = coal->rx_coalesce_usecs;
1185 
1186 	int_ctrl =  readl(&cpsw->wr_regs->int_control);
1187 	prescale = cpsw->bus_freq_mhz * 4;
1188 
1189 	if (!coal->rx_coalesce_usecs) {
1190 		int_ctrl &= ~(CPSW_INTPRESCALE_MASK | CPSW_INTPACEEN);
1191 		goto update_return;
1192 	}
1193 
1194 	if (coal_intvl < CPSW_CMINTMIN_INTVL)
1195 		coal_intvl = CPSW_CMINTMIN_INTVL;
1196 
1197 	if (coal_intvl > CPSW_CMINTMAX_INTVL) {
1198 		/* Interrupt pacer works with 4us Pulse, we can
1199 		 * throttle further by dilating the 4us pulse.
1200 		 */
1201 		addnl_dvdr = CPSW_INTPRESCALE_MASK / prescale;
1202 
1203 		if (addnl_dvdr > 1) {
1204 			prescale *= addnl_dvdr;
1205 			if (coal_intvl > (CPSW_CMINTMAX_INTVL * addnl_dvdr))
1206 				coal_intvl = (CPSW_CMINTMAX_INTVL
1207 						* addnl_dvdr);
1208 		} else {
1209 			addnl_dvdr = 1;
1210 			coal_intvl = CPSW_CMINTMAX_INTVL;
1211 		}
1212 	}
1213 
1214 	num_interrupts = (1000 * addnl_dvdr) / coal_intvl;
1215 	writel(num_interrupts, &cpsw->wr_regs->rx_imax);
1216 	writel(num_interrupts, &cpsw->wr_regs->tx_imax);
1217 
1218 	int_ctrl |= CPSW_INTPACEEN;
1219 	int_ctrl &= (~CPSW_INTPRESCALE_MASK);
1220 	int_ctrl |= (prescale & CPSW_INTPRESCALE_MASK);
1221 
1222 update_return:
1223 	writel(int_ctrl, &cpsw->wr_regs->int_control);
1224 
1225 	cpsw_notice(priv, timer, "Set coalesce to %d usecs.\n", coal_intvl);
1226 	cpsw->coal_intvl = coal_intvl;
1227 
1228 	return 0;
1229 }
1230 
1231 static int cpsw_get_sset_count(struct net_device *ndev, int sset)
1232 {
1233 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1234 
1235 	switch (sset) {
1236 	case ETH_SS_STATS:
1237 		return (CPSW_STATS_COMMON_LEN +
1238 		       (cpsw->rx_ch_num + cpsw->tx_ch_num) *
1239 		       CPSW_STATS_CH_LEN);
1240 	default:
1241 		return -EOPNOTSUPP;
1242 	}
1243 }
1244 
1245 static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir)
1246 {
1247 	int ch_stats_len;
1248 	int line;
1249 	int i;
1250 
1251 	ch_stats_len = CPSW_STATS_CH_LEN * ch_num;
1252 	for (i = 0; i < ch_stats_len; i++) {
1253 		line = i % CPSW_STATS_CH_LEN;
1254 		snprintf(*p, ETH_GSTRING_LEN,
1255 			 "%s DMA chan %d: %s", rx_dir ? "Rx" : "Tx",
1256 			 i / CPSW_STATS_CH_LEN,
1257 			 cpsw_gstrings_ch_stats[line].stat_string);
1258 		*p += ETH_GSTRING_LEN;
1259 	}
1260 }
1261 
1262 static void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
1263 {
1264 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1265 	u8 *p = data;
1266 	int i;
1267 
1268 	switch (stringset) {
1269 	case ETH_SS_STATS:
1270 		for (i = 0; i < CPSW_STATS_COMMON_LEN; i++) {
1271 			memcpy(p, cpsw_gstrings_stats[i].stat_string,
1272 			       ETH_GSTRING_LEN);
1273 			p += ETH_GSTRING_LEN;
1274 		}
1275 
1276 		cpsw_add_ch_strings(&p, cpsw->rx_ch_num, 1);
1277 		cpsw_add_ch_strings(&p, cpsw->tx_ch_num, 0);
1278 		break;
1279 	}
1280 }
1281 
1282 static void cpsw_get_ethtool_stats(struct net_device *ndev,
1283 				    struct ethtool_stats *stats, u64 *data)
1284 {
1285 	u8 *p;
1286 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1287 	struct cpdma_chan_stats ch_stats;
1288 	int i, l, ch;
1289 
1290 	/* Collect Davinci CPDMA stats for Rx and Tx Channel */
1291 	for (l = 0; l < CPSW_STATS_COMMON_LEN; l++)
1292 		data[l] = readl(cpsw->hw_stats +
1293 				cpsw_gstrings_stats[l].stat_offset);
1294 
1295 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1296 		cpdma_chan_get_stats(cpsw->rxv[ch].ch, &ch_stats);
1297 		for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1298 			p = (u8 *)&ch_stats +
1299 				cpsw_gstrings_ch_stats[i].stat_offset;
1300 			data[l] = *(u32 *)p;
1301 		}
1302 	}
1303 
1304 	for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
1305 		cpdma_chan_get_stats(cpsw->txv[ch].ch, &ch_stats);
1306 		for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1307 			p = (u8 *)&ch_stats +
1308 				cpsw_gstrings_ch_stats[i].stat_offset;
1309 			data[l] = *(u32 *)p;
1310 		}
1311 	}
1312 }
1313 
1314 static inline int cpsw_tx_packet_submit(struct cpsw_priv *priv,
1315 					struct sk_buff *skb,
1316 					struct cpdma_chan *txch)
1317 {
1318 	struct cpsw_common *cpsw = priv->cpsw;
1319 
1320 	skb_tx_timestamp(skb);
1321 	return cpdma_chan_submit(txch, skb, skb->data, skb->len,
1322 				 priv->emac_port + cpsw->data.dual_emac);
1323 }
1324 
1325 static inline void cpsw_add_dual_emac_def_ale_entries(
1326 		struct cpsw_priv *priv, struct cpsw_slave *slave,
1327 		u32 slave_port)
1328 {
1329 	struct cpsw_common *cpsw = priv->cpsw;
1330 	u32 port_mask = 1 << slave_port | ALE_PORT_HOST;
1331 
1332 	if (cpsw->version == CPSW_VERSION_1)
1333 		slave_write(slave, slave->port_vlan, CPSW1_PORT_VLAN);
1334 	else
1335 		slave_write(slave, slave->port_vlan, CPSW2_PORT_VLAN);
1336 	cpsw_ale_add_vlan(cpsw->ale, slave->port_vlan, port_mask,
1337 			  port_mask, port_mask, 0);
1338 	cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1339 			   port_mask, ALE_VLAN, slave->port_vlan, 0);
1340 	cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
1341 			   HOST_PORT_NUM, ALE_VLAN |
1342 			   ALE_SECURE, slave->port_vlan);
1343 }
1344 
1345 static void soft_reset_slave(struct cpsw_slave *slave)
1346 {
1347 	char name[32];
1348 
1349 	snprintf(name, sizeof(name), "slave-%d", slave->slave_num);
1350 	soft_reset(name, &slave->sliver->soft_reset);
1351 }
1352 
1353 static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
1354 {
1355 	u32 slave_port;
1356 	struct phy_device *phy;
1357 	struct cpsw_common *cpsw = priv->cpsw;
1358 
1359 	soft_reset_slave(slave);
1360 
1361 	/* setup priority mapping */
1362 	writel_relaxed(RX_PRIORITY_MAPPING, &slave->sliver->rx_pri_map);
1363 
1364 	switch (cpsw->version) {
1365 	case CPSW_VERSION_1:
1366 		slave_write(slave, TX_PRIORITY_MAPPING, CPSW1_TX_PRI_MAP);
1367 		/* Increase RX FIFO size to 5 for supporting fullduplex
1368 		 * flow control mode
1369 		 */
1370 		slave_write(slave,
1371 			    (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1372 			    CPSW_MAX_BLKS_RX, CPSW1_MAX_BLKS);
1373 		break;
1374 	case CPSW_VERSION_2:
1375 	case CPSW_VERSION_3:
1376 	case CPSW_VERSION_4:
1377 		slave_write(slave, TX_PRIORITY_MAPPING, CPSW2_TX_PRI_MAP);
1378 		/* Increase RX FIFO size to 5 for supporting fullduplex
1379 		 * flow control mode
1380 		 */
1381 		slave_write(slave,
1382 			    (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1383 			    CPSW_MAX_BLKS_RX, CPSW2_MAX_BLKS);
1384 		break;
1385 	}
1386 
1387 	/* setup max packet size, and mac address */
1388 	writel_relaxed(cpsw->rx_packet_max, &slave->sliver->rx_maxlen);
1389 	cpsw_set_slave_mac(slave, priv);
1390 
1391 	slave->mac_control = 0;	/* no link yet */
1392 
1393 	slave_port = cpsw_get_slave_port(slave->slave_num);
1394 
1395 	if (cpsw->data.dual_emac)
1396 		cpsw_add_dual_emac_def_ale_entries(priv, slave, slave_port);
1397 	else
1398 		cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1399 				   1 << slave_port, 0, 0, ALE_MCAST_FWD_2);
1400 
1401 	if (slave->data->phy_node) {
1402 		phy = of_phy_connect(priv->ndev, slave->data->phy_node,
1403 				 &cpsw_adjust_link, 0, slave->data->phy_if);
1404 		if (!phy) {
1405 			dev_err(priv->dev, "phy \"%pOF\" not found on slave %d\n",
1406 				slave->data->phy_node,
1407 				slave->slave_num);
1408 			return;
1409 		}
1410 	} else {
1411 		phy = phy_connect(priv->ndev, slave->data->phy_id,
1412 				 &cpsw_adjust_link, slave->data->phy_if);
1413 		if (IS_ERR(phy)) {
1414 			dev_err(priv->dev,
1415 				"phy \"%s\" not found on slave %d, err %ld\n",
1416 				slave->data->phy_id, slave->slave_num,
1417 				PTR_ERR(phy));
1418 			return;
1419 		}
1420 	}
1421 
1422 	slave->phy = phy;
1423 
1424 	phy_attached_info(slave->phy);
1425 
1426 	phy_start(slave->phy);
1427 
1428 	/* Configure GMII_SEL register */
1429 	cpsw_phy_sel(cpsw->dev, slave->phy->interface, slave->slave_num);
1430 }
1431 
1432 static inline void cpsw_add_default_vlan(struct cpsw_priv *priv)
1433 {
1434 	struct cpsw_common *cpsw = priv->cpsw;
1435 	const int vlan = cpsw->data.default_vlan;
1436 	u32 reg;
1437 	int i;
1438 	int unreg_mcast_mask;
1439 
1440 	reg = (cpsw->version == CPSW_VERSION_1) ? CPSW1_PORT_VLAN :
1441 	       CPSW2_PORT_VLAN;
1442 
1443 	writel(vlan, &cpsw->host_port_regs->port_vlan);
1444 
1445 	for (i = 0; i < cpsw->data.slaves; i++)
1446 		slave_write(cpsw->slaves + i, vlan, reg);
1447 
1448 	if (priv->ndev->flags & IFF_ALLMULTI)
1449 		unreg_mcast_mask = ALE_ALL_PORTS;
1450 	else
1451 		unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
1452 
1453 	cpsw_ale_add_vlan(cpsw->ale, vlan, ALE_ALL_PORTS,
1454 			  ALE_ALL_PORTS, ALE_ALL_PORTS,
1455 			  unreg_mcast_mask);
1456 }
1457 
1458 static void cpsw_init_host_port(struct cpsw_priv *priv)
1459 {
1460 	u32 fifo_mode;
1461 	u32 control_reg;
1462 	struct cpsw_common *cpsw = priv->cpsw;
1463 
1464 	/* soft reset the controller and initialize ale */
1465 	soft_reset("cpsw", &cpsw->regs->soft_reset);
1466 	cpsw_ale_start(cpsw->ale);
1467 
1468 	/* switch to vlan unaware mode */
1469 	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_VLAN_AWARE,
1470 			     CPSW_ALE_VLAN_AWARE);
1471 	control_reg = readl(&cpsw->regs->control);
1472 	control_reg |= CPSW_VLAN_AWARE | CPSW_RX_VLAN_ENCAP;
1473 	writel(control_reg, &cpsw->regs->control);
1474 	fifo_mode = (cpsw->data.dual_emac) ? CPSW_FIFO_DUAL_MAC_MODE :
1475 		     CPSW_FIFO_NORMAL_MODE;
1476 	writel(fifo_mode, &cpsw->host_port_regs->tx_in_ctl);
1477 
1478 	/* setup host port priority mapping */
1479 	writel_relaxed(CPDMA_TX_PRIORITY_MAP,
1480 		       &cpsw->host_port_regs->cpdma_tx_pri_map);
1481 	writel_relaxed(0, &cpsw->host_port_regs->cpdma_rx_chan_map);
1482 
1483 	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM,
1484 			     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1485 
1486 	if (!cpsw->data.dual_emac) {
1487 		cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
1488 				   0, 0);
1489 		cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1490 				   ALE_PORT_HOST, 0, 0, ALE_MCAST_FWD_2);
1491 	}
1492 }
1493 
1494 static int cpsw_fill_rx_channels(struct cpsw_priv *priv)
1495 {
1496 	struct cpsw_common *cpsw = priv->cpsw;
1497 	struct sk_buff *skb;
1498 	int ch_buf_num;
1499 	int ch, i, ret;
1500 
1501 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1502 		ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);
1503 		for (i = 0; i < ch_buf_num; i++) {
1504 			skb = __netdev_alloc_skb_ip_align(priv->ndev,
1505 							  cpsw->rx_packet_max,
1506 							  GFP_KERNEL);
1507 			if (!skb) {
1508 				cpsw_err(priv, ifup, "cannot allocate skb\n");
1509 				return -ENOMEM;
1510 			}
1511 
1512 			skb_set_queue_mapping(skb, ch);
1513 			ret = cpdma_chan_submit(cpsw->rxv[ch].ch, skb,
1514 						skb->data, skb_tailroom(skb),
1515 						0);
1516 			if (ret < 0) {
1517 				cpsw_err(priv, ifup,
1518 					 "cannot submit skb to channel %d rx, error %d\n",
1519 					 ch, ret);
1520 				kfree_skb(skb);
1521 				return ret;
1522 			}
1523 			kmemleak_not_leak(skb);
1524 		}
1525 
1526 		cpsw_info(priv, ifup, "ch %d rx, submitted %d descriptors\n",
1527 			  ch, ch_buf_num);
1528 	}
1529 
1530 	return 0;
1531 }
1532 
1533 static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_common *cpsw)
1534 {
1535 	u32 slave_port;
1536 
1537 	slave_port = cpsw_get_slave_port(slave->slave_num);
1538 
1539 	if (!slave->phy)
1540 		return;
1541 	phy_stop(slave->phy);
1542 	phy_disconnect(slave->phy);
1543 	slave->phy = NULL;
1544 	cpsw_ale_control_set(cpsw->ale, slave_port,
1545 			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1546 	soft_reset_slave(slave);
1547 }
1548 
1549 static int cpsw_ndo_open(struct net_device *ndev)
1550 {
1551 	struct cpsw_priv *priv = netdev_priv(ndev);
1552 	struct cpsw_common *cpsw = priv->cpsw;
1553 	int ret;
1554 	u32 reg;
1555 
1556 	ret = pm_runtime_get_sync(cpsw->dev);
1557 	if (ret < 0) {
1558 		pm_runtime_put_noidle(cpsw->dev);
1559 		return ret;
1560 	}
1561 
1562 	netif_carrier_off(ndev);
1563 
1564 	/* Notify the stack of the actual queue counts. */
1565 	ret = netif_set_real_num_tx_queues(ndev, cpsw->tx_ch_num);
1566 	if (ret) {
1567 		dev_err(priv->dev, "cannot set real number of tx queues\n");
1568 		goto err_cleanup;
1569 	}
1570 
1571 	ret = netif_set_real_num_rx_queues(ndev, cpsw->rx_ch_num);
1572 	if (ret) {
1573 		dev_err(priv->dev, "cannot set real number of rx queues\n");
1574 		goto err_cleanup;
1575 	}
1576 
1577 	reg = cpsw->version;
1578 
1579 	dev_info(priv->dev, "initializing cpsw version %d.%d (%d)\n",
1580 		 CPSW_MAJOR_VERSION(reg), CPSW_MINOR_VERSION(reg),
1581 		 CPSW_RTL_VERSION(reg));
1582 
1583 	/* Initialize host and slave ports */
1584 	if (!cpsw->usage_count)
1585 		cpsw_init_host_port(priv);
1586 	for_each_slave(priv, cpsw_slave_open, priv);
1587 
1588 	/* Add default VLAN */
1589 	if (!cpsw->data.dual_emac)
1590 		cpsw_add_default_vlan(priv);
1591 	else
1592 		cpsw_ale_add_vlan(cpsw->ale, cpsw->data.default_vlan,
1593 				  ALE_ALL_PORTS, ALE_ALL_PORTS, 0, 0);
1594 
1595 	/* initialize shared resources for every ndev */
1596 	if (!cpsw->usage_count) {
1597 		/* disable priority elevation */
1598 		writel_relaxed(0, &cpsw->regs->ptype);
1599 
1600 		/* enable statistics collection only on all ports */
1601 		writel_relaxed(0x7, &cpsw->regs->stat_port_en);
1602 
1603 		/* Enable internal fifo flow control */
1604 		writel(0x7, &cpsw->regs->flow_control);
1605 
1606 		napi_enable(&cpsw->napi_rx);
1607 		napi_enable(&cpsw->napi_tx);
1608 
1609 		if (cpsw->tx_irq_disabled) {
1610 			cpsw->tx_irq_disabled = false;
1611 			enable_irq(cpsw->irqs_table[1]);
1612 		}
1613 
1614 		if (cpsw->rx_irq_disabled) {
1615 			cpsw->rx_irq_disabled = false;
1616 			enable_irq(cpsw->irqs_table[0]);
1617 		}
1618 
1619 		ret = cpsw_fill_rx_channels(priv);
1620 		if (ret < 0)
1621 			goto err_cleanup;
1622 
1623 		if (cpts_register(cpsw->cpts))
1624 			dev_err(priv->dev, "error registering cpts device\n");
1625 
1626 	}
1627 
1628 	/* Enable Interrupt pacing if configured */
1629 	if (cpsw->coal_intvl != 0) {
1630 		struct ethtool_coalesce coal;
1631 
1632 		coal.rx_coalesce_usecs = cpsw->coal_intvl;
1633 		cpsw_set_coalesce(ndev, &coal);
1634 	}
1635 
1636 	cpdma_ctlr_start(cpsw->dma);
1637 	cpsw_intr_enable(cpsw);
1638 	cpsw->usage_count++;
1639 
1640 	return 0;
1641 
1642 err_cleanup:
1643 	cpdma_ctlr_stop(cpsw->dma);
1644 	for_each_slave(priv, cpsw_slave_stop, cpsw);
1645 	pm_runtime_put_sync(cpsw->dev);
1646 	netif_carrier_off(priv->ndev);
1647 	return ret;
1648 }
1649 
1650 static int cpsw_ndo_stop(struct net_device *ndev)
1651 {
1652 	struct cpsw_priv *priv = netdev_priv(ndev);
1653 	struct cpsw_common *cpsw = priv->cpsw;
1654 
1655 	cpsw_info(priv, ifdown, "shutting down cpsw device\n");
1656 	netif_tx_stop_all_queues(priv->ndev);
1657 	netif_carrier_off(priv->ndev);
1658 
1659 	if (cpsw->usage_count <= 1) {
1660 		napi_disable(&cpsw->napi_rx);
1661 		napi_disable(&cpsw->napi_tx);
1662 		cpts_unregister(cpsw->cpts);
1663 		cpsw_intr_disable(cpsw);
1664 		cpdma_ctlr_stop(cpsw->dma);
1665 		cpsw_ale_stop(cpsw->ale);
1666 	}
1667 	for_each_slave(priv, cpsw_slave_stop, cpsw);
1668 
1669 	if (cpsw_need_resplit(cpsw))
1670 		cpsw_split_res(ndev);
1671 
1672 	cpsw->usage_count--;
1673 	pm_runtime_put_sync(cpsw->dev);
1674 	return 0;
1675 }
1676 
1677 static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
1678 				       struct net_device *ndev)
1679 {
1680 	struct cpsw_priv *priv = netdev_priv(ndev);
1681 	struct cpsw_common *cpsw = priv->cpsw;
1682 	struct cpts *cpts = cpsw->cpts;
1683 	struct netdev_queue *txq;
1684 	struct cpdma_chan *txch;
1685 	int ret, q_idx;
1686 
1687 	if (skb_padto(skb, CPSW_MIN_PACKET_SIZE)) {
1688 		cpsw_err(priv, tx_err, "packet pad failed\n");
1689 		ndev->stats.tx_dropped++;
1690 		return NET_XMIT_DROP;
1691 	}
1692 
1693 	if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
1694 	    cpts_is_tx_enabled(cpts) && cpts_can_timestamp(cpts, skb))
1695 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1696 
1697 	q_idx = skb_get_queue_mapping(skb);
1698 	if (q_idx >= cpsw->tx_ch_num)
1699 		q_idx = q_idx % cpsw->tx_ch_num;
1700 
1701 	txch = cpsw->txv[q_idx].ch;
1702 	txq = netdev_get_tx_queue(ndev, q_idx);
1703 	ret = cpsw_tx_packet_submit(priv, skb, txch);
1704 	if (unlikely(ret != 0)) {
1705 		cpsw_err(priv, tx_err, "desc submit failed\n");
1706 		goto fail;
1707 	}
1708 
1709 	/* If there is no more tx desc left free then we need to
1710 	 * tell the kernel to stop sending us tx frames.
1711 	 */
1712 	if (unlikely(!cpdma_check_free_tx_desc(txch))) {
1713 		netif_tx_stop_queue(txq);
1714 
1715 		/* Barrier, so that stop_queue visible to other cpus */
1716 		smp_mb__after_atomic();
1717 
1718 		if (cpdma_check_free_tx_desc(txch))
1719 			netif_tx_wake_queue(txq);
1720 	}
1721 
1722 	return NETDEV_TX_OK;
1723 fail:
1724 	ndev->stats.tx_dropped++;
1725 	netif_tx_stop_queue(txq);
1726 
1727 	/* Barrier, so that stop_queue visible to other cpus */
1728 	smp_mb__after_atomic();
1729 
1730 	if (cpdma_check_free_tx_desc(txch))
1731 		netif_tx_wake_queue(txq);
1732 
1733 	return NETDEV_TX_BUSY;
1734 }
1735 
1736 #if IS_ENABLED(CONFIG_TI_CPTS)
1737 
1738 static void cpsw_hwtstamp_v1(struct cpsw_common *cpsw)
1739 {
1740 	struct cpsw_slave *slave = &cpsw->slaves[cpsw->data.active_slave];
1741 	u32 ts_en, seq_id;
1742 
1743 	if (!cpts_is_tx_enabled(cpsw->cpts) &&
1744 	    !cpts_is_rx_enabled(cpsw->cpts)) {
1745 		slave_write(slave, 0, CPSW1_TS_CTL);
1746 		return;
1747 	}
1748 
1749 	seq_id = (30 << CPSW_V1_SEQ_ID_OFS_SHIFT) | ETH_P_1588;
1750 	ts_en = EVENT_MSG_BITS << CPSW_V1_MSG_TYPE_OFS;
1751 
1752 	if (cpts_is_tx_enabled(cpsw->cpts))
1753 		ts_en |= CPSW_V1_TS_TX_EN;
1754 
1755 	if (cpts_is_rx_enabled(cpsw->cpts))
1756 		ts_en |= CPSW_V1_TS_RX_EN;
1757 
1758 	slave_write(slave, ts_en, CPSW1_TS_CTL);
1759 	slave_write(slave, seq_id, CPSW1_TS_SEQ_LTYPE);
1760 }
1761 
1762 static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
1763 {
1764 	struct cpsw_slave *slave;
1765 	struct cpsw_common *cpsw = priv->cpsw;
1766 	u32 ctrl, mtype;
1767 
1768 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1769 
1770 	ctrl = slave_read(slave, CPSW2_CONTROL);
1771 	switch (cpsw->version) {
1772 	case CPSW_VERSION_2:
1773 		ctrl &= ~CTRL_V2_ALL_TS_MASK;
1774 
1775 		if (cpts_is_tx_enabled(cpsw->cpts))
1776 			ctrl |= CTRL_V2_TX_TS_BITS;
1777 
1778 		if (cpts_is_rx_enabled(cpsw->cpts))
1779 			ctrl |= CTRL_V2_RX_TS_BITS;
1780 		break;
1781 	case CPSW_VERSION_3:
1782 	default:
1783 		ctrl &= ~CTRL_V3_ALL_TS_MASK;
1784 
1785 		if (cpts_is_tx_enabled(cpsw->cpts))
1786 			ctrl |= CTRL_V3_TX_TS_BITS;
1787 
1788 		if (cpts_is_rx_enabled(cpsw->cpts))
1789 			ctrl |= CTRL_V3_RX_TS_BITS;
1790 		break;
1791 	}
1792 
1793 	mtype = (30 << TS_SEQ_ID_OFFSET_SHIFT) | EVENT_MSG_BITS;
1794 
1795 	slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
1796 	slave_write(slave, ctrl, CPSW2_CONTROL);
1797 	writel_relaxed(ETH_P_1588, &cpsw->regs->ts_ltype);
1798 }
1799 
1800 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
1801 {
1802 	struct cpsw_priv *priv = netdev_priv(dev);
1803 	struct hwtstamp_config cfg;
1804 	struct cpsw_common *cpsw = priv->cpsw;
1805 	struct cpts *cpts = cpsw->cpts;
1806 
1807 	if (cpsw->version != CPSW_VERSION_1 &&
1808 	    cpsw->version != CPSW_VERSION_2 &&
1809 	    cpsw->version != CPSW_VERSION_3)
1810 		return -EOPNOTSUPP;
1811 
1812 	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1813 		return -EFAULT;
1814 
1815 	/* reserved for future extensions */
1816 	if (cfg.flags)
1817 		return -EINVAL;
1818 
1819 	if (cfg.tx_type != HWTSTAMP_TX_OFF && cfg.tx_type != HWTSTAMP_TX_ON)
1820 		return -ERANGE;
1821 
1822 	switch (cfg.rx_filter) {
1823 	case HWTSTAMP_FILTER_NONE:
1824 		cpts_rx_enable(cpts, 0);
1825 		break;
1826 	case HWTSTAMP_FILTER_ALL:
1827 	case HWTSTAMP_FILTER_NTP_ALL:
1828 		return -ERANGE;
1829 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1830 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1831 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1832 		cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V1_L4_EVENT);
1833 		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
1834 		break;
1835 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1836 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1837 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1838 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1839 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1840 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1841 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1842 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1843 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1844 		cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V2_EVENT);
1845 		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
1846 		break;
1847 	default:
1848 		return -ERANGE;
1849 	}
1850 
1851 	cpts_tx_enable(cpts, cfg.tx_type == HWTSTAMP_TX_ON);
1852 
1853 	switch (cpsw->version) {
1854 	case CPSW_VERSION_1:
1855 		cpsw_hwtstamp_v1(cpsw);
1856 		break;
1857 	case CPSW_VERSION_2:
1858 	case CPSW_VERSION_3:
1859 		cpsw_hwtstamp_v2(priv);
1860 		break;
1861 	default:
1862 		WARN_ON(1);
1863 	}
1864 
1865 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1866 }
1867 
1868 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
1869 {
1870 	struct cpsw_common *cpsw = ndev_to_cpsw(dev);
1871 	struct cpts *cpts = cpsw->cpts;
1872 	struct hwtstamp_config cfg;
1873 
1874 	if (cpsw->version != CPSW_VERSION_1 &&
1875 	    cpsw->version != CPSW_VERSION_2 &&
1876 	    cpsw->version != CPSW_VERSION_3)
1877 		return -EOPNOTSUPP;
1878 
1879 	cfg.flags = 0;
1880 	cfg.tx_type = cpts_is_tx_enabled(cpts) ?
1881 		      HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1882 	cfg.rx_filter = (cpts_is_rx_enabled(cpts) ?
1883 			 cpts->rx_enable : HWTSTAMP_FILTER_NONE);
1884 
1885 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1886 }
1887 #else
1888 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
1889 {
1890 	return -EOPNOTSUPP;
1891 }
1892 
1893 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
1894 {
1895 	return -EOPNOTSUPP;
1896 }
1897 #endif /*CONFIG_TI_CPTS*/
1898 
1899 static int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
1900 {
1901 	struct cpsw_priv *priv = netdev_priv(dev);
1902 	struct cpsw_common *cpsw = priv->cpsw;
1903 	int slave_no = cpsw_slave_index(cpsw, priv);
1904 
1905 	if (!netif_running(dev))
1906 		return -EINVAL;
1907 
1908 	switch (cmd) {
1909 	case SIOCSHWTSTAMP:
1910 		return cpsw_hwtstamp_set(dev, req);
1911 	case SIOCGHWTSTAMP:
1912 		return cpsw_hwtstamp_get(dev, req);
1913 	}
1914 
1915 	if (!cpsw->slaves[slave_no].phy)
1916 		return -EOPNOTSUPP;
1917 	return phy_mii_ioctl(cpsw->slaves[slave_no].phy, req, cmd);
1918 }
1919 
1920 static void cpsw_ndo_tx_timeout(struct net_device *ndev)
1921 {
1922 	struct cpsw_priv *priv = netdev_priv(ndev);
1923 	struct cpsw_common *cpsw = priv->cpsw;
1924 	int ch;
1925 
1926 	cpsw_err(priv, tx_err, "transmit timeout, restarting dma\n");
1927 	ndev->stats.tx_errors++;
1928 	cpsw_intr_disable(cpsw);
1929 	for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
1930 		cpdma_chan_stop(cpsw->txv[ch].ch);
1931 		cpdma_chan_start(cpsw->txv[ch].ch);
1932 	}
1933 
1934 	cpsw_intr_enable(cpsw);
1935 	netif_trans_update(ndev);
1936 	netif_tx_wake_all_queues(ndev);
1937 }
1938 
1939 static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
1940 {
1941 	struct cpsw_priv *priv = netdev_priv(ndev);
1942 	struct sockaddr *addr = (struct sockaddr *)p;
1943 	struct cpsw_common *cpsw = priv->cpsw;
1944 	int flags = 0;
1945 	u16 vid = 0;
1946 	int ret;
1947 
1948 	if (!is_valid_ether_addr(addr->sa_data))
1949 		return -EADDRNOTAVAIL;
1950 
1951 	ret = pm_runtime_get_sync(cpsw->dev);
1952 	if (ret < 0) {
1953 		pm_runtime_put_noidle(cpsw->dev);
1954 		return ret;
1955 	}
1956 
1957 	if (cpsw->data.dual_emac) {
1958 		vid = cpsw->slaves[priv->emac_port].port_vlan;
1959 		flags = ALE_VLAN;
1960 	}
1961 
1962 	cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
1963 			   flags, vid);
1964 	cpsw_ale_add_ucast(cpsw->ale, addr->sa_data, HOST_PORT_NUM,
1965 			   flags, vid);
1966 
1967 	memcpy(priv->mac_addr, addr->sa_data, ETH_ALEN);
1968 	memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
1969 	for_each_slave(priv, cpsw_set_slave_mac, priv);
1970 
1971 	pm_runtime_put(cpsw->dev);
1972 
1973 	return 0;
1974 }
1975 
1976 #ifdef CONFIG_NET_POLL_CONTROLLER
1977 static void cpsw_ndo_poll_controller(struct net_device *ndev)
1978 {
1979 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1980 
1981 	cpsw_intr_disable(cpsw);
1982 	cpsw_rx_interrupt(cpsw->irqs_table[0], cpsw);
1983 	cpsw_tx_interrupt(cpsw->irqs_table[1], cpsw);
1984 	cpsw_intr_enable(cpsw);
1985 }
1986 #endif
1987 
1988 static inline int cpsw_add_vlan_ale_entry(struct cpsw_priv *priv,
1989 				unsigned short vid)
1990 {
1991 	int ret;
1992 	int unreg_mcast_mask = 0;
1993 	u32 port_mask;
1994 	struct cpsw_common *cpsw = priv->cpsw;
1995 
1996 	if (cpsw->data.dual_emac) {
1997 		port_mask = (1 << (priv->emac_port + 1)) | ALE_PORT_HOST;
1998 
1999 		if (priv->ndev->flags & IFF_ALLMULTI)
2000 			unreg_mcast_mask = port_mask;
2001 	} else {
2002 		port_mask = ALE_ALL_PORTS;
2003 
2004 		if (priv->ndev->flags & IFF_ALLMULTI)
2005 			unreg_mcast_mask = ALE_ALL_PORTS;
2006 		else
2007 			unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
2008 	}
2009 
2010 	ret = cpsw_ale_add_vlan(cpsw->ale, vid, port_mask, 0, port_mask,
2011 				unreg_mcast_mask);
2012 	if (ret != 0)
2013 		return ret;
2014 
2015 	ret = cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
2016 				 HOST_PORT_NUM, ALE_VLAN, vid);
2017 	if (ret != 0)
2018 		goto clean_vid;
2019 
2020 	ret = cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
2021 				 port_mask, ALE_VLAN, vid, 0);
2022 	if (ret != 0)
2023 		goto clean_vlan_ucast;
2024 	return 0;
2025 
2026 clean_vlan_ucast:
2027 	cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
2028 			   HOST_PORT_NUM, ALE_VLAN, vid);
2029 clean_vid:
2030 	cpsw_ale_del_vlan(cpsw->ale, vid, 0);
2031 	return ret;
2032 }
2033 
2034 static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
2035 				    __be16 proto, u16 vid)
2036 {
2037 	struct cpsw_priv *priv = netdev_priv(ndev);
2038 	struct cpsw_common *cpsw = priv->cpsw;
2039 	int ret;
2040 
2041 	if (vid == cpsw->data.default_vlan)
2042 		return 0;
2043 
2044 	ret = pm_runtime_get_sync(cpsw->dev);
2045 	if (ret < 0) {
2046 		pm_runtime_put_noidle(cpsw->dev);
2047 		return ret;
2048 	}
2049 
2050 	if (cpsw->data.dual_emac) {
2051 		/* In dual EMAC, reserved VLAN id should not be used for
2052 		 * creating VLAN interfaces as this can break the dual
2053 		 * EMAC port separation
2054 		 */
2055 		int i;
2056 
2057 		for (i = 0; i < cpsw->data.slaves; i++) {
2058 			if (vid == cpsw->slaves[i].port_vlan)
2059 				return -EINVAL;
2060 		}
2061 	}
2062 
2063 	dev_info(priv->dev, "Adding vlanid %d to vlan filter\n", vid);
2064 	ret = cpsw_add_vlan_ale_entry(priv, vid);
2065 
2066 	pm_runtime_put(cpsw->dev);
2067 	return ret;
2068 }
2069 
2070 static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
2071 				     __be16 proto, u16 vid)
2072 {
2073 	struct cpsw_priv *priv = netdev_priv(ndev);
2074 	struct cpsw_common *cpsw = priv->cpsw;
2075 	int ret;
2076 
2077 	if (vid == cpsw->data.default_vlan)
2078 		return 0;
2079 
2080 	ret = pm_runtime_get_sync(cpsw->dev);
2081 	if (ret < 0) {
2082 		pm_runtime_put_noidle(cpsw->dev);
2083 		return ret;
2084 	}
2085 
2086 	if (cpsw->data.dual_emac) {
2087 		int i;
2088 
2089 		for (i = 0; i < cpsw->data.slaves; i++) {
2090 			if (vid == cpsw->slaves[i].port_vlan)
2091 				return -EINVAL;
2092 		}
2093 	}
2094 
2095 	dev_info(priv->dev, "removing vlanid %d from vlan filter\n", vid);
2096 	ret = cpsw_ale_del_vlan(cpsw->ale, vid, 0);
2097 	if (ret != 0)
2098 		return ret;
2099 
2100 	ret = cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
2101 				 HOST_PORT_NUM, ALE_VLAN, vid);
2102 	if (ret != 0)
2103 		return ret;
2104 
2105 	ret = cpsw_ale_del_mcast(cpsw->ale, priv->ndev->broadcast,
2106 				 0, ALE_VLAN, vid);
2107 	pm_runtime_put(cpsw->dev);
2108 	return ret;
2109 }
2110 
2111 static int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate)
2112 {
2113 	struct cpsw_priv *priv = netdev_priv(ndev);
2114 	struct cpsw_common *cpsw = priv->cpsw;
2115 	struct cpsw_slave *slave;
2116 	u32 min_rate;
2117 	u32 ch_rate;
2118 	int i, ret;
2119 
2120 	ch_rate = netdev_get_tx_queue(ndev, queue)->tx_maxrate;
2121 	if (ch_rate == rate)
2122 		return 0;
2123 
2124 	ch_rate = rate * 1000;
2125 	min_rate = cpdma_chan_get_min_rate(cpsw->dma);
2126 	if ((ch_rate < min_rate && ch_rate)) {
2127 		dev_err(priv->dev, "The channel rate cannot be less than %dMbps",
2128 			min_rate);
2129 		return -EINVAL;
2130 	}
2131 
2132 	if (rate > cpsw->speed) {
2133 		dev_err(priv->dev, "The channel rate cannot be more than 2Gbps");
2134 		return -EINVAL;
2135 	}
2136 
2137 	ret = pm_runtime_get_sync(cpsw->dev);
2138 	if (ret < 0) {
2139 		pm_runtime_put_noidle(cpsw->dev);
2140 		return ret;
2141 	}
2142 
2143 	ret = cpdma_chan_set_rate(cpsw->txv[queue].ch, ch_rate);
2144 	pm_runtime_put(cpsw->dev);
2145 
2146 	if (ret)
2147 		return ret;
2148 
2149 	/* update rates for slaves tx queues */
2150 	for (i = 0; i < cpsw->data.slaves; i++) {
2151 		slave = &cpsw->slaves[i];
2152 		if (!slave->ndev)
2153 			continue;
2154 
2155 		netdev_get_tx_queue(slave->ndev, queue)->tx_maxrate = rate;
2156 	}
2157 
2158 	cpsw_split_res(ndev);
2159 	return ret;
2160 }
2161 
2162 static const struct net_device_ops cpsw_netdev_ops = {
2163 	.ndo_open		= cpsw_ndo_open,
2164 	.ndo_stop		= cpsw_ndo_stop,
2165 	.ndo_start_xmit		= cpsw_ndo_start_xmit,
2166 	.ndo_set_mac_address	= cpsw_ndo_set_mac_address,
2167 	.ndo_do_ioctl		= cpsw_ndo_ioctl,
2168 	.ndo_validate_addr	= eth_validate_addr,
2169 	.ndo_tx_timeout		= cpsw_ndo_tx_timeout,
2170 	.ndo_set_rx_mode	= cpsw_ndo_set_rx_mode,
2171 	.ndo_set_tx_maxrate	= cpsw_ndo_set_tx_maxrate,
2172 #ifdef CONFIG_NET_POLL_CONTROLLER
2173 	.ndo_poll_controller	= cpsw_ndo_poll_controller,
2174 #endif
2175 	.ndo_vlan_rx_add_vid	= cpsw_ndo_vlan_rx_add_vid,
2176 	.ndo_vlan_rx_kill_vid	= cpsw_ndo_vlan_rx_kill_vid,
2177 };
2178 
2179 static int cpsw_get_regs_len(struct net_device *ndev)
2180 {
2181 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2182 
2183 	return cpsw->data.ale_entries * ALE_ENTRY_WORDS * sizeof(u32);
2184 }
2185 
2186 static void cpsw_get_regs(struct net_device *ndev,
2187 			  struct ethtool_regs *regs, void *p)
2188 {
2189 	u32 *reg = p;
2190 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2191 
2192 	/* update CPSW IP version */
2193 	regs->version = cpsw->version;
2194 
2195 	cpsw_ale_dump(cpsw->ale, reg);
2196 }
2197 
2198 static void cpsw_get_drvinfo(struct net_device *ndev,
2199 			     struct ethtool_drvinfo *info)
2200 {
2201 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2202 	struct platform_device	*pdev = to_platform_device(cpsw->dev);
2203 
2204 	strlcpy(info->driver, "cpsw", sizeof(info->driver));
2205 	strlcpy(info->version, "1.0", sizeof(info->version));
2206 	strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
2207 }
2208 
2209 static u32 cpsw_get_msglevel(struct net_device *ndev)
2210 {
2211 	struct cpsw_priv *priv = netdev_priv(ndev);
2212 	return priv->msg_enable;
2213 }
2214 
2215 static void cpsw_set_msglevel(struct net_device *ndev, u32 value)
2216 {
2217 	struct cpsw_priv *priv = netdev_priv(ndev);
2218 	priv->msg_enable = value;
2219 }
2220 
2221 #if IS_ENABLED(CONFIG_TI_CPTS)
2222 static int cpsw_get_ts_info(struct net_device *ndev,
2223 			    struct ethtool_ts_info *info)
2224 {
2225 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2226 
2227 	info->so_timestamping =
2228 		SOF_TIMESTAMPING_TX_HARDWARE |
2229 		SOF_TIMESTAMPING_TX_SOFTWARE |
2230 		SOF_TIMESTAMPING_RX_HARDWARE |
2231 		SOF_TIMESTAMPING_RX_SOFTWARE |
2232 		SOF_TIMESTAMPING_SOFTWARE |
2233 		SOF_TIMESTAMPING_RAW_HARDWARE;
2234 	info->phc_index = cpsw->cpts->phc_index;
2235 	info->tx_types =
2236 		(1 << HWTSTAMP_TX_OFF) |
2237 		(1 << HWTSTAMP_TX_ON);
2238 	info->rx_filters =
2239 		(1 << HWTSTAMP_FILTER_NONE) |
2240 		(1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
2241 		(1 << HWTSTAMP_FILTER_PTP_V2_EVENT);
2242 	return 0;
2243 }
2244 #else
2245 static int cpsw_get_ts_info(struct net_device *ndev,
2246 			    struct ethtool_ts_info *info)
2247 {
2248 	info->so_timestamping =
2249 		SOF_TIMESTAMPING_TX_SOFTWARE |
2250 		SOF_TIMESTAMPING_RX_SOFTWARE |
2251 		SOF_TIMESTAMPING_SOFTWARE;
2252 	info->phc_index = -1;
2253 	info->tx_types = 0;
2254 	info->rx_filters = 0;
2255 	return 0;
2256 }
2257 #endif
2258 
2259 static int cpsw_get_link_ksettings(struct net_device *ndev,
2260 				   struct ethtool_link_ksettings *ecmd)
2261 {
2262 	struct cpsw_priv *priv = netdev_priv(ndev);
2263 	struct cpsw_common *cpsw = priv->cpsw;
2264 	int slave_no = cpsw_slave_index(cpsw, priv);
2265 
2266 	if (!cpsw->slaves[slave_no].phy)
2267 		return -EOPNOTSUPP;
2268 
2269 	phy_ethtool_ksettings_get(cpsw->slaves[slave_no].phy, ecmd);
2270 	return 0;
2271 }
2272 
2273 static int cpsw_set_link_ksettings(struct net_device *ndev,
2274 				   const struct ethtool_link_ksettings *ecmd)
2275 {
2276 	struct cpsw_priv *priv = netdev_priv(ndev);
2277 	struct cpsw_common *cpsw = priv->cpsw;
2278 	int slave_no = cpsw_slave_index(cpsw, priv);
2279 
2280 	if (cpsw->slaves[slave_no].phy)
2281 		return phy_ethtool_ksettings_set(cpsw->slaves[slave_no].phy,
2282 						 ecmd);
2283 	else
2284 		return -EOPNOTSUPP;
2285 }
2286 
2287 static void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2288 {
2289 	struct cpsw_priv *priv = netdev_priv(ndev);
2290 	struct cpsw_common *cpsw = priv->cpsw;
2291 	int slave_no = cpsw_slave_index(cpsw, priv);
2292 
2293 	wol->supported = 0;
2294 	wol->wolopts = 0;
2295 
2296 	if (cpsw->slaves[slave_no].phy)
2297 		phy_ethtool_get_wol(cpsw->slaves[slave_no].phy, wol);
2298 }
2299 
2300 static int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2301 {
2302 	struct cpsw_priv *priv = netdev_priv(ndev);
2303 	struct cpsw_common *cpsw = priv->cpsw;
2304 	int slave_no = cpsw_slave_index(cpsw, priv);
2305 
2306 	if (cpsw->slaves[slave_no].phy)
2307 		return phy_ethtool_set_wol(cpsw->slaves[slave_no].phy, wol);
2308 	else
2309 		return -EOPNOTSUPP;
2310 }
2311 
2312 static void cpsw_get_pauseparam(struct net_device *ndev,
2313 				struct ethtool_pauseparam *pause)
2314 {
2315 	struct cpsw_priv *priv = netdev_priv(ndev);
2316 
2317 	pause->autoneg = AUTONEG_DISABLE;
2318 	pause->rx_pause = priv->rx_pause ? true : false;
2319 	pause->tx_pause = priv->tx_pause ? true : false;
2320 }
2321 
2322 static int cpsw_set_pauseparam(struct net_device *ndev,
2323 			       struct ethtool_pauseparam *pause)
2324 {
2325 	struct cpsw_priv *priv = netdev_priv(ndev);
2326 	bool link;
2327 
2328 	priv->rx_pause = pause->rx_pause ? true : false;
2329 	priv->tx_pause = pause->tx_pause ? true : false;
2330 
2331 	for_each_slave(priv, _cpsw_adjust_link, priv, &link);
2332 	return 0;
2333 }
2334 
2335 static int cpsw_ethtool_op_begin(struct net_device *ndev)
2336 {
2337 	struct cpsw_priv *priv = netdev_priv(ndev);
2338 	struct cpsw_common *cpsw = priv->cpsw;
2339 	int ret;
2340 
2341 	ret = pm_runtime_get_sync(cpsw->dev);
2342 	if (ret < 0) {
2343 		cpsw_err(priv, drv, "ethtool begin failed %d\n", ret);
2344 		pm_runtime_put_noidle(cpsw->dev);
2345 	}
2346 
2347 	return ret;
2348 }
2349 
2350 static void cpsw_ethtool_op_complete(struct net_device *ndev)
2351 {
2352 	struct cpsw_priv *priv = netdev_priv(ndev);
2353 	int ret;
2354 
2355 	ret = pm_runtime_put(priv->cpsw->dev);
2356 	if (ret < 0)
2357 		cpsw_err(priv, drv, "ethtool complete failed %d\n", ret);
2358 }
2359 
2360 static void cpsw_get_channels(struct net_device *ndev,
2361 			      struct ethtool_channels *ch)
2362 {
2363 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2364 
2365 	ch->max_combined = 0;
2366 	ch->max_rx = CPSW_MAX_QUEUES;
2367 	ch->max_tx = CPSW_MAX_QUEUES;
2368 	ch->max_other = 0;
2369 	ch->other_count = 0;
2370 	ch->rx_count = cpsw->rx_ch_num;
2371 	ch->tx_count = cpsw->tx_ch_num;
2372 	ch->combined_count = 0;
2373 }
2374 
2375 static int cpsw_check_ch_settings(struct cpsw_common *cpsw,
2376 				  struct ethtool_channels *ch)
2377 {
2378 	if (ch->combined_count)
2379 		return -EINVAL;
2380 
2381 	/* verify we have at least one channel in each direction */
2382 	if (!ch->rx_count || !ch->tx_count)
2383 		return -EINVAL;
2384 
2385 	if (ch->rx_count > cpsw->data.channels ||
2386 	    ch->tx_count > cpsw->data.channels)
2387 		return -EINVAL;
2388 
2389 	return 0;
2390 }
2391 
2392 static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx)
2393 {
2394 	struct cpsw_common *cpsw = priv->cpsw;
2395 	void (*handler)(void *, int, int);
2396 	struct netdev_queue *queue;
2397 	struct cpsw_vector *vec;
2398 	int ret, *ch;
2399 
2400 	if (rx) {
2401 		ch = &cpsw->rx_ch_num;
2402 		vec = cpsw->rxv;
2403 		handler = cpsw_rx_handler;
2404 	} else {
2405 		ch = &cpsw->tx_ch_num;
2406 		vec = cpsw->txv;
2407 		handler = cpsw_tx_handler;
2408 	}
2409 
2410 	while (*ch < ch_num) {
2411 		vec[*ch].ch = cpdma_chan_create(cpsw->dma, *ch, handler, rx);
2412 		queue = netdev_get_tx_queue(priv->ndev, *ch);
2413 		queue->tx_maxrate = 0;
2414 
2415 		if (IS_ERR(vec[*ch].ch))
2416 			return PTR_ERR(vec[*ch].ch);
2417 
2418 		if (!vec[*ch].ch)
2419 			return -EINVAL;
2420 
2421 		cpsw_info(priv, ifup, "created new %d %s channel\n", *ch,
2422 			  (rx ? "rx" : "tx"));
2423 		(*ch)++;
2424 	}
2425 
2426 	while (*ch > ch_num) {
2427 		(*ch)--;
2428 
2429 		ret = cpdma_chan_destroy(vec[*ch].ch);
2430 		if (ret)
2431 			return ret;
2432 
2433 		cpsw_info(priv, ifup, "destroyed %d %s channel\n", *ch,
2434 			  (rx ? "rx" : "tx"));
2435 	}
2436 
2437 	return 0;
2438 }
2439 
2440 static int cpsw_update_channels(struct cpsw_priv *priv,
2441 				struct ethtool_channels *ch)
2442 {
2443 	int ret;
2444 
2445 	ret = cpsw_update_channels_res(priv, ch->rx_count, 1);
2446 	if (ret)
2447 		return ret;
2448 
2449 	ret = cpsw_update_channels_res(priv, ch->tx_count, 0);
2450 	if (ret)
2451 		return ret;
2452 
2453 	return 0;
2454 }
2455 
2456 static void cpsw_suspend_data_pass(struct net_device *ndev)
2457 {
2458 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2459 	struct cpsw_slave *slave;
2460 	int i;
2461 
2462 	/* Disable NAPI scheduling */
2463 	cpsw_intr_disable(cpsw);
2464 
2465 	/* Stop all transmit queues for every network device.
2466 	 * Disable re-using rx descriptors with dormant_on.
2467 	 */
2468 	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2469 		if (!(slave->ndev && netif_running(slave->ndev)))
2470 			continue;
2471 
2472 		netif_tx_stop_all_queues(slave->ndev);
2473 		netif_dormant_on(slave->ndev);
2474 	}
2475 
2476 	/* Handle rest of tx packets and stop cpdma channels */
2477 	cpdma_ctlr_stop(cpsw->dma);
2478 }
2479 
2480 static int cpsw_resume_data_pass(struct net_device *ndev)
2481 {
2482 	struct cpsw_priv *priv = netdev_priv(ndev);
2483 	struct cpsw_common *cpsw = priv->cpsw;
2484 	struct cpsw_slave *slave;
2485 	int i, ret;
2486 
2487 	/* Allow rx packets handling */
2488 	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++)
2489 		if (slave->ndev && netif_running(slave->ndev))
2490 			netif_dormant_off(slave->ndev);
2491 
2492 	/* After this receive is started */
2493 	if (cpsw->usage_count) {
2494 		ret = cpsw_fill_rx_channels(priv);
2495 		if (ret)
2496 			return ret;
2497 
2498 		cpdma_ctlr_start(cpsw->dma);
2499 		cpsw_intr_enable(cpsw);
2500 	}
2501 
2502 	/* Resume transmit for every affected interface */
2503 	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++)
2504 		if (slave->ndev && netif_running(slave->ndev))
2505 			netif_tx_start_all_queues(slave->ndev);
2506 
2507 	return 0;
2508 }
2509 
2510 static int cpsw_set_channels(struct net_device *ndev,
2511 			     struct ethtool_channels *chs)
2512 {
2513 	struct cpsw_priv *priv = netdev_priv(ndev);
2514 	struct cpsw_common *cpsw = priv->cpsw;
2515 	struct cpsw_slave *slave;
2516 	int i, ret;
2517 
2518 	ret = cpsw_check_ch_settings(cpsw, chs);
2519 	if (ret < 0)
2520 		return ret;
2521 
2522 	cpsw_suspend_data_pass(ndev);
2523 	ret = cpsw_update_channels(priv, chs);
2524 	if (ret)
2525 		goto err;
2526 
2527 	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2528 		if (!(slave->ndev && netif_running(slave->ndev)))
2529 			continue;
2530 
2531 		/* Inform stack about new count of queues */
2532 		ret = netif_set_real_num_tx_queues(slave->ndev,
2533 						   cpsw->tx_ch_num);
2534 		if (ret) {
2535 			dev_err(priv->dev, "cannot set real number of tx queues\n");
2536 			goto err;
2537 		}
2538 
2539 		ret = netif_set_real_num_rx_queues(slave->ndev,
2540 						   cpsw->rx_ch_num);
2541 		if (ret) {
2542 			dev_err(priv->dev, "cannot set real number of rx queues\n");
2543 			goto err;
2544 		}
2545 	}
2546 
2547 	if (cpsw->usage_count)
2548 		cpsw_split_res(ndev);
2549 
2550 	ret = cpsw_resume_data_pass(ndev);
2551 	if (!ret)
2552 		return 0;
2553 err:
2554 	dev_err(priv->dev, "cannot update channels number, closing device\n");
2555 	dev_close(ndev);
2556 	return ret;
2557 }
2558 
2559 static int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata)
2560 {
2561 	struct cpsw_priv *priv = netdev_priv(ndev);
2562 	struct cpsw_common *cpsw = priv->cpsw;
2563 	int slave_no = cpsw_slave_index(cpsw, priv);
2564 
2565 	if (cpsw->slaves[slave_no].phy)
2566 		return phy_ethtool_get_eee(cpsw->slaves[slave_no].phy, edata);
2567 	else
2568 		return -EOPNOTSUPP;
2569 }
2570 
2571 static int cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
2572 {
2573 	struct cpsw_priv *priv = netdev_priv(ndev);
2574 	struct cpsw_common *cpsw = priv->cpsw;
2575 	int slave_no = cpsw_slave_index(cpsw, priv);
2576 
2577 	if (cpsw->slaves[slave_no].phy)
2578 		return phy_ethtool_set_eee(cpsw->slaves[slave_no].phy, edata);
2579 	else
2580 		return -EOPNOTSUPP;
2581 }
2582 
2583 static int cpsw_nway_reset(struct net_device *ndev)
2584 {
2585 	struct cpsw_priv *priv = netdev_priv(ndev);
2586 	struct cpsw_common *cpsw = priv->cpsw;
2587 	int slave_no = cpsw_slave_index(cpsw, priv);
2588 
2589 	if (cpsw->slaves[slave_no].phy)
2590 		return genphy_restart_aneg(cpsw->slaves[slave_no].phy);
2591 	else
2592 		return -EOPNOTSUPP;
2593 }
2594 
2595 static void cpsw_get_ringparam(struct net_device *ndev,
2596 			       struct ethtool_ringparam *ering)
2597 {
2598 	struct cpsw_priv *priv = netdev_priv(ndev);
2599 	struct cpsw_common *cpsw = priv->cpsw;
2600 
2601 	/* not supported */
2602 	ering->tx_max_pending = 0;
2603 	ering->tx_pending = cpdma_get_num_tx_descs(cpsw->dma);
2604 	ering->rx_max_pending = descs_pool_size - CPSW_MAX_QUEUES;
2605 	ering->rx_pending = cpdma_get_num_rx_descs(cpsw->dma);
2606 }
2607 
2608 static int cpsw_set_ringparam(struct net_device *ndev,
2609 			      struct ethtool_ringparam *ering)
2610 {
2611 	struct cpsw_priv *priv = netdev_priv(ndev);
2612 	struct cpsw_common *cpsw = priv->cpsw;
2613 	int ret;
2614 
2615 	/* ignore ering->tx_pending - only rx_pending adjustment is supported */
2616 
2617 	if (ering->rx_mini_pending || ering->rx_jumbo_pending ||
2618 	    ering->rx_pending < CPSW_MAX_QUEUES ||
2619 	    ering->rx_pending > (descs_pool_size - CPSW_MAX_QUEUES))
2620 		return -EINVAL;
2621 
2622 	if (ering->rx_pending == cpdma_get_num_rx_descs(cpsw->dma))
2623 		return 0;
2624 
2625 	cpsw_suspend_data_pass(ndev);
2626 
2627 	cpdma_set_num_rx_descs(cpsw->dma, ering->rx_pending);
2628 
2629 	if (cpsw->usage_count)
2630 		cpdma_chan_split_pool(cpsw->dma);
2631 
2632 	ret = cpsw_resume_data_pass(ndev);
2633 	if (!ret)
2634 		return 0;
2635 
2636 	dev_err(&ndev->dev, "cannot set ring params, closing device\n");
2637 	dev_close(ndev);
2638 	return ret;
2639 }
2640 
2641 static const struct ethtool_ops cpsw_ethtool_ops = {
2642 	.get_drvinfo	= cpsw_get_drvinfo,
2643 	.get_msglevel	= cpsw_get_msglevel,
2644 	.set_msglevel	= cpsw_set_msglevel,
2645 	.get_link	= ethtool_op_get_link,
2646 	.get_ts_info	= cpsw_get_ts_info,
2647 	.get_coalesce	= cpsw_get_coalesce,
2648 	.set_coalesce	= cpsw_set_coalesce,
2649 	.get_sset_count		= cpsw_get_sset_count,
2650 	.get_strings		= cpsw_get_strings,
2651 	.get_ethtool_stats	= cpsw_get_ethtool_stats,
2652 	.get_pauseparam		= cpsw_get_pauseparam,
2653 	.set_pauseparam		= cpsw_set_pauseparam,
2654 	.get_wol	= cpsw_get_wol,
2655 	.set_wol	= cpsw_set_wol,
2656 	.get_regs_len	= cpsw_get_regs_len,
2657 	.get_regs	= cpsw_get_regs,
2658 	.begin		= cpsw_ethtool_op_begin,
2659 	.complete	= cpsw_ethtool_op_complete,
2660 	.get_channels	= cpsw_get_channels,
2661 	.set_channels	= cpsw_set_channels,
2662 	.get_link_ksettings	= cpsw_get_link_ksettings,
2663 	.set_link_ksettings	= cpsw_set_link_ksettings,
2664 	.get_eee	= cpsw_get_eee,
2665 	.set_eee	= cpsw_set_eee,
2666 	.nway_reset	= cpsw_nway_reset,
2667 	.get_ringparam = cpsw_get_ringparam,
2668 	.set_ringparam = cpsw_set_ringparam,
2669 };
2670 
2671 static void cpsw_slave_init(struct cpsw_slave *slave, struct cpsw_common *cpsw,
2672 			    u32 slave_reg_ofs, u32 sliver_reg_ofs)
2673 {
2674 	void __iomem		*regs = cpsw->regs;
2675 	int			slave_num = slave->slave_num;
2676 	struct cpsw_slave_data	*data = cpsw->data.slave_data + slave_num;
2677 
2678 	slave->data	= data;
2679 	slave->regs	= regs + slave_reg_ofs;
2680 	slave->sliver	= regs + sliver_reg_ofs;
2681 	slave->port_vlan = data->dual_emac_res_vlan;
2682 }
2683 
2684 static int cpsw_probe_dt(struct cpsw_platform_data *data,
2685 			 struct platform_device *pdev)
2686 {
2687 	struct device_node *node = pdev->dev.of_node;
2688 	struct device_node *slave_node;
2689 	int i = 0, ret;
2690 	u32 prop;
2691 
2692 	if (!node)
2693 		return -EINVAL;
2694 
2695 	if (of_property_read_u32(node, "slaves", &prop)) {
2696 		dev_err(&pdev->dev, "Missing slaves property in the DT.\n");
2697 		return -EINVAL;
2698 	}
2699 	data->slaves = prop;
2700 
2701 	if (of_property_read_u32(node, "active_slave", &prop)) {
2702 		dev_err(&pdev->dev, "Missing active_slave property in the DT.\n");
2703 		return -EINVAL;
2704 	}
2705 	data->active_slave = prop;
2706 
2707 	data->slave_data = devm_kzalloc(&pdev->dev, data->slaves
2708 					* sizeof(struct cpsw_slave_data),
2709 					GFP_KERNEL);
2710 	if (!data->slave_data)
2711 		return -ENOMEM;
2712 
2713 	if (of_property_read_u32(node, "cpdma_channels", &prop)) {
2714 		dev_err(&pdev->dev, "Missing cpdma_channels property in the DT.\n");
2715 		return -EINVAL;
2716 	}
2717 	data->channels = prop;
2718 
2719 	if (of_property_read_u32(node, "ale_entries", &prop)) {
2720 		dev_err(&pdev->dev, "Missing ale_entries property in the DT.\n");
2721 		return -EINVAL;
2722 	}
2723 	data->ale_entries = prop;
2724 
2725 	if (of_property_read_u32(node, "bd_ram_size", &prop)) {
2726 		dev_err(&pdev->dev, "Missing bd_ram_size property in the DT.\n");
2727 		return -EINVAL;
2728 	}
2729 	data->bd_ram_size = prop;
2730 
2731 	if (of_property_read_u32(node, "mac_control", &prop)) {
2732 		dev_err(&pdev->dev, "Missing mac_control property in the DT.\n");
2733 		return -EINVAL;
2734 	}
2735 	data->mac_control = prop;
2736 
2737 	if (of_property_read_bool(node, "dual_emac"))
2738 		data->dual_emac = 1;
2739 
2740 	/*
2741 	 * Populate all the child nodes here...
2742 	 */
2743 	ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
2744 	/* We do not want to force this, as in some cases may not have child */
2745 	if (ret)
2746 		dev_warn(&pdev->dev, "Doesn't have any child node\n");
2747 
2748 	for_each_available_child_of_node(node, slave_node) {
2749 		struct cpsw_slave_data *slave_data = data->slave_data + i;
2750 		const void *mac_addr = NULL;
2751 		int lenp;
2752 		const __be32 *parp;
2753 
2754 		/* This is no slave child node, continue */
2755 		if (strcmp(slave_node->name, "slave"))
2756 			continue;
2757 
2758 		slave_data->phy_node = of_parse_phandle(slave_node,
2759 							"phy-handle", 0);
2760 		parp = of_get_property(slave_node, "phy_id", &lenp);
2761 		if (slave_data->phy_node) {
2762 			dev_dbg(&pdev->dev,
2763 				"slave[%d] using phy-handle=\"%pOF\"\n",
2764 				i, slave_data->phy_node);
2765 		} else if (of_phy_is_fixed_link(slave_node)) {
2766 			/* In the case of a fixed PHY, the DT node associated
2767 			 * to the PHY is the Ethernet MAC DT node.
2768 			 */
2769 			ret = of_phy_register_fixed_link(slave_node);
2770 			if (ret) {
2771 				if (ret != -EPROBE_DEFER)
2772 					dev_err(&pdev->dev, "failed to register fixed-link phy: %d\n", ret);
2773 				return ret;
2774 			}
2775 			slave_data->phy_node = of_node_get(slave_node);
2776 		} else if (parp) {
2777 			u32 phyid;
2778 			struct device_node *mdio_node;
2779 			struct platform_device *mdio;
2780 
2781 			if (lenp != (sizeof(__be32) * 2)) {
2782 				dev_err(&pdev->dev, "Invalid slave[%d] phy_id property\n", i);
2783 				goto no_phy_slave;
2784 			}
2785 			mdio_node = of_find_node_by_phandle(be32_to_cpup(parp));
2786 			phyid = be32_to_cpup(parp+1);
2787 			mdio = of_find_device_by_node(mdio_node);
2788 			of_node_put(mdio_node);
2789 			if (!mdio) {
2790 				dev_err(&pdev->dev, "Missing mdio platform device\n");
2791 				return -EINVAL;
2792 			}
2793 			snprintf(slave_data->phy_id, sizeof(slave_data->phy_id),
2794 				 PHY_ID_FMT, mdio->name, phyid);
2795 			put_device(&mdio->dev);
2796 		} else {
2797 			dev_err(&pdev->dev,
2798 				"No slave[%d] phy_id, phy-handle, or fixed-link property\n",
2799 				i);
2800 			goto no_phy_slave;
2801 		}
2802 		slave_data->phy_if = of_get_phy_mode(slave_node);
2803 		if (slave_data->phy_if < 0) {
2804 			dev_err(&pdev->dev, "Missing or malformed slave[%d] phy-mode property\n",
2805 				i);
2806 			return slave_data->phy_if;
2807 		}
2808 
2809 no_phy_slave:
2810 		mac_addr = of_get_mac_address(slave_node);
2811 		if (mac_addr) {
2812 			memcpy(slave_data->mac_addr, mac_addr, ETH_ALEN);
2813 		} else {
2814 			ret = ti_cm_get_macid(&pdev->dev, i,
2815 					      slave_data->mac_addr);
2816 			if (ret)
2817 				return ret;
2818 		}
2819 		if (data->dual_emac) {
2820 			if (of_property_read_u32(slave_node, "dual_emac_res_vlan",
2821 						 &prop)) {
2822 				dev_err(&pdev->dev, "Missing dual_emac_res_vlan in DT.\n");
2823 				slave_data->dual_emac_res_vlan = i+1;
2824 				dev_err(&pdev->dev, "Using %d as Reserved VLAN for %d slave\n",
2825 					slave_data->dual_emac_res_vlan, i);
2826 			} else {
2827 				slave_data->dual_emac_res_vlan = prop;
2828 			}
2829 		}
2830 
2831 		i++;
2832 		if (i == data->slaves)
2833 			break;
2834 	}
2835 
2836 	return 0;
2837 }
2838 
2839 static void cpsw_remove_dt(struct platform_device *pdev)
2840 {
2841 	struct net_device *ndev = platform_get_drvdata(pdev);
2842 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2843 	struct cpsw_platform_data *data = &cpsw->data;
2844 	struct device_node *node = pdev->dev.of_node;
2845 	struct device_node *slave_node;
2846 	int i = 0;
2847 
2848 	for_each_available_child_of_node(node, slave_node) {
2849 		struct cpsw_slave_data *slave_data = &data->slave_data[i];
2850 
2851 		if (strcmp(slave_node->name, "slave"))
2852 			continue;
2853 
2854 		if (of_phy_is_fixed_link(slave_node))
2855 			of_phy_deregister_fixed_link(slave_node);
2856 
2857 		of_node_put(slave_data->phy_node);
2858 
2859 		i++;
2860 		if (i == data->slaves)
2861 			break;
2862 	}
2863 
2864 	of_platform_depopulate(&pdev->dev);
2865 }
2866 
2867 static int cpsw_probe_dual_emac(struct cpsw_priv *priv)
2868 {
2869 	struct cpsw_common		*cpsw = priv->cpsw;
2870 	struct cpsw_platform_data	*data = &cpsw->data;
2871 	struct net_device		*ndev;
2872 	struct cpsw_priv		*priv_sl2;
2873 	int ret = 0;
2874 
2875 	ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
2876 	if (!ndev) {
2877 		dev_err(cpsw->dev, "cpsw: error allocating net_device\n");
2878 		return -ENOMEM;
2879 	}
2880 
2881 	priv_sl2 = netdev_priv(ndev);
2882 	priv_sl2->cpsw = cpsw;
2883 	priv_sl2->ndev = ndev;
2884 	priv_sl2->dev  = &ndev->dev;
2885 	priv_sl2->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
2886 
2887 	if (is_valid_ether_addr(data->slave_data[1].mac_addr)) {
2888 		memcpy(priv_sl2->mac_addr, data->slave_data[1].mac_addr,
2889 			ETH_ALEN);
2890 		dev_info(cpsw->dev, "cpsw: Detected MACID = %pM\n",
2891 			 priv_sl2->mac_addr);
2892 	} else {
2893 		random_ether_addr(priv_sl2->mac_addr);
2894 		dev_info(cpsw->dev, "cpsw: Random MACID = %pM\n",
2895 			 priv_sl2->mac_addr);
2896 	}
2897 	memcpy(ndev->dev_addr, priv_sl2->mac_addr, ETH_ALEN);
2898 
2899 	priv_sl2->emac_port = 1;
2900 	cpsw->slaves[1].ndev = ndev;
2901 	ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2902 
2903 	ndev->netdev_ops = &cpsw_netdev_ops;
2904 	ndev->ethtool_ops = &cpsw_ethtool_ops;
2905 
2906 	/* register the network device */
2907 	SET_NETDEV_DEV(ndev, cpsw->dev);
2908 	ret = register_netdev(ndev);
2909 	if (ret) {
2910 		dev_err(cpsw->dev, "cpsw: error registering net device\n");
2911 		free_netdev(ndev);
2912 		ret = -ENODEV;
2913 	}
2914 
2915 	return ret;
2916 }
2917 
2918 #define CPSW_QUIRK_IRQ		BIT(0)
2919 
2920 static const struct platform_device_id cpsw_devtype[] = {
2921 	{
2922 		/* keep it for existing comaptibles */
2923 		.name = "cpsw",
2924 		.driver_data = CPSW_QUIRK_IRQ,
2925 	}, {
2926 		.name = "am335x-cpsw",
2927 		.driver_data = CPSW_QUIRK_IRQ,
2928 	}, {
2929 		.name = "am4372-cpsw",
2930 		.driver_data = 0,
2931 	}, {
2932 		.name = "dra7-cpsw",
2933 		.driver_data = 0,
2934 	}, {
2935 		/* sentinel */
2936 	}
2937 };
2938 MODULE_DEVICE_TABLE(platform, cpsw_devtype);
2939 
2940 enum ti_cpsw_type {
2941 	CPSW = 0,
2942 	AM335X_CPSW,
2943 	AM4372_CPSW,
2944 	DRA7_CPSW,
2945 };
2946 
2947 static const struct of_device_id cpsw_of_mtable[] = {
2948 	{ .compatible = "ti,cpsw", .data = &cpsw_devtype[CPSW], },
2949 	{ .compatible = "ti,am335x-cpsw", .data = &cpsw_devtype[AM335X_CPSW], },
2950 	{ .compatible = "ti,am4372-cpsw", .data = &cpsw_devtype[AM4372_CPSW], },
2951 	{ .compatible = "ti,dra7-cpsw", .data = &cpsw_devtype[DRA7_CPSW], },
2952 	{ /* sentinel */ },
2953 };
2954 MODULE_DEVICE_TABLE(of, cpsw_of_mtable);
2955 
2956 static int cpsw_probe(struct platform_device *pdev)
2957 {
2958 	struct clk			*clk;
2959 	struct cpsw_platform_data	*data;
2960 	struct net_device		*ndev;
2961 	struct cpsw_priv		*priv;
2962 	struct cpdma_params		dma_params;
2963 	struct cpsw_ale_params		ale_params;
2964 	void __iomem			*ss_regs;
2965 	void __iomem			*cpts_regs;
2966 	struct resource			*res, *ss_res;
2967 	const struct of_device_id	*of_id;
2968 	struct gpio_descs		*mode;
2969 	u32 slave_offset, sliver_offset, slave_size;
2970 	struct cpsw_common		*cpsw;
2971 	int ret = 0, i;
2972 	int irq;
2973 
2974 	cpsw = devm_kzalloc(&pdev->dev, sizeof(struct cpsw_common), GFP_KERNEL);
2975 	if (!cpsw)
2976 		return -ENOMEM;
2977 
2978 	cpsw->dev = &pdev->dev;
2979 
2980 	ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
2981 	if (!ndev) {
2982 		dev_err(&pdev->dev, "error allocating net_device\n");
2983 		return -ENOMEM;
2984 	}
2985 
2986 	platform_set_drvdata(pdev, ndev);
2987 	priv = netdev_priv(ndev);
2988 	priv->cpsw = cpsw;
2989 	priv->ndev = ndev;
2990 	priv->dev  = &ndev->dev;
2991 	priv->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
2992 	cpsw->rx_packet_max = max(rx_packet_max, 128);
2993 
2994 	mode = devm_gpiod_get_array_optional(&pdev->dev, "mode", GPIOD_OUT_LOW);
2995 	if (IS_ERR(mode)) {
2996 		ret = PTR_ERR(mode);
2997 		dev_err(&pdev->dev, "gpio request failed, ret %d\n", ret);
2998 		goto clean_ndev_ret;
2999 	}
3000 
3001 	/*
3002 	 * This may be required here for child devices.
3003 	 */
3004 	pm_runtime_enable(&pdev->dev);
3005 
3006 	/* Select default pin state */
3007 	pinctrl_pm_select_default_state(&pdev->dev);
3008 
3009 	/* Need to enable clocks with runtime PM api to access module
3010 	 * registers
3011 	 */
3012 	ret = pm_runtime_get_sync(&pdev->dev);
3013 	if (ret < 0) {
3014 		pm_runtime_put_noidle(&pdev->dev);
3015 		goto clean_runtime_disable_ret;
3016 	}
3017 
3018 	ret = cpsw_probe_dt(&cpsw->data, pdev);
3019 	if (ret)
3020 		goto clean_dt_ret;
3021 
3022 	data = &cpsw->data;
3023 	cpsw->rx_ch_num = 1;
3024 	cpsw->tx_ch_num = 1;
3025 
3026 	if (is_valid_ether_addr(data->slave_data[0].mac_addr)) {
3027 		memcpy(priv->mac_addr, data->slave_data[0].mac_addr, ETH_ALEN);
3028 		dev_info(&pdev->dev, "Detected MACID = %pM\n", priv->mac_addr);
3029 	} else {
3030 		eth_random_addr(priv->mac_addr);
3031 		dev_info(&pdev->dev, "Random MACID = %pM\n", priv->mac_addr);
3032 	}
3033 
3034 	memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
3035 
3036 	cpsw->slaves = devm_kzalloc(&pdev->dev,
3037 				    sizeof(struct cpsw_slave) * data->slaves,
3038 				    GFP_KERNEL);
3039 	if (!cpsw->slaves) {
3040 		ret = -ENOMEM;
3041 		goto clean_dt_ret;
3042 	}
3043 	for (i = 0; i < data->slaves; i++)
3044 		cpsw->slaves[i].slave_num = i;
3045 
3046 	cpsw->slaves[0].ndev = ndev;
3047 	priv->emac_port = 0;
3048 
3049 	clk = devm_clk_get(&pdev->dev, "fck");
3050 	if (IS_ERR(clk)) {
3051 		dev_err(priv->dev, "fck is not found\n");
3052 		ret = -ENODEV;
3053 		goto clean_dt_ret;
3054 	}
3055 	cpsw->bus_freq_mhz = clk_get_rate(clk) / 1000000;
3056 
3057 	ss_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3058 	ss_regs = devm_ioremap_resource(&pdev->dev, ss_res);
3059 	if (IS_ERR(ss_regs)) {
3060 		ret = PTR_ERR(ss_regs);
3061 		goto clean_dt_ret;
3062 	}
3063 	cpsw->regs = ss_regs;
3064 
3065 	cpsw->version = readl(&cpsw->regs->id_ver);
3066 
3067 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
3068 	cpsw->wr_regs = devm_ioremap_resource(&pdev->dev, res);
3069 	if (IS_ERR(cpsw->wr_regs)) {
3070 		ret = PTR_ERR(cpsw->wr_regs);
3071 		goto clean_dt_ret;
3072 	}
3073 
3074 	memset(&dma_params, 0, sizeof(dma_params));
3075 	memset(&ale_params, 0, sizeof(ale_params));
3076 
3077 	switch (cpsw->version) {
3078 	case CPSW_VERSION_1:
3079 		cpsw->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;
3080 		cpts_regs		= ss_regs + CPSW1_CPTS_OFFSET;
3081 		cpsw->hw_stats	     = ss_regs + CPSW1_HW_STATS;
3082 		dma_params.dmaregs   = ss_regs + CPSW1_CPDMA_OFFSET;
3083 		dma_params.txhdp     = ss_regs + CPSW1_STATERAM_OFFSET;
3084 		ale_params.ale_regs  = ss_regs + CPSW1_ALE_OFFSET;
3085 		slave_offset         = CPSW1_SLAVE_OFFSET;
3086 		slave_size           = CPSW1_SLAVE_SIZE;
3087 		sliver_offset        = CPSW1_SLIVER_OFFSET;
3088 		dma_params.desc_mem_phys = 0;
3089 		break;
3090 	case CPSW_VERSION_2:
3091 	case CPSW_VERSION_3:
3092 	case CPSW_VERSION_4:
3093 		cpsw->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;
3094 		cpts_regs		= ss_regs + CPSW2_CPTS_OFFSET;
3095 		cpsw->hw_stats	     = ss_regs + CPSW2_HW_STATS;
3096 		dma_params.dmaregs   = ss_regs + CPSW2_CPDMA_OFFSET;
3097 		dma_params.txhdp     = ss_regs + CPSW2_STATERAM_OFFSET;
3098 		ale_params.ale_regs  = ss_regs + CPSW2_ALE_OFFSET;
3099 		slave_offset         = CPSW2_SLAVE_OFFSET;
3100 		slave_size           = CPSW2_SLAVE_SIZE;
3101 		sliver_offset        = CPSW2_SLIVER_OFFSET;
3102 		dma_params.desc_mem_phys =
3103 			(u32 __force) ss_res->start + CPSW2_BD_OFFSET;
3104 		break;
3105 	default:
3106 		dev_err(priv->dev, "unknown version 0x%08x\n", cpsw->version);
3107 		ret = -ENODEV;
3108 		goto clean_dt_ret;
3109 	}
3110 	for (i = 0; i < cpsw->data.slaves; i++) {
3111 		struct cpsw_slave *slave = &cpsw->slaves[i];
3112 
3113 		cpsw_slave_init(slave, cpsw, slave_offset, sliver_offset);
3114 		slave_offset  += slave_size;
3115 		sliver_offset += SLIVER_SIZE;
3116 	}
3117 
3118 	dma_params.dev		= &pdev->dev;
3119 	dma_params.rxthresh	= dma_params.dmaregs + CPDMA_RXTHRESH;
3120 	dma_params.rxfree	= dma_params.dmaregs + CPDMA_RXFREE;
3121 	dma_params.rxhdp	= dma_params.txhdp + CPDMA_RXHDP;
3122 	dma_params.txcp		= dma_params.txhdp + CPDMA_TXCP;
3123 	dma_params.rxcp		= dma_params.txhdp + CPDMA_RXCP;
3124 
3125 	dma_params.num_chan		= data->channels;
3126 	dma_params.has_soft_reset	= true;
3127 	dma_params.min_packet_size	= CPSW_MIN_PACKET_SIZE;
3128 	dma_params.desc_mem_size	= data->bd_ram_size;
3129 	dma_params.desc_align		= 16;
3130 	dma_params.has_ext_regs		= true;
3131 	dma_params.desc_hw_addr         = dma_params.desc_mem_phys;
3132 	dma_params.bus_freq_mhz		= cpsw->bus_freq_mhz;
3133 	dma_params.descs_pool_size	= descs_pool_size;
3134 
3135 	cpsw->dma = cpdma_ctlr_create(&dma_params);
3136 	if (!cpsw->dma) {
3137 		dev_err(priv->dev, "error initializing dma\n");
3138 		ret = -ENOMEM;
3139 		goto clean_dt_ret;
3140 	}
3141 
3142 	cpsw->txv[0].ch = cpdma_chan_create(cpsw->dma, 0, cpsw_tx_handler, 0);
3143 	if (IS_ERR(cpsw->txv[0].ch)) {
3144 		dev_err(priv->dev, "error initializing tx dma channel\n");
3145 		ret = PTR_ERR(cpsw->txv[0].ch);
3146 		goto clean_dma_ret;
3147 	}
3148 
3149 	cpsw->rxv[0].ch = cpdma_chan_create(cpsw->dma, 0, cpsw_rx_handler, 1);
3150 	if (IS_ERR(cpsw->rxv[0].ch)) {
3151 		dev_err(priv->dev, "error initializing rx dma channel\n");
3152 		ret = PTR_ERR(cpsw->rxv[0].ch);
3153 		goto clean_dma_ret;
3154 	}
3155 
3156 	ale_params.dev			= &pdev->dev;
3157 	ale_params.ale_ageout		= ale_ageout;
3158 	ale_params.ale_entries		= data->ale_entries;
3159 	ale_params.ale_ports		= CPSW_ALE_PORTS_NUM;
3160 
3161 	cpsw->ale = cpsw_ale_create(&ale_params);
3162 	if (!cpsw->ale) {
3163 		dev_err(priv->dev, "error initializing ale engine\n");
3164 		ret = -ENODEV;
3165 		goto clean_dma_ret;
3166 	}
3167 
3168 	cpsw->cpts = cpts_create(cpsw->dev, cpts_regs, cpsw->dev->of_node);
3169 	if (IS_ERR(cpsw->cpts)) {
3170 		ret = PTR_ERR(cpsw->cpts);
3171 		goto clean_dma_ret;
3172 	}
3173 
3174 	ndev->irq = platform_get_irq(pdev, 1);
3175 	if (ndev->irq < 0) {
3176 		dev_err(priv->dev, "error getting irq resource\n");
3177 		ret = ndev->irq;
3178 		goto clean_dma_ret;
3179 	}
3180 
3181 	of_id = of_match_device(cpsw_of_mtable, &pdev->dev);
3182 	if (of_id) {
3183 		pdev->id_entry = of_id->data;
3184 		if (pdev->id_entry->driver_data)
3185 			cpsw->quirk_irq = true;
3186 	}
3187 
3188 	ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
3189 
3190 	ndev->netdev_ops = &cpsw_netdev_ops;
3191 	ndev->ethtool_ops = &cpsw_ethtool_ops;
3192 	netif_napi_add(ndev, &cpsw->napi_rx, cpsw_rx_poll, CPSW_POLL_WEIGHT);
3193 	netif_tx_napi_add(ndev, &cpsw->napi_tx, cpsw_tx_poll, CPSW_POLL_WEIGHT);
3194 	cpsw_split_res(ndev);
3195 
3196 	/* register the network device */
3197 	SET_NETDEV_DEV(ndev, &pdev->dev);
3198 	ret = register_netdev(ndev);
3199 	if (ret) {
3200 		dev_err(priv->dev, "error registering net device\n");
3201 		ret = -ENODEV;
3202 		goto clean_dma_ret;
3203 	}
3204 
3205 	if (cpsw->data.dual_emac) {
3206 		ret = cpsw_probe_dual_emac(priv);
3207 		if (ret) {
3208 			cpsw_err(priv, probe, "error probe slave 2 emac interface\n");
3209 			goto clean_unregister_netdev_ret;
3210 		}
3211 	}
3212 
3213 	/* Grab RX and TX IRQs. Note that we also have RX_THRESHOLD and
3214 	 * MISC IRQs which are always kept disabled with this driver so
3215 	 * we will not request them.
3216 	 *
3217 	 * If anyone wants to implement support for those, make sure to
3218 	 * first request and append them to irqs_table array.
3219 	 */
3220 
3221 	/* RX IRQ */
3222 	irq = platform_get_irq(pdev, 1);
3223 	if (irq < 0) {
3224 		ret = irq;
3225 		goto clean_dma_ret;
3226 	}
3227 
3228 	cpsw->irqs_table[0] = irq;
3229 	ret = devm_request_irq(&pdev->dev, irq, cpsw_rx_interrupt,
3230 			       0, dev_name(&pdev->dev), cpsw);
3231 	if (ret < 0) {
3232 		dev_err(priv->dev, "error attaching irq (%d)\n", ret);
3233 		goto clean_dma_ret;
3234 	}
3235 
3236 	/* TX IRQ */
3237 	irq = platform_get_irq(pdev, 2);
3238 	if (irq < 0) {
3239 		ret = irq;
3240 		goto clean_dma_ret;
3241 	}
3242 
3243 	cpsw->irqs_table[1] = irq;
3244 	ret = devm_request_irq(&pdev->dev, irq, cpsw_tx_interrupt,
3245 			       0, dev_name(&pdev->dev), cpsw);
3246 	if (ret < 0) {
3247 		dev_err(priv->dev, "error attaching irq (%d)\n", ret);
3248 		goto clean_dma_ret;
3249 	}
3250 
3251 	cpsw_notice(priv, probe,
3252 		    "initialized device (regs %pa, irq %d, pool size %d)\n",
3253 		    &ss_res->start, ndev->irq, dma_params.descs_pool_size);
3254 
3255 	pm_runtime_put(&pdev->dev);
3256 
3257 	return 0;
3258 
3259 clean_unregister_netdev_ret:
3260 	unregister_netdev(ndev);
3261 clean_dma_ret:
3262 	cpdma_ctlr_destroy(cpsw->dma);
3263 clean_dt_ret:
3264 	cpsw_remove_dt(pdev);
3265 	pm_runtime_put_sync(&pdev->dev);
3266 clean_runtime_disable_ret:
3267 	pm_runtime_disable(&pdev->dev);
3268 clean_ndev_ret:
3269 	free_netdev(priv->ndev);
3270 	return ret;
3271 }
3272 
3273 static int cpsw_remove(struct platform_device *pdev)
3274 {
3275 	struct net_device *ndev = platform_get_drvdata(pdev);
3276 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
3277 	int ret;
3278 
3279 	ret = pm_runtime_get_sync(&pdev->dev);
3280 	if (ret < 0) {
3281 		pm_runtime_put_noidle(&pdev->dev);
3282 		return ret;
3283 	}
3284 
3285 	if (cpsw->data.dual_emac)
3286 		unregister_netdev(cpsw->slaves[1].ndev);
3287 	unregister_netdev(ndev);
3288 
3289 	cpts_release(cpsw->cpts);
3290 	cpdma_ctlr_destroy(cpsw->dma);
3291 	cpsw_remove_dt(pdev);
3292 	pm_runtime_put_sync(&pdev->dev);
3293 	pm_runtime_disable(&pdev->dev);
3294 	if (cpsw->data.dual_emac)
3295 		free_netdev(cpsw->slaves[1].ndev);
3296 	free_netdev(ndev);
3297 	return 0;
3298 }
3299 
3300 #ifdef CONFIG_PM_SLEEP
3301 static int cpsw_suspend(struct device *dev)
3302 {
3303 	struct platform_device	*pdev = to_platform_device(dev);
3304 	struct net_device	*ndev = platform_get_drvdata(pdev);
3305 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
3306 
3307 	if (cpsw->data.dual_emac) {
3308 		int i;
3309 
3310 		for (i = 0; i < cpsw->data.slaves; i++) {
3311 			if (netif_running(cpsw->slaves[i].ndev))
3312 				cpsw_ndo_stop(cpsw->slaves[i].ndev);
3313 		}
3314 	} else {
3315 		if (netif_running(ndev))
3316 			cpsw_ndo_stop(ndev);
3317 	}
3318 
3319 	/* Select sleep pin state */
3320 	pinctrl_pm_select_sleep_state(dev);
3321 
3322 	return 0;
3323 }
3324 
3325 static int cpsw_resume(struct device *dev)
3326 {
3327 	struct platform_device	*pdev = to_platform_device(dev);
3328 	struct net_device	*ndev = platform_get_drvdata(pdev);
3329 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
3330 
3331 	/* Select default pin state */
3332 	pinctrl_pm_select_default_state(dev);
3333 
3334 	/* shut up ASSERT_RTNL() warning in netif_set_real_num_tx/rx_queues */
3335 	rtnl_lock();
3336 	if (cpsw->data.dual_emac) {
3337 		int i;
3338 
3339 		for (i = 0; i < cpsw->data.slaves; i++) {
3340 			if (netif_running(cpsw->slaves[i].ndev))
3341 				cpsw_ndo_open(cpsw->slaves[i].ndev);
3342 		}
3343 	} else {
3344 		if (netif_running(ndev))
3345 			cpsw_ndo_open(ndev);
3346 	}
3347 	rtnl_unlock();
3348 
3349 	return 0;
3350 }
3351 #endif
3352 
3353 static SIMPLE_DEV_PM_OPS(cpsw_pm_ops, cpsw_suspend, cpsw_resume);
3354 
3355 static struct platform_driver cpsw_driver = {
3356 	.driver = {
3357 		.name	 = "cpsw",
3358 		.pm	 = &cpsw_pm_ops,
3359 		.of_match_table = cpsw_of_mtable,
3360 	},
3361 	.probe = cpsw_probe,
3362 	.remove = cpsw_remove,
3363 };
3364 
3365 module_platform_driver(cpsw_driver);
3366 
3367 MODULE_LICENSE("GPL");
3368 MODULE_AUTHOR("Cyril Chemparathy <cyril@ti.com>");
3369 MODULE_AUTHOR("Mugunthan V N <mugunthanvnm@ti.com>");
3370 MODULE_DESCRIPTION("TI CPSW Ethernet driver");
3371