xref: /openbmc/linux/drivers/net/ethernet/ti/cpsw.c (revision a36954f5)
1 /*
2  * Texas Instruments Ethernet Switch Driver
3  *
4  * Copyright (C) 2012 Texas Instruments
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License as
8  * published by the Free Software Foundation version 2.
9  *
10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11  * kind, whether express or implied; without even the implied warranty
12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  */
15 
16 #include <linux/kernel.h>
17 #include <linux/io.h>
18 #include <linux/clk.h>
19 #include <linux/timer.h>
20 #include <linux/module.h>
21 #include <linux/platform_device.h>
22 #include <linux/irqreturn.h>
23 #include <linux/interrupt.h>
24 #include <linux/if_ether.h>
25 #include <linux/etherdevice.h>
26 #include <linux/netdevice.h>
27 #include <linux/net_tstamp.h>
28 #include <linux/phy.h>
29 #include <linux/workqueue.h>
30 #include <linux/delay.h>
31 #include <linux/pm_runtime.h>
32 #include <linux/gpio.h>
33 #include <linux/of.h>
34 #include <linux/of_mdio.h>
35 #include <linux/of_net.h>
36 #include <linux/of_device.h>
37 #include <linux/if_vlan.h>
38 
39 #include <linux/pinctrl/consumer.h>
40 
41 #include "cpsw.h"
42 #include "cpsw_ale.h"
43 #include "cpts.h"
44 #include "davinci_cpdma.h"
45 
46 #define CPSW_DEBUG	(NETIF_MSG_HW		| NETIF_MSG_WOL		| \
47 			 NETIF_MSG_DRV		| NETIF_MSG_LINK	| \
48 			 NETIF_MSG_IFUP		| NETIF_MSG_INTR	| \
49 			 NETIF_MSG_PROBE	| NETIF_MSG_TIMER	| \
50 			 NETIF_MSG_IFDOWN	| NETIF_MSG_RX_ERR	| \
51 			 NETIF_MSG_TX_ERR	| NETIF_MSG_TX_DONE	| \
52 			 NETIF_MSG_PKTDATA	| NETIF_MSG_TX_QUEUED	| \
53 			 NETIF_MSG_RX_STATUS)
54 
55 #define cpsw_info(priv, type, format, ...)		\
56 do {								\
57 	if (netif_msg_##type(priv) && net_ratelimit())		\
58 		dev_info(priv->dev, format, ## __VA_ARGS__);	\
59 } while (0)
60 
61 #define cpsw_err(priv, type, format, ...)		\
62 do {								\
63 	if (netif_msg_##type(priv) && net_ratelimit())		\
64 		dev_err(priv->dev, format, ## __VA_ARGS__);	\
65 } while (0)
66 
67 #define cpsw_dbg(priv, type, format, ...)		\
68 do {								\
69 	if (netif_msg_##type(priv) && net_ratelimit())		\
70 		dev_dbg(priv->dev, format, ## __VA_ARGS__);	\
71 } while (0)
72 
73 #define cpsw_notice(priv, type, format, ...)		\
74 do {								\
75 	if (netif_msg_##type(priv) && net_ratelimit())		\
76 		dev_notice(priv->dev, format, ## __VA_ARGS__);	\
77 } while (0)
78 
79 #define ALE_ALL_PORTS		0x7
80 
81 #define CPSW_MAJOR_VERSION(reg)		(reg >> 8 & 0x7)
82 #define CPSW_MINOR_VERSION(reg)		(reg & 0xff)
83 #define CPSW_RTL_VERSION(reg)		((reg >> 11) & 0x1f)
84 
85 #define CPSW_VERSION_1		0x19010a
86 #define CPSW_VERSION_2		0x19010c
87 #define CPSW_VERSION_3		0x19010f
88 #define CPSW_VERSION_4		0x190112
89 
90 #define HOST_PORT_NUM		0
91 #define SLIVER_SIZE		0x40
92 
93 #define CPSW1_HOST_PORT_OFFSET	0x028
94 #define CPSW1_SLAVE_OFFSET	0x050
95 #define CPSW1_SLAVE_SIZE	0x040
96 #define CPSW1_CPDMA_OFFSET	0x100
97 #define CPSW1_STATERAM_OFFSET	0x200
98 #define CPSW1_HW_STATS		0x400
99 #define CPSW1_CPTS_OFFSET	0x500
100 #define CPSW1_ALE_OFFSET	0x600
101 #define CPSW1_SLIVER_OFFSET	0x700
102 
103 #define CPSW2_HOST_PORT_OFFSET	0x108
104 #define CPSW2_SLAVE_OFFSET	0x200
105 #define CPSW2_SLAVE_SIZE	0x100
106 #define CPSW2_CPDMA_OFFSET	0x800
107 #define CPSW2_HW_STATS		0x900
108 #define CPSW2_STATERAM_OFFSET	0xa00
109 #define CPSW2_CPTS_OFFSET	0xc00
110 #define CPSW2_ALE_OFFSET	0xd00
111 #define CPSW2_SLIVER_OFFSET	0xd80
112 #define CPSW2_BD_OFFSET		0x2000
113 
114 #define CPDMA_RXTHRESH		0x0c0
115 #define CPDMA_RXFREE		0x0e0
116 #define CPDMA_TXHDP		0x00
117 #define CPDMA_RXHDP		0x20
118 #define CPDMA_TXCP		0x40
119 #define CPDMA_RXCP		0x60
120 
121 #define CPSW_POLL_WEIGHT	64
122 #define CPSW_MIN_PACKET_SIZE	60
123 #define CPSW_MAX_PACKET_SIZE	(1500 + 14 + 4 + 4)
124 
125 #define RX_PRIORITY_MAPPING	0x76543210
126 #define TX_PRIORITY_MAPPING	0x33221100
127 #define CPDMA_TX_PRIORITY_MAP	0x01234567
128 
129 #define CPSW_VLAN_AWARE		BIT(1)
130 #define CPSW_ALE_VLAN_AWARE	1
131 
132 #define CPSW_FIFO_NORMAL_MODE		(0 << 16)
133 #define CPSW_FIFO_DUAL_MAC_MODE		(1 << 16)
134 #define CPSW_FIFO_RATE_LIMIT_MODE	(2 << 16)
135 
136 #define CPSW_INTPACEEN		(0x3f << 16)
137 #define CPSW_INTPRESCALE_MASK	(0x7FF << 0)
138 #define CPSW_CMINTMAX_CNT	63
139 #define CPSW_CMINTMIN_CNT	2
140 #define CPSW_CMINTMAX_INTVL	(1000 / CPSW_CMINTMIN_CNT)
141 #define CPSW_CMINTMIN_INTVL	((1000 / CPSW_CMINTMAX_CNT) + 1)
142 
143 #define cpsw_slave_index(cpsw, priv)				\
144 		((cpsw->data.dual_emac) ? priv->emac_port :	\
145 		cpsw->data.active_slave)
146 #define IRQ_NUM			2
147 #define CPSW_MAX_QUEUES		8
148 #define CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT 256
149 
150 static int debug_level;
151 module_param(debug_level, int, 0);
152 MODULE_PARM_DESC(debug_level, "cpsw debug level (NETIF_MSG bits)");
153 
154 static int ale_ageout = 10;
155 module_param(ale_ageout, int, 0);
156 MODULE_PARM_DESC(ale_ageout, "cpsw ale ageout interval (seconds)");
157 
158 static int rx_packet_max = CPSW_MAX_PACKET_SIZE;
159 module_param(rx_packet_max, int, 0);
160 MODULE_PARM_DESC(rx_packet_max, "maximum receive packet size (bytes)");
161 
162 static int descs_pool_size = CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT;
163 module_param(descs_pool_size, int, 0444);
164 MODULE_PARM_DESC(descs_pool_size, "Number of CPDMA CPPI descriptors in pool");
165 
166 struct cpsw_wr_regs {
167 	u32	id_ver;
168 	u32	soft_reset;
169 	u32	control;
170 	u32	int_control;
171 	u32	rx_thresh_en;
172 	u32	rx_en;
173 	u32	tx_en;
174 	u32	misc_en;
175 	u32	mem_allign1[8];
176 	u32	rx_thresh_stat;
177 	u32	rx_stat;
178 	u32	tx_stat;
179 	u32	misc_stat;
180 	u32	mem_allign2[8];
181 	u32	rx_imax;
182 	u32	tx_imax;
183 
184 };
185 
186 struct cpsw_ss_regs {
187 	u32	id_ver;
188 	u32	control;
189 	u32	soft_reset;
190 	u32	stat_port_en;
191 	u32	ptype;
192 	u32	soft_idle;
193 	u32	thru_rate;
194 	u32	gap_thresh;
195 	u32	tx_start_wds;
196 	u32	flow_control;
197 	u32	vlan_ltype;
198 	u32	ts_ltype;
199 	u32	dlr_ltype;
200 };
201 
202 /* CPSW_PORT_V1 */
203 #define CPSW1_MAX_BLKS      0x00 /* Maximum FIFO Blocks */
204 #define CPSW1_BLK_CNT       0x04 /* FIFO Block Usage Count (Read Only) */
205 #define CPSW1_TX_IN_CTL     0x08 /* Transmit FIFO Control */
206 #define CPSW1_PORT_VLAN     0x0c /* VLAN Register */
207 #define CPSW1_TX_PRI_MAP    0x10 /* Tx Header Priority to Switch Pri Mapping */
208 #define CPSW1_TS_CTL        0x14 /* Time Sync Control */
209 #define CPSW1_TS_SEQ_LTYPE  0x18 /* Time Sync Sequence ID Offset and Msg Type */
210 #define CPSW1_TS_VLAN       0x1c /* Time Sync VLAN1 and VLAN2 */
211 
212 /* CPSW_PORT_V2 */
213 #define CPSW2_CONTROL       0x00 /* Control Register */
214 #define CPSW2_MAX_BLKS      0x08 /* Maximum FIFO Blocks */
215 #define CPSW2_BLK_CNT       0x0c /* FIFO Block Usage Count (Read Only) */
216 #define CPSW2_TX_IN_CTL     0x10 /* Transmit FIFO Control */
217 #define CPSW2_PORT_VLAN     0x14 /* VLAN Register */
218 #define CPSW2_TX_PRI_MAP    0x18 /* Tx Header Priority to Switch Pri Mapping */
219 #define CPSW2_TS_SEQ_MTYPE  0x1c /* Time Sync Sequence ID Offset and Msg Type */
220 
221 /* CPSW_PORT_V1 and V2 */
222 #define SA_LO               0x20 /* CPGMAC_SL Source Address Low */
223 #define SA_HI               0x24 /* CPGMAC_SL Source Address High */
224 #define SEND_PERCENT        0x28 /* Transmit Queue Send Percentages */
225 
226 /* CPSW_PORT_V2 only */
227 #define RX_DSCP_PRI_MAP0    0x30 /* Rx DSCP Priority to Rx Packet Mapping */
228 #define RX_DSCP_PRI_MAP1    0x34 /* Rx DSCP Priority to Rx Packet Mapping */
229 #define RX_DSCP_PRI_MAP2    0x38 /* Rx DSCP Priority to Rx Packet Mapping */
230 #define RX_DSCP_PRI_MAP3    0x3c /* Rx DSCP Priority to Rx Packet Mapping */
231 #define RX_DSCP_PRI_MAP4    0x40 /* Rx DSCP Priority to Rx Packet Mapping */
232 #define RX_DSCP_PRI_MAP5    0x44 /* Rx DSCP Priority to Rx Packet Mapping */
233 #define RX_DSCP_PRI_MAP6    0x48 /* Rx DSCP Priority to Rx Packet Mapping */
234 #define RX_DSCP_PRI_MAP7    0x4c /* Rx DSCP Priority to Rx Packet Mapping */
235 
236 /* Bit definitions for the CPSW2_CONTROL register */
237 #define PASS_PRI_TAGGED     (1<<24) /* Pass Priority Tagged */
238 #define VLAN_LTYPE2_EN      (1<<21) /* VLAN LTYPE 2 enable */
239 #define VLAN_LTYPE1_EN      (1<<20) /* VLAN LTYPE 1 enable */
240 #define DSCP_PRI_EN         (1<<16) /* DSCP Priority Enable */
241 #define TS_320              (1<<14) /* Time Sync Dest Port 320 enable */
242 #define TS_319              (1<<13) /* Time Sync Dest Port 319 enable */
243 #define TS_132              (1<<12) /* Time Sync Dest IP Addr 132 enable */
244 #define TS_131              (1<<11) /* Time Sync Dest IP Addr 131 enable */
245 #define TS_130              (1<<10) /* Time Sync Dest IP Addr 130 enable */
246 #define TS_129              (1<<9)  /* Time Sync Dest IP Addr 129 enable */
247 #define TS_TTL_NONZERO      (1<<8)  /* Time Sync Time To Live Non-zero enable */
248 #define TS_ANNEX_F_EN       (1<<6)  /* Time Sync Annex F enable */
249 #define TS_ANNEX_D_EN       (1<<4)  /* Time Sync Annex D enable */
250 #define TS_LTYPE2_EN        (1<<3)  /* Time Sync LTYPE 2 enable */
251 #define TS_LTYPE1_EN        (1<<2)  /* Time Sync LTYPE 1 enable */
252 #define TS_TX_EN            (1<<1)  /* Time Sync Transmit Enable */
253 #define TS_RX_EN            (1<<0)  /* Time Sync Receive Enable */
254 
255 #define CTRL_V2_TS_BITS \
256 	(TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
257 	 TS_TTL_NONZERO  | TS_ANNEX_D_EN | TS_LTYPE1_EN)
258 
259 #define CTRL_V2_ALL_TS_MASK (CTRL_V2_TS_BITS | TS_TX_EN | TS_RX_EN)
260 #define CTRL_V2_TX_TS_BITS  (CTRL_V2_TS_BITS | TS_TX_EN)
261 #define CTRL_V2_RX_TS_BITS  (CTRL_V2_TS_BITS | TS_RX_EN)
262 
263 
264 #define CTRL_V3_TS_BITS \
265 	(TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
266 	 TS_TTL_NONZERO | TS_ANNEX_F_EN | TS_ANNEX_D_EN |\
267 	 TS_LTYPE1_EN)
268 
269 #define CTRL_V3_ALL_TS_MASK (CTRL_V3_TS_BITS | TS_TX_EN | TS_RX_EN)
270 #define CTRL_V3_TX_TS_BITS  (CTRL_V3_TS_BITS | TS_TX_EN)
271 #define CTRL_V3_RX_TS_BITS  (CTRL_V3_TS_BITS | TS_RX_EN)
272 
273 /* Bit definitions for the CPSW2_TS_SEQ_MTYPE register */
274 #define TS_SEQ_ID_OFFSET_SHIFT   (16)    /* Time Sync Sequence ID Offset */
275 #define TS_SEQ_ID_OFFSET_MASK    (0x3f)
276 #define TS_MSG_TYPE_EN_SHIFT     (0)     /* Time Sync Message Type Enable */
277 #define TS_MSG_TYPE_EN_MASK      (0xffff)
278 
279 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
280 #define EVENT_MSG_BITS ((1<<0) | (1<<1) | (1<<2) | (1<<3))
281 
282 /* Bit definitions for the CPSW1_TS_CTL register */
283 #define CPSW_V1_TS_RX_EN		BIT(0)
284 #define CPSW_V1_TS_TX_EN		BIT(4)
285 #define CPSW_V1_MSG_TYPE_OFS		16
286 
287 /* Bit definitions for the CPSW1_TS_SEQ_LTYPE register */
288 #define CPSW_V1_SEQ_ID_OFS_SHIFT	16
289 
290 #define CPSW_MAX_BLKS_TX		15
291 #define CPSW_MAX_BLKS_TX_SHIFT		4
292 #define CPSW_MAX_BLKS_RX		5
293 
294 struct cpsw_host_regs {
295 	u32	max_blks;
296 	u32	blk_cnt;
297 	u32	tx_in_ctl;
298 	u32	port_vlan;
299 	u32	tx_pri_map;
300 	u32	cpdma_tx_pri_map;
301 	u32	cpdma_rx_chan_map;
302 };
303 
304 struct cpsw_sliver_regs {
305 	u32	id_ver;
306 	u32	mac_control;
307 	u32	mac_status;
308 	u32	soft_reset;
309 	u32	rx_maxlen;
310 	u32	__reserved_0;
311 	u32	rx_pause;
312 	u32	tx_pause;
313 	u32	__reserved_1;
314 	u32	rx_pri_map;
315 };
316 
317 struct cpsw_hw_stats {
318 	u32	rxgoodframes;
319 	u32	rxbroadcastframes;
320 	u32	rxmulticastframes;
321 	u32	rxpauseframes;
322 	u32	rxcrcerrors;
323 	u32	rxaligncodeerrors;
324 	u32	rxoversizedframes;
325 	u32	rxjabberframes;
326 	u32	rxundersizedframes;
327 	u32	rxfragments;
328 	u32	__pad_0[2];
329 	u32	rxoctets;
330 	u32	txgoodframes;
331 	u32	txbroadcastframes;
332 	u32	txmulticastframes;
333 	u32	txpauseframes;
334 	u32	txdeferredframes;
335 	u32	txcollisionframes;
336 	u32	txsinglecollframes;
337 	u32	txmultcollframes;
338 	u32	txexcessivecollisions;
339 	u32	txlatecollisions;
340 	u32	txunderrun;
341 	u32	txcarriersenseerrors;
342 	u32	txoctets;
343 	u32	octetframes64;
344 	u32	octetframes65t127;
345 	u32	octetframes128t255;
346 	u32	octetframes256t511;
347 	u32	octetframes512t1023;
348 	u32	octetframes1024tup;
349 	u32	netoctets;
350 	u32	rxsofoverruns;
351 	u32	rxmofoverruns;
352 	u32	rxdmaoverruns;
353 };
354 
355 struct cpsw_slave {
356 	void __iomem			*regs;
357 	struct cpsw_sliver_regs __iomem	*sliver;
358 	int				slave_num;
359 	u32				mac_control;
360 	struct cpsw_slave_data		*data;
361 	struct phy_device		*phy;
362 	struct net_device		*ndev;
363 	u32				port_vlan;
364 };
365 
366 static inline u32 slave_read(struct cpsw_slave *slave, u32 offset)
367 {
368 	return __raw_readl(slave->regs + offset);
369 }
370 
371 static inline void slave_write(struct cpsw_slave *slave, u32 val, u32 offset)
372 {
373 	__raw_writel(val, slave->regs + offset);
374 }
375 
376 struct cpsw_vector {
377 	struct cpdma_chan *ch;
378 	int budget;
379 };
380 
381 struct cpsw_common {
382 	struct device			*dev;
383 	struct cpsw_platform_data	data;
384 	struct napi_struct		napi_rx;
385 	struct napi_struct		napi_tx;
386 	struct cpsw_ss_regs __iomem	*regs;
387 	struct cpsw_wr_regs __iomem	*wr_regs;
388 	u8 __iomem			*hw_stats;
389 	struct cpsw_host_regs __iomem	*host_port_regs;
390 	u32				version;
391 	u32				coal_intvl;
392 	u32				bus_freq_mhz;
393 	int				rx_packet_max;
394 	struct cpsw_slave		*slaves;
395 	struct cpdma_ctlr		*dma;
396 	struct cpsw_vector		txv[CPSW_MAX_QUEUES];
397 	struct cpsw_vector		rxv[CPSW_MAX_QUEUES];
398 	struct cpsw_ale			*ale;
399 	bool				quirk_irq;
400 	bool				rx_irq_disabled;
401 	bool				tx_irq_disabled;
402 	u32 irqs_table[IRQ_NUM];
403 	struct cpts			*cpts;
404 	int				rx_ch_num, tx_ch_num;
405 	int				speed;
406 	int				usage_count;
407 };
408 
409 struct cpsw_priv {
410 	struct net_device		*ndev;
411 	struct device			*dev;
412 	u32				msg_enable;
413 	u8				mac_addr[ETH_ALEN];
414 	bool				rx_pause;
415 	bool				tx_pause;
416 	u32 emac_port;
417 	struct cpsw_common *cpsw;
418 };
419 
420 struct cpsw_stats {
421 	char stat_string[ETH_GSTRING_LEN];
422 	int type;
423 	int sizeof_stat;
424 	int stat_offset;
425 };
426 
427 enum {
428 	CPSW_STATS,
429 	CPDMA_RX_STATS,
430 	CPDMA_TX_STATS,
431 };
432 
433 #define CPSW_STAT(m)		CPSW_STATS,				\
434 				sizeof(((struct cpsw_hw_stats *)0)->m), \
435 				offsetof(struct cpsw_hw_stats, m)
436 #define CPDMA_RX_STAT(m)	CPDMA_RX_STATS,				   \
437 				sizeof(((struct cpdma_chan_stats *)0)->m), \
438 				offsetof(struct cpdma_chan_stats, m)
439 #define CPDMA_TX_STAT(m)	CPDMA_TX_STATS,				   \
440 				sizeof(((struct cpdma_chan_stats *)0)->m), \
441 				offsetof(struct cpdma_chan_stats, m)
442 
443 static const struct cpsw_stats cpsw_gstrings_stats[] = {
444 	{ "Good Rx Frames", CPSW_STAT(rxgoodframes) },
445 	{ "Broadcast Rx Frames", CPSW_STAT(rxbroadcastframes) },
446 	{ "Multicast Rx Frames", CPSW_STAT(rxmulticastframes) },
447 	{ "Pause Rx Frames", CPSW_STAT(rxpauseframes) },
448 	{ "Rx CRC Errors", CPSW_STAT(rxcrcerrors) },
449 	{ "Rx Align/Code Errors", CPSW_STAT(rxaligncodeerrors) },
450 	{ "Oversize Rx Frames", CPSW_STAT(rxoversizedframes) },
451 	{ "Rx Jabbers", CPSW_STAT(rxjabberframes) },
452 	{ "Undersize (Short) Rx Frames", CPSW_STAT(rxundersizedframes) },
453 	{ "Rx Fragments", CPSW_STAT(rxfragments) },
454 	{ "Rx Octets", CPSW_STAT(rxoctets) },
455 	{ "Good Tx Frames", CPSW_STAT(txgoodframes) },
456 	{ "Broadcast Tx Frames", CPSW_STAT(txbroadcastframes) },
457 	{ "Multicast Tx Frames", CPSW_STAT(txmulticastframes) },
458 	{ "Pause Tx Frames", CPSW_STAT(txpauseframes) },
459 	{ "Deferred Tx Frames", CPSW_STAT(txdeferredframes) },
460 	{ "Collisions", CPSW_STAT(txcollisionframes) },
461 	{ "Single Collision Tx Frames", CPSW_STAT(txsinglecollframes) },
462 	{ "Multiple Collision Tx Frames", CPSW_STAT(txmultcollframes) },
463 	{ "Excessive Collisions", CPSW_STAT(txexcessivecollisions) },
464 	{ "Late Collisions", CPSW_STAT(txlatecollisions) },
465 	{ "Tx Underrun", CPSW_STAT(txunderrun) },
466 	{ "Carrier Sense Errors", CPSW_STAT(txcarriersenseerrors) },
467 	{ "Tx Octets", CPSW_STAT(txoctets) },
468 	{ "Rx + Tx 64 Octet Frames", CPSW_STAT(octetframes64) },
469 	{ "Rx + Tx 65-127 Octet Frames", CPSW_STAT(octetframes65t127) },
470 	{ "Rx + Tx 128-255 Octet Frames", CPSW_STAT(octetframes128t255) },
471 	{ "Rx + Tx 256-511 Octet Frames", CPSW_STAT(octetframes256t511) },
472 	{ "Rx + Tx 512-1023 Octet Frames", CPSW_STAT(octetframes512t1023) },
473 	{ "Rx + Tx 1024-Up Octet Frames", CPSW_STAT(octetframes1024tup) },
474 	{ "Net Octets", CPSW_STAT(netoctets) },
475 	{ "Rx Start of Frame Overruns", CPSW_STAT(rxsofoverruns) },
476 	{ "Rx Middle of Frame Overruns", CPSW_STAT(rxmofoverruns) },
477 	{ "Rx DMA Overruns", CPSW_STAT(rxdmaoverruns) },
478 };
479 
480 static const struct cpsw_stats cpsw_gstrings_ch_stats[] = {
481 	{ "head_enqueue", CPDMA_RX_STAT(head_enqueue) },
482 	{ "tail_enqueue", CPDMA_RX_STAT(tail_enqueue) },
483 	{ "pad_enqueue", CPDMA_RX_STAT(pad_enqueue) },
484 	{ "misqueued", CPDMA_RX_STAT(misqueued) },
485 	{ "desc_alloc_fail", CPDMA_RX_STAT(desc_alloc_fail) },
486 	{ "pad_alloc_fail", CPDMA_RX_STAT(pad_alloc_fail) },
487 	{ "runt_receive_buf", CPDMA_RX_STAT(runt_receive_buff) },
488 	{ "runt_transmit_buf", CPDMA_RX_STAT(runt_transmit_buff) },
489 	{ "empty_dequeue", CPDMA_RX_STAT(empty_dequeue) },
490 	{ "busy_dequeue", CPDMA_RX_STAT(busy_dequeue) },
491 	{ "good_dequeue", CPDMA_RX_STAT(good_dequeue) },
492 	{ "requeue", CPDMA_RX_STAT(requeue) },
493 	{ "teardown_dequeue", CPDMA_RX_STAT(teardown_dequeue) },
494 };
495 
496 #define CPSW_STATS_COMMON_LEN	ARRAY_SIZE(cpsw_gstrings_stats)
497 #define CPSW_STATS_CH_LEN	ARRAY_SIZE(cpsw_gstrings_ch_stats)
498 
499 #define ndev_to_cpsw(ndev) (((struct cpsw_priv *)netdev_priv(ndev))->cpsw)
500 #define napi_to_cpsw(napi)	container_of(napi, struct cpsw_common, napi)
501 #define for_each_slave(priv, func, arg...)				\
502 	do {								\
503 		struct cpsw_slave *slave;				\
504 		struct cpsw_common *cpsw = (priv)->cpsw;		\
505 		int n;							\
506 		if (cpsw->data.dual_emac)				\
507 			(func)((cpsw)->slaves + priv->emac_port, ##arg);\
508 		else							\
509 			for (n = cpsw->data.slaves,			\
510 					slave = cpsw->slaves;		\
511 					n; n--)				\
512 				(func)(slave++, ##arg);			\
513 	} while (0)
514 
515 #define cpsw_dual_emac_src_port_detect(cpsw, status, ndev, skb)		\
516 	do {								\
517 		if (!cpsw->data.dual_emac)				\
518 			break;						\
519 		if (CPDMA_RX_SOURCE_PORT(status) == 1) {		\
520 			ndev = cpsw->slaves[0].ndev;			\
521 			skb->dev = ndev;				\
522 		} else if (CPDMA_RX_SOURCE_PORT(status) == 2) {		\
523 			ndev = cpsw->slaves[1].ndev;			\
524 			skb->dev = ndev;				\
525 		}							\
526 	} while (0)
527 #define cpsw_add_mcast(cpsw, priv, addr)				\
528 	do {								\
529 		if (cpsw->data.dual_emac) {				\
530 			struct cpsw_slave *slave = cpsw->slaves +	\
531 						priv->emac_port;	\
532 			int slave_port = cpsw_get_slave_port(		\
533 						slave->slave_num);	\
534 			cpsw_ale_add_mcast(cpsw->ale, addr,		\
535 				1 << slave_port | ALE_PORT_HOST,	\
536 				ALE_VLAN, slave->port_vlan, 0);		\
537 		} else {						\
538 			cpsw_ale_add_mcast(cpsw->ale, addr,		\
539 				ALE_ALL_PORTS,				\
540 				0, 0, 0);				\
541 		}							\
542 	} while (0)
543 
544 static inline int cpsw_get_slave_port(u32 slave_num)
545 {
546 	return slave_num + 1;
547 }
548 
549 static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
550 {
551 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
552 	struct cpsw_ale *ale = cpsw->ale;
553 	int i;
554 
555 	if (cpsw->data.dual_emac) {
556 		bool flag = false;
557 
558 		/* Enabling promiscuous mode for one interface will be
559 		 * common for both the interface as the interface shares
560 		 * the same hardware resource.
561 		 */
562 		for (i = 0; i < cpsw->data.slaves; i++)
563 			if (cpsw->slaves[i].ndev->flags & IFF_PROMISC)
564 				flag = true;
565 
566 		if (!enable && flag) {
567 			enable = true;
568 			dev_err(&ndev->dev, "promiscuity not disabled as the other interface is still in promiscuity mode\n");
569 		}
570 
571 		if (enable) {
572 			/* Enable Bypass */
573 			cpsw_ale_control_set(ale, 0, ALE_BYPASS, 1);
574 
575 			dev_dbg(&ndev->dev, "promiscuity enabled\n");
576 		} else {
577 			/* Disable Bypass */
578 			cpsw_ale_control_set(ale, 0, ALE_BYPASS, 0);
579 			dev_dbg(&ndev->dev, "promiscuity disabled\n");
580 		}
581 	} else {
582 		if (enable) {
583 			unsigned long timeout = jiffies + HZ;
584 
585 			/* Disable Learn for all ports (host is port 0 and slaves are port 1 and up */
586 			for (i = 0; i <= cpsw->data.slaves; i++) {
587 				cpsw_ale_control_set(ale, i,
588 						     ALE_PORT_NOLEARN, 1);
589 				cpsw_ale_control_set(ale, i,
590 						     ALE_PORT_NO_SA_UPDATE, 1);
591 			}
592 
593 			/* Clear All Untouched entries */
594 			cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
595 			do {
596 				cpu_relax();
597 				if (cpsw_ale_control_get(ale, 0, ALE_AGEOUT))
598 					break;
599 			} while (time_after(timeout, jiffies));
600 			cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
601 
602 			/* Clear all mcast from ALE */
603 			cpsw_ale_flush_multicast(ale, ALE_ALL_PORTS, -1);
604 
605 			/* Flood All Unicast Packets to Host port */
606 			cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 1);
607 			dev_dbg(&ndev->dev, "promiscuity enabled\n");
608 		} else {
609 			/* Don't Flood All Unicast Packets to Host port */
610 			cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 0);
611 
612 			/* Enable Learn for all ports (host is port 0 and slaves are port 1 and up */
613 			for (i = 0; i <= cpsw->data.slaves; i++) {
614 				cpsw_ale_control_set(ale, i,
615 						     ALE_PORT_NOLEARN, 0);
616 				cpsw_ale_control_set(ale, i,
617 						     ALE_PORT_NO_SA_UPDATE, 0);
618 			}
619 			dev_dbg(&ndev->dev, "promiscuity disabled\n");
620 		}
621 	}
622 }
623 
624 static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
625 {
626 	struct cpsw_priv *priv = netdev_priv(ndev);
627 	struct cpsw_common *cpsw = priv->cpsw;
628 	int vid;
629 
630 	if (cpsw->data.dual_emac)
631 		vid = cpsw->slaves[priv->emac_port].port_vlan;
632 	else
633 		vid = cpsw->data.default_vlan;
634 
635 	if (ndev->flags & IFF_PROMISC) {
636 		/* Enable promiscuous mode */
637 		cpsw_set_promiscious(ndev, true);
638 		cpsw_ale_set_allmulti(cpsw->ale, IFF_ALLMULTI);
639 		return;
640 	} else {
641 		/* Disable promiscuous mode */
642 		cpsw_set_promiscious(ndev, false);
643 	}
644 
645 	/* Restore allmulti on vlans if necessary */
646 	cpsw_ale_set_allmulti(cpsw->ale, priv->ndev->flags & IFF_ALLMULTI);
647 
648 	/* Clear all mcast from ALE */
649 	cpsw_ale_flush_multicast(cpsw->ale, ALE_ALL_PORTS, vid);
650 
651 	if (!netdev_mc_empty(ndev)) {
652 		struct netdev_hw_addr *ha;
653 
654 		/* program multicast address list into ALE register */
655 		netdev_for_each_mc_addr(ha, ndev) {
656 			cpsw_add_mcast(cpsw, priv, (u8 *)ha->addr);
657 		}
658 	}
659 }
660 
661 static void cpsw_intr_enable(struct cpsw_common *cpsw)
662 {
663 	__raw_writel(0xFF, &cpsw->wr_regs->tx_en);
664 	__raw_writel(0xFF, &cpsw->wr_regs->rx_en);
665 
666 	cpdma_ctlr_int_ctrl(cpsw->dma, true);
667 	return;
668 }
669 
670 static void cpsw_intr_disable(struct cpsw_common *cpsw)
671 {
672 	__raw_writel(0, &cpsw->wr_regs->tx_en);
673 	__raw_writel(0, &cpsw->wr_regs->rx_en);
674 
675 	cpdma_ctlr_int_ctrl(cpsw->dma, false);
676 	return;
677 }
678 
679 static void cpsw_tx_handler(void *token, int len, int status)
680 {
681 	struct netdev_queue	*txq;
682 	struct sk_buff		*skb = token;
683 	struct net_device	*ndev = skb->dev;
684 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
685 
686 	/* Check whether the queue is stopped due to stalled tx dma, if the
687 	 * queue is stopped then start the queue as we have free desc for tx
688 	 */
689 	txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
690 	if (unlikely(netif_tx_queue_stopped(txq)))
691 		netif_tx_wake_queue(txq);
692 
693 	cpts_tx_timestamp(cpsw->cpts, skb);
694 	ndev->stats.tx_packets++;
695 	ndev->stats.tx_bytes += len;
696 	dev_kfree_skb_any(skb);
697 }
698 
699 static void cpsw_rx_handler(void *token, int len, int status)
700 {
701 	struct cpdma_chan	*ch;
702 	struct sk_buff		*skb = token;
703 	struct sk_buff		*new_skb;
704 	struct net_device	*ndev = skb->dev;
705 	int			ret = 0;
706 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
707 
708 	cpsw_dual_emac_src_port_detect(cpsw, status, ndev, skb);
709 
710 	if (unlikely(status < 0) || unlikely(!netif_running(ndev))) {
711 		/* In dual emac mode check for all interfaces */
712 		if (cpsw->data.dual_emac && cpsw->usage_count &&
713 		    (status >= 0)) {
714 			/* The packet received is for the interface which
715 			 * is already down and the other interface is up
716 			 * and running, instead of freeing which results
717 			 * in reducing of the number of rx descriptor in
718 			 * DMA engine, requeue skb back to cpdma.
719 			 */
720 			new_skb = skb;
721 			goto requeue;
722 		}
723 
724 		/* the interface is going down, skbs are purged */
725 		dev_kfree_skb_any(skb);
726 		return;
727 	}
728 
729 	new_skb = netdev_alloc_skb_ip_align(ndev, cpsw->rx_packet_max);
730 	if (new_skb) {
731 		skb_copy_queue_mapping(new_skb, skb);
732 		skb_put(skb, len);
733 		cpts_rx_timestamp(cpsw->cpts, skb);
734 		skb->protocol = eth_type_trans(skb, ndev);
735 		netif_receive_skb(skb);
736 		ndev->stats.rx_bytes += len;
737 		ndev->stats.rx_packets++;
738 		kmemleak_not_leak(new_skb);
739 	} else {
740 		ndev->stats.rx_dropped++;
741 		new_skb = skb;
742 	}
743 
744 requeue:
745 	if (netif_dormant(ndev)) {
746 		dev_kfree_skb_any(new_skb);
747 		return;
748 	}
749 
750 	ch = cpsw->rxv[skb_get_queue_mapping(new_skb)].ch;
751 	ret = cpdma_chan_submit(ch, new_skb, new_skb->data,
752 				skb_tailroom(new_skb), 0);
753 	if (WARN_ON(ret < 0))
754 		dev_kfree_skb_any(new_skb);
755 }
756 
757 static void cpsw_split_res(struct net_device *ndev)
758 {
759 	struct cpsw_priv *priv = netdev_priv(ndev);
760 	u32 consumed_rate = 0, bigest_rate = 0;
761 	struct cpsw_common *cpsw = priv->cpsw;
762 	struct cpsw_vector *txv = cpsw->txv;
763 	int i, ch_weight, rlim_ch_num = 0;
764 	int budget, bigest_rate_ch = 0;
765 	u32 ch_rate, max_rate;
766 	int ch_budget = 0;
767 
768 	for (i = 0; i < cpsw->tx_ch_num; i++) {
769 		ch_rate = cpdma_chan_get_rate(txv[i].ch);
770 		if (!ch_rate)
771 			continue;
772 
773 		rlim_ch_num++;
774 		consumed_rate += ch_rate;
775 	}
776 
777 	if (cpsw->tx_ch_num == rlim_ch_num) {
778 		max_rate = consumed_rate;
779 	} else if (!rlim_ch_num) {
780 		ch_budget = CPSW_POLL_WEIGHT / cpsw->tx_ch_num;
781 		bigest_rate = 0;
782 		max_rate = consumed_rate;
783 	} else {
784 		max_rate = cpsw->speed * 1000;
785 
786 		/* if max_rate is less then expected due to reduced link speed,
787 		 * split proportionally according next potential max speed
788 		 */
789 		if (max_rate < consumed_rate)
790 			max_rate *= 10;
791 
792 		if (max_rate < consumed_rate)
793 			max_rate *= 10;
794 
795 		ch_budget = (consumed_rate * CPSW_POLL_WEIGHT) / max_rate;
796 		ch_budget = (CPSW_POLL_WEIGHT - ch_budget) /
797 			    (cpsw->tx_ch_num - rlim_ch_num);
798 		bigest_rate = (max_rate - consumed_rate) /
799 			      (cpsw->tx_ch_num - rlim_ch_num);
800 	}
801 
802 	/* split tx weight/budget */
803 	budget = CPSW_POLL_WEIGHT;
804 	for (i = 0; i < cpsw->tx_ch_num; i++) {
805 		ch_rate = cpdma_chan_get_rate(txv[i].ch);
806 		if (ch_rate) {
807 			txv[i].budget = (ch_rate * CPSW_POLL_WEIGHT) / max_rate;
808 			if (!txv[i].budget)
809 				txv[i].budget++;
810 			if (ch_rate > bigest_rate) {
811 				bigest_rate_ch = i;
812 				bigest_rate = ch_rate;
813 			}
814 
815 			ch_weight = (ch_rate * 100) / max_rate;
816 			if (!ch_weight)
817 				ch_weight++;
818 			cpdma_chan_set_weight(cpsw->txv[i].ch, ch_weight);
819 		} else {
820 			txv[i].budget = ch_budget;
821 			if (!bigest_rate_ch)
822 				bigest_rate_ch = i;
823 			cpdma_chan_set_weight(cpsw->txv[i].ch, 0);
824 		}
825 
826 		budget -= txv[i].budget;
827 	}
828 
829 	if (budget)
830 		txv[bigest_rate_ch].budget += budget;
831 
832 	/* split rx budget */
833 	budget = CPSW_POLL_WEIGHT;
834 	ch_budget = budget / cpsw->rx_ch_num;
835 	for (i = 0; i < cpsw->rx_ch_num; i++) {
836 		cpsw->rxv[i].budget = ch_budget;
837 		budget -= ch_budget;
838 	}
839 
840 	if (budget)
841 		cpsw->rxv[0].budget += budget;
842 }
843 
844 static irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id)
845 {
846 	struct cpsw_common *cpsw = dev_id;
847 
848 	writel(0, &cpsw->wr_regs->tx_en);
849 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);
850 
851 	if (cpsw->quirk_irq) {
852 		disable_irq_nosync(cpsw->irqs_table[1]);
853 		cpsw->tx_irq_disabled = true;
854 	}
855 
856 	napi_schedule(&cpsw->napi_tx);
857 	return IRQ_HANDLED;
858 }
859 
860 static irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id)
861 {
862 	struct cpsw_common *cpsw = dev_id;
863 
864 	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);
865 	writel(0, &cpsw->wr_regs->rx_en);
866 
867 	if (cpsw->quirk_irq) {
868 		disable_irq_nosync(cpsw->irqs_table[0]);
869 		cpsw->rx_irq_disabled = true;
870 	}
871 
872 	napi_schedule(&cpsw->napi_rx);
873 	return IRQ_HANDLED;
874 }
875 
876 static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
877 {
878 	u32			ch_map;
879 	int			num_tx, cur_budget, ch;
880 	struct cpsw_common	*cpsw = napi_to_cpsw(napi_tx);
881 	struct cpsw_vector	*txv;
882 
883 	/* process every unprocessed channel */
884 	ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
885 	for (ch = 0, num_tx = 0; ch_map; ch_map >>= 1, ch++) {
886 		if (!(ch_map & 0x01))
887 			continue;
888 
889 		txv = &cpsw->txv[ch];
890 		if (unlikely(txv->budget > budget - num_tx))
891 			cur_budget = budget - num_tx;
892 		else
893 			cur_budget = txv->budget;
894 
895 		num_tx += cpdma_chan_process(txv->ch, cur_budget);
896 		if (num_tx >= budget)
897 			break;
898 	}
899 
900 	if (num_tx < budget) {
901 		napi_complete(napi_tx);
902 		writel(0xff, &cpsw->wr_regs->tx_en);
903 		if (cpsw->quirk_irq && cpsw->tx_irq_disabled) {
904 			cpsw->tx_irq_disabled = false;
905 			enable_irq(cpsw->irqs_table[1]);
906 		}
907 	}
908 
909 	return num_tx;
910 }
911 
912 static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
913 {
914 	u32			ch_map;
915 	int			num_rx, cur_budget, ch;
916 	struct cpsw_common	*cpsw = napi_to_cpsw(napi_rx);
917 	struct cpsw_vector	*rxv;
918 
919 	/* process every unprocessed channel */
920 	ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
921 	for (ch = 0, num_rx = 0; ch_map; ch_map >>= 1, ch++) {
922 		if (!(ch_map & 0x01))
923 			continue;
924 
925 		rxv = &cpsw->rxv[ch];
926 		if (unlikely(rxv->budget > budget - num_rx))
927 			cur_budget = budget - num_rx;
928 		else
929 			cur_budget = rxv->budget;
930 
931 		num_rx += cpdma_chan_process(rxv->ch, cur_budget);
932 		if (num_rx >= budget)
933 			break;
934 	}
935 
936 	if (num_rx < budget) {
937 		napi_complete_done(napi_rx, num_rx);
938 		writel(0xff, &cpsw->wr_regs->rx_en);
939 		if (cpsw->quirk_irq && cpsw->rx_irq_disabled) {
940 			cpsw->rx_irq_disabled = false;
941 			enable_irq(cpsw->irqs_table[0]);
942 		}
943 	}
944 
945 	return num_rx;
946 }
947 
948 static inline void soft_reset(const char *module, void __iomem *reg)
949 {
950 	unsigned long timeout = jiffies + HZ;
951 
952 	__raw_writel(1, reg);
953 	do {
954 		cpu_relax();
955 	} while ((__raw_readl(reg) & 1) && time_after(timeout, jiffies));
956 
957 	WARN(__raw_readl(reg) & 1, "failed to soft-reset %s\n", module);
958 }
959 
960 #define mac_hi(mac)	(((mac)[0] << 0) | ((mac)[1] << 8) |	\
961 			 ((mac)[2] << 16) | ((mac)[3] << 24))
962 #define mac_lo(mac)	(((mac)[4] << 0) | ((mac)[5] << 8))
963 
964 static void cpsw_set_slave_mac(struct cpsw_slave *slave,
965 			       struct cpsw_priv *priv)
966 {
967 	slave_write(slave, mac_hi(priv->mac_addr), SA_HI);
968 	slave_write(slave, mac_lo(priv->mac_addr), SA_LO);
969 }
970 
971 static void _cpsw_adjust_link(struct cpsw_slave *slave,
972 			      struct cpsw_priv *priv, bool *link)
973 {
974 	struct phy_device	*phy = slave->phy;
975 	u32			mac_control = 0;
976 	u32			slave_port;
977 	struct cpsw_common *cpsw = priv->cpsw;
978 
979 	if (!phy)
980 		return;
981 
982 	slave_port = cpsw_get_slave_port(slave->slave_num);
983 
984 	if (phy->link) {
985 		mac_control = cpsw->data.mac_control;
986 
987 		/* enable forwarding */
988 		cpsw_ale_control_set(cpsw->ale, slave_port,
989 				     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
990 
991 		if (phy->speed == 1000)
992 			mac_control |= BIT(7);	/* GIGABITEN	*/
993 		if (phy->duplex)
994 			mac_control |= BIT(0);	/* FULLDUPLEXEN	*/
995 
996 		/* set speed_in input in case RMII mode is used in 100Mbps */
997 		if (phy->speed == 100)
998 			mac_control |= BIT(15);
999 		else if (phy->speed == 10)
1000 			mac_control |= BIT(18); /* In Band mode */
1001 
1002 		if (priv->rx_pause)
1003 			mac_control |= BIT(3);
1004 
1005 		if (priv->tx_pause)
1006 			mac_control |= BIT(4);
1007 
1008 		*link = true;
1009 	} else {
1010 		mac_control = 0;
1011 		/* disable forwarding */
1012 		cpsw_ale_control_set(cpsw->ale, slave_port,
1013 				     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1014 	}
1015 
1016 	if (mac_control != slave->mac_control) {
1017 		phy_print_status(phy);
1018 		__raw_writel(mac_control, &slave->sliver->mac_control);
1019 	}
1020 
1021 	slave->mac_control = mac_control;
1022 }
1023 
1024 static int cpsw_get_common_speed(struct cpsw_common *cpsw)
1025 {
1026 	int i, speed;
1027 
1028 	for (i = 0, speed = 0; i < cpsw->data.slaves; i++)
1029 		if (cpsw->slaves[i].phy && cpsw->slaves[i].phy->link)
1030 			speed += cpsw->slaves[i].phy->speed;
1031 
1032 	return speed;
1033 }
1034 
1035 static int cpsw_need_resplit(struct cpsw_common *cpsw)
1036 {
1037 	int i, rlim_ch_num;
1038 	int speed, ch_rate;
1039 
1040 	/* re-split resources only in case speed was changed */
1041 	speed = cpsw_get_common_speed(cpsw);
1042 	if (speed == cpsw->speed || !speed)
1043 		return 0;
1044 
1045 	cpsw->speed = speed;
1046 
1047 	for (i = 0, rlim_ch_num = 0; i < cpsw->tx_ch_num; i++) {
1048 		ch_rate = cpdma_chan_get_rate(cpsw->txv[i].ch);
1049 		if (!ch_rate)
1050 			break;
1051 
1052 		rlim_ch_num++;
1053 	}
1054 
1055 	/* cases not dependent on speed */
1056 	if (!rlim_ch_num || rlim_ch_num == cpsw->tx_ch_num)
1057 		return 0;
1058 
1059 	return 1;
1060 }
1061 
1062 static void cpsw_adjust_link(struct net_device *ndev)
1063 {
1064 	struct cpsw_priv	*priv = netdev_priv(ndev);
1065 	struct cpsw_common	*cpsw = priv->cpsw;
1066 	bool			link = false;
1067 
1068 	for_each_slave(priv, _cpsw_adjust_link, priv, &link);
1069 
1070 	if (link) {
1071 		if (cpsw_need_resplit(cpsw))
1072 			cpsw_split_res(ndev);
1073 
1074 		netif_carrier_on(ndev);
1075 		if (netif_running(ndev))
1076 			netif_tx_wake_all_queues(ndev);
1077 	} else {
1078 		netif_carrier_off(ndev);
1079 		netif_tx_stop_all_queues(ndev);
1080 	}
1081 }
1082 
1083 static int cpsw_get_coalesce(struct net_device *ndev,
1084 				struct ethtool_coalesce *coal)
1085 {
1086 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1087 
1088 	coal->rx_coalesce_usecs = cpsw->coal_intvl;
1089 	return 0;
1090 }
1091 
1092 static int cpsw_set_coalesce(struct net_device *ndev,
1093 				struct ethtool_coalesce *coal)
1094 {
1095 	struct cpsw_priv *priv = netdev_priv(ndev);
1096 	u32 int_ctrl;
1097 	u32 num_interrupts = 0;
1098 	u32 prescale = 0;
1099 	u32 addnl_dvdr = 1;
1100 	u32 coal_intvl = 0;
1101 	struct cpsw_common *cpsw = priv->cpsw;
1102 
1103 	coal_intvl = coal->rx_coalesce_usecs;
1104 
1105 	int_ctrl =  readl(&cpsw->wr_regs->int_control);
1106 	prescale = cpsw->bus_freq_mhz * 4;
1107 
1108 	if (!coal->rx_coalesce_usecs) {
1109 		int_ctrl &= ~(CPSW_INTPRESCALE_MASK | CPSW_INTPACEEN);
1110 		goto update_return;
1111 	}
1112 
1113 	if (coal_intvl < CPSW_CMINTMIN_INTVL)
1114 		coal_intvl = CPSW_CMINTMIN_INTVL;
1115 
1116 	if (coal_intvl > CPSW_CMINTMAX_INTVL) {
1117 		/* Interrupt pacer works with 4us Pulse, we can
1118 		 * throttle further by dilating the 4us pulse.
1119 		 */
1120 		addnl_dvdr = CPSW_INTPRESCALE_MASK / prescale;
1121 
1122 		if (addnl_dvdr > 1) {
1123 			prescale *= addnl_dvdr;
1124 			if (coal_intvl > (CPSW_CMINTMAX_INTVL * addnl_dvdr))
1125 				coal_intvl = (CPSW_CMINTMAX_INTVL
1126 						* addnl_dvdr);
1127 		} else {
1128 			addnl_dvdr = 1;
1129 			coal_intvl = CPSW_CMINTMAX_INTVL;
1130 		}
1131 	}
1132 
1133 	num_interrupts = (1000 * addnl_dvdr) / coal_intvl;
1134 	writel(num_interrupts, &cpsw->wr_regs->rx_imax);
1135 	writel(num_interrupts, &cpsw->wr_regs->tx_imax);
1136 
1137 	int_ctrl |= CPSW_INTPACEEN;
1138 	int_ctrl &= (~CPSW_INTPRESCALE_MASK);
1139 	int_ctrl |= (prescale & CPSW_INTPRESCALE_MASK);
1140 
1141 update_return:
1142 	writel(int_ctrl, &cpsw->wr_regs->int_control);
1143 
1144 	cpsw_notice(priv, timer, "Set coalesce to %d usecs.\n", coal_intvl);
1145 	cpsw->coal_intvl = coal_intvl;
1146 
1147 	return 0;
1148 }
1149 
1150 static int cpsw_get_sset_count(struct net_device *ndev, int sset)
1151 {
1152 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1153 
1154 	switch (sset) {
1155 	case ETH_SS_STATS:
1156 		return (CPSW_STATS_COMMON_LEN +
1157 		       (cpsw->rx_ch_num + cpsw->tx_ch_num) *
1158 		       CPSW_STATS_CH_LEN);
1159 	default:
1160 		return -EOPNOTSUPP;
1161 	}
1162 }
1163 
1164 static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir)
1165 {
1166 	int ch_stats_len;
1167 	int line;
1168 	int i;
1169 
1170 	ch_stats_len = CPSW_STATS_CH_LEN * ch_num;
1171 	for (i = 0; i < ch_stats_len; i++) {
1172 		line = i % CPSW_STATS_CH_LEN;
1173 		snprintf(*p, ETH_GSTRING_LEN,
1174 			 "%s DMA chan %d: %s", rx_dir ? "Rx" : "Tx",
1175 			 i / CPSW_STATS_CH_LEN,
1176 			 cpsw_gstrings_ch_stats[line].stat_string);
1177 		*p += ETH_GSTRING_LEN;
1178 	}
1179 }
1180 
1181 static void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
1182 {
1183 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1184 	u8 *p = data;
1185 	int i;
1186 
1187 	switch (stringset) {
1188 	case ETH_SS_STATS:
1189 		for (i = 0; i < CPSW_STATS_COMMON_LEN; i++) {
1190 			memcpy(p, cpsw_gstrings_stats[i].stat_string,
1191 			       ETH_GSTRING_LEN);
1192 			p += ETH_GSTRING_LEN;
1193 		}
1194 
1195 		cpsw_add_ch_strings(&p, cpsw->rx_ch_num, 1);
1196 		cpsw_add_ch_strings(&p, cpsw->tx_ch_num, 0);
1197 		break;
1198 	}
1199 }
1200 
1201 static void cpsw_get_ethtool_stats(struct net_device *ndev,
1202 				    struct ethtool_stats *stats, u64 *data)
1203 {
1204 	u8 *p;
1205 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1206 	struct cpdma_chan_stats ch_stats;
1207 	int i, l, ch;
1208 
1209 	/* Collect Davinci CPDMA stats for Rx and Tx Channel */
1210 	for (l = 0; l < CPSW_STATS_COMMON_LEN; l++)
1211 		data[l] = readl(cpsw->hw_stats +
1212 				cpsw_gstrings_stats[l].stat_offset);
1213 
1214 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1215 		cpdma_chan_get_stats(cpsw->rxv[ch].ch, &ch_stats);
1216 		for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1217 			p = (u8 *)&ch_stats +
1218 				cpsw_gstrings_ch_stats[i].stat_offset;
1219 			data[l] = *(u32 *)p;
1220 		}
1221 	}
1222 
1223 	for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
1224 		cpdma_chan_get_stats(cpsw->txv[ch].ch, &ch_stats);
1225 		for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1226 			p = (u8 *)&ch_stats +
1227 				cpsw_gstrings_ch_stats[i].stat_offset;
1228 			data[l] = *(u32 *)p;
1229 		}
1230 	}
1231 }
1232 
1233 static inline int cpsw_tx_packet_submit(struct cpsw_priv *priv,
1234 					struct sk_buff *skb,
1235 					struct cpdma_chan *txch)
1236 {
1237 	struct cpsw_common *cpsw = priv->cpsw;
1238 
1239 	return cpdma_chan_submit(txch, skb, skb->data, skb->len,
1240 				 priv->emac_port + cpsw->data.dual_emac);
1241 }
1242 
1243 static inline void cpsw_add_dual_emac_def_ale_entries(
1244 		struct cpsw_priv *priv, struct cpsw_slave *slave,
1245 		u32 slave_port)
1246 {
1247 	struct cpsw_common *cpsw = priv->cpsw;
1248 	u32 port_mask = 1 << slave_port | ALE_PORT_HOST;
1249 
1250 	if (cpsw->version == CPSW_VERSION_1)
1251 		slave_write(slave, slave->port_vlan, CPSW1_PORT_VLAN);
1252 	else
1253 		slave_write(slave, slave->port_vlan, CPSW2_PORT_VLAN);
1254 	cpsw_ale_add_vlan(cpsw->ale, slave->port_vlan, port_mask,
1255 			  port_mask, port_mask, 0);
1256 	cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1257 			   port_mask, ALE_VLAN, slave->port_vlan, 0);
1258 	cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
1259 			   HOST_PORT_NUM, ALE_VLAN |
1260 			   ALE_SECURE, slave->port_vlan);
1261 }
1262 
1263 static void soft_reset_slave(struct cpsw_slave *slave)
1264 {
1265 	char name[32];
1266 
1267 	snprintf(name, sizeof(name), "slave-%d", slave->slave_num);
1268 	soft_reset(name, &slave->sliver->soft_reset);
1269 }
1270 
1271 static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
1272 {
1273 	u32 slave_port;
1274 	struct phy_device *phy;
1275 	struct cpsw_common *cpsw = priv->cpsw;
1276 
1277 	soft_reset_slave(slave);
1278 
1279 	/* setup priority mapping */
1280 	__raw_writel(RX_PRIORITY_MAPPING, &slave->sliver->rx_pri_map);
1281 
1282 	switch (cpsw->version) {
1283 	case CPSW_VERSION_1:
1284 		slave_write(slave, TX_PRIORITY_MAPPING, CPSW1_TX_PRI_MAP);
1285 		/* Increase RX FIFO size to 5 for supporting fullduplex
1286 		 * flow control mode
1287 		 */
1288 		slave_write(slave,
1289 			    (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1290 			    CPSW_MAX_BLKS_RX, CPSW1_MAX_BLKS);
1291 		break;
1292 	case CPSW_VERSION_2:
1293 	case CPSW_VERSION_3:
1294 	case CPSW_VERSION_4:
1295 		slave_write(slave, TX_PRIORITY_MAPPING, CPSW2_TX_PRI_MAP);
1296 		/* Increase RX FIFO size to 5 for supporting fullduplex
1297 		 * flow control mode
1298 		 */
1299 		slave_write(slave,
1300 			    (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1301 			    CPSW_MAX_BLKS_RX, CPSW2_MAX_BLKS);
1302 		break;
1303 	}
1304 
1305 	/* setup max packet size, and mac address */
1306 	__raw_writel(cpsw->rx_packet_max, &slave->sliver->rx_maxlen);
1307 	cpsw_set_slave_mac(slave, priv);
1308 
1309 	slave->mac_control = 0;	/* no link yet */
1310 
1311 	slave_port = cpsw_get_slave_port(slave->slave_num);
1312 
1313 	if (cpsw->data.dual_emac)
1314 		cpsw_add_dual_emac_def_ale_entries(priv, slave, slave_port);
1315 	else
1316 		cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1317 				   1 << slave_port, 0, 0, ALE_MCAST_FWD_2);
1318 
1319 	if (slave->data->phy_node) {
1320 		phy = of_phy_connect(priv->ndev, slave->data->phy_node,
1321 				 &cpsw_adjust_link, 0, slave->data->phy_if);
1322 		if (!phy) {
1323 			dev_err(priv->dev, "phy \"%s\" not found on slave %d\n",
1324 				slave->data->phy_node->full_name,
1325 				slave->slave_num);
1326 			return;
1327 		}
1328 	} else {
1329 		phy = phy_connect(priv->ndev, slave->data->phy_id,
1330 				 &cpsw_adjust_link, slave->data->phy_if);
1331 		if (IS_ERR(phy)) {
1332 			dev_err(priv->dev,
1333 				"phy \"%s\" not found on slave %d, err %ld\n",
1334 				slave->data->phy_id, slave->slave_num,
1335 				PTR_ERR(phy));
1336 			return;
1337 		}
1338 	}
1339 
1340 	slave->phy = phy;
1341 
1342 	phy_attached_info(slave->phy);
1343 
1344 	phy_start(slave->phy);
1345 
1346 	/* Configure GMII_SEL register */
1347 	cpsw_phy_sel(cpsw->dev, slave->phy->interface, slave->slave_num);
1348 }
1349 
1350 static inline void cpsw_add_default_vlan(struct cpsw_priv *priv)
1351 {
1352 	struct cpsw_common *cpsw = priv->cpsw;
1353 	const int vlan = cpsw->data.default_vlan;
1354 	u32 reg;
1355 	int i;
1356 	int unreg_mcast_mask;
1357 
1358 	reg = (cpsw->version == CPSW_VERSION_1) ? CPSW1_PORT_VLAN :
1359 	       CPSW2_PORT_VLAN;
1360 
1361 	writel(vlan, &cpsw->host_port_regs->port_vlan);
1362 
1363 	for (i = 0; i < cpsw->data.slaves; i++)
1364 		slave_write(cpsw->slaves + i, vlan, reg);
1365 
1366 	if (priv->ndev->flags & IFF_ALLMULTI)
1367 		unreg_mcast_mask = ALE_ALL_PORTS;
1368 	else
1369 		unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
1370 
1371 	cpsw_ale_add_vlan(cpsw->ale, vlan, ALE_ALL_PORTS,
1372 			  ALE_ALL_PORTS, ALE_ALL_PORTS,
1373 			  unreg_mcast_mask);
1374 }
1375 
1376 static void cpsw_init_host_port(struct cpsw_priv *priv)
1377 {
1378 	u32 fifo_mode;
1379 	u32 control_reg;
1380 	struct cpsw_common *cpsw = priv->cpsw;
1381 
1382 	/* soft reset the controller and initialize ale */
1383 	soft_reset("cpsw", &cpsw->regs->soft_reset);
1384 	cpsw_ale_start(cpsw->ale);
1385 
1386 	/* switch to vlan unaware mode */
1387 	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_VLAN_AWARE,
1388 			     CPSW_ALE_VLAN_AWARE);
1389 	control_reg = readl(&cpsw->regs->control);
1390 	control_reg |= CPSW_VLAN_AWARE;
1391 	writel(control_reg, &cpsw->regs->control);
1392 	fifo_mode = (cpsw->data.dual_emac) ? CPSW_FIFO_DUAL_MAC_MODE :
1393 		     CPSW_FIFO_NORMAL_MODE;
1394 	writel(fifo_mode, &cpsw->host_port_regs->tx_in_ctl);
1395 
1396 	/* setup host port priority mapping */
1397 	__raw_writel(CPDMA_TX_PRIORITY_MAP,
1398 		     &cpsw->host_port_regs->cpdma_tx_pri_map);
1399 	__raw_writel(0, &cpsw->host_port_regs->cpdma_rx_chan_map);
1400 
1401 	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM,
1402 			     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1403 
1404 	if (!cpsw->data.dual_emac) {
1405 		cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
1406 				   0, 0);
1407 		cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1408 				   ALE_PORT_HOST, 0, 0, ALE_MCAST_FWD_2);
1409 	}
1410 }
1411 
1412 static int cpsw_fill_rx_channels(struct cpsw_priv *priv)
1413 {
1414 	struct cpsw_common *cpsw = priv->cpsw;
1415 	struct sk_buff *skb;
1416 	int ch_buf_num;
1417 	int ch, i, ret;
1418 
1419 	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1420 		ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);
1421 		for (i = 0; i < ch_buf_num; i++) {
1422 			skb = __netdev_alloc_skb_ip_align(priv->ndev,
1423 							  cpsw->rx_packet_max,
1424 							  GFP_KERNEL);
1425 			if (!skb) {
1426 				cpsw_err(priv, ifup, "cannot allocate skb\n");
1427 				return -ENOMEM;
1428 			}
1429 
1430 			skb_set_queue_mapping(skb, ch);
1431 			ret = cpdma_chan_submit(cpsw->rxv[ch].ch, skb,
1432 						skb->data, skb_tailroom(skb),
1433 						0);
1434 			if (ret < 0) {
1435 				cpsw_err(priv, ifup,
1436 					 "cannot submit skb to channel %d rx, error %d\n",
1437 					 ch, ret);
1438 				kfree_skb(skb);
1439 				return ret;
1440 			}
1441 			kmemleak_not_leak(skb);
1442 		}
1443 
1444 		cpsw_info(priv, ifup, "ch %d rx, submitted %d descriptors\n",
1445 			  ch, ch_buf_num);
1446 	}
1447 
1448 	return 0;
1449 }
1450 
1451 static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_common *cpsw)
1452 {
1453 	u32 slave_port;
1454 
1455 	slave_port = cpsw_get_slave_port(slave->slave_num);
1456 
1457 	if (!slave->phy)
1458 		return;
1459 	phy_stop(slave->phy);
1460 	phy_disconnect(slave->phy);
1461 	slave->phy = NULL;
1462 	cpsw_ale_control_set(cpsw->ale, slave_port,
1463 			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1464 	soft_reset_slave(slave);
1465 }
1466 
1467 static int cpsw_ndo_open(struct net_device *ndev)
1468 {
1469 	struct cpsw_priv *priv = netdev_priv(ndev);
1470 	struct cpsw_common *cpsw = priv->cpsw;
1471 	int ret;
1472 	u32 reg;
1473 
1474 	ret = pm_runtime_get_sync(cpsw->dev);
1475 	if (ret < 0) {
1476 		pm_runtime_put_noidle(cpsw->dev);
1477 		return ret;
1478 	}
1479 
1480 	netif_carrier_off(ndev);
1481 
1482 	/* Notify the stack of the actual queue counts. */
1483 	ret = netif_set_real_num_tx_queues(ndev, cpsw->tx_ch_num);
1484 	if (ret) {
1485 		dev_err(priv->dev, "cannot set real number of tx queues\n");
1486 		goto err_cleanup;
1487 	}
1488 
1489 	ret = netif_set_real_num_rx_queues(ndev, cpsw->rx_ch_num);
1490 	if (ret) {
1491 		dev_err(priv->dev, "cannot set real number of rx queues\n");
1492 		goto err_cleanup;
1493 	}
1494 
1495 	reg = cpsw->version;
1496 
1497 	dev_info(priv->dev, "initializing cpsw version %d.%d (%d)\n",
1498 		 CPSW_MAJOR_VERSION(reg), CPSW_MINOR_VERSION(reg),
1499 		 CPSW_RTL_VERSION(reg));
1500 
1501 	/* Initialize host and slave ports */
1502 	if (!cpsw->usage_count)
1503 		cpsw_init_host_port(priv);
1504 	for_each_slave(priv, cpsw_slave_open, priv);
1505 
1506 	/* Add default VLAN */
1507 	if (!cpsw->data.dual_emac)
1508 		cpsw_add_default_vlan(priv);
1509 	else
1510 		cpsw_ale_add_vlan(cpsw->ale, cpsw->data.default_vlan,
1511 				  ALE_ALL_PORTS, ALE_ALL_PORTS, 0, 0);
1512 
1513 	/* initialize shared resources for every ndev */
1514 	if (!cpsw->usage_count) {
1515 		/* disable priority elevation */
1516 		__raw_writel(0, &cpsw->regs->ptype);
1517 
1518 		/* enable statistics collection only on all ports */
1519 		__raw_writel(0x7, &cpsw->regs->stat_port_en);
1520 
1521 		/* Enable internal fifo flow control */
1522 		writel(0x7, &cpsw->regs->flow_control);
1523 
1524 		napi_enable(&cpsw->napi_rx);
1525 		napi_enable(&cpsw->napi_tx);
1526 
1527 		if (cpsw->tx_irq_disabled) {
1528 			cpsw->tx_irq_disabled = false;
1529 			enable_irq(cpsw->irqs_table[1]);
1530 		}
1531 
1532 		if (cpsw->rx_irq_disabled) {
1533 			cpsw->rx_irq_disabled = false;
1534 			enable_irq(cpsw->irqs_table[0]);
1535 		}
1536 
1537 		ret = cpsw_fill_rx_channels(priv);
1538 		if (ret < 0)
1539 			goto err_cleanup;
1540 
1541 		if (cpts_register(cpsw->cpts))
1542 			dev_err(priv->dev, "error registering cpts device\n");
1543 
1544 	}
1545 
1546 	/* Enable Interrupt pacing if configured */
1547 	if (cpsw->coal_intvl != 0) {
1548 		struct ethtool_coalesce coal;
1549 
1550 		coal.rx_coalesce_usecs = cpsw->coal_intvl;
1551 		cpsw_set_coalesce(ndev, &coal);
1552 	}
1553 
1554 	cpdma_ctlr_start(cpsw->dma);
1555 	cpsw_intr_enable(cpsw);
1556 	cpsw->usage_count++;
1557 
1558 	return 0;
1559 
1560 err_cleanup:
1561 	cpdma_ctlr_stop(cpsw->dma);
1562 	for_each_slave(priv, cpsw_slave_stop, cpsw);
1563 	pm_runtime_put_sync(cpsw->dev);
1564 	netif_carrier_off(priv->ndev);
1565 	return ret;
1566 }
1567 
1568 static int cpsw_ndo_stop(struct net_device *ndev)
1569 {
1570 	struct cpsw_priv *priv = netdev_priv(ndev);
1571 	struct cpsw_common *cpsw = priv->cpsw;
1572 
1573 	cpsw_info(priv, ifdown, "shutting down cpsw device\n");
1574 	netif_tx_stop_all_queues(priv->ndev);
1575 	netif_carrier_off(priv->ndev);
1576 
1577 	if (cpsw->usage_count <= 1) {
1578 		napi_disable(&cpsw->napi_rx);
1579 		napi_disable(&cpsw->napi_tx);
1580 		cpts_unregister(cpsw->cpts);
1581 		cpsw_intr_disable(cpsw);
1582 		cpdma_ctlr_stop(cpsw->dma);
1583 		cpsw_ale_stop(cpsw->ale);
1584 	}
1585 	for_each_slave(priv, cpsw_slave_stop, cpsw);
1586 
1587 	if (cpsw_need_resplit(cpsw))
1588 		cpsw_split_res(ndev);
1589 
1590 	cpsw->usage_count--;
1591 	pm_runtime_put_sync(cpsw->dev);
1592 	return 0;
1593 }
1594 
1595 static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
1596 				       struct net_device *ndev)
1597 {
1598 	struct cpsw_priv *priv = netdev_priv(ndev);
1599 	struct cpsw_common *cpsw = priv->cpsw;
1600 	struct netdev_queue *txq;
1601 	struct cpdma_chan *txch;
1602 	int ret, q_idx;
1603 
1604 	if (skb_padto(skb, CPSW_MIN_PACKET_SIZE)) {
1605 		cpsw_err(priv, tx_err, "packet pad failed\n");
1606 		ndev->stats.tx_dropped++;
1607 		return NET_XMIT_DROP;
1608 	}
1609 
1610 	if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
1611 	    cpts_is_tx_enabled(cpsw->cpts))
1612 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1613 
1614 	skb_tx_timestamp(skb);
1615 
1616 	q_idx = skb_get_queue_mapping(skb);
1617 	if (q_idx >= cpsw->tx_ch_num)
1618 		q_idx = q_idx % cpsw->tx_ch_num;
1619 
1620 	txch = cpsw->txv[q_idx].ch;
1621 	ret = cpsw_tx_packet_submit(priv, skb, txch);
1622 	if (unlikely(ret != 0)) {
1623 		cpsw_err(priv, tx_err, "desc submit failed\n");
1624 		goto fail;
1625 	}
1626 
1627 	/* If there is no more tx desc left free then we need to
1628 	 * tell the kernel to stop sending us tx frames.
1629 	 */
1630 	if (unlikely(!cpdma_check_free_tx_desc(txch))) {
1631 		txq = netdev_get_tx_queue(ndev, q_idx);
1632 		netif_tx_stop_queue(txq);
1633 	}
1634 
1635 	return NETDEV_TX_OK;
1636 fail:
1637 	ndev->stats.tx_dropped++;
1638 	txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
1639 	netif_tx_stop_queue(txq);
1640 	return NETDEV_TX_BUSY;
1641 }
1642 
1643 #if IS_ENABLED(CONFIG_TI_CPTS)
1644 
1645 static void cpsw_hwtstamp_v1(struct cpsw_common *cpsw)
1646 {
1647 	struct cpsw_slave *slave = &cpsw->slaves[cpsw->data.active_slave];
1648 	u32 ts_en, seq_id;
1649 
1650 	if (!cpts_is_tx_enabled(cpsw->cpts) &&
1651 	    !cpts_is_rx_enabled(cpsw->cpts)) {
1652 		slave_write(slave, 0, CPSW1_TS_CTL);
1653 		return;
1654 	}
1655 
1656 	seq_id = (30 << CPSW_V1_SEQ_ID_OFS_SHIFT) | ETH_P_1588;
1657 	ts_en = EVENT_MSG_BITS << CPSW_V1_MSG_TYPE_OFS;
1658 
1659 	if (cpts_is_tx_enabled(cpsw->cpts))
1660 		ts_en |= CPSW_V1_TS_TX_EN;
1661 
1662 	if (cpts_is_rx_enabled(cpsw->cpts))
1663 		ts_en |= CPSW_V1_TS_RX_EN;
1664 
1665 	slave_write(slave, ts_en, CPSW1_TS_CTL);
1666 	slave_write(slave, seq_id, CPSW1_TS_SEQ_LTYPE);
1667 }
1668 
1669 static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
1670 {
1671 	struct cpsw_slave *slave;
1672 	struct cpsw_common *cpsw = priv->cpsw;
1673 	u32 ctrl, mtype;
1674 
1675 	slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1676 
1677 	ctrl = slave_read(slave, CPSW2_CONTROL);
1678 	switch (cpsw->version) {
1679 	case CPSW_VERSION_2:
1680 		ctrl &= ~CTRL_V2_ALL_TS_MASK;
1681 
1682 		if (cpts_is_tx_enabled(cpsw->cpts))
1683 			ctrl |= CTRL_V2_TX_TS_BITS;
1684 
1685 		if (cpts_is_rx_enabled(cpsw->cpts))
1686 			ctrl |= CTRL_V2_RX_TS_BITS;
1687 		break;
1688 	case CPSW_VERSION_3:
1689 	default:
1690 		ctrl &= ~CTRL_V3_ALL_TS_MASK;
1691 
1692 		if (cpts_is_tx_enabled(cpsw->cpts))
1693 			ctrl |= CTRL_V3_TX_TS_BITS;
1694 
1695 		if (cpts_is_rx_enabled(cpsw->cpts))
1696 			ctrl |= CTRL_V3_RX_TS_BITS;
1697 		break;
1698 	}
1699 
1700 	mtype = (30 << TS_SEQ_ID_OFFSET_SHIFT) | EVENT_MSG_BITS;
1701 
1702 	slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
1703 	slave_write(slave, ctrl, CPSW2_CONTROL);
1704 	__raw_writel(ETH_P_1588, &cpsw->regs->ts_ltype);
1705 }
1706 
1707 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
1708 {
1709 	struct cpsw_priv *priv = netdev_priv(dev);
1710 	struct hwtstamp_config cfg;
1711 	struct cpsw_common *cpsw = priv->cpsw;
1712 	struct cpts *cpts = cpsw->cpts;
1713 
1714 	if (cpsw->version != CPSW_VERSION_1 &&
1715 	    cpsw->version != CPSW_VERSION_2 &&
1716 	    cpsw->version != CPSW_VERSION_3)
1717 		return -EOPNOTSUPP;
1718 
1719 	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1720 		return -EFAULT;
1721 
1722 	/* reserved for future extensions */
1723 	if (cfg.flags)
1724 		return -EINVAL;
1725 
1726 	if (cfg.tx_type != HWTSTAMP_TX_OFF && cfg.tx_type != HWTSTAMP_TX_ON)
1727 		return -ERANGE;
1728 
1729 	switch (cfg.rx_filter) {
1730 	case HWTSTAMP_FILTER_NONE:
1731 		cpts_rx_enable(cpts, 0);
1732 		break;
1733 	case HWTSTAMP_FILTER_ALL:
1734 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1735 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1736 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1737 		return -ERANGE;
1738 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1739 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1740 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1741 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1742 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1743 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1744 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1745 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1746 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1747 		cpts_rx_enable(cpts, 1);
1748 		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
1749 		break;
1750 	default:
1751 		return -ERANGE;
1752 	}
1753 
1754 	cpts_tx_enable(cpts, cfg.tx_type == HWTSTAMP_TX_ON);
1755 
1756 	switch (cpsw->version) {
1757 	case CPSW_VERSION_1:
1758 		cpsw_hwtstamp_v1(cpsw);
1759 		break;
1760 	case CPSW_VERSION_2:
1761 	case CPSW_VERSION_3:
1762 		cpsw_hwtstamp_v2(priv);
1763 		break;
1764 	default:
1765 		WARN_ON(1);
1766 	}
1767 
1768 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1769 }
1770 
1771 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
1772 {
1773 	struct cpsw_common *cpsw = ndev_to_cpsw(dev);
1774 	struct cpts *cpts = cpsw->cpts;
1775 	struct hwtstamp_config cfg;
1776 
1777 	if (cpsw->version != CPSW_VERSION_1 &&
1778 	    cpsw->version != CPSW_VERSION_2 &&
1779 	    cpsw->version != CPSW_VERSION_3)
1780 		return -EOPNOTSUPP;
1781 
1782 	cfg.flags = 0;
1783 	cfg.tx_type = cpts_is_tx_enabled(cpts) ?
1784 		      HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1785 	cfg.rx_filter = (cpts_is_rx_enabled(cpts) ?
1786 			 HWTSTAMP_FILTER_PTP_V2_EVENT : HWTSTAMP_FILTER_NONE);
1787 
1788 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1789 }
1790 #else
1791 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
1792 {
1793 	return -EOPNOTSUPP;
1794 }
1795 
1796 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
1797 {
1798 	return -EOPNOTSUPP;
1799 }
1800 #endif /*CONFIG_TI_CPTS*/
1801 
1802 static int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
1803 {
1804 	struct cpsw_priv *priv = netdev_priv(dev);
1805 	struct cpsw_common *cpsw = priv->cpsw;
1806 	int slave_no = cpsw_slave_index(cpsw, priv);
1807 
1808 	if (!netif_running(dev))
1809 		return -EINVAL;
1810 
1811 	switch (cmd) {
1812 	case SIOCSHWTSTAMP:
1813 		return cpsw_hwtstamp_set(dev, req);
1814 	case SIOCGHWTSTAMP:
1815 		return cpsw_hwtstamp_get(dev, req);
1816 	}
1817 
1818 	if (!cpsw->slaves[slave_no].phy)
1819 		return -EOPNOTSUPP;
1820 	return phy_mii_ioctl(cpsw->slaves[slave_no].phy, req, cmd);
1821 }
1822 
1823 static void cpsw_ndo_tx_timeout(struct net_device *ndev)
1824 {
1825 	struct cpsw_priv *priv = netdev_priv(ndev);
1826 	struct cpsw_common *cpsw = priv->cpsw;
1827 	int ch;
1828 
1829 	cpsw_err(priv, tx_err, "transmit timeout, restarting dma\n");
1830 	ndev->stats.tx_errors++;
1831 	cpsw_intr_disable(cpsw);
1832 	for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
1833 		cpdma_chan_stop(cpsw->txv[ch].ch);
1834 		cpdma_chan_start(cpsw->txv[ch].ch);
1835 	}
1836 
1837 	cpsw_intr_enable(cpsw);
1838 	netif_trans_update(ndev);
1839 	netif_tx_wake_all_queues(ndev);
1840 }
1841 
1842 static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
1843 {
1844 	struct cpsw_priv *priv = netdev_priv(ndev);
1845 	struct sockaddr *addr = (struct sockaddr *)p;
1846 	struct cpsw_common *cpsw = priv->cpsw;
1847 	int flags = 0;
1848 	u16 vid = 0;
1849 	int ret;
1850 
1851 	if (!is_valid_ether_addr(addr->sa_data))
1852 		return -EADDRNOTAVAIL;
1853 
1854 	ret = pm_runtime_get_sync(cpsw->dev);
1855 	if (ret < 0) {
1856 		pm_runtime_put_noidle(cpsw->dev);
1857 		return ret;
1858 	}
1859 
1860 	if (cpsw->data.dual_emac) {
1861 		vid = cpsw->slaves[priv->emac_port].port_vlan;
1862 		flags = ALE_VLAN;
1863 	}
1864 
1865 	cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
1866 			   flags, vid);
1867 	cpsw_ale_add_ucast(cpsw->ale, addr->sa_data, HOST_PORT_NUM,
1868 			   flags, vid);
1869 
1870 	memcpy(priv->mac_addr, addr->sa_data, ETH_ALEN);
1871 	memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
1872 	for_each_slave(priv, cpsw_set_slave_mac, priv);
1873 
1874 	pm_runtime_put(cpsw->dev);
1875 
1876 	return 0;
1877 }
1878 
1879 #ifdef CONFIG_NET_POLL_CONTROLLER
1880 static void cpsw_ndo_poll_controller(struct net_device *ndev)
1881 {
1882 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1883 
1884 	cpsw_intr_disable(cpsw);
1885 	cpsw_rx_interrupt(cpsw->irqs_table[0], cpsw);
1886 	cpsw_tx_interrupt(cpsw->irqs_table[1], cpsw);
1887 	cpsw_intr_enable(cpsw);
1888 }
1889 #endif
1890 
1891 static inline int cpsw_add_vlan_ale_entry(struct cpsw_priv *priv,
1892 				unsigned short vid)
1893 {
1894 	int ret;
1895 	int unreg_mcast_mask = 0;
1896 	u32 port_mask;
1897 	struct cpsw_common *cpsw = priv->cpsw;
1898 
1899 	if (cpsw->data.dual_emac) {
1900 		port_mask = (1 << (priv->emac_port + 1)) | ALE_PORT_HOST;
1901 
1902 		if (priv->ndev->flags & IFF_ALLMULTI)
1903 			unreg_mcast_mask = port_mask;
1904 	} else {
1905 		port_mask = ALE_ALL_PORTS;
1906 
1907 		if (priv->ndev->flags & IFF_ALLMULTI)
1908 			unreg_mcast_mask = ALE_ALL_PORTS;
1909 		else
1910 			unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
1911 	}
1912 
1913 	ret = cpsw_ale_add_vlan(cpsw->ale, vid, port_mask, 0, port_mask,
1914 				unreg_mcast_mask);
1915 	if (ret != 0)
1916 		return ret;
1917 
1918 	ret = cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
1919 				 HOST_PORT_NUM, ALE_VLAN, vid);
1920 	if (ret != 0)
1921 		goto clean_vid;
1922 
1923 	ret = cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1924 				 port_mask, ALE_VLAN, vid, 0);
1925 	if (ret != 0)
1926 		goto clean_vlan_ucast;
1927 	return 0;
1928 
1929 clean_vlan_ucast:
1930 	cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
1931 			   HOST_PORT_NUM, ALE_VLAN, vid);
1932 clean_vid:
1933 	cpsw_ale_del_vlan(cpsw->ale, vid, 0);
1934 	return ret;
1935 }
1936 
1937 static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
1938 				    __be16 proto, u16 vid)
1939 {
1940 	struct cpsw_priv *priv = netdev_priv(ndev);
1941 	struct cpsw_common *cpsw = priv->cpsw;
1942 	int ret;
1943 
1944 	if (vid == cpsw->data.default_vlan)
1945 		return 0;
1946 
1947 	ret = pm_runtime_get_sync(cpsw->dev);
1948 	if (ret < 0) {
1949 		pm_runtime_put_noidle(cpsw->dev);
1950 		return ret;
1951 	}
1952 
1953 	if (cpsw->data.dual_emac) {
1954 		/* In dual EMAC, reserved VLAN id should not be used for
1955 		 * creating VLAN interfaces as this can break the dual
1956 		 * EMAC port separation
1957 		 */
1958 		int i;
1959 
1960 		for (i = 0; i < cpsw->data.slaves; i++) {
1961 			if (vid == cpsw->slaves[i].port_vlan)
1962 				return -EINVAL;
1963 		}
1964 	}
1965 
1966 	dev_info(priv->dev, "Adding vlanid %d to vlan filter\n", vid);
1967 	ret = cpsw_add_vlan_ale_entry(priv, vid);
1968 
1969 	pm_runtime_put(cpsw->dev);
1970 	return ret;
1971 }
1972 
1973 static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
1974 				     __be16 proto, u16 vid)
1975 {
1976 	struct cpsw_priv *priv = netdev_priv(ndev);
1977 	struct cpsw_common *cpsw = priv->cpsw;
1978 	int ret;
1979 
1980 	if (vid == cpsw->data.default_vlan)
1981 		return 0;
1982 
1983 	ret = pm_runtime_get_sync(cpsw->dev);
1984 	if (ret < 0) {
1985 		pm_runtime_put_noidle(cpsw->dev);
1986 		return ret;
1987 	}
1988 
1989 	if (cpsw->data.dual_emac) {
1990 		int i;
1991 
1992 		for (i = 0; i < cpsw->data.slaves; i++) {
1993 			if (vid == cpsw->slaves[i].port_vlan)
1994 				return -EINVAL;
1995 		}
1996 	}
1997 
1998 	dev_info(priv->dev, "removing vlanid %d from vlan filter\n", vid);
1999 	ret = cpsw_ale_del_vlan(cpsw->ale, vid, 0);
2000 	if (ret != 0)
2001 		return ret;
2002 
2003 	ret = cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
2004 				 HOST_PORT_NUM, ALE_VLAN, vid);
2005 	if (ret != 0)
2006 		return ret;
2007 
2008 	ret = cpsw_ale_del_mcast(cpsw->ale, priv->ndev->broadcast,
2009 				 0, ALE_VLAN, vid);
2010 	pm_runtime_put(cpsw->dev);
2011 	return ret;
2012 }
2013 
2014 static int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate)
2015 {
2016 	struct cpsw_priv *priv = netdev_priv(ndev);
2017 	struct cpsw_common *cpsw = priv->cpsw;
2018 	struct cpsw_slave *slave;
2019 	u32 min_rate;
2020 	u32 ch_rate;
2021 	int i, ret;
2022 
2023 	ch_rate = netdev_get_tx_queue(ndev, queue)->tx_maxrate;
2024 	if (ch_rate == rate)
2025 		return 0;
2026 
2027 	ch_rate = rate * 1000;
2028 	min_rate = cpdma_chan_get_min_rate(cpsw->dma);
2029 	if ((ch_rate < min_rate && ch_rate)) {
2030 		dev_err(priv->dev, "The channel rate cannot be less than %dMbps",
2031 			min_rate);
2032 		return -EINVAL;
2033 	}
2034 
2035 	if (rate > cpsw->speed) {
2036 		dev_err(priv->dev, "The channel rate cannot be more than 2Gbps");
2037 		return -EINVAL;
2038 	}
2039 
2040 	ret = pm_runtime_get_sync(cpsw->dev);
2041 	if (ret < 0) {
2042 		pm_runtime_put_noidle(cpsw->dev);
2043 		return ret;
2044 	}
2045 
2046 	ret = cpdma_chan_set_rate(cpsw->txv[queue].ch, ch_rate);
2047 	pm_runtime_put(cpsw->dev);
2048 
2049 	if (ret)
2050 		return ret;
2051 
2052 	/* update rates for slaves tx queues */
2053 	for (i = 0; i < cpsw->data.slaves; i++) {
2054 		slave = &cpsw->slaves[i];
2055 		if (!slave->ndev)
2056 			continue;
2057 
2058 		netdev_get_tx_queue(slave->ndev, queue)->tx_maxrate = rate;
2059 	}
2060 
2061 	cpsw_split_res(ndev);
2062 	return ret;
2063 }
2064 
2065 static const struct net_device_ops cpsw_netdev_ops = {
2066 	.ndo_open		= cpsw_ndo_open,
2067 	.ndo_stop		= cpsw_ndo_stop,
2068 	.ndo_start_xmit		= cpsw_ndo_start_xmit,
2069 	.ndo_set_mac_address	= cpsw_ndo_set_mac_address,
2070 	.ndo_do_ioctl		= cpsw_ndo_ioctl,
2071 	.ndo_validate_addr	= eth_validate_addr,
2072 	.ndo_tx_timeout		= cpsw_ndo_tx_timeout,
2073 	.ndo_set_rx_mode	= cpsw_ndo_set_rx_mode,
2074 	.ndo_set_tx_maxrate	= cpsw_ndo_set_tx_maxrate,
2075 #ifdef CONFIG_NET_POLL_CONTROLLER
2076 	.ndo_poll_controller	= cpsw_ndo_poll_controller,
2077 #endif
2078 	.ndo_vlan_rx_add_vid	= cpsw_ndo_vlan_rx_add_vid,
2079 	.ndo_vlan_rx_kill_vid	= cpsw_ndo_vlan_rx_kill_vid,
2080 };
2081 
2082 static int cpsw_get_regs_len(struct net_device *ndev)
2083 {
2084 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2085 
2086 	return cpsw->data.ale_entries * ALE_ENTRY_WORDS * sizeof(u32);
2087 }
2088 
2089 static void cpsw_get_regs(struct net_device *ndev,
2090 			  struct ethtool_regs *regs, void *p)
2091 {
2092 	u32 *reg = p;
2093 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2094 
2095 	/* update CPSW IP version */
2096 	regs->version = cpsw->version;
2097 
2098 	cpsw_ale_dump(cpsw->ale, reg);
2099 }
2100 
2101 static void cpsw_get_drvinfo(struct net_device *ndev,
2102 			     struct ethtool_drvinfo *info)
2103 {
2104 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2105 	struct platform_device	*pdev = to_platform_device(cpsw->dev);
2106 
2107 	strlcpy(info->driver, "cpsw", sizeof(info->driver));
2108 	strlcpy(info->version, "1.0", sizeof(info->version));
2109 	strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
2110 }
2111 
2112 static u32 cpsw_get_msglevel(struct net_device *ndev)
2113 {
2114 	struct cpsw_priv *priv = netdev_priv(ndev);
2115 	return priv->msg_enable;
2116 }
2117 
2118 static void cpsw_set_msglevel(struct net_device *ndev, u32 value)
2119 {
2120 	struct cpsw_priv *priv = netdev_priv(ndev);
2121 	priv->msg_enable = value;
2122 }
2123 
2124 #if IS_ENABLED(CONFIG_TI_CPTS)
2125 static int cpsw_get_ts_info(struct net_device *ndev,
2126 			    struct ethtool_ts_info *info)
2127 {
2128 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2129 
2130 	info->so_timestamping =
2131 		SOF_TIMESTAMPING_TX_HARDWARE |
2132 		SOF_TIMESTAMPING_TX_SOFTWARE |
2133 		SOF_TIMESTAMPING_RX_HARDWARE |
2134 		SOF_TIMESTAMPING_RX_SOFTWARE |
2135 		SOF_TIMESTAMPING_SOFTWARE |
2136 		SOF_TIMESTAMPING_RAW_HARDWARE;
2137 	info->phc_index = cpsw->cpts->phc_index;
2138 	info->tx_types =
2139 		(1 << HWTSTAMP_TX_OFF) |
2140 		(1 << HWTSTAMP_TX_ON);
2141 	info->rx_filters =
2142 		(1 << HWTSTAMP_FILTER_NONE) |
2143 		(1 << HWTSTAMP_FILTER_PTP_V2_EVENT);
2144 	return 0;
2145 }
2146 #else
2147 static int cpsw_get_ts_info(struct net_device *ndev,
2148 			    struct ethtool_ts_info *info)
2149 {
2150 	info->so_timestamping =
2151 		SOF_TIMESTAMPING_TX_SOFTWARE |
2152 		SOF_TIMESTAMPING_RX_SOFTWARE |
2153 		SOF_TIMESTAMPING_SOFTWARE;
2154 	info->phc_index = -1;
2155 	info->tx_types = 0;
2156 	info->rx_filters = 0;
2157 	return 0;
2158 }
2159 #endif
2160 
2161 static int cpsw_get_link_ksettings(struct net_device *ndev,
2162 				   struct ethtool_link_ksettings *ecmd)
2163 {
2164 	struct cpsw_priv *priv = netdev_priv(ndev);
2165 	struct cpsw_common *cpsw = priv->cpsw;
2166 	int slave_no = cpsw_slave_index(cpsw, priv);
2167 
2168 	if (cpsw->slaves[slave_no].phy)
2169 		return phy_ethtool_ksettings_get(cpsw->slaves[slave_no].phy,
2170 						 ecmd);
2171 	else
2172 		return -EOPNOTSUPP;
2173 }
2174 
2175 static int cpsw_set_link_ksettings(struct net_device *ndev,
2176 				   const struct ethtool_link_ksettings *ecmd)
2177 {
2178 	struct cpsw_priv *priv = netdev_priv(ndev);
2179 	struct cpsw_common *cpsw = priv->cpsw;
2180 	int slave_no = cpsw_slave_index(cpsw, priv);
2181 
2182 	if (cpsw->slaves[slave_no].phy)
2183 		return phy_ethtool_ksettings_set(cpsw->slaves[slave_no].phy,
2184 						 ecmd);
2185 	else
2186 		return -EOPNOTSUPP;
2187 }
2188 
2189 static void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2190 {
2191 	struct cpsw_priv *priv = netdev_priv(ndev);
2192 	struct cpsw_common *cpsw = priv->cpsw;
2193 	int slave_no = cpsw_slave_index(cpsw, priv);
2194 
2195 	wol->supported = 0;
2196 	wol->wolopts = 0;
2197 
2198 	if (cpsw->slaves[slave_no].phy)
2199 		phy_ethtool_get_wol(cpsw->slaves[slave_no].phy, wol);
2200 }
2201 
2202 static int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2203 {
2204 	struct cpsw_priv *priv = netdev_priv(ndev);
2205 	struct cpsw_common *cpsw = priv->cpsw;
2206 	int slave_no = cpsw_slave_index(cpsw, priv);
2207 
2208 	if (cpsw->slaves[slave_no].phy)
2209 		return phy_ethtool_set_wol(cpsw->slaves[slave_no].phy, wol);
2210 	else
2211 		return -EOPNOTSUPP;
2212 }
2213 
2214 static void cpsw_get_pauseparam(struct net_device *ndev,
2215 				struct ethtool_pauseparam *pause)
2216 {
2217 	struct cpsw_priv *priv = netdev_priv(ndev);
2218 
2219 	pause->autoneg = AUTONEG_DISABLE;
2220 	pause->rx_pause = priv->rx_pause ? true : false;
2221 	pause->tx_pause = priv->tx_pause ? true : false;
2222 }
2223 
2224 static int cpsw_set_pauseparam(struct net_device *ndev,
2225 			       struct ethtool_pauseparam *pause)
2226 {
2227 	struct cpsw_priv *priv = netdev_priv(ndev);
2228 	bool link;
2229 
2230 	priv->rx_pause = pause->rx_pause ? true : false;
2231 	priv->tx_pause = pause->tx_pause ? true : false;
2232 
2233 	for_each_slave(priv, _cpsw_adjust_link, priv, &link);
2234 	return 0;
2235 }
2236 
2237 static int cpsw_ethtool_op_begin(struct net_device *ndev)
2238 {
2239 	struct cpsw_priv *priv = netdev_priv(ndev);
2240 	struct cpsw_common *cpsw = priv->cpsw;
2241 	int ret;
2242 
2243 	ret = pm_runtime_get_sync(cpsw->dev);
2244 	if (ret < 0) {
2245 		cpsw_err(priv, drv, "ethtool begin failed %d\n", ret);
2246 		pm_runtime_put_noidle(cpsw->dev);
2247 	}
2248 
2249 	return ret;
2250 }
2251 
2252 static void cpsw_ethtool_op_complete(struct net_device *ndev)
2253 {
2254 	struct cpsw_priv *priv = netdev_priv(ndev);
2255 	int ret;
2256 
2257 	ret = pm_runtime_put(priv->cpsw->dev);
2258 	if (ret < 0)
2259 		cpsw_err(priv, drv, "ethtool complete failed %d\n", ret);
2260 }
2261 
2262 static void cpsw_get_channels(struct net_device *ndev,
2263 			      struct ethtool_channels *ch)
2264 {
2265 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2266 
2267 	ch->max_combined = 0;
2268 	ch->max_rx = CPSW_MAX_QUEUES;
2269 	ch->max_tx = CPSW_MAX_QUEUES;
2270 	ch->max_other = 0;
2271 	ch->other_count = 0;
2272 	ch->rx_count = cpsw->rx_ch_num;
2273 	ch->tx_count = cpsw->tx_ch_num;
2274 	ch->combined_count = 0;
2275 }
2276 
2277 static int cpsw_check_ch_settings(struct cpsw_common *cpsw,
2278 				  struct ethtool_channels *ch)
2279 {
2280 	if (ch->combined_count)
2281 		return -EINVAL;
2282 
2283 	/* verify we have at least one channel in each direction */
2284 	if (!ch->rx_count || !ch->tx_count)
2285 		return -EINVAL;
2286 
2287 	if (ch->rx_count > cpsw->data.channels ||
2288 	    ch->tx_count > cpsw->data.channels)
2289 		return -EINVAL;
2290 
2291 	return 0;
2292 }
2293 
2294 static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx)
2295 {
2296 	int (*poll)(struct napi_struct *, int);
2297 	struct cpsw_common *cpsw = priv->cpsw;
2298 	void (*handler)(void *, int, int);
2299 	struct netdev_queue *queue;
2300 	struct cpsw_vector *vec;
2301 	int ret, *ch;
2302 
2303 	if (rx) {
2304 		ch = &cpsw->rx_ch_num;
2305 		vec = cpsw->rxv;
2306 		handler = cpsw_rx_handler;
2307 		poll = cpsw_rx_poll;
2308 	} else {
2309 		ch = &cpsw->tx_ch_num;
2310 		vec = cpsw->txv;
2311 		handler = cpsw_tx_handler;
2312 		poll = cpsw_tx_poll;
2313 	}
2314 
2315 	while (*ch < ch_num) {
2316 		vec[*ch].ch = cpdma_chan_create(cpsw->dma, *ch, handler, rx);
2317 		queue = netdev_get_tx_queue(priv->ndev, *ch);
2318 		queue->tx_maxrate = 0;
2319 
2320 		if (IS_ERR(vec[*ch].ch))
2321 			return PTR_ERR(vec[*ch].ch);
2322 
2323 		if (!vec[*ch].ch)
2324 			return -EINVAL;
2325 
2326 		cpsw_info(priv, ifup, "created new %d %s channel\n", *ch,
2327 			  (rx ? "rx" : "tx"));
2328 		(*ch)++;
2329 	}
2330 
2331 	while (*ch > ch_num) {
2332 		(*ch)--;
2333 
2334 		ret = cpdma_chan_destroy(vec[*ch].ch);
2335 		if (ret)
2336 			return ret;
2337 
2338 		cpsw_info(priv, ifup, "destroyed %d %s channel\n", *ch,
2339 			  (rx ? "rx" : "tx"));
2340 	}
2341 
2342 	return 0;
2343 }
2344 
2345 static int cpsw_update_channels(struct cpsw_priv *priv,
2346 				struct ethtool_channels *ch)
2347 {
2348 	int ret;
2349 
2350 	ret = cpsw_update_channels_res(priv, ch->rx_count, 1);
2351 	if (ret)
2352 		return ret;
2353 
2354 	ret = cpsw_update_channels_res(priv, ch->tx_count, 0);
2355 	if (ret)
2356 		return ret;
2357 
2358 	return 0;
2359 }
2360 
2361 static void cpsw_suspend_data_pass(struct net_device *ndev)
2362 {
2363 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2364 	struct cpsw_slave *slave;
2365 	int i;
2366 
2367 	/* Disable NAPI scheduling */
2368 	cpsw_intr_disable(cpsw);
2369 
2370 	/* Stop all transmit queues for every network device.
2371 	 * Disable re-using rx descriptors with dormant_on.
2372 	 */
2373 	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2374 		if (!(slave->ndev && netif_running(slave->ndev)))
2375 			continue;
2376 
2377 		netif_tx_stop_all_queues(slave->ndev);
2378 		netif_dormant_on(slave->ndev);
2379 	}
2380 
2381 	/* Handle rest of tx packets and stop cpdma channels */
2382 	cpdma_ctlr_stop(cpsw->dma);
2383 }
2384 
2385 static int cpsw_resume_data_pass(struct net_device *ndev)
2386 {
2387 	struct cpsw_priv *priv = netdev_priv(ndev);
2388 	struct cpsw_common *cpsw = priv->cpsw;
2389 	struct cpsw_slave *slave;
2390 	int i, ret;
2391 
2392 	/* Allow rx packets handling */
2393 	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++)
2394 		if (slave->ndev && netif_running(slave->ndev))
2395 			netif_dormant_off(slave->ndev);
2396 
2397 	/* After this receive is started */
2398 	if (cpsw->usage_count) {
2399 		ret = cpsw_fill_rx_channels(priv);
2400 		if (ret)
2401 			return ret;
2402 
2403 		cpdma_ctlr_start(cpsw->dma);
2404 		cpsw_intr_enable(cpsw);
2405 	}
2406 
2407 	/* Resume transmit for every affected interface */
2408 	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++)
2409 		if (slave->ndev && netif_running(slave->ndev))
2410 			netif_tx_start_all_queues(slave->ndev);
2411 
2412 	return 0;
2413 }
2414 
2415 static int cpsw_set_channels(struct net_device *ndev,
2416 			     struct ethtool_channels *chs)
2417 {
2418 	struct cpsw_priv *priv = netdev_priv(ndev);
2419 	struct cpsw_common *cpsw = priv->cpsw;
2420 	struct cpsw_slave *slave;
2421 	int i, ret;
2422 
2423 	ret = cpsw_check_ch_settings(cpsw, chs);
2424 	if (ret < 0)
2425 		return ret;
2426 
2427 	cpsw_suspend_data_pass(ndev);
2428 	ret = cpsw_update_channels(priv, chs);
2429 	if (ret)
2430 		goto err;
2431 
2432 	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2433 		if (!(slave->ndev && netif_running(slave->ndev)))
2434 			continue;
2435 
2436 		/* Inform stack about new count of queues */
2437 		ret = netif_set_real_num_tx_queues(slave->ndev,
2438 						   cpsw->tx_ch_num);
2439 		if (ret) {
2440 			dev_err(priv->dev, "cannot set real number of tx queues\n");
2441 			goto err;
2442 		}
2443 
2444 		ret = netif_set_real_num_rx_queues(slave->ndev,
2445 						   cpsw->rx_ch_num);
2446 		if (ret) {
2447 			dev_err(priv->dev, "cannot set real number of rx queues\n");
2448 			goto err;
2449 		}
2450 	}
2451 
2452 	if (cpsw->usage_count)
2453 		cpsw_split_res(ndev);
2454 
2455 	ret = cpsw_resume_data_pass(ndev);
2456 	if (!ret)
2457 		return 0;
2458 err:
2459 	dev_err(priv->dev, "cannot update channels number, closing device\n");
2460 	dev_close(ndev);
2461 	return ret;
2462 }
2463 
2464 static int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata)
2465 {
2466 	struct cpsw_priv *priv = netdev_priv(ndev);
2467 	struct cpsw_common *cpsw = priv->cpsw;
2468 	int slave_no = cpsw_slave_index(cpsw, priv);
2469 
2470 	if (cpsw->slaves[slave_no].phy)
2471 		return phy_ethtool_get_eee(cpsw->slaves[slave_no].phy, edata);
2472 	else
2473 		return -EOPNOTSUPP;
2474 }
2475 
2476 static int cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
2477 {
2478 	struct cpsw_priv *priv = netdev_priv(ndev);
2479 	struct cpsw_common *cpsw = priv->cpsw;
2480 	int slave_no = cpsw_slave_index(cpsw, priv);
2481 
2482 	if (cpsw->slaves[slave_no].phy)
2483 		return phy_ethtool_set_eee(cpsw->slaves[slave_no].phy, edata);
2484 	else
2485 		return -EOPNOTSUPP;
2486 }
2487 
2488 static int cpsw_nway_reset(struct net_device *ndev)
2489 {
2490 	struct cpsw_priv *priv = netdev_priv(ndev);
2491 	struct cpsw_common *cpsw = priv->cpsw;
2492 	int slave_no = cpsw_slave_index(cpsw, priv);
2493 
2494 	if (cpsw->slaves[slave_no].phy)
2495 		return genphy_restart_aneg(cpsw->slaves[slave_no].phy);
2496 	else
2497 		return -EOPNOTSUPP;
2498 }
2499 
2500 static void cpsw_get_ringparam(struct net_device *ndev,
2501 			       struct ethtool_ringparam *ering)
2502 {
2503 	struct cpsw_priv *priv = netdev_priv(ndev);
2504 	struct cpsw_common *cpsw = priv->cpsw;
2505 
2506 	/* not supported */
2507 	ering->tx_max_pending = 0;
2508 	ering->tx_pending = cpdma_get_num_tx_descs(cpsw->dma);
2509 	ering->rx_max_pending = descs_pool_size - CPSW_MAX_QUEUES;
2510 	ering->rx_pending = cpdma_get_num_rx_descs(cpsw->dma);
2511 }
2512 
2513 static int cpsw_set_ringparam(struct net_device *ndev,
2514 			      struct ethtool_ringparam *ering)
2515 {
2516 	struct cpsw_priv *priv = netdev_priv(ndev);
2517 	struct cpsw_common *cpsw = priv->cpsw;
2518 	int ret;
2519 
2520 	/* ignore ering->tx_pending - only rx_pending adjustment is supported */
2521 
2522 	if (ering->rx_mini_pending || ering->rx_jumbo_pending ||
2523 	    ering->rx_pending < CPSW_MAX_QUEUES ||
2524 	    ering->rx_pending > (descs_pool_size - CPSW_MAX_QUEUES))
2525 		return -EINVAL;
2526 
2527 	if (ering->rx_pending == cpdma_get_num_rx_descs(cpsw->dma))
2528 		return 0;
2529 
2530 	cpsw_suspend_data_pass(ndev);
2531 
2532 	cpdma_set_num_rx_descs(cpsw->dma, ering->rx_pending);
2533 
2534 	if (cpsw->usage_count)
2535 		cpdma_chan_split_pool(cpsw->dma);
2536 
2537 	ret = cpsw_resume_data_pass(ndev);
2538 	if (!ret)
2539 		return 0;
2540 
2541 	dev_err(&ndev->dev, "cannot set ring params, closing device\n");
2542 	dev_close(ndev);
2543 	return ret;
2544 }
2545 
2546 static const struct ethtool_ops cpsw_ethtool_ops = {
2547 	.get_drvinfo	= cpsw_get_drvinfo,
2548 	.get_msglevel	= cpsw_get_msglevel,
2549 	.set_msglevel	= cpsw_set_msglevel,
2550 	.get_link	= ethtool_op_get_link,
2551 	.get_ts_info	= cpsw_get_ts_info,
2552 	.get_coalesce	= cpsw_get_coalesce,
2553 	.set_coalesce	= cpsw_set_coalesce,
2554 	.get_sset_count		= cpsw_get_sset_count,
2555 	.get_strings		= cpsw_get_strings,
2556 	.get_ethtool_stats	= cpsw_get_ethtool_stats,
2557 	.get_pauseparam		= cpsw_get_pauseparam,
2558 	.set_pauseparam		= cpsw_set_pauseparam,
2559 	.get_wol	= cpsw_get_wol,
2560 	.set_wol	= cpsw_set_wol,
2561 	.get_regs_len	= cpsw_get_regs_len,
2562 	.get_regs	= cpsw_get_regs,
2563 	.begin		= cpsw_ethtool_op_begin,
2564 	.complete	= cpsw_ethtool_op_complete,
2565 	.get_channels	= cpsw_get_channels,
2566 	.set_channels	= cpsw_set_channels,
2567 	.get_link_ksettings	= cpsw_get_link_ksettings,
2568 	.set_link_ksettings	= cpsw_set_link_ksettings,
2569 	.get_eee	= cpsw_get_eee,
2570 	.set_eee	= cpsw_set_eee,
2571 	.nway_reset	= cpsw_nway_reset,
2572 	.get_ringparam = cpsw_get_ringparam,
2573 	.set_ringparam = cpsw_set_ringparam,
2574 };
2575 
2576 static void cpsw_slave_init(struct cpsw_slave *slave, struct cpsw_common *cpsw,
2577 			    u32 slave_reg_ofs, u32 sliver_reg_ofs)
2578 {
2579 	void __iomem		*regs = cpsw->regs;
2580 	int			slave_num = slave->slave_num;
2581 	struct cpsw_slave_data	*data = cpsw->data.slave_data + slave_num;
2582 
2583 	slave->data	= data;
2584 	slave->regs	= regs + slave_reg_ofs;
2585 	slave->sliver	= regs + sliver_reg_ofs;
2586 	slave->port_vlan = data->dual_emac_res_vlan;
2587 }
2588 
2589 static int cpsw_probe_dt(struct cpsw_platform_data *data,
2590 			 struct platform_device *pdev)
2591 {
2592 	struct device_node *node = pdev->dev.of_node;
2593 	struct device_node *slave_node;
2594 	int i = 0, ret;
2595 	u32 prop;
2596 
2597 	if (!node)
2598 		return -EINVAL;
2599 
2600 	if (of_property_read_u32(node, "slaves", &prop)) {
2601 		dev_err(&pdev->dev, "Missing slaves property in the DT.\n");
2602 		return -EINVAL;
2603 	}
2604 	data->slaves = prop;
2605 
2606 	if (of_property_read_u32(node, "active_slave", &prop)) {
2607 		dev_err(&pdev->dev, "Missing active_slave property in the DT.\n");
2608 		return -EINVAL;
2609 	}
2610 	data->active_slave = prop;
2611 
2612 	data->slave_data = devm_kzalloc(&pdev->dev, data->slaves
2613 					* sizeof(struct cpsw_slave_data),
2614 					GFP_KERNEL);
2615 	if (!data->slave_data)
2616 		return -ENOMEM;
2617 
2618 	if (of_property_read_u32(node, "cpdma_channels", &prop)) {
2619 		dev_err(&pdev->dev, "Missing cpdma_channels property in the DT.\n");
2620 		return -EINVAL;
2621 	}
2622 	data->channels = prop;
2623 
2624 	if (of_property_read_u32(node, "ale_entries", &prop)) {
2625 		dev_err(&pdev->dev, "Missing ale_entries property in the DT.\n");
2626 		return -EINVAL;
2627 	}
2628 	data->ale_entries = prop;
2629 
2630 	if (of_property_read_u32(node, "bd_ram_size", &prop)) {
2631 		dev_err(&pdev->dev, "Missing bd_ram_size property in the DT.\n");
2632 		return -EINVAL;
2633 	}
2634 	data->bd_ram_size = prop;
2635 
2636 	if (of_property_read_u32(node, "mac_control", &prop)) {
2637 		dev_err(&pdev->dev, "Missing mac_control property in the DT.\n");
2638 		return -EINVAL;
2639 	}
2640 	data->mac_control = prop;
2641 
2642 	if (of_property_read_bool(node, "dual_emac"))
2643 		data->dual_emac = 1;
2644 
2645 	/*
2646 	 * Populate all the child nodes here...
2647 	 */
2648 	ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
2649 	/* We do not want to force this, as in some cases may not have child */
2650 	if (ret)
2651 		dev_warn(&pdev->dev, "Doesn't have any child node\n");
2652 
2653 	for_each_available_child_of_node(node, slave_node) {
2654 		struct cpsw_slave_data *slave_data = data->slave_data + i;
2655 		const void *mac_addr = NULL;
2656 		int lenp;
2657 		const __be32 *parp;
2658 
2659 		/* This is no slave child node, continue */
2660 		if (strcmp(slave_node->name, "slave"))
2661 			continue;
2662 
2663 		slave_data->phy_node = of_parse_phandle(slave_node,
2664 							"phy-handle", 0);
2665 		parp = of_get_property(slave_node, "phy_id", &lenp);
2666 		if (slave_data->phy_node) {
2667 			dev_dbg(&pdev->dev,
2668 				"slave[%d] using phy-handle=\"%s\"\n",
2669 				i, slave_data->phy_node->full_name);
2670 		} else if (of_phy_is_fixed_link(slave_node)) {
2671 			/* In the case of a fixed PHY, the DT node associated
2672 			 * to the PHY is the Ethernet MAC DT node.
2673 			 */
2674 			ret = of_phy_register_fixed_link(slave_node);
2675 			if (ret) {
2676 				if (ret != -EPROBE_DEFER)
2677 					dev_err(&pdev->dev, "failed to register fixed-link phy: %d\n", ret);
2678 				return ret;
2679 			}
2680 			slave_data->phy_node = of_node_get(slave_node);
2681 		} else if (parp) {
2682 			u32 phyid;
2683 			struct device_node *mdio_node;
2684 			struct platform_device *mdio;
2685 
2686 			if (lenp != (sizeof(__be32) * 2)) {
2687 				dev_err(&pdev->dev, "Invalid slave[%d] phy_id property\n", i);
2688 				goto no_phy_slave;
2689 			}
2690 			mdio_node = of_find_node_by_phandle(be32_to_cpup(parp));
2691 			phyid = be32_to_cpup(parp+1);
2692 			mdio = of_find_device_by_node(mdio_node);
2693 			of_node_put(mdio_node);
2694 			if (!mdio) {
2695 				dev_err(&pdev->dev, "Missing mdio platform device\n");
2696 				return -EINVAL;
2697 			}
2698 			snprintf(slave_data->phy_id, sizeof(slave_data->phy_id),
2699 				 PHY_ID_FMT, mdio->name, phyid);
2700 			put_device(&mdio->dev);
2701 		} else {
2702 			dev_err(&pdev->dev,
2703 				"No slave[%d] phy_id, phy-handle, or fixed-link property\n",
2704 				i);
2705 			goto no_phy_slave;
2706 		}
2707 		slave_data->phy_if = of_get_phy_mode(slave_node);
2708 		if (slave_data->phy_if < 0) {
2709 			dev_err(&pdev->dev, "Missing or malformed slave[%d] phy-mode property\n",
2710 				i);
2711 			return slave_data->phy_if;
2712 		}
2713 
2714 no_phy_slave:
2715 		mac_addr = of_get_mac_address(slave_node);
2716 		if (mac_addr) {
2717 			memcpy(slave_data->mac_addr, mac_addr, ETH_ALEN);
2718 		} else {
2719 			ret = ti_cm_get_macid(&pdev->dev, i,
2720 					      slave_data->mac_addr);
2721 			if (ret)
2722 				return ret;
2723 		}
2724 		if (data->dual_emac) {
2725 			if (of_property_read_u32(slave_node, "dual_emac_res_vlan",
2726 						 &prop)) {
2727 				dev_err(&pdev->dev, "Missing dual_emac_res_vlan in DT.\n");
2728 				slave_data->dual_emac_res_vlan = i+1;
2729 				dev_err(&pdev->dev, "Using %d as Reserved VLAN for %d slave\n",
2730 					slave_data->dual_emac_res_vlan, i);
2731 			} else {
2732 				slave_data->dual_emac_res_vlan = prop;
2733 			}
2734 		}
2735 
2736 		i++;
2737 		if (i == data->slaves)
2738 			break;
2739 	}
2740 
2741 	return 0;
2742 }
2743 
2744 static void cpsw_remove_dt(struct platform_device *pdev)
2745 {
2746 	struct net_device *ndev = platform_get_drvdata(pdev);
2747 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2748 	struct cpsw_platform_data *data = &cpsw->data;
2749 	struct device_node *node = pdev->dev.of_node;
2750 	struct device_node *slave_node;
2751 	int i = 0;
2752 
2753 	for_each_available_child_of_node(node, slave_node) {
2754 		struct cpsw_slave_data *slave_data = &data->slave_data[i];
2755 
2756 		if (strcmp(slave_node->name, "slave"))
2757 			continue;
2758 
2759 		if (of_phy_is_fixed_link(slave_node))
2760 			of_phy_deregister_fixed_link(slave_node);
2761 
2762 		of_node_put(slave_data->phy_node);
2763 
2764 		i++;
2765 		if (i == data->slaves)
2766 			break;
2767 	}
2768 
2769 	of_platform_depopulate(&pdev->dev);
2770 }
2771 
2772 static int cpsw_probe_dual_emac(struct cpsw_priv *priv)
2773 {
2774 	struct cpsw_common		*cpsw = priv->cpsw;
2775 	struct cpsw_platform_data	*data = &cpsw->data;
2776 	struct net_device		*ndev;
2777 	struct cpsw_priv		*priv_sl2;
2778 	int ret = 0;
2779 
2780 	ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
2781 	if (!ndev) {
2782 		dev_err(cpsw->dev, "cpsw: error allocating net_device\n");
2783 		return -ENOMEM;
2784 	}
2785 
2786 	priv_sl2 = netdev_priv(ndev);
2787 	priv_sl2->cpsw = cpsw;
2788 	priv_sl2->ndev = ndev;
2789 	priv_sl2->dev  = &ndev->dev;
2790 	priv_sl2->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
2791 
2792 	if (is_valid_ether_addr(data->slave_data[1].mac_addr)) {
2793 		memcpy(priv_sl2->mac_addr, data->slave_data[1].mac_addr,
2794 			ETH_ALEN);
2795 		dev_info(cpsw->dev, "cpsw: Detected MACID = %pM\n",
2796 			 priv_sl2->mac_addr);
2797 	} else {
2798 		random_ether_addr(priv_sl2->mac_addr);
2799 		dev_info(cpsw->dev, "cpsw: Random MACID = %pM\n",
2800 			 priv_sl2->mac_addr);
2801 	}
2802 	memcpy(ndev->dev_addr, priv_sl2->mac_addr, ETH_ALEN);
2803 
2804 	priv_sl2->emac_port = 1;
2805 	cpsw->slaves[1].ndev = ndev;
2806 	ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2807 
2808 	ndev->netdev_ops = &cpsw_netdev_ops;
2809 	ndev->ethtool_ops = &cpsw_ethtool_ops;
2810 
2811 	/* register the network device */
2812 	SET_NETDEV_DEV(ndev, cpsw->dev);
2813 	ret = register_netdev(ndev);
2814 	if (ret) {
2815 		dev_err(cpsw->dev, "cpsw: error registering net device\n");
2816 		free_netdev(ndev);
2817 		ret = -ENODEV;
2818 	}
2819 
2820 	return ret;
2821 }
2822 
2823 #define CPSW_QUIRK_IRQ		BIT(0)
2824 
2825 static struct platform_device_id cpsw_devtype[] = {
2826 	{
2827 		/* keep it for existing comaptibles */
2828 		.name = "cpsw",
2829 		.driver_data = CPSW_QUIRK_IRQ,
2830 	}, {
2831 		.name = "am335x-cpsw",
2832 		.driver_data = CPSW_QUIRK_IRQ,
2833 	}, {
2834 		.name = "am4372-cpsw",
2835 		.driver_data = 0,
2836 	}, {
2837 		.name = "dra7-cpsw",
2838 		.driver_data = 0,
2839 	}, {
2840 		/* sentinel */
2841 	}
2842 };
2843 MODULE_DEVICE_TABLE(platform, cpsw_devtype);
2844 
2845 enum ti_cpsw_type {
2846 	CPSW = 0,
2847 	AM335X_CPSW,
2848 	AM4372_CPSW,
2849 	DRA7_CPSW,
2850 };
2851 
2852 static const struct of_device_id cpsw_of_mtable[] = {
2853 	{ .compatible = "ti,cpsw", .data = &cpsw_devtype[CPSW], },
2854 	{ .compatible = "ti,am335x-cpsw", .data = &cpsw_devtype[AM335X_CPSW], },
2855 	{ .compatible = "ti,am4372-cpsw", .data = &cpsw_devtype[AM4372_CPSW], },
2856 	{ .compatible = "ti,dra7-cpsw", .data = &cpsw_devtype[DRA7_CPSW], },
2857 	{ /* sentinel */ },
2858 };
2859 MODULE_DEVICE_TABLE(of, cpsw_of_mtable);
2860 
2861 static int cpsw_probe(struct platform_device *pdev)
2862 {
2863 	struct clk			*clk;
2864 	struct cpsw_platform_data	*data;
2865 	struct net_device		*ndev;
2866 	struct cpsw_priv		*priv;
2867 	struct cpdma_params		dma_params;
2868 	struct cpsw_ale_params		ale_params;
2869 	void __iomem			*ss_regs;
2870 	void __iomem			*cpts_regs;
2871 	struct resource			*res, *ss_res;
2872 	const struct of_device_id	*of_id;
2873 	struct gpio_descs		*mode;
2874 	u32 slave_offset, sliver_offset, slave_size;
2875 	struct cpsw_common		*cpsw;
2876 	int ret = 0, i;
2877 	int irq;
2878 
2879 	cpsw = devm_kzalloc(&pdev->dev, sizeof(struct cpsw_common), GFP_KERNEL);
2880 	if (!cpsw)
2881 		return -ENOMEM;
2882 
2883 	cpsw->dev = &pdev->dev;
2884 
2885 	ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
2886 	if (!ndev) {
2887 		dev_err(&pdev->dev, "error allocating net_device\n");
2888 		return -ENOMEM;
2889 	}
2890 
2891 	platform_set_drvdata(pdev, ndev);
2892 	priv = netdev_priv(ndev);
2893 	priv->cpsw = cpsw;
2894 	priv->ndev = ndev;
2895 	priv->dev  = &ndev->dev;
2896 	priv->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
2897 	cpsw->rx_packet_max = max(rx_packet_max, 128);
2898 
2899 	mode = devm_gpiod_get_array_optional(&pdev->dev, "mode", GPIOD_OUT_LOW);
2900 	if (IS_ERR(mode)) {
2901 		ret = PTR_ERR(mode);
2902 		dev_err(&pdev->dev, "gpio request failed, ret %d\n", ret);
2903 		goto clean_ndev_ret;
2904 	}
2905 
2906 	/*
2907 	 * This may be required here for child devices.
2908 	 */
2909 	pm_runtime_enable(&pdev->dev);
2910 
2911 	/* Select default pin state */
2912 	pinctrl_pm_select_default_state(&pdev->dev);
2913 
2914 	/* Need to enable clocks with runtime PM api to access module
2915 	 * registers
2916 	 */
2917 	ret = pm_runtime_get_sync(&pdev->dev);
2918 	if (ret < 0) {
2919 		pm_runtime_put_noidle(&pdev->dev);
2920 		goto clean_runtime_disable_ret;
2921 	}
2922 
2923 	ret = cpsw_probe_dt(&cpsw->data, pdev);
2924 	if (ret)
2925 		goto clean_dt_ret;
2926 
2927 	data = &cpsw->data;
2928 	cpsw->rx_ch_num = 1;
2929 	cpsw->tx_ch_num = 1;
2930 
2931 	if (is_valid_ether_addr(data->slave_data[0].mac_addr)) {
2932 		memcpy(priv->mac_addr, data->slave_data[0].mac_addr, ETH_ALEN);
2933 		dev_info(&pdev->dev, "Detected MACID = %pM\n", priv->mac_addr);
2934 	} else {
2935 		eth_random_addr(priv->mac_addr);
2936 		dev_info(&pdev->dev, "Random MACID = %pM\n", priv->mac_addr);
2937 	}
2938 
2939 	memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
2940 
2941 	cpsw->slaves = devm_kzalloc(&pdev->dev,
2942 				    sizeof(struct cpsw_slave) * data->slaves,
2943 				    GFP_KERNEL);
2944 	if (!cpsw->slaves) {
2945 		ret = -ENOMEM;
2946 		goto clean_dt_ret;
2947 	}
2948 	for (i = 0; i < data->slaves; i++)
2949 		cpsw->slaves[i].slave_num = i;
2950 
2951 	cpsw->slaves[0].ndev = ndev;
2952 	priv->emac_port = 0;
2953 
2954 	clk = devm_clk_get(&pdev->dev, "fck");
2955 	if (IS_ERR(clk)) {
2956 		dev_err(priv->dev, "fck is not found\n");
2957 		ret = -ENODEV;
2958 		goto clean_dt_ret;
2959 	}
2960 	cpsw->bus_freq_mhz = clk_get_rate(clk) / 1000000;
2961 
2962 	ss_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2963 	ss_regs = devm_ioremap_resource(&pdev->dev, ss_res);
2964 	if (IS_ERR(ss_regs)) {
2965 		ret = PTR_ERR(ss_regs);
2966 		goto clean_dt_ret;
2967 	}
2968 	cpsw->regs = ss_regs;
2969 
2970 	cpsw->version = readl(&cpsw->regs->id_ver);
2971 
2972 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2973 	cpsw->wr_regs = devm_ioremap_resource(&pdev->dev, res);
2974 	if (IS_ERR(cpsw->wr_regs)) {
2975 		ret = PTR_ERR(cpsw->wr_regs);
2976 		goto clean_dt_ret;
2977 	}
2978 
2979 	memset(&dma_params, 0, sizeof(dma_params));
2980 	memset(&ale_params, 0, sizeof(ale_params));
2981 
2982 	switch (cpsw->version) {
2983 	case CPSW_VERSION_1:
2984 		cpsw->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;
2985 		cpts_regs		= ss_regs + CPSW1_CPTS_OFFSET;
2986 		cpsw->hw_stats	     = ss_regs + CPSW1_HW_STATS;
2987 		dma_params.dmaregs   = ss_regs + CPSW1_CPDMA_OFFSET;
2988 		dma_params.txhdp     = ss_regs + CPSW1_STATERAM_OFFSET;
2989 		ale_params.ale_regs  = ss_regs + CPSW1_ALE_OFFSET;
2990 		slave_offset         = CPSW1_SLAVE_OFFSET;
2991 		slave_size           = CPSW1_SLAVE_SIZE;
2992 		sliver_offset        = CPSW1_SLIVER_OFFSET;
2993 		dma_params.desc_mem_phys = 0;
2994 		break;
2995 	case CPSW_VERSION_2:
2996 	case CPSW_VERSION_3:
2997 	case CPSW_VERSION_4:
2998 		cpsw->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;
2999 		cpts_regs		= ss_regs + CPSW2_CPTS_OFFSET;
3000 		cpsw->hw_stats	     = ss_regs + CPSW2_HW_STATS;
3001 		dma_params.dmaregs   = ss_regs + CPSW2_CPDMA_OFFSET;
3002 		dma_params.txhdp     = ss_regs + CPSW2_STATERAM_OFFSET;
3003 		ale_params.ale_regs  = ss_regs + CPSW2_ALE_OFFSET;
3004 		slave_offset         = CPSW2_SLAVE_OFFSET;
3005 		slave_size           = CPSW2_SLAVE_SIZE;
3006 		sliver_offset        = CPSW2_SLIVER_OFFSET;
3007 		dma_params.desc_mem_phys =
3008 			(u32 __force) ss_res->start + CPSW2_BD_OFFSET;
3009 		break;
3010 	default:
3011 		dev_err(priv->dev, "unknown version 0x%08x\n", cpsw->version);
3012 		ret = -ENODEV;
3013 		goto clean_dt_ret;
3014 	}
3015 	for (i = 0; i < cpsw->data.slaves; i++) {
3016 		struct cpsw_slave *slave = &cpsw->slaves[i];
3017 
3018 		cpsw_slave_init(slave, cpsw, slave_offset, sliver_offset);
3019 		slave_offset  += slave_size;
3020 		sliver_offset += SLIVER_SIZE;
3021 	}
3022 
3023 	dma_params.dev		= &pdev->dev;
3024 	dma_params.rxthresh	= dma_params.dmaregs + CPDMA_RXTHRESH;
3025 	dma_params.rxfree	= dma_params.dmaregs + CPDMA_RXFREE;
3026 	dma_params.rxhdp	= dma_params.txhdp + CPDMA_RXHDP;
3027 	dma_params.txcp		= dma_params.txhdp + CPDMA_TXCP;
3028 	dma_params.rxcp		= dma_params.txhdp + CPDMA_RXCP;
3029 
3030 	dma_params.num_chan		= data->channels;
3031 	dma_params.has_soft_reset	= true;
3032 	dma_params.min_packet_size	= CPSW_MIN_PACKET_SIZE;
3033 	dma_params.desc_mem_size	= data->bd_ram_size;
3034 	dma_params.desc_align		= 16;
3035 	dma_params.has_ext_regs		= true;
3036 	dma_params.desc_hw_addr         = dma_params.desc_mem_phys;
3037 	dma_params.bus_freq_mhz		= cpsw->bus_freq_mhz;
3038 	dma_params.descs_pool_size	= descs_pool_size;
3039 
3040 	cpsw->dma = cpdma_ctlr_create(&dma_params);
3041 	if (!cpsw->dma) {
3042 		dev_err(priv->dev, "error initializing dma\n");
3043 		ret = -ENOMEM;
3044 		goto clean_dt_ret;
3045 	}
3046 
3047 	cpsw->txv[0].ch = cpdma_chan_create(cpsw->dma, 0, cpsw_tx_handler, 0);
3048 	cpsw->rxv[0].ch = cpdma_chan_create(cpsw->dma, 0, cpsw_rx_handler, 1);
3049 	if (WARN_ON(!cpsw->rxv[0].ch || !cpsw->txv[0].ch)) {
3050 		dev_err(priv->dev, "error initializing dma channels\n");
3051 		ret = -ENOMEM;
3052 		goto clean_dma_ret;
3053 	}
3054 
3055 	ale_params.dev			= &pdev->dev;
3056 	ale_params.ale_ageout		= ale_ageout;
3057 	ale_params.ale_entries		= data->ale_entries;
3058 	ale_params.ale_ports		= data->slaves;
3059 
3060 	cpsw->ale = cpsw_ale_create(&ale_params);
3061 	if (!cpsw->ale) {
3062 		dev_err(priv->dev, "error initializing ale engine\n");
3063 		ret = -ENODEV;
3064 		goto clean_dma_ret;
3065 	}
3066 
3067 	cpsw->cpts = cpts_create(cpsw->dev, cpts_regs, cpsw->dev->of_node);
3068 	if (IS_ERR(cpsw->cpts)) {
3069 		ret = PTR_ERR(cpsw->cpts);
3070 		goto clean_ale_ret;
3071 	}
3072 
3073 	ndev->irq = platform_get_irq(pdev, 1);
3074 	if (ndev->irq < 0) {
3075 		dev_err(priv->dev, "error getting irq resource\n");
3076 		ret = ndev->irq;
3077 		goto clean_ale_ret;
3078 	}
3079 
3080 	of_id = of_match_device(cpsw_of_mtable, &pdev->dev);
3081 	if (of_id) {
3082 		pdev->id_entry = of_id->data;
3083 		if (pdev->id_entry->driver_data)
3084 			cpsw->quirk_irq = true;
3085 	}
3086 
3087 	/* Grab RX and TX IRQs. Note that we also have RX_THRESHOLD and
3088 	 * MISC IRQs which are always kept disabled with this driver so
3089 	 * we will not request them.
3090 	 *
3091 	 * If anyone wants to implement support for those, make sure to
3092 	 * first request and append them to irqs_table array.
3093 	 */
3094 
3095 	/* RX IRQ */
3096 	irq = platform_get_irq(pdev, 1);
3097 	if (irq < 0) {
3098 		ret = irq;
3099 		goto clean_ale_ret;
3100 	}
3101 
3102 	cpsw->irqs_table[0] = irq;
3103 	ret = devm_request_irq(&pdev->dev, irq, cpsw_rx_interrupt,
3104 			       0, dev_name(&pdev->dev), cpsw);
3105 	if (ret < 0) {
3106 		dev_err(priv->dev, "error attaching irq (%d)\n", ret);
3107 		goto clean_ale_ret;
3108 	}
3109 
3110 	/* TX IRQ */
3111 	irq = platform_get_irq(pdev, 2);
3112 	if (irq < 0) {
3113 		ret = irq;
3114 		goto clean_ale_ret;
3115 	}
3116 
3117 	cpsw->irqs_table[1] = irq;
3118 	ret = devm_request_irq(&pdev->dev, irq, cpsw_tx_interrupt,
3119 			       0, dev_name(&pdev->dev), cpsw);
3120 	if (ret < 0) {
3121 		dev_err(priv->dev, "error attaching irq (%d)\n", ret);
3122 		goto clean_ale_ret;
3123 	}
3124 
3125 	ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
3126 
3127 	ndev->netdev_ops = &cpsw_netdev_ops;
3128 	ndev->ethtool_ops = &cpsw_ethtool_ops;
3129 	netif_napi_add(ndev, &cpsw->napi_rx, cpsw_rx_poll, CPSW_POLL_WEIGHT);
3130 	netif_tx_napi_add(ndev, &cpsw->napi_tx, cpsw_tx_poll, CPSW_POLL_WEIGHT);
3131 	cpsw_split_res(ndev);
3132 
3133 	/* register the network device */
3134 	SET_NETDEV_DEV(ndev, &pdev->dev);
3135 	ret = register_netdev(ndev);
3136 	if (ret) {
3137 		dev_err(priv->dev, "error registering net device\n");
3138 		ret = -ENODEV;
3139 		goto clean_ale_ret;
3140 	}
3141 
3142 	cpsw_notice(priv, probe,
3143 		    "initialized device (regs %pa, irq %d, pool size %d)\n",
3144 		    &ss_res->start, ndev->irq, dma_params.descs_pool_size);
3145 	if (cpsw->data.dual_emac) {
3146 		ret = cpsw_probe_dual_emac(priv);
3147 		if (ret) {
3148 			cpsw_err(priv, probe, "error probe slave 2 emac interface\n");
3149 			goto clean_unregister_netdev_ret;
3150 		}
3151 	}
3152 
3153 	pm_runtime_put(&pdev->dev);
3154 
3155 	return 0;
3156 
3157 clean_unregister_netdev_ret:
3158 	unregister_netdev(ndev);
3159 clean_ale_ret:
3160 	cpsw_ale_destroy(cpsw->ale);
3161 clean_dma_ret:
3162 	cpdma_ctlr_destroy(cpsw->dma);
3163 clean_dt_ret:
3164 	cpsw_remove_dt(pdev);
3165 	pm_runtime_put_sync(&pdev->dev);
3166 clean_runtime_disable_ret:
3167 	pm_runtime_disable(&pdev->dev);
3168 clean_ndev_ret:
3169 	free_netdev(priv->ndev);
3170 	return ret;
3171 }
3172 
3173 static int cpsw_remove(struct platform_device *pdev)
3174 {
3175 	struct net_device *ndev = platform_get_drvdata(pdev);
3176 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
3177 	int ret;
3178 
3179 	ret = pm_runtime_get_sync(&pdev->dev);
3180 	if (ret < 0) {
3181 		pm_runtime_put_noidle(&pdev->dev);
3182 		return ret;
3183 	}
3184 
3185 	if (cpsw->data.dual_emac)
3186 		unregister_netdev(cpsw->slaves[1].ndev);
3187 	unregister_netdev(ndev);
3188 
3189 	cpts_release(cpsw->cpts);
3190 	cpsw_ale_destroy(cpsw->ale);
3191 	cpdma_ctlr_destroy(cpsw->dma);
3192 	cpsw_remove_dt(pdev);
3193 	pm_runtime_put_sync(&pdev->dev);
3194 	pm_runtime_disable(&pdev->dev);
3195 	if (cpsw->data.dual_emac)
3196 		free_netdev(cpsw->slaves[1].ndev);
3197 	free_netdev(ndev);
3198 	return 0;
3199 }
3200 
3201 #ifdef CONFIG_PM_SLEEP
3202 static int cpsw_suspend(struct device *dev)
3203 {
3204 	struct platform_device	*pdev = to_platform_device(dev);
3205 	struct net_device	*ndev = platform_get_drvdata(pdev);
3206 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
3207 
3208 	if (cpsw->data.dual_emac) {
3209 		int i;
3210 
3211 		for (i = 0; i < cpsw->data.slaves; i++) {
3212 			if (netif_running(cpsw->slaves[i].ndev))
3213 				cpsw_ndo_stop(cpsw->slaves[i].ndev);
3214 		}
3215 	} else {
3216 		if (netif_running(ndev))
3217 			cpsw_ndo_stop(ndev);
3218 	}
3219 
3220 	/* Select sleep pin state */
3221 	pinctrl_pm_select_sleep_state(dev);
3222 
3223 	return 0;
3224 }
3225 
3226 static int cpsw_resume(struct device *dev)
3227 {
3228 	struct platform_device	*pdev = to_platform_device(dev);
3229 	struct net_device	*ndev = platform_get_drvdata(pdev);
3230 	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
3231 
3232 	/* Select default pin state */
3233 	pinctrl_pm_select_default_state(dev);
3234 
3235 	/* shut up ASSERT_RTNL() warning in netif_set_real_num_tx/rx_queues */
3236 	rtnl_lock();
3237 	if (cpsw->data.dual_emac) {
3238 		int i;
3239 
3240 		for (i = 0; i < cpsw->data.slaves; i++) {
3241 			if (netif_running(cpsw->slaves[i].ndev))
3242 				cpsw_ndo_open(cpsw->slaves[i].ndev);
3243 		}
3244 	} else {
3245 		if (netif_running(ndev))
3246 			cpsw_ndo_open(ndev);
3247 	}
3248 	rtnl_unlock();
3249 
3250 	return 0;
3251 }
3252 #endif
3253 
3254 static SIMPLE_DEV_PM_OPS(cpsw_pm_ops, cpsw_suspend, cpsw_resume);
3255 
3256 static struct platform_driver cpsw_driver = {
3257 	.driver = {
3258 		.name	 = "cpsw",
3259 		.pm	 = &cpsw_pm_ops,
3260 		.of_match_table = cpsw_of_mtable,
3261 	},
3262 	.probe = cpsw_probe,
3263 	.remove = cpsw_remove,
3264 };
3265 
3266 module_platform_driver(cpsw_driver);
3267 
3268 MODULE_LICENSE("GPL");
3269 MODULE_AUTHOR("Cyril Chemparathy <cyril@ti.com>");
3270 MODULE_AUTHOR("Mugunthan V N <mugunthanvnm@ti.com>");
3271 MODULE_DESCRIPTION("TI CPSW Ethernet driver");
3272