1 // SPDX-License-Identifier: GPL-2.0
2 /* Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver
3  *
4  * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
5  *
6  */
7 
8 #include <linux/clk.h>
9 #include <linux/etherdevice.h>
10 #include <linux/if_vlan.h>
11 #include <linux/interrupt.h>
12 #include <linux/kernel.h>
13 #include <linux/kmemleak.h>
14 #include <linux/module.h>
15 #include <linux/netdevice.h>
16 #include <linux/net_tstamp.h>
17 #include <linux/of.h>
18 #include <linux/of_mdio.h>
19 #include <linux/of_net.h>
20 #include <linux/of_device.h>
21 #include <linux/phy.h>
22 #include <linux/phy/phy.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/regmap.h>
26 #include <linux/mfd/syscon.h>
27 #include <linux/sys_soc.h>
28 #include <linux/dma/ti-cppi5.h>
29 #include <linux/dma/k3-udma-glue.h>
30 
31 #include "cpsw_ale.h"
32 #include "cpsw_sl.h"
33 #include "am65-cpsw-nuss.h"
34 #include "am65-cpsw-switchdev.h"
35 #include "k3-cppi-desc-pool.h"
36 #include "am65-cpts.h"
37 
38 #define AM65_CPSW_SS_BASE	0x0
39 #define AM65_CPSW_SGMII_BASE	0x100
40 #define AM65_CPSW_XGMII_BASE	0x2100
41 #define AM65_CPSW_CPSW_NU_BASE	0x20000
42 #define AM65_CPSW_NU_PORTS_BASE	0x1000
43 #define AM65_CPSW_NU_FRAM_BASE	0x12000
44 #define AM65_CPSW_NU_STATS_BASE	0x1a000
45 #define AM65_CPSW_NU_ALE_BASE	0x1e000
46 #define AM65_CPSW_NU_CPTS_BASE	0x1d000
47 
48 #define AM65_CPSW_NU_PORTS_OFFSET	0x1000
49 #define AM65_CPSW_NU_STATS_PORT_OFFSET	0x200
50 #define AM65_CPSW_NU_FRAM_PORT_OFFSET	0x200
51 
52 #define AM65_CPSW_MAX_PORTS	8
53 
54 #define AM65_CPSW_MIN_PACKET_SIZE	VLAN_ETH_ZLEN
55 #define AM65_CPSW_MAX_PACKET_SIZE	(VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)
56 
57 #define AM65_CPSW_REG_CTL		0x004
58 #define AM65_CPSW_REG_STAT_PORT_EN	0x014
59 #define AM65_CPSW_REG_PTYPE		0x018
60 
61 #define AM65_CPSW_P0_REG_CTL			0x004
62 #define AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET	0x008
63 
64 #define AM65_CPSW_PORT_REG_PRI_CTL		0x01c
65 #define AM65_CPSW_PORT_REG_RX_PRI_MAP		0x020
66 #define AM65_CPSW_PORT_REG_RX_MAXLEN		0x024
67 
68 #define AM65_CPSW_PORTN_REG_SA_L		0x308
69 #define AM65_CPSW_PORTN_REG_SA_H		0x30c
70 #define AM65_CPSW_PORTN_REG_TS_CTL              0x310
71 #define AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG	0x314
72 #define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG	0x318
73 #define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2       0x31C
74 
75 #define AM65_CPSW_CTL_VLAN_AWARE		BIT(1)
76 #define AM65_CPSW_CTL_P0_ENABLE			BIT(2)
77 #define AM65_CPSW_CTL_P0_TX_CRC_REMOVE		BIT(13)
78 #define AM65_CPSW_CTL_P0_RX_PAD			BIT(14)
79 
80 /* AM65_CPSW_P0_REG_CTL */
81 #define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN	BIT(0)
82 
83 /* AM65_CPSW_PORT_REG_PRI_CTL */
84 #define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN	BIT(8)
85 
86 /* AM65_CPSW_PN_TS_CTL register fields */
87 #define AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN		BIT(4)
88 #define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN	BIT(5)
89 #define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT2_EN	BIT(6)
90 #define AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN		BIT(7)
91 #define AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN		BIT(10)
92 #define AM65_CPSW_PN_TS_CTL_TX_HOST_TS_EN	BIT(11)
93 #define AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT	16
94 
95 /* AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG register fields */
96 #define AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT	16
97 
98 /* AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 */
99 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107	BIT(16)
100 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129	BIT(17)
101 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130	BIT(18)
102 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131	BIT(19)
103 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132	BIT(20)
104 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319	BIT(21)
105 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320	BIT(22)
106 #define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO BIT(23)
107 
108 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
109 #define AM65_CPSW_TS_EVENT_MSG_TYPE_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
110 
111 #define AM65_CPSW_TS_SEQ_ID_OFFSET (0x1e)
112 
113 #define AM65_CPSW_TS_TX_ANX_ALL_EN		\
114 	(AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN |	\
115 	 AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN |	\
116 	 AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN)
117 
118 #define AM65_CPSW_ALE_AGEOUT_DEFAULT	30
119 /* Number of TX/RX descriptors */
120 #define AM65_CPSW_MAX_TX_DESC	500
121 #define AM65_CPSW_MAX_RX_DESC	500
122 
123 #define AM65_CPSW_NAV_PS_DATA_SIZE 16
124 #define AM65_CPSW_NAV_SW_DATA_SIZE 16
125 
126 #define AM65_CPSW_DEBUG	(NETIF_MSG_HW | NETIF_MSG_DRV | NETIF_MSG_LINK | \
127 			 NETIF_MSG_IFUP	| NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \
128 			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
129 
130 static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
131 				      const u8 *dev_addr)
132 {
133 	u32 mac_hi = (dev_addr[0] << 0) | (dev_addr[1] << 8) |
134 		     (dev_addr[2] << 16) | (dev_addr[3] << 24);
135 	u32 mac_lo = (dev_addr[4] << 0) | (dev_addr[5] << 8);
136 
137 	writel(mac_hi, slave->port_base + AM65_CPSW_PORTN_REG_SA_H);
138 	writel(mac_lo, slave->port_base + AM65_CPSW_PORTN_REG_SA_L);
139 }
140 
141 static void am65_cpsw_sl_ctl_reset(struct am65_cpsw_port *port)
142 {
143 	cpsw_sl_reset(port->slave.mac_sl, 100);
144 	/* Max length register has to be restored after MAC SL reset */
145 	writel(AM65_CPSW_MAX_PACKET_SIZE,
146 	       port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
147 }
148 
149 static void am65_cpsw_nuss_get_ver(struct am65_cpsw_common *common)
150 {
151 	common->nuss_ver = readl(common->ss_base);
152 	common->cpsw_ver = readl(common->cpsw_base);
153 	dev_info(common->dev,
154 		 "initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u quirks:%08x\n",
155 		common->nuss_ver,
156 		common->cpsw_ver,
157 		common->port_num + 1,
158 		common->pdata.quirks);
159 }
160 
161 void am65_cpsw_nuss_adjust_link(struct net_device *ndev)
162 {
163 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
164 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
165 	struct phy_device *phy = port->slave.phy;
166 	u32 mac_control = 0;
167 
168 	if (!phy)
169 		return;
170 
171 	if (phy->link) {
172 		mac_control = CPSW_SL_CTL_GMII_EN;
173 
174 		if (phy->speed == 1000)
175 			mac_control |= CPSW_SL_CTL_GIG;
176 		if (phy->speed == 10 && phy_interface_is_rgmii(phy))
177 			/* Can be used with in band mode only */
178 			mac_control |= CPSW_SL_CTL_EXT_EN;
179 		if (phy->speed == 100 && phy->interface == PHY_INTERFACE_MODE_RMII)
180 			mac_control |= CPSW_SL_CTL_IFCTL_A;
181 		if (phy->duplex)
182 			mac_control |= CPSW_SL_CTL_FULLDUPLEX;
183 
184 		/* RGMII speed is 100M if !CPSW_SL_CTL_GIG*/
185 
186 		/* rx_pause/tx_pause */
187 		if (port->slave.rx_pause)
188 			mac_control |= CPSW_SL_CTL_RX_FLOW_EN;
189 
190 		if (port->slave.tx_pause)
191 			mac_control |= CPSW_SL_CTL_TX_FLOW_EN;
192 
193 		cpsw_sl_ctl_set(port->slave.mac_sl, mac_control);
194 
195 		/* enable forwarding */
196 		cpsw_ale_control_set(common->ale, port->port_id,
197 				     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
198 
199 		am65_cpsw_qos_link_up(ndev, phy->speed);
200 		netif_tx_wake_all_queues(ndev);
201 	} else {
202 		int tmo;
203 
204 		/* disable forwarding */
205 		cpsw_ale_control_set(common->ale, port->port_id,
206 				     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
207 
208 		cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
209 
210 		tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);
211 		dev_dbg(common->dev, "donw msc_sl %08x tmo %d\n",
212 			cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS),
213 			tmo);
214 
215 		cpsw_sl_ctl_reset(port->slave.mac_sl);
216 
217 		am65_cpsw_qos_link_down(ndev);
218 		netif_tx_stop_all_queues(ndev);
219 	}
220 
221 	phy_print_status(phy);
222 }
223 
224 static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
225 					    __be16 proto, u16 vid)
226 {
227 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
228 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
229 	u32 port_mask, unreg_mcast = 0;
230 	int ret;
231 
232 	if (!common->is_emac_mode)
233 		return 0;
234 
235 	if (!netif_running(ndev) || !vid)
236 		return 0;
237 
238 	ret = pm_runtime_get_sync(common->dev);
239 	if (ret < 0) {
240 		pm_runtime_put_noidle(common->dev);
241 		return ret;
242 	}
243 
244 	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
245 	if (!vid)
246 		unreg_mcast = port_mask;
247 	dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);
248 	ret = cpsw_ale_vlan_add_modify(common->ale, vid, port_mask,
249 				       unreg_mcast, port_mask, 0);
250 
251 	pm_runtime_put(common->dev);
252 	return ret;
253 }
254 
255 static int am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device *ndev,
256 					     __be16 proto, u16 vid)
257 {
258 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
259 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
260 	int ret;
261 
262 	if (!common->is_emac_mode)
263 		return 0;
264 
265 	if (!netif_running(ndev) || !vid)
266 		return 0;
267 
268 	ret = pm_runtime_get_sync(common->dev);
269 	if (ret < 0) {
270 		pm_runtime_put_noidle(common->dev);
271 		return ret;
272 	}
273 
274 	dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid);
275 	ret = cpsw_ale_del_vlan(common->ale, vid,
276 				BIT(port->port_id) | ALE_PORT_HOST);
277 
278 	pm_runtime_put(common->dev);
279 	return ret;
280 }
281 
282 static void am65_cpsw_slave_set_promisc(struct am65_cpsw_port *port,
283 					bool promisc)
284 {
285 	struct am65_cpsw_common *common = port->common;
286 
287 	if (promisc && !common->is_emac_mode) {
288 		dev_dbg(common->dev, "promisc mode requested in switch mode");
289 		return;
290 	}
291 
292 	if (promisc) {
293 		/* Enable promiscuous mode */
294 		cpsw_ale_control_set(common->ale, port->port_id,
295 				     ALE_PORT_MACONLY_CAF, 1);
296 		dev_dbg(common->dev, "promisc enabled\n");
297 	} else {
298 		/* Disable promiscuous mode */
299 		cpsw_ale_control_set(common->ale, port->port_id,
300 				     ALE_PORT_MACONLY_CAF, 0);
301 		dev_dbg(common->dev, "promisc disabled\n");
302 	}
303 }
304 
305 static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev)
306 {
307 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
308 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
309 	u32 port_mask;
310 	bool promisc;
311 
312 	promisc = !!(ndev->flags & IFF_PROMISC);
313 	am65_cpsw_slave_set_promisc(port, promisc);
314 
315 	if (promisc)
316 		return;
317 
318 	/* Restore allmulti on vlans if necessary */
319 	cpsw_ale_set_allmulti(common->ale,
320 			      ndev->flags & IFF_ALLMULTI, port->port_id);
321 
322 	port_mask = ALE_PORT_HOST;
323 	/* Clear all mcast from ALE */
324 	cpsw_ale_flush_multicast(common->ale, port_mask, -1);
325 
326 	if (!netdev_mc_empty(ndev)) {
327 		struct netdev_hw_addr *ha;
328 
329 		/* program multicast address list into ALE register */
330 		netdev_for_each_mc_addr(ha, ndev) {
331 			cpsw_ale_add_mcast(common->ale, ha->addr,
332 					   port_mask, 0, 0, 0);
333 		}
334 	}
335 }
336 
337 static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev,
338 					       unsigned int txqueue)
339 {
340 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
341 	struct am65_cpsw_tx_chn *tx_chn;
342 	struct netdev_queue *netif_txq;
343 	unsigned long trans_start;
344 
345 	netif_txq = netdev_get_tx_queue(ndev, txqueue);
346 	tx_chn = &common->tx_chns[txqueue];
347 	trans_start = netif_txq->trans_start;
348 
349 	netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n",
350 		   txqueue,
351 		   netif_tx_queue_stopped(netif_txq),
352 		   jiffies_to_msecs(jiffies - trans_start),
353 		   dql_avail(&netif_txq->dql),
354 		   k3_cppi_desc_pool_avail(tx_chn->desc_pool));
355 
356 	if (netif_tx_queue_stopped(netif_txq)) {
357 		/* try recover if stopped by us */
358 		txq_trans_update(netif_txq);
359 		netif_tx_wake_queue(netif_txq);
360 	}
361 }
362 
363 static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
364 				  struct sk_buff *skb)
365 {
366 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
367 	struct cppi5_host_desc_t *desc_rx;
368 	struct device *dev = common->dev;
369 	u32 pkt_len = skb_tailroom(skb);
370 	dma_addr_t desc_dma;
371 	dma_addr_t buf_dma;
372 	void *swdata;
373 
374 	desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);
375 	if (!desc_rx) {
376 		dev_err(dev, "Failed to allocate RXFDQ descriptor\n");
377 		return -ENOMEM;
378 	}
379 	desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
380 
381 	buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len,
382 				 DMA_FROM_DEVICE);
383 	if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {
384 		k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
385 		dev_err(dev, "Failed to map rx skb buffer\n");
386 		return -EINVAL;
387 	}
388 
389 	cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
390 			 AM65_CPSW_NAV_PS_DATA_SIZE);
391 	k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
392 	cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb));
393 	swdata = cppi5_hdesc_get_swdata(desc_rx);
394 	*((void **)swdata) = skb;
395 
396 	return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, desc_rx, desc_dma);
397 }
398 
399 void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common)
400 {
401 	struct am65_cpsw_host *host_p = am65_common_get_host(common);
402 	u32 val, pri_map;
403 
404 	/* P0 set Receive Priority Type */
405 	val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
406 
407 	if (common->pf_p0_rx_ptype_rrobin) {
408 		val |= AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
409 		/* Enet Ports fifos works in fixed priority mode only, so
410 		 * reset P0_Rx_Pri_Map so all packet will go in Enet fifo 0
411 		 */
412 		pri_map = 0x0;
413 	} else {
414 		val &= ~AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
415 		/* restore P0_Rx_Pri_Map */
416 		pri_map = 0x76543210;
417 	}
418 
419 	writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP);
420 	writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
421 }
422 
423 static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common);
424 static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
425 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
426 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
427 
428 static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common,
429 				      netdev_features_t features)
430 {
431 	struct am65_cpsw_host *host_p = am65_common_get_host(common);
432 	int port_idx, i, ret;
433 	struct sk_buff *skb;
434 	u32 val, port_mask;
435 
436 	if (common->usage_count)
437 		return 0;
438 
439 	/* Control register */
440 	writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE |
441 	       AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD,
442 	       common->cpsw_base + AM65_CPSW_REG_CTL);
443 	/* Max length register */
444 	writel(AM65_CPSW_MAX_PACKET_SIZE,
445 	       host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
446 	/* set base flow_id */
447 	writel(common->rx_flow_id_base,
448 	       host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);
449 	/* en tx crc offload */
450 	writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN, host_p->port_base + AM65_CPSW_P0_REG_CTL);
451 
452 	am65_cpsw_nuss_set_p0_ptype(common);
453 
454 	/* enable statistic */
455 	val = BIT(HOST_PORT_NUM);
456 	for (port_idx = 0; port_idx < common->port_num; port_idx++) {
457 		struct am65_cpsw_port *port = &common->ports[port_idx];
458 
459 		if (!port->disabled)
460 			val |=  BIT(port->port_id);
461 	}
462 	writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
463 
464 	/* disable priority elevation */
465 	writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE);
466 
467 	cpsw_ale_start(common->ale);
468 
469 	/* limit to one RX flow only */
470 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
471 			     ALE_DEFAULT_THREAD_ID, 0);
472 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
473 			     ALE_DEFAULT_THREAD_ENABLE, 1);
474 	/* switch to vlan unaware mode */
475 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1);
476 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
477 			     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
478 
479 	/* default vlan cfg: create mask based on enabled ports */
480 	port_mask = GENMASK(common->port_num, 0) &
481 		    ~common->disabled_ports_mask;
482 
483 	cpsw_ale_add_vlan(common->ale, 0, port_mask,
484 			  port_mask, port_mask,
485 			  port_mask & ~ALE_PORT_HOST);
486 
487 	if (common->is_emac_mode)
488 		am65_cpsw_init_host_port_emac(common);
489 	else
490 		am65_cpsw_init_host_port_switch(common);
491 
492 	for (i = 0; i < common->rx_chns.descs_num; i++) {
493 		skb = __netdev_alloc_skb_ip_align(NULL,
494 						  AM65_CPSW_MAX_PACKET_SIZE,
495 						  GFP_KERNEL);
496 		if (!skb) {
497 			dev_err(common->dev, "cannot allocate skb\n");
498 			return -ENOMEM;
499 		}
500 
501 		ret = am65_cpsw_nuss_rx_push(common, skb);
502 		if (ret < 0) {
503 			dev_err(common->dev,
504 				"cannot submit skb to channel rx, error %d\n",
505 				ret);
506 			kfree_skb(skb);
507 			return ret;
508 		}
509 		kmemleak_not_leak(skb);
510 	}
511 	k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);
512 
513 	for (i = 0; i < common->tx_ch_num; i++) {
514 		ret = k3_udma_glue_enable_tx_chn(common->tx_chns[i].tx_chn);
515 		if (ret)
516 			return ret;
517 		napi_enable(&common->tx_chns[i].napi_tx);
518 	}
519 
520 	napi_enable(&common->napi_rx);
521 
522 	dev_dbg(common->dev, "cpsw_nuss started\n");
523 	return 0;
524 }
525 
526 static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma);
527 static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma);
528 
529 static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
530 {
531 	int i;
532 
533 	if (common->usage_count != 1)
534 		return 0;
535 
536 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
537 			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
538 
539 	/* shutdown tx channels */
540 	atomic_set(&common->tdown_cnt, common->tx_ch_num);
541 	/* ensure new tdown_cnt value is visible */
542 	smp_mb__after_atomic();
543 	reinit_completion(&common->tdown_complete);
544 
545 	for (i = 0; i < common->tx_ch_num; i++)
546 		k3_udma_glue_tdown_tx_chn(common->tx_chns[i].tx_chn, false);
547 
548 	i = wait_for_completion_timeout(&common->tdown_complete,
549 					msecs_to_jiffies(1000));
550 	if (!i)
551 		dev_err(common->dev, "tx timeout\n");
552 	for (i = 0; i < common->tx_ch_num; i++)
553 		napi_disable(&common->tx_chns[i].napi_tx);
554 
555 	for (i = 0; i < common->tx_ch_num; i++) {
556 		k3_udma_glue_reset_tx_chn(common->tx_chns[i].tx_chn,
557 					  &common->tx_chns[i],
558 					  am65_cpsw_nuss_tx_cleanup);
559 		k3_udma_glue_disable_tx_chn(common->tx_chns[i].tx_chn);
560 	}
561 
562 	k3_udma_glue_tdown_rx_chn(common->rx_chns.rx_chn, true);
563 	napi_disable(&common->napi_rx);
564 
565 	for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++)
566 		k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, i,
567 					  &common->rx_chns,
568 					  am65_cpsw_nuss_rx_cleanup, !!i);
569 
570 	k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);
571 
572 	cpsw_ale_stop(common->ale);
573 
574 	writel(0, common->cpsw_base + AM65_CPSW_REG_CTL);
575 	writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
576 
577 	dev_dbg(common->dev, "cpsw_nuss stopped\n");
578 	return 0;
579 }
580 
581 static int am65_cpsw_nuss_ndo_slave_stop(struct net_device *ndev)
582 {
583 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
584 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
585 	int ret;
586 
587 	if (port->slave.phy)
588 		phy_stop(port->slave.phy);
589 
590 	netif_tx_stop_all_queues(ndev);
591 
592 	if (port->slave.phy) {
593 		phy_disconnect(port->slave.phy);
594 		port->slave.phy = NULL;
595 	}
596 
597 	ret = am65_cpsw_nuss_common_stop(common);
598 	if (ret)
599 		return ret;
600 
601 	common->usage_count--;
602 	pm_runtime_put(common->dev);
603 	return 0;
604 }
605 
606 static int cpsw_restore_vlans(struct net_device *vdev, int vid, void *arg)
607 {
608 	struct am65_cpsw_port *port = arg;
609 
610 	if (!vdev)
611 		return 0;
612 
613 	return am65_cpsw_nuss_ndo_slave_add_vid(port->ndev, 0, vid);
614 }
615 
616 static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
617 {
618 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
619 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
620 	int ret, i;
621 
622 	ret = pm_runtime_get_sync(common->dev);
623 	if (ret < 0) {
624 		pm_runtime_put_noidle(common->dev);
625 		return ret;
626 	}
627 
628 	/* Notify the stack of the actual queue counts. */
629 	ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num);
630 	if (ret) {
631 		dev_err(common->dev, "cannot set real number of tx queues\n");
632 		return ret;
633 	}
634 
635 	ret = netif_set_real_num_rx_queues(ndev, AM65_CPSW_MAX_RX_QUEUES);
636 	if (ret) {
637 		dev_err(common->dev, "cannot set real number of rx queues\n");
638 		return ret;
639 	}
640 
641 	for (i = 0; i < common->tx_ch_num; i++)
642 		netdev_tx_reset_queue(netdev_get_tx_queue(ndev, i));
643 
644 	ret = am65_cpsw_nuss_common_open(common, ndev->features);
645 	if (ret)
646 		return ret;
647 
648 	common->usage_count++;
649 
650 	am65_cpsw_port_set_sl_mac(port, ndev->dev_addr);
651 
652 	if (common->is_emac_mode)
653 		am65_cpsw_init_port_emac_ale(port);
654 	else
655 		am65_cpsw_init_port_switch_ale(port);
656 
657 	/* mac_sl should be configured via phy-link interface */
658 	am65_cpsw_sl_ctl_reset(port);
659 
660 	ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET,
661 			       port->slave.phy_if);
662 	if (ret)
663 		goto error_cleanup;
664 
665 	if (port->slave.phy_node) {
666 		port->slave.phy = of_phy_connect(ndev,
667 						 port->slave.phy_node,
668 						 &am65_cpsw_nuss_adjust_link,
669 						 0, port->slave.phy_if);
670 		if (!port->slave.phy) {
671 			dev_err(common->dev, "phy %pOF not found on slave %d\n",
672 				port->slave.phy_node,
673 				port->port_id);
674 			ret = -ENODEV;
675 			goto error_cleanup;
676 		}
677 	}
678 
679 	/* restore vlan configurations */
680 	vlan_for_each(ndev, cpsw_restore_vlans, port);
681 
682 	phy_attached_info(port->slave.phy);
683 	phy_start(port->slave.phy);
684 
685 	return 0;
686 
687 error_cleanup:
688 	am65_cpsw_nuss_ndo_slave_stop(ndev);
689 	return ret;
690 }
691 
692 static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
693 {
694 	struct am65_cpsw_rx_chn *rx_chn = data;
695 	struct cppi5_host_desc_t *desc_rx;
696 	struct sk_buff *skb;
697 	dma_addr_t buf_dma;
698 	u32 buf_dma_len;
699 	void **swdata;
700 
701 	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
702 	swdata = cppi5_hdesc_get_swdata(desc_rx);
703 	skb = *swdata;
704 	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
705 	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
706 
707 	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
708 	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
709 
710 	dev_kfree_skb_any(skb);
711 }
712 
713 static void am65_cpsw_nuss_rx_ts(struct sk_buff *skb, u32 *psdata)
714 {
715 	struct skb_shared_hwtstamps *ssh;
716 	u64 ns;
717 
718 	ns = ((u64)psdata[1] << 32) | psdata[0];
719 
720 	ssh = skb_hwtstamps(skb);
721 	memset(ssh, 0, sizeof(*ssh));
722 	ssh->hwtstamp = ns_to_ktime(ns);
723 }
724 
725 /* RX psdata[2] word format - checksum information */
726 #define AM65_CPSW_RX_PSD_CSUM_ADD	GENMASK(15, 0)
727 #define AM65_CPSW_RX_PSD_CSUM_ERR	BIT(16)
728 #define AM65_CPSW_RX_PSD_IS_FRAGMENT	BIT(17)
729 #define AM65_CPSW_RX_PSD_IS_TCP		BIT(18)
730 #define AM65_CPSW_RX_PSD_IPV6_VALID	BIT(19)
731 #define AM65_CPSW_RX_PSD_IPV4_VALID	BIT(20)
732 
733 static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
734 {
735 	/* HW can verify IPv4/IPv6 TCP/UDP packets checksum
736 	 * csum information provides in psdata[2] word:
737 	 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error
738 	 * AM65_CPSW_RX_PSD_IPV6_VALID and AM65_CPSW_RX_PSD_IPV4_VALID
739 	 * bits - indicates IPv4/IPv6 packet
740 	 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet
741 	 * AM65_CPSW_RX_PSD_CSUM_ADD has value 0xFFFF for non fragmented packets
742 	 * or csum value for fragmented packets if !AM65_CPSW_RX_PSD_CSUM_ERR
743 	 */
744 	skb_checksum_none_assert(skb);
745 
746 	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
747 		return;
748 
749 	if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID |
750 			  AM65_CPSW_RX_PSD_IPV4_VALID)) &&
751 			  !(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) {
752 		/* csum for fragmented packets is unsupported */
753 		if (!(csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT))
754 			skb->ip_summed = CHECKSUM_UNNECESSARY;
755 	}
756 }
757 
758 static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
759 				     u32 flow_idx)
760 {
761 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
762 	u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
763 	struct am65_cpsw_ndev_priv *ndev_priv;
764 	struct am65_cpsw_ndev_stats *stats;
765 	struct cppi5_host_desc_t *desc_rx;
766 	struct device *dev = common->dev;
767 	struct sk_buff *skb, *new_skb;
768 	dma_addr_t desc_dma, buf_dma;
769 	struct am65_cpsw_port *port;
770 	struct net_device *ndev;
771 	void **swdata;
772 	u32 *psdata;
773 	int ret = 0;
774 
775 	ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);
776 	if (ret) {
777 		if (ret != -ENODATA)
778 			dev_err(dev, "RX: pop chn fail %d\n", ret);
779 		return ret;
780 	}
781 
782 	if (cppi5_desc_is_tdcm(desc_dma)) {
783 		dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx);
784 		return 0;
785 	}
786 
787 	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
788 	dev_dbg(dev, "%s flow_idx: %u desc %pad\n",
789 		__func__, flow_idx, &desc_dma);
790 
791 	swdata = cppi5_hdesc_get_swdata(desc_rx);
792 	skb = *swdata;
793 	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
794 	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
795 	pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
796 	cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
797 	dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
798 	port = am65_common_get_port(common, port_id);
799 	ndev = port->ndev;
800 	skb->dev = ndev;
801 
802 	psdata = cppi5_hdesc_get_psdata(desc_rx);
803 	/* add RX timestamp */
804 	if (port->rx_ts_enabled)
805 		am65_cpsw_nuss_rx_ts(skb, psdata);
806 	csum_info = psdata[2];
807 	dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
808 
809 	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
810 
811 	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
812 
813 	new_skb = netdev_alloc_skb_ip_align(ndev, AM65_CPSW_MAX_PACKET_SIZE);
814 	if (new_skb) {
815 		ndev_priv = netdev_priv(ndev);
816 		am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark);
817 		skb_put(skb, pkt_len);
818 		skb->protocol = eth_type_trans(skb, ndev);
819 		am65_cpsw_nuss_rx_csum(skb, csum_info);
820 		napi_gro_receive(&common->napi_rx, skb);
821 
822 		stats = this_cpu_ptr(ndev_priv->stats);
823 
824 		u64_stats_update_begin(&stats->syncp);
825 		stats->rx_packets++;
826 		stats->rx_bytes += pkt_len;
827 		u64_stats_update_end(&stats->syncp);
828 		kmemleak_not_leak(new_skb);
829 	} else {
830 		ndev->stats.rx_dropped++;
831 		new_skb = skb;
832 	}
833 
834 	if (netif_dormant(ndev)) {
835 		dev_kfree_skb_any(new_skb);
836 		ndev->stats.rx_dropped++;
837 		return 0;
838 	}
839 
840 	ret = am65_cpsw_nuss_rx_push(common, new_skb);
841 	if (WARN_ON(ret < 0)) {
842 		dev_kfree_skb_any(new_skb);
843 		ndev->stats.rx_errors++;
844 		ndev->stats.rx_dropped++;
845 	}
846 
847 	return ret;
848 }
849 
850 static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
851 {
852 	struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_rx);
853 	int flow = AM65_CPSW_MAX_RX_FLOWS;
854 	int cur_budget, ret;
855 	int num_rx = 0;
856 
857 	/* process every flow */
858 	while (flow--) {
859 		cur_budget = budget - num_rx;
860 
861 		while (cur_budget--) {
862 			ret = am65_cpsw_nuss_rx_packets(common, flow);
863 			if (ret)
864 				break;
865 			num_rx++;
866 		}
867 
868 		if (num_rx >= budget)
869 			break;
870 	}
871 
872 	dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
873 
874 	if (num_rx < budget && napi_complete_done(napi_rx, num_rx))
875 		enable_irq(common->rx_chns.irq);
876 
877 	return num_rx;
878 }
879 
880 static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
881 				     struct cppi5_host_desc_t *desc)
882 {
883 	struct cppi5_host_desc_t *first_desc, *next_desc;
884 	dma_addr_t buf_dma, next_desc_dma;
885 	u32 buf_dma_len;
886 
887 	first_desc = desc;
888 	next_desc = first_desc;
889 
890 	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
891 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
892 
893 	dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);
894 
895 	next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
896 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
897 	while (next_desc_dma) {
898 		next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
899 						       next_desc_dma);
900 		cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
901 		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
902 
903 		dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
904 			       DMA_TO_DEVICE);
905 
906 		next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
907 		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
908 
909 		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
910 	}
911 
912 	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
913 }
914 
915 static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
916 {
917 	struct am65_cpsw_tx_chn *tx_chn = data;
918 	struct cppi5_host_desc_t *desc_tx;
919 	struct sk_buff *skb;
920 	void **swdata;
921 
922 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
923 	swdata = cppi5_hdesc_get_swdata(desc_tx);
924 	skb = *(swdata);
925 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
926 
927 	dev_kfree_skb_any(skb);
928 }
929 
930 static struct sk_buff *
931 am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn,
932 			       dma_addr_t desc_dma)
933 {
934 	struct am65_cpsw_ndev_priv *ndev_priv;
935 	struct am65_cpsw_ndev_stats *stats;
936 	struct cppi5_host_desc_t *desc_tx;
937 	struct net_device *ndev;
938 	struct sk_buff *skb;
939 	void **swdata;
940 
941 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
942 					     desc_dma);
943 	swdata = cppi5_hdesc_get_swdata(desc_tx);
944 	skb = *(swdata);
945 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
946 
947 	ndev = skb->dev;
948 
949 	am65_cpts_tx_timestamp(tx_chn->common->cpts, skb);
950 
951 	ndev_priv = netdev_priv(ndev);
952 	stats = this_cpu_ptr(ndev_priv->stats);
953 	u64_stats_update_begin(&stats->syncp);
954 	stats->tx_packets++;
955 	stats->tx_bytes += skb->len;
956 	u64_stats_update_end(&stats->syncp);
957 
958 	return skb;
959 }
960 
961 static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,
962 				   struct netdev_queue *netif_txq)
963 {
964 	if (netif_tx_queue_stopped(netif_txq)) {
965 		/* Check whether the queue is stopped due to stalled
966 		 * tx dma, if the queue is stopped then wake the queue
967 		 * as we have free desc for tx
968 		 */
969 		__netif_tx_lock(netif_txq, smp_processor_id());
970 		if (netif_running(ndev) &&
971 		    (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))
972 			netif_tx_wake_queue(netif_txq);
973 
974 		__netif_tx_unlock(netif_txq);
975 	}
976 }
977 
978 static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
979 					   int chn, unsigned int budget)
980 {
981 	struct device *dev = common->dev;
982 	struct am65_cpsw_tx_chn *tx_chn;
983 	struct netdev_queue *netif_txq;
984 	unsigned int total_bytes = 0;
985 	struct net_device *ndev;
986 	struct sk_buff *skb;
987 	dma_addr_t desc_dma;
988 	int res, num_tx = 0;
989 
990 	tx_chn = &common->tx_chns[chn];
991 
992 	while (true) {
993 		spin_lock(&tx_chn->lock);
994 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
995 		spin_unlock(&tx_chn->lock);
996 		if (res == -ENODATA)
997 			break;
998 
999 		if (cppi5_desc_is_tdcm(desc_dma)) {
1000 			if (atomic_dec_and_test(&common->tdown_cnt))
1001 				complete(&common->tdown_complete);
1002 			break;
1003 		}
1004 
1005 		skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);
1006 		total_bytes = skb->len;
1007 		ndev = skb->dev;
1008 		napi_consume_skb(skb, budget);
1009 		num_tx++;
1010 
1011 		netif_txq = netdev_get_tx_queue(ndev, chn);
1012 
1013 		netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1014 
1015 		am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1016 	}
1017 
1018 	dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1019 
1020 	return num_tx;
1021 }
1022 
1023 static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common,
1024 					      int chn, unsigned int budget)
1025 {
1026 	struct device *dev = common->dev;
1027 	struct am65_cpsw_tx_chn *tx_chn;
1028 	struct netdev_queue *netif_txq;
1029 	unsigned int total_bytes = 0;
1030 	struct net_device *ndev;
1031 	struct sk_buff *skb;
1032 	dma_addr_t desc_dma;
1033 	int res, num_tx = 0;
1034 
1035 	tx_chn = &common->tx_chns[chn];
1036 
1037 	while (true) {
1038 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1039 		if (res == -ENODATA)
1040 			break;
1041 
1042 		if (cppi5_desc_is_tdcm(desc_dma)) {
1043 			if (atomic_dec_and_test(&common->tdown_cnt))
1044 				complete(&common->tdown_complete);
1045 			break;
1046 		}
1047 
1048 		skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);
1049 
1050 		ndev = skb->dev;
1051 		total_bytes += skb->len;
1052 		napi_consume_skb(skb, budget);
1053 		num_tx++;
1054 	}
1055 
1056 	if (!num_tx)
1057 		return 0;
1058 
1059 	netif_txq = netdev_get_tx_queue(ndev, chn);
1060 
1061 	netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1062 
1063 	am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1064 
1065 	dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1066 
1067 	return num_tx;
1068 }
1069 
1070 static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget)
1071 {
1072 	struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);
1073 	int num_tx;
1074 
1075 	if (AM65_CPSW_IS_CPSW2G(tx_chn->common))
1076 		num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id, budget);
1077 	else
1078 		num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, tx_chn->id, budget);
1079 
1080 	num_tx = min(num_tx, budget);
1081 	if (num_tx < budget) {
1082 		napi_complete(napi_tx);
1083 		enable_irq(tx_chn->irq);
1084 	}
1085 
1086 	return num_tx;
1087 }
1088 
1089 static irqreturn_t am65_cpsw_nuss_rx_irq(int irq, void *dev_id)
1090 {
1091 	struct am65_cpsw_common *common = dev_id;
1092 
1093 	disable_irq_nosync(irq);
1094 	napi_schedule(&common->napi_rx);
1095 
1096 	return IRQ_HANDLED;
1097 }
1098 
1099 static irqreturn_t am65_cpsw_nuss_tx_irq(int irq, void *dev_id)
1100 {
1101 	struct am65_cpsw_tx_chn *tx_chn = dev_id;
1102 
1103 	disable_irq_nosync(irq);
1104 	napi_schedule(&tx_chn->napi_tx);
1105 
1106 	return IRQ_HANDLED;
1107 }
1108 
1109 static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
1110 						 struct net_device *ndev)
1111 {
1112 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1113 	struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
1114 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1115 	struct device *dev = common->dev;
1116 	struct am65_cpsw_tx_chn *tx_chn;
1117 	struct netdev_queue *netif_txq;
1118 	dma_addr_t desc_dma, buf_dma;
1119 	int ret, q_idx, i;
1120 	void **swdata;
1121 	u32 *psdata;
1122 	u32 pkt_len;
1123 
1124 	/* padding enabled in hw */
1125 	pkt_len = skb_headlen(skb);
1126 
1127 	/* SKB TX timestamp */
1128 	if (port->tx_ts_enabled)
1129 		am65_cpts_prep_tx_timestamp(common->cpts, skb);
1130 
1131 	q_idx = skb_get_queue_mapping(skb);
1132 	dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx);
1133 
1134 	tx_chn = &common->tx_chns[q_idx];
1135 	netif_txq = netdev_get_tx_queue(ndev, q_idx);
1136 
1137 	/* Map the linear buffer */
1138 	buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,
1139 				 DMA_TO_DEVICE);
1140 	if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1141 		dev_err(dev, "Failed to map tx skb buffer\n");
1142 		ndev->stats.tx_errors++;
1143 		goto err_free_skb;
1144 	}
1145 
1146 	first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1147 	if (!first_desc) {
1148 		dev_dbg(dev, "Failed to allocate descriptor\n");
1149 		dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,
1150 				 DMA_TO_DEVICE);
1151 		goto busy_stop_q;
1152 	}
1153 
1154 	cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1155 			 AM65_CPSW_NAV_PS_DATA_SIZE);
1156 	cppi5_desc_set_pktids(&first_desc->hdr, 0, 0x3FFF);
1157 	cppi5_hdesc_set_pkttype(first_desc, 0x7);
1158 	cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id);
1159 
1160 	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1161 	cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
1162 	swdata = cppi5_hdesc_get_swdata(first_desc);
1163 	*(swdata) = skb;
1164 	psdata = cppi5_hdesc_get_psdata(first_desc);
1165 
1166 	/* HW csum offload if enabled */
1167 	psdata[2] = 0;
1168 	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1169 		unsigned int cs_start, cs_offset;
1170 
1171 		cs_start = skb_transport_offset(skb);
1172 		cs_offset = cs_start + skb->csum_offset;
1173 		/* HW numerates bytes starting from 1 */
1174 		psdata[2] = ((cs_offset + 1) << 24) |
1175 			    ((cs_start + 1) << 16) | (skb->len - cs_start);
1176 		dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]);
1177 	}
1178 
1179 	if (!skb_is_nonlinear(skb))
1180 		goto done_tx;
1181 
1182 	dev_dbg(dev, "fragmented SKB\n");
1183 
1184 	/* Handle the case where skb is fragmented in pages */
1185 	cur_desc = first_desc;
1186 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1187 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1188 		u32 frag_size = skb_frag_size(frag);
1189 
1190 		next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1191 		if (!next_desc) {
1192 			dev_err(dev, "Failed to allocate descriptor\n");
1193 			goto busy_free_descs;
1194 		}
1195 
1196 		buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,
1197 					   DMA_TO_DEVICE);
1198 		if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1199 			dev_err(dev, "Failed to map tx skb page\n");
1200 			k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
1201 			ndev->stats.tx_errors++;
1202 			goto err_free_descs;
1203 		}
1204 
1205 		cppi5_hdesc_reset_hbdesc(next_desc);
1206 		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1207 		cppi5_hdesc_attach_buf(next_desc,
1208 				       buf_dma, frag_size, buf_dma, frag_size);
1209 
1210 		desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,
1211 						      next_desc);
1212 		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);
1213 		cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
1214 
1215 		pkt_len += frag_size;
1216 		cur_desc = next_desc;
1217 	}
1218 	WARN_ON(pkt_len != skb->len);
1219 
1220 done_tx:
1221 	skb_tx_timestamp(skb);
1222 
1223 	/* report bql before sending packet */
1224 	netdev_tx_sent_queue(netif_txq, pkt_len);
1225 
1226 	cppi5_hdesc_set_pktlen(first_desc, pkt_len);
1227 	desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);
1228 	if (AM65_CPSW_IS_CPSW2G(common)) {
1229 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1230 	} else {
1231 		spin_lock_bh(&tx_chn->lock);
1232 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1233 		spin_unlock_bh(&tx_chn->lock);
1234 	}
1235 	if (ret) {
1236 		dev_err(dev, "can't push desc %d\n", ret);
1237 		/* inform bql */
1238 		netdev_tx_completed_queue(netif_txq, 1, pkt_len);
1239 		ndev->stats.tx_errors++;
1240 		goto err_free_descs;
1241 	}
1242 
1243 	if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
1244 		netif_tx_stop_queue(netif_txq);
1245 		/* Barrier, so that stop_queue visible to other cpus */
1246 		smp_mb__after_atomic();
1247 		dev_dbg(dev, "netif_tx_stop_queue %d\n", q_idx);
1248 
1249 		/* re-check for smp */
1250 		if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
1251 		    MAX_SKB_FRAGS) {
1252 			netif_tx_wake_queue(netif_txq);
1253 			dev_dbg(dev, "netif_tx_wake_queue %d\n", q_idx);
1254 		}
1255 	}
1256 
1257 	return NETDEV_TX_OK;
1258 
1259 err_free_descs:
1260 	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1261 err_free_skb:
1262 	ndev->stats.tx_dropped++;
1263 	dev_kfree_skb_any(skb);
1264 	return NETDEV_TX_OK;
1265 
1266 busy_free_descs:
1267 	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1268 busy_stop_q:
1269 	netif_tx_stop_queue(netif_txq);
1270 	return NETDEV_TX_BUSY;
1271 }
1272 
1273 static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev,
1274 						    void *addr)
1275 {
1276 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1277 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1278 	struct sockaddr *sockaddr = (struct sockaddr *)addr;
1279 	int ret;
1280 
1281 	ret = eth_prepare_mac_addr_change(ndev, addr);
1282 	if (ret < 0)
1283 		return ret;
1284 
1285 	ret = pm_runtime_get_sync(common->dev);
1286 	if (ret < 0) {
1287 		pm_runtime_put_noidle(common->dev);
1288 		return ret;
1289 	}
1290 
1291 	cpsw_ale_del_ucast(common->ale, ndev->dev_addr,
1292 			   HOST_PORT_NUM, 0, 0);
1293 	cpsw_ale_add_ucast(common->ale, sockaddr->sa_data,
1294 			   HOST_PORT_NUM, ALE_SECURE, 0);
1295 
1296 	am65_cpsw_port_set_sl_mac(port, addr);
1297 	eth_commit_mac_addr_change(ndev, sockaddr);
1298 
1299 	pm_runtime_put(common->dev);
1300 
1301 	return 0;
1302 }
1303 
1304 static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev,
1305 				       struct ifreq *ifr)
1306 {
1307 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1308 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1309 	u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype;
1310 	struct hwtstamp_config cfg;
1311 
1312 	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1313 		return -EOPNOTSUPP;
1314 
1315 	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1316 		return -EFAULT;
1317 
1318 	/* TX HW timestamp */
1319 	switch (cfg.tx_type) {
1320 	case HWTSTAMP_TX_OFF:
1321 	case HWTSTAMP_TX_ON:
1322 		break;
1323 	default:
1324 		return -ERANGE;
1325 	}
1326 
1327 	switch (cfg.rx_filter) {
1328 	case HWTSTAMP_FILTER_NONE:
1329 		port->rx_ts_enabled = false;
1330 		break;
1331 	case HWTSTAMP_FILTER_ALL:
1332 	case HWTSTAMP_FILTER_SOME:
1333 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1334 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1335 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1336 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1337 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1338 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1339 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1340 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1341 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1342 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1343 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1344 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1345 	case HWTSTAMP_FILTER_NTP_ALL:
1346 		port->rx_ts_enabled = true;
1347 		cfg.rx_filter = HWTSTAMP_FILTER_ALL;
1348 		break;
1349 	default:
1350 		return -ERANGE;
1351 	}
1352 
1353 	port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON);
1354 
1355 	/* cfg TX timestamp */
1356 	seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET <<
1357 		  AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT) | ETH_P_1588;
1358 
1359 	ts_vlan_ltype = ETH_P_8021Q;
1360 
1361 	ts_ctrl_ltype2 = ETH_P_1588 |
1362 			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 |
1363 			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 |
1364 			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 |
1365 			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 |
1366 			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 |
1367 			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 |
1368 			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 |
1369 			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO;
1370 
1371 	ts_ctrl = AM65_CPSW_TS_EVENT_MSG_TYPE_BITS <<
1372 		  AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT;
1373 
1374 	if (port->tx_ts_enabled)
1375 		ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN |
1376 			   AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN;
1377 
1378 	writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG);
1379 	writel(ts_vlan_ltype, port->port_base +
1380 	       AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG);
1381 	writel(ts_ctrl_ltype2, port->port_base +
1382 	       AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2);
1383 	writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL);
1384 
1385 	/* en/dis RX timestamp */
1386 	am65_cpts_rx_enable(common->cpts, port->rx_ts_enabled);
1387 
1388 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1389 }
1390 
1391 static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev,
1392 				       struct ifreq *ifr)
1393 {
1394 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1395 	struct hwtstamp_config cfg;
1396 
1397 	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1398 		return -EOPNOTSUPP;
1399 
1400 	cfg.flags = 0;
1401 	cfg.tx_type = port->tx_ts_enabled ?
1402 		      HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1403 	cfg.rx_filter = port->rx_ts_enabled ?
1404 			HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1405 
1406 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1407 }
1408 
1409 static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev,
1410 					  struct ifreq *req, int cmd)
1411 {
1412 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1413 
1414 	if (!netif_running(ndev))
1415 		return -EINVAL;
1416 
1417 	switch (cmd) {
1418 	case SIOCSHWTSTAMP:
1419 		return am65_cpsw_nuss_hwtstamp_set(ndev, req);
1420 	case SIOCGHWTSTAMP:
1421 		return am65_cpsw_nuss_hwtstamp_get(ndev, req);
1422 	}
1423 
1424 	if (!port->slave.phy)
1425 		return -EOPNOTSUPP;
1426 
1427 	return phy_mii_ioctl(port->slave.phy, req, cmd);
1428 }
1429 
1430 static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev,
1431 					 struct rtnl_link_stats64 *stats)
1432 {
1433 	struct am65_cpsw_ndev_priv *ndev_priv = netdev_priv(dev);
1434 	unsigned int start;
1435 	int cpu;
1436 
1437 	for_each_possible_cpu(cpu) {
1438 		struct am65_cpsw_ndev_stats *cpu_stats;
1439 		u64 rx_packets;
1440 		u64 rx_bytes;
1441 		u64 tx_packets;
1442 		u64 tx_bytes;
1443 
1444 		cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu);
1445 		do {
1446 			start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
1447 			rx_packets = cpu_stats->rx_packets;
1448 			rx_bytes   = cpu_stats->rx_bytes;
1449 			tx_packets = cpu_stats->tx_packets;
1450 			tx_bytes   = cpu_stats->tx_bytes;
1451 		} while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
1452 
1453 		stats->rx_packets += rx_packets;
1454 		stats->rx_bytes   += rx_bytes;
1455 		stats->tx_packets += tx_packets;
1456 		stats->tx_bytes   += tx_bytes;
1457 	}
1458 
1459 	stats->rx_errors	= dev->stats.rx_errors;
1460 	stats->rx_dropped	= dev->stats.rx_dropped;
1461 	stats->tx_dropped	= dev->stats.tx_dropped;
1462 }
1463 
1464 static struct devlink_port *am65_cpsw_ndo_get_devlink_port(struct net_device *ndev)
1465 {
1466 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1467 
1468 	return &port->devlink_port;
1469 }
1470 
1471 static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
1472 	.ndo_open		= am65_cpsw_nuss_ndo_slave_open,
1473 	.ndo_stop		= am65_cpsw_nuss_ndo_slave_stop,
1474 	.ndo_start_xmit		= am65_cpsw_nuss_ndo_slave_xmit,
1475 	.ndo_set_rx_mode	= am65_cpsw_nuss_ndo_slave_set_rx_mode,
1476 	.ndo_get_stats64        = am65_cpsw_nuss_ndo_get_stats,
1477 	.ndo_validate_addr	= eth_validate_addr,
1478 	.ndo_set_mac_address	= am65_cpsw_nuss_ndo_slave_set_mac_address,
1479 	.ndo_tx_timeout		= am65_cpsw_nuss_ndo_host_tx_timeout,
1480 	.ndo_vlan_rx_add_vid	= am65_cpsw_nuss_ndo_slave_add_vid,
1481 	.ndo_vlan_rx_kill_vid	= am65_cpsw_nuss_ndo_slave_kill_vid,
1482 	.ndo_do_ioctl		= am65_cpsw_nuss_ndo_slave_ioctl,
1483 	.ndo_setup_tc           = am65_cpsw_qos_ndo_setup_tc,
1484 	.ndo_get_devlink_port   = am65_cpsw_ndo_get_devlink_port,
1485 };
1486 
1487 static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port)
1488 {
1489 	struct am65_cpsw_common *common = port->common;
1490 
1491 	if (!port->disabled)
1492 		return;
1493 
1494 	cpsw_ale_control_set(common->ale, port->port_id,
1495 			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1496 
1497 	cpsw_sl_reset(port->slave.mac_sl, 100);
1498 	cpsw_sl_ctl_reset(port->slave.mac_sl);
1499 }
1500 
1501 static void am65_cpsw_nuss_free_tx_chns(void *data)
1502 {
1503 	struct am65_cpsw_common *common = data;
1504 	int i;
1505 
1506 	for (i = 0; i < common->tx_ch_num; i++) {
1507 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1508 
1509 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
1510 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
1511 
1512 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
1513 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
1514 
1515 		memset(tx_chn, 0, sizeof(*tx_chn));
1516 	}
1517 }
1518 
1519 void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common)
1520 {
1521 	struct device *dev = common->dev;
1522 	int i;
1523 
1524 	devm_remove_action(dev, am65_cpsw_nuss_free_tx_chns, common);
1525 
1526 	for (i = 0; i < common->tx_ch_num; i++) {
1527 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1528 
1529 		if (tx_chn->irq)
1530 			devm_free_irq(dev, tx_chn->irq, tx_chn);
1531 
1532 		netif_napi_del(&tx_chn->napi_tx);
1533 
1534 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
1535 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
1536 
1537 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
1538 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
1539 
1540 		memset(tx_chn, 0, sizeof(*tx_chn));
1541 	}
1542 }
1543 
1544 static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
1545 {
1546 	u32  max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS);
1547 	struct k3_udma_glue_tx_channel_cfg tx_cfg = { 0 };
1548 	struct device *dev = common->dev;
1549 	struct k3_ring_cfg ring_cfg = {
1550 		.elm_size = K3_RINGACC_RING_ELSIZE_8,
1551 		.mode = K3_RINGACC_RING_MODE_RING,
1552 		.flags = 0
1553 	};
1554 	u32 hdesc_size;
1555 	int i, ret = 0;
1556 
1557 	hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
1558 					   AM65_CPSW_NAV_SW_DATA_SIZE);
1559 
1560 	tx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
1561 	tx_cfg.tx_cfg = ring_cfg;
1562 	tx_cfg.txcq_cfg = ring_cfg;
1563 	tx_cfg.tx_cfg.size = max_desc_num;
1564 	tx_cfg.txcq_cfg.size = max_desc_num;
1565 
1566 	for (i = 0; i < common->tx_ch_num; i++) {
1567 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1568 
1569 		snprintf(tx_chn->tx_chn_name,
1570 			 sizeof(tx_chn->tx_chn_name), "tx%d", i);
1571 
1572 		spin_lock_init(&tx_chn->lock);
1573 		tx_chn->common = common;
1574 		tx_chn->id = i;
1575 		tx_chn->descs_num = max_desc_num;
1576 
1577 		tx_chn->tx_chn =
1578 			k3_udma_glue_request_tx_chn(dev,
1579 						    tx_chn->tx_chn_name,
1580 						    &tx_cfg);
1581 		if (IS_ERR(tx_chn->tx_chn)) {
1582 			ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn),
1583 					    "Failed to request tx dma channel\n");
1584 			goto err;
1585 		}
1586 		tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);
1587 
1588 		tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev,
1589 								  tx_chn->descs_num,
1590 								  hdesc_size,
1591 								  tx_chn->tx_chn_name);
1592 		if (IS_ERR(tx_chn->desc_pool)) {
1593 			ret = PTR_ERR(tx_chn->desc_pool);
1594 			dev_err(dev, "Failed to create poll %d\n", ret);
1595 			goto err;
1596 		}
1597 
1598 		tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);
1599 		if (tx_chn->irq <= 0) {
1600 			dev_err(dev, "Failed to get tx dma irq %d\n",
1601 				tx_chn->irq);
1602 			goto err;
1603 		}
1604 
1605 		snprintf(tx_chn->tx_chn_name,
1606 			 sizeof(tx_chn->tx_chn_name), "%s-tx%d",
1607 			 dev_name(dev), tx_chn->id);
1608 	}
1609 
1610 err:
1611 	i = devm_add_action(dev, am65_cpsw_nuss_free_tx_chns, common);
1612 	if (i) {
1613 		dev_err(dev, "Failed to add free_tx_chns action %d\n", i);
1614 		return i;
1615 	}
1616 
1617 	return ret;
1618 }
1619 
1620 static void am65_cpsw_nuss_free_rx_chns(void *data)
1621 {
1622 	struct am65_cpsw_common *common = data;
1623 	struct am65_cpsw_rx_chn *rx_chn;
1624 
1625 	rx_chn = &common->rx_chns;
1626 
1627 	if (!IS_ERR_OR_NULL(rx_chn->rx_chn))
1628 		k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
1629 
1630 	if (!IS_ERR_OR_NULL(rx_chn->desc_pool))
1631 		k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
1632 }
1633 
1634 static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
1635 {
1636 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
1637 	struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 };
1638 	u32  max_desc_num = AM65_CPSW_MAX_RX_DESC;
1639 	struct device *dev = common->dev;
1640 	u32 hdesc_size;
1641 	u32 fdqring_id;
1642 	int i, ret = 0;
1643 
1644 	hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
1645 					   AM65_CPSW_NAV_SW_DATA_SIZE);
1646 
1647 	rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
1648 	rx_cfg.flow_id_num = AM65_CPSW_MAX_RX_FLOWS;
1649 	rx_cfg.flow_id_base = common->rx_flow_id_base;
1650 
1651 	/* init all flows */
1652 	rx_chn->dev = dev;
1653 	rx_chn->descs_num = max_desc_num;
1654 
1655 	rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg);
1656 	if (IS_ERR(rx_chn->rx_chn)) {
1657 		ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn),
1658 				    "Failed to request rx dma channel\n");
1659 		goto err;
1660 	}
1661 	rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);
1662 
1663 	rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,
1664 							  rx_chn->descs_num,
1665 							  hdesc_size, "rx");
1666 	if (IS_ERR(rx_chn->desc_pool)) {
1667 		ret = PTR_ERR(rx_chn->desc_pool);
1668 		dev_err(dev, "Failed to create rx poll %d\n", ret);
1669 		goto err;
1670 	}
1671 
1672 	common->rx_flow_id_base =
1673 			k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);
1674 	dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);
1675 
1676 	fdqring_id = K3_RINGACC_RING_ID_ANY;
1677 	for (i = 0; i < rx_cfg.flow_id_num; i++) {
1678 		struct k3_ring_cfg rxring_cfg = {
1679 			.elm_size = K3_RINGACC_RING_ELSIZE_8,
1680 			.mode = K3_RINGACC_RING_MODE_RING,
1681 			.flags = 0,
1682 		};
1683 		struct k3_ring_cfg fdqring_cfg = {
1684 			.elm_size = K3_RINGACC_RING_ELSIZE_8,
1685 			.flags = K3_RINGACC_RING_SHARED,
1686 		};
1687 		struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = {
1688 			.rx_cfg = rxring_cfg,
1689 			.rxfdq_cfg = fdqring_cfg,
1690 			.ring_rxq_id = K3_RINGACC_RING_ID_ANY,
1691 			.src_tag_lo_sel =
1692 				K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
1693 		};
1694 
1695 		rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
1696 		rx_flow_cfg.rx_cfg.size = max_desc_num;
1697 		rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
1698 		rx_flow_cfg.rxfdq_cfg.mode = common->pdata.fdqring_mode;
1699 
1700 		ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,
1701 						i, &rx_flow_cfg);
1702 		if (ret) {
1703 			dev_err(dev, "Failed to init rx flow%d %d\n", i, ret);
1704 			goto err;
1705 		}
1706 		if (!i)
1707 			fdqring_id =
1708 				k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,
1709 								i);
1710 
1711 		rx_chn->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);
1712 
1713 		if (rx_chn->irq <= 0) {
1714 			dev_err(dev, "Failed to get rx dma irq %d\n",
1715 				rx_chn->irq);
1716 			ret = -ENXIO;
1717 			goto err;
1718 		}
1719 	}
1720 
1721 err:
1722 	i = devm_add_action(dev, am65_cpsw_nuss_free_rx_chns, common);
1723 	if (i) {
1724 		dev_err(dev, "Failed to add free_rx_chns action %d\n", i);
1725 		return i;
1726 	}
1727 
1728 	return ret;
1729 }
1730 
1731 static int am65_cpsw_nuss_init_host_p(struct am65_cpsw_common *common)
1732 {
1733 	struct am65_cpsw_host *host_p = am65_common_get_host(common);
1734 
1735 	host_p->common = common;
1736 	host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE;
1737 	host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE;
1738 
1739 	return 0;
1740 }
1741 
1742 static int am65_cpsw_am654_get_efuse_macid(struct device_node *of_node,
1743 					   int slave, u8 *mac_addr)
1744 {
1745 	u32 mac_lo, mac_hi, offset;
1746 	struct regmap *syscon;
1747 	int ret;
1748 
1749 	syscon = syscon_regmap_lookup_by_phandle(of_node, "ti,syscon-efuse");
1750 	if (IS_ERR(syscon)) {
1751 		if (PTR_ERR(syscon) == -ENODEV)
1752 			return 0;
1753 		return PTR_ERR(syscon);
1754 	}
1755 
1756 	ret = of_property_read_u32_index(of_node, "ti,syscon-efuse", 1,
1757 					 &offset);
1758 	if (ret)
1759 		return ret;
1760 
1761 	regmap_read(syscon, offset, &mac_lo);
1762 	regmap_read(syscon, offset + 4, &mac_hi);
1763 
1764 	mac_addr[0] = (mac_hi >> 8) & 0xff;
1765 	mac_addr[1] = mac_hi & 0xff;
1766 	mac_addr[2] = (mac_lo >> 24) & 0xff;
1767 	mac_addr[3] = (mac_lo >> 16) & 0xff;
1768 	mac_addr[4] = (mac_lo >> 8) & 0xff;
1769 	mac_addr[5] = mac_lo & 0xff;
1770 
1771 	return 0;
1772 }
1773 
1774 static int am65_cpsw_init_cpts(struct am65_cpsw_common *common)
1775 {
1776 	struct device *dev = common->dev;
1777 	struct device_node *node;
1778 	struct am65_cpts *cpts;
1779 	void __iomem *reg_base;
1780 
1781 	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1782 		return 0;
1783 
1784 	node = of_get_child_by_name(dev->of_node, "cpts");
1785 	if (!node) {
1786 		dev_err(dev, "%s cpts not found\n", __func__);
1787 		return -ENOENT;
1788 	}
1789 
1790 	reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE;
1791 	cpts = am65_cpts_create(dev, reg_base, node);
1792 	if (IS_ERR(cpts)) {
1793 		int ret = PTR_ERR(cpts);
1794 
1795 		if (ret == -EOPNOTSUPP) {
1796 			dev_info(dev, "cpts disabled\n");
1797 			return 0;
1798 		}
1799 
1800 		dev_err(dev, "cpts create err %d\n", ret);
1801 		return ret;
1802 	}
1803 	common->cpts = cpts;
1804 	/* Forbid PM runtime if CPTS is running.
1805 	 * K3 CPSWxG modules may completely lose context during ON->OFF
1806 	 * transitions depending on integration.
1807 	 * AM65x/J721E MCU CPSW2G: false
1808 	 * J721E MAIN_CPSW9G: true
1809 	 */
1810 	pm_runtime_forbid(dev);
1811 
1812 	return 0;
1813 }
1814 
1815 static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
1816 {
1817 	struct device_node *node, *port_np;
1818 	struct device *dev = common->dev;
1819 	int ret;
1820 
1821 	node = of_get_child_by_name(dev->of_node, "ethernet-ports");
1822 	if (!node)
1823 		return -ENOENT;
1824 
1825 	for_each_child_of_node(node, port_np) {
1826 		struct am65_cpsw_port *port;
1827 		u32 port_id;
1828 
1829 		/* it is not a slave port node, continue */
1830 		if (strcmp(port_np->name, "port"))
1831 			continue;
1832 
1833 		ret = of_property_read_u32(port_np, "reg", &port_id);
1834 		if (ret < 0) {
1835 			dev_err(dev, "%pOF error reading port_id %d\n",
1836 				port_np, ret);
1837 			return ret;
1838 		}
1839 
1840 		if (!port_id || port_id > common->port_num) {
1841 			dev_err(dev, "%pOF has invalid port_id %u %s\n",
1842 				port_np, port_id, port_np->name);
1843 			return -EINVAL;
1844 		}
1845 
1846 		port = am65_common_get_port(common, port_id);
1847 		port->port_id = port_id;
1848 		port->common = common;
1849 		port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +
1850 				  AM65_CPSW_NU_PORTS_OFFSET * (port_id);
1851 		port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +
1852 				  (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id);
1853 		port->name = of_get_property(port_np, "label", NULL);
1854 		port->fetch_ram_base =
1855 				common->cpsw_base + AM65_CPSW_NU_FRAM_BASE +
1856 				(AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1));
1857 
1858 		port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base);
1859 		if (IS_ERR(port->slave.mac_sl))
1860 			return PTR_ERR(port->slave.mac_sl);
1861 
1862 		port->disabled = !of_device_is_available(port_np);
1863 		if (port->disabled) {
1864 			common->disabled_ports_mask |= BIT(port->port_id);
1865 			continue;
1866 		}
1867 
1868 		port->slave.ifphy = devm_of_phy_get(dev, port_np, NULL);
1869 		if (IS_ERR(port->slave.ifphy)) {
1870 			ret = PTR_ERR(port->slave.ifphy);
1871 			dev_err(dev, "%pOF error retrieving port phy: %d\n",
1872 				port_np, ret);
1873 			return ret;
1874 		}
1875 
1876 		port->slave.mac_only =
1877 				of_property_read_bool(port_np, "ti,mac-only");
1878 
1879 		/* get phy/link info */
1880 		if (of_phy_is_fixed_link(port_np)) {
1881 			ret = of_phy_register_fixed_link(port_np);
1882 			if (ret)
1883 				return dev_err_probe(dev, ret,
1884 						     "failed to register fixed-link phy %pOF\n",
1885 						     port_np);
1886 			port->slave.phy_node = of_node_get(port_np);
1887 		} else {
1888 			port->slave.phy_node =
1889 				of_parse_phandle(port_np, "phy-handle", 0);
1890 		}
1891 
1892 		if (!port->slave.phy_node) {
1893 			dev_err(dev,
1894 				"slave[%d] no phy found\n", port_id);
1895 			return -ENODEV;
1896 		}
1897 
1898 		ret = of_get_phy_mode(port_np, &port->slave.phy_if);
1899 		if (ret) {
1900 			dev_err(dev, "%pOF read phy-mode err %d\n",
1901 				port_np, ret);
1902 			return ret;
1903 		}
1904 
1905 		ret = of_get_mac_address(port_np, port->slave.mac_addr);
1906 		if (ret) {
1907 			am65_cpsw_am654_get_efuse_macid(port_np,
1908 							port->port_id,
1909 							port->slave.mac_addr);
1910 			if (!is_valid_ether_addr(port->slave.mac_addr)) {
1911 				random_ether_addr(port->slave.mac_addr);
1912 				dev_err(dev, "Use random MAC address\n");
1913 			}
1914 		}
1915 	}
1916 	of_node_put(node);
1917 
1918 	/* is there at least one ext.port */
1919 	if (!(~common->disabled_ports_mask & GENMASK(common->port_num, 1))) {
1920 		dev_err(dev, "No Ext. port are available\n");
1921 		return -ENODEV;
1922 	}
1923 
1924 	return 0;
1925 }
1926 
1927 static void am65_cpsw_pcpu_stats_free(void *data)
1928 {
1929 	struct am65_cpsw_ndev_stats __percpu *stats = data;
1930 
1931 	free_percpu(stats);
1932 }
1933 
1934 static int
1935 am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
1936 {
1937 	struct am65_cpsw_ndev_priv *ndev_priv;
1938 	struct device *dev = common->dev;
1939 	struct am65_cpsw_port *port;
1940 	int ret;
1941 
1942 	port = &common->ports[port_idx];
1943 
1944 	if (port->disabled)
1945 		return 0;
1946 
1947 	/* alloc netdev */
1948 	port->ndev = devm_alloc_etherdev_mqs(common->dev,
1949 					     sizeof(struct am65_cpsw_ndev_priv),
1950 					     AM65_CPSW_MAX_TX_QUEUES,
1951 					     AM65_CPSW_MAX_RX_QUEUES);
1952 	if (!port->ndev) {
1953 		dev_err(dev, "error allocating slave net_device %u\n",
1954 			port->port_id);
1955 		return -ENOMEM;
1956 	}
1957 
1958 	ndev_priv = netdev_priv(port->ndev);
1959 	ndev_priv->port = port;
1960 	ndev_priv->msg_enable = AM65_CPSW_DEBUG;
1961 	SET_NETDEV_DEV(port->ndev, dev);
1962 
1963 	ether_addr_copy(port->ndev->dev_addr, port->slave.mac_addr);
1964 
1965 	port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE;
1966 	port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE;
1967 	port->ndev->hw_features = NETIF_F_SG |
1968 				  NETIF_F_RXCSUM |
1969 				  NETIF_F_HW_CSUM |
1970 				  NETIF_F_HW_TC;
1971 	port->ndev->features = port->ndev->hw_features |
1972 			       NETIF_F_HW_VLAN_CTAG_FILTER;
1973 	port->ndev->vlan_features |=  NETIF_F_SG;
1974 	port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops;
1975 	port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;
1976 
1977 	/* Disable TX checksum offload by default due to HW bug */
1978 	if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM)
1979 		port->ndev->features &= ~NETIF_F_HW_CSUM;
1980 
1981 	ndev_priv->stats = netdev_alloc_pcpu_stats(struct am65_cpsw_ndev_stats);
1982 	if (!ndev_priv->stats)
1983 		return -ENOMEM;
1984 
1985 	ret = devm_add_action_or_reset(dev, am65_cpsw_pcpu_stats_free,
1986 				       ndev_priv->stats);
1987 	if (ret)
1988 		dev_err(dev, "failed to add percpu stat free action %d\n", ret);
1989 
1990 	if (!common->dma_ndev)
1991 		common->dma_ndev = port->ndev;
1992 
1993 	return ret;
1994 }
1995 
1996 static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common)
1997 {
1998 	int ret;
1999 	int i;
2000 
2001 	for (i = 0; i < common->port_num; i++) {
2002 		ret = am65_cpsw_nuss_init_port_ndev(common, i);
2003 		if (ret)
2004 			return ret;
2005 	}
2006 
2007 	netif_napi_add(common->dma_ndev, &common->napi_rx,
2008 		       am65_cpsw_nuss_rx_poll, NAPI_POLL_WEIGHT);
2009 
2010 	return ret;
2011 }
2012 
2013 static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common)
2014 {
2015 	struct device *dev = common->dev;
2016 	int i, ret = 0;
2017 
2018 	for (i = 0; i < common->tx_ch_num; i++) {
2019 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2020 
2021 		netif_tx_napi_add(common->dma_ndev, &tx_chn->napi_tx,
2022 				  am65_cpsw_nuss_tx_poll, NAPI_POLL_WEIGHT);
2023 
2024 		ret = devm_request_irq(dev, tx_chn->irq,
2025 				       am65_cpsw_nuss_tx_irq,
2026 				       IRQF_TRIGGER_HIGH,
2027 				       tx_chn->tx_chn_name, tx_chn);
2028 		if (ret) {
2029 			dev_err(dev, "failure requesting tx%u irq %u, %d\n",
2030 				tx_chn->id, tx_chn->irq, ret);
2031 			goto err;
2032 		}
2033 	}
2034 
2035 err:
2036 	return ret;
2037 }
2038 
2039 static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common)
2040 {
2041 	struct am65_cpsw_port *port;
2042 	int i;
2043 
2044 	for (i = 0; i < common->port_num; i++) {
2045 		port = &common->ports[i];
2046 		if (port->ndev)
2047 			unregister_netdev(port->ndev);
2048 	}
2049 }
2050 
2051 static void am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common *common)
2052 {
2053 	int set_val = 0;
2054 	int i;
2055 
2056 	if (common->br_members == (GENMASK(common->port_num, 1) & ~common->disabled_ports_mask))
2057 		set_val = 1;
2058 
2059 	dev_dbg(common->dev, "set offload_fwd_mark %d\n", set_val);
2060 
2061 	for (i = 1; i <= common->port_num; i++) {
2062 		struct am65_cpsw_port *port = am65_common_get_port(common, i);
2063 		struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(port->ndev);
2064 
2065 		priv->offload_fwd_mark = set_val;
2066 	}
2067 }
2068 
2069 bool am65_cpsw_port_dev_check(const struct net_device *ndev)
2070 {
2071 	if (ndev->netdev_ops == &am65_cpsw_nuss_netdev_ops) {
2072 		struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2073 
2074 		return !common->is_emac_mode;
2075 	}
2076 
2077 	return false;
2078 }
2079 
2080 static int am65_cpsw_netdevice_port_link(struct net_device *ndev, struct net_device *br_ndev)
2081 {
2082 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2083 	struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2084 
2085 	if (!common->br_members) {
2086 		common->hw_bridge_dev = br_ndev;
2087 	} else {
2088 		/* This is adding the port to a second bridge, this is
2089 		 * unsupported
2090 		 */
2091 		if (common->hw_bridge_dev != br_ndev)
2092 			return -EOPNOTSUPP;
2093 	}
2094 
2095 	common->br_members |= BIT(priv->port->port_id);
2096 
2097 	am65_cpsw_port_offload_fwd_mark_update(common);
2098 
2099 	return NOTIFY_DONE;
2100 }
2101 
2102 static void am65_cpsw_netdevice_port_unlink(struct net_device *ndev)
2103 {
2104 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2105 	struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2106 
2107 	common->br_members &= ~BIT(priv->port->port_id);
2108 
2109 	am65_cpsw_port_offload_fwd_mark_update(common);
2110 
2111 	if (!common->br_members)
2112 		common->hw_bridge_dev = NULL;
2113 }
2114 
2115 /* netdev notifier */
2116 static int am65_cpsw_netdevice_event(struct notifier_block *unused,
2117 				     unsigned long event, void *ptr)
2118 {
2119 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
2120 	struct netdev_notifier_changeupper_info *info;
2121 	int ret = NOTIFY_DONE;
2122 
2123 	if (!am65_cpsw_port_dev_check(ndev))
2124 		return NOTIFY_DONE;
2125 
2126 	switch (event) {
2127 	case NETDEV_CHANGEUPPER:
2128 		info = ptr;
2129 
2130 		if (netif_is_bridge_master(info->upper_dev)) {
2131 			if (info->linking)
2132 				ret = am65_cpsw_netdevice_port_link(ndev, info->upper_dev);
2133 			else
2134 				am65_cpsw_netdevice_port_unlink(ndev);
2135 		}
2136 		break;
2137 	default:
2138 		return NOTIFY_DONE;
2139 	}
2140 
2141 	return notifier_from_errno(ret);
2142 }
2143 
2144 static int am65_cpsw_register_notifiers(struct am65_cpsw_common *cpsw)
2145 {
2146 	int ret = 0;
2147 
2148 	if (AM65_CPSW_IS_CPSW2G(cpsw) ||
2149 	    !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2150 		return 0;
2151 
2152 	cpsw->am65_cpsw_netdevice_nb.notifier_call = &am65_cpsw_netdevice_event;
2153 	ret = register_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2154 	if (ret) {
2155 		dev_err(cpsw->dev, "can't register netdevice notifier\n");
2156 		return ret;
2157 	}
2158 
2159 	ret = am65_cpsw_switchdev_register_notifiers(cpsw);
2160 	if (ret)
2161 		unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2162 
2163 	return ret;
2164 }
2165 
2166 static void am65_cpsw_unregister_notifiers(struct am65_cpsw_common *cpsw)
2167 {
2168 	if (AM65_CPSW_IS_CPSW2G(cpsw) ||
2169 	    !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2170 		return;
2171 
2172 	am65_cpsw_switchdev_unregister_notifiers(cpsw);
2173 	unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2174 }
2175 
2176 static const struct devlink_ops am65_cpsw_devlink_ops = {};
2177 
2178 static void am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common *cpsw)
2179 {
2180 	cpsw_ale_add_mcast(cpsw->ale, eth_stp_addr, ALE_PORT_HOST, ALE_SUPER, 0,
2181 			   ALE_MCAST_BLOCK_LEARN_FWD);
2182 }
2183 
2184 static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common)
2185 {
2186 	struct am65_cpsw_host *host = am65_common_get_host(common);
2187 
2188 	writel(common->default_vlan, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2189 
2190 	am65_cpsw_init_stp_ale_entry(common);
2191 
2192 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 1);
2193 	dev_dbg(common->dev, "Set P0_UNI_FLOOD\n");
2194 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 0);
2195 }
2196 
2197 static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common)
2198 {
2199 	struct am65_cpsw_host *host = am65_common_get_host(common);
2200 
2201 	writel(0, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2202 
2203 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 0);
2204 	dev_dbg(common->dev, "unset P0_UNI_FLOOD\n");
2205 
2206 	/* learning make no sense in multi-mac mode */
2207 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 1);
2208 }
2209 
2210 static int am65_cpsw_dl_switch_mode_get(struct devlink *dl, u32 id,
2211 					struct devlink_param_gset_ctx *ctx)
2212 {
2213 	struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
2214 	struct am65_cpsw_common *common = dl_priv->common;
2215 
2216 	dev_dbg(common->dev, "%s id:%u\n", __func__, id);
2217 
2218 	if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
2219 		return -EOPNOTSUPP;
2220 
2221 	ctx->val.vbool = !common->is_emac_mode;
2222 
2223 	return 0;
2224 }
2225 
2226 static void am65_cpsw_init_port_emac_ale(struct  am65_cpsw_port *port)
2227 {
2228 	struct am65_cpsw_slave_data *slave = &port->slave;
2229 	struct am65_cpsw_common *common = port->common;
2230 	u32 port_mask;
2231 
2232 	writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2233 
2234 	if (slave->mac_only)
2235 		/* enable mac-only mode on port */
2236 		cpsw_ale_control_set(common->ale, port->port_id,
2237 				     ALE_PORT_MACONLY, 1);
2238 
2239 	cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_NOLEARN, 1);
2240 
2241 	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
2242 
2243 	cpsw_ale_add_ucast(common->ale, port->ndev->dev_addr,
2244 			   HOST_PORT_NUM, ALE_SECURE, slave->port_vlan);
2245 	cpsw_ale_add_mcast(common->ale, port->ndev->broadcast,
2246 			   port_mask, ALE_VLAN, slave->port_vlan, ALE_MCAST_FWD_2);
2247 }
2248 
2249 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port)
2250 {
2251 	struct am65_cpsw_slave_data *slave = &port->slave;
2252 	struct am65_cpsw_common *cpsw = port->common;
2253 	u32 port_mask;
2254 
2255 	cpsw_ale_control_set(cpsw->ale, port->port_id,
2256 			     ALE_PORT_NOLEARN, 0);
2257 
2258 	cpsw_ale_add_ucast(cpsw->ale, port->ndev->dev_addr,
2259 			   HOST_PORT_NUM, ALE_SECURE | ALE_BLOCKED | ALE_VLAN,
2260 			   slave->port_vlan);
2261 
2262 	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
2263 
2264 	cpsw_ale_add_mcast(cpsw->ale, port->ndev->broadcast,
2265 			   port_mask, ALE_VLAN, slave->port_vlan,
2266 			   ALE_MCAST_FWD_2);
2267 
2268 	writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2269 
2270 	cpsw_ale_control_set(cpsw->ale, port->port_id,
2271 			     ALE_PORT_MACONLY, 0);
2272 }
2273 
2274 static int am65_cpsw_dl_switch_mode_set(struct devlink *dl, u32 id,
2275 					struct devlink_param_gset_ctx *ctx)
2276 {
2277 	struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
2278 	struct am65_cpsw_common *cpsw = dl_priv->common;
2279 	bool switch_en = ctx->val.vbool;
2280 	bool if_running = false;
2281 	int i;
2282 
2283 	dev_dbg(cpsw->dev, "%s id:%u\n", __func__, id);
2284 
2285 	if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
2286 		return -EOPNOTSUPP;
2287 
2288 	if (switch_en == !cpsw->is_emac_mode)
2289 		return 0;
2290 
2291 	if (!switch_en && cpsw->br_members) {
2292 		dev_err(cpsw->dev, "Remove ports from bridge before disabling switch mode\n");
2293 		return -EINVAL;
2294 	}
2295 
2296 	rtnl_lock();
2297 
2298 	cpsw->is_emac_mode = !switch_en;
2299 
2300 	for (i = 0; i < cpsw->port_num; i++) {
2301 		struct net_device *sl_ndev = cpsw->ports[i].ndev;
2302 
2303 		if (!sl_ndev || !netif_running(sl_ndev))
2304 			continue;
2305 
2306 		if_running = true;
2307 	}
2308 
2309 	if (!if_running) {
2310 		/* all ndevs are down */
2311 		for (i = 0; i < cpsw->port_num; i++) {
2312 			struct net_device *sl_ndev = cpsw->ports[i].ndev;
2313 			struct am65_cpsw_slave_data *slave;
2314 
2315 			if (!sl_ndev)
2316 				continue;
2317 
2318 			slave = am65_ndev_to_slave(sl_ndev);
2319 			if (switch_en)
2320 				slave->port_vlan = cpsw->default_vlan;
2321 			else
2322 				slave->port_vlan = 0;
2323 		}
2324 
2325 		goto exit;
2326 	}
2327 
2328 	cpsw_ale_control_set(cpsw->ale, 0, ALE_BYPASS, 1);
2329 	/* clean up ALE table */
2330 	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_CLEAR, 1);
2331 	cpsw_ale_control_get(cpsw->ale, HOST_PORT_NUM, ALE_AGEOUT);
2332 
2333 	if (switch_en) {
2334 		dev_info(cpsw->dev, "Enable switch mode\n");
2335 
2336 		am65_cpsw_init_host_port_switch(cpsw);
2337 
2338 		for (i = 0; i < cpsw->port_num; i++) {
2339 			struct net_device *sl_ndev = cpsw->ports[i].ndev;
2340 			struct am65_cpsw_slave_data *slave;
2341 			struct am65_cpsw_port *port;
2342 
2343 			if (!sl_ndev)
2344 				continue;
2345 
2346 			port = am65_ndev_to_port(sl_ndev);
2347 			slave = am65_ndev_to_slave(sl_ndev);
2348 			slave->port_vlan = cpsw->default_vlan;
2349 
2350 			if (netif_running(sl_ndev))
2351 				am65_cpsw_init_port_switch_ale(port);
2352 		}
2353 
2354 	} else {
2355 		dev_info(cpsw->dev, "Disable switch mode\n");
2356 
2357 		am65_cpsw_init_host_port_emac(cpsw);
2358 
2359 		for (i = 0; i < cpsw->port_num; i++) {
2360 			struct net_device *sl_ndev = cpsw->ports[i].ndev;
2361 			struct am65_cpsw_port *port;
2362 
2363 			if (!sl_ndev)
2364 				continue;
2365 
2366 			port = am65_ndev_to_port(sl_ndev);
2367 			port->slave.port_vlan = 0;
2368 			if (netif_running(sl_ndev))
2369 				am65_cpsw_init_port_emac_ale(port);
2370 		}
2371 	}
2372 	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_BYPASS, 0);
2373 exit:
2374 	rtnl_unlock();
2375 
2376 	return 0;
2377 }
2378 
2379 static const struct devlink_param am65_cpsw_devlink_params[] = {
2380 	DEVLINK_PARAM_DRIVER(AM65_CPSW_DL_PARAM_SWITCH_MODE, "switch_mode",
2381 			     DEVLINK_PARAM_TYPE_BOOL,
2382 			     BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2383 			     am65_cpsw_dl_switch_mode_get,
2384 			     am65_cpsw_dl_switch_mode_set, NULL),
2385 };
2386 
2387 static void am65_cpsw_unregister_devlink_ports(struct am65_cpsw_common *common)
2388 {
2389 	struct devlink_port *dl_port;
2390 	struct am65_cpsw_port *port;
2391 	int i;
2392 
2393 	for (i = 1; i <= common->port_num; i++) {
2394 		port = am65_common_get_port(common, i);
2395 		dl_port = &port->devlink_port;
2396 
2397 		if (dl_port->registered)
2398 			devlink_port_unregister(dl_port);
2399 	}
2400 }
2401 
2402 static int am65_cpsw_nuss_register_devlink(struct am65_cpsw_common *common)
2403 {
2404 	struct devlink_port_attrs attrs = {};
2405 	struct am65_cpsw_devlink *dl_priv;
2406 	struct device *dev = common->dev;
2407 	struct devlink_port *dl_port;
2408 	struct am65_cpsw_port *port;
2409 	int ret = 0;
2410 	int i;
2411 
2412 	common->devlink =
2413 		devlink_alloc(&am65_cpsw_devlink_ops, sizeof(*dl_priv));
2414 	if (!common->devlink)
2415 		return -ENOMEM;
2416 
2417 	dl_priv = devlink_priv(common->devlink);
2418 	dl_priv->common = common;
2419 
2420 	ret = devlink_register(common->devlink, dev);
2421 	if (ret) {
2422 		dev_err(dev, "devlink reg fail ret:%d\n", ret);
2423 		goto dl_free;
2424 	}
2425 
2426 	/* Provide devlink hook to switch mode when multiple external ports
2427 	 * are present NUSS switchdev driver is enabled.
2428 	 */
2429 	if (!AM65_CPSW_IS_CPSW2G(common) &&
2430 	    IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) {
2431 		ret = devlink_params_register(common->devlink,
2432 					      am65_cpsw_devlink_params,
2433 					      ARRAY_SIZE(am65_cpsw_devlink_params));
2434 		if (ret) {
2435 			dev_err(dev, "devlink params reg fail ret:%d\n", ret);
2436 			goto dl_unreg;
2437 		}
2438 		devlink_params_publish(common->devlink);
2439 	}
2440 
2441 	for (i = 1; i <= common->port_num; i++) {
2442 		port = am65_common_get_port(common, i);
2443 		dl_port = &port->devlink_port;
2444 
2445 		attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
2446 		attrs.phys.port_number = port->port_id;
2447 		attrs.switch_id.id_len = sizeof(resource_size_t);
2448 		memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len);
2449 		devlink_port_attrs_set(dl_port, &attrs);
2450 
2451 		ret = devlink_port_register(common->devlink, dl_port, port->port_id);
2452 		if (ret) {
2453 			dev_err(dev, "devlink_port reg fail for port %d, ret:%d\n",
2454 				port->port_id, ret);
2455 			goto dl_port_unreg;
2456 		}
2457 		devlink_port_type_eth_set(dl_port, port->ndev);
2458 	}
2459 
2460 	return ret;
2461 
2462 dl_port_unreg:
2463 	am65_cpsw_unregister_devlink_ports(common);
2464 dl_unreg:
2465 	devlink_unregister(common->devlink);
2466 dl_free:
2467 	devlink_free(common->devlink);
2468 
2469 	return ret;
2470 }
2471 
2472 static void am65_cpsw_unregister_devlink(struct am65_cpsw_common *common)
2473 {
2474 	if (!AM65_CPSW_IS_CPSW2G(common) &&
2475 	    IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) {
2476 		devlink_params_unpublish(common->devlink);
2477 		devlink_params_unregister(common->devlink, am65_cpsw_devlink_params,
2478 					  ARRAY_SIZE(am65_cpsw_devlink_params));
2479 	}
2480 
2481 	am65_cpsw_unregister_devlink_ports(common);
2482 	devlink_unregister(common->devlink);
2483 	devlink_free(common->devlink);
2484 }
2485 
2486 static int am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common *common)
2487 {
2488 	struct device *dev = common->dev;
2489 	struct am65_cpsw_port *port;
2490 	int ret = 0, i;
2491 
2492 	ret = am65_cpsw_nuss_ndev_add_tx_napi(common);
2493 	if (ret)
2494 		return ret;
2495 
2496 	ret = devm_request_irq(dev, common->rx_chns.irq,
2497 			       am65_cpsw_nuss_rx_irq,
2498 			       IRQF_TRIGGER_HIGH, dev_name(dev), common);
2499 	if (ret) {
2500 		dev_err(dev, "failure requesting rx irq %u, %d\n",
2501 			common->rx_chns.irq, ret);
2502 		return ret;
2503 	}
2504 
2505 	for (i = 0; i < common->port_num; i++) {
2506 		port = &common->ports[i];
2507 
2508 		if (!port->ndev)
2509 			continue;
2510 
2511 		ret = register_netdev(port->ndev);
2512 		if (ret) {
2513 			dev_err(dev, "error registering slave net device%i %d\n",
2514 				i, ret);
2515 			goto err_cleanup_ndev;
2516 		}
2517 	}
2518 
2519 	ret = am65_cpsw_register_notifiers(common);
2520 	if (ret)
2521 		goto err_cleanup_ndev;
2522 
2523 	ret = am65_cpsw_nuss_register_devlink(common);
2524 	if (ret)
2525 		goto clean_unregister_notifiers;
2526 
2527 	/* can't auto unregister ndev using devm_add_action() due to
2528 	 * devres release sequence in DD core for DMA
2529 	 */
2530 
2531 	return 0;
2532 clean_unregister_notifiers:
2533 	am65_cpsw_unregister_notifiers(common);
2534 err_cleanup_ndev:
2535 	am65_cpsw_nuss_cleanup_ndev(common);
2536 
2537 	return ret;
2538 }
2539 
2540 int am65_cpsw_nuss_update_tx_chns(struct am65_cpsw_common *common, int num_tx)
2541 {
2542 	int ret;
2543 
2544 	common->tx_ch_num = num_tx;
2545 	ret = am65_cpsw_nuss_init_tx_chns(common);
2546 	if (ret)
2547 		return ret;
2548 
2549 	return am65_cpsw_nuss_ndev_add_tx_napi(common);
2550 }
2551 
2552 struct am65_cpsw_soc_pdata {
2553 	u32	quirks_dis;
2554 };
2555 
2556 static const struct am65_cpsw_soc_pdata am65x_soc_sr2_0 = {
2557 	.quirks_dis = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
2558 };
2559 
2560 static const struct soc_device_attribute am65_cpsw_socinfo[] = {
2561 	{ .family = "AM65X",
2562 	  .revision = "SR2.0",
2563 	  .data = &am65x_soc_sr2_0
2564 	},
2565 	{/* sentinel */}
2566 };
2567 
2568 static const struct am65_cpsw_pdata am65x_sr1_0 = {
2569 	.quirks = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
2570 	.ale_dev_id = "am65x-cpsw2g",
2571 	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2572 };
2573 
2574 static const struct am65_cpsw_pdata j721e_pdata = {
2575 	.quirks = 0,
2576 	.ale_dev_id = "am65x-cpsw2g",
2577 	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2578 };
2579 
2580 static const struct am65_cpsw_pdata am64x_cpswxg_pdata = {
2581 	.quirks = 0,
2582 	.ale_dev_id = "am64-cpswxg",
2583 	.fdqring_mode = K3_RINGACC_RING_MODE_RING,
2584 };
2585 
2586 static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
2587 	{ .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
2588 	{ .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
2589 	{ .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
2590 	{ /* sentinel */ },
2591 };
2592 MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
2593 
2594 static void am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common *common)
2595 {
2596 	const struct soc_device_attribute *soc;
2597 
2598 	soc = soc_device_match(am65_cpsw_socinfo);
2599 	if (soc && soc->data) {
2600 		const struct am65_cpsw_soc_pdata *socdata = soc->data;
2601 
2602 		/* disable quirks */
2603 		common->pdata.quirks &= ~socdata->quirks_dis;
2604 	}
2605 }
2606 
2607 static int am65_cpsw_nuss_probe(struct platform_device *pdev)
2608 {
2609 	struct cpsw_ale_params ale_params = { 0 };
2610 	const struct of_device_id *of_id;
2611 	struct device *dev = &pdev->dev;
2612 	struct am65_cpsw_common *common;
2613 	struct device_node *node;
2614 	struct resource *res;
2615 	struct clk *clk;
2616 	u64 id_temp;
2617 	int ret, i;
2618 
2619 	common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL);
2620 	if (!common)
2621 		return -ENOMEM;
2622 	common->dev = dev;
2623 
2624 	of_id = of_match_device(am65_cpsw_nuss_of_mtable, dev);
2625 	if (!of_id)
2626 		return -EINVAL;
2627 	common->pdata = *(const struct am65_cpsw_pdata *)of_id->data;
2628 
2629 	am65_cpsw_nuss_apply_socinfo(common);
2630 
2631 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpsw_nuss");
2632 	common->ss_base = devm_ioremap_resource(&pdev->dev, res);
2633 	if (IS_ERR(common->ss_base))
2634 		return PTR_ERR(common->ss_base);
2635 	common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE;
2636 	/* Use device's physical base address as switch id */
2637 	id_temp = cpu_to_be64(res->start);
2638 	memcpy(common->switch_id, &id_temp, sizeof(res->start));
2639 
2640 	node = of_get_child_by_name(dev->of_node, "ethernet-ports");
2641 	if (!node)
2642 		return -ENOENT;
2643 	common->port_num = of_get_child_count(node);
2644 	if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS)
2645 		return -ENOENT;
2646 	of_node_put(node);
2647 
2648 	common->rx_flow_id_base = -1;
2649 	init_completion(&common->tdown_complete);
2650 	common->tx_ch_num = 1;
2651 	common->pf_p0_rx_ptype_rrobin = false;
2652 	common->default_vlan = 1;
2653 
2654 	common->ports = devm_kcalloc(dev, common->port_num,
2655 				     sizeof(*common->ports),
2656 				     GFP_KERNEL);
2657 	if (!common->ports)
2658 		return -ENOMEM;
2659 
2660 	clk = devm_clk_get(dev, "fck");
2661 	if (IS_ERR(clk))
2662 		return dev_err_probe(dev, PTR_ERR(clk), "getting fck clock\n");
2663 	common->bus_freq = clk_get_rate(clk);
2664 
2665 	pm_runtime_enable(dev);
2666 	ret = pm_runtime_get_sync(dev);
2667 	if (ret < 0) {
2668 		pm_runtime_put_noidle(dev);
2669 		pm_runtime_disable(dev);
2670 		return ret;
2671 	}
2672 
2673 	node = of_get_child_by_name(dev->of_node, "mdio");
2674 	if (!node) {
2675 		dev_warn(dev, "MDIO node not found\n");
2676 	} else if (of_device_is_available(node)) {
2677 		struct platform_device *mdio_pdev;
2678 
2679 		mdio_pdev = of_platform_device_create(node, NULL, dev);
2680 		if (!mdio_pdev) {
2681 			ret = -ENODEV;
2682 			goto err_pm_clear;
2683 		}
2684 
2685 		common->mdio_dev =  &mdio_pdev->dev;
2686 	}
2687 	of_node_put(node);
2688 
2689 	am65_cpsw_nuss_get_ver(common);
2690 
2691 	/* init tx channels */
2692 	ret = am65_cpsw_nuss_init_tx_chns(common);
2693 	if (ret)
2694 		goto err_of_clear;
2695 	ret = am65_cpsw_nuss_init_rx_chns(common);
2696 	if (ret)
2697 		goto err_of_clear;
2698 
2699 	ret = am65_cpsw_nuss_init_host_p(common);
2700 	if (ret)
2701 		goto err_of_clear;
2702 
2703 	ret = am65_cpsw_nuss_init_slave_ports(common);
2704 	if (ret)
2705 		goto err_of_clear;
2706 
2707 	/* init common data */
2708 	ale_params.dev = dev;
2709 	ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT;
2710 	ale_params.ale_ports = common->port_num + 1;
2711 	ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE;
2712 	ale_params.dev_id = common->pdata.ale_dev_id;
2713 	ale_params.bus_freq = common->bus_freq;
2714 
2715 	common->ale = cpsw_ale_create(&ale_params);
2716 	if (IS_ERR(common->ale)) {
2717 		dev_err(dev, "error initializing ale engine\n");
2718 		ret = PTR_ERR(common->ale);
2719 		goto err_of_clear;
2720 	}
2721 
2722 	ret = am65_cpsw_init_cpts(common);
2723 	if (ret)
2724 		goto err_of_clear;
2725 
2726 	/* init ports */
2727 	for (i = 0; i < common->port_num; i++)
2728 		am65_cpsw_nuss_slave_disable_unused(&common->ports[i]);
2729 
2730 	dev_set_drvdata(dev, common);
2731 
2732 	common->is_emac_mode = true;
2733 
2734 	ret = am65_cpsw_nuss_init_ndevs(common);
2735 	if (ret)
2736 		goto err_of_clear;
2737 
2738 	ret = am65_cpsw_nuss_register_ndevs(common);
2739 	if (ret)
2740 		goto err_of_clear;
2741 
2742 	pm_runtime_put(dev);
2743 	return 0;
2744 
2745 err_of_clear:
2746 	of_platform_device_destroy(common->mdio_dev, NULL);
2747 err_pm_clear:
2748 	pm_runtime_put_sync(dev);
2749 	pm_runtime_disable(dev);
2750 	return ret;
2751 }
2752 
2753 static int am65_cpsw_nuss_remove(struct platform_device *pdev)
2754 {
2755 	struct device *dev = &pdev->dev;
2756 	struct am65_cpsw_common *common;
2757 	int ret;
2758 
2759 	common = dev_get_drvdata(dev);
2760 
2761 	ret = pm_runtime_get_sync(&pdev->dev);
2762 	if (ret < 0) {
2763 		pm_runtime_put_noidle(&pdev->dev);
2764 		return ret;
2765 	}
2766 
2767 	am65_cpsw_unregister_devlink(common);
2768 	am65_cpsw_unregister_notifiers(common);
2769 
2770 	/* must unregister ndevs here because DD release_driver routine calls
2771 	 * dma_deconfigure(dev) before devres_release_all(dev)
2772 	 */
2773 	am65_cpsw_nuss_cleanup_ndev(common);
2774 
2775 	of_platform_device_destroy(common->mdio_dev, NULL);
2776 
2777 	pm_runtime_put_sync(&pdev->dev);
2778 	pm_runtime_disable(&pdev->dev);
2779 	return 0;
2780 }
2781 
2782 static struct platform_driver am65_cpsw_nuss_driver = {
2783 	.driver = {
2784 		.name	 = AM65_CPSW_DRV_NAME,
2785 		.of_match_table = am65_cpsw_nuss_of_mtable,
2786 	},
2787 	.probe = am65_cpsw_nuss_probe,
2788 	.remove = am65_cpsw_nuss_remove,
2789 };
2790 
2791 module_platform_driver(am65_cpsw_nuss_driver);
2792 
2793 MODULE_LICENSE("GPL v2");
2794 MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
2795 MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver");
2796