1 // SPDX-License-Identifier: GPL-2.0+
2 
3 #include <linux/types.h>
4 #include <linux/clk.h>
5 #include <linux/platform_device.h>
6 #include <linux/pm_runtime.h>
7 #include <linux/acpi.h>
8 #include <linux/of_mdio.h>
9 #include <linux/etherdevice.h>
10 #include <linux/interrupt.h>
11 #include <linux/io.h>
12 #include <linux/netlink.h>
13 #include <linux/bpf.h>
14 #include <linux/bpf_trace.h>
15 
16 #include <net/tcp.h>
17 #include <net/page_pool.h>
18 #include <net/ip6_checksum.h>
19 
20 #define NETSEC_REG_SOFT_RST			0x104
21 #define NETSEC_REG_COM_INIT			0x120
22 
23 #define NETSEC_REG_TOP_STATUS			0x200
24 #define NETSEC_IRQ_RX				BIT(1)
25 #define NETSEC_IRQ_TX				BIT(0)
26 
27 #define NETSEC_REG_TOP_INTEN			0x204
28 #define NETSEC_REG_INTEN_SET			0x234
29 #define NETSEC_REG_INTEN_CLR			0x238
30 
31 #define NETSEC_REG_NRM_TX_STATUS		0x400
32 #define NETSEC_REG_NRM_TX_INTEN			0x404
33 #define NETSEC_REG_NRM_TX_INTEN_SET		0x428
34 #define NETSEC_REG_NRM_TX_INTEN_CLR		0x42c
35 #define NRM_TX_ST_NTOWNR	BIT(17)
36 #define NRM_TX_ST_TR_ERR	BIT(16)
37 #define NRM_TX_ST_TXDONE	BIT(15)
38 #define NRM_TX_ST_TMREXP	BIT(14)
39 
40 #define NETSEC_REG_NRM_RX_STATUS		0x440
41 #define NETSEC_REG_NRM_RX_INTEN			0x444
42 #define NETSEC_REG_NRM_RX_INTEN_SET		0x468
43 #define NETSEC_REG_NRM_RX_INTEN_CLR		0x46c
44 #define NRM_RX_ST_RC_ERR	BIT(16)
45 #define NRM_RX_ST_PKTCNT	BIT(15)
46 #define NRM_RX_ST_TMREXP	BIT(14)
47 
48 #define NETSEC_REG_PKT_CMD_BUF			0xd0
49 
50 #define NETSEC_REG_CLK_EN			0x100
51 
52 #define NETSEC_REG_PKT_CTRL			0x140
53 
54 #define NETSEC_REG_DMA_TMR_CTRL			0x20c
55 #define NETSEC_REG_F_TAIKI_MC_VER		0x22c
56 #define NETSEC_REG_F_TAIKI_VER			0x230
57 #define NETSEC_REG_DMA_HM_CTRL			0x214
58 #define NETSEC_REG_DMA_MH_CTRL			0x220
59 #define NETSEC_REG_ADDR_DIS_CORE		0x218
60 #define NETSEC_REG_DMAC_HM_CMD_BUF		0x210
61 #define NETSEC_REG_DMAC_MH_CMD_BUF		0x21c
62 
63 #define NETSEC_REG_NRM_TX_PKTCNT		0x410
64 
65 #define NETSEC_REG_NRM_TX_DONE_PKTCNT		0x414
66 #define NETSEC_REG_NRM_TX_DONE_TXINT_PKTCNT	0x418
67 
68 #define NETSEC_REG_NRM_TX_TMR			0x41c
69 
70 #define NETSEC_REG_NRM_RX_PKTCNT		0x454
71 #define NETSEC_REG_NRM_RX_RXINT_PKTCNT		0x458
72 #define NETSEC_REG_NRM_TX_TXINT_TMR		0x420
73 #define NETSEC_REG_NRM_RX_RXINT_TMR		0x460
74 
75 #define NETSEC_REG_NRM_RX_TMR			0x45c
76 
77 #define NETSEC_REG_NRM_TX_DESC_START_UP		0x434
78 #define NETSEC_REG_NRM_TX_DESC_START_LW		0x408
79 #define NETSEC_REG_NRM_RX_DESC_START_UP		0x474
80 #define NETSEC_REG_NRM_RX_DESC_START_LW		0x448
81 
82 #define NETSEC_REG_NRM_TX_CONFIG		0x430
83 #define NETSEC_REG_NRM_RX_CONFIG		0x470
84 
85 #define MAC_REG_STATUS				0x1024
86 #define MAC_REG_DATA				0x11c0
87 #define MAC_REG_CMD				0x11c4
88 #define MAC_REG_FLOW_TH				0x11cc
89 #define MAC_REG_INTF_SEL			0x11d4
90 #define MAC_REG_DESC_INIT			0x11fc
91 #define MAC_REG_DESC_SOFT_RST			0x1204
92 #define NETSEC_REG_MODE_TRANS_COMP_STATUS	0x500
93 
94 #define GMAC_REG_MCR				0x0000
95 #define GMAC_REG_MFFR				0x0004
96 #define GMAC_REG_GAR				0x0010
97 #define GMAC_REG_GDR				0x0014
98 #define GMAC_REG_FCR				0x0018
99 #define GMAC_REG_BMR				0x1000
100 #define GMAC_REG_RDLAR				0x100c
101 #define GMAC_REG_TDLAR				0x1010
102 #define GMAC_REG_OMR				0x1018
103 
104 #define MHZ(n)		((n) * 1000 * 1000)
105 
106 #define NETSEC_TX_SHIFT_OWN_FIELD		31
107 #define NETSEC_TX_SHIFT_LD_FIELD		30
108 #define NETSEC_TX_SHIFT_DRID_FIELD		24
109 #define NETSEC_TX_SHIFT_PT_FIELD		21
110 #define NETSEC_TX_SHIFT_TDRID_FIELD		16
111 #define NETSEC_TX_SHIFT_CC_FIELD		15
112 #define NETSEC_TX_SHIFT_FS_FIELD		9
113 #define NETSEC_TX_LAST				8
114 #define NETSEC_TX_SHIFT_CO			7
115 #define NETSEC_TX_SHIFT_SO			6
116 #define NETSEC_TX_SHIFT_TRS_FIELD		4
117 
118 #define NETSEC_RX_PKT_OWN_FIELD			31
119 #define NETSEC_RX_PKT_LD_FIELD			30
120 #define NETSEC_RX_PKT_SDRID_FIELD		24
121 #define NETSEC_RX_PKT_FR_FIELD			23
122 #define NETSEC_RX_PKT_ER_FIELD			21
123 #define NETSEC_RX_PKT_ERR_FIELD			16
124 #define NETSEC_RX_PKT_TDRID_FIELD		12
125 #define NETSEC_RX_PKT_FS_FIELD			9
126 #define NETSEC_RX_PKT_LS_FIELD			8
127 #define NETSEC_RX_PKT_CO_FIELD			6
128 
129 #define NETSEC_RX_PKT_ERR_MASK			3
130 
131 #define NETSEC_MAX_TX_PKT_LEN			1518
132 #define NETSEC_MAX_TX_JUMBO_PKT_LEN		9018
133 
134 #define NETSEC_RING_GMAC			15
135 #define NETSEC_RING_MAX				2
136 
137 #define NETSEC_TCP_SEG_LEN_MAX			1460
138 #define NETSEC_TCP_JUMBO_SEG_LEN_MAX		8960
139 
140 #define NETSEC_RX_CKSUM_NOTAVAIL		0
141 #define NETSEC_RX_CKSUM_OK			1
142 #define NETSEC_RX_CKSUM_NG			2
143 
144 #define NETSEC_TOP_IRQ_REG_CODE_LOAD_END	BIT(20)
145 #define NETSEC_IRQ_TRANSITION_COMPLETE		BIT(4)
146 
147 #define NETSEC_MODE_TRANS_COMP_IRQ_N2T		BIT(20)
148 #define NETSEC_MODE_TRANS_COMP_IRQ_T2N		BIT(19)
149 
150 #define NETSEC_INT_PKTCNT_MAX			2047
151 
152 #define NETSEC_FLOW_START_TH_MAX		95
153 #define NETSEC_FLOW_STOP_TH_MAX			95
154 #define NETSEC_FLOW_PAUSE_TIME_MIN		5
155 
156 #define NETSEC_CLK_EN_REG_DOM_ALL		0x3f
157 
158 #define NETSEC_PKT_CTRL_REG_MODE_NRM		BIT(28)
159 #define NETSEC_PKT_CTRL_REG_EN_JUMBO		BIT(27)
160 #define NETSEC_PKT_CTRL_REG_LOG_CHKSUM_ER	BIT(3)
161 #define NETSEC_PKT_CTRL_REG_LOG_HD_INCOMPLETE	BIT(2)
162 #define NETSEC_PKT_CTRL_REG_LOG_HD_ER		BIT(1)
163 #define NETSEC_PKT_CTRL_REG_DRP_NO_MATCH	BIT(0)
164 
165 #define NETSEC_CLK_EN_REG_DOM_G			BIT(5)
166 #define NETSEC_CLK_EN_REG_DOM_C			BIT(1)
167 #define NETSEC_CLK_EN_REG_DOM_D			BIT(0)
168 
169 #define NETSEC_COM_INIT_REG_DB			BIT(2)
170 #define NETSEC_COM_INIT_REG_CLS			BIT(1)
171 #define NETSEC_COM_INIT_REG_ALL			(NETSEC_COM_INIT_REG_CLS | \
172 						 NETSEC_COM_INIT_REG_DB)
173 
174 #define NETSEC_SOFT_RST_REG_RESET		0
175 #define NETSEC_SOFT_RST_REG_RUN			BIT(31)
176 
177 #define NETSEC_DMA_CTRL_REG_STOP		1
178 #define MH_CTRL__MODE_TRANS			BIT(20)
179 
180 #define NETSEC_GMAC_CMD_ST_READ			0
181 #define NETSEC_GMAC_CMD_ST_WRITE		BIT(28)
182 #define NETSEC_GMAC_CMD_ST_BUSY			BIT(31)
183 
184 #define NETSEC_GMAC_BMR_REG_COMMON		0x00412080
185 #define NETSEC_GMAC_BMR_REG_RESET		0x00020181
186 #define NETSEC_GMAC_BMR_REG_SWR			0x00000001
187 
188 #define NETSEC_GMAC_OMR_REG_ST			BIT(13)
189 #define NETSEC_GMAC_OMR_REG_SR			BIT(1)
190 
191 #define NETSEC_GMAC_MCR_REG_IBN			BIT(30)
192 #define NETSEC_GMAC_MCR_REG_CST			BIT(25)
193 #define NETSEC_GMAC_MCR_REG_JE			BIT(20)
194 #define NETSEC_MCR_PS				BIT(15)
195 #define NETSEC_GMAC_MCR_REG_FES			BIT(14)
196 #define NETSEC_GMAC_MCR_REG_FULL_DUPLEX_COMMON	0x0000280c
197 #define NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON	0x0001a00c
198 
199 #define NETSEC_FCR_RFE				BIT(2)
200 #define NETSEC_FCR_TFE				BIT(1)
201 
202 #define NETSEC_GMAC_GAR_REG_GW			BIT(1)
203 #define NETSEC_GMAC_GAR_REG_GB			BIT(0)
204 
205 #define NETSEC_GMAC_GAR_REG_SHIFT_PA		11
206 #define NETSEC_GMAC_GAR_REG_SHIFT_GR		6
207 #define GMAC_REG_SHIFT_CR_GAR			2
208 
209 #define NETSEC_GMAC_GAR_REG_CR_25_35_MHZ	2
210 #define NETSEC_GMAC_GAR_REG_CR_35_60_MHZ	3
211 #define NETSEC_GMAC_GAR_REG_CR_60_100_MHZ	0
212 #define NETSEC_GMAC_GAR_REG_CR_100_150_MHZ	1
213 #define NETSEC_GMAC_GAR_REG_CR_150_250_MHZ	4
214 #define NETSEC_GMAC_GAR_REG_CR_250_300_MHZ	5
215 
216 #define NETSEC_GMAC_RDLAR_REG_COMMON		0x18000
217 #define NETSEC_GMAC_TDLAR_REG_COMMON		0x1c000
218 
219 #define NETSEC_REG_NETSEC_VER_F_TAIKI		0x50000
220 
221 #define NETSEC_REG_DESC_RING_CONFIG_CFG_UP	BIT(31)
222 #define NETSEC_REG_DESC_RING_CONFIG_CH_RST	BIT(30)
223 #define NETSEC_REG_DESC_TMR_MODE		4
224 #define NETSEC_REG_DESC_ENDIAN			0
225 
226 #define NETSEC_MAC_DESC_SOFT_RST_SOFT_RST	1
227 #define NETSEC_MAC_DESC_INIT_REG_INIT		1
228 
229 #define NETSEC_EEPROM_MAC_ADDRESS		0x00
230 #define NETSEC_EEPROM_HM_ME_ADDRESS_H		0x08
231 #define NETSEC_EEPROM_HM_ME_ADDRESS_L		0x0C
232 #define NETSEC_EEPROM_HM_ME_SIZE		0x10
233 #define NETSEC_EEPROM_MH_ME_ADDRESS_H		0x14
234 #define NETSEC_EEPROM_MH_ME_ADDRESS_L		0x18
235 #define NETSEC_EEPROM_MH_ME_SIZE		0x1C
236 #define NETSEC_EEPROM_PKT_ME_ADDRESS		0x20
237 #define NETSEC_EEPROM_PKT_ME_SIZE		0x24
238 
239 #define DESC_NUM	256
240 
241 #define NETSEC_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN)
242 #define NETSEC_RXBUF_HEADROOM (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \
243 			       NET_IP_ALIGN)
244 #define NETSEC_RX_BUF_NON_DATA (NETSEC_RXBUF_HEADROOM + \
245 				SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
246 #define NETSEC_RX_BUF_SIZE	(PAGE_SIZE - NETSEC_RX_BUF_NON_DATA)
247 
248 #define DESC_SZ	sizeof(struct netsec_de)
249 
250 #define NETSEC_F_NETSEC_VER_MAJOR_NUM(x)	((x) & 0xffff0000)
251 
252 #define NETSEC_XDP_PASS          0
253 #define NETSEC_XDP_CONSUMED      BIT(0)
254 #define NETSEC_XDP_TX            BIT(1)
255 #define NETSEC_XDP_REDIR         BIT(2)
256 
257 enum ring_id {
258 	NETSEC_RING_TX = 0,
259 	NETSEC_RING_RX
260 };
261 
262 enum buf_type {
263 	TYPE_NETSEC_SKB = 0,
264 	TYPE_NETSEC_XDP_TX,
265 	TYPE_NETSEC_XDP_NDO,
266 };
267 
268 struct netsec_desc {
269 	union {
270 		struct sk_buff *skb;
271 		struct xdp_frame *xdpf;
272 	};
273 	dma_addr_t dma_addr;
274 	void *addr;
275 	u16 len;
276 	u8 buf_type;
277 };
278 
279 struct netsec_desc_ring {
280 	dma_addr_t desc_dma;
281 	struct netsec_desc *desc;
282 	void *vaddr;
283 	u16 head, tail;
284 	u16 xdp_xmit; /* netsec_xdp_xmit packets */
285 	struct page_pool *page_pool;
286 	struct xdp_rxq_info xdp_rxq;
287 	spinlock_t lock; /* XDP tx queue locking */
288 };
289 
290 struct netsec_priv {
291 	struct netsec_desc_ring desc_ring[NETSEC_RING_MAX];
292 	struct ethtool_coalesce et_coalesce;
293 	struct bpf_prog *xdp_prog;
294 	spinlock_t reglock; /* protect reg access */
295 	struct napi_struct napi;
296 	phy_interface_t phy_interface;
297 	struct net_device *ndev;
298 	struct device_node *phy_np;
299 	struct phy_device *phydev;
300 	struct mii_bus *mii_bus;
301 	void __iomem *ioaddr;
302 	void __iomem *eeprom_base;
303 	struct device *dev;
304 	struct clk *clk;
305 	u32 msg_enable;
306 	u32 freq;
307 	u32 phy_addr;
308 	bool rx_cksum_offload_flag;
309 };
310 
311 struct netsec_de { /* Netsec Descriptor layout */
312 	u32 attr;
313 	u32 data_buf_addr_up;
314 	u32 data_buf_addr_lw;
315 	u32 buf_len_info;
316 };
317 
318 struct netsec_tx_pkt_ctrl {
319 	u16 tcp_seg_len;
320 	bool tcp_seg_offload_flag;
321 	bool cksum_offload_flag;
322 };
323 
324 struct netsec_rx_pkt_info {
325 	int rx_cksum_result;
326 	int err_code;
327 	bool err_flag;
328 };
329 
330 static void netsec_write(struct netsec_priv *priv, u32 reg_addr, u32 val)
331 {
332 	writel(val, priv->ioaddr + reg_addr);
333 }
334 
335 static u32 netsec_read(struct netsec_priv *priv, u32 reg_addr)
336 {
337 	return readl(priv->ioaddr + reg_addr);
338 }
339 
340 /************* MDIO BUS OPS FOLLOW *************/
341 
342 #define TIMEOUT_SPINS_MAC		1000
343 #define TIMEOUT_SECONDARY_MS_MAC	100
344 
345 static u32 netsec_clk_type(u32 freq)
346 {
347 	if (freq < MHZ(35))
348 		return NETSEC_GMAC_GAR_REG_CR_25_35_MHZ;
349 	if (freq < MHZ(60))
350 		return NETSEC_GMAC_GAR_REG_CR_35_60_MHZ;
351 	if (freq < MHZ(100))
352 		return NETSEC_GMAC_GAR_REG_CR_60_100_MHZ;
353 	if (freq < MHZ(150))
354 		return NETSEC_GMAC_GAR_REG_CR_100_150_MHZ;
355 	if (freq < MHZ(250))
356 		return NETSEC_GMAC_GAR_REG_CR_150_250_MHZ;
357 
358 	return NETSEC_GMAC_GAR_REG_CR_250_300_MHZ;
359 }
360 
361 static int netsec_wait_while_busy(struct netsec_priv *priv, u32 addr, u32 mask)
362 {
363 	u32 timeout = TIMEOUT_SPINS_MAC;
364 
365 	while (--timeout && netsec_read(priv, addr) & mask)
366 		cpu_relax();
367 	if (timeout)
368 		return 0;
369 
370 	timeout = TIMEOUT_SECONDARY_MS_MAC;
371 	while (--timeout && netsec_read(priv, addr) & mask)
372 		usleep_range(1000, 2000);
373 
374 	if (timeout)
375 		return 0;
376 
377 	netdev_WARN(priv->ndev, "%s: timeout\n", __func__);
378 
379 	return -ETIMEDOUT;
380 }
381 
382 static int netsec_mac_write(struct netsec_priv *priv, u32 addr, u32 value)
383 {
384 	netsec_write(priv, MAC_REG_DATA, value);
385 	netsec_write(priv, MAC_REG_CMD, addr | NETSEC_GMAC_CMD_ST_WRITE);
386 	return netsec_wait_while_busy(priv,
387 				      MAC_REG_CMD, NETSEC_GMAC_CMD_ST_BUSY);
388 }
389 
390 static int netsec_mac_read(struct netsec_priv *priv, u32 addr, u32 *read)
391 {
392 	int ret;
393 
394 	netsec_write(priv, MAC_REG_CMD, addr | NETSEC_GMAC_CMD_ST_READ);
395 	ret = netsec_wait_while_busy(priv,
396 				     MAC_REG_CMD, NETSEC_GMAC_CMD_ST_BUSY);
397 	if (ret)
398 		return ret;
399 
400 	*read = netsec_read(priv, MAC_REG_DATA);
401 
402 	return 0;
403 }
404 
405 static int netsec_mac_wait_while_busy(struct netsec_priv *priv,
406 				      u32 addr, u32 mask)
407 {
408 	u32 timeout = TIMEOUT_SPINS_MAC;
409 	int ret, data;
410 
411 	do {
412 		ret = netsec_mac_read(priv, addr, &data);
413 		if (ret)
414 			break;
415 		cpu_relax();
416 	} while (--timeout && (data & mask));
417 
418 	if (timeout)
419 		return 0;
420 
421 	timeout = TIMEOUT_SECONDARY_MS_MAC;
422 	do {
423 		usleep_range(1000, 2000);
424 
425 		ret = netsec_mac_read(priv, addr, &data);
426 		if (ret)
427 			break;
428 		cpu_relax();
429 	} while (--timeout && (data & mask));
430 
431 	if (timeout && !ret)
432 		return 0;
433 
434 	netdev_WARN(priv->ndev, "%s: timeout\n", __func__);
435 
436 	return -ETIMEDOUT;
437 }
438 
439 static int netsec_mac_update_to_phy_state(struct netsec_priv *priv)
440 {
441 	struct phy_device *phydev = priv->ndev->phydev;
442 	u32 value = 0;
443 
444 	value = phydev->duplex ? NETSEC_GMAC_MCR_REG_FULL_DUPLEX_COMMON :
445 				 NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON;
446 
447 	if (phydev->speed != SPEED_1000)
448 		value |= NETSEC_MCR_PS;
449 
450 	if (priv->phy_interface != PHY_INTERFACE_MODE_GMII &&
451 	    phydev->speed == SPEED_100)
452 		value |= NETSEC_GMAC_MCR_REG_FES;
453 
454 	value |= NETSEC_GMAC_MCR_REG_CST | NETSEC_GMAC_MCR_REG_JE;
455 
456 	if (phy_interface_mode_is_rgmii(priv->phy_interface))
457 		value |= NETSEC_GMAC_MCR_REG_IBN;
458 
459 	if (netsec_mac_write(priv, GMAC_REG_MCR, value))
460 		return -ETIMEDOUT;
461 
462 	return 0;
463 }
464 
465 static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr);
466 
467 static int netsec_phy_write(struct mii_bus *bus,
468 			    int phy_addr, int reg, u16 val)
469 {
470 	int status;
471 	struct netsec_priv *priv = bus->priv;
472 
473 	if (netsec_mac_write(priv, GMAC_REG_GDR, val))
474 		return -ETIMEDOUT;
475 	if (netsec_mac_write(priv, GMAC_REG_GAR,
476 			     phy_addr << NETSEC_GMAC_GAR_REG_SHIFT_PA |
477 			     reg << NETSEC_GMAC_GAR_REG_SHIFT_GR |
478 			     NETSEC_GMAC_GAR_REG_GW | NETSEC_GMAC_GAR_REG_GB |
479 			     (netsec_clk_type(priv->freq) <<
480 			      GMAC_REG_SHIFT_CR_GAR)))
481 		return -ETIMEDOUT;
482 
483 	status = netsec_mac_wait_while_busy(priv, GMAC_REG_GAR,
484 					    NETSEC_GMAC_GAR_REG_GB);
485 
486 	/* Developerbox implements RTL8211E PHY and there is
487 	 * a compatibility problem with F_GMAC4.
488 	 * RTL8211E expects MDC clock must be kept toggling for several
489 	 * clock cycle with MDIO high before entering the IDLE state.
490 	 * To meet this requirement, netsec driver needs to issue dummy
491 	 * read(e.g. read PHYID1(offset 0x2) register) right after write.
492 	 */
493 	netsec_phy_read(bus, phy_addr, MII_PHYSID1);
494 
495 	return status;
496 }
497 
498 static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr)
499 {
500 	struct netsec_priv *priv = bus->priv;
501 	u32 data;
502 	int ret;
503 
504 	if (netsec_mac_write(priv, GMAC_REG_GAR, NETSEC_GMAC_GAR_REG_GB |
505 			     phy_addr << NETSEC_GMAC_GAR_REG_SHIFT_PA |
506 			     reg_addr << NETSEC_GMAC_GAR_REG_SHIFT_GR |
507 			     (netsec_clk_type(priv->freq) <<
508 			      GMAC_REG_SHIFT_CR_GAR)))
509 		return -ETIMEDOUT;
510 
511 	ret = netsec_mac_wait_while_busy(priv, GMAC_REG_GAR,
512 					 NETSEC_GMAC_GAR_REG_GB);
513 	if (ret)
514 		return ret;
515 
516 	ret = netsec_mac_read(priv, GMAC_REG_GDR, &data);
517 	if (ret)
518 		return ret;
519 
520 	return data;
521 }
522 
523 /************* ETHTOOL_OPS FOLLOW *************/
524 
525 static void netsec_et_get_drvinfo(struct net_device *net_device,
526 				  struct ethtool_drvinfo *info)
527 {
528 	strlcpy(info->driver, "netsec", sizeof(info->driver));
529 	strlcpy(info->bus_info, dev_name(net_device->dev.parent),
530 		sizeof(info->bus_info));
531 }
532 
533 static int netsec_et_get_coalesce(struct net_device *net_device,
534 				  struct ethtool_coalesce *et_coalesce)
535 {
536 	struct netsec_priv *priv = netdev_priv(net_device);
537 
538 	*et_coalesce = priv->et_coalesce;
539 
540 	return 0;
541 }
542 
543 static int netsec_et_set_coalesce(struct net_device *net_device,
544 				  struct ethtool_coalesce *et_coalesce)
545 {
546 	struct netsec_priv *priv = netdev_priv(net_device);
547 
548 	priv->et_coalesce = *et_coalesce;
549 
550 	if (priv->et_coalesce.tx_coalesce_usecs < 50)
551 		priv->et_coalesce.tx_coalesce_usecs = 50;
552 	if (priv->et_coalesce.tx_max_coalesced_frames < 1)
553 		priv->et_coalesce.tx_max_coalesced_frames = 1;
554 
555 	netsec_write(priv, NETSEC_REG_NRM_TX_DONE_TXINT_PKTCNT,
556 		     priv->et_coalesce.tx_max_coalesced_frames);
557 	netsec_write(priv, NETSEC_REG_NRM_TX_TXINT_TMR,
558 		     priv->et_coalesce.tx_coalesce_usecs);
559 	netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_SET, NRM_TX_ST_TXDONE);
560 	netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_SET, NRM_TX_ST_TMREXP);
561 
562 	if (priv->et_coalesce.rx_coalesce_usecs < 50)
563 		priv->et_coalesce.rx_coalesce_usecs = 50;
564 	if (priv->et_coalesce.rx_max_coalesced_frames < 1)
565 		priv->et_coalesce.rx_max_coalesced_frames = 1;
566 
567 	netsec_write(priv, NETSEC_REG_NRM_RX_RXINT_PKTCNT,
568 		     priv->et_coalesce.rx_max_coalesced_frames);
569 	netsec_write(priv, NETSEC_REG_NRM_RX_RXINT_TMR,
570 		     priv->et_coalesce.rx_coalesce_usecs);
571 	netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_SET, NRM_RX_ST_PKTCNT);
572 	netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_SET, NRM_RX_ST_TMREXP);
573 
574 	return 0;
575 }
576 
577 static u32 netsec_et_get_msglevel(struct net_device *dev)
578 {
579 	struct netsec_priv *priv = netdev_priv(dev);
580 
581 	return priv->msg_enable;
582 }
583 
584 static void netsec_et_set_msglevel(struct net_device *dev, u32 datum)
585 {
586 	struct netsec_priv *priv = netdev_priv(dev);
587 
588 	priv->msg_enable = datum;
589 }
590 
591 static const struct ethtool_ops netsec_ethtool_ops = {
592 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
593 				     ETHTOOL_COALESCE_MAX_FRAMES,
594 	.get_drvinfo		= netsec_et_get_drvinfo,
595 	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
596 	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
597 	.get_link		= ethtool_op_get_link,
598 	.get_coalesce		= netsec_et_get_coalesce,
599 	.set_coalesce		= netsec_et_set_coalesce,
600 	.get_msglevel		= netsec_et_get_msglevel,
601 	.set_msglevel		= netsec_et_set_msglevel,
602 };
603 
604 /************* NETDEV_OPS FOLLOW *************/
605 
606 
607 static void netsec_set_rx_de(struct netsec_priv *priv,
608 			     struct netsec_desc_ring *dring, u16 idx,
609 			     const struct netsec_desc *desc)
610 {
611 	struct netsec_de *de = dring->vaddr + DESC_SZ * idx;
612 	u32 attr = (1 << NETSEC_RX_PKT_OWN_FIELD) |
613 		   (1 << NETSEC_RX_PKT_FS_FIELD) |
614 		   (1 << NETSEC_RX_PKT_LS_FIELD);
615 
616 	if (idx == DESC_NUM - 1)
617 		attr |= (1 << NETSEC_RX_PKT_LD_FIELD);
618 
619 	de->data_buf_addr_up = upper_32_bits(desc->dma_addr);
620 	de->data_buf_addr_lw = lower_32_bits(desc->dma_addr);
621 	de->buf_len_info = desc->len;
622 	de->attr = attr;
623 	dma_wmb();
624 
625 	dring->desc[idx].dma_addr = desc->dma_addr;
626 	dring->desc[idx].addr = desc->addr;
627 	dring->desc[idx].len = desc->len;
628 }
629 
630 static bool netsec_clean_tx_dring(struct netsec_priv *priv)
631 {
632 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
633 	struct netsec_de *entry;
634 	int tail = dring->tail;
635 	unsigned int bytes;
636 	int cnt = 0;
637 
638 	spin_lock(&dring->lock);
639 
640 	bytes = 0;
641 	entry = dring->vaddr + DESC_SZ * tail;
642 
643 	while (!(entry->attr & (1U << NETSEC_TX_SHIFT_OWN_FIELD)) &&
644 	       cnt < DESC_NUM) {
645 		struct netsec_desc *desc;
646 		int eop;
647 
648 		desc = &dring->desc[tail];
649 		eop = (entry->attr >> NETSEC_TX_LAST) & 1;
650 		dma_rmb();
651 
652 		/* if buf_type is either TYPE_NETSEC_SKB or
653 		 * TYPE_NETSEC_XDP_NDO we mapped it
654 		 */
655 		if (desc->buf_type != TYPE_NETSEC_XDP_TX)
656 			dma_unmap_single(priv->dev, desc->dma_addr, desc->len,
657 					 DMA_TO_DEVICE);
658 
659 		if (!eop)
660 			goto next;
661 
662 		if (desc->buf_type == TYPE_NETSEC_SKB) {
663 			bytes += desc->skb->len;
664 			dev_kfree_skb(desc->skb);
665 		} else {
666 			bytes += desc->xdpf->len;
667 			xdp_return_frame(desc->xdpf);
668 		}
669 next:
670 		/* clean up so netsec_uninit_pkt_dring() won't free the skb
671 		 * again
672 		 */
673 		*desc = (struct netsec_desc){};
674 
675 		/* entry->attr is not going to be accessed by the NIC until
676 		 * netsec_set_tx_de() is called. No need for a dma_wmb() here
677 		 */
678 		entry->attr = 1U << NETSEC_TX_SHIFT_OWN_FIELD;
679 		/* move tail ahead */
680 		dring->tail = (tail + 1) % DESC_NUM;
681 
682 		tail = dring->tail;
683 		entry = dring->vaddr + DESC_SZ * tail;
684 		cnt++;
685 	}
686 
687 	spin_unlock(&dring->lock);
688 
689 	if (!cnt)
690 		return false;
691 
692 	/* reading the register clears the irq */
693 	netsec_read(priv, NETSEC_REG_NRM_TX_DONE_PKTCNT);
694 
695 	priv->ndev->stats.tx_packets += cnt;
696 	priv->ndev->stats.tx_bytes += bytes;
697 
698 	netdev_completed_queue(priv->ndev, cnt, bytes);
699 
700 	return true;
701 }
702 
703 static void netsec_process_tx(struct netsec_priv *priv)
704 {
705 	struct net_device *ndev = priv->ndev;
706 	bool cleaned;
707 
708 	cleaned = netsec_clean_tx_dring(priv);
709 
710 	if (cleaned && netif_queue_stopped(ndev)) {
711 		/* Make sure we update the value, anyone stopping the queue
712 		 * after this will read the proper consumer idx
713 		 */
714 		smp_wmb();
715 		netif_wake_queue(ndev);
716 	}
717 }
718 
719 static void *netsec_alloc_rx_data(struct netsec_priv *priv,
720 				  dma_addr_t *dma_handle, u16 *desc_len)
721 
722 {
723 
724 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
725 	struct page *page;
726 
727 	page = page_pool_dev_alloc_pages(dring->page_pool);
728 	if (!page)
729 		return NULL;
730 
731 	/* We allocate the same buffer length for XDP and non-XDP cases.
732 	 * page_pool API will map the whole page, skip what's needed for
733 	 * network payloads and/or XDP
734 	 */
735 	*dma_handle = page_pool_get_dma_addr(page) + NETSEC_RXBUF_HEADROOM;
736 	/* Make sure the incoming payload fits in the page for XDP and non-XDP
737 	 * cases and reserve enough space for headroom + skb_shared_info
738 	 */
739 	*desc_len = NETSEC_RX_BUF_SIZE;
740 
741 	return page_address(page);
742 }
743 
744 static void netsec_rx_fill(struct netsec_priv *priv, u16 from, u16 num)
745 {
746 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
747 	u16 idx = from;
748 
749 	while (num) {
750 		netsec_set_rx_de(priv, dring, idx, &dring->desc[idx]);
751 		idx++;
752 		if (idx >= DESC_NUM)
753 			idx = 0;
754 		num--;
755 	}
756 }
757 
758 static void netsec_xdp_ring_tx_db(struct netsec_priv *priv, u16 pkts)
759 {
760 	if (likely(pkts))
761 		netsec_write(priv, NETSEC_REG_NRM_TX_PKTCNT, pkts);
762 }
763 
764 static void netsec_finalize_xdp_rx(struct netsec_priv *priv, u32 xdp_res,
765 				   u16 pkts)
766 {
767 	if (xdp_res & NETSEC_XDP_REDIR)
768 		xdp_do_flush_map();
769 
770 	if (xdp_res & NETSEC_XDP_TX)
771 		netsec_xdp_ring_tx_db(priv, pkts);
772 }
773 
774 static void netsec_set_tx_de(struct netsec_priv *priv,
775 			     struct netsec_desc_ring *dring,
776 			     const struct netsec_tx_pkt_ctrl *tx_ctrl,
777 			     const struct netsec_desc *desc, void *buf)
778 {
779 	int idx = dring->head;
780 	struct netsec_de *de;
781 	u32 attr;
782 
783 	de = dring->vaddr + (DESC_SZ * idx);
784 
785 	attr = (1 << NETSEC_TX_SHIFT_OWN_FIELD) |
786 	       (1 << NETSEC_TX_SHIFT_PT_FIELD) |
787 	       (NETSEC_RING_GMAC << NETSEC_TX_SHIFT_TDRID_FIELD) |
788 	       (1 << NETSEC_TX_SHIFT_FS_FIELD) |
789 	       (1 << NETSEC_TX_LAST) |
790 	       (tx_ctrl->cksum_offload_flag << NETSEC_TX_SHIFT_CO) |
791 	       (tx_ctrl->tcp_seg_offload_flag << NETSEC_TX_SHIFT_SO) |
792 	       (1 << NETSEC_TX_SHIFT_TRS_FIELD);
793 	if (idx == DESC_NUM - 1)
794 		attr |= (1 << NETSEC_TX_SHIFT_LD_FIELD);
795 
796 	de->data_buf_addr_up = upper_32_bits(desc->dma_addr);
797 	de->data_buf_addr_lw = lower_32_bits(desc->dma_addr);
798 	de->buf_len_info = (tx_ctrl->tcp_seg_len << 16) | desc->len;
799 	de->attr = attr;
800 
801 	dring->desc[idx] = *desc;
802 	if (desc->buf_type == TYPE_NETSEC_SKB)
803 		dring->desc[idx].skb = buf;
804 	else if (desc->buf_type == TYPE_NETSEC_XDP_TX ||
805 		 desc->buf_type == TYPE_NETSEC_XDP_NDO)
806 		dring->desc[idx].xdpf = buf;
807 
808 	/* move head ahead */
809 	dring->head = (dring->head + 1) % DESC_NUM;
810 }
811 
812 /* The current driver only supports 1 Txq, this should run under spin_lock() */
813 static u32 netsec_xdp_queue_one(struct netsec_priv *priv,
814 				struct xdp_frame *xdpf, bool is_ndo)
815 
816 {
817 	struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX];
818 	struct page *page = virt_to_page(xdpf->data);
819 	struct netsec_tx_pkt_ctrl tx_ctrl = {};
820 	struct netsec_desc tx_desc;
821 	dma_addr_t dma_handle;
822 	u16 filled;
823 
824 	if (tx_ring->head >= tx_ring->tail)
825 		filled = tx_ring->head - tx_ring->tail;
826 	else
827 		filled = tx_ring->head + DESC_NUM - tx_ring->tail;
828 
829 	if (DESC_NUM - filled <= 1)
830 		return NETSEC_XDP_CONSUMED;
831 
832 	if (is_ndo) {
833 		/* this is for ndo_xdp_xmit, the buffer needs mapping before
834 		 * sending
835 		 */
836 		dma_handle = dma_map_single(priv->dev, xdpf->data, xdpf->len,
837 					    DMA_TO_DEVICE);
838 		if (dma_mapping_error(priv->dev, dma_handle))
839 			return NETSEC_XDP_CONSUMED;
840 		tx_desc.buf_type = TYPE_NETSEC_XDP_NDO;
841 	} else {
842 		/* This is the device Rx buffer from page_pool. No need to remap
843 		 * just sync and send it
844 		 */
845 		struct netsec_desc_ring *rx_ring =
846 			&priv->desc_ring[NETSEC_RING_RX];
847 		enum dma_data_direction dma_dir =
848 			page_pool_get_dma_dir(rx_ring->page_pool);
849 
850 		dma_handle = page_pool_get_dma_addr(page) + xdpf->headroom +
851 			sizeof(*xdpf);
852 		dma_sync_single_for_device(priv->dev, dma_handle, xdpf->len,
853 					   dma_dir);
854 		tx_desc.buf_type = TYPE_NETSEC_XDP_TX;
855 	}
856 
857 	tx_desc.dma_addr = dma_handle;
858 	tx_desc.addr = xdpf->data;
859 	tx_desc.len = xdpf->len;
860 
861 	netdev_sent_queue(priv->ndev, xdpf->len);
862 	netsec_set_tx_de(priv, tx_ring, &tx_ctrl, &tx_desc, xdpf);
863 
864 	return NETSEC_XDP_TX;
865 }
866 
867 static u32 netsec_xdp_xmit_back(struct netsec_priv *priv, struct xdp_buff *xdp)
868 {
869 	struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX];
870 	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
871 	u32 ret;
872 
873 	if (unlikely(!xdpf))
874 		return NETSEC_XDP_CONSUMED;
875 
876 	spin_lock(&tx_ring->lock);
877 	ret = netsec_xdp_queue_one(priv, xdpf, false);
878 	spin_unlock(&tx_ring->lock);
879 
880 	return ret;
881 }
882 
883 static u32 netsec_run_xdp(struct netsec_priv *priv, struct bpf_prog *prog,
884 			  struct xdp_buff *xdp)
885 {
886 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
887 	unsigned int sync, len = xdp->data_end - xdp->data;
888 	u32 ret = NETSEC_XDP_PASS;
889 	struct page *page;
890 	int err;
891 	u32 act;
892 
893 	act = bpf_prog_run_xdp(prog, xdp);
894 
895 	/* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
896 	sync = xdp->data_end - xdp->data_hard_start - NETSEC_RXBUF_HEADROOM;
897 	sync = max(sync, len);
898 
899 	switch (act) {
900 	case XDP_PASS:
901 		ret = NETSEC_XDP_PASS;
902 		break;
903 	case XDP_TX:
904 		ret = netsec_xdp_xmit_back(priv, xdp);
905 		if (ret != NETSEC_XDP_TX) {
906 			page = virt_to_head_page(xdp->data);
907 			page_pool_put_page(dring->page_pool, page, sync, true);
908 		}
909 		break;
910 	case XDP_REDIRECT:
911 		err = xdp_do_redirect(priv->ndev, xdp, prog);
912 		if (!err) {
913 			ret = NETSEC_XDP_REDIR;
914 		} else {
915 			ret = NETSEC_XDP_CONSUMED;
916 			page = virt_to_head_page(xdp->data);
917 			page_pool_put_page(dring->page_pool, page, sync, true);
918 		}
919 		break;
920 	default:
921 		bpf_warn_invalid_xdp_action(act);
922 		/* fall through */
923 	case XDP_ABORTED:
924 		trace_xdp_exception(priv->ndev, prog, act);
925 		/* fall through -- handle aborts by dropping packet */
926 	case XDP_DROP:
927 		ret = NETSEC_XDP_CONSUMED;
928 		page = virt_to_head_page(xdp->data);
929 		page_pool_put_page(dring->page_pool, page, sync, true);
930 		break;
931 	}
932 
933 	return ret;
934 }
935 
936 static int netsec_process_rx(struct netsec_priv *priv, int budget)
937 {
938 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
939 	struct net_device *ndev = priv->ndev;
940 	struct netsec_rx_pkt_info rx_info;
941 	enum dma_data_direction dma_dir;
942 	struct bpf_prog *xdp_prog;
943 	struct xdp_buff xdp;
944 	u16 xdp_xmit = 0;
945 	u32 xdp_act = 0;
946 	int done = 0;
947 
948 	xdp.rxq = &dring->xdp_rxq;
949 	xdp.frame_sz = PAGE_SIZE;
950 
951 	rcu_read_lock();
952 	xdp_prog = READ_ONCE(priv->xdp_prog);
953 	dma_dir = page_pool_get_dma_dir(dring->page_pool);
954 
955 	while (done < budget) {
956 		u16 idx = dring->tail;
957 		struct netsec_de *de = dring->vaddr + (DESC_SZ * idx);
958 		struct netsec_desc *desc = &dring->desc[idx];
959 		struct page *page = virt_to_page(desc->addr);
960 		u32 xdp_result = NETSEC_XDP_PASS;
961 		struct sk_buff *skb = NULL;
962 		u16 pkt_len, desc_len;
963 		dma_addr_t dma_handle;
964 		void *buf_addr;
965 
966 		if (de->attr & (1U << NETSEC_RX_PKT_OWN_FIELD)) {
967 			/* reading the register clears the irq */
968 			netsec_read(priv, NETSEC_REG_NRM_RX_PKTCNT);
969 			break;
970 		}
971 
972 		/* This  barrier is needed to keep us from reading
973 		 * any other fields out of the netsec_de until we have
974 		 * verified the descriptor has been written back
975 		 */
976 		dma_rmb();
977 		done++;
978 
979 		pkt_len = de->buf_len_info >> 16;
980 		rx_info.err_code = (de->attr >> NETSEC_RX_PKT_ERR_FIELD) &
981 			NETSEC_RX_PKT_ERR_MASK;
982 		rx_info.err_flag = (de->attr >> NETSEC_RX_PKT_ER_FIELD) & 1;
983 		if (rx_info.err_flag) {
984 			netif_err(priv, drv, priv->ndev,
985 				  "%s: rx fail err(%d)\n", __func__,
986 				  rx_info.err_code);
987 			ndev->stats.rx_dropped++;
988 			dring->tail = (dring->tail + 1) % DESC_NUM;
989 			/* reuse buffer page frag */
990 			netsec_rx_fill(priv, idx, 1);
991 			continue;
992 		}
993 		rx_info.rx_cksum_result =
994 			(de->attr >> NETSEC_RX_PKT_CO_FIELD) & 3;
995 
996 		/* allocate a fresh buffer and map it to the hardware.
997 		 * This will eventually replace the old buffer in the hardware
998 		 */
999 		buf_addr = netsec_alloc_rx_data(priv, &dma_handle, &desc_len);
1000 
1001 		if (unlikely(!buf_addr))
1002 			break;
1003 
1004 		dma_sync_single_for_cpu(priv->dev, desc->dma_addr, pkt_len,
1005 					dma_dir);
1006 		prefetch(desc->addr);
1007 
1008 		xdp.data_hard_start = desc->addr;
1009 		xdp.data = desc->addr + NETSEC_RXBUF_HEADROOM;
1010 		xdp_set_data_meta_invalid(&xdp);
1011 		xdp.data_end = xdp.data + pkt_len;
1012 
1013 		if (xdp_prog) {
1014 			xdp_result = netsec_run_xdp(priv, xdp_prog, &xdp);
1015 			if (xdp_result != NETSEC_XDP_PASS) {
1016 				xdp_act |= xdp_result;
1017 				if (xdp_result == NETSEC_XDP_TX)
1018 					xdp_xmit++;
1019 				goto next;
1020 			}
1021 		}
1022 		skb = build_skb(desc->addr, desc->len + NETSEC_RX_BUF_NON_DATA);
1023 
1024 		if (unlikely(!skb)) {
1025 			/* If skb fails recycle_direct will either unmap and
1026 			 * free the page or refill the cache depending on the
1027 			 * cache state. Since we paid the allocation cost if
1028 			 * building an skb fails try to put the page into cache
1029 			 */
1030 			page_pool_put_page(dring->page_pool, page, pkt_len,
1031 					   true);
1032 			netif_err(priv, drv, priv->ndev,
1033 				  "rx failed to build skb\n");
1034 			break;
1035 		}
1036 		page_pool_release_page(dring->page_pool, page);
1037 
1038 		skb_reserve(skb, xdp.data - xdp.data_hard_start);
1039 		skb_put(skb, xdp.data_end - xdp.data);
1040 		skb->protocol = eth_type_trans(skb, priv->ndev);
1041 
1042 		if (priv->rx_cksum_offload_flag &&
1043 		    rx_info.rx_cksum_result == NETSEC_RX_CKSUM_OK)
1044 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1045 
1046 next:
1047 		if (skb)
1048 			napi_gro_receive(&priv->napi, skb);
1049 		if (skb || xdp_result) {
1050 			ndev->stats.rx_packets++;
1051 			ndev->stats.rx_bytes += xdp.data_end - xdp.data;
1052 		}
1053 
1054 		/* Update the descriptor with fresh buffers */
1055 		desc->len = desc_len;
1056 		desc->dma_addr = dma_handle;
1057 		desc->addr = buf_addr;
1058 
1059 		netsec_rx_fill(priv, idx, 1);
1060 		dring->tail = (dring->tail + 1) % DESC_NUM;
1061 	}
1062 	netsec_finalize_xdp_rx(priv, xdp_act, xdp_xmit);
1063 
1064 	rcu_read_unlock();
1065 
1066 	return done;
1067 }
1068 
1069 static int netsec_napi_poll(struct napi_struct *napi, int budget)
1070 {
1071 	struct netsec_priv *priv;
1072 	int done;
1073 
1074 	priv = container_of(napi, struct netsec_priv, napi);
1075 
1076 	netsec_process_tx(priv);
1077 	done = netsec_process_rx(priv, budget);
1078 
1079 	if (done < budget && napi_complete_done(napi, done)) {
1080 		unsigned long flags;
1081 
1082 		spin_lock_irqsave(&priv->reglock, flags);
1083 		netsec_write(priv, NETSEC_REG_INTEN_SET,
1084 			     NETSEC_IRQ_RX | NETSEC_IRQ_TX);
1085 		spin_unlock_irqrestore(&priv->reglock, flags);
1086 	}
1087 
1088 	return done;
1089 }
1090 
1091 
1092 static int netsec_desc_used(struct netsec_desc_ring *dring)
1093 {
1094 	int used;
1095 
1096 	if (dring->head >= dring->tail)
1097 		used = dring->head - dring->tail;
1098 	else
1099 		used = dring->head + DESC_NUM - dring->tail;
1100 
1101 	return used;
1102 }
1103 
1104 static int netsec_check_stop_tx(struct netsec_priv *priv, int used)
1105 {
1106 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
1107 
1108 	/* keep tail from touching the queue */
1109 	if (DESC_NUM - used < 2) {
1110 		netif_stop_queue(priv->ndev);
1111 
1112 		/* Make sure we read the updated value in case
1113 		 * descriptors got freed
1114 		 */
1115 		smp_rmb();
1116 
1117 		used = netsec_desc_used(dring);
1118 		if (DESC_NUM - used < 2)
1119 			return NETDEV_TX_BUSY;
1120 
1121 		netif_wake_queue(priv->ndev);
1122 	}
1123 
1124 	return 0;
1125 }
1126 
1127 static netdev_tx_t netsec_netdev_start_xmit(struct sk_buff *skb,
1128 					    struct net_device *ndev)
1129 {
1130 	struct netsec_priv *priv = netdev_priv(ndev);
1131 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
1132 	struct netsec_tx_pkt_ctrl tx_ctrl = {};
1133 	struct netsec_desc tx_desc;
1134 	u16 tso_seg_len = 0;
1135 	int filled;
1136 
1137 	spin_lock_bh(&dring->lock);
1138 	filled = netsec_desc_used(dring);
1139 	if (netsec_check_stop_tx(priv, filled)) {
1140 		spin_unlock_bh(&dring->lock);
1141 		net_warn_ratelimited("%s %s Tx queue full\n",
1142 				     dev_name(priv->dev), ndev->name);
1143 		return NETDEV_TX_BUSY;
1144 	}
1145 
1146 	if (skb->ip_summed == CHECKSUM_PARTIAL)
1147 		tx_ctrl.cksum_offload_flag = true;
1148 
1149 	if (skb_is_gso(skb))
1150 		tso_seg_len = skb_shinfo(skb)->gso_size;
1151 
1152 	if (tso_seg_len > 0) {
1153 		if (skb->protocol == htons(ETH_P_IP)) {
1154 			ip_hdr(skb)->tot_len = 0;
1155 			tcp_hdr(skb)->check =
1156 				~tcp_v4_check(0, ip_hdr(skb)->saddr,
1157 					      ip_hdr(skb)->daddr, 0);
1158 		} else {
1159 			tcp_v6_gso_csum_prep(skb);
1160 		}
1161 
1162 		tx_ctrl.tcp_seg_offload_flag = true;
1163 		tx_ctrl.tcp_seg_len = tso_seg_len;
1164 	}
1165 
1166 	tx_desc.dma_addr = dma_map_single(priv->dev, skb->data,
1167 					  skb_headlen(skb), DMA_TO_DEVICE);
1168 	if (dma_mapping_error(priv->dev, tx_desc.dma_addr)) {
1169 		spin_unlock_bh(&dring->lock);
1170 		netif_err(priv, drv, priv->ndev,
1171 			  "%s: DMA mapping failed\n", __func__);
1172 		ndev->stats.tx_dropped++;
1173 		dev_kfree_skb_any(skb);
1174 		return NETDEV_TX_OK;
1175 	}
1176 	tx_desc.addr = skb->data;
1177 	tx_desc.len = skb_headlen(skb);
1178 	tx_desc.buf_type = TYPE_NETSEC_SKB;
1179 
1180 	skb_tx_timestamp(skb);
1181 	netdev_sent_queue(priv->ndev, skb->len);
1182 
1183 	netsec_set_tx_de(priv, dring, &tx_ctrl, &tx_desc, skb);
1184 	spin_unlock_bh(&dring->lock);
1185 	netsec_write(priv, NETSEC_REG_NRM_TX_PKTCNT, 1); /* submit another tx */
1186 
1187 	return NETDEV_TX_OK;
1188 }
1189 
1190 static void netsec_uninit_pkt_dring(struct netsec_priv *priv, int id)
1191 {
1192 	struct netsec_desc_ring *dring = &priv->desc_ring[id];
1193 	struct netsec_desc *desc;
1194 	u16 idx;
1195 
1196 	if (!dring->vaddr || !dring->desc)
1197 		return;
1198 	for (idx = 0; idx < DESC_NUM; idx++) {
1199 		desc = &dring->desc[idx];
1200 		if (!desc->addr)
1201 			continue;
1202 
1203 		if (id == NETSEC_RING_RX) {
1204 			struct page *page = virt_to_page(desc->addr);
1205 
1206 			page_pool_put_full_page(dring->page_pool, page, false);
1207 		} else if (id == NETSEC_RING_TX) {
1208 			dma_unmap_single(priv->dev, desc->dma_addr, desc->len,
1209 					 DMA_TO_DEVICE);
1210 			dev_kfree_skb(desc->skb);
1211 		}
1212 	}
1213 
1214 	/* Rx is currently using page_pool */
1215 	if (id == NETSEC_RING_RX) {
1216 		if (xdp_rxq_info_is_reg(&dring->xdp_rxq))
1217 			xdp_rxq_info_unreg(&dring->xdp_rxq);
1218 		page_pool_destroy(dring->page_pool);
1219 	}
1220 
1221 	memset(dring->desc, 0, sizeof(struct netsec_desc) * DESC_NUM);
1222 	memset(dring->vaddr, 0, DESC_SZ * DESC_NUM);
1223 
1224 	dring->head = 0;
1225 	dring->tail = 0;
1226 
1227 	if (id == NETSEC_RING_TX)
1228 		netdev_reset_queue(priv->ndev);
1229 }
1230 
1231 static void netsec_free_dring(struct netsec_priv *priv, int id)
1232 {
1233 	struct netsec_desc_ring *dring = &priv->desc_ring[id];
1234 
1235 	if (dring->vaddr) {
1236 		dma_free_coherent(priv->dev, DESC_SZ * DESC_NUM,
1237 				  dring->vaddr, dring->desc_dma);
1238 		dring->vaddr = NULL;
1239 	}
1240 
1241 	kfree(dring->desc);
1242 	dring->desc = NULL;
1243 }
1244 
1245 static int netsec_alloc_dring(struct netsec_priv *priv, enum ring_id id)
1246 {
1247 	struct netsec_desc_ring *dring = &priv->desc_ring[id];
1248 
1249 	dring->vaddr = dma_alloc_coherent(priv->dev, DESC_SZ * DESC_NUM,
1250 					  &dring->desc_dma, GFP_KERNEL);
1251 	if (!dring->vaddr)
1252 		goto err;
1253 
1254 	dring->desc = kcalloc(DESC_NUM, sizeof(*dring->desc), GFP_KERNEL);
1255 	if (!dring->desc)
1256 		goto err;
1257 
1258 	return 0;
1259 err:
1260 	netsec_free_dring(priv, id);
1261 
1262 	return -ENOMEM;
1263 }
1264 
1265 static void netsec_setup_tx_dring(struct netsec_priv *priv)
1266 {
1267 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
1268 	int i;
1269 
1270 	for (i = 0; i < DESC_NUM; i++) {
1271 		struct netsec_de *de;
1272 
1273 		de = dring->vaddr + (DESC_SZ * i);
1274 		/* de->attr is not going to be accessed by the NIC
1275 		 * until netsec_set_tx_de() is called.
1276 		 * No need for a dma_wmb() here
1277 		 */
1278 		de->attr = 1U << NETSEC_TX_SHIFT_OWN_FIELD;
1279 	}
1280 }
1281 
1282 static int netsec_setup_rx_dring(struct netsec_priv *priv)
1283 {
1284 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
1285 	struct bpf_prog *xdp_prog = READ_ONCE(priv->xdp_prog);
1286 	struct page_pool_params pp_params = {
1287 		.order = 0,
1288 		/* internal DMA mapping in page_pool */
1289 		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
1290 		.pool_size = DESC_NUM,
1291 		.nid = NUMA_NO_NODE,
1292 		.dev = priv->dev,
1293 		.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
1294 		.offset = NETSEC_RXBUF_HEADROOM,
1295 		.max_len = NETSEC_RX_BUF_SIZE,
1296 	};
1297 	int i, err;
1298 
1299 	dring->page_pool = page_pool_create(&pp_params);
1300 	if (IS_ERR(dring->page_pool)) {
1301 		err = PTR_ERR(dring->page_pool);
1302 		dring->page_pool = NULL;
1303 		goto err_out;
1304 	}
1305 
1306 	err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0);
1307 	if (err)
1308 		goto err_out;
1309 
1310 	err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_POOL,
1311 					 dring->page_pool);
1312 	if (err)
1313 		goto err_out;
1314 
1315 	for (i = 0; i < DESC_NUM; i++) {
1316 		struct netsec_desc *desc = &dring->desc[i];
1317 		dma_addr_t dma_handle;
1318 		void *buf;
1319 		u16 len;
1320 
1321 		buf = netsec_alloc_rx_data(priv, &dma_handle, &len);
1322 
1323 		if (!buf) {
1324 			err = -ENOMEM;
1325 			goto err_out;
1326 		}
1327 		desc->dma_addr = dma_handle;
1328 		desc->addr = buf;
1329 		desc->len = len;
1330 	}
1331 
1332 	netsec_rx_fill(priv, 0, DESC_NUM);
1333 
1334 	return 0;
1335 
1336 err_out:
1337 	netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
1338 	return err;
1339 }
1340 
1341 static int netsec_netdev_load_ucode_region(struct netsec_priv *priv, u32 reg,
1342 					   u32 addr_h, u32 addr_l, u32 size)
1343 {
1344 	u64 base = (u64)addr_h << 32 | addr_l;
1345 	void __iomem *ucode;
1346 	u32 i;
1347 
1348 	ucode = ioremap(base, size * sizeof(u32));
1349 	if (!ucode)
1350 		return -ENOMEM;
1351 
1352 	for (i = 0; i < size; i++)
1353 		netsec_write(priv, reg, readl(ucode + i * 4));
1354 
1355 	iounmap(ucode);
1356 	return 0;
1357 }
1358 
1359 static int netsec_netdev_load_microcode(struct netsec_priv *priv)
1360 {
1361 	u32 addr_h, addr_l, size;
1362 	int err;
1363 
1364 	addr_h = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_ADDRESS_H);
1365 	addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_ADDRESS_L);
1366 	size = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_SIZE);
1367 	err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_DMAC_HM_CMD_BUF,
1368 					      addr_h, addr_l, size);
1369 	if (err)
1370 		return err;
1371 
1372 	addr_h = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_ADDRESS_H);
1373 	addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_ADDRESS_L);
1374 	size = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_SIZE);
1375 	err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_DMAC_MH_CMD_BUF,
1376 					      addr_h, addr_l, size);
1377 	if (err)
1378 		return err;
1379 
1380 	addr_h = 0;
1381 	addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_PKT_ME_ADDRESS);
1382 	size = readl(priv->eeprom_base + NETSEC_EEPROM_PKT_ME_SIZE);
1383 	err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_PKT_CMD_BUF,
1384 					      addr_h, addr_l, size);
1385 	if (err)
1386 		return err;
1387 
1388 	return 0;
1389 }
1390 
1391 static int netsec_reset_hardware(struct netsec_priv *priv,
1392 				 bool load_ucode)
1393 {
1394 	u32 value;
1395 	int err;
1396 
1397 	/* stop DMA engines */
1398 	if (!netsec_read(priv, NETSEC_REG_ADDR_DIS_CORE)) {
1399 		netsec_write(priv, NETSEC_REG_DMA_HM_CTRL,
1400 			     NETSEC_DMA_CTRL_REG_STOP);
1401 		netsec_write(priv, NETSEC_REG_DMA_MH_CTRL,
1402 			     NETSEC_DMA_CTRL_REG_STOP);
1403 
1404 		while (netsec_read(priv, NETSEC_REG_DMA_HM_CTRL) &
1405 		       NETSEC_DMA_CTRL_REG_STOP)
1406 			cpu_relax();
1407 
1408 		while (netsec_read(priv, NETSEC_REG_DMA_MH_CTRL) &
1409 		       NETSEC_DMA_CTRL_REG_STOP)
1410 			cpu_relax();
1411 	}
1412 
1413 	netsec_write(priv, NETSEC_REG_SOFT_RST, NETSEC_SOFT_RST_REG_RESET);
1414 	netsec_write(priv, NETSEC_REG_SOFT_RST, NETSEC_SOFT_RST_REG_RUN);
1415 	netsec_write(priv, NETSEC_REG_COM_INIT, NETSEC_COM_INIT_REG_ALL);
1416 
1417 	while (netsec_read(priv, NETSEC_REG_COM_INIT) != 0)
1418 		cpu_relax();
1419 
1420 	/* set desc_start addr */
1421 	netsec_write(priv, NETSEC_REG_NRM_RX_DESC_START_UP,
1422 		     upper_32_bits(priv->desc_ring[NETSEC_RING_RX].desc_dma));
1423 	netsec_write(priv, NETSEC_REG_NRM_RX_DESC_START_LW,
1424 		     lower_32_bits(priv->desc_ring[NETSEC_RING_RX].desc_dma));
1425 
1426 	netsec_write(priv, NETSEC_REG_NRM_TX_DESC_START_UP,
1427 		     upper_32_bits(priv->desc_ring[NETSEC_RING_TX].desc_dma));
1428 	netsec_write(priv, NETSEC_REG_NRM_TX_DESC_START_LW,
1429 		     lower_32_bits(priv->desc_ring[NETSEC_RING_TX].desc_dma));
1430 
1431 	/* set normal tx dring ring config */
1432 	netsec_write(priv, NETSEC_REG_NRM_TX_CONFIG,
1433 		     1 << NETSEC_REG_DESC_ENDIAN);
1434 	netsec_write(priv, NETSEC_REG_NRM_RX_CONFIG,
1435 		     1 << NETSEC_REG_DESC_ENDIAN);
1436 
1437 	if (load_ucode) {
1438 		err = netsec_netdev_load_microcode(priv);
1439 		if (err) {
1440 			netif_err(priv, probe, priv->ndev,
1441 				  "%s: failed to load microcode (%d)\n",
1442 				  __func__, err);
1443 			return err;
1444 		}
1445 	}
1446 
1447 	/* start DMA engines */
1448 	netsec_write(priv, NETSEC_REG_DMA_TMR_CTRL, priv->freq / 1000000 - 1);
1449 	netsec_write(priv, NETSEC_REG_ADDR_DIS_CORE, 0);
1450 
1451 	usleep_range(1000, 2000);
1452 
1453 	if (!(netsec_read(priv, NETSEC_REG_TOP_STATUS) &
1454 	      NETSEC_TOP_IRQ_REG_CODE_LOAD_END)) {
1455 		netif_err(priv, probe, priv->ndev,
1456 			  "microengine start failed\n");
1457 		return -ENXIO;
1458 	}
1459 	netsec_write(priv, NETSEC_REG_TOP_STATUS,
1460 		     NETSEC_TOP_IRQ_REG_CODE_LOAD_END);
1461 
1462 	value = NETSEC_PKT_CTRL_REG_MODE_NRM;
1463 	if (priv->ndev->mtu > ETH_DATA_LEN)
1464 		value |= NETSEC_PKT_CTRL_REG_EN_JUMBO;
1465 
1466 	/* change to normal mode */
1467 	netsec_write(priv, NETSEC_REG_DMA_MH_CTRL, MH_CTRL__MODE_TRANS);
1468 	netsec_write(priv, NETSEC_REG_PKT_CTRL, value);
1469 
1470 	while ((netsec_read(priv, NETSEC_REG_MODE_TRANS_COMP_STATUS) &
1471 		NETSEC_MODE_TRANS_COMP_IRQ_T2N) == 0)
1472 		cpu_relax();
1473 
1474 	/* clear any pending EMPTY/ERR irq status */
1475 	netsec_write(priv, NETSEC_REG_NRM_TX_STATUS, ~0);
1476 
1477 	/* Disable TX & RX intr */
1478 	netsec_write(priv, NETSEC_REG_INTEN_CLR, ~0);
1479 
1480 	return 0;
1481 }
1482 
1483 static int netsec_start_gmac(struct netsec_priv *priv)
1484 {
1485 	struct phy_device *phydev = priv->ndev->phydev;
1486 	u32 value = 0;
1487 	int ret;
1488 
1489 	if (phydev->speed != SPEED_1000)
1490 		value = (NETSEC_GMAC_MCR_REG_CST |
1491 			 NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON);
1492 
1493 	if (netsec_mac_write(priv, GMAC_REG_MCR, value))
1494 		return -ETIMEDOUT;
1495 	if (netsec_mac_write(priv, GMAC_REG_BMR,
1496 			     NETSEC_GMAC_BMR_REG_RESET))
1497 		return -ETIMEDOUT;
1498 
1499 	/* Wait soft reset */
1500 	usleep_range(1000, 5000);
1501 
1502 	ret = netsec_mac_read(priv, GMAC_REG_BMR, &value);
1503 	if (ret)
1504 		return ret;
1505 	if (value & NETSEC_GMAC_BMR_REG_SWR)
1506 		return -EAGAIN;
1507 
1508 	netsec_write(priv, MAC_REG_DESC_SOFT_RST, 1);
1509 	if (netsec_wait_while_busy(priv, MAC_REG_DESC_SOFT_RST, 1))
1510 		return -ETIMEDOUT;
1511 
1512 	netsec_write(priv, MAC_REG_DESC_INIT, 1);
1513 	if (netsec_wait_while_busy(priv, MAC_REG_DESC_INIT, 1))
1514 		return -ETIMEDOUT;
1515 
1516 	if (netsec_mac_write(priv, GMAC_REG_BMR,
1517 			     NETSEC_GMAC_BMR_REG_COMMON))
1518 		return -ETIMEDOUT;
1519 	if (netsec_mac_write(priv, GMAC_REG_RDLAR,
1520 			     NETSEC_GMAC_RDLAR_REG_COMMON))
1521 		return -ETIMEDOUT;
1522 	if (netsec_mac_write(priv, GMAC_REG_TDLAR,
1523 			     NETSEC_GMAC_TDLAR_REG_COMMON))
1524 		return -ETIMEDOUT;
1525 	if (netsec_mac_write(priv, GMAC_REG_MFFR, 0x80000001))
1526 		return -ETIMEDOUT;
1527 
1528 	ret = netsec_mac_update_to_phy_state(priv);
1529 	if (ret)
1530 		return ret;
1531 
1532 	ret = netsec_mac_read(priv, GMAC_REG_OMR, &value);
1533 	if (ret)
1534 		return ret;
1535 
1536 	value |= NETSEC_GMAC_OMR_REG_SR;
1537 	value |= NETSEC_GMAC_OMR_REG_ST;
1538 
1539 	netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_CLR, ~0);
1540 	netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_CLR, ~0);
1541 
1542 	netsec_et_set_coalesce(priv->ndev, &priv->et_coalesce);
1543 
1544 	if (netsec_mac_write(priv, GMAC_REG_OMR, value))
1545 		return -ETIMEDOUT;
1546 
1547 	return 0;
1548 }
1549 
1550 static int netsec_stop_gmac(struct netsec_priv *priv)
1551 {
1552 	u32 value;
1553 	int ret;
1554 
1555 	ret = netsec_mac_read(priv, GMAC_REG_OMR, &value);
1556 	if (ret)
1557 		return ret;
1558 	value &= ~NETSEC_GMAC_OMR_REG_SR;
1559 	value &= ~NETSEC_GMAC_OMR_REG_ST;
1560 
1561 	/* disable all interrupts */
1562 	netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_CLR, ~0);
1563 	netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_CLR, ~0);
1564 
1565 	return netsec_mac_write(priv, GMAC_REG_OMR, value);
1566 }
1567 
1568 static void netsec_phy_adjust_link(struct net_device *ndev)
1569 {
1570 	struct netsec_priv *priv = netdev_priv(ndev);
1571 
1572 	if (ndev->phydev->link)
1573 		netsec_start_gmac(priv);
1574 	else
1575 		netsec_stop_gmac(priv);
1576 
1577 	phy_print_status(ndev->phydev);
1578 }
1579 
1580 static irqreturn_t netsec_irq_handler(int irq, void *dev_id)
1581 {
1582 	struct netsec_priv *priv = dev_id;
1583 	u32 val, status = netsec_read(priv, NETSEC_REG_TOP_STATUS);
1584 	unsigned long flags;
1585 
1586 	/* Disable interrupts */
1587 	if (status & NETSEC_IRQ_TX) {
1588 		val = netsec_read(priv, NETSEC_REG_NRM_TX_STATUS);
1589 		netsec_write(priv, NETSEC_REG_NRM_TX_STATUS, val);
1590 	}
1591 	if (status & NETSEC_IRQ_RX) {
1592 		val = netsec_read(priv, NETSEC_REG_NRM_RX_STATUS);
1593 		netsec_write(priv, NETSEC_REG_NRM_RX_STATUS, val);
1594 	}
1595 
1596 	spin_lock_irqsave(&priv->reglock, flags);
1597 	netsec_write(priv, NETSEC_REG_INTEN_CLR, NETSEC_IRQ_RX | NETSEC_IRQ_TX);
1598 	spin_unlock_irqrestore(&priv->reglock, flags);
1599 
1600 	napi_schedule(&priv->napi);
1601 
1602 	return IRQ_HANDLED;
1603 }
1604 
1605 static int netsec_netdev_open(struct net_device *ndev)
1606 {
1607 	struct netsec_priv *priv = netdev_priv(ndev);
1608 	int ret;
1609 
1610 	pm_runtime_get_sync(priv->dev);
1611 
1612 	netsec_setup_tx_dring(priv);
1613 	ret = netsec_setup_rx_dring(priv);
1614 	if (ret) {
1615 		netif_err(priv, probe, priv->ndev,
1616 			  "%s: fail setup ring\n", __func__);
1617 		goto err1;
1618 	}
1619 
1620 	ret = request_irq(priv->ndev->irq, netsec_irq_handler,
1621 			  IRQF_SHARED, "netsec", priv);
1622 	if (ret) {
1623 		netif_err(priv, drv, priv->ndev, "request_irq failed\n");
1624 		goto err2;
1625 	}
1626 
1627 	if (dev_of_node(priv->dev)) {
1628 		if (!of_phy_connect(priv->ndev, priv->phy_np,
1629 				    netsec_phy_adjust_link, 0,
1630 				    priv->phy_interface)) {
1631 			netif_err(priv, link, priv->ndev, "missing PHY\n");
1632 			ret = -ENODEV;
1633 			goto err3;
1634 		}
1635 	} else {
1636 		ret = phy_connect_direct(priv->ndev, priv->phydev,
1637 					 netsec_phy_adjust_link,
1638 					 priv->phy_interface);
1639 		if (ret) {
1640 			netif_err(priv, link, priv->ndev,
1641 				  "phy_connect_direct() failed (%d)\n", ret);
1642 			goto err3;
1643 		}
1644 	}
1645 
1646 	phy_start(ndev->phydev);
1647 
1648 	netsec_start_gmac(priv);
1649 	napi_enable(&priv->napi);
1650 	netif_start_queue(ndev);
1651 
1652 	/* Enable TX+RX intr. */
1653 	netsec_write(priv, NETSEC_REG_INTEN_SET, NETSEC_IRQ_RX | NETSEC_IRQ_TX);
1654 
1655 	return 0;
1656 err3:
1657 	free_irq(priv->ndev->irq, priv);
1658 err2:
1659 	netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
1660 err1:
1661 	pm_runtime_put_sync(priv->dev);
1662 	return ret;
1663 }
1664 
1665 static int netsec_netdev_stop(struct net_device *ndev)
1666 {
1667 	int ret;
1668 	struct netsec_priv *priv = netdev_priv(ndev);
1669 
1670 	netif_stop_queue(priv->ndev);
1671 	dma_wmb();
1672 
1673 	napi_disable(&priv->napi);
1674 
1675 	netsec_write(priv, NETSEC_REG_INTEN_CLR, ~0);
1676 	netsec_stop_gmac(priv);
1677 
1678 	free_irq(priv->ndev->irq, priv);
1679 
1680 	netsec_uninit_pkt_dring(priv, NETSEC_RING_TX);
1681 	netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
1682 
1683 	phy_stop(ndev->phydev);
1684 	phy_disconnect(ndev->phydev);
1685 
1686 	ret = netsec_reset_hardware(priv, false);
1687 
1688 	pm_runtime_put_sync(priv->dev);
1689 
1690 	return ret;
1691 }
1692 
1693 static int netsec_netdev_init(struct net_device *ndev)
1694 {
1695 	struct netsec_priv *priv = netdev_priv(ndev);
1696 	int ret;
1697 	u16 data;
1698 
1699 	BUILD_BUG_ON_NOT_POWER_OF_2(DESC_NUM);
1700 
1701 	ret = netsec_alloc_dring(priv, NETSEC_RING_TX);
1702 	if (ret)
1703 		return ret;
1704 
1705 	ret = netsec_alloc_dring(priv, NETSEC_RING_RX);
1706 	if (ret)
1707 		goto err1;
1708 
1709 	/* set phy power down */
1710 	data = netsec_phy_read(priv->mii_bus, priv->phy_addr, MII_BMCR) |
1711 		BMCR_PDOWN;
1712 	netsec_phy_write(priv->mii_bus, priv->phy_addr, MII_BMCR, data);
1713 
1714 	ret = netsec_reset_hardware(priv, true);
1715 	if (ret)
1716 		goto err2;
1717 
1718 	spin_lock_init(&priv->desc_ring[NETSEC_RING_TX].lock);
1719 	spin_lock_init(&priv->desc_ring[NETSEC_RING_RX].lock);
1720 
1721 	return 0;
1722 err2:
1723 	netsec_free_dring(priv, NETSEC_RING_RX);
1724 err1:
1725 	netsec_free_dring(priv, NETSEC_RING_TX);
1726 	return ret;
1727 }
1728 
1729 static void netsec_netdev_uninit(struct net_device *ndev)
1730 {
1731 	struct netsec_priv *priv = netdev_priv(ndev);
1732 
1733 	netsec_free_dring(priv, NETSEC_RING_RX);
1734 	netsec_free_dring(priv, NETSEC_RING_TX);
1735 }
1736 
1737 static int netsec_netdev_set_features(struct net_device *ndev,
1738 				      netdev_features_t features)
1739 {
1740 	struct netsec_priv *priv = netdev_priv(ndev);
1741 
1742 	priv->rx_cksum_offload_flag = !!(features & NETIF_F_RXCSUM);
1743 
1744 	return 0;
1745 }
1746 
1747 static int netsec_xdp_xmit(struct net_device *ndev, int n,
1748 			   struct xdp_frame **frames, u32 flags)
1749 {
1750 	struct netsec_priv *priv = netdev_priv(ndev);
1751 	struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX];
1752 	int drops = 0;
1753 	int i;
1754 
1755 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1756 		return -EINVAL;
1757 
1758 	spin_lock(&tx_ring->lock);
1759 	for (i = 0; i < n; i++) {
1760 		struct xdp_frame *xdpf = frames[i];
1761 		int err;
1762 
1763 		err = netsec_xdp_queue_one(priv, xdpf, true);
1764 		if (err != NETSEC_XDP_TX) {
1765 			xdp_return_frame_rx_napi(xdpf);
1766 			drops++;
1767 		} else {
1768 			tx_ring->xdp_xmit++;
1769 		}
1770 	}
1771 	spin_unlock(&tx_ring->lock);
1772 
1773 	if (unlikely(flags & XDP_XMIT_FLUSH)) {
1774 		netsec_xdp_ring_tx_db(priv, tx_ring->xdp_xmit);
1775 		tx_ring->xdp_xmit = 0;
1776 	}
1777 
1778 	return n - drops;
1779 }
1780 
1781 static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,
1782 			    struct netlink_ext_ack *extack)
1783 {
1784 	struct net_device *dev = priv->ndev;
1785 	struct bpf_prog *old_prog;
1786 
1787 	/* For now just support only the usual MTU sized frames */
1788 	if (prog && dev->mtu > 1500) {
1789 		NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP");
1790 		return -EOPNOTSUPP;
1791 	}
1792 
1793 	if (netif_running(dev))
1794 		netsec_netdev_stop(dev);
1795 
1796 	/* Detach old prog, if any */
1797 	old_prog = xchg(&priv->xdp_prog, prog);
1798 	if (old_prog)
1799 		bpf_prog_put(old_prog);
1800 
1801 	if (netif_running(dev))
1802 		netsec_netdev_open(dev);
1803 
1804 	return 0;
1805 }
1806 
1807 static int netsec_xdp(struct net_device *ndev, struct netdev_bpf *xdp)
1808 {
1809 	struct netsec_priv *priv = netdev_priv(ndev);
1810 
1811 	switch (xdp->command) {
1812 	case XDP_SETUP_PROG:
1813 		return netsec_xdp_setup(priv, xdp->prog, xdp->extack);
1814 	case XDP_QUERY_PROG:
1815 		xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
1816 		return 0;
1817 	default:
1818 		return -EINVAL;
1819 	}
1820 }
1821 
1822 static const struct net_device_ops netsec_netdev_ops = {
1823 	.ndo_init		= netsec_netdev_init,
1824 	.ndo_uninit		= netsec_netdev_uninit,
1825 	.ndo_open		= netsec_netdev_open,
1826 	.ndo_stop		= netsec_netdev_stop,
1827 	.ndo_start_xmit		= netsec_netdev_start_xmit,
1828 	.ndo_set_features	= netsec_netdev_set_features,
1829 	.ndo_set_mac_address    = eth_mac_addr,
1830 	.ndo_validate_addr	= eth_validate_addr,
1831 	.ndo_do_ioctl		= phy_do_ioctl,
1832 	.ndo_xdp_xmit		= netsec_xdp_xmit,
1833 	.ndo_bpf		= netsec_xdp,
1834 };
1835 
1836 static int netsec_of_probe(struct platform_device *pdev,
1837 			   struct netsec_priv *priv, u32 *phy_addr)
1838 {
1839 	priv->phy_np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0);
1840 	if (!priv->phy_np) {
1841 		dev_err(&pdev->dev, "missing required property 'phy-handle'\n");
1842 		return -EINVAL;
1843 	}
1844 
1845 	*phy_addr = of_mdio_parse_addr(&pdev->dev, priv->phy_np);
1846 
1847 	priv->clk = devm_clk_get(&pdev->dev, NULL); /* get by 'phy_ref_clk' */
1848 	if (IS_ERR(priv->clk)) {
1849 		dev_err(&pdev->dev, "phy_ref_clk not found\n");
1850 		return PTR_ERR(priv->clk);
1851 	}
1852 	priv->freq = clk_get_rate(priv->clk);
1853 
1854 	return 0;
1855 }
1856 
1857 static int netsec_acpi_probe(struct platform_device *pdev,
1858 			     struct netsec_priv *priv, u32 *phy_addr)
1859 {
1860 	int ret;
1861 
1862 	if (!IS_ENABLED(CONFIG_ACPI))
1863 		return -ENODEV;
1864 
1865 	ret = device_property_read_u32(&pdev->dev, "phy-channel", phy_addr);
1866 	if (ret) {
1867 		dev_err(&pdev->dev,
1868 			"missing required property 'phy-channel'\n");
1869 		return ret;
1870 	}
1871 
1872 	ret = device_property_read_u32(&pdev->dev,
1873 				       "socionext,phy-clock-frequency",
1874 				       &priv->freq);
1875 	if (ret)
1876 		dev_err(&pdev->dev,
1877 			"missing required property 'socionext,phy-clock-frequency'\n");
1878 	return ret;
1879 }
1880 
1881 static void netsec_unregister_mdio(struct netsec_priv *priv)
1882 {
1883 	struct phy_device *phydev = priv->phydev;
1884 
1885 	if (!dev_of_node(priv->dev) && phydev) {
1886 		phy_device_remove(phydev);
1887 		phy_device_free(phydev);
1888 	}
1889 
1890 	mdiobus_unregister(priv->mii_bus);
1891 }
1892 
1893 static int netsec_register_mdio(struct netsec_priv *priv, u32 phy_addr)
1894 {
1895 	struct mii_bus *bus;
1896 	int ret;
1897 
1898 	bus = devm_mdiobus_alloc(priv->dev);
1899 	if (!bus)
1900 		return -ENOMEM;
1901 
1902 	snprintf(bus->id, MII_BUS_ID_SIZE, "%s", dev_name(priv->dev));
1903 	bus->priv = priv;
1904 	bus->name = "SNI NETSEC MDIO";
1905 	bus->read = netsec_phy_read;
1906 	bus->write = netsec_phy_write;
1907 	bus->parent = priv->dev;
1908 	priv->mii_bus = bus;
1909 
1910 	if (dev_of_node(priv->dev)) {
1911 		struct device_node *mdio_node, *parent = dev_of_node(priv->dev);
1912 
1913 		mdio_node = of_get_child_by_name(parent, "mdio");
1914 		if (mdio_node) {
1915 			parent = mdio_node;
1916 		} else {
1917 			/* older f/w doesn't populate the mdio subnode,
1918 			 * allow relaxed upgrade of f/w in due time.
1919 			 */
1920 			dev_info(priv->dev, "Upgrade f/w for mdio subnode!\n");
1921 		}
1922 
1923 		ret = of_mdiobus_register(bus, parent);
1924 		of_node_put(mdio_node);
1925 
1926 		if (ret) {
1927 			dev_err(priv->dev, "mdiobus register err(%d)\n", ret);
1928 			return ret;
1929 		}
1930 	} else {
1931 		/* Mask out all PHYs from auto probing. */
1932 		bus->phy_mask = ~0;
1933 		ret = mdiobus_register(bus);
1934 		if (ret) {
1935 			dev_err(priv->dev, "mdiobus register err(%d)\n", ret);
1936 			return ret;
1937 		}
1938 
1939 		priv->phydev = get_phy_device(bus, phy_addr, false);
1940 		if (IS_ERR(priv->phydev)) {
1941 			ret = PTR_ERR(priv->phydev);
1942 			dev_err(priv->dev, "get_phy_device err(%d)\n", ret);
1943 			priv->phydev = NULL;
1944 			return -ENODEV;
1945 		}
1946 
1947 		ret = phy_device_register(priv->phydev);
1948 		if (ret) {
1949 			mdiobus_unregister(bus);
1950 			dev_err(priv->dev,
1951 				"phy_device_register err(%d)\n", ret);
1952 		}
1953 	}
1954 
1955 	return ret;
1956 }
1957 
1958 static int netsec_probe(struct platform_device *pdev)
1959 {
1960 	struct resource *mmio_res, *eeprom_res, *irq_res;
1961 	u8 *mac, macbuf[ETH_ALEN];
1962 	struct netsec_priv *priv;
1963 	u32 hw_ver, phy_addr = 0;
1964 	struct net_device *ndev;
1965 	int ret;
1966 
1967 	mmio_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1968 	if (!mmio_res) {
1969 		dev_err(&pdev->dev, "No MMIO resource found.\n");
1970 		return -ENODEV;
1971 	}
1972 
1973 	eeprom_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1974 	if (!eeprom_res) {
1975 		dev_info(&pdev->dev, "No EEPROM resource found.\n");
1976 		return -ENODEV;
1977 	}
1978 
1979 	irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1980 	if (!irq_res) {
1981 		dev_err(&pdev->dev, "No IRQ resource found.\n");
1982 		return -ENODEV;
1983 	}
1984 
1985 	ndev = alloc_etherdev(sizeof(*priv));
1986 	if (!ndev)
1987 		return -ENOMEM;
1988 
1989 	priv = netdev_priv(ndev);
1990 
1991 	spin_lock_init(&priv->reglock);
1992 	SET_NETDEV_DEV(ndev, &pdev->dev);
1993 	platform_set_drvdata(pdev, priv);
1994 	ndev->irq = irq_res->start;
1995 	priv->dev = &pdev->dev;
1996 	priv->ndev = ndev;
1997 
1998 	priv->msg_enable = NETIF_MSG_TX_ERR | NETIF_MSG_HW | NETIF_MSG_DRV |
1999 			   NETIF_MSG_LINK | NETIF_MSG_PROBE;
2000 
2001 	priv->phy_interface = device_get_phy_mode(&pdev->dev);
2002 	if ((int)priv->phy_interface < 0) {
2003 		dev_err(&pdev->dev, "missing required property 'phy-mode'\n");
2004 		ret = -ENODEV;
2005 		goto free_ndev;
2006 	}
2007 
2008 	priv->ioaddr = devm_ioremap(&pdev->dev, mmio_res->start,
2009 				    resource_size(mmio_res));
2010 	if (!priv->ioaddr) {
2011 		dev_err(&pdev->dev, "devm_ioremap() failed\n");
2012 		ret = -ENXIO;
2013 		goto free_ndev;
2014 	}
2015 
2016 	priv->eeprom_base = devm_ioremap(&pdev->dev, eeprom_res->start,
2017 					 resource_size(eeprom_res));
2018 	if (!priv->eeprom_base) {
2019 		dev_err(&pdev->dev, "devm_ioremap() failed for EEPROM\n");
2020 		ret = -ENXIO;
2021 		goto free_ndev;
2022 	}
2023 
2024 	mac = device_get_mac_address(&pdev->dev, macbuf, sizeof(macbuf));
2025 	if (mac)
2026 		ether_addr_copy(ndev->dev_addr, mac);
2027 
2028 	if (priv->eeprom_base &&
2029 	    (!mac || !is_valid_ether_addr(ndev->dev_addr))) {
2030 		void __iomem *macp = priv->eeprom_base +
2031 					NETSEC_EEPROM_MAC_ADDRESS;
2032 
2033 		ndev->dev_addr[0] = readb(macp + 3);
2034 		ndev->dev_addr[1] = readb(macp + 2);
2035 		ndev->dev_addr[2] = readb(macp + 1);
2036 		ndev->dev_addr[3] = readb(macp + 0);
2037 		ndev->dev_addr[4] = readb(macp + 7);
2038 		ndev->dev_addr[5] = readb(macp + 6);
2039 	}
2040 
2041 	if (!is_valid_ether_addr(ndev->dev_addr)) {
2042 		dev_warn(&pdev->dev, "No MAC address found, using random\n");
2043 		eth_hw_addr_random(ndev);
2044 	}
2045 
2046 	if (dev_of_node(&pdev->dev))
2047 		ret = netsec_of_probe(pdev, priv, &phy_addr);
2048 	else
2049 		ret = netsec_acpi_probe(pdev, priv, &phy_addr);
2050 	if (ret)
2051 		goto free_ndev;
2052 
2053 	priv->phy_addr = phy_addr;
2054 
2055 	if (!priv->freq) {
2056 		dev_err(&pdev->dev, "missing PHY reference clock frequency\n");
2057 		ret = -ENODEV;
2058 		goto free_ndev;
2059 	}
2060 
2061 	/* default for throughput */
2062 	priv->et_coalesce.rx_coalesce_usecs = 500;
2063 	priv->et_coalesce.rx_max_coalesced_frames = 8;
2064 	priv->et_coalesce.tx_coalesce_usecs = 500;
2065 	priv->et_coalesce.tx_max_coalesced_frames = 8;
2066 
2067 	ret = device_property_read_u32(&pdev->dev, "max-frame-size",
2068 				       &ndev->max_mtu);
2069 	if (ret < 0)
2070 		ndev->max_mtu = ETH_DATA_LEN;
2071 
2072 	/* runtime_pm coverage just for probe, open/close also cover it */
2073 	pm_runtime_enable(&pdev->dev);
2074 	pm_runtime_get_sync(&pdev->dev);
2075 
2076 	hw_ver = netsec_read(priv, NETSEC_REG_F_TAIKI_VER);
2077 	/* this driver only supports F_TAIKI style NETSEC */
2078 	if (NETSEC_F_NETSEC_VER_MAJOR_NUM(hw_ver) !=
2079 	    NETSEC_F_NETSEC_VER_MAJOR_NUM(NETSEC_REG_NETSEC_VER_F_TAIKI)) {
2080 		ret = -ENODEV;
2081 		goto pm_disable;
2082 	}
2083 
2084 	dev_info(&pdev->dev, "hardware revision %d.%d\n",
2085 		 hw_ver >> 16, hw_ver & 0xffff);
2086 
2087 	netif_napi_add(ndev, &priv->napi, netsec_napi_poll, NAPI_POLL_WEIGHT);
2088 
2089 	ndev->netdev_ops = &netsec_netdev_ops;
2090 	ndev->ethtool_ops = &netsec_ethtool_ops;
2091 
2092 	ndev->features |= NETIF_F_HIGHDMA | NETIF_F_RXCSUM | NETIF_F_GSO |
2093 				NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2094 	ndev->hw_features = ndev->features;
2095 
2096 	priv->rx_cksum_offload_flag = true;
2097 
2098 	ret = netsec_register_mdio(priv, phy_addr);
2099 	if (ret)
2100 		goto unreg_napi;
2101 
2102 	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)))
2103 		dev_warn(&pdev->dev, "Failed to set DMA mask\n");
2104 
2105 	ret = register_netdev(ndev);
2106 	if (ret) {
2107 		netif_err(priv, probe, ndev, "register_netdev() failed\n");
2108 		goto unreg_mii;
2109 	}
2110 
2111 	pm_runtime_put_sync(&pdev->dev);
2112 	return 0;
2113 
2114 unreg_mii:
2115 	netsec_unregister_mdio(priv);
2116 unreg_napi:
2117 	netif_napi_del(&priv->napi);
2118 pm_disable:
2119 	pm_runtime_put_sync(&pdev->dev);
2120 	pm_runtime_disable(&pdev->dev);
2121 free_ndev:
2122 	free_netdev(ndev);
2123 	dev_err(&pdev->dev, "init failed\n");
2124 
2125 	return ret;
2126 }
2127 
2128 static int netsec_remove(struct platform_device *pdev)
2129 {
2130 	struct netsec_priv *priv = platform_get_drvdata(pdev);
2131 
2132 	unregister_netdev(priv->ndev);
2133 
2134 	netsec_unregister_mdio(priv);
2135 
2136 	netif_napi_del(&priv->napi);
2137 
2138 	pm_runtime_disable(&pdev->dev);
2139 	free_netdev(priv->ndev);
2140 
2141 	return 0;
2142 }
2143 
2144 #ifdef CONFIG_PM
2145 static int netsec_runtime_suspend(struct device *dev)
2146 {
2147 	struct netsec_priv *priv = dev_get_drvdata(dev);
2148 
2149 	netsec_write(priv, NETSEC_REG_CLK_EN, 0);
2150 
2151 	clk_disable_unprepare(priv->clk);
2152 
2153 	return 0;
2154 }
2155 
2156 static int netsec_runtime_resume(struct device *dev)
2157 {
2158 	struct netsec_priv *priv = dev_get_drvdata(dev);
2159 
2160 	clk_prepare_enable(priv->clk);
2161 
2162 	netsec_write(priv, NETSEC_REG_CLK_EN, NETSEC_CLK_EN_REG_DOM_D |
2163 					       NETSEC_CLK_EN_REG_DOM_C |
2164 					       NETSEC_CLK_EN_REG_DOM_G);
2165 	return 0;
2166 }
2167 #endif
2168 
2169 static const struct dev_pm_ops netsec_pm_ops = {
2170 	SET_RUNTIME_PM_OPS(netsec_runtime_suspend, netsec_runtime_resume, NULL)
2171 };
2172 
2173 static const struct of_device_id netsec_dt_ids[] = {
2174 	{ .compatible = "socionext,synquacer-netsec" },
2175 	{ }
2176 };
2177 MODULE_DEVICE_TABLE(of, netsec_dt_ids);
2178 
2179 #ifdef CONFIG_ACPI
2180 static const struct acpi_device_id netsec_acpi_ids[] = {
2181 	{ "SCX0001" },
2182 	{ }
2183 };
2184 MODULE_DEVICE_TABLE(acpi, netsec_acpi_ids);
2185 #endif
2186 
2187 static struct platform_driver netsec_driver = {
2188 	.probe	= netsec_probe,
2189 	.remove	= netsec_remove,
2190 	.driver = {
2191 		.name = "netsec",
2192 		.pm = &netsec_pm_ops,
2193 		.of_match_table = netsec_dt_ids,
2194 		.acpi_match_table = ACPI_PTR(netsec_acpi_ids),
2195 	},
2196 };
2197 module_platform_driver(netsec_driver);
2198 
2199 MODULE_AUTHOR("Jassi Brar <jaswinder.singh@linaro.org>");
2200 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
2201 MODULE_DESCRIPTION("NETSEC Ethernet driver");
2202 MODULE_LICENSE("GPL");
2203