1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
3 
4 #include <linux/kernel.h>
5 #include <linux/io.h>
6 #include <linux/iopoll.h>
7 #include <linux/etherdevice.h>
8 #include <linux/platform_device.h>
9 #include <linux/if_ether.h>
10 #include <linux/if_vlan.h>
11 #include <net/dst_metadata.h>
12 #include <net/dsa.h>
13 #include "mtk_eth_soc.h"
14 #include "mtk_ppe.h"
15 #include "mtk_ppe_regs.h"
16 
17 static DEFINE_SPINLOCK(ppe_lock);
18 
19 static const struct rhashtable_params mtk_flow_l2_ht_params = {
20 	.head_offset = offsetof(struct mtk_flow_entry, l2_node),
21 	.key_offset = offsetof(struct mtk_flow_entry, data.bridge),
22 	.key_len = offsetof(struct mtk_foe_bridge, key_end),
23 	.automatic_shrinking = true,
24 };
25 
26 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)
27 {
28 	writel(val, ppe->base + reg);
29 }
30 
31 static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg)
32 {
33 	return readl(ppe->base + reg);
34 }
35 
36 static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set)
37 {
38 	u32 val;
39 
40 	val = ppe_r32(ppe, reg);
41 	val &= ~mask;
42 	val |= set;
43 	ppe_w32(ppe, reg, val);
44 
45 	return val;
46 }
47 
48 static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val)
49 {
50 	return ppe_m32(ppe, reg, 0, val);
51 }
52 
53 static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val)
54 {
55 	return ppe_m32(ppe, reg, val, 0);
56 }
57 
58 static u32 mtk_eth_timestamp(struct mtk_eth *eth)
59 {
60 	return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth);
61 }
62 
63 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
64 {
65 	int ret;
66 	u32 val;
67 
68 	ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val,
69 				 !(val & MTK_PPE_GLO_CFG_BUSY),
70 				 20, MTK_PPE_WAIT_TIMEOUT_US);
71 
72 	if (ret)
73 		dev_err(ppe->dev, "PPE table busy");
74 
75 	return ret;
76 }
77 
78 static int mtk_ppe_mib_wait_busy(struct mtk_ppe *ppe)
79 {
80 	int ret;
81 	u32 val;
82 
83 	ret = readl_poll_timeout(ppe->base + MTK_PPE_MIB_SER_CR, val,
84 				 !(val & MTK_PPE_MIB_SER_CR_ST),
85 				 20, MTK_PPE_WAIT_TIMEOUT_US);
86 
87 	if (ret)
88 		dev_err(ppe->dev, "MIB table busy");
89 
90 	return ret;
91 }
92 
93 static int mtk_mib_entry_read(struct mtk_ppe *ppe, u16 index, u64 *bytes, u64 *packets)
94 {
95 	u32 val, cnt_r0, cnt_r1, cnt_r2;
96 	int ret;
97 
98 	val = FIELD_PREP(MTK_PPE_MIB_SER_CR_ADDR, index) | MTK_PPE_MIB_SER_CR_ST;
99 	ppe_w32(ppe, MTK_PPE_MIB_SER_CR, val);
100 
101 	ret = mtk_ppe_mib_wait_busy(ppe);
102 	if (ret)
103 		return ret;
104 
105 	cnt_r0 = readl(ppe->base + MTK_PPE_MIB_SER_R0);
106 	cnt_r1 = readl(ppe->base + MTK_PPE_MIB_SER_R1);
107 	cnt_r2 = readl(ppe->base + MTK_PPE_MIB_SER_R2);
108 
109 	if (mtk_is_netsys_v3_or_greater(ppe->eth)) {
110 		/* 64 bit for each counter */
111 		u32 cnt_r3 = readl(ppe->base + MTK_PPE_MIB_SER_R3);
112 		*bytes = ((u64)cnt_r1 << 32) | cnt_r0;
113 		*packets = ((u64)cnt_r3 << 32) | cnt_r2;
114 	} else {
115 		/* 48 bit byte counter, 40 bit packet counter */
116 		u32 byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0);
117 		u32 byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH, cnt_r1);
118 		u32 pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R1_PKT_CNT_LOW, cnt_r1);
119 		u32 pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH, cnt_r2);
120 		*bytes = ((u64)byte_cnt_high << 32) | byte_cnt_low;
121 		*packets = ((u64)pkt_cnt_high << 16) | pkt_cnt_low;
122 	}
123 
124 	return 0;
125 }
126 
127 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe)
128 {
129 	ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
130 	ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
131 }
132 
133 static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable)
134 {
135 	mtk_ppe_cache_clear(ppe);
136 
137 	ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN,
138 		enable * MTK_PPE_CACHE_CTL_EN);
139 }
140 
141 static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e)
142 {
143 	u32 hv1, hv2, hv3;
144 	u32 hash;
145 
146 	switch (mtk_get_ib1_pkt_type(eth, e->ib1)) {
147 		case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
148 		case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
149 			hv1 = e->ipv4.orig.ports;
150 			hv2 = e->ipv4.orig.dest_ip;
151 			hv3 = e->ipv4.orig.src_ip;
152 			break;
153 		case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
154 		case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
155 			hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3];
156 			hv1 ^= e->ipv6.ports;
157 
158 			hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2];
159 			hv2 ^= e->ipv6.dest_ip[0];
160 
161 			hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1];
162 			hv3 ^= e->ipv6.src_ip[0];
163 			break;
164 		case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
165 		case MTK_PPE_PKT_TYPE_IPV6_6RD:
166 		default:
167 			WARN_ON_ONCE(1);
168 			return MTK_PPE_HASH_MASK;
169 	}
170 
171 	hash = (hv1 & hv2) | ((~hv1) & hv3);
172 	hash = (hash >> 24) | ((hash & 0xffffff) << 8);
173 	hash ^= hv1 ^ hv2 ^ hv3;
174 	hash ^= hash >> 16;
175 	hash <<= (ffs(eth->soc->hash_offset) - 1);
176 	hash &= MTK_PPE_ENTRIES - 1;
177 
178 	return hash;
179 }
180 
181 static inline struct mtk_foe_mac_info *
182 mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry)
183 {
184 	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
185 
186 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
187 		return &entry->bridge.l2;
188 
189 	if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
190 		return &entry->ipv6.l2;
191 
192 	return &entry->ipv4.l2;
193 }
194 
195 static inline u32 *
196 mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry)
197 {
198 	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
199 
200 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
201 		return &entry->bridge.ib2;
202 
203 	if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
204 		return &entry->ipv6.ib2;
205 
206 	return &entry->ipv4.ib2;
207 }
208 
209 int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry,
210 			  int type, int l4proto, u8 pse_port, u8 *src_mac,
211 			  u8 *dest_mac)
212 {
213 	struct mtk_foe_mac_info *l2;
214 	u32 ports_pad, val;
215 
216 	memset(entry, 0, sizeof(*entry));
217 
218 	if (mtk_is_netsys_v2_or_greater(eth)) {
219 		val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
220 		      FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) |
221 		      FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
222 		      MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2;
223 		entry->ib1 = val;
224 
225 		val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) |
226 		      FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf);
227 	} else {
228 		int port_mg = eth->soc->offload_version > 1 ? 0 : 0x3f;
229 
230 		val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
231 		      FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) |
232 		      FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
233 		      MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL;
234 		entry->ib1 = val;
235 
236 		val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) |
237 		      FIELD_PREP(MTK_FOE_IB2_PORT_MG, port_mg) |
238 		      FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f);
239 	}
240 
241 	if (is_multicast_ether_addr(dest_mac))
242 		val |= mtk_get_ib2_multicast_mask(eth);
243 
244 	ports_pad = 0xa5a5a500 | (l4proto & 0xff);
245 	if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
246 		entry->ipv4.orig.ports = ports_pad;
247 	if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
248 		entry->ipv6.ports = ports_pad;
249 
250 	if (type == MTK_PPE_PKT_TYPE_BRIDGE) {
251 		ether_addr_copy(entry->bridge.src_mac, src_mac);
252 		ether_addr_copy(entry->bridge.dest_mac, dest_mac);
253 		entry->bridge.ib2 = val;
254 		l2 = &entry->bridge.l2;
255 	} else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) {
256 		entry->ipv6.ib2 = val;
257 		l2 = &entry->ipv6.l2;
258 	} else {
259 		entry->ipv4.ib2 = val;
260 		l2 = &entry->ipv4.l2;
261 	}
262 
263 	l2->dest_mac_hi = get_unaligned_be32(dest_mac);
264 	l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4);
265 	l2->src_mac_hi = get_unaligned_be32(src_mac);
266 	l2->src_mac_lo = get_unaligned_be16(src_mac + 4);
267 
268 	if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
269 		l2->etype = ETH_P_IPV6;
270 	else
271 		l2->etype = ETH_P_IP;
272 
273 	return 0;
274 }
275 
276 int mtk_foe_entry_set_pse_port(struct mtk_eth *eth,
277 			       struct mtk_foe_entry *entry, u8 port)
278 {
279 	u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
280 	u32 val = *ib2;
281 
282 	if (mtk_is_netsys_v2_or_greater(eth)) {
283 		val &= ~MTK_FOE_IB2_DEST_PORT_V2;
284 		val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port);
285 	} else {
286 		val &= ~MTK_FOE_IB2_DEST_PORT;
287 		val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port);
288 	}
289 	*ib2 = val;
290 
291 	return 0;
292 }
293 
294 int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth,
295 				 struct mtk_foe_entry *entry, bool egress,
296 				 __be32 src_addr, __be16 src_port,
297 				 __be32 dest_addr, __be16 dest_port)
298 {
299 	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
300 	struct mtk_ipv4_tuple *t;
301 
302 	switch (type) {
303 	case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
304 		if (egress) {
305 			t = &entry->ipv4.new;
306 			break;
307 		}
308 		fallthrough;
309 	case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
310 	case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
311 		t = &entry->ipv4.orig;
312 		break;
313 	case MTK_PPE_PKT_TYPE_IPV6_6RD:
314 		entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr);
315 		entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr);
316 		return 0;
317 	default:
318 		WARN_ON_ONCE(1);
319 		return -EINVAL;
320 	}
321 
322 	t->src_ip = be32_to_cpu(src_addr);
323 	t->dest_ip = be32_to_cpu(dest_addr);
324 
325 	if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
326 		return 0;
327 
328 	t->src_port = be16_to_cpu(src_port);
329 	t->dest_port = be16_to_cpu(dest_port);
330 
331 	return 0;
332 }
333 
334 int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth,
335 				 struct mtk_foe_entry *entry,
336 				 __be32 *src_addr, __be16 src_port,
337 				 __be32 *dest_addr, __be16 dest_port)
338 {
339 	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
340 	u32 *src, *dest;
341 	int i;
342 
343 	switch (type) {
344 	case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
345 		src = entry->dslite.tunnel_src_ip;
346 		dest = entry->dslite.tunnel_dest_ip;
347 		break;
348 	case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
349 	case MTK_PPE_PKT_TYPE_IPV6_6RD:
350 		entry->ipv6.src_port = be16_to_cpu(src_port);
351 		entry->ipv6.dest_port = be16_to_cpu(dest_port);
352 		fallthrough;
353 	case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
354 		src = entry->ipv6.src_ip;
355 		dest = entry->ipv6.dest_ip;
356 		break;
357 	default:
358 		WARN_ON_ONCE(1);
359 		return -EINVAL;
360 	}
361 
362 	for (i = 0; i < 4; i++)
363 		src[i] = be32_to_cpu(src_addr[i]);
364 	for (i = 0; i < 4; i++)
365 		dest[i] = be32_to_cpu(dest_addr[i]);
366 
367 	return 0;
368 }
369 
370 int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry,
371 			  int port)
372 {
373 	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
374 
375 	l2->etype = BIT(port);
376 
377 	if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)))
378 		entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1);
379 	else
380 		l2->etype |= BIT(8);
381 
382 	entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth);
383 
384 	return 0;
385 }
386 
387 int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry,
388 			   int vid)
389 {
390 	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
391 
392 	switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) {
393 	case 0:
394 		entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) |
395 			      mtk_prep_ib1_vlan_layer(eth, 1);
396 		l2->vlan1 = vid;
397 		return 0;
398 	case 1:
399 		if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) {
400 			l2->vlan1 = vid;
401 			l2->etype |= BIT(8);
402 		} else {
403 			l2->vlan2 = vid;
404 			entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1);
405 		}
406 		return 0;
407 	default:
408 		return -ENOSPC;
409 	}
410 }
411 
412 int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry,
413 			    int sid)
414 {
415 	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
416 
417 	if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) ||
418 	    (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth)))
419 		l2->etype = ETH_P_PPP_SES;
420 
421 	entry->ib1 |= mtk_get_ib1_ppoe_mask(eth);
422 	l2->pppoe_id = sid;
423 
424 	return 0;
425 }
426 
427 int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry,
428 			   int wdma_idx, int txq, int bss, int wcid)
429 {
430 	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
431 	u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
432 
433 	switch (eth->soc->version) {
434 	case 3:
435 		*ib2 &= ~MTK_FOE_IB2_PORT_MG_V2;
436 		*ib2 |=  FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) |
437 			 MTK_FOE_IB2_WDMA_WINFO_V2;
438 		l2->w3info = FIELD_PREP(MTK_FOE_WINFO_WCID_V3, wcid) |
439 			     FIELD_PREP(MTK_FOE_WINFO_BSS_V3, bss);
440 		break;
441 	case 2:
442 		*ib2 &= ~MTK_FOE_IB2_PORT_MG_V2;
443 		*ib2 |=  FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) |
444 			 MTK_FOE_IB2_WDMA_WINFO_V2;
445 		l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) |
446 			    FIELD_PREP(MTK_FOE_WINFO_BSS, bss);
447 		break;
448 	default:
449 		*ib2 &= ~MTK_FOE_IB2_PORT_MG;
450 		*ib2 |= MTK_FOE_IB2_WDMA_WINFO;
451 		if (wdma_idx)
452 			*ib2 |= MTK_FOE_IB2_WDMA_DEVIDX;
453 		l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) |
454 			    FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) |
455 			    FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq);
456 		break;
457 	}
458 
459 	return 0;
460 }
461 
462 int mtk_foe_entry_set_queue(struct mtk_eth *eth, struct mtk_foe_entry *entry,
463 			    unsigned int queue)
464 {
465 	u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
466 
467 	if (mtk_is_netsys_v2_or_greater(eth)) {
468 		*ib2 &= ~MTK_FOE_IB2_QID_V2;
469 		*ib2 |= FIELD_PREP(MTK_FOE_IB2_QID_V2, queue);
470 		*ib2 |= MTK_FOE_IB2_PSE_QOS_V2;
471 	} else {
472 		*ib2 &= ~MTK_FOE_IB2_QID;
473 		*ib2 |= FIELD_PREP(MTK_FOE_IB2_QID, queue);
474 		*ib2 |= MTK_FOE_IB2_PSE_QOS;
475 	}
476 
477 	return 0;
478 }
479 
480 static bool
481 mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry,
482 		     struct mtk_foe_entry *data)
483 {
484 	int type, len;
485 
486 	if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP)
487 		return false;
488 
489 	type = mtk_get_ib1_pkt_type(eth, entry->data.ib1);
490 	if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE)
491 		len = offsetof(struct mtk_foe_entry, ipv6._rsv);
492 	else
493 		len = offsetof(struct mtk_foe_entry, ipv4.ib2);
494 
495 	return !memcmp(&entry->data.data, &data->data, len - 4);
496 }
497 
498 static void
499 __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
500 {
501 	struct hlist_head *head;
502 	struct hlist_node *tmp;
503 
504 	if (entry->type == MTK_FLOW_TYPE_L2) {
505 		rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node,
506 				       mtk_flow_l2_ht_params);
507 
508 		head = &entry->l2_flows;
509 		hlist_for_each_entry_safe(entry, tmp, head, l2_data.list)
510 			__mtk_foe_entry_clear(ppe, entry);
511 		return;
512 	}
513 
514 	hlist_del_init(&entry->list);
515 	if (entry->hash != 0xffff) {
516 		struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash);
517 
518 		hwe->ib1 &= ~MTK_FOE_IB1_STATE;
519 		hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID);
520 		dma_wmb();
521 		mtk_ppe_cache_clear(ppe);
522 
523 		if (ppe->accounting) {
524 			struct mtk_foe_accounting *acct;
525 
526 			acct = ppe->acct_table + entry->hash * sizeof(*acct);
527 			acct->packets = 0;
528 			acct->bytes = 0;
529 		}
530 	}
531 	entry->hash = 0xffff;
532 
533 	if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW)
534 		return;
535 
536 	hlist_del_init(&entry->l2_data.list);
537 	kfree(entry);
538 }
539 
540 static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1)
541 {
542 	u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);
543 	u16 now = mtk_eth_timestamp(ppe->eth);
544 	u16 timestamp = ib1 & ib1_ts_mask;
545 
546 	if (timestamp > now)
547 		return ib1_ts_mask + 1 - timestamp + now;
548 	else
549 		return now - timestamp;
550 }
551 
552 static void
553 mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
554 {
555 	u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);
556 	struct mtk_flow_entry *cur;
557 	struct mtk_foe_entry *hwe;
558 	struct hlist_node *tmp;
559 	int idle;
560 
561 	idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1);
562 	hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) {
563 		int cur_idle;
564 		u32 ib1;
565 
566 		hwe = mtk_foe_get_entry(ppe, cur->hash);
567 		ib1 = READ_ONCE(hwe->ib1);
568 
569 		if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) {
570 			cur->hash = 0xffff;
571 			__mtk_foe_entry_clear(ppe, cur);
572 			continue;
573 		}
574 
575 		cur_idle = __mtk_foe_entry_idle_time(ppe, ib1);
576 		if (cur_idle >= idle)
577 			continue;
578 
579 		idle = cur_idle;
580 		entry->data.ib1 &= ~ib1_ts_mask;
581 		entry->data.ib1 |= hwe->ib1 & ib1_ts_mask;
582 	}
583 }
584 
585 static void
586 mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
587 {
588 	struct mtk_foe_entry foe = {};
589 	struct mtk_foe_entry *hwe;
590 
591 	spin_lock_bh(&ppe_lock);
592 
593 	if (entry->type == MTK_FLOW_TYPE_L2) {
594 		mtk_flow_entry_update_l2(ppe, entry);
595 		goto out;
596 	}
597 
598 	if (entry->hash == 0xffff)
599 		goto out;
600 
601 	hwe = mtk_foe_get_entry(ppe, entry->hash);
602 	memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size);
603 	if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) {
604 		entry->hash = 0xffff;
605 		goto out;
606 	}
607 
608 	entry->data.ib1 = foe.ib1;
609 
610 out:
611 	spin_unlock_bh(&ppe_lock);
612 }
613 
614 static void
615 __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
616 		       u16 hash)
617 {
618 	struct mtk_eth *eth = ppe->eth;
619 	u16 timestamp = mtk_eth_timestamp(eth);
620 	struct mtk_foe_entry *hwe;
621 	u32 val;
622 
623 	if (mtk_is_netsys_v2_or_greater(eth)) {
624 		entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2;
625 		entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2,
626 					 timestamp);
627 	} else {
628 		entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
629 		entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP,
630 					 timestamp);
631 	}
632 
633 	hwe = mtk_foe_get_entry(ppe, hash);
634 	memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1));
635 	wmb();
636 	hwe->ib1 = entry->ib1;
637 
638 	if (ppe->accounting) {
639 		if (mtk_is_netsys_v2_or_greater(eth))
640 			val = MTK_FOE_IB2_MIB_CNT_V2;
641 		else
642 			val = MTK_FOE_IB2_MIB_CNT;
643 		*mtk_foe_entry_ib2(eth, hwe) |= val;
644 	}
645 
646 	dma_wmb();
647 
648 	mtk_ppe_cache_clear(ppe);
649 }
650 
651 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
652 {
653 	spin_lock_bh(&ppe_lock);
654 	__mtk_foe_entry_clear(ppe, entry);
655 	spin_unlock_bh(&ppe_lock);
656 }
657 
658 static int
659 mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
660 {
661 	struct mtk_flow_entry *prev;
662 
663 	entry->type = MTK_FLOW_TYPE_L2;
664 
665 	prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &entry->l2_node,
666 						 mtk_flow_l2_ht_params);
667 	if (likely(!prev))
668 		return 0;
669 
670 	if (IS_ERR(prev))
671 		return PTR_ERR(prev);
672 
673 	return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node,
674 				       &entry->l2_node, mtk_flow_l2_ht_params);
675 }
676 
677 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
678 {
679 	const struct mtk_soc_data *soc = ppe->eth->soc;
680 	int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1);
681 	u32 hash;
682 
683 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
684 		return mtk_foe_entry_commit_l2(ppe, entry);
685 
686 	hash = mtk_ppe_hash_entry(ppe->eth, &entry->data);
687 	entry->hash = 0xffff;
688 	spin_lock_bh(&ppe_lock);
689 	hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]);
690 	spin_unlock_bh(&ppe_lock);
691 
692 	return 0;
693 }
694 
695 static void
696 mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry,
697 			     u16 hash)
698 {
699 	const struct mtk_soc_data *soc = ppe->eth->soc;
700 	struct mtk_flow_entry *flow_info;
701 	struct mtk_foe_entry foe = {}, *hwe;
702 	struct mtk_foe_mac_info *l2;
703 	u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP;
704 	int type;
705 
706 	flow_info = kzalloc(sizeof(*flow_info), GFP_ATOMIC);
707 	if (!flow_info)
708 		return;
709 
710 	flow_info->l2_data.base_flow = entry;
711 	flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW;
712 	flow_info->hash = hash;
713 	hlist_add_head(&flow_info->list,
714 		       &ppe->foe_flow[hash / soc->hash_offset]);
715 	hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows);
716 
717 	hwe = mtk_foe_get_entry(ppe, hash);
718 	memcpy(&foe, hwe, soc->foe_entry_size);
719 	foe.ib1 &= ib1_mask;
720 	foe.ib1 |= entry->data.ib1 & ~ib1_mask;
721 
722 	l2 = mtk_foe_entry_l2(ppe->eth, &foe);
723 	memcpy(l2, &entry->data.bridge.l2, sizeof(*l2));
724 
725 	type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1);
726 	if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT)
727 		memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new));
728 	else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP)
729 		l2->etype = ETH_P_IPV6;
730 
731 	*mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2;
732 
733 	__mtk_foe_entry_commit(ppe, &foe, hash);
734 }
735 
736 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
737 {
738 	const struct mtk_soc_data *soc = ppe->eth->soc;
739 	struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset];
740 	struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash);
741 	struct mtk_flow_entry *entry;
742 	struct mtk_foe_bridge key = {};
743 	struct hlist_node *n;
744 	struct ethhdr *eh;
745 	bool found = false;
746 	u8 *tag;
747 
748 	spin_lock_bh(&ppe_lock);
749 
750 	if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND)
751 		goto out;
752 
753 	hlist_for_each_entry_safe(entry, n, head, list) {
754 		if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) {
755 			if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) ==
756 				     MTK_FOE_STATE_BIND))
757 				continue;
758 
759 			entry->hash = 0xffff;
760 			__mtk_foe_entry_clear(ppe, entry);
761 			continue;
762 		}
763 
764 		if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) {
765 			if (entry->hash != 0xffff)
766 				entry->hash = 0xffff;
767 			continue;
768 		}
769 
770 		entry->hash = hash;
771 		__mtk_foe_entry_commit(ppe, &entry->data, hash);
772 		found = true;
773 	}
774 
775 	if (found)
776 		goto out;
777 
778 	eh = eth_hdr(skb);
779 	ether_addr_copy(key.dest_mac, eh->h_dest);
780 	ether_addr_copy(key.src_mac, eh->h_source);
781 	tag = skb->data - 2;
782 	key.vlan = 0;
783 	switch (skb->protocol) {
784 #if IS_ENABLED(CONFIG_NET_DSA)
785 	case htons(ETH_P_XDSA):
786 		if (!netdev_uses_dsa(skb->dev) ||
787 		    skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK)
788 			goto out;
789 
790 		if (!skb_metadata_dst(skb))
791 			tag += 4;
792 
793 		if (get_unaligned_be16(tag) != ETH_P_8021Q)
794 			break;
795 
796 		fallthrough;
797 #endif
798 	case htons(ETH_P_8021Q):
799 		key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK;
800 		break;
801 	default:
802 		break;
803 	}
804 
805 	entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params);
806 	if (!entry)
807 		goto out;
808 
809 	mtk_foe_entry_commit_subflow(ppe, entry, hash);
810 
811 out:
812 	spin_unlock_bh(&ppe_lock);
813 }
814 
815 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
816 {
817 	mtk_flow_entry_update(ppe, entry);
818 
819 	return __mtk_foe_entry_idle_time(ppe, entry->data.ib1);
820 }
821 
822 int mtk_ppe_prepare_reset(struct mtk_ppe *ppe)
823 {
824 	if (!ppe)
825 		return -EINVAL;
826 
827 	/* disable KA */
828 	ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE);
829 	ppe_clear(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE);
830 	ppe_w32(ppe, MTK_PPE_KEEPALIVE, 0);
831 	usleep_range(10000, 11000);
832 
833 	/* set KA timer to maximum */
834 	ppe_set(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE);
835 	ppe_w32(ppe, MTK_PPE_KEEPALIVE, 0xffffffff);
836 
837 	/* set KA tick select */
838 	ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_TICK_SEL);
839 	ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE);
840 	usleep_range(10000, 11000);
841 
842 	/* disable scan mode */
843 	ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_SCAN_MODE);
844 	usleep_range(10000, 11000);
845 
846 	return mtk_ppe_wait_busy(ppe);
847 }
848 
849 struct mtk_foe_accounting *mtk_foe_entry_get_mib(struct mtk_ppe *ppe, u32 index,
850 						 struct mtk_foe_accounting *diff)
851 {
852 	struct mtk_foe_accounting *acct;
853 	int size = sizeof(struct mtk_foe_accounting);
854 	u64 bytes, packets;
855 
856 	if (!ppe->accounting)
857 		return NULL;
858 
859 	if (mtk_mib_entry_read(ppe, index, &bytes, &packets))
860 		return NULL;
861 
862 	acct = ppe->acct_table + index * size;
863 
864 	acct->bytes += bytes;
865 	acct->packets += packets;
866 
867 	if (diff) {
868 		diff->bytes = bytes;
869 		diff->packets = packets;
870 	}
871 
872 	return acct;
873 }
874 
875 struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index)
876 {
877 	bool accounting = eth->soc->has_accounting;
878 	const struct mtk_soc_data *soc = eth->soc;
879 	struct mtk_foe_accounting *acct;
880 	struct device *dev = eth->dev;
881 	struct mtk_mib_entry *mib;
882 	struct mtk_ppe *ppe;
883 	u32 foe_flow_size;
884 	void *foe;
885 
886 	ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL);
887 	if (!ppe)
888 		return NULL;
889 
890 	rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params);
891 
892 	/* need to allocate a separate device, since it PPE DMA access is
893 	 * not coherent.
894 	 */
895 	ppe->base = base;
896 	ppe->eth = eth;
897 	ppe->dev = dev;
898 	ppe->version = eth->soc->offload_version;
899 	ppe->accounting = accounting;
900 
901 	foe = dmam_alloc_coherent(ppe->dev,
902 				  MTK_PPE_ENTRIES * soc->foe_entry_size,
903 				  &ppe->foe_phys, GFP_KERNEL);
904 	if (!foe)
905 		goto err_free_l2_flows;
906 
907 	ppe->foe_table = foe;
908 
909 	foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) *
910 			sizeof(*ppe->foe_flow);
911 	ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL);
912 	if (!ppe->foe_flow)
913 		goto err_free_l2_flows;
914 
915 	if (accounting) {
916 		mib = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*mib),
917 					  &ppe->mib_phys, GFP_KERNEL);
918 		if (!mib)
919 			return NULL;
920 
921 		ppe->mib_table = mib;
922 
923 		acct = devm_kzalloc(dev, MTK_PPE_ENTRIES * sizeof(*acct),
924 				    GFP_KERNEL);
925 
926 		if (!acct)
927 			return NULL;
928 
929 		ppe->acct_table = acct;
930 	}
931 
932 	mtk_ppe_debugfs_init(ppe, index);
933 
934 	return ppe;
935 
936 err_free_l2_flows:
937 	rhashtable_destroy(&ppe->l2_flows);
938 	return NULL;
939 }
940 
941 void mtk_ppe_deinit(struct mtk_eth *eth)
942 {
943 	int i;
944 
945 	for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) {
946 		if (!eth->ppe[i])
947 			return;
948 		rhashtable_destroy(&eth->ppe[i]->l2_flows);
949 	}
950 }
951 
952 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
953 {
954 	static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 };
955 	int i, k;
956 
957 	memset(ppe->foe_table, 0,
958 	       MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size);
959 
960 	if (!IS_ENABLED(CONFIG_SOC_MT7621))
961 		return;
962 
963 	/* skip all entries that cross the 1024 byte boundary */
964 	for (i = 0; i < MTK_PPE_ENTRIES; i += 128) {
965 		for (k = 0; k < ARRAY_SIZE(skip); k++) {
966 			struct mtk_foe_entry *hwe;
967 
968 			hwe = mtk_foe_get_entry(ppe, i + skip[k]);
969 			hwe->ib1 |= MTK_FOE_IB1_STATIC;
970 		}
971 	}
972 }
973 
974 void mtk_ppe_start(struct mtk_ppe *ppe)
975 {
976 	u32 val;
977 
978 	if (!ppe)
979 		return;
980 
981 	mtk_ppe_init_foe_table(ppe);
982 	ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys);
983 
984 	val = MTK_PPE_TB_CFG_AGE_NON_L4 |
985 	      MTK_PPE_TB_CFG_AGE_UNBIND |
986 	      MTK_PPE_TB_CFG_AGE_TCP |
987 	      MTK_PPE_TB_CFG_AGE_UDP |
988 	      MTK_PPE_TB_CFG_AGE_TCP_FIN |
989 	      FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS,
990 			 MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) |
991 	      FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE,
992 			 MTK_PPE_KEEPALIVE_DISABLE) |
993 	      FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) |
994 	      FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE,
995 			 MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) |
996 	      FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM,
997 			 MTK_PPE_ENTRIES_SHIFT);
998 	if (mtk_is_netsys_v2_or_greater(ppe->eth))
999 		val |= MTK_PPE_TB_CFG_INFO_SEL;
1000 	if (!mtk_is_netsys_v3_or_greater(ppe->eth))
1001 		val |= MTK_PPE_TB_CFG_ENTRY_80B;
1002 	ppe_w32(ppe, MTK_PPE_TB_CFG, val);
1003 
1004 	ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK,
1005 		MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6);
1006 
1007 	mtk_ppe_cache_enable(ppe, true);
1008 
1009 	val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE |
1010 	      MTK_PPE_FLOW_CFG_IP6_5T_ROUTE |
1011 	      MTK_PPE_FLOW_CFG_IP6_6RD |
1012 	      MTK_PPE_FLOW_CFG_IP4_NAT |
1013 	      MTK_PPE_FLOW_CFG_IP4_NAPT |
1014 	      MTK_PPE_FLOW_CFG_IP4_DSLITE |
1015 	      MTK_PPE_FLOW_CFG_IP4_NAT_FRAG;
1016 	if (mtk_is_netsys_v2_or_greater(ppe->eth))
1017 		val |= MTK_PPE_MD_TOAP_BYP_CRSN0 |
1018 		       MTK_PPE_MD_TOAP_BYP_CRSN1 |
1019 		       MTK_PPE_MD_TOAP_BYP_CRSN2 |
1020 		       MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY;
1021 	else
1022 		val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG |
1023 		       MTK_PPE_FLOW_CFG_IP4_UDP_FRAG;
1024 	ppe_w32(ppe, MTK_PPE_FLOW_CFG, val);
1025 
1026 	val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) |
1027 	      FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3);
1028 	ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val);
1029 
1030 	val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) |
1031 	      FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1);
1032 	ppe_w32(ppe, MTK_PPE_BIND_AGE0, val);
1033 
1034 	val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) |
1035 	      FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7);
1036 	ppe_w32(ppe, MTK_PPE_BIND_AGE1, val);
1037 
1038 	val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF;
1039 	ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val);
1040 
1041 	val = MTK_PPE_BIND_LIMIT1_FULL |
1042 	      FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1);
1043 	ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val);
1044 
1045 	val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) |
1046 	      FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1);
1047 	ppe_w32(ppe, MTK_PPE_BIND_RATE, val);
1048 
1049 	/* enable PPE */
1050 	val = MTK_PPE_GLO_CFG_EN |
1051 	      MTK_PPE_GLO_CFG_IP4_L4_CS_DROP |
1052 	      MTK_PPE_GLO_CFG_IP4_CS_DROP |
1053 	      MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE;
1054 	ppe_w32(ppe, MTK_PPE_GLO_CFG, val);
1055 
1056 	ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0);
1057 
1058 	if (mtk_is_netsys_v2_or_greater(ppe->eth)) {
1059 		ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777);
1060 		ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f);
1061 	}
1062 
1063 	if (ppe->accounting && ppe->mib_phys) {
1064 		ppe_w32(ppe, MTK_PPE_MIB_TB_BASE, ppe->mib_phys);
1065 		ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_EN,
1066 			MTK_PPE_MIB_CFG_EN);
1067 		ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_RD_CLR,
1068 			MTK_PPE_MIB_CFG_RD_CLR);
1069 		ppe_m32(ppe, MTK_PPE_MIB_CACHE_CTL, MTK_PPE_MIB_CACHE_CTL_EN,
1070 			MTK_PPE_MIB_CFG_RD_CLR);
1071 	}
1072 }
1073 
1074 int mtk_ppe_stop(struct mtk_ppe *ppe)
1075 {
1076 	u32 val;
1077 	int i;
1078 
1079 	if (!ppe)
1080 		return 0;
1081 
1082 	for (i = 0; i < MTK_PPE_ENTRIES; i++) {
1083 		struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i);
1084 
1085 		hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE,
1086 				      MTK_FOE_STATE_INVALID);
1087 	}
1088 
1089 	mtk_ppe_cache_enable(ppe, false);
1090 
1091 	/* disable offload engine */
1092 	ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN);
1093 	ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0);
1094 
1095 	/* disable aging */
1096 	val = MTK_PPE_TB_CFG_AGE_NON_L4 |
1097 	      MTK_PPE_TB_CFG_AGE_UNBIND |
1098 	      MTK_PPE_TB_CFG_AGE_TCP |
1099 	      MTK_PPE_TB_CFG_AGE_UDP |
1100 	      MTK_PPE_TB_CFG_AGE_TCP_FIN;
1101 	ppe_clear(ppe, MTK_PPE_TB_CFG, val);
1102 
1103 	return mtk_ppe_wait_busy(ppe);
1104 }
1105