1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
3 
4 #include <linux/kernel.h>
5 #include <linux/io.h>
6 #include <linux/iopoll.h>
7 #include <linux/etherdevice.h>
8 #include <linux/platform_device.h>
9 #include <linux/if_ether.h>
10 #include <linux/if_vlan.h>
11 #include <net/dst_metadata.h>
12 #include <net/dsa.h>
13 #include "mtk_eth_soc.h"
14 #include "mtk_ppe.h"
15 #include "mtk_ppe_regs.h"
16 
17 static DEFINE_SPINLOCK(ppe_lock);
18 
19 static const struct rhashtable_params mtk_flow_l2_ht_params = {
20 	.head_offset = offsetof(struct mtk_flow_entry, l2_node),
21 	.key_offset = offsetof(struct mtk_flow_entry, data.bridge),
22 	.key_len = offsetof(struct mtk_foe_bridge, key_end),
23 	.automatic_shrinking = true,
24 };
25 
26 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)
27 {
28 	writel(val, ppe->base + reg);
29 }
30 
31 static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg)
32 {
33 	return readl(ppe->base + reg);
34 }
35 
36 static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set)
37 {
38 	u32 val;
39 
40 	val = ppe_r32(ppe, reg);
41 	val &= ~mask;
42 	val |= set;
43 	ppe_w32(ppe, reg, val);
44 
45 	return val;
46 }
47 
48 static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val)
49 {
50 	return ppe_m32(ppe, reg, 0, val);
51 }
52 
53 static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val)
54 {
55 	return ppe_m32(ppe, reg, val, 0);
56 }
57 
58 static u32 mtk_eth_timestamp(struct mtk_eth *eth)
59 {
60 	return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth);
61 }
62 
63 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
64 {
65 	int ret;
66 	u32 val;
67 
68 	ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val,
69 				 !(val & MTK_PPE_GLO_CFG_BUSY),
70 				 20, MTK_PPE_WAIT_TIMEOUT_US);
71 
72 	if (ret)
73 		dev_err(ppe->dev, "PPE table busy");
74 
75 	return ret;
76 }
77 
78 static int mtk_ppe_mib_wait_busy(struct mtk_ppe *ppe)
79 {
80 	int ret;
81 	u32 val;
82 
83 	ret = readl_poll_timeout(ppe->base + MTK_PPE_MIB_SER_CR, val,
84 				 !(val & MTK_PPE_MIB_SER_CR_ST),
85 				 20, MTK_PPE_WAIT_TIMEOUT_US);
86 
87 	if (ret)
88 		dev_err(ppe->dev, "MIB table busy");
89 
90 	return ret;
91 }
92 
93 static int mtk_mib_entry_read(struct mtk_ppe *ppe, u16 index, u64 *bytes, u64 *packets)
94 {
95 	u32 byte_cnt_low, byte_cnt_high, pkt_cnt_low, pkt_cnt_high;
96 	u32 val, cnt_r0, cnt_r1, cnt_r2;
97 	int ret;
98 
99 	val = FIELD_PREP(MTK_PPE_MIB_SER_CR_ADDR, index) | MTK_PPE_MIB_SER_CR_ST;
100 	ppe_w32(ppe, MTK_PPE_MIB_SER_CR, val);
101 
102 	ret = mtk_ppe_mib_wait_busy(ppe);
103 	if (ret)
104 		return ret;
105 
106 	cnt_r0 = readl(ppe->base + MTK_PPE_MIB_SER_R0);
107 	cnt_r1 = readl(ppe->base + MTK_PPE_MIB_SER_R1);
108 	cnt_r2 = readl(ppe->base + MTK_PPE_MIB_SER_R2);
109 
110 	byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0);
111 	byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH, cnt_r1);
112 	pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R1_PKT_CNT_LOW, cnt_r1);
113 	pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH, cnt_r2);
114 	*bytes = ((u64)byte_cnt_high << 32) | byte_cnt_low;
115 	*packets = (pkt_cnt_high << 16) | pkt_cnt_low;
116 
117 	return 0;
118 }
119 
120 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe)
121 {
122 	ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
123 	ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
124 }
125 
126 static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable)
127 {
128 	mtk_ppe_cache_clear(ppe);
129 
130 	ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN,
131 		enable * MTK_PPE_CACHE_CTL_EN);
132 }
133 
134 static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e)
135 {
136 	u32 hv1, hv2, hv3;
137 	u32 hash;
138 
139 	switch (mtk_get_ib1_pkt_type(eth, e->ib1)) {
140 		case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
141 		case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
142 			hv1 = e->ipv4.orig.ports;
143 			hv2 = e->ipv4.orig.dest_ip;
144 			hv3 = e->ipv4.orig.src_ip;
145 			break;
146 		case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
147 		case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
148 			hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3];
149 			hv1 ^= e->ipv6.ports;
150 
151 			hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2];
152 			hv2 ^= e->ipv6.dest_ip[0];
153 
154 			hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1];
155 			hv3 ^= e->ipv6.src_ip[0];
156 			break;
157 		case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
158 		case MTK_PPE_PKT_TYPE_IPV6_6RD:
159 		default:
160 			WARN_ON_ONCE(1);
161 			return MTK_PPE_HASH_MASK;
162 	}
163 
164 	hash = (hv1 & hv2) | ((~hv1) & hv3);
165 	hash = (hash >> 24) | ((hash & 0xffffff) << 8);
166 	hash ^= hv1 ^ hv2 ^ hv3;
167 	hash ^= hash >> 16;
168 	hash <<= (ffs(eth->soc->hash_offset) - 1);
169 	hash &= MTK_PPE_ENTRIES - 1;
170 
171 	return hash;
172 }
173 
174 static inline struct mtk_foe_mac_info *
175 mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry)
176 {
177 	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
178 
179 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
180 		return &entry->bridge.l2;
181 
182 	if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
183 		return &entry->ipv6.l2;
184 
185 	return &entry->ipv4.l2;
186 }
187 
188 static inline u32 *
189 mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry)
190 {
191 	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
192 
193 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
194 		return &entry->bridge.ib2;
195 
196 	if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
197 		return &entry->ipv6.ib2;
198 
199 	return &entry->ipv4.ib2;
200 }
201 
202 int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry,
203 			  int type, int l4proto, u8 pse_port, u8 *src_mac,
204 			  u8 *dest_mac)
205 {
206 	struct mtk_foe_mac_info *l2;
207 	u32 ports_pad, val;
208 
209 	memset(entry, 0, sizeof(*entry));
210 
211 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
212 		val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
213 		      FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) |
214 		      FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
215 		      MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2;
216 		entry->ib1 = val;
217 
218 		val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) |
219 		      FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf);
220 	} else {
221 		int port_mg = eth->soc->offload_version > 1 ? 0 : 0x3f;
222 
223 		val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
224 		      FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) |
225 		      FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
226 		      MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL;
227 		entry->ib1 = val;
228 
229 		val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) |
230 		      FIELD_PREP(MTK_FOE_IB2_PORT_MG, port_mg) |
231 		      FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f);
232 	}
233 
234 	if (is_multicast_ether_addr(dest_mac))
235 		val |= mtk_get_ib2_multicast_mask(eth);
236 
237 	ports_pad = 0xa5a5a500 | (l4proto & 0xff);
238 	if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
239 		entry->ipv4.orig.ports = ports_pad;
240 	if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
241 		entry->ipv6.ports = ports_pad;
242 
243 	if (type == MTK_PPE_PKT_TYPE_BRIDGE) {
244 		ether_addr_copy(entry->bridge.src_mac, src_mac);
245 		ether_addr_copy(entry->bridge.dest_mac, dest_mac);
246 		entry->bridge.ib2 = val;
247 		l2 = &entry->bridge.l2;
248 	} else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) {
249 		entry->ipv6.ib2 = val;
250 		l2 = &entry->ipv6.l2;
251 	} else {
252 		entry->ipv4.ib2 = val;
253 		l2 = &entry->ipv4.l2;
254 	}
255 
256 	l2->dest_mac_hi = get_unaligned_be32(dest_mac);
257 	l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4);
258 	l2->src_mac_hi = get_unaligned_be32(src_mac);
259 	l2->src_mac_lo = get_unaligned_be16(src_mac + 4);
260 
261 	if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
262 		l2->etype = ETH_P_IPV6;
263 	else
264 		l2->etype = ETH_P_IP;
265 
266 	return 0;
267 }
268 
269 int mtk_foe_entry_set_pse_port(struct mtk_eth *eth,
270 			       struct mtk_foe_entry *entry, u8 port)
271 {
272 	u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
273 	u32 val = *ib2;
274 
275 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
276 		val &= ~MTK_FOE_IB2_DEST_PORT_V2;
277 		val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port);
278 	} else {
279 		val &= ~MTK_FOE_IB2_DEST_PORT;
280 		val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port);
281 	}
282 	*ib2 = val;
283 
284 	return 0;
285 }
286 
287 int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth,
288 				 struct mtk_foe_entry *entry, bool egress,
289 				 __be32 src_addr, __be16 src_port,
290 				 __be32 dest_addr, __be16 dest_port)
291 {
292 	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
293 	struct mtk_ipv4_tuple *t;
294 
295 	switch (type) {
296 	case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
297 		if (egress) {
298 			t = &entry->ipv4.new;
299 			break;
300 		}
301 		fallthrough;
302 	case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
303 	case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
304 		t = &entry->ipv4.orig;
305 		break;
306 	case MTK_PPE_PKT_TYPE_IPV6_6RD:
307 		entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr);
308 		entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr);
309 		return 0;
310 	default:
311 		WARN_ON_ONCE(1);
312 		return -EINVAL;
313 	}
314 
315 	t->src_ip = be32_to_cpu(src_addr);
316 	t->dest_ip = be32_to_cpu(dest_addr);
317 
318 	if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
319 		return 0;
320 
321 	t->src_port = be16_to_cpu(src_port);
322 	t->dest_port = be16_to_cpu(dest_port);
323 
324 	return 0;
325 }
326 
327 int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth,
328 				 struct mtk_foe_entry *entry,
329 				 __be32 *src_addr, __be16 src_port,
330 				 __be32 *dest_addr, __be16 dest_port)
331 {
332 	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
333 	u32 *src, *dest;
334 	int i;
335 
336 	switch (type) {
337 	case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
338 		src = entry->dslite.tunnel_src_ip;
339 		dest = entry->dslite.tunnel_dest_ip;
340 		break;
341 	case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
342 	case MTK_PPE_PKT_TYPE_IPV6_6RD:
343 		entry->ipv6.src_port = be16_to_cpu(src_port);
344 		entry->ipv6.dest_port = be16_to_cpu(dest_port);
345 		fallthrough;
346 	case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
347 		src = entry->ipv6.src_ip;
348 		dest = entry->ipv6.dest_ip;
349 		break;
350 	default:
351 		WARN_ON_ONCE(1);
352 		return -EINVAL;
353 	}
354 
355 	for (i = 0; i < 4; i++)
356 		src[i] = be32_to_cpu(src_addr[i]);
357 	for (i = 0; i < 4; i++)
358 		dest[i] = be32_to_cpu(dest_addr[i]);
359 
360 	return 0;
361 }
362 
363 int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry,
364 			  int port)
365 {
366 	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
367 
368 	l2->etype = BIT(port);
369 
370 	if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)))
371 		entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1);
372 	else
373 		l2->etype |= BIT(8);
374 
375 	entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth);
376 
377 	return 0;
378 }
379 
380 int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry,
381 			   int vid)
382 {
383 	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
384 
385 	switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) {
386 	case 0:
387 		entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) |
388 			      mtk_prep_ib1_vlan_layer(eth, 1);
389 		l2->vlan1 = vid;
390 		return 0;
391 	case 1:
392 		if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) {
393 			l2->vlan1 = vid;
394 			l2->etype |= BIT(8);
395 		} else {
396 			l2->vlan2 = vid;
397 			entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1);
398 		}
399 		return 0;
400 	default:
401 		return -ENOSPC;
402 	}
403 }
404 
405 int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry,
406 			    int sid)
407 {
408 	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
409 
410 	if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) ||
411 	    (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth)))
412 		l2->etype = ETH_P_PPP_SES;
413 
414 	entry->ib1 |= mtk_get_ib1_ppoe_mask(eth);
415 	l2->pppoe_id = sid;
416 
417 	return 0;
418 }
419 
420 int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry,
421 			   int wdma_idx, int txq, int bss, int wcid)
422 {
423 	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
424 	u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
425 
426 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
427 		*ib2 &= ~MTK_FOE_IB2_PORT_MG_V2;
428 		*ib2 |=  FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) |
429 			 MTK_FOE_IB2_WDMA_WINFO_V2;
430 		l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) |
431 			    FIELD_PREP(MTK_FOE_WINFO_BSS, bss);
432 	} else {
433 		*ib2 &= ~MTK_FOE_IB2_PORT_MG;
434 		*ib2 |= MTK_FOE_IB2_WDMA_WINFO;
435 		if (wdma_idx)
436 			*ib2 |= MTK_FOE_IB2_WDMA_DEVIDX;
437 		l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) |
438 			    FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) |
439 			    FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq);
440 	}
441 
442 	return 0;
443 }
444 
445 int mtk_foe_entry_set_queue(struct mtk_eth *eth, struct mtk_foe_entry *entry,
446 			    unsigned int queue)
447 {
448 	u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
449 
450 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
451 		*ib2 &= ~MTK_FOE_IB2_QID_V2;
452 		*ib2 |= FIELD_PREP(MTK_FOE_IB2_QID_V2, queue);
453 		*ib2 |= MTK_FOE_IB2_PSE_QOS_V2;
454 	} else {
455 		*ib2 &= ~MTK_FOE_IB2_QID;
456 		*ib2 |= FIELD_PREP(MTK_FOE_IB2_QID, queue);
457 		*ib2 |= MTK_FOE_IB2_PSE_QOS;
458 	}
459 
460 	return 0;
461 }
462 
463 static bool
464 mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry,
465 		     struct mtk_foe_entry *data)
466 {
467 	int type, len;
468 
469 	if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP)
470 		return false;
471 
472 	type = mtk_get_ib1_pkt_type(eth, entry->data.ib1);
473 	if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE)
474 		len = offsetof(struct mtk_foe_entry, ipv6._rsv);
475 	else
476 		len = offsetof(struct mtk_foe_entry, ipv4.ib2);
477 
478 	return !memcmp(&entry->data.data, &data->data, len - 4);
479 }
480 
481 static void
482 __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
483 {
484 	struct hlist_head *head;
485 	struct hlist_node *tmp;
486 
487 	if (entry->type == MTK_FLOW_TYPE_L2) {
488 		rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node,
489 				       mtk_flow_l2_ht_params);
490 
491 		head = &entry->l2_flows;
492 		hlist_for_each_entry_safe(entry, tmp, head, l2_data.list)
493 			__mtk_foe_entry_clear(ppe, entry);
494 		return;
495 	}
496 
497 	hlist_del_init(&entry->list);
498 	if (entry->hash != 0xffff) {
499 		struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash);
500 
501 		hwe->ib1 &= ~MTK_FOE_IB1_STATE;
502 		hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID);
503 		dma_wmb();
504 		mtk_ppe_cache_clear(ppe);
505 
506 		if (ppe->accounting) {
507 			struct mtk_foe_accounting *acct;
508 
509 			acct = ppe->acct_table + entry->hash * sizeof(*acct);
510 			acct->packets = 0;
511 			acct->bytes = 0;
512 		}
513 	}
514 	entry->hash = 0xffff;
515 
516 	if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW)
517 		return;
518 
519 	hlist_del_init(&entry->l2_data.list);
520 	kfree(entry);
521 }
522 
523 static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1)
524 {
525 	u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);
526 	u16 now = mtk_eth_timestamp(ppe->eth);
527 	u16 timestamp = ib1 & ib1_ts_mask;
528 
529 	if (timestamp > now)
530 		return ib1_ts_mask + 1 - timestamp + now;
531 	else
532 		return now - timestamp;
533 }
534 
535 static void
536 mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
537 {
538 	u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);
539 	struct mtk_flow_entry *cur;
540 	struct mtk_foe_entry *hwe;
541 	struct hlist_node *tmp;
542 	int idle;
543 
544 	idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1);
545 	hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) {
546 		int cur_idle;
547 		u32 ib1;
548 
549 		hwe = mtk_foe_get_entry(ppe, cur->hash);
550 		ib1 = READ_ONCE(hwe->ib1);
551 
552 		if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) {
553 			cur->hash = 0xffff;
554 			__mtk_foe_entry_clear(ppe, cur);
555 			continue;
556 		}
557 
558 		cur_idle = __mtk_foe_entry_idle_time(ppe, ib1);
559 		if (cur_idle >= idle)
560 			continue;
561 
562 		idle = cur_idle;
563 		entry->data.ib1 &= ~ib1_ts_mask;
564 		entry->data.ib1 |= hwe->ib1 & ib1_ts_mask;
565 	}
566 }
567 
568 static void
569 mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
570 {
571 	struct mtk_foe_entry foe = {};
572 	struct mtk_foe_entry *hwe;
573 
574 	spin_lock_bh(&ppe_lock);
575 
576 	if (entry->type == MTK_FLOW_TYPE_L2) {
577 		mtk_flow_entry_update_l2(ppe, entry);
578 		goto out;
579 	}
580 
581 	if (entry->hash == 0xffff)
582 		goto out;
583 
584 	hwe = mtk_foe_get_entry(ppe, entry->hash);
585 	memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size);
586 	if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) {
587 		entry->hash = 0xffff;
588 		goto out;
589 	}
590 
591 	entry->data.ib1 = foe.ib1;
592 
593 out:
594 	spin_unlock_bh(&ppe_lock);
595 }
596 
597 static void
598 __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
599 		       u16 hash)
600 {
601 	struct mtk_eth *eth = ppe->eth;
602 	u16 timestamp = mtk_eth_timestamp(eth);
603 	struct mtk_foe_entry *hwe;
604 	u32 val;
605 
606 	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
607 		entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2;
608 		entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2,
609 					 timestamp);
610 	} else {
611 		entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
612 		entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP,
613 					 timestamp);
614 	}
615 
616 	hwe = mtk_foe_get_entry(ppe, hash);
617 	memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1));
618 	wmb();
619 	hwe->ib1 = entry->ib1;
620 
621 	if (ppe->accounting) {
622 		if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
623 			val = MTK_FOE_IB2_MIB_CNT_V2;
624 		else
625 			val = MTK_FOE_IB2_MIB_CNT;
626 		*mtk_foe_entry_ib2(eth, hwe) |= val;
627 	}
628 
629 	dma_wmb();
630 
631 	mtk_ppe_cache_clear(ppe);
632 }
633 
634 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
635 {
636 	spin_lock_bh(&ppe_lock);
637 	__mtk_foe_entry_clear(ppe, entry);
638 	spin_unlock_bh(&ppe_lock);
639 }
640 
641 static int
642 mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
643 {
644 	struct mtk_flow_entry *prev;
645 
646 	entry->type = MTK_FLOW_TYPE_L2;
647 
648 	prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &entry->l2_node,
649 						 mtk_flow_l2_ht_params);
650 	if (likely(!prev))
651 		return 0;
652 
653 	if (IS_ERR(prev))
654 		return PTR_ERR(prev);
655 
656 	return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node,
657 				       &entry->l2_node, mtk_flow_l2_ht_params);
658 }
659 
660 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
661 {
662 	const struct mtk_soc_data *soc = ppe->eth->soc;
663 	int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1);
664 	u32 hash;
665 
666 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
667 		return mtk_foe_entry_commit_l2(ppe, entry);
668 
669 	hash = mtk_ppe_hash_entry(ppe->eth, &entry->data);
670 	entry->hash = 0xffff;
671 	spin_lock_bh(&ppe_lock);
672 	hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]);
673 	spin_unlock_bh(&ppe_lock);
674 
675 	return 0;
676 }
677 
678 static void
679 mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry,
680 			     u16 hash)
681 {
682 	const struct mtk_soc_data *soc = ppe->eth->soc;
683 	struct mtk_flow_entry *flow_info;
684 	struct mtk_foe_entry foe = {}, *hwe;
685 	struct mtk_foe_mac_info *l2;
686 	u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP;
687 	int type;
688 
689 	flow_info = kzalloc(sizeof(*flow_info), GFP_ATOMIC);
690 	if (!flow_info)
691 		return;
692 
693 	flow_info->l2_data.base_flow = entry;
694 	flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW;
695 	flow_info->hash = hash;
696 	hlist_add_head(&flow_info->list,
697 		       &ppe->foe_flow[hash / soc->hash_offset]);
698 	hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows);
699 
700 	hwe = mtk_foe_get_entry(ppe, hash);
701 	memcpy(&foe, hwe, soc->foe_entry_size);
702 	foe.ib1 &= ib1_mask;
703 	foe.ib1 |= entry->data.ib1 & ~ib1_mask;
704 
705 	l2 = mtk_foe_entry_l2(ppe->eth, &foe);
706 	memcpy(l2, &entry->data.bridge.l2, sizeof(*l2));
707 
708 	type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1);
709 	if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT)
710 		memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new));
711 	else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP)
712 		l2->etype = ETH_P_IPV6;
713 
714 	*mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2;
715 
716 	__mtk_foe_entry_commit(ppe, &foe, hash);
717 }
718 
719 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
720 {
721 	const struct mtk_soc_data *soc = ppe->eth->soc;
722 	struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset];
723 	struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash);
724 	struct mtk_flow_entry *entry;
725 	struct mtk_foe_bridge key = {};
726 	struct hlist_node *n;
727 	struct ethhdr *eh;
728 	bool found = false;
729 	u8 *tag;
730 
731 	spin_lock_bh(&ppe_lock);
732 
733 	if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND)
734 		goto out;
735 
736 	hlist_for_each_entry_safe(entry, n, head, list) {
737 		if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) {
738 			if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) ==
739 				     MTK_FOE_STATE_BIND))
740 				continue;
741 
742 			entry->hash = 0xffff;
743 			__mtk_foe_entry_clear(ppe, entry);
744 			continue;
745 		}
746 
747 		if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) {
748 			if (entry->hash != 0xffff)
749 				entry->hash = 0xffff;
750 			continue;
751 		}
752 
753 		entry->hash = hash;
754 		__mtk_foe_entry_commit(ppe, &entry->data, hash);
755 		found = true;
756 	}
757 
758 	if (found)
759 		goto out;
760 
761 	eh = eth_hdr(skb);
762 	ether_addr_copy(key.dest_mac, eh->h_dest);
763 	ether_addr_copy(key.src_mac, eh->h_source);
764 	tag = skb->data - 2;
765 	key.vlan = 0;
766 	switch (skb->protocol) {
767 #if IS_ENABLED(CONFIG_NET_DSA)
768 	case htons(ETH_P_XDSA):
769 		if (!netdev_uses_dsa(skb->dev) ||
770 		    skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK)
771 			goto out;
772 
773 		if (!skb_metadata_dst(skb))
774 			tag += 4;
775 
776 		if (get_unaligned_be16(tag) != ETH_P_8021Q)
777 			break;
778 
779 		fallthrough;
780 #endif
781 	case htons(ETH_P_8021Q):
782 		key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK;
783 		break;
784 	default:
785 		break;
786 	}
787 
788 	entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params);
789 	if (!entry)
790 		goto out;
791 
792 	mtk_foe_entry_commit_subflow(ppe, entry, hash);
793 
794 out:
795 	spin_unlock_bh(&ppe_lock);
796 }
797 
798 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
799 {
800 	mtk_flow_entry_update(ppe, entry);
801 
802 	return __mtk_foe_entry_idle_time(ppe, entry->data.ib1);
803 }
804 
805 int mtk_ppe_prepare_reset(struct mtk_ppe *ppe)
806 {
807 	if (!ppe)
808 		return -EINVAL;
809 
810 	/* disable KA */
811 	ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE);
812 	ppe_clear(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE);
813 	ppe_w32(ppe, MTK_PPE_KEEPALIVE, 0);
814 	usleep_range(10000, 11000);
815 
816 	/* set KA timer to maximum */
817 	ppe_set(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE);
818 	ppe_w32(ppe, MTK_PPE_KEEPALIVE, 0xffffffff);
819 
820 	/* set KA tick select */
821 	ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_TICK_SEL);
822 	ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE);
823 	usleep_range(10000, 11000);
824 
825 	/* disable scan mode */
826 	ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_SCAN_MODE);
827 	usleep_range(10000, 11000);
828 
829 	return mtk_ppe_wait_busy(ppe);
830 }
831 
832 struct mtk_foe_accounting *mtk_foe_entry_get_mib(struct mtk_ppe *ppe, u32 index,
833 						 struct mtk_foe_accounting *diff)
834 {
835 	struct mtk_foe_accounting *acct;
836 	int size = sizeof(struct mtk_foe_accounting);
837 	u64 bytes, packets;
838 
839 	if (!ppe->accounting)
840 		return NULL;
841 
842 	if (mtk_mib_entry_read(ppe, index, &bytes, &packets))
843 		return NULL;
844 
845 	acct = ppe->acct_table + index * size;
846 
847 	acct->bytes += bytes;
848 	acct->packets += packets;
849 
850 	if (diff) {
851 		diff->bytes = bytes;
852 		diff->packets = packets;
853 	}
854 
855 	return acct;
856 }
857 
858 struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index)
859 {
860 	bool accounting = eth->soc->has_accounting;
861 	const struct mtk_soc_data *soc = eth->soc;
862 	struct mtk_foe_accounting *acct;
863 	struct device *dev = eth->dev;
864 	struct mtk_mib_entry *mib;
865 	struct mtk_ppe *ppe;
866 	u32 foe_flow_size;
867 	void *foe;
868 
869 	ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL);
870 	if (!ppe)
871 		return NULL;
872 
873 	rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params);
874 
875 	/* need to allocate a separate device, since it PPE DMA access is
876 	 * not coherent.
877 	 */
878 	ppe->base = base;
879 	ppe->eth = eth;
880 	ppe->dev = dev;
881 	ppe->version = eth->soc->offload_version;
882 	ppe->accounting = accounting;
883 
884 	foe = dmam_alloc_coherent(ppe->dev,
885 				  MTK_PPE_ENTRIES * soc->foe_entry_size,
886 				  &ppe->foe_phys, GFP_KERNEL);
887 	if (!foe)
888 		goto err_free_l2_flows;
889 
890 	ppe->foe_table = foe;
891 
892 	foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) *
893 			sizeof(*ppe->foe_flow);
894 	ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL);
895 	if (!ppe->foe_flow)
896 		goto err_free_l2_flows;
897 
898 	if (accounting) {
899 		mib = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*mib),
900 					  &ppe->mib_phys, GFP_KERNEL);
901 		if (!mib)
902 			return NULL;
903 
904 		ppe->mib_table = mib;
905 
906 		acct = devm_kzalloc(dev, MTK_PPE_ENTRIES * sizeof(*acct),
907 				    GFP_KERNEL);
908 
909 		if (!acct)
910 			return NULL;
911 
912 		ppe->acct_table = acct;
913 	}
914 
915 	mtk_ppe_debugfs_init(ppe, index);
916 
917 	return ppe;
918 
919 err_free_l2_flows:
920 	rhashtable_destroy(&ppe->l2_flows);
921 	return NULL;
922 }
923 
924 void mtk_ppe_deinit(struct mtk_eth *eth)
925 {
926 	int i;
927 
928 	for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) {
929 		if (!eth->ppe[i])
930 			return;
931 		rhashtable_destroy(&eth->ppe[i]->l2_flows);
932 	}
933 }
934 
935 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
936 {
937 	static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 };
938 	int i, k;
939 
940 	memset(ppe->foe_table, 0,
941 	       MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size);
942 
943 	if (!IS_ENABLED(CONFIG_SOC_MT7621))
944 		return;
945 
946 	/* skip all entries that cross the 1024 byte boundary */
947 	for (i = 0; i < MTK_PPE_ENTRIES; i += 128) {
948 		for (k = 0; k < ARRAY_SIZE(skip); k++) {
949 			struct mtk_foe_entry *hwe;
950 
951 			hwe = mtk_foe_get_entry(ppe, i + skip[k]);
952 			hwe->ib1 |= MTK_FOE_IB1_STATIC;
953 		}
954 	}
955 }
956 
957 void mtk_ppe_start(struct mtk_ppe *ppe)
958 {
959 	u32 val;
960 
961 	if (!ppe)
962 		return;
963 
964 	mtk_ppe_init_foe_table(ppe);
965 	ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys);
966 
967 	val = MTK_PPE_TB_CFG_ENTRY_80B |
968 	      MTK_PPE_TB_CFG_AGE_NON_L4 |
969 	      MTK_PPE_TB_CFG_AGE_UNBIND |
970 	      MTK_PPE_TB_CFG_AGE_TCP |
971 	      MTK_PPE_TB_CFG_AGE_UDP |
972 	      MTK_PPE_TB_CFG_AGE_TCP_FIN |
973 	      FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS,
974 			 MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) |
975 	      FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE,
976 			 MTK_PPE_KEEPALIVE_DISABLE) |
977 	      FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) |
978 	      FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE,
979 			 MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) |
980 	      FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM,
981 			 MTK_PPE_ENTRIES_SHIFT);
982 	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2))
983 		val |= MTK_PPE_TB_CFG_INFO_SEL;
984 	ppe_w32(ppe, MTK_PPE_TB_CFG, val);
985 
986 	ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK,
987 		MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6);
988 
989 	mtk_ppe_cache_enable(ppe, true);
990 
991 	val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE |
992 	      MTK_PPE_FLOW_CFG_IP6_5T_ROUTE |
993 	      MTK_PPE_FLOW_CFG_IP6_6RD |
994 	      MTK_PPE_FLOW_CFG_IP4_NAT |
995 	      MTK_PPE_FLOW_CFG_IP4_NAPT |
996 	      MTK_PPE_FLOW_CFG_IP4_DSLITE |
997 	      MTK_PPE_FLOW_CFG_IP4_NAT_FRAG;
998 	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2))
999 		val |= MTK_PPE_MD_TOAP_BYP_CRSN0 |
1000 		       MTK_PPE_MD_TOAP_BYP_CRSN1 |
1001 		       MTK_PPE_MD_TOAP_BYP_CRSN2 |
1002 		       MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY;
1003 	else
1004 		val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG |
1005 		       MTK_PPE_FLOW_CFG_IP4_UDP_FRAG;
1006 	ppe_w32(ppe, MTK_PPE_FLOW_CFG, val);
1007 
1008 	val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) |
1009 	      FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3);
1010 	ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val);
1011 
1012 	val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) |
1013 	      FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1);
1014 	ppe_w32(ppe, MTK_PPE_BIND_AGE0, val);
1015 
1016 	val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) |
1017 	      FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7);
1018 	ppe_w32(ppe, MTK_PPE_BIND_AGE1, val);
1019 
1020 	val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF;
1021 	ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val);
1022 
1023 	val = MTK_PPE_BIND_LIMIT1_FULL |
1024 	      FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1);
1025 	ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val);
1026 
1027 	val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) |
1028 	      FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1);
1029 	ppe_w32(ppe, MTK_PPE_BIND_RATE, val);
1030 
1031 	/* enable PPE */
1032 	val = MTK_PPE_GLO_CFG_EN |
1033 	      MTK_PPE_GLO_CFG_IP4_L4_CS_DROP |
1034 	      MTK_PPE_GLO_CFG_IP4_CS_DROP |
1035 	      MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE;
1036 	ppe_w32(ppe, MTK_PPE_GLO_CFG, val);
1037 
1038 	ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0);
1039 
1040 	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) {
1041 		ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777);
1042 		ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f);
1043 	}
1044 
1045 	if (ppe->accounting && ppe->mib_phys) {
1046 		ppe_w32(ppe, MTK_PPE_MIB_TB_BASE, ppe->mib_phys);
1047 		ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_EN,
1048 			MTK_PPE_MIB_CFG_EN);
1049 		ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_RD_CLR,
1050 			MTK_PPE_MIB_CFG_RD_CLR);
1051 		ppe_m32(ppe, MTK_PPE_MIB_CACHE_CTL, MTK_PPE_MIB_CACHE_CTL_EN,
1052 			MTK_PPE_MIB_CFG_RD_CLR);
1053 	}
1054 }
1055 
1056 int mtk_ppe_stop(struct mtk_ppe *ppe)
1057 {
1058 	u32 val;
1059 	int i;
1060 
1061 	if (!ppe)
1062 		return 0;
1063 
1064 	for (i = 0; i < MTK_PPE_ENTRIES; i++) {
1065 		struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i);
1066 
1067 		hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE,
1068 				      MTK_FOE_STATE_INVALID);
1069 	}
1070 
1071 	mtk_ppe_cache_enable(ppe, false);
1072 
1073 	/* disable offload engine */
1074 	ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN);
1075 	ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0);
1076 
1077 	/* disable aging */
1078 	val = MTK_PPE_TB_CFG_AGE_NON_L4 |
1079 	      MTK_PPE_TB_CFG_AGE_UNBIND |
1080 	      MTK_PPE_TB_CFG_AGE_TCP |
1081 	      MTK_PPE_TB_CFG_AGE_UDP |
1082 	      MTK_PPE_TB_CFG_AGE_TCP_FIN;
1083 	ppe_clear(ppe, MTK_PPE_TB_CFG, val);
1084 
1085 	return mtk_ppe_wait_busy(ppe);
1086 }
1087