1ba37b7caSFelix Fietkau // SPDX-License-Identifier: GPL-2.0-only 2ba37b7caSFelix Fietkau /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ 3ba37b7caSFelix Fietkau 4ba37b7caSFelix Fietkau #include <linux/kernel.h> 5ba37b7caSFelix Fietkau #include <linux/io.h> 6c5d66587SIlya Lipnitskiy #include <linux/iopoll.h> 7ba37b7caSFelix Fietkau #include <linux/etherdevice.h> 8ba37b7caSFelix Fietkau #include <linux/platform_device.h> 933fc42deSFelix Fietkau #include <linux/if_ether.h> 1033fc42deSFelix Fietkau #include <linux/if_vlan.h> 1133fc42deSFelix Fietkau #include <net/dsa.h> 12c4f033d9SFelix Fietkau #include "mtk_eth_soc.h" 13ba37b7caSFelix Fietkau #include "mtk_ppe.h" 14ba37b7caSFelix Fietkau #include "mtk_ppe_regs.h" 15ba37b7caSFelix Fietkau 16c4f033d9SFelix Fietkau static DEFINE_SPINLOCK(ppe_lock); 17c4f033d9SFelix Fietkau 1833fc42deSFelix Fietkau static const struct rhashtable_params mtk_flow_l2_ht_params = { 1933fc42deSFelix Fietkau .head_offset = offsetof(struct mtk_flow_entry, l2_node), 2033fc42deSFelix Fietkau .key_offset = offsetof(struct mtk_flow_entry, data.bridge), 2133fc42deSFelix Fietkau .key_len = offsetof(struct mtk_foe_bridge, key_end), 2233fc42deSFelix Fietkau .automatic_shrinking = true, 2333fc42deSFelix Fietkau }; 2433fc42deSFelix Fietkau 25ba37b7caSFelix Fietkau static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val) 26ba37b7caSFelix Fietkau { 27ba37b7caSFelix Fietkau writel(val, ppe->base + reg); 28ba37b7caSFelix Fietkau } 29ba37b7caSFelix Fietkau 30ba37b7caSFelix Fietkau static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg) 31ba37b7caSFelix Fietkau { 32ba37b7caSFelix Fietkau return readl(ppe->base + reg); 33ba37b7caSFelix Fietkau } 34ba37b7caSFelix Fietkau 35ba37b7caSFelix Fietkau static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set) 36ba37b7caSFelix Fietkau { 37ba37b7caSFelix Fietkau u32 val; 38ba37b7caSFelix Fietkau 39ba37b7caSFelix Fietkau val = ppe_r32(ppe, reg); 40ba37b7caSFelix Fietkau val &= ~mask; 41ba37b7caSFelix Fietkau val |= set; 42ba37b7caSFelix Fietkau ppe_w32(ppe, reg, val); 43ba37b7caSFelix Fietkau 44ba37b7caSFelix Fietkau return val; 45ba37b7caSFelix Fietkau } 46ba37b7caSFelix Fietkau 47ba37b7caSFelix Fietkau static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val) 48ba37b7caSFelix Fietkau { 49ba37b7caSFelix Fietkau return ppe_m32(ppe, reg, 0, val); 50ba37b7caSFelix Fietkau } 51ba37b7caSFelix Fietkau 52ba37b7caSFelix Fietkau static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) 53ba37b7caSFelix Fietkau { 54ba37b7caSFelix Fietkau return ppe_m32(ppe, reg, val, 0); 55ba37b7caSFelix Fietkau } 56ba37b7caSFelix Fietkau 57c4f033d9SFelix Fietkau static u32 mtk_eth_timestamp(struct mtk_eth *eth) 58c4f033d9SFelix Fietkau { 5903a3180eSLorenzo Bianconi return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth); 60c4f033d9SFelix Fietkau } 61c4f033d9SFelix Fietkau 62ba37b7caSFelix Fietkau static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) 63ba37b7caSFelix Fietkau { 64c5d66587SIlya Lipnitskiy int ret; 65c5d66587SIlya Lipnitskiy u32 val; 66ba37b7caSFelix Fietkau 67c5d66587SIlya Lipnitskiy ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val, 68c5d66587SIlya Lipnitskiy !(val & MTK_PPE_GLO_CFG_BUSY), 69c5d66587SIlya Lipnitskiy 20, MTK_PPE_WAIT_TIMEOUT_US); 70ba37b7caSFelix Fietkau 71c5d66587SIlya Lipnitskiy if (ret) 72ba37b7caSFelix Fietkau dev_err(ppe->dev, "PPE table busy"); 73ba37b7caSFelix Fietkau 74c5d66587SIlya Lipnitskiy return ret; 75ba37b7caSFelix Fietkau } 76ba37b7caSFelix Fietkau 77ba37b7caSFelix Fietkau static void mtk_ppe_cache_clear(struct mtk_ppe *ppe) 78ba37b7caSFelix Fietkau { 79ba37b7caSFelix Fietkau ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 80ba37b7caSFelix Fietkau ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 81ba37b7caSFelix Fietkau } 82ba37b7caSFelix Fietkau 83ba37b7caSFelix Fietkau static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable) 84ba37b7caSFelix Fietkau { 85ba37b7caSFelix Fietkau mtk_ppe_cache_clear(ppe); 86ba37b7caSFelix Fietkau 87ba37b7caSFelix Fietkau ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN, 88ba37b7caSFelix Fietkau enable * MTK_PPE_CACHE_CTL_EN); 89ba37b7caSFelix Fietkau } 90ba37b7caSFelix Fietkau 91ba2fc48cSLorenzo Bianconi static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e) 92ba37b7caSFelix Fietkau { 93ba37b7caSFelix Fietkau u32 hv1, hv2, hv3; 94ba37b7caSFelix Fietkau u32 hash; 95ba37b7caSFelix Fietkau 9603a3180eSLorenzo Bianconi switch (mtk_get_ib1_pkt_type(eth, e->ib1)) { 97ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 98ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 99ba37b7caSFelix Fietkau hv1 = e->ipv4.orig.ports; 100ba37b7caSFelix Fietkau hv2 = e->ipv4.orig.dest_ip; 101ba37b7caSFelix Fietkau hv3 = e->ipv4.orig.src_ip; 102ba37b7caSFelix Fietkau break; 103ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 104ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 105ba37b7caSFelix Fietkau hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3]; 106ba37b7caSFelix Fietkau hv1 ^= e->ipv6.ports; 107ba37b7caSFelix Fietkau 108ba37b7caSFelix Fietkau hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2]; 109ba37b7caSFelix Fietkau hv2 ^= e->ipv6.dest_ip[0]; 110ba37b7caSFelix Fietkau 111ba37b7caSFelix Fietkau hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1]; 112ba37b7caSFelix Fietkau hv3 ^= e->ipv6.src_ip[0]; 113ba37b7caSFelix Fietkau break; 114ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 115ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_6RD: 116ba37b7caSFelix Fietkau default: 117ba37b7caSFelix Fietkau WARN_ON_ONCE(1); 118ba37b7caSFelix Fietkau return MTK_PPE_HASH_MASK; 119ba37b7caSFelix Fietkau } 120ba37b7caSFelix Fietkau 121ba37b7caSFelix Fietkau hash = (hv1 & hv2) | ((~hv1) & hv3); 122ba37b7caSFelix Fietkau hash = (hash >> 24) | ((hash & 0xffffff) << 8); 123ba37b7caSFelix Fietkau hash ^= hv1 ^ hv2 ^ hv3; 124ba37b7caSFelix Fietkau hash ^= hash >> 16; 125ba2fc48cSLorenzo Bianconi hash <<= (ffs(eth->soc->hash_offset) - 1); 126ba37b7caSFelix Fietkau hash &= MTK_PPE_ENTRIES - 1; 127ba37b7caSFelix Fietkau 128ba37b7caSFelix Fietkau return hash; 129ba37b7caSFelix Fietkau } 130ba37b7caSFelix Fietkau 131ba37b7caSFelix Fietkau static inline struct mtk_foe_mac_info * 13203a3180eSLorenzo Bianconi mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 133ba37b7caSFelix Fietkau { 13403a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 135ba37b7caSFelix Fietkau 13633fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_BRIDGE) 13733fc42deSFelix Fietkau return &entry->bridge.l2; 13833fc42deSFelix Fietkau 139ba37b7caSFelix Fietkau if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 140ba37b7caSFelix Fietkau return &entry->ipv6.l2; 141ba37b7caSFelix Fietkau 142ba37b7caSFelix Fietkau return &entry->ipv4.l2; 143ba37b7caSFelix Fietkau } 144ba37b7caSFelix Fietkau 145ba37b7caSFelix Fietkau static inline u32 * 14603a3180eSLorenzo Bianconi mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 147ba37b7caSFelix Fietkau { 14803a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 149ba37b7caSFelix Fietkau 15033fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_BRIDGE) 15133fc42deSFelix Fietkau return &entry->bridge.ib2; 15233fc42deSFelix Fietkau 153ba37b7caSFelix Fietkau if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 154ba37b7caSFelix Fietkau return &entry->ipv6.ib2; 155ba37b7caSFelix Fietkau 156ba37b7caSFelix Fietkau return &entry->ipv4.ib2; 157ba37b7caSFelix Fietkau } 158ba37b7caSFelix Fietkau 15903a3180eSLorenzo Bianconi int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, 16003a3180eSLorenzo Bianconi int type, int l4proto, u8 pse_port, u8 *src_mac, 16103a3180eSLorenzo Bianconi u8 *dest_mac) 162ba37b7caSFelix Fietkau { 163ba37b7caSFelix Fietkau struct mtk_foe_mac_info *l2; 164ba37b7caSFelix Fietkau u32 ports_pad, val; 165ba37b7caSFelix Fietkau 166ba37b7caSFelix Fietkau memset(entry, 0, sizeof(*entry)); 167ba37b7caSFelix Fietkau 16803a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 16903a3180eSLorenzo Bianconi val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 17003a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | 17103a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 17203a3180eSLorenzo Bianconi MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2; 17303a3180eSLorenzo Bianconi entry->ib1 = val; 17403a3180eSLorenzo Bianconi 17503a3180eSLorenzo Bianconi val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) | 17603a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf); 17703a3180eSLorenzo Bianconi } else { 178ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 179ba37b7caSFelix Fietkau FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | 180ba37b7caSFelix Fietkau FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 18103a3180eSLorenzo Bianconi MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL; 182ba37b7caSFelix Fietkau entry->ib1 = val; 183ba37b7caSFelix Fietkau 18403a3180eSLorenzo Bianconi val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) | 18503a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) | 18603a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f); 18703a3180eSLorenzo Bianconi } 188ba37b7caSFelix Fietkau 189ba37b7caSFelix Fietkau if (is_multicast_ether_addr(dest_mac)) 19003a3180eSLorenzo Bianconi val |= mtk_get_ib2_multicast_mask(eth); 191ba37b7caSFelix Fietkau 192ba37b7caSFelix Fietkau ports_pad = 0xa5a5a500 | (l4proto & 0xff); 193ba37b7caSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 194ba37b7caSFelix Fietkau entry->ipv4.orig.ports = ports_pad; 195ba37b7caSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 196ba37b7caSFelix Fietkau entry->ipv6.ports = ports_pad; 197ba37b7caSFelix Fietkau 19833fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_BRIDGE) { 19933fc42deSFelix Fietkau ether_addr_copy(entry->bridge.src_mac, src_mac); 20033fc42deSFelix Fietkau ether_addr_copy(entry->bridge.dest_mac, dest_mac); 20133fc42deSFelix Fietkau entry->bridge.ib2 = val; 20233fc42deSFelix Fietkau l2 = &entry->bridge.l2; 20333fc42deSFelix Fietkau } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { 204ba37b7caSFelix Fietkau entry->ipv6.ib2 = val; 205ba37b7caSFelix Fietkau l2 = &entry->ipv6.l2; 206ba37b7caSFelix Fietkau } else { 207ba37b7caSFelix Fietkau entry->ipv4.ib2 = val; 208ba37b7caSFelix Fietkau l2 = &entry->ipv4.l2; 209ba37b7caSFelix Fietkau } 210ba37b7caSFelix Fietkau 211ba37b7caSFelix Fietkau l2->dest_mac_hi = get_unaligned_be32(dest_mac); 212ba37b7caSFelix Fietkau l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4); 213ba37b7caSFelix Fietkau l2->src_mac_hi = get_unaligned_be32(src_mac); 214ba37b7caSFelix Fietkau l2->src_mac_lo = get_unaligned_be16(src_mac + 4); 215ba37b7caSFelix Fietkau 216ba37b7caSFelix Fietkau if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 217ba37b7caSFelix Fietkau l2->etype = ETH_P_IPV6; 218ba37b7caSFelix Fietkau else 219ba37b7caSFelix Fietkau l2->etype = ETH_P_IP; 220ba37b7caSFelix Fietkau 221ba37b7caSFelix Fietkau return 0; 222ba37b7caSFelix Fietkau } 223ba37b7caSFelix Fietkau 22403a3180eSLorenzo Bianconi int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, 22503a3180eSLorenzo Bianconi struct mtk_foe_entry *entry, u8 port) 226ba37b7caSFelix Fietkau { 22703a3180eSLorenzo Bianconi u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 22803a3180eSLorenzo Bianconi u32 val = *ib2; 229ba37b7caSFelix Fietkau 23003a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 23103a3180eSLorenzo Bianconi val &= ~MTK_FOE_IB2_DEST_PORT_V2; 23203a3180eSLorenzo Bianconi val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); 23303a3180eSLorenzo Bianconi } else { 234ba37b7caSFelix Fietkau val &= ~MTK_FOE_IB2_DEST_PORT; 235ba37b7caSFelix Fietkau val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); 23603a3180eSLorenzo Bianconi } 237ba37b7caSFelix Fietkau *ib2 = val; 238ba37b7caSFelix Fietkau 239ba37b7caSFelix Fietkau return 0; 240ba37b7caSFelix Fietkau } 241ba37b7caSFelix Fietkau 24203a3180eSLorenzo Bianconi int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, 24303a3180eSLorenzo Bianconi struct mtk_foe_entry *entry, bool egress, 244ba37b7caSFelix Fietkau __be32 src_addr, __be16 src_port, 245ba37b7caSFelix Fietkau __be32 dest_addr, __be16 dest_port) 246ba37b7caSFelix Fietkau { 24703a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 248ba37b7caSFelix Fietkau struct mtk_ipv4_tuple *t; 249ba37b7caSFelix Fietkau 250ba37b7caSFelix Fietkau switch (type) { 251ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 252ba37b7caSFelix Fietkau if (egress) { 253ba37b7caSFelix Fietkau t = &entry->ipv4.new; 254ba37b7caSFelix Fietkau break; 255ba37b7caSFelix Fietkau } 256ba37b7caSFelix Fietkau fallthrough; 257ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 258ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 259ba37b7caSFelix Fietkau t = &entry->ipv4.orig; 260ba37b7caSFelix Fietkau break; 261ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_6RD: 262ba37b7caSFelix Fietkau entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr); 263ba37b7caSFelix Fietkau entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr); 264ba37b7caSFelix Fietkau return 0; 265ba37b7caSFelix Fietkau default: 266ba37b7caSFelix Fietkau WARN_ON_ONCE(1); 267ba37b7caSFelix Fietkau return -EINVAL; 268ba37b7caSFelix Fietkau } 269ba37b7caSFelix Fietkau 270ba37b7caSFelix Fietkau t->src_ip = be32_to_cpu(src_addr); 271ba37b7caSFelix Fietkau t->dest_ip = be32_to_cpu(dest_addr); 272ba37b7caSFelix Fietkau 273ba37b7caSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 274ba37b7caSFelix Fietkau return 0; 275ba37b7caSFelix Fietkau 276ba37b7caSFelix Fietkau t->src_port = be16_to_cpu(src_port); 277ba37b7caSFelix Fietkau t->dest_port = be16_to_cpu(dest_port); 278ba37b7caSFelix Fietkau 279ba37b7caSFelix Fietkau return 0; 280ba37b7caSFelix Fietkau } 281ba37b7caSFelix Fietkau 28203a3180eSLorenzo Bianconi int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, 28303a3180eSLorenzo Bianconi struct mtk_foe_entry *entry, 284ba37b7caSFelix Fietkau __be32 *src_addr, __be16 src_port, 285ba37b7caSFelix Fietkau __be32 *dest_addr, __be16 dest_port) 286ba37b7caSFelix Fietkau { 28703a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 288ba37b7caSFelix Fietkau u32 *src, *dest; 289ba37b7caSFelix Fietkau int i; 290ba37b7caSFelix Fietkau 291ba37b7caSFelix Fietkau switch (type) { 292ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 293ba37b7caSFelix Fietkau src = entry->dslite.tunnel_src_ip; 294ba37b7caSFelix Fietkau dest = entry->dslite.tunnel_dest_ip; 295ba37b7caSFelix Fietkau break; 296ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 297ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_6RD: 298ba37b7caSFelix Fietkau entry->ipv6.src_port = be16_to_cpu(src_port); 299ba37b7caSFelix Fietkau entry->ipv6.dest_port = be16_to_cpu(dest_port); 300ba37b7caSFelix Fietkau fallthrough; 301ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 302ba37b7caSFelix Fietkau src = entry->ipv6.src_ip; 303ba37b7caSFelix Fietkau dest = entry->ipv6.dest_ip; 304ba37b7caSFelix Fietkau break; 305ba37b7caSFelix Fietkau default: 306ba37b7caSFelix Fietkau WARN_ON_ONCE(1); 307ba37b7caSFelix Fietkau return -EINVAL; 3083b2c32f9SQiheng Lin } 309ba37b7caSFelix Fietkau 310ba37b7caSFelix Fietkau for (i = 0; i < 4; i++) 311ba37b7caSFelix Fietkau src[i] = be32_to_cpu(src_addr[i]); 312ba37b7caSFelix Fietkau for (i = 0; i < 4; i++) 313ba37b7caSFelix Fietkau dest[i] = be32_to_cpu(dest_addr[i]); 314ba37b7caSFelix Fietkau 315ba37b7caSFelix Fietkau return 0; 316ba37b7caSFelix Fietkau } 317ba37b7caSFelix Fietkau 31803a3180eSLorenzo Bianconi int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, 31903a3180eSLorenzo Bianconi int port) 320ba37b7caSFelix Fietkau { 32103a3180eSLorenzo Bianconi struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 322ba37b7caSFelix Fietkau 323ba37b7caSFelix Fietkau l2->etype = BIT(port); 324ba37b7caSFelix Fietkau 32503a3180eSLorenzo Bianconi if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth))) 32603a3180eSLorenzo Bianconi entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1); 327ba37b7caSFelix Fietkau else 328ba37b7caSFelix Fietkau l2->etype |= BIT(8); 329ba37b7caSFelix Fietkau 33003a3180eSLorenzo Bianconi entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth); 331ba37b7caSFelix Fietkau 332ba37b7caSFelix Fietkau return 0; 333ba37b7caSFelix Fietkau } 334ba37b7caSFelix Fietkau 33503a3180eSLorenzo Bianconi int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, 33603a3180eSLorenzo Bianconi int vid) 337ba37b7caSFelix Fietkau { 33803a3180eSLorenzo Bianconi struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 339ba37b7caSFelix Fietkau 340fb7da771SDaniel Golle switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) { 341ba37b7caSFelix Fietkau case 0: 34203a3180eSLorenzo Bianconi entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) | 34303a3180eSLorenzo Bianconi mtk_prep_ib1_vlan_layer(eth, 1); 344ba37b7caSFelix Fietkau l2->vlan1 = vid; 345ba37b7caSFelix Fietkau return 0; 346ba37b7caSFelix Fietkau case 1: 34703a3180eSLorenzo Bianconi if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) { 348ba37b7caSFelix Fietkau l2->vlan1 = vid; 349ba37b7caSFelix Fietkau l2->etype |= BIT(8); 350ba37b7caSFelix Fietkau } else { 351ba37b7caSFelix Fietkau l2->vlan2 = vid; 35203a3180eSLorenzo Bianconi entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1); 353ba37b7caSFelix Fietkau } 354ba37b7caSFelix Fietkau return 0; 355ba37b7caSFelix Fietkau default: 356ba37b7caSFelix Fietkau return -ENOSPC; 357ba37b7caSFelix Fietkau } 358ba37b7caSFelix Fietkau } 359ba37b7caSFelix Fietkau 36003a3180eSLorenzo Bianconi int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, 36103a3180eSLorenzo Bianconi int sid) 362ba37b7caSFelix Fietkau { 36303a3180eSLorenzo Bianconi struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 364ba37b7caSFelix Fietkau 36503a3180eSLorenzo Bianconi if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) || 36603a3180eSLorenzo Bianconi (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) 367ba37b7caSFelix Fietkau l2->etype = ETH_P_PPP_SES; 368ba37b7caSFelix Fietkau 36903a3180eSLorenzo Bianconi entry->ib1 |= mtk_get_ib1_ppoe_mask(eth); 370ba37b7caSFelix Fietkau l2->pppoe_id = sid; 371ba37b7caSFelix Fietkau 372ba37b7caSFelix Fietkau return 0; 373ba37b7caSFelix Fietkau } 374ba37b7caSFelix Fietkau 37503a3180eSLorenzo Bianconi int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, 37603a3180eSLorenzo Bianconi int wdma_idx, int txq, int bss, int wcid) 377a333215eSFelix Fietkau { 37803a3180eSLorenzo Bianconi struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 37903a3180eSLorenzo Bianconi u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 380a333215eSFelix Fietkau 38103a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 38203a3180eSLorenzo Bianconi *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; 38303a3180eSLorenzo Bianconi *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | 38403a3180eSLorenzo Bianconi MTK_FOE_IB2_WDMA_WINFO_V2; 38503a3180eSLorenzo Bianconi l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | 38603a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_WINFO_BSS, bss); 38703a3180eSLorenzo Bianconi } else { 388a333215eSFelix Fietkau *ib2 &= ~MTK_FOE_IB2_PORT_MG; 389a333215eSFelix Fietkau *ib2 |= MTK_FOE_IB2_WDMA_WINFO; 390a333215eSFelix Fietkau if (wdma_idx) 391a333215eSFelix Fietkau *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; 392a333215eSFelix Fietkau l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | 393a333215eSFelix Fietkau FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | 394a333215eSFelix Fietkau FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); 39503a3180eSLorenzo Bianconi } 396a333215eSFelix Fietkau 397a333215eSFelix Fietkau return 0; 398a333215eSFelix Fietkau } 399a333215eSFelix Fietkau 400ba37b7caSFelix Fietkau static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry) 401ba37b7caSFelix Fietkau { 402ba37b7caSFelix Fietkau return !(entry->ib1 & MTK_FOE_IB1_STATIC) && 403ba37b7caSFelix Fietkau FIELD_GET(MTK_FOE_IB1_STATE, entry->ib1) != MTK_FOE_STATE_BIND; 404ba37b7caSFelix Fietkau } 405ba37b7caSFelix Fietkau 406c4f033d9SFelix Fietkau static bool 40703a3180eSLorenzo Bianconi mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry, 40803a3180eSLorenzo Bianconi struct mtk_foe_entry *data) 409c4f033d9SFelix Fietkau { 410c4f033d9SFelix Fietkau int type, len; 411c4f033d9SFelix Fietkau 412c4f033d9SFelix Fietkau if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) 413c4f033d9SFelix Fietkau return false; 414c4f033d9SFelix Fietkau 41503a3180eSLorenzo Bianconi type = mtk_get_ib1_pkt_type(eth, entry->data.ib1); 416c4f033d9SFelix Fietkau if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) 417c4f033d9SFelix Fietkau len = offsetof(struct mtk_foe_entry, ipv6._rsv); 418c4f033d9SFelix Fietkau else 419c4f033d9SFelix Fietkau len = offsetof(struct mtk_foe_entry, ipv4.ib2); 420c4f033d9SFelix Fietkau 421c4f033d9SFelix Fietkau return !memcmp(&entry->data.data, &data->data, len - 4); 422c4f033d9SFelix Fietkau } 423c4f033d9SFelix Fietkau 424c4f033d9SFelix Fietkau static void 42533fc42deSFelix Fietkau __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 42633fc42deSFelix Fietkau { 42733fc42deSFelix Fietkau struct hlist_head *head; 42833fc42deSFelix Fietkau struct hlist_node *tmp; 42933fc42deSFelix Fietkau 43033fc42deSFelix Fietkau if (entry->type == MTK_FLOW_TYPE_L2) { 43133fc42deSFelix Fietkau rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node, 43233fc42deSFelix Fietkau mtk_flow_l2_ht_params); 43333fc42deSFelix Fietkau 43433fc42deSFelix Fietkau head = &entry->l2_flows; 43533fc42deSFelix Fietkau hlist_for_each_entry_safe(entry, tmp, head, l2_data.list) 43633fc42deSFelix Fietkau __mtk_foe_entry_clear(ppe, entry); 43733fc42deSFelix Fietkau return; 43833fc42deSFelix Fietkau } 43933fc42deSFelix Fietkau 44033fc42deSFelix Fietkau hlist_del_init(&entry->list); 44133fc42deSFelix Fietkau if (entry->hash != 0xffff) { 4429d8cb4c0SLorenzo Bianconi struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash); 4439d8cb4c0SLorenzo Bianconi 4449d8cb4c0SLorenzo Bianconi hwe->ib1 &= ~MTK_FOE_IB1_STATE; 4459d8cb4c0SLorenzo Bianconi hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_UNBIND); 44633fc42deSFelix Fietkau dma_wmb(); 44733fc42deSFelix Fietkau } 44833fc42deSFelix Fietkau entry->hash = 0xffff; 44933fc42deSFelix Fietkau 45033fc42deSFelix Fietkau if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW) 45133fc42deSFelix Fietkau return; 45233fc42deSFelix Fietkau 45333fc42deSFelix Fietkau hlist_del_init(&entry->l2_data.list); 45433fc42deSFelix Fietkau kfree(entry); 45533fc42deSFelix Fietkau } 45633fc42deSFelix Fietkau 45733fc42deSFelix Fietkau static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) 45833fc42deSFelix Fietkau { 45903a3180eSLorenzo Bianconi u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 46003a3180eSLorenzo Bianconi u16 now = mtk_eth_timestamp(ppe->eth); 46103a3180eSLorenzo Bianconi u16 timestamp = ib1 & ib1_ts_mask; 46233fc42deSFelix Fietkau 46333fc42deSFelix Fietkau if (timestamp > now) 46403a3180eSLorenzo Bianconi return ib1_ts_mask + 1 - timestamp + now; 46533fc42deSFelix Fietkau else 46633fc42deSFelix Fietkau return now - timestamp; 46733fc42deSFelix Fietkau } 46833fc42deSFelix Fietkau 46933fc42deSFelix Fietkau static void 47033fc42deSFelix Fietkau mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 47133fc42deSFelix Fietkau { 47203a3180eSLorenzo Bianconi u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 47333fc42deSFelix Fietkau struct mtk_flow_entry *cur; 47433fc42deSFelix Fietkau struct mtk_foe_entry *hwe; 47533fc42deSFelix Fietkau struct hlist_node *tmp; 47633fc42deSFelix Fietkau int idle; 47733fc42deSFelix Fietkau 47833fc42deSFelix Fietkau idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 47933fc42deSFelix Fietkau hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) { 48033fc42deSFelix Fietkau int cur_idle; 48133fc42deSFelix Fietkau u32 ib1; 48233fc42deSFelix Fietkau 4839d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, cur->hash); 48433fc42deSFelix Fietkau ib1 = READ_ONCE(hwe->ib1); 48533fc42deSFelix Fietkau 48633fc42deSFelix Fietkau if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { 48733fc42deSFelix Fietkau cur->hash = 0xffff; 48833fc42deSFelix Fietkau __mtk_foe_entry_clear(ppe, cur); 48933fc42deSFelix Fietkau continue; 49033fc42deSFelix Fietkau } 49133fc42deSFelix Fietkau 49233fc42deSFelix Fietkau cur_idle = __mtk_foe_entry_idle_time(ppe, ib1); 49333fc42deSFelix Fietkau if (cur_idle >= idle) 49433fc42deSFelix Fietkau continue; 49533fc42deSFelix Fietkau 49633fc42deSFelix Fietkau idle = cur_idle; 49703a3180eSLorenzo Bianconi entry->data.ib1 &= ~ib1_ts_mask; 49803a3180eSLorenzo Bianconi entry->data.ib1 |= hwe->ib1 & ib1_ts_mask; 49933fc42deSFelix Fietkau } 50033fc42deSFelix Fietkau } 50133fc42deSFelix Fietkau 50233fc42deSFelix Fietkau static void 503c4f033d9SFelix Fietkau mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 504ba37b7caSFelix Fietkau { 5059d8cb4c0SLorenzo Bianconi struct mtk_foe_entry foe = {}; 506ba37b7caSFelix Fietkau struct mtk_foe_entry *hwe; 507ba37b7caSFelix Fietkau 508c4f033d9SFelix Fietkau spin_lock_bh(&ppe_lock); 50933fc42deSFelix Fietkau 51033fc42deSFelix Fietkau if (entry->type == MTK_FLOW_TYPE_L2) { 51133fc42deSFelix Fietkau mtk_flow_entry_update_l2(ppe, entry); 51233fc42deSFelix Fietkau goto out; 51333fc42deSFelix Fietkau } 51433fc42deSFelix Fietkau 515c4f033d9SFelix Fietkau if (entry->hash == 0xffff) 516c4f033d9SFelix Fietkau goto out; 517c4f033d9SFelix Fietkau 5189d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, entry->hash); 5199d8cb4c0SLorenzo Bianconi memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size); 52003a3180eSLorenzo Bianconi if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) { 521c4f033d9SFelix Fietkau entry->hash = 0xffff; 522c4f033d9SFelix Fietkau goto out; 523c4f033d9SFelix Fietkau } 524c4f033d9SFelix Fietkau 525c4f033d9SFelix Fietkau entry->data.ib1 = foe.ib1; 526c4f033d9SFelix Fietkau 527c4f033d9SFelix Fietkau out: 528c4f033d9SFelix Fietkau spin_unlock_bh(&ppe_lock); 529c4f033d9SFelix Fietkau } 530c4f033d9SFelix Fietkau 531c4f033d9SFelix Fietkau static void 532c4f033d9SFelix Fietkau __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, 533c4f033d9SFelix Fietkau u16 hash) 534c4f033d9SFelix Fietkau { 53503a3180eSLorenzo Bianconi struct mtk_eth *eth = ppe->eth; 53603a3180eSLorenzo Bianconi u16 timestamp = mtk_eth_timestamp(eth); 537c4f033d9SFelix Fietkau struct mtk_foe_entry *hwe; 538c4f033d9SFelix Fietkau 53903a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 54003a3180eSLorenzo Bianconi entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; 54103a3180eSLorenzo Bianconi entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, 54203a3180eSLorenzo Bianconi timestamp); 54303a3180eSLorenzo Bianconi } else { 544ba37b7caSFelix Fietkau entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; 54503a3180eSLorenzo Bianconi entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, 54603a3180eSLorenzo Bianconi timestamp); 54703a3180eSLorenzo Bianconi } 548ba37b7caSFelix Fietkau 5499d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, hash); 550*454b20e1SDaniel Golle memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1)); 551ba37b7caSFelix Fietkau wmb(); 552ba37b7caSFelix Fietkau hwe->ib1 = entry->ib1; 553ba37b7caSFelix Fietkau 554ba37b7caSFelix Fietkau dma_wmb(); 555ba37b7caSFelix Fietkau 556ba37b7caSFelix Fietkau mtk_ppe_cache_clear(ppe); 557ba37b7caSFelix Fietkau } 558ba37b7caSFelix Fietkau 559c4f033d9SFelix Fietkau void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 560c4f033d9SFelix Fietkau { 561c4f033d9SFelix Fietkau spin_lock_bh(&ppe_lock); 56233fc42deSFelix Fietkau __mtk_foe_entry_clear(ppe, entry); 563c4f033d9SFelix Fietkau spin_unlock_bh(&ppe_lock); 564c4f033d9SFelix Fietkau } 565c4f033d9SFelix Fietkau 56633fc42deSFelix Fietkau static int 56733fc42deSFelix Fietkau mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 56833fc42deSFelix Fietkau { 56933fc42deSFelix Fietkau entry->type = MTK_FLOW_TYPE_L2; 57033fc42deSFelix Fietkau 57133fc42deSFelix Fietkau return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node, 57233fc42deSFelix Fietkau mtk_flow_l2_ht_params); 57333fc42deSFelix Fietkau } 57433fc42deSFelix Fietkau 575c4f033d9SFelix Fietkau int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 576c4f033d9SFelix Fietkau { 577ba2fc48cSLorenzo Bianconi const struct mtk_soc_data *soc = ppe->eth->soc; 57803a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1); 57933fc42deSFelix Fietkau u32 hash; 580c4f033d9SFelix Fietkau 58133fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_BRIDGE) 58233fc42deSFelix Fietkau return mtk_foe_entry_commit_l2(ppe, entry); 58333fc42deSFelix Fietkau 584ba2fc48cSLorenzo Bianconi hash = mtk_ppe_hash_entry(ppe->eth, &entry->data); 585c4f033d9SFelix Fietkau entry->hash = 0xffff; 586c4f033d9SFelix Fietkau spin_lock_bh(&ppe_lock); 587ba2fc48cSLorenzo Bianconi hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]); 588c4f033d9SFelix Fietkau spin_unlock_bh(&ppe_lock); 589c4f033d9SFelix Fietkau 590c4f033d9SFelix Fietkau return 0; 591c4f033d9SFelix Fietkau } 592c4f033d9SFelix Fietkau 59333fc42deSFelix Fietkau static void 59433fc42deSFelix Fietkau mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, 59533fc42deSFelix Fietkau u16 hash) 59633fc42deSFelix Fietkau { 597ba2fc48cSLorenzo Bianconi const struct mtk_soc_data *soc = ppe->eth->soc; 59833fc42deSFelix Fietkau struct mtk_flow_entry *flow_info; 5999d8cb4c0SLorenzo Bianconi struct mtk_foe_entry foe = {}, *hwe; 60033fc42deSFelix Fietkau struct mtk_foe_mac_info *l2; 60103a3180eSLorenzo Bianconi u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP; 60233fc42deSFelix Fietkau int type; 60333fc42deSFelix Fietkau 60433fc42deSFelix Fietkau flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end), 60533fc42deSFelix Fietkau GFP_ATOMIC); 60633fc42deSFelix Fietkau if (!flow_info) 60733fc42deSFelix Fietkau return; 60833fc42deSFelix Fietkau 60933fc42deSFelix Fietkau flow_info->l2_data.base_flow = entry; 61033fc42deSFelix Fietkau flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; 61133fc42deSFelix Fietkau flow_info->hash = hash; 612ba2fc48cSLorenzo Bianconi hlist_add_head(&flow_info->list, 613ba2fc48cSLorenzo Bianconi &ppe->foe_flow[hash / soc->hash_offset]); 61433fc42deSFelix Fietkau hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows); 61533fc42deSFelix Fietkau 6169d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, hash); 6179d8cb4c0SLorenzo Bianconi memcpy(&foe, hwe, soc->foe_entry_size); 61833fc42deSFelix Fietkau foe.ib1 &= ib1_mask; 61933fc42deSFelix Fietkau foe.ib1 |= entry->data.ib1 & ~ib1_mask; 62033fc42deSFelix Fietkau 62103a3180eSLorenzo Bianconi l2 = mtk_foe_entry_l2(ppe->eth, &foe); 62233fc42deSFelix Fietkau memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); 62333fc42deSFelix Fietkau 62403a3180eSLorenzo Bianconi type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1); 62533fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) 62633fc42deSFelix Fietkau memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); 62733fc42deSFelix Fietkau else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) 62833fc42deSFelix Fietkau l2->etype = ETH_P_IPV6; 62933fc42deSFelix Fietkau 63003a3180eSLorenzo Bianconi *mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2; 63133fc42deSFelix Fietkau 63233fc42deSFelix Fietkau __mtk_foe_entry_commit(ppe, &foe, hash); 63333fc42deSFelix Fietkau } 63433fc42deSFelix Fietkau 635c4f033d9SFelix Fietkau void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) 636c4f033d9SFelix Fietkau { 637ba2fc48cSLorenzo Bianconi const struct mtk_soc_data *soc = ppe->eth->soc; 638ba2fc48cSLorenzo Bianconi struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset]; 6399d8cb4c0SLorenzo Bianconi struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash); 64033fc42deSFelix Fietkau struct mtk_flow_entry *entry; 64133fc42deSFelix Fietkau struct mtk_foe_bridge key = {}; 64217a5f6a7SDan Carpenter struct hlist_node *n; 64333fc42deSFelix Fietkau struct ethhdr *eh; 644c4f033d9SFelix Fietkau bool found = false; 64533fc42deSFelix Fietkau u8 *tag; 646c4f033d9SFelix Fietkau 647c4f033d9SFelix Fietkau spin_lock_bh(&ppe_lock); 64833fc42deSFelix Fietkau 64933fc42deSFelix Fietkau if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND) 65033fc42deSFelix Fietkau goto out; 65133fc42deSFelix Fietkau 65217a5f6a7SDan Carpenter hlist_for_each_entry_safe(entry, n, head, list) { 65333fc42deSFelix Fietkau if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) { 65433fc42deSFelix Fietkau if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == 65533fc42deSFelix Fietkau MTK_FOE_STATE_BIND)) 65633fc42deSFelix Fietkau continue; 65733fc42deSFelix Fietkau 65833fc42deSFelix Fietkau entry->hash = 0xffff; 65933fc42deSFelix Fietkau __mtk_foe_entry_clear(ppe, entry); 66033fc42deSFelix Fietkau continue; 66133fc42deSFelix Fietkau } 66233fc42deSFelix Fietkau 66303a3180eSLorenzo Bianconi if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) { 664c4f033d9SFelix Fietkau if (entry->hash != 0xffff) 665c4f033d9SFelix Fietkau entry->hash = 0xffff; 666c4f033d9SFelix Fietkau continue; 667c4f033d9SFelix Fietkau } 668c4f033d9SFelix Fietkau 669c4f033d9SFelix Fietkau entry->hash = hash; 670c4f033d9SFelix Fietkau __mtk_foe_entry_commit(ppe, &entry->data, hash); 671c4f033d9SFelix Fietkau found = true; 672c4f033d9SFelix Fietkau } 67333fc42deSFelix Fietkau 67433fc42deSFelix Fietkau if (found) 67533fc42deSFelix Fietkau goto out; 67633fc42deSFelix Fietkau 67733fc42deSFelix Fietkau eh = eth_hdr(skb); 67833fc42deSFelix Fietkau ether_addr_copy(key.dest_mac, eh->h_dest); 67933fc42deSFelix Fietkau ether_addr_copy(key.src_mac, eh->h_source); 68033fc42deSFelix Fietkau tag = skb->data - 2; 68133fc42deSFelix Fietkau key.vlan = 0; 68233fc42deSFelix Fietkau switch (skb->protocol) { 68333fc42deSFelix Fietkau #if IS_ENABLED(CONFIG_NET_DSA) 68433fc42deSFelix Fietkau case htons(ETH_P_XDSA): 68533fc42deSFelix Fietkau if (!netdev_uses_dsa(skb->dev) || 68633fc42deSFelix Fietkau skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) 68733fc42deSFelix Fietkau goto out; 68833fc42deSFelix Fietkau 68933fc42deSFelix Fietkau tag += 4; 69033fc42deSFelix Fietkau if (get_unaligned_be16(tag) != ETH_P_8021Q) 69133fc42deSFelix Fietkau break; 69233fc42deSFelix Fietkau 69333fc42deSFelix Fietkau fallthrough; 69433fc42deSFelix Fietkau #endif 69533fc42deSFelix Fietkau case htons(ETH_P_8021Q): 69633fc42deSFelix Fietkau key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK; 69733fc42deSFelix Fietkau break; 69833fc42deSFelix Fietkau default: 69933fc42deSFelix Fietkau break; 70033fc42deSFelix Fietkau } 70133fc42deSFelix Fietkau 70233fc42deSFelix Fietkau entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params); 70333fc42deSFelix Fietkau if (!entry) 70433fc42deSFelix Fietkau goto out; 70533fc42deSFelix Fietkau 70633fc42deSFelix Fietkau mtk_foe_entry_commit_subflow(ppe, entry, hash); 70733fc42deSFelix Fietkau 70833fc42deSFelix Fietkau out: 709c4f033d9SFelix Fietkau spin_unlock_bh(&ppe_lock); 710c4f033d9SFelix Fietkau } 711c4f033d9SFelix Fietkau 712c4f033d9SFelix Fietkau int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 713c4f033d9SFelix Fietkau { 714c4f033d9SFelix Fietkau mtk_flow_entry_update(ppe, entry); 715c4f033d9SFelix Fietkau 71633fc42deSFelix Fietkau return __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 717c4f033d9SFelix Fietkau } 718c4f033d9SFelix Fietkau 719c4f033d9SFelix Fietkau struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, 7204ff1a3fcSLorenzo Bianconi int version, int index) 721ba37b7caSFelix Fietkau { 722ba2fc48cSLorenzo Bianconi const struct mtk_soc_data *soc = eth->soc; 723c4f033d9SFelix Fietkau struct device *dev = eth->dev; 7241ccc723bSFelix Fietkau struct mtk_ppe *ppe; 725ba2fc48cSLorenzo Bianconi u32 foe_flow_size; 7269d8cb4c0SLorenzo Bianconi void *foe; 7271ccc723bSFelix Fietkau 7281ccc723bSFelix Fietkau ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL); 7291ccc723bSFelix Fietkau if (!ppe) 7301ccc723bSFelix Fietkau return NULL; 731ba37b7caSFelix Fietkau 73233fc42deSFelix Fietkau rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params); 73333fc42deSFelix Fietkau 734ba37b7caSFelix Fietkau /* need to allocate a separate device, since it PPE DMA access is 735ba37b7caSFelix Fietkau * not coherent. 736ba37b7caSFelix Fietkau */ 737ba37b7caSFelix Fietkau ppe->base = base; 738c4f033d9SFelix Fietkau ppe->eth = eth; 739ba37b7caSFelix Fietkau ppe->dev = dev; 740ba37b7caSFelix Fietkau ppe->version = version; 741ba37b7caSFelix Fietkau 7429d8cb4c0SLorenzo Bianconi foe = dmam_alloc_coherent(ppe->dev, 7439d8cb4c0SLorenzo Bianconi MTK_PPE_ENTRIES * soc->foe_entry_size, 744ba37b7caSFelix Fietkau &ppe->foe_phys, GFP_KERNEL); 745ba37b7caSFelix Fietkau if (!foe) 7461ccc723bSFelix Fietkau return NULL; 747ba37b7caSFelix Fietkau 748ba37b7caSFelix Fietkau ppe->foe_table = foe; 749ba37b7caSFelix Fietkau 750ba2fc48cSLorenzo Bianconi foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) * 751ba2fc48cSLorenzo Bianconi sizeof(*ppe->foe_flow); 752ba2fc48cSLorenzo Bianconi ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL); 753ba2fc48cSLorenzo Bianconi if (!ppe->foe_flow) 754ba2fc48cSLorenzo Bianconi return NULL; 755ba2fc48cSLorenzo Bianconi 7564ff1a3fcSLorenzo Bianconi mtk_ppe_debugfs_init(ppe, index); 757ba37b7caSFelix Fietkau 7581ccc723bSFelix Fietkau return ppe; 759ba37b7caSFelix Fietkau } 760ba37b7caSFelix Fietkau 761ba37b7caSFelix Fietkau static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) 762ba37b7caSFelix Fietkau { 763ba37b7caSFelix Fietkau static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 }; 764ba37b7caSFelix Fietkau int i, k; 765ba37b7caSFelix Fietkau 7669d8cb4c0SLorenzo Bianconi memset(ppe->foe_table, 0, 7679d8cb4c0SLorenzo Bianconi MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size); 768ba37b7caSFelix Fietkau 769ba37b7caSFelix Fietkau if (!IS_ENABLED(CONFIG_SOC_MT7621)) 770ba37b7caSFelix Fietkau return; 771ba37b7caSFelix Fietkau 772ba37b7caSFelix Fietkau /* skip all entries that cross the 1024 byte boundary */ 7739d8cb4c0SLorenzo Bianconi for (i = 0; i < MTK_PPE_ENTRIES; i += 128) { 7749d8cb4c0SLorenzo Bianconi for (k = 0; k < ARRAY_SIZE(skip); k++) { 7759d8cb4c0SLorenzo Bianconi struct mtk_foe_entry *hwe; 7769d8cb4c0SLorenzo Bianconi 7779d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, i + skip[k]); 7789d8cb4c0SLorenzo Bianconi hwe->ib1 |= MTK_FOE_IB1_STATIC; 7799d8cb4c0SLorenzo Bianconi } 7809d8cb4c0SLorenzo Bianconi } 781ba37b7caSFelix Fietkau } 782ba37b7caSFelix Fietkau 7834ff1a3fcSLorenzo Bianconi void mtk_ppe_start(struct mtk_ppe *ppe) 784ba37b7caSFelix Fietkau { 785ba37b7caSFelix Fietkau u32 val; 786ba37b7caSFelix Fietkau 7874ff1a3fcSLorenzo Bianconi if (!ppe) 7884ff1a3fcSLorenzo Bianconi return; 7894ff1a3fcSLorenzo Bianconi 790ba37b7caSFelix Fietkau mtk_ppe_init_foe_table(ppe); 791ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys); 792ba37b7caSFelix Fietkau 793ba37b7caSFelix Fietkau val = MTK_PPE_TB_CFG_ENTRY_80B | 794ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_NON_L4 | 795ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_UNBIND | 796ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_TCP | 797ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_UDP | 798ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_TCP_FIN | 799ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS, 800ba37b7caSFelix Fietkau MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) | 801ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE, 802ba37b7caSFelix Fietkau MTK_PPE_KEEPALIVE_DISABLE) | 803ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) | 804ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE, 805ba37b7caSFelix Fietkau MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) | 806ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, 807ba37b7caSFelix Fietkau MTK_PPE_ENTRIES_SHIFT); 80803a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 80903a3180eSLorenzo Bianconi val |= MTK_PPE_TB_CFG_INFO_SEL; 810ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_TB_CFG, val); 811ba37b7caSFelix Fietkau 812ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, 813ba37b7caSFelix Fietkau MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6); 814ba37b7caSFelix Fietkau 815ba37b7caSFelix Fietkau mtk_ppe_cache_enable(ppe, true); 816ba37b7caSFelix Fietkau 81703a3180eSLorenzo Bianconi val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | 818ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP6_5T_ROUTE | 819ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP6_6RD | 820ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP4_NAT | 821ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP4_NAPT | 822ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP4_DSLITE | 823ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; 82403a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 82503a3180eSLorenzo Bianconi val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | 82603a3180eSLorenzo Bianconi MTK_PPE_MD_TOAP_BYP_CRSN1 | 82703a3180eSLorenzo Bianconi MTK_PPE_MD_TOAP_BYP_CRSN2 | 82803a3180eSLorenzo Bianconi MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY; 82903a3180eSLorenzo Bianconi else 83003a3180eSLorenzo Bianconi val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | 83103a3180eSLorenzo Bianconi MTK_PPE_FLOW_CFG_IP4_UDP_FRAG; 832ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); 833ba37b7caSFelix Fietkau 834ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) | 835ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3); 836ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val); 837ba37b7caSFelix Fietkau 838ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) | 839ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1); 840ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_AGE0, val); 841ba37b7caSFelix Fietkau 842ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 843ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7); 844ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_AGE1, val); 845ba37b7caSFelix Fietkau 846ba37b7caSFelix Fietkau val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF; 847ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val); 848ba37b7caSFelix Fietkau 849ba37b7caSFelix Fietkau val = MTK_PPE_BIND_LIMIT1_FULL | 850ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1); 851ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val); 852ba37b7caSFelix Fietkau 853ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) | 854ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1); 855ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_RATE, val); 856ba37b7caSFelix Fietkau 857ba37b7caSFelix Fietkau /* enable PPE */ 858ba37b7caSFelix Fietkau val = MTK_PPE_GLO_CFG_EN | 859ba37b7caSFelix Fietkau MTK_PPE_GLO_CFG_IP4_L4_CS_DROP | 860ba37b7caSFelix Fietkau MTK_PPE_GLO_CFG_IP4_CS_DROP | 861ba37b7caSFelix Fietkau MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE; 862ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_GLO_CFG, val); 863ba37b7caSFelix Fietkau 864ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0); 86503a3180eSLorenzo Bianconi 86603a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) { 86703a3180eSLorenzo Bianconi ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777); 86803a3180eSLorenzo Bianconi ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f); 86903a3180eSLorenzo Bianconi } 870ba37b7caSFelix Fietkau } 871ba37b7caSFelix Fietkau 872ba37b7caSFelix Fietkau int mtk_ppe_stop(struct mtk_ppe *ppe) 873ba37b7caSFelix Fietkau { 874ba37b7caSFelix Fietkau u32 val; 875ba37b7caSFelix Fietkau int i; 876ba37b7caSFelix Fietkau 8774ff1a3fcSLorenzo Bianconi if (!ppe) 8784ff1a3fcSLorenzo Bianconi return 0; 8794ff1a3fcSLorenzo Bianconi 8809d8cb4c0SLorenzo Bianconi for (i = 0; i < MTK_PPE_ENTRIES; i++) { 8819d8cb4c0SLorenzo Bianconi struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i); 8829d8cb4c0SLorenzo Bianconi 8839d8cb4c0SLorenzo Bianconi hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, 884ba37b7caSFelix Fietkau MTK_FOE_STATE_INVALID); 8859d8cb4c0SLorenzo Bianconi } 886ba37b7caSFelix Fietkau 887ba37b7caSFelix Fietkau mtk_ppe_cache_enable(ppe, false); 888ba37b7caSFelix Fietkau 889ba37b7caSFelix Fietkau /* disable offload engine */ 890ba37b7caSFelix Fietkau ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN); 891ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0); 892ba37b7caSFelix Fietkau 893ba37b7caSFelix Fietkau /* disable aging */ 894ba37b7caSFelix Fietkau val = MTK_PPE_TB_CFG_AGE_NON_L4 | 895ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_UNBIND | 896ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_TCP | 897ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_UDP | 898ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_TCP_FIN; 899ba37b7caSFelix Fietkau ppe_clear(ppe, MTK_PPE_TB_CFG, val); 900ba37b7caSFelix Fietkau 901ba37b7caSFelix Fietkau return mtk_ppe_wait_busy(ppe); 902ba37b7caSFelix Fietkau } 903