1ba37b7caSFelix Fietkau // SPDX-License-Identifier: GPL-2.0-only 2ba37b7caSFelix Fietkau /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ 3ba37b7caSFelix Fietkau 4ba37b7caSFelix Fietkau #include <linux/kernel.h> 5ba37b7caSFelix Fietkau #include <linux/io.h> 6c5d66587SIlya Lipnitskiy #include <linux/iopoll.h> 7ba37b7caSFelix Fietkau #include <linux/etherdevice.h> 8ba37b7caSFelix Fietkau #include <linux/platform_device.h> 933fc42deSFelix Fietkau #include <linux/if_ether.h> 1033fc42deSFelix Fietkau #include <linux/if_vlan.h> 1133fc42deSFelix Fietkau #include <net/dsa.h> 12c4f033d9SFelix Fietkau #include "mtk_eth_soc.h" 13ba37b7caSFelix Fietkau #include "mtk_ppe.h" 14ba37b7caSFelix Fietkau #include "mtk_ppe_regs.h" 15ba37b7caSFelix Fietkau 16c4f033d9SFelix Fietkau static DEFINE_SPINLOCK(ppe_lock); 17c4f033d9SFelix Fietkau 1833fc42deSFelix Fietkau static const struct rhashtable_params mtk_flow_l2_ht_params = { 1933fc42deSFelix Fietkau .head_offset = offsetof(struct mtk_flow_entry, l2_node), 2033fc42deSFelix Fietkau .key_offset = offsetof(struct mtk_flow_entry, data.bridge), 2133fc42deSFelix Fietkau .key_len = offsetof(struct mtk_foe_bridge, key_end), 2233fc42deSFelix Fietkau .automatic_shrinking = true, 2333fc42deSFelix Fietkau }; 2433fc42deSFelix Fietkau 25ba37b7caSFelix Fietkau static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val) 26ba37b7caSFelix Fietkau { 27ba37b7caSFelix Fietkau writel(val, ppe->base + reg); 28ba37b7caSFelix Fietkau } 29ba37b7caSFelix Fietkau 30ba37b7caSFelix Fietkau static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg) 31ba37b7caSFelix Fietkau { 32ba37b7caSFelix Fietkau return readl(ppe->base + reg); 33ba37b7caSFelix Fietkau } 34ba37b7caSFelix Fietkau 35ba37b7caSFelix Fietkau static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set) 36ba37b7caSFelix Fietkau { 37ba37b7caSFelix Fietkau u32 val; 38ba37b7caSFelix Fietkau 39ba37b7caSFelix Fietkau val = ppe_r32(ppe, reg); 40ba37b7caSFelix Fietkau val &= ~mask; 41ba37b7caSFelix Fietkau val |= set; 42ba37b7caSFelix Fietkau ppe_w32(ppe, reg, val); 43ba37b7caSFelix Fietkau 44ba37b7caSFelix Fietkau return val; 45ba37b7caSFelix Fietkau } 46ba37b7caSFelix Fietkau 47ba37b7caSFelix Fietkau static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val) 48ba37b7caSFelix Fietkau { 49ba37b7caSFelix Fietkau return ppe_m32(ppe, reg, 0, val); 50ba37b7caSFelix Fietkau } 51ba37b7caSFelix Fietkau 52ba37b7caSFelix Fietkau static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) 53ba37b7caSFelix Fietkau { 54ba37b7caSFelix Fietkau return ppe_m32(ppe, reg, val, 0); 55ba37b7caSFelix Fietkau } 56ba37b7caSFelix Fietkau 57c4f033d9SFelix Fietkau static u32 mtk_eth_timestamp(struct mtk_eth *eth) 58c4f033d9SFelix Fietkau { 5903a3180eSLorenzo Bianconi return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth); 60c4f033d9SFelix Fietkau } 61c4f033d9SFelix Fietkau 62ba37b7caSFelix Fietkau static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) 63ba37b7caSFelix Fietkau { 64c5d66587SIlya Lipnitskiy int ret; 65c5d66587SIlya Lipnitskiy u32 val; 66ba37b7caSFelix Fietkau 67c5d66587SIlya Lipnitskiy ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val, 68c5d66587SIlya Lipnitskiy !(val & MTK_PPE_GLO_CFG_BUSY), 69c5d66587SIlya Lipnitskiy 20, MTK_PPE_WAIT_TIMEOUT_US); 70ba37b7caSFelix Fietkau 71c5d66587SIlya Lipnitskiy if (ret) 72ba37b7caSFelix Fietkau dev_err(ppe->dev, "PPE table busy"); 73ba37b7caSFelix Fietkau 74c5d66587SIlya Lipnitskiy return ret; 75ba37b7caSFelix Fietkau } 76ba37b7caSFelix Fietkau 77ba37b7caSFelix Fietkau static void mtk_ppe_cache_clear(struct mtk_ppe *ppe) 78ba37b7caSFelix Fietkau { 79ba37b7caSFelix Fietkau ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 80ba37b7caSFelix Fietkau ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 81ba37b7caSFelix Fietkau } 82ba37b7caSFelix Fietkau 83ba37b7caSFelix Fietkau static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable) 84ba37b7caSFelix Fietkau { 85ba37b7caSFelix Fietkau mtk_ppe_cache_clear(ppe); 86ba37b7caSFelix Fietkau 87ba37b7caSFelix Fietkau ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN, 88ba37b7caSFelix Fietkau enable * MTK_PPE_CACHE_CTL_EN); 89ba37b7caSFelix Fietkau } 90ba37b7caSFelix Fietkau 91ba2fc48cSLorenzo Bianconi static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e) 92ba37b7caSFelix Fietkau { 93ba37b7caSFelix Fietkau u32 hv1, hv2, hv3; 94ba37b7caSFelix Fietkau u32 hash; 95ba37b7caSFelix Fietkau 9603a3180eSLorenzo Bianconi switch (mtk_get_ib1_pkt_type(eth, e->ib1)) { 97ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 98ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 99ba37b7caSFelix Fietkau hv1 = e->ipv4.orig.ports; 100ba37b7caSFelix Fietkau hv2 = e->ipv4.orig.dest_ip; 101ba37b7caSFelix Fietkau hv3 = e->ipv4.orig.src_ip; 102ba37b7caSFelix Fietkau break; 103ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 104ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 105ba37b7caSFelix Fietkau hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3]; 106ba37b7caSFelix Fietkau hv1 ^= e->ipv6.ports; 107ba37b7caSFelix Fietkau 108ba37b7caSFelix Fietkau hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2]; 109ba37b7caSFelix Fietkau hv2 ^= e->ipv6.dest_ip[0]; 110ba37b7caSFelix Fietkau 111ba37b7caSFelix Fietkau hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1]; 112ba37b7caSFelix Fietkau hv3 ^= e->ipv6.src_ip[0]; 113ba37b7caSFelix Fietkau break; 114ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 115ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_6RD: 116ba37b7caSFelix Fietkau default: 117ba37b7caSFelix Fietkau WARN_ON_ONCE(1); 118ba37b7caSFelix Fietkau return MTK_PPE_HASH_MASK; 119ba37b7caSFelix Fietkau } 120ba37b7caSFelix Fietkau 121ba37b7caSFelix Fietkau hash = (hv1 & hv2) | ((~hv1) & hv3); 122ba37b7caSFelix Fietkau hash = (hash >> 24) | ((hash & 0xffffff) << 8); 123ba37b7caSFelix Fietkau hash ^= hv1 ^ hv2 ^ hv3; 124ba37b7caSFelix Fietkau hash ^= hash >> 16; 125ba2fc48cSLorenzo Bianconi hash <<= (ffs(eth->soc->hash_offset) - 1); 126ba37b7caSFelix Fietkau hash &= MTK_PPE_ENTRIES - 1; 127ba37b7caSFelix Fietkau 128ba37b7caSFelix Fietkau return hash; 129ba37b7caSFelix Fietkau } 130ba37b7caSFelix Fietkau 131ba37b7caSFelix Fietkau static inline struct mtk_foe_mac_info * 13203a3180eSLorenzo Bianconi mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 133ba37b7caSFelix Fietkau { 13403a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 135ba37b7caSFelix Fietkau 13633fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_BRIDGE) 13733fc42deSFelix Fietkau return &entry->bridge.l2; 13833fc42deSFelix Fietkau 139ba37b7caSFelix Fietkau if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 140ba37b7caSFelix Fietkau return &entry->ipv6.l2; 141ba37b7caSFelix Fietkau 142ba37b7caSFelix Fietkau return &entry->ipv4.l2; 143ba37b7caSFelix Fietkau } 144ba37b7caSFelix Fietkau 145ba37b7caSFelix Fietkau static inline u32 * 14603a3180eSLorenzo Bianconi mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 147ba37b7caSFelix Fietkau { 14803a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 149ba37b7caSFelix Fietkau 15033fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_BRIDGE) 15133fc42deSFelix Fietkau return &entry->bridge.ib2; 15233fc42deSFelix Fietkau 153ba37b7caSFelix Fietkau if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 154ba37b7caSFelix Fietkau return &entry->ipv6.ib2; 155ba37b7caSFelix Fietkau 156ba37b7caSFelix Fietkau return &entry->ipv4.ib2; 157ba37b7caSFelix Fietkau } 158ba37b7caSFelix Fietkau 15903a3180eSLorenzo Bianconi int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, 16003a3180eSLorenzo Bianconi int type, int l4proto, u8 pse_port, u8 *src_mac, 16103a3180eSLorenzo Bianconi u8 *dest_mac) 162ba37b7caSFelix Fietkau { 163ba37b7caSFelix Fietkau struct mtk_foe_mac_info *l2; 164ba37b7caSFelix Fietkau u32 ports_pad, val; 165ba37b7caSFelix Fietkau 166ba37b7caSFelix Fietkau memset(entry, 0, sizeof(*entry)); 167ba37b7caSFelix Fietkau 16803a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 16903a3180eSLorenzo Bianconi val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 17003a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | 17103a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 17203a3180eSLorenzo Bianconi MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2; 17303a3180eSLorenzo Bianconi entry->ib1 = val; 17403a3180eSLorenzo Bianconi 17503a3180eSLorenzo Bianconi val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) | 17603a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf); 17703a3180eSLorenzo Bianconi } else { 178*71ba8e48SFelix Fietkau int port_mg = eth->soc->offload_version > 1 ? 0 : 0x3f; 179*71ba8e48SFelix Fietkau 180ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 181ba37b7caSFelix Fietkau FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | 182ba37b7caSFelix Fietkau FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 18303a3180eSLorenzo Bianconi MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL; 184ba37b7caSFelix Fietkau entry->ib1 = val; 185ba37b7caSFelix Fietkau 18603a3180eSLorenzo Bianconi val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) | 187*71ba8e48SFelix Fietkau FIELD_PREP(MTK_FOE_IB2_PORT_MG, port_mg) | 18803a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f); 18903a3180eSLorenzo Bianconi } 190ba37b7caSFelix Fietkau 191ba37b7caSFelix Fietkau if (is_multicast_ether_addr(dest_mac)) 19203a3180eSLorenzo Bianconi val |= mtk_get_ib2_multicast_mask(eth); 193ba37b7caSFelix Fietkau 194ba37b7caSFelix Fietkau ports_pad = 0xa5a5a500 | (l4proto & 0xff); 195ba37b7caSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 196ba37b7caSFelix Fietkau entry->ipv4.orig.ports = ports_pad; 197ba37b7caSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 198ba37b7caSFelix Fietkau entry->ipv6.ports = ports_pad; 199ba37b7caSFelix Fietkau 20033fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_BRIDGE) { 20133fc42deSFelix Fietkau ether_addr_copy(entry->bridge.src_mac, src_mac); 20233fc42deSFelix Fietkau ether_addr_copy(entry->bridge.dest_mac, dest_mac); 20333fc42deSFelix Fietkau entry->bridge.ib2 = val; 20433fc42deSFelix Fietkau l2 = &entry->bridge.l2; 20533fc42deSFelix Fietkau } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { 206ba37b7caSFelix Fietkau entry->ipv6.ib2 = val; 207ba37b7caSFelix Fietkau l2 = &entry->ipv6.l2; 208ba37b7caSFelix Fietkau } else { 209ba37b7caSFelix Fietkau entry->ipv4.ib2 = val; 210ba37b7caSFelix Fietkau l2 = &entry->ipv4.l2; 211ba37b7caSFelix Fietkau } 212ba37b7caSFelix Fietkau 213ba37b7caSFelix Fietkau l2->dest_mac_hi = get_unaligned_be32(dest_mac); 214ba37b7caSFelix Fietkau l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4); 215ba37b7caSFelix Fietkau l2->src_mac_hi = get_unaligned_be32(src_mac); 216ba37b7caSFelix Fietkau l2->src_mac_lo = get_unaligned_be16(src_mac + 4); 217ba37b7caSFelix Fietkau 218ba37b7caSFelix Fietkau if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 219ba37b7caSFelix Fietkau l2->etype = ETH_P_IPV6; 220ba37b7caSFelix Fietkau else 221ba37b7caSFelix Fietkau l2->etype = ETH_P_IP; 222ba37b7caSFelix Fietkau 223ba37b7caSFelix Fietkau return 0; 224ba37b7caSFelix Fietkau } 225ba37b7caSFelix Fietkau 22603a3180eSLorenzo Bianconi int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, 22703a3180eSLorenzo Bianconi struct mtk_foe_entry *entry, u8 port) 228ba37b7caSFelix Fietkau { 22903a3180eSLorenzo Bianconi u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 23003a3180eSLorenzo Bianconi u32 val = *ib2; 231ba37b7caSFelix Fietkau 23203a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 23303a3180eSLorenzo Bianconi val &= ~MTK_FOE_IB2_DEST_PORT_V2; 23403a3180eSLorenzo Bianconi val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); 23503a3180eSLorenzo Bianconi } else { 236ba37b7caSFelix Fietkau val &= ~MTK_FOE_IB2_DEST_PORT; 237ba37b7caSFelix Fietkau val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); 23803a3180eSLorenzo Bianconi } 239ba37b7caSFelix Fietkau *ib2 = val; 240ba37b7caSFelix Fietkau 241ba37b7caSFelix Fietkau return 0; 242ba37b7caSFelix Fietkau } 243ba37b7caSFelix Fietkau 24403a3180eSLorenzo Bianconi int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, 24503a3180eSLorenzo Bianconi struct mtk_foe_entry *entry, bool egress, 246ba37b7caSFelix Fietkau __be32 src_addr, __be16 src_port, 247ba37b7caSFelix Fietkau __be32 dest_addr, __be16 dest_port) 248ba37b7caSFelix Fietkau { 24903a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 250ba37b7caSFelix Fietkau struct mtk_ipv4_tuple *t; 251ba37b7caSFelix Fietkau 252ba37b7caSFelix Fietkau switch (type) { 253ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 254ba37b7caSFelix Fietkau if (egress) { 255ba37b7caSFelix Fietkau t = &entry->ipv4.new; 256ba37b7caSFelix Fietkau break; 257ba37b7caSFelix Fietkau } 258ba37b7caSFelix Fietkau fallthrough; 259ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 260ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 261ba37b7caSFelix Fietkau t = &entry->ipv4.orig; 262ba37b7caSFelix Fietkau break; 263ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_6RD: 264ba37b7caSFelix Fietkau entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr); 265ba37b7caSFelix Fietkau entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr); 266ba37b7caSFelix Fietkau return 0; 267ba37b7caSFelix Fietkau default: 268ba37b7caSFelix Fietkau WARN_ON_ONCE(1); 269ba37b7caSFelix Fietkau return -EINVAL; 270ba37b7caSFelix Fietkau } 271ba37b7caSFelix Fietkau 272ba37b7caSFelix Fietkau t->src_ip = be32_to_cpu(src_addr); 273ba37b7caSFelix Fietkau t->dest_ip = be32_to_cpu(dest_addr); 274ba37b7caSFelix Fietkau 275ba37b7caSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 276ba37b7caSFelix Fietkau return 0; 277ba37b7caSFelix Fietkau 278ba37b7caSFelix Fietkau t->src_port = be16_to_cpu(src_port); 279ba37b7caSFelix Fietkau t->dest_port = be16_to_cpu(dest_port); 280ba37b7caSFelix Fietkau 281ba37b7caSFelix Fietkau return 0; 282ba37b7caSFelix Fietkau } 283ba37b7caSFelix Fietkau 28403a3180eSLorenzo Bianconi int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, 28503a3180eSLorenzo Bianconi struct mtk_foe_entry *entry, 286ba37b7caSFelix Fietkau __be32 *src_addr, __be16 src_port, 287ba37b7caSFelix Fietkau __be32 *dest_addr, __be16 dest_port) 288ba37b7caSFelix Fietkau { 28903a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 290ba37b7caSFelix Fietkau u32 *src, *dest; 291ba37b7caSFelix Fietkau int i; 292ba37b7caSFelix Fietkau 293ba37b7caSFelix Fietkau switch (type) { 294ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 295ba37b7caSFelix Fietkau src = entry->dslite.tunnel_src_ip; 296ba37b7caSFelix Fietkau dest = entry->dslite.tunnel_dest_ip; 297ba37b7caSFelix Fietkau break; 298ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 299ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_6RD: 300ba37b7caSFelix Fietkau entry->ipv6.src_port = be16_to_cpu(src_port); 301ba37b7caSFelix Fietkau entry->ipv6.dest_port = be16_to_cpu(dest_port); 302ba37b7caSFelix Fietkau fallthrough; 303ba37b7caSFelix Fietkau case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 304ba37b7caSFelix Fietkau src = entry->ipv6.src_ip; 305ba37b7caSFelix Fietkau dest = entry->ipv6.dest_ip; 306ba37b7caSFelix Fietkau break; 307ba37b7caSFelix Fietkau default: 308ba37b7caSFelix Fietkau WARN_ON_ONCE(1); 309ba37b7caSFelix Fietkau return -EINVAL; 3103b2c32f9SQiheng Lin } 311ba37b7caSFelix Fietkau 312ba37b7caSFelix Fietkau for (i = 0; i < 4; i++) 313ba37b7caSFelix Fietkau src[i] = be32_to_cpu(src_addr[i]); 314ba37b7caSFelix Fietkau for (i = 0; i < 4; i++) 315ba37b7caSFelix Fietkau dest[i] = be32_to_cpu(dest_addr[i]); 316ba37b7caSFelix Fietkau 317ba37b7caSFelix Fietkau return 0; 318ba37b7caSFelix Fietkau } 319ba37b7caSFelix Fietkau 32003a3180eSLorenzo Bianconi int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, 32103a3180eSLorenzo Bianconi int port) 322ba37b7caSFelix Fietkau { 32303a3180eSLorenzo Bianconi struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 324ba37b7caSFelix Fietkau 325ba37b7caSFelix Fietkau l2->etype = BIT(port); 326ba37b7caSFelix Fietkau 32703a3180eSLorenzo Bianconi if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth))) 32803a3180eSLorenzo Bianconi entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1); 329ba37b7caSFelix Fietkau else 330ba37b7caSFelix Fietkau l2->etype |= BIT(8); 331ba37b7caSFelix Fietkau 33203a3180eSLorenzo Bianconi entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth); 333ba37b7caSFelix Fietkau 334ba37b7caSFelix Fietkau return 0; 335ba37b7caSFelix Fietkau } 336ba37b7caSFelix Fietkau 33703a3180eSLorenzo Bianconi int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, 33803a3180eSLorenzo Bianconi int vid) 339ba37b7caSFelix Fietkau { 34003a3180eSLorenzo Bianconi struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 341ba37b7caSFelix Fietkau 342fb7da771SDaniel Golle switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) { 343ba37b7caSFelix Fietkau case 0: 34403a3180eSLorenzo Bianconi entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) | 34503a3180eSLorenzo Bianconi mtk_prep_ib1_vlan_layer(eth, 1); 346ba37b7caSFelix Fietkau l2->vlan1 = vid; 347ba37b7caSFelix Fietkau return 0; 348ba37b7caSFelix Fietkau case 1: 34903a3180eSLorenzo Bianconi if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) { 350ba37b7caSFelix Fietkau l2->vlan1 = vid; 351ba37b7caSFelix Fietkau l2->etype |= BIT(8); 352ba37b7caSFelix Fietkau } else { 353ba37b7caSFelix Fietkau l2->vlan2 = vid; 35403a3180eSLorenzo Bianconi entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1); 355ba37b7caSFelix Fietkau } 356ba37b7caSFelix Fietkau return 0; 357ba37b7caSFelix Fietkau default: 358ba37b7caSFelix Fietkau return -ENOSPC; 359ba37b7caSFelix Fietkau } 360ba37b7caSFelix Fietkau } 361ba37b7caSFelix Fietkau 36203a3180eSLorenzo Bianconi int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, 36303a3180eSLorenzo Bianconi int sid) 364ba37b7caSFelix Fietkau { 36503a3180eSLorenzo Bianconi struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 366ba37b7caSFelix Fietkau 36703a3180eSLorenzo Bianconi if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) || 36803a3180eSLorenzo Bianconi (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) 369ba37b7caSFelix Fietkau l2->etype = ETH_P_PPP_SES; 370ba37b7caSFelix Fietkau 37103a3180eSLorenzo Bianconi entry->ib1 |= mtk_get_ib1_ppoe_mask(eth); 372ba37b7caSFelix Fietkau l2->pppoe_id = sid; 373ba37b7caSFelix Fietkau 374ba37b7caSFelix Fietkau return 0; 375ba37b7caSFelix Fietkau } 376ba37b7caSFelix Fietkau 37703a3180eSLorenzo Bianconi int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, 37803a3180eSLorenzo Bianconi int wdma_idx, int txq, int bss, int wcid) 379a333215eSFelix Fietkau { 38003a3180eSLorenzo Bianconi struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 38103a3180eSLorenzo Bianconi u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 382a333215eSFelix Fietkau 38303a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 38403a3180eSLorenzo Bianconi *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; 38503a3180eSLorenzo Bianconi *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | 38603a3180eSLorenzo Bianconi MTK_FOE_IB2_WDMA_WINFO_V2; 38703a3180eSLorenzo Bianconi l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | 38803a3180eSLorenzo Bianconi FIELD_PREP(MTK_FOE_WINFO_BSS, bss); 38903a3180eSLorenzo Bianconi } else { 390a333215eSFelix Fietkau *ib2 &= ~MTK_FOE_IB2_PORT_MG; 391a333215eSFelix Fietkau *ib2 |= MTK_FOE_IB2_WDMA_WINFO; 392a333215eSFelix Fietkau if (wdma_idx) 393a333215eSFelix Fietkau *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; 394a333215eSFelix Fietkau l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | 395a333215eSFelix Fietkau FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | 396a333215eSFelix Fietkau FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); 39703a3180eSLorenzo Bianconi } 398a333215eSFelix Fietkau 399a333215eSFelix Fietkau return 0; 400a333215eSFelix Fietkau } 401a333215eSFelix Fietkau 402c4f033d9SFelix Fietkau static bool 40303a3180eSLorenzo Bianconi mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry, 40403a3180eSLorenzo Bianconi struct mtk_foe_entry *data) 405c4f033d9SFelix Fietkau { 406c4f033d9SFelix Fietkau int type, len; 407c4f033d9SFelix Fietkau 408c4f033d9SFelix Fietkau if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) 409c4f033d9SFelix Fietkau return false; 410c4f033d9SFelix Fietkau 41103a3180eSLorenzo Bianconi type = mtk_get_ib1_pkt_type(eth, entry->data.ib1); 412c4f033d9SFelix Fietkau if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) 413c4f033d9SFelix Fietkau len = offsetof(struct mtk_foe_entry, ipv6._rsv); 414c4f033d9SFelix Fietkau else 415c4f033d9SFelix Fietkau len = offsetof(struct mtk_foe_entry, ipv4.ib2); 416c4f033d9SFelix Fietkau 417c4f033d9SFelix Fietkau return !memcmp(&entry->data.data, &data->data, len - 4); 418c4f033d9SFelix Fietkau } 419c4f033d9SFelix Fietkau 420c4f033d9SFelix Fietkau static void 42133fc42deSFelix Fietkau __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 42233fc42deSFelix Fietkau { 42333fc42deSFelix Fietkau struct hlist_head *head; 42433fc42deSFelix Fietkau struct hlist_node *tmp; 42533fc42deSFelix Fietkau 42633fc42deSFelix Fietkau if (entry->type == MTK_FLOW_TYPE_L2) { 42733fc42deSFelix Fietkau rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node, 42833fc42deSFelix Fietkau mtk_flow_l2_ht_params); 42933fc42deSFelix Fietkau 43033fc42deSFelix Fietkau head = &entry->l2_flows; 43133fc42deSFelix Fietkau hlist_for_each_entry_safe(entry, tmp, head, l2_data.list) 43233fc42deSFelix Fietkau __mtk_foe_entry_clear(ppe, entry); 43333fc42deSFelix Fietkau return; 43433fc42deSFelix Fietkau } 43533fc42deSFelix Fietkau 43633fc42deSFelix Fietkau hlist_del_init(&entry->list); 43733fc42deSFelix Fietkau if (entry->hash != 0xffff) { 4389d8cb4c0SLorenzo Bianconi struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash); 4399d8cb4c0SLorenzo Bianconi 4409d8cb4c0SLorenzo Bianconi hwe->ib1 &= ~MTK_FOE_IB1_STATE; 441e52f7c1dSJakub Kicinski hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID); 44233fc42deSFelix Fietkau dma_wmb(); 44333fc42deSFelix Fietkau } 44433fc42deSFelix Fietkau entry->hash = 0xffff; 44533fc42deSFelix Fietkau 44633fc42deSFelix Fietkau if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW) 44733fc42deSFelix Fietkau return; 44833fc42deSFelix Fietkau 44933fc42deSFelix Fietkau hlist_del_init(&entry->l2_data.list); 45033fc42deSFelix Fietkau kfree(entry); 45133fc42deSFelix Fietkau } 45233fc42deSFelix Fietkau 45333fc42deSFelix Fietkau static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) 45433fc42deSFelix Fietkau { 45503a3180eSLorenzo Bianconi u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 45603a3180eSLorenzo Bianconi u16 now = mtk_eth_timestamp(ppe->eth); 45703a3180eSLorenzo Bianconi u16 timestamp = ib1 & ib1_ts_mask; 45833fc42deSFelix Fietkau 45933fc42deSFelix Fietkau if (timestamp > now) 46003a3180eSLorenzo Bianconi return ib1_ts_mask + 1 - timestamp + now; 46133fc42deSFelix Fietkau else 46233fc42deSFelix Fietkau return now - timestamp; 46333fc42deSFelix Fietkau } 46433fc42deSFelix Fietkau 46533fc42deSFelix Fietkau static void 46633fc42deSFelix Fietkau mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 46733fc42deSFelix Fietkau { 46803a3180eSLorenzo Bianconi u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 46933fc42deSFelix Fietkau struct mtk_flow_entry *cur; 47033fc42deSFelix Fietkau struct mtk_foe_entry *hwe; 47133fc42deSFelix Fietkau struct hlist_node *tmp; 47233fc42deSFelix Fietkau int idle; 47333fc42deSFelix Fietkau 47433fc42deSFelix Fietkau idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 47533fc42deSFelix Fietkau hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) { 47633fc42deSFelix Fietkau int cur_idle; 47733fc42deSFelix Fietkau u32 ib1; 47833fc42deSFelix Fietkau 4799d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, cur->hash); 48033fc42deSFelix Fietkau ib1 = READ_ONCE(hwe->ib1); 48133fc42deSFelix Fietkau 48233fc42deSFelix Fietkau if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { 48333fc42deSFelix Fietkau cur->hash = 0xffff; 48433fc42deSFelix Fietkau __mtk_foe_entry_clear(ppe, cur); 48533fc42deSFelix Fietkau continue; 48633fc42deSFelix Fietkau } 48733fc42deSFelix Fietkau 48833fc42deSFelix Fietkau cur_idle = __mtk_foe_entry_idle_time(ppe, ib1); 48933fc42deSFelix Fietkau if (cur_idle >= idle) 49033fc42deSFelix Fietkau continue; 49133fc42deSFelix Fietkau 49233fc42deSFelix Fietkau idle = cur_idle; 49303a3180eSLorenzo Bianconi entry->data.ib1 &= ~ib1_ts_mask; 49403a3180eSLorenzo Bianconi entry->data.ib1 |= hwe->ib1 & ib1_ts_mask; 49533fc42deSFelix Fietkau } 49633fc42deSFelix Fietkau } 49733fc42deSFelix Fietkau 49833fc42deSFelix Fietkau static void 499c4f033d9SFelix Fietkau mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 500ba37b7caSFelix Fietkau { 5019d8cb4c0SLorenzo Bianconi struct mtk_foe_entry foe = {}; 502ba37b7caSFelix Fietkau struct mtk_foe_entry *hwe; 503ba37b7caSFelix Fietkau 504c4f033d9SFelix Fietkau spin_lock_bh(&ppe_lock); 50533fc42deSFelix Fietkau 50633fc42deSFelix Fietkau if (entry->type == MTK_FLOW_TYPE_L2) { 50733fc42deSFelix Fietkau mtk_flow_entry_update_l2(ppe, entry); 50833fc42deSFelix Fietkau goto out; 50933fc42deSFelix Fietkau } 51033fc42deSFelix Fietkau 511c4f033d9SFelix Fietkau if (entry->hash == 0xffff) 512c4f033d9SFelix Fietkau goto out; 513c4f033d9SFelix Fietkau 5149d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, entry->hash); 5159d8cb4c0SLorenzo Bianconi memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size); 51603a3180eSLorenzo Bianconi if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) { 517c4f033d9SFelix Fietkau entry->hash = 0xffff; 518c4f033d9SFelix Fietkau goto out; 519c4f033d9SFelix Fietkau } 520c4f033d9SFelix Fietkau 521c4f033d9SFelix Fietkau entry->data.ib1 = foe.ib1; 522c4f033d9SFelix Fietkau 523c4f033d9SFelix Fietkau out: 524c4f033d9SFelix Fietkau spin_unlock_bh(&ppe_lock); 525c4f033d9SFelix Fietkau } 526c4f033d9SFelix Fietkau 527c4f033d9SFelix Fietkau static void 528c4f033d9SFelix Fietkau __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, 529c4f033d9SFelix Fietkau u16 hash) 530c4f033d9SFelix Fietkau { 53103a3180eSLorenzo Bianconi struct mtk_eth *eth = ppe->eth; 53203a3180eSLorenzo Bianconi u16 timestamp = mtk_eth_timestamp(eth); 533c4f033d9SFelix Fietkau struct mtk_foe_entry *hwe; 534c4f033d9SFelix Fietkau 53503a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 53603a3180eSLorenzo Bianconi entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; 53703a3180eSLorenzo Bianconi entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, 53803a3180eSLorenzo Bianconi timestamp); 53903a3180eSLorenzo Bianconi } else { 540ba37b7caSFelix Fietkau entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; 54103a3180eSLorenzo Bianconi entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, 54203a3180eSLorenzo Bianconi timestamp); 54303a3180eSLorenzo Bianconi } 544ba37b7caSFelix Fietkau 5459d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, hash); 546454b20e1SDaniel Golle memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1)); 547ba37b7caSFelix Fietkau wmb(); 548ba37b7caSFelix Fietkau hwe->ib1 = entry->ib1; 549ba37b7caSFelix Fietkau 550ba37b7caSFelix Fietkau dma_wmb(); 551ba37b7caSFelix Fietkau 552ba37b7caSFelix Fietkau mtk_ppe_cache_clear(ppe); 553ba37b7caSFelix Fietkau } 554ba37b7caSFelix Fietkau 555c4f033d9SFelix Fietkau void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 556c4f033d9SFelix Fietkau { 557c4f033d9SFelix Fietkau spin_lock_bh(&ppe_lock); 55833fc42deSFelix Fietkau __mtk_foe_entry_clear(ppe, entry); 559c4f033d9SFelix Fietkau spin_unlock_bh(&ppe_lock); 560c4f033d9SFelix Fietkau } 561c4f033d9SFelix Fietkau 56233fc42deSFelix Fietkau static int 56333fc42deSFelix Fietkau mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 56433fc42deSFelix Fietkau { 56533fc42deSFelix Fietkau entry->type = MTK_FLOW_TYPE_L2; 56633fc42deSFelix Fietkau 56733fc42deSFelix Fietkau return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node, 56833fc42deSFelix Fietkau mtk_flow_l2_ht_params); 56933fc42deSFelix Fietkau } 57033fc42deSFelix Fietkau 571c4f033d9SFelix Fietkau int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 572c4f033d9SFelix Fietkau { 573ba2fc48cSLorenzo Bianconi const struct mtk_soc_data *soc = ppe->eth->soc; 57403a3180eSLorenzo Bianconi int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1); 57533fc42deSFelix Fietkau u32 hash; 576c4f033d9SFelix Fietkau 57733fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_BRIDGE) 57833fc42deSFelix Fietkau return mtk_foe_entry_commit_l2(ppe, entry); 57933fc42deSFelix Fietkau 580ba2fc48cSLorenzo Bianconi hash = mtk_ppe_hash_entry(ppe->eth, &entry->data); 581c4f033d9SFelix Fietkau entry->hash = 0xffff; 582c4f033d9SFelix Fietkau spin_lock_bh(&ppe_lock); 583ba2fc48cSLorenzo Bianconi hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]); 584c4f033d9SFelix Fietkau spin_unlock_bh(&ppe_lock); 585c4f033d9SFelix Fietkau 586c4f033d9SFelix Fietkau return 0; 587c4f033d9SFelix Fietkau } 588c4f033d9SFelix Fietkau 58933fc42deSFelix Fietkau static void 59033fc42deSFelix Fietkau mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, 59133fc42deSFelix Fietkau u16 hash) 59233fc42deSFelix Fietkau { 593ba2fc48cSLorenzo Bianconi const struct mtk_soc_data *soc = ppe->eth->soc; 59433fc42deSFelix Fietkau struct mtk_flow_entry *flow_info; 5959d8cb4c0SLorenzo Bianconi struct mtk_foe_entry foe = {}, *hwe; 59633fc42deSFelix Fietkau struct mtk_foe_mac_info *l2; 59703a3180eSLorenzo Bianconi u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP; 59833fc42deSFelix Fietkau int type; 59933fc42deSFelix Fietkau 60033fc42deSFelix Fietkau flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end), 60133fc42deSFelix Fietkau GFP_ATOMIC); 60233fc42deSFelix Fietkau if (!flow_info) 60333fc42deSFelix Fietkau return; 60433fc42deSFelix Fietkau 60533fc42deSFelix Fietkau flow_info->l2_data.base_flow = entry; 60633fc42deSFelix Fietkau flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; 60733fc42deSFelix Fietkau flow_info->hash = hash; 608ba2fc48cSLorenzo Bianconi hlist_add_head(&flow_info->list, 609ba2fc48cSLorenzo Bianconi &ppe->foe_flow[hash / soc->hash_offset]); 61033fc42deSFelix Fietkau hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows); 61133fc42deSFelix Fietkau 6129d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, hash); 6139d8cb4c0SLorenzo Bianconi memcpy(&foe, hwe, soc->foe_entry_size); 61433fc42deSFelix Fietkau foe.ib1 &= ib1_mask; 61533fc42deSFelix Fietkau foe.ib1 |= entry->data.ib1 & ~ib1_mask; 61633fc42deSFelix Fietkau 61703a3180eSLorenzo Bianconi l2 = mtk_foe_entry_l2(ppe->eth, &foe); 61833fc42deSFelix Fietkau memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); 61933fc42deSFelix Fietkau 62003a3180eSLorenzo Bianconi type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1); 62133fc42deSFelix Fietkau if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) 62233fc42deSFelix Fietkau memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); 62333fc42deSFelix Fietkau else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) 62433fc42deSFelix Fietkau l2->etype = ETH_P_IPV6; 62533fc42deSFelix Fietkau 62603a3180eSLorenzo Bianconi *mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2; 62733fc42deSFelix Fietkau 62833fc42deSFelix Fietkau __mtk_foe_entry_commit(ppe, &foe, hash); 62933fc42deSFelix Fietkau } 63033fc42deSFelix Fietkau 631c4f033d9SFelix Fietkau void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) 632c4f033d9SFelix Fietkau { 633ba2fc48cSLorenzo Bianconi const struct mtk_soc_data *soc = ppe->eth->soc; 634ba2fc48cSLorenzo Bianconi struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset]; 6359d8cb4c0SLorenzo Bianconi struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash); 63633fc42deSFelix Fietkau struct mtk_flow_entry *entry; 63733fc42deSFelix Fietkau struct mtk_foe_bridge key = {}; 63817a5f6a7SDan Carpenter struct hlist_node *n; 63933fc42deSFelix Fietkau struct ethhdr *eh; 640c4f033d9SFelix Fietkau bool found = false; 64133fc42deSFelix Fietkau u8 *tag; 642c4f033d9SFelix Fietkau 643c4f033d9SFelix Fietkau spin_lock_bh(&ppe_lock); 64433fc42deSFelix Fietkau 64533fc42deSFelix Fietkau if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND) 64633fc42deSFelix Fietkau goto out; 64733fc42deSFelix Fietkau 64817a5f6a7SDan Carpenter hlist_for_each_entry_safe(entry, n, head, list) { 64933fc42deSFelix Fietkau if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) { 65033fc42deSFelix Fietkau if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == 65133fc42deSFelix Fietkau MTK_FOE_STATE_BIND)) 65233fc42deSFelix Fietkau continue; 65333fc42deSFelix Fietkau 65433fc42deSFelix Fietkau entry->hash = 0xffff; 65533fc42deSFelix Fietkau __mtk_foe_entry_clear(ppe, entry); 65633fc42deSFelix Fietkau continue; 65733fc42deSFelix Fietkau } 65833fc42deSFelix Fietkau 65903a3180eSLorenzo Bianconi if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) { 660c4f033d9SFelix Fietkau if (entry->hash != 0xffff) 661c4f033d9SFelix Fietkau entry->hash = 0xffff; 662c4f033d9SFelix Fietkau continue; 663c4f033d9SFelix Fietkau } 664c4f033d9SFelix Fietkau 665c4f033d9SFelix Fietkau entry->hash = hash; 666c4f033d9SFelix Fietkau __mtk_foe_entry_commit(ppe, &entry->data, hash); 667c4f033d9SFelix Fietkau found = true; 668c4f033d9SFelix Fietkau } 66933fc42deSFelix Fietkau 67033fc42deSFelix Fietkau if (found) 67133fc42deSFelix Fietkau goto out; 67233fc42deSFelix Fietkau 67333fc42deSFelix Fietkau eh = eth_hdr(skb); 67433fc42deSFelix Fietkau ether_addr_copy(key.dest_mac, eh->h_dest); 67533fc42deSFelix Fietkau ether_addr_copy(key.src_mac, eh->h_source); 67633fc42deSFelix Fietkau tag = skb->data - 2; 67733fc42deSFelix Fietkau key.vlan = 0; 67833fc42deSFelix Fietkau switch (skb->protocol) { 67933fc42deSFelix Fietkau #if IS_ENABLED(CONFIG_NET_DSA) 68033fc42deSFelix Fietkau case htons(ETH_P_XDSA): 68133fc42deSFelix Fietkau if (!netdev_uses_dsa(skb->dev) || 68233fc42deSFelix Fietkau skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) 68333fc42deSFelix Fietkau goto out; 68433fc42deSFelix Fietkau 68533fc42deSFelix Fietkau tag += 4; 68633fc42deSFelix Fietkau if (get_unaligned_be16(tag) != ETH_P_8021Q) 68733fc42deSFelix Fietkau break; 68833fc42deSFelix Fietkau 68933fc42deSFelix Fietkau fallthrough; 69033fc42deSFelix Fietkau #endif 69133fc42deSFelix Fietkau case htons(ETH_P_8021Q): 69233fc42deSFelix Fietkau key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK; 69333fc42deSFelix Fietkau break; 69433fc42deSFelix Fietkau default: 69533fc42deSFelix Fietkau break; 69633fc42deSFelix Fietkau } 69733fc42deSFelix Fietkau 69833fc42deSFelix Fietkau entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params); 69933fc42deSFelix Fietkau if (!entry) 70033fc42deSFelix Fietkau goto out; 70133fc42deSFelix Fietkau 70233fc42deSFelix Fietkau mtk_foe_entry_commit_subflow(ppe, entry, hash); 70333fc42deSFelix Fietkau 70433fc42deSFelix Fietkau out: 705c4f033d9SFelix Fietkau spin_unlock_bh(&ppe_lock); 706c4f033d9SFelix Fietkau } 707c4f033d9SFelix Fietkau 708c4f033d9SFelix Fietkau int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 709c4f033d9SFelix Fietkau { 710c4f033d9SFelix Fietkau mtk_flow_entry_update(ppe, entry); 711c4f033d9SFelix Fietkau 71233fc42deSFelix Fietkau return __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 713c4f033d9SFelix Fietkau } 714c4f033d9SFelix Fietkau 715c4f033d9SFelix Fietkau struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, 7164ff1a3fcSLorenzo Bianconi int version, int index) 717ba37b7caSFelix Fietkau { 718ba2fc48cSLorenzo Bianconi const struct mtk_soc_data *soc = eth->soc; 719c4f033d9SFelix Fietkau struct device *dev = eth->dev; 7201ccc723bSFelix Fietkau struct mtk_ppe *ppe; 721ba2fc48cSLorenzo Bianconi u32 foe_flow_size; 7229d8cb4c0SLorenzo Bianconi void *foe; 7231ccc723bSFelix Fietkau 7241ccc723bSFelix Fietkau ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL); 7251ccc723bSFelix Fietkau if (!ppe) 7261ccc723bSFelix Fietkau return NULL; 727ba37b7caSFelix Fietkau 72833fc42deSFelix Fietkau rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params); 72933fc42deSFelix Fietkau 730ba37b7caSFelix Fietkau /* need to allocate a separate device, since it PPE DMA access is 731ba37b7caSFelix Fietkau * not coherent. 732ba37b7caSFelix Fietkau */ 733ba37b7caSFelix Fietkau ppe->base = base; 734c4f033d9SFelix Fietkau ppe->eth = eth; 735ba37b7caSFelix Fietkau ppe->dev = dev; 736ba37b7caSFelix Fietkau ppe->version = version; 737ba37b7caSFelix Fietkau 7389d8cb4c0SLorenzo Bianconi foe = dmam_alloc_coherent(ppe->dev, 7399d8cb4c0SLorenzo Bianconi MTK_PPE_ENTRIES * soc->foe_entry_size, 740ba37b7caSFelix Fietkau &ppe->foe_phys, GFP_KERNEL); 741ba37b7caSFelix Fietkau if (!foe) 7421ccc723bSFelix Fietkau return NULL; 743ba37b7caSFelix Fietkau 744ba37b7caSFelix Fietkau ppe->foe_table = foe; 745ba37b7caSFelix Fietkau 746ba2fc48cSLorenzo Bianconi foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) * 747ba2fc48cSLorenzo Bianconi sizeof(*ppe->foe_flow); 748ba2fc48cSLorenzo Bianconi ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL); 749ba2fc48cSLorenzo Bianconi if (!ppe->foe_flow) 750ba2fc48cSLorenzo Bianconi return NULL; 751ba2fc48cSLorenzo Bianconi 7524ff1a3fcSLorenzo Bianconi mtk_ppe_debugfs_init(ppe, index); 753ba37b7caSFelix Fietkau 7541ccc723bSFelix Fietkau return ppe; 755ba37b7caSFelix Fietkau } 756ba37b7caSFelix Fietkau 757ba37b7caSFelix Fietkau static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) 758ba37b7caSFelix Fietkau { 759ba37b7caSFelix Fietkau static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 }; 760ba37b7caSFelix Fietkau int i, k; 761ba37b7caSFelix Fietkau 7629d8cb4c0SLorenzo Bianconi memset(ppe->foe_table, 0, 7639d8cb4c0SLorenzo Bianconi MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size); 764ba37b7caSFelix Fietkau 765ba37b7caSFelix Fietkau if (!IS_ENABLED(CONFIG_SOC_MT7621)) 766ba37b7caSFelix Fietkau return; 767ba37b7caSFelix Fietkau 768ba37b7caSFelix Fietkau /* skip all entries that cross the 1024 byte boundary */ 7699d8cb4c0SLorenzo Bianconi for (i = 0; i < MTK_PPE_ENTRIES; i += 128) { 7709d8cb4c0SLorenzo Bianconi for (k = 0; k < ARRAY_SIZE(skip); k++) { 7719d8cb4c0SLorenzo Bianconi struct mtk_foe_entry *hwe; 7729d8cb4c0SLorenzo Bianconi 7739d8cb4c0SLorenzo Bianconi hwe = mtk_foe_get_entry(ppe, i + skip[k]); 7749d8cb4c0SLorenzo Bianconi hwe->ib1 |= MTK_FOE_IB1_STATIC; 7759d8cb4c0SLorenzo Bianconi } 7769d8cb4c0SLorenzo Bianconi } 777ba37b7caSFelix Fietkau } 778ba37b7caSFelix Fietkau 7794ff1a3fcSLorenzo Bianconi void mtk_ppe_start(struct mtk_ppe *ppe) 780ba37b7caSFelix Fietkau { 781ba37b7caSFelix Fietkau u32 val; 782ba37b7caSFelix Fietkau 7834ff1a3fcSLorenzo Bianconi if (!ppe) 7844ff1a3fcSLorenzo Bianconi return; 7854ff1a3fcSLorenzo Bianconi 786ba37b7caSFelix Fietkau mtk_ppe_init_foe_table(ppe); 787ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys); 788ba37b7caSFelix Fietkau 789ba37b7caSFelix Fietkau val = MTK_PPE_TB_CFG_ENTRY_80B | 790ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_NON_L4 | 791ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_UNBIND | 792ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_TCP | 793ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_UDP | 794ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_TCP_FIN | 795ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS, 796ba37b7caSFelix Fietkau MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) | 797ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE, 798ba37b7caSFelix Fietkau MTK_PPE_KEEPALIVE_DISABLE) | 799ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) | 800ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE, 801ba37b7caSFelix Fietkau MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) | 802ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, 803ba37b7caSFelix Fietkau MTK_PPE_ENTRIES_SHIFT); 80403a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 80503a3180eSLorenzo Bianconi val |= MTK_PPE_TB_CFG_INFO_SEL; 806ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_TB_CFG, val); 807ba37b7caSFelix Fietkau 808ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, 809ba37b7caSFelix Fietkau MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6); 810ba37b7caSFelix Fietkau 811ba37b7caSFelix Fietkau mtk_ppe_cache_enable(ppe, true); 812ba37b7caSFelix Fietkau 81303a3180eSLorenzo Bianconi val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | 814ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP6_5T_ROUTE | 815ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP6_6RD | 816ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP4_NAT | 817ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP4_NAPT | 818ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP4_DSLITE | 819ba37b7caSFelix Fietkau MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; 82003a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 82103a3180eSLorenzo Bianconi val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | 82203a3180eSLorenzo Bianconi MTK_PPE_MD_TOAP_BYP_CRSN1 | 82303a3180eSLorenzo Bianconi MTK_PPE_MD_TOAP_BYP_CRSN2 | 82403a3180eSLorenzo Bianconi MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY; 82503a3180eSLorenzo Bianconi else 82603a3180eSLorenzo Bianconi val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | 82703a3180eSLorenzo Bianconi MTK_PPE_FLOW_CFG_IP4_UDP_FRAG; 828ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); 829ba37b7caSFelix Fietkau 830ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) | 831ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3); 832ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val); 833ba37b7caSFelix Fietkau 834ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) | 835ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1); 836ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_AGE0, val); 837ba37b7caSFelix Fietkau 838ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 839ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7); 840ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_AGE1, val); 841ba37b7caSFelix Fietkau 842ba37b7caSFelix Fietkau val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF; 843ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val); 844ba37b7caSFelix Fietkau 845ba37b7caSFelix Fietkau val = MTK_PPE_BIND_LIMIT1_FULL | 846ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1); 847ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val); 848ba37b7caSFelix Fietkau 849ba37b7caSFelix Fietkau val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) | 850ba37b7caSFelix Fietkau FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1); 851ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_BIND_RATE, val); 852ba37b7caSFelix Fietkau 853ba37b7caSFelix Fietkau /* enable PPE */ 854ba37b7caSFelix Fietkau val = MTK_PPE_GLO_CFG_EN | 855ba37b7caSFelix Fietkau MTK_PPE_GLO_CFG_IP4_L4_CS_DROP | 856ba37b7caSFelix Fietkau MTK_PPE_GLO_CFG_IP4_CS_DROP | 857ba37b7caSFelix Fietkau MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE; 858ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_GLO_CFG, val); 859ba37b7caSFelix Fietkau 860ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0); 86103a3180eSLorenzo Bianconi 86203a3180eSLorenzo Bianconi if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) { 86303a3180eSLorenzo Bianconi ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777); 86403a3180eSLorenzo Bianconi ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f); 86503a3180eSLorenzo Bianconi } 866ba37b7caSFelix Fietkau } 867ba37b7caSFelix Fietkau 868ba37b7caSFelix Fietkau int mtk_ppe_stop(struct mtk_ppe *ppe) 869ba37b7caSFelix Fietkau { 870ba37b7caSFelix Fietkau u32 val; 871ba37b7caSFelix Fietkau int i; 872ba37b7caSFelix Fietkau 8734ff1a3fcSLorenzo Bianconi if (!ppe) 8744ff1a3fcSLorenzo Bianconi return 0; 8754ff1a3fcSLorenzo Bianconi 8769d8cb4c0SLorenzo Bianconi for (i = 0; i < MTK_PPE_ENTRIES; i++) { 8779d8cb4c0SLorenzo Bianconi struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i); 8789d8cb4c0SLorenzo Bianconi 8799d8cb4c0SLorenzo Bianconi hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, 880ba37b7caSFelix Fietkau MTK_FOE_STATE_INVALID); 8819d8cb4c0SLorenzo Bianconi } 882ba37b7caSFelix Fietkau 883ba37b7caSFelix Fietkau mtk_ppe_cache_enable(ppe, false); 884ba37b7caSFelix Fietkau 885ba37b7caSFelix Fietkau /* disable offload engine */ 886ba37b7caSFelix Fietkau ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN); 887ba37b7caSFelix Fietkau ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0); 888ba37b7caSFelix Fietkau 889ba37b7caSFelix Fietkau /* disable aging */ 890ba37b7caSFelix Fietkau val = MTK_PPE_TB_CFG_AGE_NON_L4 | 891ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_UNBIND | 892ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_TCP | 893ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_UDP | 894ba37b7caSFelix Fietkau MTK_PPE_TB_CFG_AGE_TCP_FIN; 895ba37b7caSFelix Fietkau ppe_clear(ppe, MTK_PPE_TB_CFG, val); 896ba37b7caSFelix Fietkau 897ba37b7caSFelix Fietkau return mtk_ppe_wait_busy(ppe); 898ba37b7caSFelix Fietkau } 899