1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ 3 4 #include <linux/kernel.h> 5 #include <linux/io.h> 6 #include <linux/iopoll.h> 7 #include <linux/etherdevice.h> 8 #include <linux/platform_device.h> 9 #include <linux/if_ether.h> 10 #include <linux/if_vlan.h> 11 #include <net/dst_metadata.h> 12 #include <net/dsa.h> 13 #include "mtk_eth_soc.h" 14 #include "mtk_ppe.h" 15 #include "mtk_ppe_regs.h" 16 17 static DEFINE_SPINLOCK(ppe_lock); 18 19 static const struct rhashtable_params mtk_flow_l2_ht_params = { 20 .head_offset = offsetof(struct mtk_flow_entry, l2_node), 21 .key_offset = offsetof(struct mtk_flow_entry, data.bridge), 22 .key_len = offsetof(struct mtk_foe_bridge, key_end), 23 .automatic_shrinking = true, 24 }; 25 26 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val) 27 { 28 writel(val, ppe->base + reg); 29 } 30 31 static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg) 32 { 33 return readl(ppe->base + reg); 34 } 35 36 static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set) 37 { 38 u32 val; 39 40 val = ppe_r32(ppe, reg); 41 val &= ~mask; 42 val |= set; 43 ppe_w32(ppe, reg, val); 44 45 return val; 46 } 47 48 static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val) 49 { 50 return ppe_m32(ppe, reg, 0, val); 51 } 52 53 static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) 54 { 55 return ppe_m32(ppe, reg, val, 0); 56 } 57 58 static u32 mtk_eth_timestamp(struct mtk_eth *eth) 59 { 60 return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth); 61 } 62 63 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) 64 { 65 int ret; 66 u32 val; 67 68 ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val, 69 !(val & MTK_PPE_GLO_CFG_BUSY), 70 20, MTK_PPE_WAIT_TIMEOUT_US); 71 72 if (ret) 73 dev_err(ppe->dev, "PPE table busy"); 74 75 return ret; 76 } 77 78 static int mtk_ppe_mib_wait_busy(struct mtk_ppe *ppe) 79 { 80 int ret; 81 u32 val; 82 83 ret = readl_poll_timeout(ppe->base + MTK_PPE_MIB_SER_CR, val, 84 !(val & MTK_PPE_MIB_SER_CR_ST), 85 20, MTK_PPE_WAIT_TIMEOUT_US); 86 87 if (ret) 88 dev_err(ppe->dev, "MIB table busy"); 89 90 return ret; 91 } 92 93 static int mtk_mib_entry_read(struct mtk_ppe *ppe, u16 index, u64 *bytes, u64 *packets) 94 { 95 u32 byte_cnt_low, byte_cnt_high, pkt_cnt_low, pkt_cnt_high; 96 u32 val, cnt_r0, cnt_r1, cnt_r2; 97 int ret; 98 99 val = FIELD_PREP(MTK_PPE_MIB_SER_CR_ADDR, index) | MTK_PPE_MIB_SER_CR_ST; 100 ppe_w32(ppe, MTK_PPE_MIB_SER_CR, val); 101 102 ret = mtk_ppe_mib_wait_busy(ppe); 103 if (ret) 104 return ret; 105 106 cnt_r0 = readl(ppe->base + MTK_PPE_MIB_SER_R0); 107 cnt_r1 = readl(ppe->base + MTK_PPE_MIB_SER_R1); 108 cnt_r2 = readl(ppe->base + MTK_PPE_MIB_SER_R2); 109 110 byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0); 111 byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH, cnt_r1); 112 pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R1_PKT_CNT_LOW, cnt_r1); 113 pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH, cnt_r2); 114 *bytes = ((u64)byte_cnt_high << 32) | byte_cnt_low; 115 *packets = (pkt_cnt_high << 16) | pkt_cnt_low; 116 117 return 0; 118 } 119 120 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe) 121 { 122 ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 123 ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 124 } 125 126 static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable) 127 { 128 mtk_ppe_cache_clear(ppe); 129 130 ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN, 131 enable * MTK_PPE_CACHE_CTL_EN); 132 } 133 134 static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e) 135 { 136 u32 hv1, hv2, hv3; 137 u32 hash; 138 139 switch (mtk_get_ib1_pkt_type(eth, e->ib1)) { 140 case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 141 case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 142 hv1 = e->ipv4.orig.ports; 143 hv2 = e->ipv4.orig.dest_ip; 144 hv3 = e->ipv4.orig.src_ip; 145 break; 146 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 147 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 148 hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3]; 149 hv1 ^= e->ipv6.ports; 150 151 hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2]; 152 hv2 ^= e->ipv6.dest_ip[0]; 153 154 hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1]; 155 hv3 ^= e->ipv6.src_ip[0]; 156 break; 157 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 158 case MTK_PPE_PKT_TYPE_IPV6_6RD: 159 default: 160 WARN_ON_ONCE(1); 161 return MTK_PPE_HASH_MASK; 162 } 163 164 hash = (hv1 & hv2) | ((~hv1) & hv3); 165 hash = (hash >> 24) | ((hash & 0xffffff) << 8); 166 hash ^= hv1 ^ hv2 ^ hv3; 167 hash ^= hash >> 16; 168 hash <<= (ffs(eth->soc->hash_offset) - 1); 169 hash &= MTK_PPE_ENTRIES - 1; 170 171 return hash; 172 } 173 174 static inline struct mtk_foe_mac_info * 175 mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 176 { 177 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 178 179 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 180 return &entry->bridge.l2; 181 182 if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 183 return &entry->ipv6.l2; 184 185 return &entry->ipv4.l2; 186 } 187 188 static inline u32 * 189 mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 190 { 191 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 192 193 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 194 return &entry->bridge.ib2; 195 196 if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 197 return &entry->ipv6.ib2; 198 199 return &entry->ipv4.ib2; 200 } 201 202 int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, 203 int type, int l4proto, u8 pse_port, u8 *src_mac, 204 u8 *dest_mac) 205 { 206 struct mtk_foe_mac_info *l2; 207 u32 ports_pad, val; 208 209 memset(entry, 0, sizeof(*entry)); 210 211 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 212 val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 213 FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | 214 FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 215 MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2; 216 entry->ib1 = val; 217 218 val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) | 219 FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf); 220 } else { 221 int port_mg = eth->soc->offload_version > 1 ? 0 : 0x3f; 222 223 val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 224 FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | 225 FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 226 MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL; 227 entry->ib1 = val; 228 229 val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) | 230 FIELD_PREP(MTK_FOE_IB2_PORT_MG, port_mg) | 231 FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f); 232 } 233 234 if (is_multicast_ether_addr(dest_mac)) 235 val |= mtk_get_ib2_multicast_mask(eth); 236 237 ports_pad = 0xa5a5a500 | (l4proto & 0xff); 238 if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 239 entry->ipv4.orig.ports = ports_pad; 240 if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 241 entry->ipv6.ports = ports_pad; 242 243 if (type == MTK_PPE_PKT_TYPE_BRIDGE) { 244 ether_addr_copy(entry->bridge.src_mac, src_mac); 245 ether_addr_copy(entry->bridge.dest_mac, dest_mac); 246 entry->bridge.ib2 = val; 247 l2 = &entry->bridge.l2; 248 } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { 249 entry->ipv6.ib2 = val; 250 l2 = &entry->ipv6.l2; 251 } else { 252 entry->ipv4.ib2 = val; 253 l2 = &entry->ipv4.l2; 254 } 255 256 l2->dest_mac_hi = get_unaligned_be32(dest_mac); 257 l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4); 258 l2->src_mac_hi = get_unaligned_be32(src_mac); 259 l2->src_mac_lo = get_unaligned_be16(src_mac + 4); 260 261 if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 262 l2->etype = ETH_P_IPV6; 263 else 264 l2->etype = ETH_P_IP; 265 266 return 0; 267 } 268 269 int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, 270 struct mtk_foe_entry *entry, u8 port) 271 { 272 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 273 u32 val = *ib2; 274 275 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 276 val &= ~MTK_FOE_IB2_DEST_PORT_V2; 277 val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); 278 } else { 279 val &= ~MTK_FOE_IB2_DEST_PORT; 280 val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); 281 } 282 *ib2 = val; 283 284 return 0; 285 } 286 287 int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, 288 struct mtk_foe_entry *entry, bool egress, 289 __be32 src_addr, __be16 src_port, 290 __be32 dest_addr, __be16 dest_port) 291 { 292 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 293 struct mtk_ipv4_tuple *t; 294 295 switch (type) { 296 case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 297 if (egress) { 298 t = &entry->ipv4.new; 299 break; 300 } 301 fallthrough; 302 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 303 case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 304 t = &entry->ipv4.orig; 305 break; 306 case MTK_PPE_PKT_TYPE_IPV6_6RD: 307 entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr); 308 entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr); 309 return 0; 310 default: 311 WARN_ON_ONCE(1); 312 return -EINVAL; 313 } 314 315 t->src_ip = be32_to_cpu(src_addr); 316 t->dest_ip = be32_to_cpu(dest_addr); 317 318 if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 319 return 0; 320 321 t->src_port = be16_to_cpu(src_port); 322 t->dest_port = be16_to_cpu(dest_port); 323 324 return 0; 325 } 326 327 int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, 328 struct mtk_foe_entry *entry, 329 __be32 *src_addr, __be16 src_port, 330 __be32 *dest_addr, __be16 dest_port) 331 { 332 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 333 u32 *src, *dest; 334 int i; 335 336 switch (type) { 337 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 338 src = entry->dslite.tunnel_src_ip; 339 dest = entry->dslite.tunnel_dest_ip; 340 break; 341 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 342 case MTK_PPE_PKT_TYPE_IPV6_6RD: 343 entry->ipv6.src_port = be16_to_cpu(src_port); 344 entry->ipv6.dest_port = be16_to_cpu(dest_port); 345 fallthrough; 346 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 347 src = entry->ipv6.src_ip; 348 dest = entry->ipv6.dest_ip; 349 break; 350 default: 351 WARN_ON_ONCE(1); 352 return -EINVAL; 353 } 354 355 for (i = 0; i < 4; i++) 356 src[i] = be32_to_cpu(src_addr[i]); 357 for (i = 0; i < 4; i++) 358 dest[i] = be32_to_cpu(dest_addr[i]); 359 360 return 0; 361 } 362 363 int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, 364 int port) 365 { 366 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 367 368 l2->etype = BIT(port); 369 370 if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth))) 371 entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1); 372 else 373 l2->etype |= BIT(8); 374 375 entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth); 376 377 return 0; 378 } 379 380 int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, 381 int vid) 382 { 383 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 384 385 switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) { 386 case 0: 387 entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) | 388 mtk_prep_ib1_vlan_layer(eth, 1); 389 l2->vlan1 = vid; 390 return 0; 391 case 1: 392 if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) { 393 l2->vlan1 = vid; 394 l2->etype |= BIT(8); 395 } else { 396 l2->vlan2 = vid; 397 entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1); 398 } 399 return 0; 400 default: 401 return -ENOSPC; 402 } 403 } 404 405 int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, 406 int sid) 407 { 408 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 409 410 if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) || 411 (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) 412 l2->etype = ETH_P_PPP_SES; 413 414 entry->ib1 |= mtk_get_ib1_ppoe_mask(eth); 415 l2->pppoe_id = sid; 416 417 return 0; 418 } 419 420 int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, 421 int wdma_idx, int txq, int bss, int wcid) 422 { 423 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 424 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 425 426 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 427 *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; 428 *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | 429 MTK_FOE_IB2_WDMA_WINFO_V2; 430 l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | 431 FIELD_PREP(MTK_FOE_WINFO_BSS, bss); 432 } else { 433 *ib2 &= ~MTK_FOE_IB2_PORT_MG; 434 *ib2 |= MTK_FOE_IB2_WDMA_WINFO; 435 if (wdma_idx) 436 *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; 437 l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | 438 FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | 439 FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); 440 } 441 442 return 0; 443 } 444 445 int mtk_foe_entry_set_queue(struct mtk_eth *eth, struct mtk_foe_entry *entry, 446 unsigned int queue) 447 { 448 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 449 450 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 451 *ib2 &= ~MTK_FOE_IB2_QID_V2; 452 *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID_V2, queue); 453 *ib2 |= MTK_FOE_IB2_PSE_QOS_V2; 454 } else { 455 *ib2 &= ~MTK_FOE_IB2_QID; 456 *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID, queue); 457 *ib2 |= MTK_FOE_IB2_PSE_QOS; 458 } 459 460 return 0; 461 } 462 463 static bool 464 mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry, 465 struct mtk_foe_entry *data) 466 { 467 int type, len; 468 469 if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) 470 return false; 471 472 type = mtk_get_ib1_pkt_type(eth, entry->data.ib1); 473 if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) 474 len = offsetof(struct mtk_foe_entry, ipv6._rsv); 475 else 476 len = offsetof(struct mtk_foe_entry, ipv4.ib2); 477 478 return !memcmp(&entry->data.data, &data->data, len - 4); 479 } 480 481 static void 482 __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 483 { 484 struct hlist_head *head; 485 struct hlist_node *tmp; 486 487 if (entry->type == MTK_FLOW_TYPE_L2) { 488 rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node, 489 mtk_flow_l2_ht_params); 490 491 head = &entry->l2_flows; 492 hlist_for_each_entry_safe(entry, tmp, head, l2_data.list) 493 __mtk_foe_entry_clear(ppe, entry); 494 return; 495 } 496 497 hlist_del_init(&entry->list); 498 if (entry->hash != 0xffff) { 499 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash); 500 501 hwe->ib1 &= ~MTK_FOE_IB1_STATE; 502 hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID); 503 dma_wmb(); 504 mtk_ppe_cache_clear(ppe); 505 506 if (ppe->accounting) { 507 struct mtk_foe_accounting *acct; 508 509 acct = ppe->acct_table + entry->hash * sizeof(*acct); 510 acct->packets = 0; 511 acct->bytes = 0; 512 } 513 } 514 entry->hash = 0xffff; 515 516 if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW) 517 return; 518 519 hlist_del_init(&entry->l2_data.list); 520 kfree(entry); 521 } 522 523 static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) 524 { 525 u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 526 u16 now = mtk_eth_timestamp(ppe->eth); 527 u16 timestamp = ib1 & ib1_ts_mask; 528 529 if (timestamp > now) 530 return ib1_ts_mask + 1 - timestamp + now; 531 else 532 return now - timestamp; 533 } 534 535 static void 536 mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 537 { 538 u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 539 struct mtk_flow_entry *cur; 540 struct mtk_foe_entry *hwe; 541 struct hlist_node *tmp; 542 int idle; 543 544 idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 545 hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) { 546 int cur_idle; 547 u32 ib1; 548 549 hwe = mtk_foe_get_entry(ppe, cur->hash); 550 ib1 = READ_ONCE(hwe->ib1); 551 552 if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { 553 cur->hash = 0xffff; 554 __mtk_foe_entry_clear(ppe, cur); 555 continue; 556 } 557 558 cur_idle = __mtk_foe_entry_idle_time(ppe, ib1); 559 if (cur_idle >= idle) 560 continue; 561 562 idle = cur_idle; 563 entry->data.ib1 &= ~ib1_ts_mask; 564 entry->data.ib1 |= hwe->ib1 & ib1_ts_mask; 565 } 566 } 567 568 static void 569 mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 570 { 571 struct mtk_foe_entry foe = {}; 572 struct mtk_foe_entry *hwe; 573 574 spin_lock_bh(&ppe_lock); 575 576 if (entry->type == MTK_FLOW_TYPE_L2) { 577 mtk_flow_entry_update_l2(ppe, entry); 578 goto out; 579 } 580 581 if (entry->hash == 0xffff) 582 goto out; 583 584 hwe = mtk_foe_get_entry(ppe, entry->hash); 585 memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size); 586 if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) { 587 entry->hash = 0xffff; 588 goto out; 589 } 590 591 entry->data.ib1 = foe.ib1; 592 593 out: 594 spin_unlock_bh(&ppe_lock); 595 } 596 597 static void 598 __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, 599 u16 hash) 600 { 601 struct mtk_eth *eth = ppe->eth; 602 u16 timestamp = mtk_eth_timestamp(eth); 603 struct mtk_foe_entry *hwe; 604 605 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 606 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; 607 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, 608 timestamp); 609 } else { 610 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; 611 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, 612 timestamp); 613 } 614 615 hwe = mtk_foe_get_entry(ppe, hash); 616 memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1)); 617 wmb(); 618 hwe->ib1 = entry->ib1; 619 620 if (ppe->accounting) 621 *mtk_foe_entry_ib2(eth, hwe) |= MTK_FOE_IB2_MIB_CNT; 622 623 dma_wmb(); 624 625 mtk_ppe_cache_clear(ppe); 626 } 627 628 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 629 { 630 spin_lock_bh(&ppe_lock); 631 __mtk_foe_entry_clear(ppe, entry); 632 spin_unlock_bh(&ppe_lock); 633 } 634 635 static int 636 mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 637 { 638 struct mtk_flow_entry *prev; 639 640 entry->type = MTK_FLOW_TYPE_L2; 641 642 prev = rhashtable_lookup_get_insert_fast(&ppe->l2_flows, &entry->l2_node, 643 mtk_flow_l2_ht_params); 644 if (likely(!prev)) 645 return 0; 646 647 if (IS_ERR(prev)) 648 return PTR_ERR(prev); 649 650 return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node, 651 &entry->l2_node, mtk_flow_l2_ht_params); 652 } 653 654 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 655 { 656 const struct mtk_soc_data *soc = ppe->eth->soc; 657 int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1); 658 u32 hash; 659 660 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 661 return mtk_foe_entry_commit_l2(ppe, entry); 662 663 hash = mtk_ppe_hash_entry(ppe->eth, &entry->data); 664 entry->hash = 0xffff; 665 spin_lock_bh(&ppe_lock); 666 hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]); 667 spin_unlock_bh(&ppe_lock); 668 669 return 0; 670 } 671 672 static void 673 mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, 674 u16 hash) 675 { 676 const struct mtk_soc_data *soc = ppe->eth->soc; 677 struct mtk_flow_entry *flow_info; 678 struct mtk_foe_entry foe = {}, *hwe; 679 struct mtk_foe_mac_info *l2; 680 u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP; 681 int type; 682 683 flow_info = kzalloc(sizeof(*flow_info), GFP_ATOMIC); 684 if (!flow_info) 685 return; 686 687 flow_info->l2_data.base_flow = entry; 688 flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; 689 flow_info->hash = hash; 690 hlist_add_head(&flow_info->list, 691 &ppe->foe_flow[hash / soc->hash_offset]); 692 hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows); 693 694 hwe = mtk_foe_get_entry(ppe, hash); 695 memcpy(&foe, hwe, soc->foe_entry_size); 696 foe.ib1 &= ib1_mask; 697 foe.ib1 |= entry->data.ib1 & ~ib1_mask; 698 699 l2 = mtk_foe_entry_l2(ppe->eth, &foe); 700 memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); 701 702 type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1); 703 if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) 704 memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); 705 else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) 706 l2->etype = ETH_P_IPV6; 707 708 *mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2; 709 710 __mtk_foe_entry_commit(ppe, &foe, hash); 711 } 712 713 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) 714 { 715 const struct mtk_soc_data *soc = ppe->eth->soc; 716 struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset]; 717 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash); 718 struct mtk_flow_entry *entry; 719 struct mtk_foe_bridge key = {}; 720 struct hlist_node *n; 721 struct ethhdr *eh; 722 bool found = false; 723 u8 *tag; 724 725 spin_lock_bh(&ppe_lock); 726 727 if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND) 728 goto out; 729 730 hlist_for_each_entry_safe(entry, n, head, list) { 731 if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) { 732 if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == 733 MTK_FOE_STATE_BIND)) 734 continue; 735 736 entry->hash = 0xffff; 737 __mtk_foe_entry_clear(ppe, entry); 738 continue; 739 } 740 741 if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) { 742 if (entry->hash != 0xffff) 743 entry->hash = 0xffff; 744 continue; 745 } 746 747 entry->hash = hash; 748 __mtk_foe_entry_commit(ppe, &entry->data, hash); 749 found = true; 750 } 751 752 if (found) 753 goto out; 754 755 eh = eth_hdr(skb); 756 ether_addr_copy(key.dest_mac, eh->h_dest); 757 ether_addr_copy(key.src_mac, eh->h_source); 758 tag = skb->data - 2; 759 key.vlan = 0; 760 switch (skb->protocol) { 761 #if IS_ENABLED(CONFIG_NET_DSA) 762 case htons(ETH_P_XDSA): 763 if (!netdev_uses_dsa(skb->dev) || 764 skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) 765 goto out; 766 767 if (!skb_metadata_dst(skb)) 768 tag += 4; 769 770 if (get_unaligned_be16(tag) != ETH_P_8021Q) 771 break; 772 773 fallthrough; 774 #endif 775 case htons(ETH_P_8021Q): 776 key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK; 777 break; 778 default: 779 break; 780 } 781 782 entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params); 783 if (!entry) 784 goto out; 785 786 mtk_foe_entry_commit_subflow(ppe, entry, hash); 787 788 out: 789 spin_unlock_bh(&ppe_lock); 790 } 791 792 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 793 { 794 mtk_flow_entry_update(ppe, entry); 795 796 return __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 797 } 798 799 int mtk_ppe_prepare_reset(struct mtk_ppe *ppe) 800 { 801 if (!ppe) 802 return -EINVAL; 803 804 /* disable KA */ 805 ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE); 806 ppe_clear(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE); 807 ppe_w32(ppe, MTK_PPE_KEEPALIVE, 0); 808 usleep_range(10000, 11000); 809 810 /* set KA timer to maximum */ 811 ppe_set(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE); 812 ppe_w32(ppe, MTK_PPE_KEEPALIVE, 0xffffffff); 813 814 /* set KA tick select */ 815 ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_TICK_SEL); 816 ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE); 817 usleep_range(10000, 11000); 818 819 /* disable scan mode */ 820 ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_SCAN_MODE); 821 usleep_range(10000, 11000); 822 823 return mtk_ppe_wait_busy(ppe); 824 } 825 826 struct mtk_foe_accounting *mtk_foe_entry_get_mib(struct mtk_ppe *ppe, u32 index, 827 struct mtk_foe_accounting *diff) 828 { 829 struct mtk_foe_accounting *acct; 830 int size = sizeof(struct mtk_foe_accounting); 831 u64 bytes, packets; 832 833 if (!ppe->accounting) 834 return NULL; 835 836 if (mtk_mib_entry_read(ppe, index, &bytes, &packets)) 837 return NULL; 838 839 acct = ppe->acct_table + index * size; 840 841 acct->bytes += bytes; 842 acct->packets += packets; 843 844 if (diff) { 845 diff->bytes = bytes; 846 diff->packets = packets; 847 } 848 849 return acct; 850 } 851 852 struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index) 853 { 854 bool accounting = eth->soc->has_accounting; 855 const struct mtk_soc_data *soc = eth->soc; 856 struct mtk_foe_accounting *acct; 857 struct device *dev = eth->dev; 858 struct mtk_mib_entry *mib; 859 struct mtk_ppe *ppe; 860 u32 foe_flow_size; 861 void *foe; 862 863 ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL); 864 if (!ppe) 865 return NULL; 866 867 rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params); 868 869 /* need to allocate a separate device, since it PPE DMA access is 870 * not coherent. 871 */ 872 ppe->base = base; 873 ppe->eth = eth; 874 ppe->dev = dev; 875 ppe->version = eth->soc->offload_version; 876 ppe->accounting = accounting; 877 878 foe = dmam_alloc_coherent(ppe->dev, 879 MTK_PPE_ENTRIES * soc->foe_entry_size, 880 &ppe->foe_phys, GFP_KERNEL); 881 if (!foe) 882 goto err_free_l2_flows; 883 884 ppe->foe_table = foe; 885 886 foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) * 887 sizeof(*ppe->foe_flow); 888 ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL); 889 if (!ppe->foe_flow) 890 goto err_free_l2_flows; 891 892 if (accounting) { 893 mib = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*mib), 894 &ppe->mib_phys, GFP_KERNEL); 895 if (!mib) 896 return NULL; 897 898 ppe->mib_table = mib; 899 900 acct = devm_kzalloc(dev, MTK_PPE_ENTRIES * sizeof(*acct), 901 GFP_KERNEL); 902 903 if (!acct) 904 return NULL; 905 906 ppe->acct_table = acct; 907 } 908 909 mtk_ppe_debugfs_init(ppe, index); 910 911 return ppe; 912 913 err_free_l2_flows: 914 rhashtable_destroy(&ppe->l2_flows); 915 return NULL; 916 } 917 918 void mtk_ppe_deinit(struct mtk_eth *eth) 919 { 920 int i; 921 922 for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) { 923 if (!eth->ppe[i]) 924 return; 925 rhashtable_destroy(ð->ppe[i]->l2_flows); 926 } 927 } 928 929 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) 930 { 931 static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 }; 932 int i, k; 933 934 memset(ppe->foe_table, 0, 935 MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size); 936 937 if (!IS_ENABLED(CONFIG_SOC_MT7621)) 938 return; 939 940 /* skip all entries that cross the 1024 byte boundary */ 941 for (i = 0; i < MTK_PPE_ENTRIES; i += 128) { 942 for (k = 0; k < ARRAY_SIZE(skip); k++) { 943 struct mtk_foe_entry *hwe; 944 945 hwe = mtk_foe_get_entry(ppe, i + skip[k]); 946 hwe->ib1 |= MTK_FOE_IB1_STATIC; 947 } 948 } 949 } 950 951 void mtk_ppe_start(struct mtk_ppe *ppe) 952 { 953 u32 val; 954 955 if (!ppe) 956 return; 957 958 mtk_ppe_init_foe_table(ppe); 959 ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys); 960 961 val = MTK_PPE_TB_CFG_ENTRY_80B | 962 MTK_PPE_TB_CFG_AGE_NON_L4 | 963 MTK_PPE_TB_CFG_AGE_UNBIND | 964 MTK_PPE_TB_CFG_AGE_TCP | 965 MTK_PPE_TB_CFG_AGE_UDP | 966 MTK_PPE_TB_CFG_AGE_TCP_FIN | 967 FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS, 968 MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) | 969 FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE, 970 MTK_PPE_KEEPALIVE_DISABLE) | 971 FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) | 972 FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE, 973 MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) | 974 FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, 975 MTK_PPE_ENTRIES_SHIFT); 976 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 977 val |= MTK_PPE_TB_CFG_INFO_SEL; 978 ppe_w32(ppe, MTK_PPE_TB_CFG, val); 979 980 ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, 981 MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6); 982 983 mtk_ppe_cache_enable(ppe, true); 984 985 val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | 986 MTK_PPE_FLOW_CFG_IP6_5T_ROUTE | 987 MTK_PPE_FLOW_CFG_IP6_6RD | 988 MTK_PPE_FLOW_CFG_IP4_NAT | 989 MTK_PPE_FLOW_CFG_IP4_NAPT | 990 MTK_PPE_FLOW_CFG_IP4_DSLITE | 991 MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; 992 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 993 val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | 994 MTK_PPE_MD_TOAP_BYP_CRSN1 | 995 MTK_PPE_MD_TOAP_BYP_CRSN2 | 996 MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY; 997 else 998 val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | 999 MTK_PPE_FLOW_CFG_IP4_UDP_FRAG; 1000 ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); 1001 1002 val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) | 1003 FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3); 1004 ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val); 1005 1006 val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) | 1007 FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1); 1008 ppe_w32(ppe, MTK_PPE_BIND_AGE0, val); 1009 1010 val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 1011 FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7); 1012 ppe_w32(ppe, MTK_PPE_BIND_AGE1, val); 1013 1014 val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF; 1015 ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val); 1016 1017 val = MTK_PPE_BIND_LIMIT1_FULL | 1018 FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1); 1019 ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val); 1020 1021 val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) | 1022 FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1); 1023 ppe_w32(ppe, MTK_PPE_BIND_RATE, val); 1024 1025 /* enable PPE */ 1026 val = MTK_PPE_GLO_CFG_EN | 1027 MTK_PPE_GLO_CFG_IP4_L4_CS_DROP | 1028 MTK_PPE_GLO_CFG_IP4_CS_DROP | 1029 MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE; 1030 ppe_w32(ppe, MTK_PPE_GLO_CFG, val); 1031 1032 ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0); 1033 1034 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) { 1035 ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777); 1036 ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f); 1037 } 1038 1039 if (ppe->accounting && ppe->mib_phys) { 1040 ppe_w32(ppe, MTK_PPE_MIB_TB_BASE, ppe->mib_phys); 1041 ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_EN, 1042 MTK_PPE_MIB_CFG_EN); 1043 ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_RD_CLR, 1044 MTK_PPE_MIB_CFG_RD_CLR); 1045 ppe_m32(ppe, MTK_PPE_MIB_CACHE_CTL, MTK_PPE_MIB_CACHE_CTL_EN, 1046 MTK_PPE_MIB_CFG_RD_CLR); 1047 } 1048 } 1049 1050 int mtk_ppe_stop(struct mtk_ppe *ppe) 1051 { 1052 u32 val; 1053 int i; 1054 1055 if (!ppe) 1056 return 0; 1057 1058 for (i = 0; i < MTK_PPE_ENTRIES; i++) { 1059 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i); 1060 1061 hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, 1062 MTK_FOE_STATE_INVALID); 1063 } 1064 1065 mtk_ppe_cache_enable(ppe, false); 1066 1067 /* disable offload engine */ 1068 ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN); 1069 ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0); 1070 1071 /* disable aging */ 1072 val = MTK_PPE_TB_CFG_AGE_NON_L4 | 1073 MTK_PPE_TB_CFG_AGE_UNBIND | 1074 MTK_PPE_TB_CFG_AGE_TCP | 1075 MTK_PPE_TB_CFG_AGE_UDP | 1076 MTK_PPE_TB_CFG_AGE_TCP_FIN; 1077 ppe_clear(ppe, MTK_PPE_TB_CFG, val); 1078 1079 return mtk_ppe_wait_busy(ppe); 1080 } 1081