1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ 3 4 #include <linux/kernel.h> 5 #include <linux/io.h> 6 #include <linux/iopoll.h> 7 #include <linux/etherdevice.h> 8 #include <linux/platform_device.h> 9 #include <linux/if_ether.h> 10 #include <linux/if_vlan.h> 11 #include <net/dsa.h> 12 #include "mtk_eth_soc.h" 13 #include "mtk_ppe.h" 14 #include "mtk_ppe_regs.h" 15 16 static DEFINE_SPINLOCK(ppe_lock); 17 18 static const struct rhashtable_params mtk_flow_l2_ht_params = { 19 .head_offset = offsetof(struct mtk_flow_entry, l2_node), 20 .key_offset = offsetof(struct mtk_flow_entry, data.bridge), 21 .key_len = offsetof(struct mtk_foe_bridge, key_end), 22 .automatic_shrinking = true, 23 }; 24 25 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val) 26 { 27 writel(val, ppe->base + reg); 28 } 29 30 static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg) 31 { 32 return readl(ppe->base + reg); 33 } 34 35 static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set) 36 { 37 u32 val; 38 39 val = ppe_r32(ppe, reg); 40 val &= ~mask; 41 val |= set; 42 ppe_w32(ppe, reg, val); 43 44 return val; 45 } 46 47 static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val) 48 { 49 return ppe_m32(ppe, reg, 0, val); 50 } 51 52 static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) 53 { 54 return ppe_m32(ppe, reg, val, 0); 55 } 56 57 static u32 mtk_eth_timestamp(struct mtk_eth *eth) 58 { 59 return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth); 60 } 61 62 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) 63 { 64 int ret; 65 u32 val; 66 67 ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val, 68 !(val & MTK_PPE_GLO_CFG_BUSY), 69 20, MTK_PPE_WAIT_TIMEOUT_US); 70 71 if (ret) 72 dev_err(ppe->dev, "PPE table busy"); 73 74 return ret; 75 } 76 77 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe) 78 { 79 ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 80 ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); 81 } 82 83 static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable) 84 { 85 mtk_ppe_cache_clear(ppe); 86 87 ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN, 88 enable * MTK_PPE_CACHE_CTL_EN); 89 } 90 91 static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e) 92 { 93 u32 hv1, hv2, hv3; 94 u32 hash; 95 96 switch (mtk_get_ib1_pkt_type(eth, e->ib1)) { 97 case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 98 case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 99 hv1 = e->ipv4.orig.ports; 100 hv2 = e->ipv4.orig.dest_ip; 101 hv3 = e->ipv4.orig.src_ip; 102 break; 103 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 104 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 105 hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3]; 106 hv1 ^= e->ipv6.ports; 107 108 hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2]; 109 hv2 ^= e->ipv6.dest_ip[0]; 110 111 hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1]; 112 hv3 ^= e->ipv6.src_ip[0]; 113 break; 114 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 115 case MTK_PPE_PKT_TYPE_IPV6_6RD: 116 default: 117 WARN_ON_ONCE(1); 118 return MTK_PPE_HASH_MASK; 119 } 120 121 hash = (hv1 & hv2) | ((~hv1) & hv3); 122 hash = (hash >> 24) | ((hash & 0xffffff) << 8); 123 hash ^= hv1 ^ hv2 ^ hv3; 124 hash ^= hash >> 16; 125 hash <<= (ffs(eth->soc->hash_offset) - 1); 126 hash &= MTK_PPE_ENTRIES - 1; 127 128 return hash; 129 } 130 131 static inline struct mtk_foe_mac_info * 132 mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 133 { 134 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 135 136 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 137 return &entry->bridge.l2; 138 139 if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 140 return &entry->ipv6.l2; 141 142 return &entry->ipv4.l2; 143 } 144 145 static inline u32 * 146 mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry) 147 { 148 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 149 150 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 151 return &entry->bridge.ib2; 152 153 if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) 154 return &entry->ipv6.ib2; 155 156 return &entry->ipv4.ib2; 157 } 158 159 int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, 160 int type, int l4proto, u8 pse_port, u8 *src_mac, 161 u8 *dest_mac) 162 { 163 struct mtk_foe_mac_info *l2; 164 u32 ports_pad, val; 165 166 memset(entry, 0, sizeof(*entry)); 167 168 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 169 val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 170 FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | 171 FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 172 MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2; 173 entry->ib1 = val; 174 175 val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) | 176 FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf); 177 } else { 178 int port_mg = eth->soc->offload_version > 1 ? 0 : 0x3f; 179 180 val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | 181 FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | 182 FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | 183 MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL; 184 entry->ib1 = val; 185 186 val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) | 187 FIELD_PREP(MTK_FOE_IB2_PORT_MG, port_mg) | 188 FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f); 189 } 190 191 if (is_multicast_ether_addr(dest_mac)) 192 val |= mtk_get_ib2_multicast_mask(eth); 193 194 ports_pad = 0xa5a5a500 | (l4proto & 0xff); 195 if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 196 entry->ipv4.orig.ports = ports_pad; 197 if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 198 entry->ipv6.ports = ports_pad; 199 200 if (type == MTK_PPE_PKT_TYPE_BRIDGE) { 201 ether_addr_copy(entry->bridge.src_mac, src_mac); 202 ether_addr_copy(entry->bridge.dest_mac, dest_mac); 203 entry->bridge.ib2 = val; 204 l2 = &entry->bridge.l2; 205 } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { 206 entry->ipv6.ib2 = val; 207 l2 = &entry->ipv6.l2; 208 } else { 209 entry->ipv4.ib2 = val; 210 l2 = &entry->ipv4.l2; 211 } 212 213 l2->dest_mac_hi = get_unaligned_be32(dest_mac); 214 l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4); 215 l2->src_mac_hi = get_unaligned_be32(src_mac); 216 l2->src_mac_lo = get_unaligned_be16(src_mac + 4); 217 218 if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) 219 l2->etype = ETH_P_IPV6; 220 else 221 l2->etype = ETH_P_IP; 222 223 return 0; 224 } 225 226 int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, 227 struct mtk_foe_entry *entry, u8 port) 228 { 229 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 230 u32 val = *ib2; 231 232 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 233 val &= ~MTK_FOE_IB2_DEST_PORT_V2; 234 val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); 235 } else { 236 val &= ~MTK_FOE_IB2_DEST_PORT; 237 val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); 238 } 239 *ib2 = val; 240 241 return 0; 242 } 243 244 int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, 245 struct mtk_foe_entry *entry, bool egress, 246 __be32 src_addr, __be16 src_port, 247 __be32 dest_addr, __be16 dest_port) 248 { 249 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 250 struct mtk_ipv4_tuple *t; 251 252 switch (type) { 253 case MTK_PPE_PKT_TYPE_IPV4_HNAPT: 254 if (egress) { 255 t = &entry->ipv4.new; 256 break; 257 } 258 fallthrough; 259 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 260 case MTK_PPE_PKT_TYPE_IPV4_ROUTE: 261 t = &entry->ipv4.orig; 262 break; 263 case MTK_PPE_PKT_TYPE_IPV6_6RD: 264 entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr); 265 entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr); 266 return 0; 267 default: 268 WARN_ON_ONCE(1); 269 return -EINVAL; 270 } 271 272 t->src_ip = be32_to_cpu(src_addr); 273 t->dest_ip = be32_to_cpu(dest_addr); 274 275 if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) 276 return 0; 277 278 t->src_port = be16_to_cpu(src_port); 279 t->dest_port = be16_to_cpu(dest_port); 280 281 return 0; 282 } 283 284 int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, 285 struct mtk_foe_entry *entry, 286 __be32 *src_addr, __be16 src_port, 287 __be32 *dest_addr, __be16 dest_port) 288 { 289 int type = mtk_get_ib1_pkt_type(eth, entry->ib1); 290 u32 *src, *dest; 291 int i; 292 293 switch (type) { 294 case MTK_PPE_PKT_TYPE_IPV4_DSLITE: 295 src = entry->dslite.tunnel_src_ip; 296 dest = entry->dslite.tunnel_dest_ip; 297 break; 298 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: 299 case MTK_PPE_PKT_TYPE_IPV6_6RD: 300 entry->ipv6.src_port = be16_to_cpu(src_port); 301 entry->ipv6.dest_port = be16_to_cpu(dest_port); 302 fallthrough; 303 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: 304 src = entry->ipv6.src_ip; 305 dest = entry->ipv6.dest_ip; 306 break; 307 default: 308 WARN_ON_ONCE(1); 309 return -EINVAL; 310 } 311 312 for (i = 0; i < 4; i++) 313 src[i] = be32_to_cpu(src_addr[i]); 314 for (i = 0; i < 4; i++) 315 dest[i] = be32_to_cpu(dest_addr[i]); 316 317 return 0; 318 } 319 320 int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, 321 int port) 322 { 323 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 324 325 l2->etype = BIT(port); 326 327 if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth))) 328 entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1); 329 else 330 l2->etype |= BIT(8); 331 332 entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth); 333 334 return 0; 335 } 336 337 int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, 338 int vid) 339 { 340 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 341 342 switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) { 343 case 0: 344 entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) | 345 mtk_prep_ib1_vlan_layer(eth, 1); 346 l2->vlan1 = vid; 347 return 0; 348 case 1: 349 if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) { 350 l2->vlan1 = vid; 351 l2->etype |= BIT(8); 352 } else { 353 l2->vlan2 = vid; 354 entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1); 355 } 356 return 0; 357 default: 358 return -ENOSPC; 359 } 360 } 361 362 int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, 363 int sid) 364 { 365 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 366 367 if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) || 368 (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) 369 l2->etype = ETH_P_PPP_SES; 370 371 entry->ib1 |= mtk_get_ib1_ppoe_mask(eth); 372 l2->pppoe_id = sid; 373 374 return 0; 375 } 376 377 int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, 378 int wdma_idx, int txq, int bss, int wcid) 379 { 380 struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); 381 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 382 383 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 384 *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; 385 *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | 386 MTK_FOE_IB2_WDMA_WINFO_V2; 387 l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | 388 FIELD_PREP(MTK_FOE_WINFO_BSS, bss); 389 } else { 390 *ib2 &= ~MTK_FOE_IB2_PORT_MG; 391 *ib2 |= MTK_FOE_IB2_WDMA_WINFO; 392 if (wdma_idx) 393 *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; 394 l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | 395 FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | 396 FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); 397 } 398 399 return 0; 400 } 401 402 int mtk_foe_entry_set_queue(struct mtk_eth *eth, struct mtk_foe_entry *entry, 403 unsigned int queue) 404 { 405 u32 *ib2 = mtk_foe_entry_ib2(eth, entry); 406 407 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 408 *ib2 &= ~MTK_FOE_IB2_QID_V2; 409 *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID_V2, queue); 410 *ib2 |= MTK_FOE_IB2_PSE_QOS_V2; 411 } else { 412 *ib2 &= ~MTK_FOE_IB2_QID; 413 *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID, queue); 414 *ib2 |= MTK_FOE_IB2_PSE_QOS; 415 } 416 417 return 0; 418 } 419 420 static bool 421 mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry, 422 struct mtk_foe_entry *data) 423 { 424 int type, len; 425 426 if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) 427 return false; 428 429 type = mtk_get_ib1_pkt_type(eth, entry->data.ib1); 430 if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) 431 len = offsetof(struct mtk_foe_entry, ipv6._rsv); 432 else 433 len = offsetof(struct mtk_foe_entry, ipv4.ib2); 434 435 return !memcmp(&entry->data.data, &data->data, len - 4); 436 } 437 438 static void 439 __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 440 { 441 struct hlist_head *head; 442 struct hlist_node *tmp; 443 444 if (entry->type == MTK_FLOW_TYPE_L2) { 445 rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node, 446 mtk_flow_l2_ht_params); 447 448 head = &entry->l2_flows; 449 hlist_for_each_entry_safe(entry, tmp, head, l2_data.list) 450 __mtk_foe_entry_clear(ppe, entry); 451 return; 452 } 453 454 hlist_del_init(&entry->list); 455 if (entry->hash != 0xffff) { 456 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash); 457 458 hwe->ib1 &= ~MTK_FOE_IB1_STATE; 459 hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID); 460 dma_wmb(); 461 } 462 entry->hash = 0xffff; 463 464 if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW) 465 return; 466 467 hlist_del_init(&entry->l2_data.list); 468 kfree(entry); 469 } 470 471 static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) 472 { 473 u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 474 u16 now = mtk_eth_timestamp(ppe->eth); 475 u16 timestamp = ib1 & ib1_ts_mask; 476 477 if (timestamp > now) 478 return ib1_ts_mask + 1 - timestamp + now; 479 else 480 return now - timestamp; 481 } 482 483 static void 484 mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 485 { 486 u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); 487 struct mtk_flow_entry *cur; 488 struct mtk_foe_entry *hwe; 489 struct hlist_node *tmp; 490 int idle; 491 492 idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 493 hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) { 494 int cur_idle; 495 u32 ib1; 496 497 hwe = mtk_foe_get_entry(ppe, cur->hash); 498 ib1 = READ_ONCE(hwe->ib1); 499 500 if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { 501 cur->hash = 0xffff; 502 __mtk_foe_entry_clear(ppe, cur); 503 continue; 504 } 505 506 cur_idle = __mtk_foe_entry_idle_time(ppe, ib1); 507 if (cur_idle >= idle) 508 continue; 509 510 idle = cur_idle; 511 entry->data.ib1 &= ~ib1_ts_mask; 512 entry->data.ib1 |= hwe->ib1 & ib1_ts_mask; 513 } 514 } 515 516 static void 517 mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 518 { 519 struct mtk_foe_entry foe = {}; 520 struct mtk_foe_entry *hwe; 521 522 spin_lock_bh(&ppe_lock); 523 524 if (entry->type == MTK_FLOW_TYPE_L2) { 525 mtk_flow_entry_update_l2(ppe, entry); 526 goto out; 527 } 528 529 if (entry->hash == 0xffff) 530 goto out; 531 532 hwe = mtk_foe_get_entry(ppe, entry->hash); 533 memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size); 534 if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) { 535 entry->hash = 0xffff; 536 goto out; 537 } 538 539 entry->data.ib1 = foe.ib1; 540 541 out: 542 spin_unlock_bh(&ppe_lock); 543 } 544 545 static void 546 __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, 547 u16 hash) 548 { 549 struct mtk_eth *eth = ppe->eth; 550 u16 timestamp = mtk_eth_timestamp(eth); 551 struct mtk_foe_entry *hwe; 552 553 if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { 554 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; 555 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, 556 timestamp); 557 } else { 558 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; 559 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, 560 timestamp); 561 } 562 563 hwe = mtk_foe_get_entry(ppe, hash); 564 memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1)); 565 wmb(); 566 hwe->ib1 = entry->ib1; 567 568 dma_wmb(); 569 570 mtk_ppe_cache_clear(ppe); 571 } 572 573 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 574 { 575 spin_lock_bh(&ppe_lock); 576 __mtk_foe_entry_clear(ppe, entry); 577 spin_unlock_bh(&ppe_lock); 578 } 579 580 static int 581 mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 582 { 583 entry->type = MTK_FLOW_TYPE_L2; 584 585 return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node, 586 mtk_flow_l2_ht_params); 587 } 588 589 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 590 { 591 const struct mtk_soc_data *soc = ppe->eth->soc; 592 int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1); 593 u32 hash; 594 595 if (type == MTK_PPE_PKT_TYPE_BRIDGE) 596 return mtk_foe_entry_commit_l2(ppe, entry); 597 598 hash = mtk_ppe_hash_entry(ppe->eth, &entry->data); 599 entry->hash = 0xffff; 600 spin_lock_bh(&ppe_lock); 601 hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]); 602 spin_unlock_bh(&ppe_lock); 603 604 return 0; 605 } 606 607 static void 608 mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, 609 u16 hash) 610 { 611 const struct mtk_soc_data *soc = ppe->eth->soc; 612 struct mtk_flow_entry *flow_info; 613 struct mtk_foe_entry foe = {}, *hwe; 614 struct mtk_foe_mac_info *l2; 615 u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP; 616 int type; 617 618 flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end), 619 GFP_ATOMIC); 620 if (!flow_info) 621 return; 622 623 flow_info->l2_data.base_flow = entry; 624 flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; 625 flow_info->hash = hash; 626 hlist_add_head(&flow_info->list, 627 &ppe->foe_flow[hash / soc->hash_offset]); 628 hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows); 629 630 hwe = mtk_foe_get_entry(ppe, hash); 631 memcpy(&foe, hwe, soc->foe_entry_size); 632 foe.ib1 &= ib1_mask; 633 foe.ib1 |= entry->data.ib1 & ~ib1_mask; 634 635 l2 = mtk_foe_entry_l2(ppe->eth, &foe); 636 memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); 637 638 type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1); 639 if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) 640 memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); 641 else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) 642 l2->etype = ETH_P_IPV6; 643 644 *mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2; 645 646 __mtk_foe_entry_commit(ppe, &foe, hash); 647 } 648 649 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) 650 { 651 const struct mtk_soc_data *soc = ppe->eth->soc; 652 struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset]; 653 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash); 654 struct mtk_flow_entry *entry; 655 struct mtk_foe_bridge key = {}; 656 struct hlist_node *n; 657 struct ethhdr *eh; 658 bool found = false; 659 u8 *tag; 660 661 spin_lock_bh(&ppe_lock); 662 663 if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND) 664 goto out; 665 666 hlist_for_each_entry_safe(entry, n, head, list) { 667 if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) { 668 if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == 669 MTK_FOE_STATE_BIND)) 670 continue; 671 672 entry->hash = 0xffff; 673 __mtk_foe_entry_clear(ppe, entry); 674 continue; 675 } 676 677 if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) { 678 if (entry->hash != 0xffff) 679 entry->hash = 0xffff; 680 continue; 681 } 682 683 entry->hash = hash; 684 __mtk_foe_entry_commit(ppe, &entry->data, hash); 685 found = true; 686 } 687 688 if (found) 689 goto out; 690 691 eh = eth_hdr(skb); 692 ether_addr_copy(key.dest_mac, eh->h_dest); 693 ether_addr_copy(key.src_mac, eh->h_source); 694 tag = skb->data - 2; 695 key.vlan = 0; 696 switch (skb->protocol) { 697 #if IS_ENABLED(CONFIG_NET_DSA) 698 case htons(ETH_P_XDSA): 699 if (!netdev_uses_dsa(skb->dev) || 700 skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) 701 goto out; 702 703 tag += 4; 704 if (get_unaligned_be16(tag) != ETH_P_8021Q) 705 break; 706 707 fallthrough; 708 #endif 709 case htons(ETH_P_8021Q): 710 key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK; 711 break; 712 default: 713 break; 714 } 715 716 entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params); 717 if (!entry) 718 goto out; 719 720 mtk_foe_entry_commit_subflow(ppe, entry, hash); 721 722 out: 723 spin_unlock_bh(&ppe_lock); 724 } 725 726 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) 727 { 728 mtk_flow_entry_update(ppe, entry); 729 730 return __mtk_foe_entry_idle_time(ppe, entry->data.ib1); 731 } 732 733 struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, 734 int version, int index) 735 { 736 const struct mtk_soc_data *soc = eth->soc; 737 struct device *dev = eth->dev; 738 struct mtk_ppe *ppe; 739 u32 foe_flow_size; 740 void *foe; 741 742 ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL); 743 if (!ppe) 744 return NULL; 745 746 rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params); 747 748 /* need to allocate a separate device, since it PPE DMA access is 749 * not coherent. 750 */ 751 ppe->base = base; 752 ppe->eth = eth; 753 ppe->dev = dev; 754 ppe->version = version; 755 756 foe = dmam_alloc_coherent(ppe->dev, 757 MTK_PPE_ENTRIES * soc->foe_entry_size, 758 &ppe->foe_phys, GFP_KERNEL); 759 if (!foe) 760 goto err_free_l2_flows; 761 762 ppe->foe_table = foe; 763 764 foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) * 765 sizeof(*ppe->foe_flow); 766 ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL); 767 if (!ppe->foe_flow) 768 goto err_free_l2_flows; 769 770 mtk_ppe_debugfs_init(ppe, index); 771 772 return ppe; 773 774 err_free_l2_flows: 775 rhashtable_destroy(&ppe->l2_flows); 776 return NULL; 777 } 778 779 void mtk_ppe_deinit(struct mtk_eth *eth) 780 { 781 int i; 782 783 for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) { 784 if (!eth->ppe[i]) 785 return; 786 rhashtable_destroy(ð->ppe[i]->l2_flows); 787 } 788 } 789 790 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) 791 { 792 static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 }; 793 int i, k; 794 795 memset(ppe->foe_table, 0, 796 MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size); 797 798 if (!IS_ENABLED(CONFIG_SOC_MT7621)) 799 return; 800 801 /* skip all entries that cross the 1024 byte boundary */ 802 for (i = 0; i < MTK_PPE_ENTRIES; i += 128) { 803 for (k = 0; k < ARRAY_SIZE(skip); k++) { 804 struct mtk_foe_entry *hwe; 805 806 hwe = mtk_foe_get_entry(ppe, i + skip[k]); 807 hwe->ib1 |= MTK_FOE_IB1_STATIC; 808 } 809 } 810 } 811 812 void mtk_ppe_start(struct mtk_ppe *ppe) 813 { 814 u32 val; 815 816 if (!ppe) 817 return; 818 819 mtk_ppe_init_foe_table(ppe); 820 ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys); 821 822 val = MTK_PPE_TB_CFG_ENTRY_80B | 823 MTK_PPE_TB_CFG_AGE_NON_L4 | 824 MTK_PPE_TB_CFG_AGE_UNBIND | 825 MTK_PPE_TB_CFG_AGE_TCP | 826 MTK_PPE_TB_CFG_AGE_UDP | 827 MTK_PPE_TB_CFG_AGE_TCP_FIN | 828 FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS, 829 MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) | 830 FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE, 831 MTK_PPE_KEEPALIVE_DISABLE) | 832 FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) | 833 FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE, 834 MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) | 835 FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, 836 MTK_PPE_ENTRIES_SHIFT); 837 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 838 val |= MTK_PPE_TB_CFG_INFO_SEL; 839 ppe_w32(ppe, MTK_PPE_TB_CFG, val); 840 841 ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, 842 MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6); 843 844 mtk_ppe_cache_enable(ppe, true); 845 846 val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | 847 MTK_PPE_FLOW_CFG_IP6_5T_ROUTE | 848 MTK_PPE_FLOW_CFG_IP6_6RD | 849 MTK_PPE_FLOW_CFG_IP4_NAT | 850 MTK_PPE_FLOW_CFG_IP4_NAPT | 851 MTK_PPE_FLOW_CFG_IP4_DSLITE | 852 MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; 853 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) 854 val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | 855 MTK_PPE_MD_TOAP_BYP_CRSN1 | 856 MTK_PPE_MD_TOAP_BYP_CRSN2 | 857 MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY; 858 else 859 val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | 860 MTK_PPE_FLOW_CFG_IP4_UDP_FRAG; 861 ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); 862 863 val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) | 864 FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3); 865 ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val); 866 867 val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) | 868 FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1); 869 ppe_w32(ppe, MTK_PPE_BIND_AGE0, val); 870 871 val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | 872 FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7); 873 ppe_w32(ppe, MTK_PPE_BIND_AGE1, val); 874 875 val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF; 876 ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val); 877 878 val = MTK_PPE_BIND_LIMIT1_FULL | 879 FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1); 880 ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val); 881 882 val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) | 883 FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1); 884 ppe_w32(ppe, MTK_PPE_BIND_RATE, val); 885 886 /* enable PPE */ 887 val = MTK_PPE_GLO_CFG_EN | 888 MTK_PPE_GLO_CFG_IP4_L4_CS_DROP | 889 MTK_PPE_GLO_CFG_IP4_CS_DROP | 890 MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE; 891 ppe_w32(ppe, MTK_PPE_GLO_CFG, val); 892 893 ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0); 894 895 if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) { 896 ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777); 897 ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f); 898 } 899 } 900 901 int mtk_ppe_stop(struct mtk_ppe *ppe) 902 { 903 u32 val; 904 int i; 905 906 if (!ppe) 907 return 0; 908 909 for (i = 0; i < MTK_PPE_ENTRIES; i++) { 910 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i); 911 912 hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, 913 MTK_FOE_STATE_INVALID); 914 } 915 916 mtk_ppe_cache_enable(ppe, false); 917 918 /* disable offload engine */ 919 ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN); 920 ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0); 921 922 /* disable aging */ 923 val = MTK_PPE_TB_CFG_AGE_NON_L4 | 924 MTK_PPE_TB_CFG_AGE_UNBIND | 925 MTK_PPE_TB_CFG_AGE_TCP | 926 MTK_PPE_TB_CFG_AGE_UDP | 927 MTK_PPE_TB_CFG_AGE_TCP_FIN; 928 ppe_clear(ppe, MTK_PPE_TB_CFG, val); 929 930 return mtk_ppe_wait_busy(ppe); 931 } 932