1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 27d208687SFelix Fietkau #include <linux/kernel.h> 37d208687SFelix Fietkau #include <linux/init.h> 47d208687SFelix Fietkau #include <linux/module.h> 57d208687SFelix Fietkau #include <linux/netfilter.h> 67d208687SFelix Fietkau #include <linux/rhashtable.h> 77d208687SFelix Fietkau #include <linux/ip.h> 8a908fdecSFelix Fietkau #include <linux/ipv6.h> 97d208687SFelix Fietkau #include <linux/netdevice.h> 107d208687SFelix Fietkau #include <net/ip.h> 11a908fdecSFelix Fietkau #include <net/ipv6.h> 12a908fdecSFelix Fietkau #include <net/ip6_route.h> 137d208687SFelix Fietkau #include <net/neighbour.h> 147d208687SFelix Fietkau #include <net/netfilter/nf_flow_table.h> 157d208687SFelix Fietkau /* For layer 4 checksum field offset. */ 167d208687SFelix Fietkau #include <linux/tcp.h> 177d208687SFelix Fietkau #include <linux/udp.h> 187d208687SFelix Fietkau 1933894c36SFelix Fietkau static int nf_flow_state_check(struct flow_offload *flow, int proto, 20b6f27d32SFelix Fietkau struct sk_buff *skb, unsigned int thoff) 21b6f27d32SFelix Fietkau { 22b6f27d32SFelix Fietkau struct tcphdr *tcph; 23b6f27d32SFelix Fietkau 2433894c36SFelix Fietkau if (proto != IPPROTO_TCP) 2533894c36SFelix Fietkau return 0; 2633894c36SFelix Fietkau 27b6f27d32SFelix Fietkau if (!pskb_may_pull(skb, thoff + sizeof(*tcph))) 28b6f27d32SFelix Fietkau return -1; 29b6f27d32SFelix Fietkau 30b6f27d32SFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 31b6f27d32SFelix Fietkau if (unlikely(tcph->fin || tcph->rst)) { 32b6f27d32SFelix Fietkau flow_offload_teardown(flow); 33b6f27d32SFelix Fietkau return -1; 34b6f27d32SFelix Fietkau } 35b6f27d32SFelix Fietkau 36b6f27d32SFelix Fietkau return 0; 37b6f27d32SFelix Fietkau } 38b6f27d32SFelix Fietkau 397d208687SFelix Fietkau static int nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff, 407d208687SFelix Fietkau __be32 addr, __be32 new_addr) 417d208687SFelix Fietkau { 427d208687SFelix Fietkau struct tcphdr *tcph; 437d208687SFelix Fietkau 447d208687SFelix Fietkau if (!pskb_may_pull(skb, thoff + sizeof(*tcph)) || 457d208687SFelix Fietkau skb_try_make_writable(skb, thoff + sizeof(*tcph))) 467d208687SFelix Fietkau return -1; 477d208687SFelix Fietkau 487d208687SFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 497d208687SFelix Fietkau inet_proto_csum_replace4(&tcph->check, skb, addr, new_addr, true); 507d208687SFelix Fietkau 517d208687SFelix Fietkau return 0; 527d208687SFelix Fietkau } 537d208687SFelix Fietkau 547d208687SFelix Fietkau static int nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff, 557d208687SFelix Fietkau __be32 addr, __be32 new_addr) 567d208687SFelix Fietkau { 577d208687SFelix Fietkau struct udphdr *udph; 587d208687SFelix Fietkau 597d208687SFelix Fietkau if (!pskb_may_pull(skb, thoff + sizeof(*udph)) || 607d208687SFelix Fietkau skb_try_make_writable(skb, thoff + sizeof(*udph))) 617d208687SFelix Fietkau return -1; 627d208687SFelix Fietkau 637d208687SFelix Fietkau udph = (void *)(skb_network_header(skb) + thoff); 647d208687SFelix Fietkau if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 657d208687SFelix Fietkau inet_proto_csum_replace4(&udph->check, skb, addr, 667d208687SFelix Fietkau new_addr, true); 677d208687SFelix Fietkau if (!udph->check) 687d208687SFelix Fietkau udph->check = CSUM_MANGLED_0; 697d208687SFelix Fietkau } 707d208687SFelix Fietkau 717d208687SFelix Fietkau return 0; 727d208687SFelix Fietkau } 737d208687SFelix Fietkau 747d208687SFelix Fietkau static int nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph, 757d208687SFelix Fietkau unsigned int thoff, __be32 addr, 767d208687SFelix Fietkau __be32 new_addr) 777d208687SFelix Fietkau { 787d208687SFelix Fietkau switch (iph->protocol) { 797d208687SFelix Fietkau case IPPROTO_TCP: 807d208687SFelix Fietkau if (nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr) < 0) 817d208687SFelix Fietkau return NF_DROP; 827d208687SFelix Fietkau break; 837d208687SFelix Fietkau case IPPROTO_UDP: 847d208687SFelix Fietkau if (nf_flow_nat_ip_udp(skb, thoff, addr, new_addr) < 0) 857d208687SFelix Fietkau return NF_DROP; 867d208687SFelix Fietkau break; 877d208687SFelix Fietkau } 887d208687SFelix Fietkau 897d208687SFelix Fietkau return 0; 907d208687SFelix Fietkau } 917d208687SFelix Fietkau 927d208687SFelix Fietkau static int nf_flow_snat_ip(const struct flow_offload *flow, struct sk_buff *skb, 937d208687SFelix Fietkau struct iphdr *iph, unsigned int thoff, 947d208687SFelix Fietkau enum flow_offload_tuple_dir dir) 957d208687SFelix Fietkau { 967d208687SFelix Fietkau __be32 addr, new_addr; 977d208687SFelix Fietkau 987d208687SFelix Fietkau switch (dir) { 997d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 1007d208687SFelix Fietkau addr = iph->saddr; 1017d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; 1027d208687SFelix Fietkau iph->saddr = new_addr; 1037d208687SFelix Fietkau break; 1047d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 1057d208687SFelix Fietkau addr = iph->daddr; 1067d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr; 1077d208687SFelix Fietkau iph->daddr = new_addr; 1087d208687SFelix Fietkau break; 1097d208687SFelix Fietkau default: 1107d208687SFelix Fietkau return -1; 1117d208687SFelix Fietkau } 1127d208687SFelix Fietkau csum_replace4(&iph->check, addr, new_addr); 1137d208687SFelix Fietkau 1147d208687SFelix Fietkau return nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); 1157d208687SFelix Fietkau } 1167d208687SFelix Fietkau 1177d208687SFelix Fietkau static int nf_flow_dnat_ip(const struct flow_offload *flow, struct sk_buff *skb, 1187d208687SFelix Fietkau struct iphdr *iph, unsigned int thoff, 1197d208687SFelix Fietkau enum flow_offload_tuple_dir dir) 1207d208687SFelix Fietkau { 1217d208687SFelix Fietkau __be32 addr, new_addr; 1227d208687SFelix Fietkau 1237d208687SFelix Fietkau switch (dir) { 1247d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 1257d208687SFelix Fietkau addr = iph->daddr; 1267d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; 1277d208687SFelix Fietkau iph->daddr = new_addr; 1287d208687SFelix Fietkau break; 1297d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 1307d208687SFelix Fietkau addr = iph->saddr; 1317d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr; 1327d208687SFelix Fietkau iph->saddr = new_addr; 1337d208687SFelix Fietkau break; 1347d208687SFelix Fietkau default: 1357d208687SFelix Fietkau return -1; 1367d208687SFelix Fietkau } 1377d208687SFelix Fietkau csum_replace4(&iph->check, addr, new_addr); 1387d208687SFelix Fietkau 1397d208687SFelix Fietkau return nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); 1407d208687SFelix Fietkau } 1417d208687SFelix Fietkau 1427d208687SFelix Fietkau static int nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb, 143b6f27d32SFelix Fietkau unsigned int thoff, enum flow_offload_tuple_dir dir) 1447d208687SFelix Fietkau { 1457d208687SFelix Fietkau struct iphdr *iph = ip_hdr(skb); 1467d208687SFelix Fietkau 147355a8b13SPablo Neira Ayuso if (test_bit(NF_FLOW_SNAT, &flow->flags) && 1487d208687SFelix Fietkau (nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir) < 0 || 1497d208687SFelix Fietkau nf_flow_snat_ip(flow, skb, iph, thoff, dir) < 0)) 1507d208687SFelix Fietkau return -1; 151355a8b13SPablo Neira Ayuso if (test_bit(NF_FLOW_DNAT, &flow->flags) && 1527d208687SFelix Fietkau (nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir) < 0 || 1537d208687SFelix Fietkau nf_flow_dnat_ip(flow, skb, iph, thoff, dir) < 0)) 1547d208687SFelix Fietkau return -1; 1557d208687SFelix Fietkau 1567d208687SFelix Fietkau return 0; 1577d208687SFelix Fietkau } 1587d208687SFelix Fietkau 1597d208687SFelix Fietkau static bool ip_has_options(unsigned int thoff) 1607d208687SFelix Fietkau { 1617d208687SFelix Fietkau return thoff != sizeof(struct iphdr); 1627d208687SFelix Fietkau } 1637d208687SFelix Fietkau 1647d208687SFelix Fietkau static int nf_flow_tuple_ip(struct sk_buff *skb, const struct net_device *dev, 1657d208687SFelix Fietkau struct flow_offload_tuple *tuple) 1667d208687SFelix Fietkau { 1677d208687SFelix Fietkau struct flow_ports *ports; 1687d208687SFelix Fietkau unsigned int thoff; 1697d208687SFelix Fietkau struct iphdr *iph; 1707d208687SFelix Fietkau 1717d208687SFelix Fietkau if (!pskb_may_pull(skb, sizeof(*iph))) 1727d208687SFelix Fietkau return -1; 1737d208687SFelix Fietkau 1747d208687SFelix Fietkau iph = ip_hdr(skb); 1757d208687SFelix Fietkau thoff = iph->ihl * 4; 1767d208687SFelix Fietkau 1777d208687SFelix Fietkau if (ip_is_fragment(iph) || 1787d208687SFelix Fietkau unlikely(ip_has_options(thoff))) 1797d208687SFelix Fietkau return -1; 1807d208687SFelix Fietkau 1817d208687SFelix Fietkau if (iph->protocol != IPPROTO_TCP && 1827d208687SFelix Fietkau iph->protocol != IPPROTO_UDP) 1837d208687SFelix Fietkau return -1; 1847d208687SFelix Fietkau 18533cc3c0cSTaehee Yoo if (iph->ttl <= 1) 18633cc3c0cSTaehee Yoo return -1; 18733cc3c0cSTaehee Yoo 1887d208687SFelix Fietkau thoff = iph->ihl * 4; 1897d208687SFelix Fietkau if (!pskb_may_pull(skb, thoff + sizeof(*ports))) 1907d208687SFelix Fietkau return -1; 1917d208687SFelix Fietkau 1927d208687SFelix Fietkau ports = (struct flow_ports *)(skb_network_header(skb) + thoff); 1937d208687SFelix Fietkau 1947d208687SFelix Fietkau tuple->src_v4.s_addr = iph->saddr; 1957d208687SFelix Fietkau tuple->dst_v4.s_addr = iph->daddr; 1967d208687SFelix Fietkau tuple->src_port = ports->source; 1977d208687SFelix Fietkau tuple->dst_port = ports->dest; 1987d208687SFelix Fietkau tuple->l3proto = AF_INET; 1997d208687SFelix Fietkau tuple->l4proto = iph->protocol; 2007d208687SFelix Fietkau tuple->iifidx = dev->ifindex; 2017d208687SFelix Fietkau 2027d208687SFelix Fietkau return 0; 2037d208687SFelix Fietkau } 2047d208687SFelix Fietkau 2057d208687SFelix Fietkau /* Based on ip_exceeds_mtu(). */ 2067d208687SFelix Fietkau static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu) 2077d208687SFelix Fietkau { 2087d208687SFelix Fietkau if (skb->len <= mtu) 2097d208687SFelix Fietkau return false; 2107d208687SFelix Fietkau 2117d208687SFelix Fietkau if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 2127d208687SFelix Fietkau return false; 2137d208687SFelix Fietkau 2147d208687SFelix Fietkau return true; 2157d208687SFelix Fietkau } 2167d208687SFelix Fietkau 217589b474aSFlorian Westphal static int nf_flow_offload_dst_check(struct dst_entry *dst) 218589b474aSFlorian Westphal { 219589b474aSFlorian Westphal if (unlikely(dst_xfrm(dst))) 220589b474aSFlorian Westphal return dst_check(dst, 0) ? 0 : -1; 221589b474aSFlorian Westphal 222589b474aSFlorian Westphal return 0; 223589b474aSFlorian Westphal } 224589b474aSFlorian Westphal 225589b474aSFlorian Westphal static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb, 226589b474aSFlorian Westphal const struct nf_hook_state *state, 227589b474aSFlorian Westphal struct dst_entry *dst) 228589b474aSFlorian Westphal { 229589b474aSFlorian Westphal skb_orphan(skb); 230589b474aSFlorian Westphal skb_dst_set_noref(skb, dst); 231589b474aSFlorian Westphal dst_output(state->net, state->sk, skb); 232589b474aSFlorian Westphal return NF_STOLEN; 233589b474aSFlorian Westphal } 234589b474aSFlorian Westphal 2357d208687SFelix Fietkau unsigned int 2367d208687SFelix Fietkau nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, 2377d208687SFelix Fietkau const struct nf_hook_state *state) 2387d208687SFelix Fietkau { 2397d208687SFelix Fietkau struct flow_offload_tuple_rhash *tuplehash; 2407d208687SFelix Fietkau struct nf_flowtable *flow_table = priv; 2417d208687SFelix Fietkau struct flow_offload_tuple tuple = {}; 2427d208687SFelix Fietkau enum flow_offload_tuple_dir dir; 2437d208687SFelix Fietkau struct flow_offload *flow; 2447d208687SFelix Fietkau struct net_device *outdev; 2452a79fd39SJason A. Donenfeld struct rtable *rt; 246b6f27d32SFelix Fietkau unsigned int thoff; 2477d208687SFelix Fietkau struct iphdr *iph; 2487d208687SFelix Fietkau __be32 nexthop; 2497d208687SFelix Fietkau 2507d208687SFelix Fietkau if (skb->protocol != htons(ETH_P_IP)) 2517d208687SFelix Fietkau return NF_ACCEPT; 2527d208687SFelix Fietkau 2537d208687SFelix Fietkau if (nf_flow_tuple_ip(skb, state->in, &tuple) < 0) 2547d208687SFelix Fietkau return NF_ACCEPT; 2557d208687SFelix Fietkau 2567d208687SFelix Fietkau tuplehash = flow_offload_lookup(flow_table, &tuple); 2577d208687SFelix Fietkau if (tuplehash == NULL) 2587d208687SFelix Fietkau return NF_ACCEPT; 2597d208687SFelix Fietkau 2607d208687SFelix Fietkau dir = tuplehash->tuple.dir; 2617d208687SFelix Fietkau flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 2622a79fd39SJason A. Donenfeld rt = (struct rtable *)flow->tuplehash[dir].tuple.dst_cache; 263227e1e4dSPablo Neira Ayuso outdev = rt->dst.dev; 2647d208687SFelix Fietkau 265e75b3e1cSFlorian Westphal if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu))) 2667d208687SFelix Fietkau return NF_ACCEPT; 2677d208687SFelix Fietkau 2687d208687SFelix Fietkau if (skb_try_make_writable(skb, sizeof(*iph))) 2697d208687SFelix Fietkau return NF_DROP; 2707d208687SFelix Fietkau 271b6f27d32SFelix Fietkau thoff = ip_hdr(skb)->ihl * 4; 27233894c36SFelix Fietkau if (nf_flow_state_check(flow, ip_hdr(skb)->protocol, skb, thoff)) 273b6f27d32SFelix Fietkau return NF_ACCEPT; 274b6f27d32SFelix Fietkau 2758b3646d6SPaul Blakey flow_offload_refresh(flow_table, flow); 276f698fe40SPablo Neira Ayuso 277589b474aSFlorian Westphal if (nf_flow_offload_dst_check(&rt->dst)) { 278589b474aSFlorian Westphal flow_offload_teardown(flow); 279589b474aSFlorian Westphal return NF_ACCEPT; 280589b474aSFlorian Westphal } 281589b474aSFlorian Westphal 28228c5ed2fSTaehee Yoo if (nf_flow_nat_ip(flow, skb, thoff, dir) < 0) 2837d208687SFelix Fietkau return NF_DROP; 2847d208687SFelix Fietkau 2857d208687SFelix Fietkau iph = ip_hdr(skb); 2867d208687SFelix Fietkau ip_decrease_ttl(iph); 287de20900fSFlorian Westphal skb->tstamp = 0; 2887d208687SFelix Fietkau 289589b474aSFlorian Westphal if (unlikely(dst_xfrm(&rt->dst))) { 290589b474aSFlorian Westphal memset(skb->cb, 0, sizeof(struct inet_skb_parm)); 291589b474aSFlorian Westphal IPCB(skb)->iif = skb->dev->ifindex; 292589b474aSFlorian Westphal IPCB(skb)->flags = IPSKB_FORWARDED; 293589b474aSFlorian Westphal return nf_flow_xmit_xfrm(skb, state, &rt->dst); 294589b474aSFlorian Westphal } 295589b474aSFlorian Westphal 2967d208687SFelix Fietkau skb->dev = outdev; 2977d208687SFelix Fietkau nexthop = rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr); 2982a79fd39SJason A. Donenfeld skb_dst_set_noref(skb, &rt->dst); 2997d208687SFelix Fietkau neigh_xmit(NEIGH_ARP_TABLE, outdev, &nexthop, skb); 3007d208687SFelix Fietkau 3017d208687SFelix Fietkau return NF_STOLEN; 3027d208687SFelix Fietkau } 3037d208687SFelix Fietkau EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook); 304a908fdecSFelix Fietkau 305a908fdecSFelix Fietkau static int nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff, 306a908fdecSFelix Fietkau struct in6_addr *addr, 307a908fdecSFelix Fietkau struct in6_addr *new_addr) 308a908fdecSFelix Fietkau { 309a908fdecSFelix Fietkau struct tcphdr *tcph; 310a908fdecSFelix Fietkau 311a908fdecSFelix Fietkau if (!pskb_may_pull(skb, thoff + sizeof(*tcph)) || 312a908fdecSFelix Fietkau skb_try_make_writable(skb, thoff + sizeof(*tcph))) 313a908fdecSFelix Fietkau return -1; 314a908fdecSFelix Fietkau 315a908fdecSFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 316a908fdecSFelix Fietkau inet_proto_csum_replace16(&tcph->check, skb, addr->s6_addr32, 317a908fdecSFelix Fietkau new_addr->s6_addr32, true); 318a908fdecSFelix Fietkau 319a908fdecSFelix Fietkau return 0; 320a908fdecSFelix Fietkau } 321a908fdecSFelix Fietkau 322a908fdecSFelix Fietkau static int nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff, 323a908fdecSFelix Fietkau struct in6_addr *addr, 324a908fdecSFelix Fietkau struct in6_addr *new_addr) 325a908fdecSFelix Fietkau { 326a908fdecSFelix Fietkau struct udphdr *udph; 327a908fdecSFelix Fietkau 328a908fdecSFelix Fietkau if (!pskb_may_pull(skb, thoff + sizeof(*udph)) || 329a908fdecSFelix Fietkau skb_try_make_writable(skb, thoff + sizeof(*udph))) 330a908fdecSFelix Fietkau return -1; 331a908fdecSFelix Fietkau 332a908fdecSFelix Fietkau udph = (void *)(skb_network_header(skb) + thoff); 333a908fdecSFelix Fietkau if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 334a908fdecSFelix Fietkau inet_proto_csum_replace16(&udph->check, skb, addr->s6_addr32, 335a908fdecSFelix Fietkau new_addr->s6_addr32, true); 336a908fdecSFelix Fietkau if (!udph->check) 337a908fdecSFelix Fietkau udph->check = CSUM_MANGLED_0; 338a908fdecSFelix Fietkau } 339a908fdecSFelix Fietkau 340a908fdecSFelix Fietkau return 0; 341a908fdecSFelix Fietkau } 342a908fdecSFelix Fietkau 343a908fdecSFelix Fietkau static int nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h, 344a908fdecSFelix Fietkau unsigned int thoff, struct in6_addr *addr, 345a908fdecSFelix Fietkau struct in6_addr *new_addr) 346a908fdecSFelix Fietkau { 347a908fdecSFelix Fietkau switch (ip6h->nexthdr) { 348a908fdecSFelix Fietkau case IPPROTO_TCP: 349a908fdecSFelix Fietkau if (nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr) < 0) 350a908fdecSFelix Fietkau return NF_DROP; 351a908fdecSFelix Fietkau break; 352a908fdecSFelix Fietkau case IPPROTO_UDP: 353a908fdecSFelix Fietkau if (nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr) < 0) 354a908fdecSFelix Fietkau return NF_DROP; 355a908fdecSFelix Fietkau break; 356a908fdecSFelix Fietkau } 357a908fdecSFelix Fietkau 358a908fdecSFelix Fietkau return 0; 359a908fdecSFelix Fietkau } 360a908fdecSFelix Fietkau 361a908fdecSFelix Fietkau static int nf_flow_snat_ipv6(const struct flow_offload *flow, 362a908fdecSFelix Fietkau struct sk_buff *skb, struct ipv6hdr *ip6h, 363a908fdecSFelix Fietkau unsigned int thoff, 364a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 365a908fdecSFelix Fietkau { 366a908fdecSFelix Fietkau struct in6_addr addr, new_addr; 367a908fdecSFelix Fietkau 368a908fdecSFelix Fietkau switch (dir) { 369a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 370a908fdecSFelix Fietkau addr = ip6h->saddr; 371a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6; 372a908fdecSFelix Fietkau ip6h->saddr = new_addr; 373a908fdecSFelix Fietkau break; 374a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 375a908fdecSFelix Fietkau addr = ip6h->daddr; 376a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6; 377a908fdecSFelix Fietkau ip6h->daddr = new_addr; 378a908fdecSFelix Fietkau break; 379a908fdecSFelix Fietkau default: 380a908fdecSFelix Fietkau return -1; 381a908fdecSFelix Fietkau } 382a908fdecSFelix Fietkau 383a908fdecSFelix Fietkau return nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr); 384a908fdecSFelix Fietkau } 385a908fdecSFelix Fietkau 386a908fdecSFelix Fietkau static int nf_flow_dnat_ipv6(const struct flow_offload *flow, 387a908fdecSFelix Fietkau struct sk_buff *skb, struct ipv6hdr *ip6h, 388a908fdecSFelix Fietkau unsigned int thoff, 389a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 390a908fdecSFelix Fietkau { 391a908fdecSFelix Fietkau struct in6_addr addr, new_addr; 392a908fdecSFelix Fietkau 393a908fdecSFelix Fietkau switch (dir) { 394a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 395a908fdecSFelix Fietkau addr = ip6h->daddr; 396a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6; 397a908fdecSFelix Fietkau ip6h->daddr = new_addr; 398a908fdecSFelix Fietkau break; 399a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 400a908fdecSFelix Fietkau addr = ip6h->saddr; 401a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6; 402a908fdecSFelix Fietkau ip6h->saddr = new_addr; 403a908fdecSFelix Fietkau break; 404a908fdecSFelix Fietkau default: 405a908fdecSFelix Fietkau return -1; 406a908fdecSFelix Fietkau } 407a908fdecSFelix Fietkau 408a908fdecSFelix Fietkau return nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr); 409a908fdecSFelix Fietkau } 410a908fdecSFelix Fietkau 411a908fdecSFelix Fietkau static int nf_flow_nat_ipv6(const struct flow_offload *flow, 412a908fdecSFelix Fietkau struct sk_buff *skb, 413a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 414a908fdecSFelix Fietkau { 415a908fdecSFelix Fietkau struct ipv6hdr *ip6h = ipv6_hdr(skb); 416a908fdecSFelix Fietkau unsigned int thoff = sizeof(*ip6h); 417a908fdecSFelix Fietkau 418355a8b13SPablo Neira Ayuso if (test_bit(NF_FLOW_SNAT, &flow->flags) && 419a908fdecSFelix Fietkau (nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir) < 0 || 420a908fdecSFelix Fietkau nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir) < 0)) 421a908fdecSFelix Fietkau return -1; 422355a8b13SPablo Neira Ayuso if (test_bit(NF_FLOW_DNAT, &flow->flags) && 423a908fdecSFelix Fietkau (nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir) < 0 || 424a908fdecSFelix Fietkau nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir) < 0)) 425a908fdecSFelix Fietkau return -1; 426a908fdecSFelix Fietkau 427a908fdecSFelix Fietkau return 0; 428a908fdecSFelix Fietkau } 429a908fdecSFelix Fietkau 430a908fdecSFelix Fietkau static int nf_flow_tuple_ipv6(struct sk_buff *skb, const struct net_device *dev, 431a908fdecSFelix Fietkau struct flow_offload_tuple *tuple) 432a908fdecSFelix Fietkau { 433a908fdecSFelix Fietkau struct flow_ports *ports; 434a908fdecSFelix Fietkau struct ipv6hdr *ip6h; 435a908fdecSFelix Fietkau unsigned int thoff; 436a908fdecSFelix Fietkau 437a908fdecSFelix Fietkau if (!pskb_may_pull(skb, sizeof(*ip6h))) 438a908fdecSFelix Fietkau return -1; 439a908fdecSFelix Fietkau 440a908fdecSFelix Fietkau ip6h = ipv6_hdr(skb); 441a908fdecSFelix Fietkau 442a908fdecSFelix Fietkau if (ip6h->nexthdr != IPPROTO_TCP && 443a908fdecSFelix Fietkau ip6h->nexthdr != IPPROTO_UDP) 444a908fdecSFelix Fietkau return -1; 445a908fdecSFelix Fietkau 44633cc3c0cSTaehee Yoo if (ip6h->hop_limit <= 1) 44733cc3c0cSTaehee Yoo return -1; 44833cc3c0cSTaehee Yoo 449a908fdecSFelix Fietkau thoff = sizeof(*ip6h); 450a908fdecSFelix Fietkau if (!pskb_may_pull(skb, thoff + sizeof(*ports))) 451a908fdecSFelix Fietkau return -1; 452a908fdecSFelix Fietkau 453a908fdecSFelix Fietkau ports = (struct flow_ports *)(skb_network_header(skb) + thoff); 454a908fdecSFelix Fietkau 455a908fdecSFelix Fietkau tuple->src_v6 = ip6h->saddr; 456a908fdecSFelix Fietkau tuple->dst_v6 = ip6h->daddr; 457a908fdecSFelix Fietkau tuple->src_port = ports->source; 458a908fdecSFelix Fietkau tuple->dst_port = ports->dest; 459a908fdecSFelix Fietkau tuple->l3proto = AF_INET6; 460a908fdecSFelix Fietkau tuple->l4proto = ip6h->nexthdr; 461a908fdecSFelix Fietkau tuple->iifidx = dev->ifindex; 462a908fdecSFelix Fietkau 463a908fdecSFelix Fietkau return 0; 464a908fdecSFelix Fietkau } 465a908fdecSFelix Fietkau 466a908fdecSFelix Fietkau unsigned int 467a908fdecSFelix Fietkau nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, 468a908fdecSFelix Fietkau const struct nf_hook_state *state) 469a908fdecSFelix Fietkau { 470a908fdecSFelix Fietkau struct flow_offload_tuple_rhash *tuplehash; 471a908fdecSFelix Fietkau struct nf_flowtable *flow_table = priv; 472a908fdecSFelix Fietkau struct flow_offload_tuple tuple = {}; 473a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir; 4749b1c1ef1SNicolas Dichtel const struct in6_addr *nexthop; 475a908fdecSFelix Fietkau struct flow_offload *flow; 476a908fdecSFelix Fietkau struct net_device *outdev; 477a908fdecSFelix Fietkau struct ipv6hdr *ip6h; 478a908fdecSFelix Fietkau struct rt6_info *rt; 479a908fdecSFelix Fietkau 480a908fdecSFelix Fietkau if (skb->protocol != htons(ETH_P_IPV6)) 481a908fdecSFelix Fietkau return NF_ACCEPT; 482a908fdecSFelix Fietkau 483a908fdecSFelix Fietkau if (nf_flow_tuple_ipv6(skb, state->in, &tuple) < 0) 484a908fdecSFelix Fietkau return NF_ACCEPT; 485a908fdecSFelix Fietkau 486a908fdecSFelix Fietkau tuplehash = flow_offload_lookup(flow_table, &tuple); 487a908fdecSFelix Fietkau if (tuplehash == NULL) 488a908fdecSFelix Fietkau return NF_ACCEPT; 489a908fdecSFelix Fietkau 490a908fdecSFelix Fietkau dir = tuplehash->tuple.dir; 491a908fdecSFelix Fietkau flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 492a908fdecSFelix Fietkau rt = (struct rt6_info *)flow->tuplehash[dir].tuple.dst_cache; 493227e1e4dSPablo Neira Ayuso outdev = rt->dst.dev; 494a908fdecSFelix Fietkau 495a908fdecSFelix Fietkau if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu))) 496a908fdecSFelix Fietkau return NF_ACCEPT; 497a908fdecSFelix Fietkau 49833894c36SFelix Fietkau if (nf_flow_state_check(flow, ipv6_hdr(skb)->nexthdr, skb, 49933894c36SFelix Fietkau sizeof(*ip6h))) 500b6f27d32SFelix Fietkau return NF_ACCEPT; 501b6f27d32SFelix Fietkau 5028b3646d6SPaul Blakey flow_offload_refresh(flow_table, flow); 503f698fe40SPablo Neira Ayuso 504589b474aSFlorian Westphal if (nf_flow_offload_dst_check(&rt->dst)) { 505589b474aSFlorian Westphal flow_offload_teardown(flow); 506589b474aSFlorian Westphal return NF_ACCEPT; 507589b474aSFlorian Westphal } 508589b474aSFlorian Westphal 509a908fdecSFelix Fietkau if (skb_try_make_writable(skb, sizeof(*ip6h))) 510a908fdecSFelix Fietkau return NF_DROP; 511a908fdecSFelix Fietkau 51228c5ed2fSTaehee Yoo if (nf_flow_nat_ipv6(flow, skb, dir) < 0) 513a908fdecSFelix Fietkau return NF_DROP; 514a908fdecSFelix Fietkau 515a908fdecSFelix Fietkau ip6h = ipv6_hdr(skb); 516a908fdecSFelix Fietkau ip6h->hop_limit--; 517de20900fSFlorian Westphal skb->tstamp = 0; 518a908fdecSFelix Fietkau 519589b474aSFlorian Westphal if (unlikely(dst_xfrm(&rt->dst))) { 520589b474aSFlorian Westphal memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); 521589b474aSFlorian Westphal IP6CB(skb)->iif = skb->dev->ifindex; 522589b474aSFlorian Westphal IP6CB(skb)->flags = IP6SKB_FORWARDED; 523589b474aSFlorian Westphal return nf_flow_xmit_xfrm(skb, state, &rt->dst); 524589b474aSFlorian Westphal } 525589b474aSFlorian Westphal 526a908fdecSFelix Fietkau skb->dev = outdev; 527a908fdecSFelix Fietkau nexthop = rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6); 5282a79fd39SJason A. Donenfeld skb_dst_set_noref(skb, &rt->dst); 529a908fdecSFelix Fietkau neigh_xmit(NEIGH_ND_TABLE, outdev, nexthop, skb); 530a908fdecSFelix Fietkau 531a908fdecSFelix Fietkau return NF_STOLEN; 532a908fdecSFelix Fietkau } 533a908fdecSFelix Fietkau EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook); 534