1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 27d208687SFelix Fietkau #include <linux/kernel.h> 37d208687SFelix Fietkau #include <linux/init.h> 47d208687SFelix Fietkau #include <linux/module.h> 57d208687SFelix Fietkau #include <linux/netfilter.h> 67d208687SFelix Fietkau #include <linux/rhashtable.h> 77d208687SFelix Fietkau #include <linux/ip.h> 8a908fdecSFelix Fietkau #include <linux/ipv6.h> 97d208687SFelix Fietkau #include <linux/netdevice.h> 107d208687SFelix Fietkau #include <net/ip.h> 11a908fdecSFelix Fietkau #include <net/ipv6.h> 12a908fdecSFelix Fietkau #include <net/ip6_route.h> 137d208687SFelix Fietkau #include <net/neighbour.h> 147d208687SFelix Fietkau #include <net/netfilter/nf_flow_table.h> 1553c2b289SPablo Neira Ayuso #include <net/netfilter/nf_conntrack_acct.h> 167d208687SFelix Fietkau /* For layer 4 checksum field offset. */ 177d208687SFelix Fietkau #include <linux/tcp.h> 187d208687SFelix Fietkau #include <linux/udp.h> 197d208687SFelix Fietkau 2033894c36SFelix Fietkau static int nf_flow_state_check(struct flow_offload *flow, int proto, 21b6f27d32SFelix Fietkau struct sk_buff *skb, unsigned int thoff) 22b6f27d32SFelix Fietkau { 23b6f27d32SFelix Fietkau struct tcphdr *tcph; 24b6f27d32SFelix Fietkau 2533894c36SFelix Fietkau if (proto != IPPROTO_TCP) 2633894c36SFelix Fietkau return 0; 2733894c36SFelix Fietkau 28b6f27d32SFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 29b6f27d32SFelix Fietkau if (unlikely(tcph->fin || tcph->rst)) { 30b6f27d32SFelix Fietkau flow_offload_teardown(flow); 31b6f27d32SFelix Fietkau return -1; 32b6f27d32SFelix Fietkau } 33b6f27d32SFelix Fietkau 34b6f27d32SFelix Fietkau return 0; 35b6f27d32SFelix Fietkau } 36b6f27d32SFelix Fietkau 37f4401262SPablo Neira Ayuso static void nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff, 387d208687SFelix Fietkau __be32 addr, __be32 new_addr) 397d208687SFelix Fietkau { 407d208687SFelix Fietkau struct tcphdr *tcph; 417d208687SFelix Fietkau 427d208687SFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 437d208687SFelix Fietkau inet_proto_csum_replace4(&tcph->check, skb, addr, new_addr, true); 447d208687SFelix Fietkau } 457d208687SFelix Fietkau 46f4401262SPablo Neira Ayuso static void nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff, 477d208687SFelix Fietkau __be32 addr, __be32 new_addr) 487d208687SFelix Fietkau { 497d208687SFelix Fietkau struct udphdr *udph; 507d208687SFelix Fietkau 517d208687SFelix Fietkau udph = (void *)(skb_network_header(skb) + thoff); 527d208687SFelix Fietkau if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 537d208687SFelix Fietkau inet_proto_csum_replace4(&udph->check, skb, addr, 547d208687SFelix Fietkau new_addr, true); 557d208687SFelix Fietkau if (!udph->check) 567d208687SFelix Fietkau udph->check = CSUM_MANGLED_0; 577d208687SFelix Fietkau } 587d208687SFelix Fietkau } 597d208687SFelix Fietkau 60f4401262SPablo Neira Ayuso static void nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph, 617d208687SFelix Fietkau unsigned int thoff, __be32 addr, 627d208687SFelix Fietkau __be32 new_addr) 637d208687SFelix Fietkau { 647d208687SFelix Fietkau switch (iph->protocol) { 657d208687SFelix Fietkau case IPPROTO_TCP: 66f4401262SPablo Neira Ayuso nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr); 677d208687SFelix Fietkau break; 687d208687SFelix Fietkau case IPPROTO_UDP: 69f4401262SPablo Neira Ayuso nf_flow_nat_ip_udp(skb, thoff, addr, new_addr); 707d208687SFelix Fietkau break; 717d208687SFelix Fietkau } 727d208687SFelix Fietkau } 737d208687SFelix Fietkau 74f4401262SPablo Neira Ayuso static void nf_flow_snat_ip(const struct flow_offload *flow, 75f4401262SPablo Neira Ayuso struct sk_buff *skb, struct iphdr *iph, 76f4401262SPablo Neira Ayuso unsigned int thoff, enum flow_offload_tuple_dir dir) 777d208687SFelix Fietkau { 787d208687SFelix Fietkau __be32 addr, new_addr; 797d208687SFelix Fietkau 807d208687SFelix Fietkau switch (dir) { 817d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 827d208687SFelix Fietkau addr = iph->saddr; 837d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; 847d208687SFelix Fietkau iph->saddr = new_addr; 857d208687SFelix Fietkau break; 867d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 877d208687SFelix Fietkau addr = iph->daddr; 887d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr; 897d208687SFelix Fietkau iph->daddr = new_addr; 907d208687SFelix Fietkau break; 917d208687SFelix Fietkau } 927d208687SFelix Fietkau csum_replace4(&iph->check, addr, new_addr); 937d208687SFelix Fietkau 94f4401262SPablo Neira Ayuso nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); 957d208687SFelix Fietkau } 967d208687SFelix Fietkau 97f4401262SPablo Neira Ayuso static void nf_flow_dnat_ip(const struct flow_offload *flow, 98f4401262SPablo Neira Ayuso struct sk_buff *skb, struct iphdr *iph, 99f4401262SPablo Neira Ayuso unsigned int thoff, enum flow_offload_tuple_dir dir) 1007d208687SFelix Fietkau { 1017d208687SFelix Fietkau __be32 addr, new_addr; 1027d208687SFelix Fietkau 1037d208687SFelix Fietkau switch (dir) { 1047d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 1057d208687SFelix Fietkau addr = iph->daddr; 1067d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; 1077d208687SFelix Fietkau iph->daddr = new_addr; 1087d208687SFelix Fietkau break; 1097d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 1107d208687SFelix Fietkau addr = iph->saddr; 1117d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr; 1127d208687SFelix Fietkau iph->saddr = new_addr; 1137d208687SFelix Fietkau break; 1147d208687SFelix Fietkau } 1157d208687SFelix Fietkau csum_replace4(&iph->check, addr, new_addr); 1167d208687SFelix Fietkau 117f4401262SPablo Neira Ayuso nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); 1187d208687SFelix Fietkau } 1197d208687SFelix Fietkau 120f4401262SPablo Neira Ayuso static void nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb, 1212fc11745SPablo Neira Ayuso unsigned int thoff, enum flow_offload_tuple_dir dir, 1222fc11745SPablo Neira Ayuso struct iphdr *iph) 1237d208687SFelix Fietkau { 124f4401262SPablo Neira Ayuso if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 125f4401262SPablo Neira Ayuso nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir); 126f4401262SPablo Neira Ayuso nf_flow_snat_ip(flow, skb, iph, thoff, dir); 127f4401262SPablo Neira Ayuso } 128f4401262SPablo Neira Ayuso if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 129f4401262SPablo Neira Ayuso nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir); 130f4401262SPablo Neira Ayuso nf_flow_dnat_ip(flow, skb, iph, thoff, dir); 131f4401262SPablo Neira Ayuso } 1327d208687SFelix Fietkau } 1337d208687SFelix Fietkau 1347d208687SFelix Fietkau static bool ip_has_options(unsigned int thoff) 1357d208687SFelix Fietkau { 1367d208687SFelix Fietkau return thoff != sizeof(struct iphdr); 1377d208687SFelix Fietkau } 1387d208687SFelix Fietkau 1397d208687SFelix Fietkau static int nf_flow_tuple_ip(struct sk_buff *skb, const struct net_device *dev, 1402fc11745SPablo Neira Ayuso struct flow_offload_tuple *tuple, u32 *hdrsize) 1417d208687SFelix Fietkau { 1427d208687SFelix Fietkau struct flow_ports *ports; 1432fc11745SPablo Neira Ayuso unsigned int thoff; 1447d208687SFelix Fietkau struct iphdr *iph; 1457d208687SFelix Fietkau 1467d208687SFelix Fietkau if (!pskb_may_pull(skb, sizeof(*iph))) 1477d208687SFelix Fietkau return -1; 1487d208687SFelix Fietkau 1497d208687SFelix Fietkau iph = ip_hdr(skb); 1507d208687SFelix Fietkau thoff = iph->ihl * 4; 1517d208687SFelix Fietkau 1527d208687SFelix Fietkau if (ip_is_fragment(iph) || 1537d208687SFelix Fietkau unlikely(ip_has_options(thoff))) 1547d208687SFelix Fietkau return -1; 1557d208687SFelix Fietkau 156793d5d61SPablo Neira Ayuso switch (iph->protocol) { 157793d5d61SPablo Neira Ayuso case IPPROTO_TCP: 1582fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct tcphdr); 159793d5d61SPablo Neira Ayuso break; 160793d5d61SPablo Neira Ayuso case IPPROTO_UDP: 1612fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct udphdr); 162793d5d61SPablo Neira Ayuso break; 163793d5d61SPablo Neira Ayuso default: 1647d208687SFelix Fietkau return -1; 165793d5d61SPablo Neira Ayuso } 1667d208687SFelix Fietkau 16733cc3c0cSTaehee Yoo if (iph->ttl <= 1) 16833cc3c0cSTaehee Yoo return -1; 16933cc3c0cSTaehee Yoo 1707d208687SFelix Fietkau thoff = iph->ihl * 4; 1712fc11745SPablo Neira Ayuso if (!pskb_may_pull(skb, thoff + *hdrsize)) 1727d208687SFelix Fietkau return -1; 1737d208687SFelix Fietkau 17441e9ec5aSHaishuang Yan iph = ip_hdr(skb); 1757d208687SFelix Fietkau ports = (struct flow_ports *)(skb_network_header(skb) + thoff); 1767d208687SFelix Fietkau 1777d208687SFelix Fietkau tuple->src_v4.s_addr = iph->saddr; 1787d208687SFelix Fietkau tuple->dst_v4.s_addr = iph->daddr; 1797d208687SFelix Fietkau tuple->src_port = ports->source; 1807d208687SFelix Fietkau tuple->dst_port = ports->dest; 1817d208687SFelix Fietkau tuple->l3proto = AF_INET; 1827d208687SFelix Fietkau tuple->l4proto = iph->protocol; 1837d208687SFelix Fietkau tuple->iifidx = dev->ifindex; 1847d208687SFelix Fietkau 1857d208687SFelix Fietkau return 0; 1867d208687SFelix Fietkau } 1877d208687SFelix Fietkau 1887d208687SFelix Fietkau /* Based on ip_exceeds_mtu(). */ 1897d208687SFelix Fietkau static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu) 1907d208687SFelix Fietkau { 1917d208687SFelix Fietkau if (skb->len <= mtu) 1927d208687SFelix Fietkau return false; 1937d208687SFelix Fietkau 1947d208687SFelix Fietkau if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 1957d208687SFelix Fietkau return false; 1967d208687SFelix Fietkau 1977d208687SFelix Fietkau return true; 1987d208687SFelix Fietkau } 1997d208687SFelix Fietkau 200589b474aSFlorian Westphal static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb, 201589b474aSFlorian Westphal const struct nf_hook_state *state, 202589b474aSFlorian Westphal struct dst_entry *dst) 203589b474aSFlorian Westphal { 204589b474aSFlorian Westphal skb_orphan(skb); 205589b474aSFlorian Westphal skb_dst_set_noref(skb, dst); 206589b474aSFlorian Westphal dst_output(state->net, state->sk, skb); 207589b474aSFlorian Westphal return NF_STOLEN; 208589b474aSFlorian Westphal } 209589b474aSFlorian Westphal 2107d208687SFelix Fietkau unsigned int 2117d208687SFelix Fietkau nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, 2127d208687SFelix Fietkau const struct nf_hook_state *state) 2137d208687SFelix Fietkau { 2147d208687SFelix Fietkau struct flow_offload_tuple_rhash *tuplehash; 2157d208687SFelix Fietkau struct nf_flowtable *flow_table = priv; 2167d208687SFelix Fietkau struct flow_offload_tuple tuple = {}; 2177d208687SFelix Fietkau enum flow_offload_tuple_dir dir; 2187d208687SFelix Fietkau struct flow_offload *flow; 2197d208687SFelix Fietkau struct net_device *outdev; 2202a79fd39SJason A. Donenfeld struct rtable *rt; 221b6f27d32SFelix Fietkau unsigned int thoff; 2227d208687SFelix Fietkau struct iphdr *iph; 2237d208687SFelix Fietkau __be32 nexthop; 2242fc11745SPablo Neira Ayuso u32 hdrsize; 2257d208687SFelix Fietkau 2267d208687SFelix Fietkau if (skb->protocol != htons(ETH_P_IP)) 2277d208687SFelix Fietkau return NF_ACCEPT; 2287d208687SFelix Fietkau 2292fc11745SPablo Neira Ayuso if (nf_flow_tuple_ip(skb, state->in, &tuple, &hdrsize) < 0) 2307d208687SFelix Fietkau return NF_ACCEPT; 2317d208687SFelix Fietkau 2327d208687SFelix Fietkau tuplehash = flow_offload_lookup(flow_table, &tuple); 2337d208687SFelix Fietkau if (tuplehash == NULL) 2347d208687SFelix Fietkau return NF_ACCEPT; 2357d208687SFelix Fietkau 2367d208687SFelix Fietkau dir = tuplehash->tuple.dir; 2377d208687SFelix Fietkau flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 2382a79fd39SJason A. Donenfeld rt = (struct rtable *)flow->tuplehash[dir].tuple.dst_cache; 239227e1e4dSPablo Neira Ayuso outdev = rt->dst.dev; 2407d208687SFelix Fietkau 241e75b3e1cSFlorian Westphal if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu))) 2427d208687SFelix Fietkau return NF_ACCEPT; 2437d208687SFelix Fietkau 2442fc11745SPablo Neira Ayuso iph = ip_hdr(skb); 2452fc11745SPablo Neira Ayuso thoff = iph->ihl * 4; 2462fc11745SPablo Neira Ayuso if (nf_flow_state_check(flow, iph->protocol, skb, thoff)) 247b6f27d32SFelix Fietkau return NF_ACCEPT; 248b6f27d32SFelix Fietkau 2498b3646d6SPaul Blakey flow_offload_refresh(flow_table, flow); 250f698fe40SPablo Neira Ayuso 251*e5075c0bSPablo Neira Ayuso if (!dst_check(&rt->dst, 0)) { 252589b474aSFlorian Westphal flow_offload_teardown(flow); 253589b474aSFlorian Westphal return NF_ACCEPT; 254589b474aSFlorian Westphal } 255589b474aSFlorian Westphal 2562babb46cSPablo Neira Ayuso if (skb_try_make_writable(skb, thoff + hdrsize)) 2572babb46cSPablo Neira Ayuso return NF_DROP; 2582babb46cSPablo Neira Ayuso 2592babb46cSPablo Neira Ayuso iph = ip_hdr(skb); 260f4401262SPablo Neira Ayuso nf_flow_nat_ip(flow, skb, thoff, dir, iph); 2617d208687SFelix Fietkau 2627d208687SFelix Fietkau ip_decrease_ttl(iph); 263de20900fSFlorian Westphal skb->tstamp = 0; 2647d208687SFelix Fietkau 26553c2b289SPablo Neira Ayuso if (flow_table->flags & NF_FLOWTABLE_COUNTER) 26653c2b289SPablo Neira Ayuso nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len); 26753c2b289SPablo Neira Ayuso 268589b474aSFlorian Westphal if (unlikely(dst_xfrm(&rt->dst))) { 269589b474aSFlorian Westphal memset(skb->cb, 0, sizeof(struct inet_skb_parm)); 270589b474aSFlorian Westphal IPCB(skb)->iif = skb->dev->ifindex; 271589b474aSFlorian Westphal IPCB(skb)->flags = IPSKB_FORWARDED; 272589b474aSFlorian Westphal return nf_flow_xmit_xfrm(skb, state, &rt->dst); 273589b474aSFlorian Westphal } 274589b474aSFlorian Westphal 2757d208687SFelix Fietkau skb->dev = outdev; 2767d208687SFelix Fietkau nexthop = rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr); 2772a79fd39SJason A. Donenfeld skb_dst_set_noref(skb, &rt->dst); 2787d208687SFelix Fietkau neigh_xmit(NEIGH_ARP_TABLE, outdev, &nexthop, skb); 2797d208687SFelix Fietkau 2807d208687SFelix Fietkau return NF_STOLEN; 2817d208687SFelix Fietkau } 2827d208687SFelix Fietkau EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook); 283a908fdecSFelix Fietkau 284f4401262SPablo Neira Ayuso static void nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff, 285a908fdecSFelix Fietkau struct in6_addr *addr, 286f4401262SPablo Neira Ayuso struct in6_addr *new_addr, 287f4401262SPablo Neira Ayuso struct ipv6hdr *ip6h) 288a908fdecSFelix Fietkau { 289a908fdecSFelix Fietkau struct tcphdr *tcph; 290a908fdecSFelix Fietkau 291a908fdecSFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 292a908fdecSFelix Fietkau inet_proto_csum_replace16(&tcph->check, skb, addr->s6_addr32, 293a908fdecSFelix Fietkau new_addr->s6_addr32, true); 294a908fdecSFelix Fietkau } 295a908fdecSFelix Fietkau 296f4401262SPablo Neira Ayuso static void nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff, 297a908fdecSFelix Fietkau struct in6_addr *addr, 298a908fdecSFelix Fietkau struct in6_addr *new_addr) 299a908fdecSFelix Fietkau { 300a908fdecSFelix Fietkau struct udphdr *udph; 301a908fdecSFelix Fietkau 302a908fdecSFelix Fietkau udph = (void *)(skb_network_header(skb) + thoff); 303a908fdecSFelix Fietkau if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 304a908fdecSFelix Fietkau inet_proto_csum_replace16(&udph->check, skb, addr->s6_addr32, 305a908fdecSFelix Fietkau new_addr->s6_addr32, true); 306a908fdecSFelix Fietkau if (!udph->check) 307a908fdecSFelix Fietkau udph->check = CSUM_MANGLED_0; 308a908fdecSFelix Fietkau } 309a908fdecSFelix Fietkau } 310a908fdecSFelix Fietkau 311f4401262SPablo Neira Ayuso static void nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h, 312a908fdecSFelix Fietkau unsigned int thoff, struct in6_addr *addr, 313a908fdecSFelix Fietkau struct in6_addr *new_addr) 314a908fdecSFelix Fietkau { 315a908fdecSFelix Fietkau switch (ip6h->nexthdr) { 316a908fdecSFelix Fietkau case IPPROTO_TCP: 317f4401262SPablo Neira Ayuso nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr, ip6h); 318a908fdecSFelix Fietkau break; 319a908fdecSFelix Fietkau case IPPROTO_UDP: 320f4401262SPablo Neira Ayuso nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr); 321a908fdecSFelix Fietkau break; 322a908fdecSFelix Fietkau } 323a908fdecSFelix Fietkau } 324a908fdecSFelix Fietkau 325f4401262SPablo Neira Ayuso static void nf_flow_snat_ipv6(const struct flow_offload *flow, 326a908fdecSFelix Fietkau struct sk_buff *skb, struct ipv6hdr *ip6h, 327a908fdecSFelix Fietkau unsigned int thoff, 328a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 329a908fdecSFelix Fietkau { 330a908fdecSFelix Fietkau struct in6_addr addr, new_addr; 331a908fdecSFelix Fietkau 332a908fdecSFelix Fietkau switch (dir) { 333a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 334a908fdecSFelix Fietkau addr = ip6h->saddr; 335a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6; 336a908fdecSFelix Fietkau ip6h->saddr = new_addr; 337a908fdecSFelix Fietkau break; 338a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 339a908fdecSFelix Fietkau addr = ip6h->daddr; 340a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6; 341a908fdecSFelix Fietkau ip6h->daddr = new_addr; 342a908fdecSFelix Fietkau break; 343a908fdecSFelix Fietkau } 344a908fdecSFelix Fietkau 345f4401262SPablo Neira Ayuso nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr); 346a908fdecSFelix Fietkau } 347a908fdecSFelix Fietkau 348f4401262SPablo Neira Ayuso static void nf_flow_dnat_ipv6(const struct flow_offload *flow, 349a908fdecSFelix Fietkau struct sk_buff *skb, struct ipv6hdr *ip6h, 350a908fdecSFelix Fietkau unsigned int thoff, 351a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 352a908fdecSFelix Fietkau { 353a908fdecSFelix Fietkau struct in6_addr addr, new_addr; 354a908fdecSFelix Fietkau 355a908fdecSFelix Fietkau switch (dir) { 356a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 357a908fdecSFelix Fietkau addr = ip6h->daddr; 358a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6; 359a908fdecSFelix Fietkau ip6h->daddr = new_addr; 360a908fdecSFelix Fietkau break; 361a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 362a908fdecSFelix Fietkau addr = ip6h->saddr; 363a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6; 364a908fdecSFelix Fietkau ip6h->saddr = new_addr; 365a908fdecSFelix Fietkau break; 366a908fdecSFelix Fietkau } 367a908fdecSFelix Fietkau 368f4401262SPablo Neira Ayuso nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr); 369a908fdecSFelix Fietkau } 370a908fdecSFelix Fietkau 371f4401262SPablo Neira Ayuso static void nf_flow_nat_ipv6(const struct flow_offload *flow, 372a908fdecSFelix Fietkau struct sk_buff *skb, 3732fc11745SPablo Neira Ayuso enum flow_offload_tuple_dir dir, 3742fc11745SPablo Neira Ayuso struct ipv6hdr *ip6h) 375a908fdecSFelix Fietkau { 376a908fdecSFelix Fietkau unsigned int thoff = sizeof(*ip6h); 377a908fdecSFelix Fietkau 378f4401262SPablo Neira Ayuso if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 379f4401262SPablo Neira Ayuso nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir); 380f4401262SPablo Neira Ayuso nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir); 381f4401262SPablo Neira Ayuso } 382f4401262SPablo Neira Ayuso if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 383f4401262SPablo Neira Ayuso nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir); 384f4401262SPablo Neira Ayuso nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir); 385f4401262SPablo Neira Ayuso } 386a908fdecSFelix Fietkau } 387a908fdecSFelix Fietkau 388a908fdecSFelix Fietkau static int nf_flow_tuple_ipv6(struct sk_buff *skb, const struct net_device *dev, 3892fc11745SPablo Neira Ayuso struct flow_offload_tuple *tuple, u32 *hdrsize) 390a908fdecSFelix Fietkau { 391a908fdecSFelix Fietkau struct flow_ports *ports; 392a908fdecSFelix Fietkau struct ipv6hdr *ip6h; 3932fc11745SPablo Neira Ayuso unsigned int thoff; 394a908fdecSFelix Fietkau 395a908fdecSFelix Fietkau if (!pskb_may_pull(skb, sizeof(*ip6h))) 396a908fdecSFelix Fietkau return -1; 397a908fdecSFelix Fietkau 398a908fdecSFelix Fietkau ip6h = ipv6_hdr(skb); 399a908fdecSFelix Fietkau 400793d5d61SPablo Neira Ayuso switch (ip6h->nexthdr) { 401793d5d61SPablo Neira Ayuso case IPPROTO_TCP: 4022fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct tcphdr); 403793d5d61SPablo Neira Ayuso break; 404793d5d61SPablo Neira Ayuso case IPPROTO_UDP: 4052fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct udphdr); 406793d5d61SPablo Neira Ayuso break; 407793d5d61SPablo Neira Ayuso default: 408a908fdecSFelix Fietkau return -1; 409793d5d61SPablo Neira Ayuso } 410a908fdecSFelix Fietkau 41133cc3c0cSTaehee Yoo if (ip6h->hop_limit <= 1) 41233cc3c0cSTaehee Yoo return -1; 41333cc3c0cSTaehee Yoo 414a908fdecSFelix Fietkau thoff = sizeof(*ip6h); 4152fc11745SPablo Neira Ayuso if (!pskb_may_pull(skb, thoff + *hdrsize)) 416a908fdecSFelix Fietkau return -1; 417a908fdecSFelix Fietkau 41841e9ec5aSHaishuang Yan ip6h = ipv6_hdr(skb); 419a908fdecSFelix Fietkau ports = (struct flow_ports *)(skb_network_header(skb) + thoff); 420a908fdecSFelix Fietkau 421a908fdecSFelix Fietkau tuple->src_v6 = ip6h->saddr; 422a908fdecSFelix Fietkau tuple->dst_v6 = ip6h->daddr; 423a908fdecSFelix Fietkau tuple->src_port = ports->source; 424a908fdecSFelix Fietkau tuple->dst_port = ports->dest; 425a908fdecSFelix Fietkau tuple->l3proto = AF_INET6; 426a908fdecSFelix Fietkau tuple->l4proto = ip6h->nexthdr; 427a908fdecSFelix Fietkau tuple->iifidx = dev->ifindex; 428a908fdecSFelix Fietkau 429a908fdecSFelix Fietkau return 0; 430a908fdecSFelix Fietkau } 431a908fdecSFelix Fietkau 432a908fdecSFelix Fietkau unsigned int 433a908fdecSFelix Fietkau nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, 434a908fdecSFelix Fietkau const struct nf_hook_state *state) 435a908fdecSFelix Fietkau { 436a908fdecSFelix Fietkau struct flow_offload_tuple_rhash *tuplehash; 437a908fdecSFelix Fietkau struct nf_flowtable *flow_table = priv; 438a908fdecSFelix Fietkau struct flow_offload_tuple tuple = {}; 439a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir; 4409b1c1ef1SNicolas Dichtel const struct in6_addr *nexthop; 441a908fdecSFelix Fietkau struct flow_offload *flow; 442a908fdecSFelix Fietkau struct net_device *outdev; 443a908fdecSFelix Fietkau struct ipv6hdr *ip6h; 444a908fdecSFelix Fietkau struct rt6_info *rt; 4452fc11745SPablo Neira Ayuso u32 hdrsize; 446a908fdecSFelix Fietkau 447a908fdecSFelix Fietkau if (skb->protocol != htons(ETH_P_IPV6)) 448a908fdecSFelix Fietkau return NF_ACCEPT; 449a908fdecSFelix Fietkau 4502fc11745SPablo Neira Ayuso if (nf_flow_tuple_ipv6(skb, state->in, &tuple, &hdrsize) < 0) 451a908fdecSFelix Fietkau return NF_ACCEPT; 452a908fdecSFelix Fietkau 453a908fdecSFelix Fietkau tuplehash = flow_offload_lookup(flow_table, &tuple); 454a908fdecSFelix Fietkau if (tuplehash == NULL) 455a908fdecSFelix Fietkau return NF_ACCEPT; 456a908fdecSFelix Fietkau 457a908fdecSFelix Fietkau dir = tuplehash->tuple.dir; 458a908fdecSFelix Fietkau flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 459a908fdecSFelix Fietkau rt = (struct rt6_info *)flow->tuplehash[dir].tuple.dst_cache; 460227e1e4dSPablo Neira Ayuso outdev = rt->dst.dev; 461a908fdecSFelix Fietkau 462a908fdecSFelix Fietkau if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu))) 463a908fdecSFelix Fietkau return NF_ACCEPT; 464a908fdecSFelix Fietkau 46533894c36SFelix Fietkau if (nf_flow_state_check(flow, ipv6_hdr(skb)->nexthdr, skb, 46633894c36SFelix Fietkau sizeof(*ip6h))) 467b6f27d32SFelix Fietkau return NF_ACCEPT; 468b6f27d32SFelix Fietkau 4698b3646d6SPaul Blakey flow_offload_refresh(flow_table, flow); 470f698fe40SPablo Neira Ayuso 471*e5075c0bSPablo Neira Ayuso if (!dst_check(&rt->dst, 0)) { 472589b474aSFlorian Westphal flow_offload_teardown(flow); 473589b474aSFlorian Westphal return NF_ACCEPT; 474589b474aSFlorian Westphal } 475589b474aSFlorian Westphal 4762fc11745SPablo Neira Ayuso if (skb_try_make_writable(skb, sizeof(*ip6h) + hdrsize)) 477a908fdecSFelix Fietkau return NF_DROP; 478a908fdecSFelix Fietkau 479a908fdecSFelix Fietkau ip6h = ipv6_hdr(skb); 480f4401262SPablo Neira Ayuso nf_flow_nat_ipv6(flow, skb, dir, ip6h); 4812fc11745SPablo Neira Ayuso 482a908fdecSFelix Fietkau ip6h->hop_limit--; 483de20900fSFlorian Westphal skb->tstamp = 0; 484a908fdecSFelix Fietkau 48553c2b289SPablo Neira Ayuso if (flow_table->flags & NF_FLOWTABLE_COUNTER) 48653c2b289SPablo Neira Ayuso nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len); 48753c2b289SPablo Neira Ayuso 488589b474aSFlorian Westphal if (unlikely(dst_xfrm(&rt->dst))) { 489589b474aSFlorian Westphal memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); 490589b474aSFlorian Westphal IP6CB(skb)->iif = skb->dev->ifindex; 491589b474aSFlorian Westphal IP6CB(skb)->flags = IP6SKB_FORWARDED; 492589b474aSFlorian Westphal return nf_flow_xmit_xfrm(skb, state, &rt->dst); 493589b474aSFlorian Westphal } 494589b474aSFlorian Westphal 495a908fdecSFelix Fietkau skb->dev = outdev; 496a908fdecSFelix Fietkau nexthop = rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6); 4972a79fd39SJason A. Donenfeld skb_dst_set_noref(skb, &rt->dst); 498a908fdecSFelix Fietkau neigh_xmit(NEIGH_ND_TABLE, outdev, nexthop, skb); 499a908fdecSFelix Fietkau 500a908fdecSFelix Fietkau return NF_STOLEN; 501a908fdecSFelix Fietkau } 502a908fdecSFelix Fietkau EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook); 503