1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 27d208687SFelix Fietkau #include <linux/kernel.h> 37d208687SFelix Fietkau #include <linux/init.h> 47d208687SFelix Fietkau #include <linux/module.h> 57d208687SFelix Fietkau #include <linux/netfilter.h> 67d208687SFelix Fietkau #include <linux/rhashtable.h> 77d208687SFelix Fietkau #include <linux/ip.h> 8a908fdecSFelix Fietkau #include <linux/ipv6.h> 97d208687SFelix Fietkau #include <linux/netdevice.h> 107d208687SFelix Fietkau #include <net/ip.h> 11a908fdecSFelix Fietkau #include <net/ipv6.h> 12a908fdecSFelix Fietkau #include <net/ip6_route.h> 137d208687SFelix Fietkau #include <net/neighbour.h> 147d208687SFelix Fietkau #include <net/netfilter/nf_flow_table.h> 1553c2b289SPablo Neira Ayuso #include <net/netfilter/nf_conntrack_acct.h> 167d208687SFelix Fietkau /* For layer 4 checksum field offset. */ 177d208687SFelix Fietkau #include <linux/tcp.h> 187d208687SFelix Fietkau #include <linux/udp.h> 197d208687SFelix Fietkau 2033894c36SFelix Fietkau static int nf_flow_state_check(struct flow_offload *flow, int proto, 21b6f27d32SFelix Fietkau struct sk_buff *skb, unsigned int thoff) 22b6f27d32SFelix Fietkau { 23b6f27d32SFelix Fietkau struct tcphdr *tcph; 24b6f27d32SFelix Fietkau 2533894c36SFelix Fietkau if (proto != IPPROTO_TCP) 2633894c36SFelix Fietkau return 0; 2733894c36SFelix Fietkau 28b6f27d32SFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 29b6f27d32SFelix Fietkau if (unlikely(tcph->fin || tcph->rst)) { 30b6f27d32SFelix Fietkau flow_offload_teardown(flow); 31b6f27d32SFelix Fietkau return -1; 32b6f27d32SFelix Fietkau } 33b6f27d32SFelix Fietkau 34b6f27d32SFelix Fietkau return 0; 35b6f27d32SFelix Fietkau } 36b6f27d32SFelix Fietkau 37f4401262SPablo Neira Ayuso static void nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff, 387d208687SFelix Fietkau __be32 addr, __be32 new_addr) 397d208687SFelix Fietkau { 407d208687SFelix Fietkau struct tcphdr *tcph; 417d208687SFelix Fietkau 427d208687SFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 437d208687SFelix Fietkau inet_proto_csum_replace4(&tcph->check, skb, addr, new_addr, true); 447d208687SFelix Fietkau } 457d208687SFelix Fietkau 46f4401262SPablo Neira Ayuso static void nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff, 477d208687SFelix Fietkau __be32 addr, __be32 new_addr) 487d208687SFelix Fietkau { 497d208687SFelix Fietkau struct udphdr *udph; 507d208687SFelix Fietkau 517d208687SFelix Fietkau udph = (void *)(skb_network_header(skb) + thoff); 527d208687SFelix Fietkau if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 537d208687SFelix Fietkau inet_proto_csum_replace4(&udph->check, skb, addr, 547d208687SFelix Fietkau new_addr, true); 557d208687SFelix Fietkau if (!udph->check) 567d208687SFelix Fietkau udph->check = CSUM_MANGLED_0; 577d208687SFelix Fietkau } 587d208687SFelix Fietkau } 597d208687SFelix Fietkau 60f4401262SPablo Neira Ayuso static void nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph, 617d208687SFelix Fietkau unsigned int thoff, __be32 addr, 627d208687SFelix Fietkau __be32 new_addr) 637d208687SFelix Fietkau { 647d208687SFelix Fietkau switch (iph->protocol) { 657d208687SFelix Fietkau case IPPROTO_TCP: 66f4401262SPablo Neira Ayuso nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr); 677d208687SFelix Fietkau break; 687d208687SFelix Fietkau case IPPROTO_UDP: 69f4401262SPablo Neira Ayuso nf_flow_nat_ip_udp(skb, thoff, addr, new_addr); 707d208687SFelix Fietkau break; 717d208687SFelix Fietkau } 727d208687SFelix Fietkau } 737d208687SFelix Fietkau 74f4401262SPablo Neira Ayuso static void nf_flow_snat_ip(const struct flow_offload *flow, 75f4401262SPablo Neira Ayuso struct sk_buff *skb, struct iphdr *iph, 76f4401262SPablo Neira Ayuso unsigned int thoff, enum flow_offload_tuple_dir dir) 777d208687SFelix Fietkau { 787d208687SFelix Fietkau __be32 addr, new_addr; 797d208687SFelix Fietkau 807d208687SFelix Fietkau switch (dir) { 817d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 827d208687SFelix Fietkau addr = iph->saddr; 837d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; 847d208687SFelix Fietkau iph->saddr = new_addr; 857d208687SFelix Fietkau break; 867d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 877d208687SFelix Fietkau addr = iph->daddr; 887d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr; 897d208687SFelix Fietkau iph->daddr = new_addr; 907d208687SFelix Fietkau break; 917d208687SFelix Fietkau } 927d208687SFelix Fietkau csum_replace4(&iph->check, addr, new_addr); 937d208687SFelix Fietkau 94f4401262SPablo Neira Ayuso nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); 957d208687SFelix Fietkau } 967d208687SFelix Fietkau 97f4401262SPablo Neira Ayuso static void nf_flow_dnat_ip(const struct flow_offload *flow, 98f4401262SPablo Neira Ayuso struct sk_buff *skb, struct iphdr *iph, 99f4401262SPablo Neira Ayuso unsigned int thoff, enum flow_offload_tuple_dir dir) 1007d208687SFelix Fietkau { 1017d208687SFelix Fietkau __be32 addr, new_addr; 1027d208687SFelix Fietkau 1037d208687SFelix Fietkau switch (dir) { 1047d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 1057d208687SFelix Fietkau addr = iph->daddr; 1067d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; 1077d208687SFelix Fietkau iph->daddr = new_addr; 1087d208687SFelix Fietkau break; 1097d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 1107d208687SFelix Fietkau addr = iph->saddr; 1117d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr; 1127d208687SFelix Fietkau iph->saddr = new_addr; 1137d208687SFelix Fietkau break; 1147d208687SFelix Fietkau } 1157d208687SFelix Fietkau csum_replace4(&iph->check, addr, new_addr); 1167d208687SFelix Fietkau 117f4401262SPablo Neira Ayuso nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); 1187d208687SFelix Fietkau } 1197d208687SFelix Fietkau 120f4401262SPablo Neira Ayuso static void nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb, 1212fc11745SPablo Neira Ayuso unsigned int thoff, enum flow_offload_tuple_dir dir, 1222fc11745SPablo Neira Ayuso struct iphdr *iph) 1237d208687SFelix Fietkau { 124f4401262SPablo Neira Ayuso if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 125f4401262SPablo Neira Ayuso nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir); 126f4401262SPablo Neira Ayuso nf_flow_snat_ip(flow, skb, iph, thoff, dir); 127f4401262SPablo Neira Ayuso } 128f4401262SPablo Neira Ayuso if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 129f4401262SPablo Neira Ayuso nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir); 130f4401262SPablo Neira Ayuso nf_flow_dnat_ip(flow, skb, iph, thoff, dir); 131f4401262SPablo Neira Ayuso } 1327d208687SFelix Fietkau } 1337d208687SFelix Fietkau 1347d208687SFelix Fietkau static bool ip_has_options(unsigned int thoff) 1357d208687SFelix Fietkau { 1367d208687SFelix Fietkau return thoff != sizeof(struct iphdr); 1377d208687SFelix Fietkau } 1387d208687SFelix Fietkau 1397d208687SFelix Fietkau static int nf_flow_tuple_ip(struct sk_buff *skb, const struct net_device *dev, 1402fc11745SPablo Neira Ayuso struct flow_offload_tuple *tuple, u32 *hdrsize) 1417d208687SFelix Fietkau { 1427d208687SFelix Fietkau struct flow_ports *ports; 1432fc11745SPablo Neira Ayuso unsigned int thoff; 1447d208687SFelix Fietkau struct iphdr *iph; 1457d208687SFelix Fietkau 1467d208687SFelix Fietkau if (!pskb_may_pull(skb, sizeof(*iph))) 1477d208687SFelix Fietkau return -1; 1487d208687SFelix Fietkau 1497d208687SFelix Fietkau iph = ip_hdr(skb); 1507d208687SFelix Fietkau thoff = iph->ihl * 4; 1517d208687SFelix Fietkau 1527d208687SFelix Fietkau if (ip_is_fragment(iph) || 1537d208687SFelix Fietkau unlikely(ip_has_options(thoff))) 1547d208687SFelix Fietkau return -1; 1557d208687SFelix Fietkau 156793d5d61SPablo Neira Ayuso switch (iph->protocol) { 157793d5d61SPablo Neira Ayuso case IPPROTO_TCP: 1582fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct tcphdr); 159793d5d61SPablo Neira Ayuso break; 160793d5d61SPablo Neira Ayuso case IPPROTO_UDP: 1612fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct udphdr); 162793d5d61SPablo Neira Ayuso break; 163793d5d61SPablo Neira Ayuso default: 1647d208687SFelix Fietkau return -1; 165793d5d61SPablo Neira Ayuso } 1667d208687SFelix Fietkau 16733cc3c0cSTaehee Yoo if (iph->ttl <= 1) 16833cc3c0cSTaehee Yoo return -1; 16933cc3c0cSTaehee Yoo 1707d208687SFelix Fietkau thoff = iph->ihl * 4; 1712fc11745SPablo Neira Ayuso if (!pskb_may_pull(skb, thoff + *hdrsize)) 1727d208687SFelix Fietkau return -1; 1737d208687SFelix Fietkau 17441e9ec5aSHaishuang Yan iph = ip_hdr(skb); 1757d208687SFelix Fietkau ports = (struct flow_ports *)(skb_network_header(skb) + thoff); 1767d208687SFelix Fietkau 1777d208687SFelix Fietkau tuple->src_v4.s_addr = iph->saddr; 1787d208687SFelix Fietkau tuple->dst_v4.s_addr = iph->daddr; 1797d208687SFelix Fietkau tuple->src_port = ports->source; 1807d208687SFelix Fietkau tuple->dst_port = ports->dest; 1817d208687SFelix Fietkau tuple->l3proto = AF_INET; 1827d208687SFelix Fietkau tuple->l4proto = iph->protocol; 1837d208687SFelix Fietkau tuple->iifidx = dev->ifindex; 1847d208687SFelix Fietkau 1857d208687SFelix Fietkau return 0; 1867d208687SFelix Fietkau } 1877d208687SFelix Fietkau 1887d208687SFelix Fietkau /* Based on ip_exceeds_mtu(). */ 1897d208687SFelix Fietkau static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu) 1907d208687SFelix Fietkau { 1917d208687SFelix Fietkau if (skb->len <= mtu) 1927d208687SFelix Fietkau return false; 1937d208687SFelix Fietkau 1947d208687SFelix Fietkau if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 1957d208687SFelix Fietkau return false; 1967d208687SFelix Fietkau 1977d208687SFelix Fietkau return true; 1987d208687SFelix Fietkau } 1997d208687SFelix Fietkau 200589b474aSFlorian Westphal static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb, 201589b474aSFlorian Westphal const struct nf_hook_state *state, 202589b474aSFlorian Westphal struct dst_entry *dst) 203589b474aSFlorian Westphal { 204589b474aSFlorian Westphal skb_orphan(skb); 205589b474aSFlorian Westphal skb_dst_set_noref(skb, dst); 206589b474aSFlorian Westphal dst_output(state->net, state->sk, skb); 207589b474aSFlorian Westphal return NF_STOLEN; 208589b474aSFlorian Westphal } 209589b474aSFlorian Westphal 2107d208687SFelix Fietkau unsigned int 2117d208687SFelix Fietkau nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, 2127d208687SFelix Fietkau const struct nf_hook_state *state) 2137d208687SFelix Fietkau { 2147d208687SFelix Fietkau struct flow_offload_tuple_rhash *tuplehash; 2157d208687SFelix Fietkau struct nf_flowtable *flow_table = priv; 2167d208687SFelix Fietkau struct flow_offload_tuple tuple = {}; 2177d208687SFelix Fietkau enum flow_offload_tuple_dir dir; 2187d208687SFelix Fietkau struct flow_offload *flow; 2197d208687SFelix Fietkau struct net_device *outdev; 2202a79fd39SJason A. Donenfeld struct rtable *rt; 221b6f27d32SFelix Fietkau unsigned int thoff; 2227d208687SFelix Fietkau struct iphdr *iph; 2237d208687SFelix Fietkau __be32 nexthop; 2242fc11745SPablo Neira Ayuso u32 hdrsize; 2257d208687SFelix Fietkau 2267d208687SFelix Fietkau if (skb->protocol != htons(ETH_P_IP)) 2277d208687SFelix Fietkau return NF_ACCEPT; 2287d208687SFelix Fietkau 2292fc11745SPablo Neira Ayuso if (nf_flow_tuple_ip(skb, state->in, &tuple, &hdrsize) < 0) 2307d208687SFelix Fietkau return NF_ACCEPT; 2317d208687SFelix Fietkau 2327d208687SFelix Fietkau tuplehash = flow_offload_lookup(flow_table, &tuple); 2337d208687SFelix Fietkau if (tuplehash == NULL) 2347d208687SFelix Fietkau return NF_ACCEPT; 2357d208687SFelix Fietkau 2367d208687SFelix Fietkau dir = tuplehash->tuple.dir; 2377d208687SFelix Fietkau flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 2387d208687SFelix Fietkau 239e75b3e1cSFlorian Westphal if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu))) 2407d208687SFelix Fietkau return NF_ACCEPT; 2417d208687SFelix Fietkau 2422fc11745SPablo Neira Ayuso iph = ip_hdr(skb); 2432fc11745SPablo Neira Ayuso thoff = iph->ihl * 4; 2442fc11745SPablo Neira Ayuso if (nf_flow_state_check(flow, iph->protocol, skb, thoff)) 245b6f27d32SFelix Fietkau return NF_ACCEPT; 246b6f27d32SFelix Fietkau 247e5075c0bSPablo Neira Ayuso if (!dst_check(&rt->dst, 0)) { 248589b474aSFlorian Westphal flow_offload_teardown(flow); 249589b474aSFlorian Westphal return NF_ACCEPT; 250589b474aSFlorian Westphal } 251589b474aSFlorian Westphal 2522babb46cSPablo Neira Ayuso if (skb_try_make_writable(skb, thoff + hdrsize)) 2532babb46cSPablo Neira Ayuso return NF_DROP; 2542babb46cSPablo Neira Ayuso 2551b9cd769SPablo Neira Ayuso flow_offload_refresh(flow_table, flow); 2561b9cd769SPablo Neira Ayuso 2572babb46cSPablo Neira Ayuso iph = ip_hdr(skb); 258f4401262SPablo Neira Ayuso nf_flow_nat_ip(flow, skb, thoff, dir, iph); 2597d208687SFelix Fietkau 2607d208687SFelix Fietkau ip_decrease_ttl(iph); 261de20900fSFlorian Westphal skb->tstamp = 0; 2627d208687SFelix Fietkau 26353c2b289SPablo Neira Ayuso if (flow_table->flags & NF_FLOWTABLE_COUNTER) 26453c2b289SPablo Neira Ayuso nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len); 26553c2b289SPablo Neira Ayuso 266*5139c0c0SPablo Neira Ayuso rt = (struct rtable *)tuplehash->tuple.dst_cache; 267*5139c0c0SPablo Neira Ayuso 268*5139c0c0SPablo Neira Ayuso if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) { 269589b474aSFlorian Westphal memset(skb->cb, 0, sizeof(struct inet_skb_parm)); 270589b474aSFlorian Westphal IPCB(skb)->iif = skb->dev->ifindex; 271589b474aSFlorian Westphal IPCB(skb)->flags = IPSKB_FORWARDED; 272589b474aSFlorian Westphal return nf_flow_xmit_xfrm(skb, state, &rt->dst); 273589b474aSFlorian Westphal } 274589b474aSFlorian Westphal 275*5139c0c0SPablo Neira Ayuso outdev = rt->dst.dev; 2767d208687SFelix Fietkau skb->dev = outdev; 2777d208687SFelix Fietkau nexthop = rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr); 2782a79fd39SJason A. Donenfeld skb_dst_set_noref(skb, &rt->dst); 2797d208687SFelix Fietkau neigh_xmit(NEIGH_ARP_TABLE, outdev, &nexthop, skb); 2807d208687SFelix Fietkau 2817d208687SFelix Fietkau return NF_STOLEN; 2827d208687SFelix Fietkau } 2837d208687SFelix Fietkau EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook); 284a908fdecSFelix Fietkau 285f4401262SPablo Neira Ayuso static void nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff, 286a908fdecSFelix Fietkau struct in6_addr *addr, 287f4401262SPablo Neira Ayuso struct in6_addr *new_addr, 288f4401262SPablo Neira Ayuso struct ipv6hdr *ip6h) 289a908fdecSFelix Fietkau { 290a908fdecSFelix Fietkau struct tcphdr *tcph; 291a908fdecSFelix Fietkau 292a908fdecSFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 293a908fdecSFelix Fietkau inet_proto_csum_replace16(&tcph->check, skb, addr->s6_addr32, 294a908fdecSFelix Fietkau new_addr->s6_addr32, true); 295a908fdecSFelix Fietkau } 296a908fdecSFelix Fietkau 297f4401262SPablo Neira Ayuso static void nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff, 298a908fdecSFelix Fietkau struct in6_addr *addr, 299a908fdecSFelix Fietkau struct in6_addr *new_addr) 300a908fdecSFelix Fietkau { 301a908fdecSFelix Fietkau struct udphdr *udph; 302a908fdecSFelix Fietkau 303a908fdecSFelix Fietkau udph = (void *)(skb_network_header(skb) + thoff); 304a908fdecSFelix Fietkau if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 305a908fdecSFelix Fietkau inet_proto_csum_replace16(&udph->check, skb, addr->s6_addr32, 306a908fdecSFelix Fietkau new_addr->s6_addr32, true); 307a908fdecSFelix Fietkau if (!udph->check) 308a908fdecSFelix Fietkau udph->check = CSUM_MANGLED_0; 309a908fdecSFelix Fietkau } 310a908fdecSFelix Fietkau } 311a908fdecSFelix Fietkau 312f4401262SPablo Neira Ayuso static void nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h, 313a908fdecSFelix Fietkau unsigned int thoff, struct in6_addr *addr, 314a908fdecSFelix Fietkau struct in6_addr *new_addr) 315a908fdecSFelix Fietkau { 316a908fdecSFelix Fietkau switch (ip6h->nexthdr) { 317a908fdecSFelix Fietkau case IPPROTO_TCP: 318f4401262SPablo Neira Ayuso nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr, ip6h); 319a908fdecSFelix Fietkau break; 320a908fdecSFelix Fietkau case IPPROTO_UDP: 321f4401262SPablo Neira Ayuso nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr); 322a908fdecSFelix Fietkau break; 323a908fdecSFelix Fietkau } 324a908fdecSFelix Fietkau } 325a908fdecSFelix Fietkau 326f4401262SPablo Neira Ayuso static void nf_flow_snat_ipv6(const struct flow_offload *flow, 327a908fdecSFelix Fietkau struct sk_buff *skb, struct ipv6hdr *ip6h, 328a908fdecSFelix Fietkau unsigned int thoff, 329a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 330a908fdecSFelix Fietkau { 331a908fdecSFelix Fietkau struct in6_addr addr, new_addr; 332a908fdecSFelix Fietkau 333a908fdecSFelix Fietkau switch (dir) { 334a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 335a908fdecSFelix Fietkau addr = ip6h->saddr; 336a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6; 337a908fdecSFelix Fietkau ip6h->saddr = new_addr; 338a908fdecSFelix Fietkau break; 339a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 340a908fdecSFelix Fietkau addr = ip6h->daddr; 341a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6; 342a908fdecSFelix Fietkau ip6h->daddr = new_addr; 343a908fdecSFelix Fietkau break; 344a908fdecSFelix Fietkau } 345a908fdecSFelix Fietkau 346f4401262SPablo Neira Ayuso nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr); 347a908fdecSFelix Fietkau } 348a908fdecSFelix Fietkau 349f4401262SPablo Neira Ayuso static void nf_flow_dnat_ipv6(const struct flow_offload *flow, 350a908fdecSFelix Fietkau struct sk_buff *skb, struct ipv6hdr *ip6h, 351a908fdecSFelix Fietkau unsigned int thoff, 352a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 353a908fdecSFelix Fietkau { 354a908fdecSFelix Fietkau struct in6_addr addr, new_addr; 355a908fdecSFelix Fietkau 356a908fdecSFelix Fietkau switch (dir) { 357a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 358a908fdecSFelix Fietkau addr = ip6h->daddr; 359a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6; 360a908fdecSFelix Fietkau ip6h->daddr = new_addr; 361a908fdecSFelix Fietkau break; 362a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 363a908fdecSFelix Fietkau addr = ip6h->saddr; 364a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6; 365a908fdecSFelix Fietkau ip6h->saddr = new_addr; 366a908fdecSFelix Fietkau break; 367a908fdecSFelix Fietkau } 368a908fdecSFelix Fietkau 369f4401262SPablo Neira Ayuso nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr); 370a908fdecSFelix Fietkau } 371a908fdecSFelix Fietkau 372f4401262SPablo Neira Ayuso static void nf_flow_nat_ipv6(const struct flow_offload *flow, 373a908fdecSFelix Fietkau struct sk_buff *skb, 3742fc11745SPablo Neira Ayuso enum flow_offload_tuple_dir dir, 3752fc11745SPablo Neira Ayuso struct ipv6hdr *ip6h) 376a908fdecSFelix Fietkau { 377a908fdecSFelix Fietkau unsigned int thoff = sizeof(*ip6h); 378a908fdecSFelix Fietkau 379f4401262SPablo Neira Ayuso if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 380f4401262SPablo Neira Ayuso nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir); 381f4401262SPablo Neira Ayuso nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir); 382f4401262SPablo Neira Ayuso } 383f4401262SPablo Neira Ayuso if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 384f4401262SPablo Neira Ayuso nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir); 385f4401262SPablo Neira Ayuso nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir); 386f4401262SPablo Neira Ayuso } 387a908fdecSFelix Fietkau } 388a908fdecSFelix Fietkau 389a908fdecSFelix Fietkau static int nf_flow_tuple_ipv6(struct sk_buff *skb, const struct net_device *dev, 3902fc11745SPablo Neira Ayuso struct flow_offload_tuple *tuple, u32 *hdrsize) 391a908fdecSFelix Fietkau { 392a908fdecSFelix Fietkau struct flow_ports *ports; 393a908fdecSFelix Fietkau struct ipv6hdr *ip6h; 3942fc11745SPablo Neira Ayuso unsigned int thoff; 395a908fdecSFelix Fietkau 396a908fdecSFelix Fietkau if (!pskb_may_pull(skb, sizeof(*ip6h))) 397a908fdecSFelix Fietkau return -1; 398a908fdecSFelix Fietkau 399a908fdecSFelix Fietkau ip6h = ipv6_hdr(skb); 400a908fdecSFelix Fietkau 401793d5d61SPablo Neira Ayuso switch (ip6h->nexthdr) { 402793d5d61SPablo Neira Ayuso case IPPROTO_TCP: 4032fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct tcphdr); 404793d5d61SPablo Neira Ayuso break; 405793d5d61SPablo Neira Ayuso case IPPROTO_UDP: 4062fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct udphdr); 407793d5d61SPablo Neira Ayuso break; 408793d5d61SPablo Neira Ayuso default: 409a908fdecSFelix Fietkau return -1; 410793d5d61SPablo Neira Ayuso } 411a908fdecSFelix Fietkau 41233cc3c0cSTaehee Yoo if (ip6h->hop_limit <= 1) 41333cc3c0cSTaehee Yoo return -1; 41433cc3c0cSTaehee Yoo 415a908fdecSFelix Fietkau thoff = sizeof(*ip6h); 4162fc11745SPablo Neira Ayuso if (!pskb_may_pull(skb, thoff + *hdrsize)) 417a908fdecSFelix Fietkau return -1; 418a908fdecSFelix Fietkau 41941e9ec5aSHaishuang Yan ip6h = ipv6_hdr(skb); 420a908fdecSFelix Fietkau ports = (struct flow_ports *)(skb_network_header(skb) + thoff); 421a908fdecSFelix Fietkau 422a908fdecSFelix Fietkau tuple->src_v6 = ip6h->saddr; 423a908fdecSFelix Fietkau tuple->dst_v6 = ip6h->daddr; 424a908fdecSFelix Fietkau tuple->src_port = ports->source; 425a908fdecSFelix Fietkau tuple->dst_port = ports->dest; 426a908fdecSFelix Fietkau tuple->l3proto = AF_INET6; 427a908fdecSFelix Fietkau tuple->l4proto = ip6h->nexthdr; 428a908fdecSFelix Fietkau tuple->iifidx = dev->ifindex; 429a908fdecSFelix Fietkau 430a908fdecSFelix Fietkau return 0; 431a908fdecSFelix Fietkau } 432a908fdecSFelix Fietkau 433a908fdecSFelix Fietkau unsigned int 434a908fdecSFelix Fietkau nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, 435a908fdecSFelix Fietkau const struct nf_hook_state *state) 436a908fdecSFelix Fietkau { 437a908fdecSFelix Fietkau struct flow_offload_tuple_rhash *tuplehash; 438a908fdecSFelix Fietkau struct nf_flowtable *flow_table = priv; 439a908fdecSFelix Fietkau struct flow_offload_tuple tuple = {}; 440a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir; 4419b1c1ef1SNicolas Dichtel const struct in6_addr *nexthop; 442a908fdecSFelix Fietkau struct flow_offload *flow; 443a908fdecSFelix Fietkau struct net_device *outdev; 444a908fdecSFelix Fietkau struct ipv6hdr *ip6h; 445a908fdecSFelix Fietkau struct rt6_info *rt; 4462fc11745SPablo Neira Ayuso u32 hdrsize; 447a908fdecSFelix Fietkau 448a908fdecSFelix Fietkau if (skb->protocol != htons(ETH_P_IPV6)) 449a908fdecSFelix Fietkau return NF_ACCEPT; 450a908fdecSFelix Fietkau 4512fc11745SPablo Neira Ayuso if (nf_flow_tuple_ipv6(skb, state->in, &tuple, &hdrsize) < 0) 452a908fdecSFelix Fietkau return NF_ACCEPT; 453a908fdecSFelix Fietkau 454a908fdecSFelix Fietkau tuplehash = flow_offload_lookup(flow_table, &tuple); 455a908fdecSFelix Fietkau if (tuplehash == NULL) 456a908fdecSFelix Fietkau return NF_ACCEPT; 457a908fdecSFelix Fietkau 458a908fdecSFelix Fietkau dir = tuplehash->tuple.dir; 459a908fdecSFelix Fietkau flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 460a908fdecSFelix Fietkau 461a908fdecSFelix Fietkau if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu))) 462a908fdecSFelix Fietkau return NF_ACCEPT; 463a908fdecSFelix Fietkau 46433894c36SFelix Fietkau if (nf_flow_state_check(flow, ipv6_hdr(skb)->nexthdr, skb, 46533894c36SFelix Fietkau sizeof(*ip6h))) 466b6f27d32SFelix Fietkau return NF_ACCEPT; 467b6f27d32SFelix Fietkau 468e5075c0bSPablo Neira Ayuso if (!dst_check(&rt->dst, 0)) { 469589b474aSFlorian Westphal flow_offload_teardown(flow); 470589b474aSFlorian Westphal return NF_ACCEPT; 471589b474aSFlorian Westphal } 472589b474aSFlorian Westphal 4732fc11745SPablo Neira Ayuso if (skb_try_make_writable(skb, sizeof(*ip6h) + hdrsize)) 474a908fdecSFelix Fietkau return NF_DROP; 475a908fdecSFelix Fietkau 4761b9cd769SPablo Neira Ayuso flow_offload_refresh(flow_table, flow); 4771b9cd769SPablo Neira Ayuso 478a908fdecSFelix Fietkau ip6h = ipv6_hdr(skb); 479f4401262SPablo Neira Ayuso nf_flow_nat_ipv6(flow, skb, dir, ip6h); 4802fc11745SPablo Neira Ayuso 481a908fdecSFelix Fietkau ip6h->hop_limit--; 482de20900fSFlorian Westphal skb->tstamp = 0; 483a908fdecSFelix Fietkau 48453c2b289SPablo Neira Ayuso if (flow_table->flags & NF_FLOWTABLE_COUNTER) 48553c2b289SPablo Neira Ayuso nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len); 48653c2b289SPablo Neira Ayuso 487*5139c0c0SPablo Neira Ayuso rt = (struct rt6_info *)tuplehash->tuple.dst_cache; 488*5139c0c0SPablo Neira Ayuso 489*5139c0c0SPablo Neira Ayuso if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) { 490589b474aSFlorian Westphal memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); 491589b474aSFlorian Westphal IP6CB(skb)->iif = skb->dev->ifindex; 492589b474aSFlorian Westphal IP6CB(skb)->flags = IP6SKB_FORWARDED; 493589b474aSFlorian Westphal return nf_flow_xmit_xfrm(skb, state, &rt->dst); 494589b474aSFlorian Westphal } 495589b474aSFlorian Westphal 496*5139c0c0SPablo Neira Ayuso outdev = rt->dst.dev; 497a908fdecSFelix Fietkau skb->dev = outdev; 498a908fdecSFelix Fietkau nexthop = rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6); 4992a79fd39SJason A. Donenfeld skb_dst_set_noref(skb, &rt->dst); 500a908fdecSFelix Fietkau neigh_xmit(NEIGH_ND_TABLE, outdev, nexthop, skb); 501a908fdecSFelix Fietkau 502a908fdecSFelix Fietkau return NF_STOLEN; 503a908fdecSFelix Fietkau } 504a908fdecSFelix Fietkau EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook); 505