1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 27d208687SFelix Fietkau #include <linux/kernel.h> 37d208687SFelix Fietkau #include <linux/init.h> 47d208687SFelix Fietkau #include <linux/module.h> 57d208687SFelix Fietkau #include <linux/netfilter.h> 67d208687SFelix Fietkau #include <linux/rhashtable.h> 77d208687SFelix Fietkau #include <linux/ip.h> 8a908fdecSFelix Fietkau #include <linux/ipv6.h> 97d208687SFelix Fietkau #include <linux/netdevice.h> 107d208687SFelix Fietkau #include <net/ip.h> 11a908fdecSFelix Fietkau #include <net/ipv6.h> 12a908fdecSFelix Fietkau #include <net/ip6_route.h> 137d208687SFelix Fietkau #include <net/neighbour.h> 147d208687SFelix Fietkau #include <net/netfilter/nf_flow_table.h> 1553c2b289SPablo Neira Ayuso #include <net/netfilter/nf_conntrack_acct.h> 167d208687SFelix Fietkau /* For layer 4 checksum field offset. */ 177d208687SFelix Fietkau #include <linux/tcp.h> 187d208687SFelix Fietkau #include <linux/udp.h> 197d208687SFelix Fietkau 2033894c36SFelix Fietkau static int nf_flow_state_check(struct flow_offload *flow, int proto, 21b6f27d32SFelix Fietkau struct sk_buff *skb, unsigned int thoff) 22b6f27d32SFelix Fietkau { 23b6f27d32SFelix Fietkau struct tcphdr *tcph; 24b6f27d32SFelix Fietkau 2533894c36SFelix Fietkau if (proto != IPPROTO_TCP) 2633894c36SFelix Fietkau return 0; 2733894c36SFelix Fietkau 28b6f27d32SFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 29b6f27d32SFelix Fietkau if (unlikely(tcph->fin || tcph->rst)) { 30b6f27d32SFelix Fietkau flow_offload_teardown(flow); 31b6f27d32SFelix Fietkau return -1; 32b6f27d32SFelix Fietkau } 33b6f27d32SFelix Fietkau 34b6f27d32SFelix Fietkau return 0; 35b6f27d32SFelix Fietkau } 36b6f27d32SFelix Fietkau 377d208687SFelix Fietkau static int nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff, 387d208687SFelix Fietkau __be32 addr, __be32 new_addr) 397d208687SFelix Fietkau { 407d208687SFelix Fietkau struct tcphdr *tcph; 417d208687SFelix Fietkau 427d208687SFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 437d208687SFelix Fietkau inet_proto_csum_replace4(&tcph->check, skb, addr, new_addr, true); 447d208687SFelix Fietkau 457d208687SFelix Fietkau return 0; 467d208687SFelix Fietkau } 477d208687SFelix Fietkau 487d208687SFelix Fietkau static int nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff, 497d208687SFelix Fietkau __be32 addr, __be32 new_addr) 507d208687SFelix Fietkau { 517d208687SFelix Fietkau struct udphdr *udph; 527d208687SFelix Fietkau 537d208687SFelix Fietkau udph = (void *)(skb_network_header(skb) + thoff); 547d208687SFelix Fietkau if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 557d208687SFelix Fietkau inet_proto_csum_replace4(&udph->check, skb, addr, 567d208687SFelix Fietkau new_addr, true); 577d208687SFelix Fietkau if (!udph->check) 587d208687SFelix Fietkau udph->check = CSUM_MANGLED_0; 597d208687SFelix Fietkau } 607d208687SFelix Fietkau 617d208687SFelix Fietkau return 0; 627d208687SFelix Fietkau } 637d208687SFelix Fietkau 647d208687SFelix Fietkau static int nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph, 657d208687SFelix Fietkau unsigned int thoff, __be32 addr, 667d208687SFelix Fietkau __be32 new_addr) 677d208687SFelix Fietkau { 687d208687SFelix Fietkau switch (iph->protocol) { 697d208687SFelix Fietkau case IPPROTO_TCP: 707d208687SFelix Fietkau if (nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr) < 0) 717d208687SFelix Fietkau return NF_DROP; 727d208687SFelix Fietkau break; 737d208687SFelix Fietkau case IPPROTO_UDP: 747d208687SFelix Fietkau if (nf_flow_nat_ip_udp(skb, thoff, addr, new_addr) < 0) 757d208687SFelix Fietkau return NF_DROP; 767d208687SFelix Fietkau break; 777d208687SFelix Fietkau } 787d208687SFelix Fietkau 797d208687SFelix Fietkau return 0; 807d208687SFelix Fietkau } 817d208687SFelix Fietkau 827d208687SFelix Fietkau static int nf_flow_snat_ip(const struct flow_offload *flow, struct sk_buff *skb, 837d208687SFelix Fietkau struct iphdr *iph, unsigned int thoff, 847d208687SFelix Fietkau enum flow_offload_tuple_dir dir) 857d208687SFelix Fietkau { 867d208687SFelix Fietkau __be32 addr, new_addr; 877d208687SFelix Fietkau 887d208687SFelix Fietkau switch (dir) { 897d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 907d208687SFelix Fietkau addr = iph->saddr; 917d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; 927d208687SFelix Fietkau iph->saddr = new_addr; 937d208687SFelix Fietkau break; 947d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 957d208687SFelix Fietkau addr = iph->daddr; 967d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr; 977d208687SFelix Fietkau iph->daddr = new_addr; 987d208687SFelix Fietkau break; 997d208687SFelix Fietkau default: 1007d208687SFelix Fietkau return -1; 1017d208687SFelix Fietkau } 1027d208687SFelix Fietkau csum_replace4(&iph->check, addr, new_addr); 1037d208687SFelix Fietkau 1047d208687SFelix Fietkau return nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); 1057d208687SFelix Fietkau } 1067d208687SFelix Fietkau 1077d208687SFelix Fietkau static int nf_flow_dnat_ip(const struct flow_offload *flow, struct sk_buff *skb, 1087d208687SFelix Fietkau struct iphdr *iph, unsigned int thoff, 1097d208687SFelix Fietkau enum flow_offload_tuple_dir dir) 1107d208687SFelix Fietkau { 1117d208687SFelix Fietkau __be32 addr, new_addr; 1127d208687SFelix Fietkau 1137d208687SFelix Fietkau switch (dir) { 1147d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 1157d208687SFelix Fietkau addr = iph->daddr; 1167d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; 1177d208687SFelix Fietkau iph->daddr = new_addr; 1187d208687SFelix Fietkau break; 1197d208687SFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 1207d208687SFelix Fietkau addr = iph->saddr; 1217d208687SFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr; 1227d208687SFelix Fietkau iph->saddr = new_addr; 1237d208687SFelix Fietkau break; 1247d208687SFelix Fietkau default: 1257d208687SFelix Fietkau return -1; 1267d208687SFelix Fietkau } 1277d208687SFelix Fietkau csum_replace4(&iph->check, addr, new_addr); 1287d208687SFelix Fietkau 1297d208687SFelix Fietkau return nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); 1307d208687SFelix Fietkau } 1317d208687SFelix Fietkau 1327d208687SFelix Fietkau static int nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb, 133*2fc11745SPablo Neira Ayuso unsigned int thoff, enum flow_offload_tuple_dir dir, 134*2fc11745SPablo Neira Ayuso struct iphdr *iph) 1357d208687SFelix Fietkau { 136355a8b13SPablo Neira Ayuso if (test_bit(NF_FLOW_SNAT, &flow->flags) && 1377d208687SFelix Fietkau (nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir) < 0 || 138*2fc11745SPablo Neira Ayuso nf_flow_snat_ip(flow, skb, iph, thoff, dir) < 0)) 1397d208687SFelix Fietkau return -1; 14061abaf02SHaishuang Yan 141355a8b13SPablo Neira Ayuso if (test_bit(NF_FLOW_DNAT, &flow->flags) && 1427d208687SFelix Fietkau (nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir) < 0 || 143*2fc11745SPablo Neira Ayuso nf_flow_dnat_ip(flow, skb, iph, thoff, dir) < 0)) 1447d208687SFelix Fietkau return -1; 1457d208687SFelix Fietkau 1467d208687SFelix Fietkau return 0; 1477d208687SFelix Fietkau } 1487d208687SFelix Fietkau 1497d208687SFelix Fietkau static bool ip_has_options(unsigned int thoff) 1507d208687SFelix Fietkau { 1517d208687SFelix Fietkau return thoff != sizeof(struct iphdr); 1527d208687SFelix Fietkau } 1537d208687SFelix Fietkau 1547d208687SFelix Fietkau static int nf_flow_tuple_ip(struct sk_buff *skb, const struct net_device *dev, 155*2fc11745SPablo Neira Ayuso struct flow_offload_tuple *tuple, u32 *hdrsize) 1567d208687SFelix Fietkau { 1577d208687SFelix Fietkau struct flow_ports *ports; 158*2fc11745SPablo Neira Ayuso unsigned int thoff; 1597d208687SFelix Fietkau struct iphdr *iph; 1607d208687SFelix Fietkau 1617d208687SFelix Fietkau if (!pskb_may_pull(skb, sizeof(*iph))) 1627d208687SFelix Fietkau return -1; 1637d208687SFelix Fietkau 1647d208687SFelix Fietkau iph = ip_hdr(skb); 1657d208687SFelix Fietkau thoff = iph->ihl * 4; 1667d208687SFelix Fietkau 1677d208687SFelix Fietkau if (ip_is_fragment(iph) || 1687d208687SFelix Fietkau unlikely(ip_has_options(thoff))) 1697d208687SFelix Fietkau return -1; 1707d208687SFelix Fietkau 171793d5d61SPablo Neira Ayuso switch (iph->protocol) { 172793d5d61SPablo Neira Ayuso case IPPROTO_TCP: 173*2fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct tcphdr); 174793d5d61SPablo Neira Ayuso break; 175793d5d61SPablo Neira Ayuso case IPPROTO_UDP: 176*2fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct udphdr); 177793d5d61SPablo Neira Ayuso break; 178793d5d61SPablo Neira Ayuso default: 1797d208687SFelix Fietkau return -1; 180793d5d61SPablo Neira Ayuso } 1817d208687SFelix Fietkau 18233cc3c0cSTaehee Yoo if (iph->ttl <= 1) 18333cc3c0cSTaehee Yoo return -1; 18433cc3c0cSTaehee Yoo 1857d208687SFelix Fietkau thoff = iph->ihl * 4; 186*2fc11745SPablo Neira Ayuso if (!pskb_may_pull(skb, thoff + *hdrsize)) 1877d208687SFelix Fietkau return -1; 1887d208687SFelix Fietkau 18941e9ec5aSHaishuang Yan iph = ip_hdr(skb); 1907d208687SFelix Fietkau ports = (struct flow_ports *)(skb_network_header(skb) + thoff); 1917d208687SFelix Fietkau 1927d208687SFelix Fietkau tuple->src_v4.s_addr = iph->saddr; 1937d208687SFelix Fietkau tuple->dst_v4.s_addr = iph->daddr; 1947d208687SFelix Fietkau tuple->src_port = ports->source; 1957d208687SFelix Fietkau tuple->dst_port = ports->dest; 1967d208687SFelix Fietkau tuple->l3proto = AF_INET; 1977d208687SFelix Fietkau tuple->l4proto = iph->protocol; 1987d208687SFelix Fietkau tuple->iifidx = dev->ifindex; 1997d208687SFelix Fietkau 2007d208687SFelix Fietkau return 0; 2017d208687SFelix Fietkau } 2027d208687SFelix Fietkau 2037d208687SFelix Fietkau /* Based on ip_exceeds_mtu(). */ 2047d208687SFelix Fietkau static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu) 2057d208687SFelix Fietkau { 2067d208687SFelix Fietkau if (skb->len <= mtu) 2077d208687SFelix Fietkau return false; 2087d208687SFelix Fietkau 2097d208687SFelix Fietkau if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 2107d208687SFelix Fietkau return false; 2117d208687SFelix Fietkau 2127d208687SFelix Fietkau return true; 2137d208687SFelix Fietkau } 2147d208687SFelix Fietkau 215589b474aSFlorian Westphal static int nf_flow_offload_dst_check(struct dst_entry *dst) 216589b474aSFlorian Westphal { 217589b474aSFlorian Westphal if (unlikely(dst_xfrm(dst))) 218589b474aSFlorian Westphal return dst_check(dst, 0) ? 0 : -1; 219589b474aSFlorian Westphal 220589b474aSFlorian Westphal return 0; 221589b474aSFlorian Westphal } 222589b474aSFlorian Westphal 223589b474aSFlorian Westphal static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb, 224589b474aSFlorian Westphal const struct nf_hook_state *state, 225589b474aSFlorian Westphal struct dst_entry *dst) 226589b474aSFlorian Westphal { 227589b474aSFlorian Westphal skb_orphan(skb); 228589b474aSFlorian Westphal skb_dst_set_noref(skb, dst); 229589b474aSFlorian Westphal dst_output(state->net, state->sk, skb); 230589b474aSFlorian Westphal return NF_STOLEN; 231589b474aSFlorian Westphal } 232589b474aSFlorian Westphal 2337d208687SFelix Fietkau unsigned int 2347d208687SFelix Fietkau nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, 2357d208687SFelix Fietkau const struct nf_hook_state *state) 2367d208687SFelix Fietkau { 2377d208687SFelix Fietkau struct flow_offload_tuple_rhash *tuplehash; 2387d208687SFelix Fietkau struct nf_flowtable *flow_table = priv; 2397d208687SFelix Fietkau struct flow_offload_tuple tuple = {}; 2407d208687SFelix Fietkau enum flow_offload_tuple_dir dir; 2417d208687SFelix Fietkau struct flow_offload *flow; 2427d208687SFelix Fietkau struct net_device *outdev; 2432a79fd39SJason A. Donenfeld struct rtable *rt; 244b6f27d32SFelix Fietkau unsigned int thoff; 2457d208687SFelix Fietkau struct iphdr *iph; 2467d208687SFelix Fietkau __be32 nexthop; 247*2fc11745SPablo Neira Ayuso u32 hdrsize; 2487d208687SFelix Fietkau 2497d208687SFelix Fietkau if (skb->protocol != htons(ETH_P_IP)) 2507d208687SFelix Fietkau return NF_ACCEPT; 2517d208687SFelix Fietkau 252*2fc11745SPablo Neira Ayuso if (nf_flow_tuple_ip(skb, state->in, &tuple, &hdrsize) < 0) 2537d208687SFelix Fietkau return NF_ACCEPT; 2547d208687SFelix Fietkau 2557d208687SFelix Fietkau tuplehash = flow_offload_lookup(flow_table, &tuple); 2567d208687SFelix Fietkau if (tuplehash == NULL) 2577d208687SFelix Fietkau return NF_ACCEPT; 2587d208687SFelix Fietkau 2597d208687SFelix Fietkau dir = tuplehash->tuple.dir; 2607d208687SFelix Fietkau flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 2612a79fd39SJason A. Donenfeld rt = (struct rtable *)flow->tuplehash[dir].tuple.dst_cache; 262227e1e4dSPablo Neira Ayuso outdev = rt->dst.dev; 2637d208687SFelix Fietkau 264e75b3e1cSFlorian Westphal if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu))) 2657d208687SFelix Fietkau return NF_ACCEPT; 2667d208687SFelix Fietkau 267*2fc11745SPablo Neira Ayuso iph = ip_hdr(skb); 268*2fc11745SPablo Neira Ayuso thoff = iph->ihl * 4; 269*2fc11745SPablo Neira Ayuso if (skb_try_make_writable(skb, thoff + hdrsize)) 2707d208687SFelix Fietkau return NF_DROP; 2717d208687SFelix Fietkau 272*2fc11745SPablo Neira Ayuso iph = ip_hdr(skb); 273*2fc11745SPablo Neira Ayuso if (nf_flow_state_check(flow, iph->protocol, skb, thoff)) 274b6f27d32SFelix Fietkau return NF_ACCEPT; 275b6f27d32SFelix Fietkau 2768b3646d6SPaul Blakey flow_offload_refresh(flow_table, flow); 277f698fe40SPablo Neira Ayuso 278589b474aSFlorian Westphal if (nf_flow_offload_dst_check(&rt->dst)) { 279589b474aSFlorian Westphal flow_offload_teardown(flow); 280589b474aSFlorian Westphal return NF_ACCEPT; 281589b474aSFlorian Westphal } 282589b474aSFlorian Westphal 283*2fc11745SPablo Neira Ayuso if (nf_flow_nat_ip(flow, skb, thoff, dir, iph) < 0) 2847d208687SFelix Fietkau return NF_DROP; 2857d208687SFelix Fietkau 2867d208687SFelix Fietkau ip_decrease_ttl(iph); 287de20900fSFlorian Westphal skb->tstamp = 0; 2887d208687SFelix Fietkau 28953c2b289SPablo Neira Ayuso if (flow_table->flags & NF_FLOWTABLE_COUNTER) 29053c2b289SPablo Neira Ayuso nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len); 29153c2b289SPablo Neira Ayuso 292589b474aSFlorian Westphal if (unlikely(dst_xfrm(&rt->dst))) { 293589b474aSFlorian Westphal memset(skb->cb, 0, sizeof(struct inet_skb_parm)); 294589b474aSFlorian Westphal IPCB(skb)->iif = skb->dev->ifindex; 295589b474aSFlorian Westphal IPCB(skb)->flags = IPSKB_FORWARDED; 296589b474aSFlorian Westphal return nf_flow_xmit_xfrm(skb, state, &rt->dst); 297589b474aSFlorian Westphal } 298589b474aSFlorian Westphal 2997d208687SFelix Fietkau skb->dev = outdev; 3007d208687SFelix Fietkau nexthop = rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr); 3012a79fd39SJason A. Donenfeld skb_dst_set_noref(skb, &rt->dst); 3027d208687SFelix Fietkau neigh_xmit(NEIGH_ARP_TABLE, outdev, &nexthop, skb); 3037d208687SFelix Fietkau 3047d208687SFelix Fietkau return NF_STOLEN; 3057d208687SFelix Fietkau } 3067d208687SFelix Fietkau EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook); 307a908fdecSFelix Fietkau 308a908fdecSFelix Fietkau static int nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff, 309a908fdecSFelix Fietkau struct in6_addr *addr, 310a908fdecSFelix Fietkau struct in6_addr *new_addr) 311a908fdecSFelix Fietkau { 312a908fdecSFelix Fietkau struct tcphdr *tcph; 313a908fdecSFelix Fietkau 314a908fdecSFelix Fietkau tcph = (void *)(skb_network_header(skb) + thoff); 315a908fdecSFelix Fietkau inet_proto_csum_replace16(&tcph->check, skb, addr->s6_addr32, 316a908fdecSFelix Fietkau new_addr->s6_addr32, true); 317a908fdecSFelix Fietkau 318a908fdecSFelix Fietkau return 0; 319a908fdecSFelix Fietkau } 320a908fdecSFelix Fietkau 321a908fdecSFelix Fietkau static int nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff, 322a908fdecSFelix Fietkau struct in6_addr *addr, 323a908fdecSFelix Fietkau struct in6_addr *new_addr) 324a908fdecSFelix Fietkau { 325a908fdecSFelix Fietkau struct udphdr *udph; 326a908fdecSFelix Fietkau 327a908fdecSFelix Fietkau udph = (void *)(skb_network_header(skb) + thoff); 328a908fdecSFelix Fietkau if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { 329a908fdecSFelix Fietkau inet_proto_csum_replace16(&udph->check, skb, addr->s6_addr32, 330a908fdecSFelix Fietkau new_addr->s6_addr32, true); 331a908fdecSFelix Fietkau if (!udph->check) 332a908fdecSFelix Fietkau udph->check = CSUM_MANGLED_0; 333a908fdecSFelix Fietkau } 334a908fdecSFelix Fietkau 335a908fdecSFelix Fietkau return 0; 336a908fdecSFelix Fietkau } 337a908fdecSFelix Fietkau 338a908fdecSFelix Fietkau static int nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h, 339a908fdecSFelix Fietkau unsigned int thoff, struct in6_addr *addr, 340a908fdecSFelix Fietkau struct in6_addr *new_addr) 341a908fdecSFelix Fietkau { 342a908fdecSFelix Fietkau switch (ip6h->nexthdr) { 343a908fdecSFelix Fietkau case IPPROTO_TCP: 344a908fdecSFelix Fietkau if (nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr) < 0) 345a908fdecSFelix Fietkau return NF_DROP; 346a908fdecSFelix Fietkau break; 347a908fdecSFelix Fietkau case IPPROTO_UDP: 348a908fdecSFelix Fietkau if (nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr) < 0) 349a908fdecSFelix Fietkau return NF_DROP; 350a908fdecSFelix Fietkau break; 351a908fdecSFelix Fietkau } 352a908fdecSFelix Fietkau 353a908fdecSFelix Fietkau return 0; 354a908fdecSFelix Fietkau } 355a908fdecSFelix Fietkau 356a908fdecSFelix Fietkau static int nf_flow_snat_ipv6(const struct flow_offload *flow, 357a908fdecSFelix Fietkau struct sk_buff *skb, struct ipv6hdr *ip6h, 358a908fdecSFelix Fietkau unsigned int thoff, 359a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 360a908fdecSFelix Fietkau { 361a908fdecSFelix Fietkau struct in6_addr addr, new_addr; 362a908fdecSFelix Fietkau 363a908fdecSFelix Fietkau switch (dir) { 364a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 365a908fdecSFelix Fietkau addr = ip6h->saddr; 366a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6; 367a908fdecSFelix Fietkau ip6h->saddr = new_addr; 368a908fdecSFelix Fietkau break; 369a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 370a908fdecSFelix Fietkau addr = ip6h->daddr; 371a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6; 372a908fdecSFelix Fietkau ip6h->daddr = new_addr; 373a908fdecSFelix Fietkau break; 374a908fdecSFelix Fietkau default: 375a908fdecSFelix Fietkau return -1; 376a908fdecSFelix Fietkau } 377a908fdecSFelix Fietkau 378a908fdecSFelix Fietkau return nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr); 379a908fdecSFelix Fietkau } 380a908fdecSFelix Fietkau 381a908fdecSFelix Fietkau static int nf_flow_dnat_ipv6(const struct flow_offload *flow, 382a908fdecSFelix Fietkau struct sk_buff *skb, struct ipv6hdr *ip6h, 383a908fdecSFelix Fietkau unsigned int thoff, 384a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir) 385a908fdecSFelix Fietkau { 386a908fdecSFelix Fietkau struct in6_addr addr, new_addr; 387a908fdecSFelix Fietkau 388a908fdecSFelix Fietkau switch (dir) { 389a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_ORIGINAL: 390a908fdecSFelix Fietkau addr = ip6h->daddr; 391a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6; 392a908fdecSFelix Fietkau ip6h->daddr = new_addr; 393a908fdecSFelix Fietkau break; 394a908fdecSFelix Fietkau case FLOW_OFFLOAD_DIR_REPLY: 395a908fdecSFelix Fietkau addr = ip6h->saddr; 396a908fdecSFelix Fietkau new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6; 397a908fdecSFelix Fietkau ip6h->saddr = new_addr; 398a908fdecSFelix Fietkau break; 399a908fdecSFelix Fietkau default: 400a908fdecSFelix Fietkau return -1; 401a908fdecSFelix Fietkau } 402a908fdecSFelix Fietkau 403a908fdecSFelix Fietkau return nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr); 404a908fdecSFelix Fietkau } 405a908fdecSFelix Fietkau 406a908fdecSFelix Fietkau static int nf_flow_nat_ipv6(const struct flow_offload *flow, 407a908fdecSFelix Fietkau struct sk_buff *skb, 408*2fc11745SPablo Neira Ayuso enum flow_offload_tuple_dir dir, 409*2fc11745SPablo Neira Ayuso struct ipv6hdr *ip6h) 410a908fdecSFelix Fietkau { 411a908fdecSFelix Fietkau unsigned int thoff = sizeof(*ip6h); 412a908fdecSFelix Fietkau 413355a8b13SPablo Neira Ayuso if (test_bit(NF_FLOW_SNAT, &flow->flags) && 414a908fdecSFelix Fietkau (nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir) < 0 || 415*2fc11745SPablo Neira Ayuso nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir) < 0)) 416a908fdecSFelix Fietkau return -1; 41761abaf02SHaishuang Yan 418355a8b13SPablo Neira Ayuso if (test_bit(NF_FLOW_DNAT, &flow->flags) && 419a908fdecSFelix Fietkau (nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir) < 0 || 420*2fc11745SPablo Neira Ayuso nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir) < 0)) 421a908fdecSFelix Fietkau return -1; 422a908fdecSFelix Fietkau 423a908fdecSFelix Fietkau return 0; 424a908fdecSFelix Fietkau } 425a908fdecSFelix Fietkau 426a908fdecSFelix Fietkau static int nf_flow_tuple_ipv6(struct sk_buff *skb, const struct net_device *dev, 427*2fc11745SPablo Neira Ayuso struct flow_offload_tuple *tuple, u32 *hdrsize) 428a908fdecSFelix Fietkau { 429a908fdecSFelix Fietkau struct flow_ports *ports; 430a908fdecSFelix Fietkau struct ipv6hdr *ip6h; 431*2fc11745SPablo Neira Ayuso unsigned int thoff; 432a908fdecSFelix Fietkau 433a908fdecSFelix Fietkau if (!pskb_may_pull(skb, sizeof(*ip6h))) 434a908fdecSFelix Fietkau return -1; 435a908fdecSFelix Fietkau 436a908fdecSFelix Fietkau ip6h = ipv6_hdr(skb); 437a908fdecSFelix Fietkau 438793d5d61SPablo Neira Ayuso switch (ip6h->nexthdr) { 439793d5d61SPablo Neira Ayuso case IPPROTO_TCP: 440*2fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct tcphdr); 441793d5d61SPablo Neira Ayuso break; 442793d5d61SPablo Neira Ayuso case IPPROTO_UDP: 443*2fc11745SPablo Neira Ayuso *hdrsize = sizeof(struct udphdr); 444793d5d61SPablo Neira Ayuso break; 445793d5d61SPablo Neira Ayuso default: 446a908fdecSFelix Fietkau return -1; 447793d5d61SPablo Neira Ayuso } 448a908fdecSFelix Fietkau 44933cc3c0cSTaehee Yoo if (ip6h->hop_limit <= 1) 45033cc3c0cSTaehee Yoo return -1; 45133cc3c0cSTaehee Yoo 452a908fdecSFelix Fietkau thoff = sizeof(*ip6h); 453*2fc11745SPablo Neira Ayuso if (!pskb_may_pull(skb, thoff + *hdrsize)) 454a908fdecSFelix Fietkau return -1; 455a908fdecSFelix Fietkau 45641e9ec5aSHaishuang Yan ip6h = ipv6_hdr(skb); 457a908fdecSFelix Fietkau ports = (struct flow_ports *)(skb_network_header(skb) + thoff); 458a908fdecSFelix Fietkau 459a908fdecSFelix Fietkau tuple->src_v6 = ip6h->saddr; 460a908fdecSFelix Fietkau tuple->dst_v6 = ip6h->daddr; 461a908fdecSFelix Fietkau tuple->src_port = ports->source; 462a908fdecSFelix Fietkau tuple->dst_port = ports->dest; 463a908fdecSFelix Fietkau tuple->l3proto = AF_INET6; 464a908fdecSFelix Fietkau tuple->l4proto = ip6h->nexthdr; 465a908fdecSFelix Fietkau tuple->iifidx = dev->ifindex; 466a908fdecSFelix Fietkau 467a908fdecSFelix Fietkau return 0; 468a908fdecSFelix Fietkau } 469a908fdecSFelix Fietkau 470a908fdecSFelix Fietkau unsigned int 471a908fdecSFelix Fietkau nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, 472a908fdecSFelix Fietkau const struct nf_hook_state *state) 473a908fdecSFelix Fietkau { 474a908fdecSFelix Fietkau struct flow_offload_tuple_rhash *tuplehash; 475a908fdecSFelix Fietkau struct nf_flowtable *flow_table = priv; 476a908fdecSFelix Fietkau struct flow_offload_tuple tuple = {}; 477a908fdecSFelix Fietkau enum flow_offload_tuple_dir dir; 4789b1c1ef1SNicolas Dichtel const struct in6_addr *nexthop; 479a908fdecSFelix Fietkau struct flow_offload *flow; 480a908fdecSFelix Fietkau struct net_device *outdev; 481a908fdecSFelix Fietkau struct ipv6hdr *ip6h; 482a908fdecSFelix Fietkau struct rt6_info *rt; 483*2fc11745SPablo Neira Ayuso u32 hdrsize; 484a908fdecSFelix Fietkau 485a908fdecSFelix Fietkau if (skb->protocol != htons(ETH_P_IPV6)) 486a908fdecSFelix Fietkau return NF_ACCEPT; 487a908fdecSFelix Fietkau 488*2fc11745SPablo Neira Ayuso if (nf_flow_tuple_ipv6(skb, state->in, &tuple, &hdrsize) < 0) 489a908fdecSFelix Fietkau return NF_ACCEPT; 490a908fdecSFelix Fietkau 491a908fdecSFelix Fietkau tuplehash = flow_offload_lookup(flow_table, &tuple); 492a908fdecSFelix Fietkau if (tuplehash == NULL) 493a908fdecSFelix Fietkau return NF_ACCEPT; 494a908fdecSFelix Fietkau 495a908fdecSFelix Fietkau dir = tuplehash->tuple.dir; 496a908fdecSFelix Fietkau flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); 497a908fdecSFelix Fietkau rt = (struct rt6_info *)flow->tuplehash[dir].tuple.dst_cache; 498227e1e4dSPablo Neira Ayuso outdev = rt->dst.dev; 499a908fdecSFelix Fietkau 500a908fdecSFelix Fietkau if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu))) 501a908fdecSFelix Fietkau return NF_ACCEPT; 502a908fdecSFelix Fietkau 50333894c36SFelix Fietkau if (nf_flow_state_check(flow, ipv6_hdr(skb)->nexthdr, skb, 50433894c36SFelix Fietkau sizeof(*ip6h))) 505b6f27d32SFelix Fietkau return NF_ACCEPT; 506b6f27d32SFelix Fietkau 5078b3646d6SPaul Blakey flow_offload_refresh(flow_table, flow); 508f698fe40SPablo Neira Ayuso 509589b474aSFlorian Westphal if (nf_flow_offload_dst_check(&rt->dst)) { 510589b474aSFlorian Westphal flow_offload_teardown(flow); 511589b474aSFlorian Westphal return NF_ACCEPT; 512589b474aSFlorian Westphal } 513589b474aSFlorian Westphal 514*2fc11745SPablo Neira Ayuso if (skb_try_make_writable(skb, sizeof(*ip6h) + hdrsize)) 515a908fdecSFelix Fietkau return NF_DROP; 516a908fdecSFelix Fietkau 517a908fdecSFelix Fietkau ip6h = ipv6_hdr(skb); 518*2fc11745SPablo Neira Ayuso if (nf_flow_nat_ipv6(flow, skb, dir, ip6h) < 0) 519*2fc11745SPablo Neira Ayuso return NF_DROP; 520*2fc11745SPablo Neira Ayuso 521a908fdecSFelix Fietkau ip6h->hop_limit--; 522de20900fSFlorian Westphal skb->tstamp = 0; 523a908fdecSFelix Fietkau 52453c2b289SPablo Neira Ayuso if (flow_table->flags & NF_FLOWTABLE_COUNTER) 52553c2b289SPablo Neira Ayuso nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len); 52653c2b289SPablo Neira Ayuso 527589b474aSFlorian Westphal if (unlikely(dst_xfrm(&rt->dst))) { 528589b474aSFlorian Westphal memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); 529589b474aSFlorian Westphal IP6CB(skb)->iif = skb->dev->ifindex; 530589b474aSFlorian Westphal IP6CB(skb)->flags = IP6SKB_FORWARDED; 531589b474aSFlorian Westphal return nf_flow_xmit_xfrm(skb, state, &rt->dst); 532589b474aSFlorian Westphal } 533589b474aSFlorian Westphal 534a908fdecSFelix Fietkau skb->dev = outdev; 535a908fdecSFelix Fietkau nexthop = rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6); 5362a79fd39SJason A. Donenfeld skb_dst_set_noref(skb, &rt->dst); 537a908fdecSFelix Fietkau neigh_xmit(NEIGH_ND_TABLE, outdev, nexthop, skb); 538a908fdecSFelix Fietkau 539a908fdecSFelix Fietkau return NF_STOLEN; 540a908fdecSFelix Fietkau } 541a908fdecSFelix Fietkau EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook); 542