12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
2406ef77cSHerbert Xu /*
3406ef77cSHerbert Xu * xfrm_output.c - Common IPsec encapsulation code.
4406ef77cSHerbert Xu *
5406ef77cSHerbert Xu * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
6406ef77cSHerbert Xu */
7406ef77cSHerbert Xu
8406ef77cSHerbert Xu #include <linux/errno.h>
9406ef77cSHerbert Xu #include <linux/module.h>
10406ef77cSHerbert Xu #include <linux/netdevice.h>
11862b82c6SHerbert Xu #include <linux/netfilter.h>
12406ef77cSHerbert Xu #include <linux/skbuff.h>
135a0e3ad6STejun Heo #include <linux/slab.h>
14406ef77cSHerbert Xu #include <linux/spinlock.h>
15406ef77cSHerbert Xu #include <net/dst.h>
16d457a0e3SEric Dumazet #include <net/gso.h>
176d64be3dSFlorian Westphal #include <net/icmp.h>
181de70830SFlorian Westphal #include <net/inet_ecn.h>
19406ef77cSHerbert Xu #include <net/xfrm.h>
20406ef77cSHerbert Xu
21f3075f48SFlorian Westphal #if IS_ENABLED(CONFIG_IPV6)
22f3075f48SFlorian Westphal #include <net/ip6_route.h>
23f3075f48SFlorian Westphal #include <net/ipv6_stubs.h>
24f3075f48SFlorian Westphal #endif
25f3075f48SFlorian Westphal
261de70830SFlorian Westphal #include "xfrm_inout.h"
271de70830SFlorian Westphal
280c4b51f0SEric W. Biederman static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb);
290c620e97SFlorian Westphal static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb);
30c6581a45SHerbert Xu
xfrm_skb_check_space(struct sk_buff * skb)3126b2072eSSteffen Klassert static int xfrm_skb_check_space(struct sk_buff *skb)
3283815deaSHerbert Xu {
33adf30907SEric Dumazet struct dst_entry *dst = skb_dst(skb);
34550ade84SHerbert Xu int nhead = dst->header_len + LL_RESERVED_SPACE(dst->dev)
3583815deaSHerbert Xu - skb_headroom(skb);
36f5184d26SJohannes Berg int ntail = dst->dev->needed_tailroom - skb_tailroom(skb);
3783815deaSHerbert Xu
38d01dbeb6SHerbert Xu if (nhead <= 0) {
39d01dbeb6SHerbert Xu if (ntail <= 0)
4083815deaSHerbert Xu return 0;
41d01dbeb6SHerbert Xu nhead = 0;
42d01dbeb6SHerbert Xu } else if (ntail < 0)
43d01dbeb6SHerbert Xu ntail = 0;
44d01dbeb6SHerbert Xu
45d01dbeb6SHerbert Xu return pskb_expand_head(skb, nhead, ntail, GFP_ATOMIC);
4683815deaSHerbert Xu }
4783815deaSHerbert Xu
489449c3cdSYing Xue /* Children define the path of the packet through the
499449c3cdSYing Xue * Linux networking. Thus, destinations are stackable.
509449c3cdSYing Xue */
519449c3cdSYing Xue
skb_dst_pop(struct sk_buff * skb)529449c3cdSYing Xue static struct dst_entry *skb_dst_pop(struct sk_buff *skb)
539449c3cdSYing Xue {
54b92cf4aaSDavid Miller struct dst_entry *child = dst_clone(xfrm_dst_child(skb_dst(skb)));
559449c3cdSYing Xue
569449c3cdSYing Xue skb_dst_drop(skb);
579449c3cdSYing Xue return child;
589449c3cdSYing Xue }
599449c3cdSYing Xue
600c620e97SFlorian Westphal /* Add encapsulation header.
610c620e97SFlorian Westphal *
620c620e97SFlorian Westphal * The IP header will be moved forward to make space for the encapsulation
630c620e97SFlorian Westphal * header.
640c620e97SFlorian Westphal */
xfrm4_transport_output(struct xfrm_state * x,struct sk_buff * skb)650c620e97SFlorian Westphal static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)
660c620e97SFlorian Westphal {
670c620e97SFlorian Westphal struct iphdr *iph = ip_hdr(skb);
680c620e97SFlorian Westphal int ihl = iph->ihl * 4;
690c620e97SFlorian Westphal
700c620e97SFlorian Westphal skb_set_inner_transport_header(skb, skb_transport_offset(skb));
710c620e97SFlorian Westphal
720c620e97SFlorian Westphal skb_set_network_header(skb, -x->props.header_len);
730c620e97SFlorian Westphal skb->mac_header = skb->network_header +
740c620e97SFlorian Westphal offsetof(struct iphdr, protocol);
750c620e97SFlorian Westphal skb->transport_header = skb->network_header + ihl;
760c620e97SFlorian Westphal __skb_pull(skb, ihl);
770c620e97SFlorian Westphal memmove(skb_network_header(skb), iph, ihl);
780c620e97SFlorian Westphal return 0;
790c620e97SFlorian Westphal }
800c620e97SFlorian Westphal
8137b9e7ebSFlorian Westphal #if IS_ENABLED(CONFIG_IPV6_MIP6)
mip6_rthdr_offset(struct sk_buff * skb,u8 ** nexthdr,int type)823ca5ca83SFlorian Westphal static int mip6_rthdr_offset(struct sk_buff *skb, u8 **nexthdr, int type)
8337b9e7ebSFlorian Westphal {
8437b9e7ebSFlorian Westphal const unsigned char *nh = skb_network_header(skb);
853ca5ca83SFlorian Westphal unsigned int offset = sizeof(struct ipv6hdr);
863ca5ca83SFlorian Westphal unsigned int packet_len;
8737b9e7ebSFlorian Westphal int found_rhdr = 0;
8837b9e7ebSFlorian Westphal
893ca5ca83SFlorian Westphal packet_len = skb_tail_pointer(skb) - nh;
9037b9e7ebSFlorian Westphal *nexthdr = &ipv6_hdr(skb)->nexthdr;
9137b9e7ebSFlorian Westphal
923ca5ca83SFlorian Westphal while (offset <= packet_len) {
933ca5ca83SFlorian Westphal struct ipv6_opt_hdr *exthdr;
943ca5ca83SFlorian Westphal
9537b9e7ebSFlorian Westphal switch (**nexthdr) {
9637b9e7ebSFlorian Westphal case NEXTHDR_HOP:
9737b9e7ebSFlorian Westphal break;
9837b9e7ebSFlorian Westphal case NEXTHDR_ROUTING:
993ca5ca83SFlorian Westphal if (type == IPPROTO_ROUTING && offset + 3 <= packet_len) {
1003ca5ca83SFlorian Westphal struct ipv6_rt_hdr *rt;
1013ca5ca83SFlorian Westphal
1023ca5ca83SFlorian Westphal rt = (struct ipv6_rt_hdr *)(nh + offset);
1033ca5ca83SFlorian Westphal if (rt->type != 0)
1043ca5ca83SFlorian Westphal return offset;
1053ca5ca83SFlorian Westphal }
10637b9e7ebSFlorian Westphal found_rhdr = 1;
10737b9e7ebSFlorian Westphal break;
10837b9e7ebSFlorian Westphal case NEXTHDR_DEST:
10937b9e7ebSFlorian Westphal /* HAO MUST NOT appear more than once.
11037b9e7ebSFlorian Westphal * XXX: It is better to try to find by the end of
11137b9e7ebSFlorian Westphal * XXX: packet if HAO exists.
11237b9e7ebSFlorian Westphal */
11337b9e7ebSFlorian Westphal if (ipv6_find_tlv(skb, offset, IPV6_TLV_HAO) >= 0) {
11437b9e7ebSFlorian Westphal net_dbg_ratelimited("mip6: hao exists already, override\n");
11537b9e7ebSFlorian Westphal return offset;
11637b9e7ebSFlorian Westphal }
11737b9e7ebSFlorian Westphal
11837b9e7ebSFlorian Westphal if (found_rhdr)
11937b9e7ebSFlorian Westphal return offset;
12037b9e7ebSFlorian Westphal
12137b9e7ebSFlorian Westphal break;
12237b9e7ebSFlorian Westphal default:
12337b9e7ebSFlorian Westphal return offset;
12437b9e7ebSFlorian Westphal }
12537b9e7ebSFlorian Westphal
1263ca5ca83SFlorian Westphal if (offset + sizeof(struct ipv6_opt_hdr) > packet_len)
1273ca5ca83SFlorian Westphal return -EINVAL;
1283ca5ca83SFlorian Westphal
1293ca5ca83SFlorian Westphal exthdr = (struct ipv6_opt_hdr *)(skb_network_header(skb) +
1303ca5ca83SFlorian Westphal offset);
13137b9e7ebSFlorian Westphal offset += ipv6_optlen(exthdr);
1323ca5ca83SFlorian Westphal if (offset > IPV6_MAXPLEN)
1333ca5ca83SFlorian Westphal return -EINVAL;
13437b9e7ebSFlorian Westphal *nexthdr = &exthdr->nexthdr;
13537b9e7ebSFlorian Westphal }
13637b9e7ebSFlorian Westphal
1373ca5ca83SFlorian Westphal return -EINVAL;
138848b18fbSFlorian Westphal }
13937b9e7ebSFlorian Westphal #endif
14037b9e7ebSFlorian Westphal
14130ad6a84SFlorian Westphal #if IS_ENABLED(CONFIG_IPV6)
xfrm6_hdr_offset(struct xfrm_state * x,struct sk_buff * skb,u8 ** prevhdr)1429acf4d3bSFlorian Westphal static int xfrm6_hdr_offset(struct xfrm_state *x, struct sk_buff *skb, u8 **prevhdr)
1439acf4d3bSFlorian Westphal {
14437b9e7ebSFlorian Westphal switch (x->type->proto) {
14537b9e7ebSFlorian Westphal #if IS_ENABLED(CONFIG_IPV6_MIP6)
14637b9e7ebSFlorian Westphal case IPPROTO_DSTOPTS:
147848b18fbSFlorian Westphal case IPPROTO_ROUTING:
1483ca5ca83SFlorian Westphal return mip6_rthdr_offset(skb, prevhdr, x->type->proto);
14937b9e7ebSFlorian Westphal #endif
15037b9e7ebSFlorian Westphal default:
15137b9e7ebSFlorian Westphal break;
15237b9e7ebSFlorian Westphal }
15337b9e7ebSFlorian Westphal
154d1002d24SFlorian Westphal return ip6_find_1stfragopt(skb, prevhdr);
1559acf4d3bSFlorian Westphal }
15630ad6a84SFlorian Westphal #endif
1579acf4d3bSFlorian Westphal
1580c620e97SFlorian Westphal /* Add encapsulation header.
1590c620e97SFlorian Westphal *
1600c620e97SFlorian Westphal * The IP header and mutable extension headers will be moved forward to make
1610c620e97SFlorian Westphal * space for the encapsulation header.
1620c620e97SFlorian Westphal */
xfrm6_transport_output(struct xfrm_state * x,struct sk_buff * skb)1630c620e97SFlorian Westphal static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)
1640c620e97SFlorian Westphal {
1654c145dceSFlorian Westphal #if IS_ENABLED(CONFIG_IPV6)
1660c620e97SFlorian Westphal struct ipv6hdr *iph;
1670c620e97SFlorian Westphal u8 *prevhdr;
1680c620e97SFlorian Westphal int hdr_len;
1690c620e97SFlorian Westphal
1700c620e97SFlorian Westphal iph = ipv6_hdr(skb);
1710c620e97SFlorian Westphal skb_set_inner_transport_header(skb, skb_transport_offset(skb));
1720c620e97SFlorian Westphal
1739acf4d3bSFlorian Westphal hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);
1740c620e97SFlorian Westphal if (hdr_len < 0)
1750c620e97SFlorian Westphal return hdr_len;
1760c620e97SFlorian Westphal skb_set_mac_header(skb,
1770c620e97SFlorian Westphal (prevhdr - x->props.header_len) - skb->data);
1780c620e97SFlorian Westphal skb_set_network_header(skb, -x->props.header_len);
1790c620e97SFlorian Westphal skb->transport_header = skb->network_header + hdr_len;
1800c620e97SFlorian Westphal __skb_pull(skb, hdr_len);
1810c620e97SFlorian Westphal memmove(ipv6_hdr(skb), iph, hdr_len);
1820c620e97SFlorian Westphal return 0;
1830c620e97SFlorian Westphal #else
1840c620e97SFlorian Westphal WARN_ON_ONCE(1);
1854c145dceSFlorian Westphal return -EAFNOSUPPORT;
1860c620e97SFlorian Westphal #endif
1870c620e97SFlorian Westphal }
1880c620e97SFlorian Westphal
1890c620e97SFlorian Westphal /* Add route optimization header space.
1900c620e97SFlorian Westphal *
1910c620e97SFlorian Westphal * The IP header and mutable extension headers will be moved forward to make
1920c620e97SFlorian Westphal * space for the route optimization header.
1930c620e97SFlorian Westphal */
xfrm6_ro_output(struct xfrm_state * x,struct sk_buff * skb)1940c620e97SFlorian Westphal static int xfrm6_ro_output(struct xfrm_state *x, struct sk_buff *skb)
1950c620e97SFlorian Westphal {
1964c145dceSFlorian Westphal #if IS_ENABLED(CONFIG_IPV6)
1970c620e97SFlorian Westphal struct ipv6hdr *iph;
1980c620e97SFlorian Westphal u8 *prevhdr;
1990c620e97SFlorian Westphal int hdr_len;
2000c620e97SFlorian Westphal
2010c620e97SFlorian Westphal iph = ipv6_hdr(skb);
2020c620e97SFlorian Westphal
2039acf4d3bSFlorian Westphal hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr);
2040c620e97SFlorian Westphal if (hdr_len < 0)
2050c620e97SFlorian Westphal return hdr_len;
2060c620e97SFlorian Westphal skb_set_mac_header(skb,
2070c620e97SFlorian Westphal (prevhdr - x->props.header_len) - skb->data);
2080c620e97SFlorian Westphal skb_set_network_header(skb, -x->props.header_len);
2090c620e97SFlorian Westphal skb->transport_header = skb->network_header + hdr_len;
2100c620e97SFlorian Westphal __skb_pull(skb, hdr_len);
2110c620e97SFlorian Westphal memmove(ipv6_hdr(skb), iph, hdr_len);
2120c620e97SFlorian Westphal
2130c620e97SFlorian Westphal return 0;
2140c620e97SFlorian Westphal #else
2150c620e97SFlorian Westphal WARN_ON_ONCE(1);
2164c145dceSFlorian Westphal return -EAFNOSUPPORT;
2170c620e97SFlorian Westphal #endif
2180c620e97SFlorian Westphal }
2190c620e97SFlorian Westphal
2201de70830SFlorian Westphal /* Add encapsulation header.
2211de70830SFlorian Westphal *
2221de70830SFlorian Westphal * The top IP header will be constructed per draft-nikander-esp-beet-mode-06.txt.
2231de70830SFlorian Westphal */
xfrm4_beet_encap_add(struct xfrm_state * x,struct sk_buff * skb)2241de70830SFlorian Westphal static int xfrm4_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
2251de70830SFlorian Westphal {
2261de70830SFlorian Westphal struct ip_beet_phdr *ph;
2271de70830SFlorian Westphal struct iphdr *top_iph;
2281de70830SFlorian Westphal int hdrlen, optlen;
2291de70830SFlorian Westphal
2301de70830SFlorian Westphal hdrlen = 0;
2311de70830SFlorian Westphal optlen = XFRM_MODE_SKB_CB(skb)->optlen;
2321de70830SFlorian Westphal if (unlikely(optlen))
2331de70830SFlorian Westphal hdrlen += IPV4_BEET_PHMAXLEN - (optlen & 4);
2341de70830SFlorian Westphal
2351de70830SFlorian Westphal skb_set_network_header(skb, -x->props.header_len - hdrlen +
2361de70830SFlorian Westphal (XFRM_MODE_SKB_CB(skb)->ihl - sizeof(*top_iph)));
2371de70830SFlorian Westphal if (x->sel.family != AF_INET6)
2381de70830SFlorian Westphal skb->network_header += IPV4_BEET_PHMAXLEN;
2391de70830SFlorian Westphal skb->mac_header = skb->network_header +
2401de70830SFlorian Westphal offsetof(struct iphdr, protocol);
2411de70830SFlorian Westphal skb->transport_header = skb->network_header + sizeof(*top_iph);
2421de70830SFlorian Westphal
2431de70830SFlorian Westphal xfrm4_beet_make_header(skb);
2441de70830SFlorian Westphal
2451de70830SFlorian Westphal ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdrlen);
2461de70830SFlorian Westphal
2471de70830SFlorian Westphal top_iph = ip_hdr(skb);
2481de70830SFlorian Westphal
2491de70830SFlorian Westphal if (unlikely(optlen)) {
2501de70830SFlorian Westphal if (WARN_ON(optlen < 0))
2511de70830SFlorian Westphal return -EINVAL;
2521de70830SFlorian Westphal
2531de70830SFlorian Westphal ph->padlen = 4 - (optlen & 4);
2541de70830SFlorian Westphal ph->hdrlen = optlen / 8;
2551de70830SFlorian Westphal ph->nexthdr = top_iph->protocol;
2561de70830SFlorian Westphal if (ph->padlen)
2571de70830SFlorian Westphal memset(ph + 1, IPOPT_NOP, ph->padlen);
2581de70830SFlorian Westphal
2591de70830SFlorian Westphal top_iph->protocol = IPPROTO_BEETPH;
2601de70830SFlorian Westphal top_iph->ihl = sizeof(struct iphdr) / 4;
2611de70830SFlorian Westphal }
2621de70830SFlorian Westphal
2631de70830SFlorian Westphal top_iph->saddr = x->props.saddr.a4;
2641de70830SFlorian Westphal top_iph->daddr = x->id.daddr.a4;
2651de70830SFlorian Westphal
2661de70830SFlorian Westphal return 0;
2671de70830SFlorian Westphal }
2681de70830SFlorian Westphal
2691de70830SFlorian Westphal /* Add encapsulation header.
2701de70830SFlorian Westphal *
2711de70830SFlorian Westphal * The top IP header will be constructed per RFC 2401.
2721de70830SFlorian Westphal */
xfrm4_tunnel_encap_add(struct xfrm_state * x,struct sk_buff * skb)2731de70830SFlorian Westphal static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
2741de70830SFlorian Westphal {
2756821ad87SMaciej Żenczykowski bool small_ipv6 = (skb->protocol == htons(ETH_P_IPV6)) && (skb->len <= IPV6_MIN_MTU);
2761de70830SFlorian Westphal struct dst_entry *dst = skb_dst(skb);
2771de70830SFlorian Westphal struct iphdr *top_iph;
2781de70830SFlorian Westphal int flags;
2791de70830SFlorian Westphal
2801de70830SFlorian Westphal skb_set_inner_network_header(skb, skb_network_offset(skb));
2811de70830SFlorian Westphal skb_set_inner_transport_header(skb, skb_transport_offset(skb));
2821de70830SFlorian Westphal
2831de70830SFlorian Westphal skb_set_network_header(skb, -x->props.header_len);
2841de70830SFlorian Westphal skb->mac_header = skb->network_header +
2851de70830SFlorian Westphal offsetof(struct iphdr, protocol);
2861de70830SFlorian Westphal skb->transport_header = skb->network_header + sizeof(*top_iph);
2871de70830SFlorian Westphal top_iph = ip_hdr(skb);
2881de70830SFlorian Westphal
2891de70830SFlorian Westphal top_iph->ihl = 5;
2901de70830SFlorian Westphal top_iph->version = 4;
2911de70830SFlorian Westphal
2921de70830SFlorian Westphal top_iph->protocol = xfrm_af2proto(skb_dst(skb)->ops->family);
2931de70830SFlorian Westphal
2941de70830SFlorian Westphal /* DS disclosing depends on XFRM_SA_XFLAG_DONT_ENCAP_DSCP */
2951de70830SFlorian Westphal if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
2961de70830SFlorian Westphal top_iph->tos = 0;
2971de70830SFlorian Westphal else
2981de70830SFlorian Westphal top_iph->tos = XFRM_MODE_SKB_CB(skb)->tos;
2991de70830SFlorian Westphal top_iph->tos = INET_ECN_encapsulate(top_iph->tos,
3001de70830SFlorian Westphal XFRM_MODE_SKB_CB(skb)->tos);
3011de70830SFlorian Westphal
3021de70830SFlorian Westphal flags = x->props.flags;
3031de70830SFlorian Westphal if (flags & XFRM_STATE_NOECN)
3041de70830SFlorian Westphal IP_ECN_clear(top_iph);
3051de70830SFlorian Westphal
3066821ad87SMaciej Żenczykowski top_iph->frag_off = (flags & XFRM_STATE_NOPMTUDISC) || small_ipv6 ?
3071de70830SFlorian Westphal 0 : (XFRM_MODE_SKB_CB(skb)->frag_off & htons(IP_DF));
3081de70830SFlorian Westphal
3091de70830SFlorian Westphal top_iph->ttl = ip4_dst_hoplimit(xfrm_dst_child(dst));
3101de70830SFlorian Westphal
3111de70830SFlorian Westphal top_iph->saddr = x->props.saddr.a4;
3121de70830SFlorian Westphal top_iph->daddr = x->id.daddr.a4;
3131de70830SFlorian Westphal ip_select_ident(dev_net(dst->dev), skb, NULL);
3141de70830SFlorian Westphal
3151de70830SFlorian Westphal return 0;
3161de70830SFlorian Westphal }
3171de70830SFlorian Westphal
3181de70830SFlorian Westphal #if IS_ENABLED(CONFIG_IPV6)
xfrm6_tunnel_encap_add(struct xfrm_state * x,struct sk_buff * skb)3191de70830SFlorian Westphal static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
3201de70830SFlorian Westphal {
3211de70830SFlorian Westphal struct dst_entry *dst = skb_dst(skb);
3221de70830SFlorian Westphal struct ipv6hdr *top_iph;
3231de70830SFlorian Westphal int dsfield;
3241de70830SFlorian Westphal
3251de70830SFlorian Westphal skb_set_inner_network_header(skb, skb_network_offset(skb));
3261de70830SFlorian Westphal skb_set_inner_transport_header(skb, skb_transport_offset(skb));
3271de70830SFlorian Westphal
3281de70830SFlorian Westphal skb_set_network_header(skb, -x->props.header_len);
3291de70830SFlorian Westphal skb->mac_header = skb->network_header +
3301de70830SFlorian Westphal offsetof(struct ipv6hdr, nexthdr);
3311de70830SFlorian Westphal skb->transport_header = skb->network_header + sizeof(*top_iph);
3321de70830SFlorian Westphal top_iph = ipv6_hdr(skb);
3331de70830SFlorian Westphal
3341de70830SFlorian Westphal top_iph->version = 6;
3351de70830SFlorian Westphal
3361de70830SFlorian Westphal memcpy(top_iph->flow_lbl, XFRM_MODE_SKB_CB(skb)->flow_lbl,
3371de70830SFlorian Westphal sizeof(top_iph->flow_lbl));
3381de70830SFlorian Westphal top_iph->nexthdr = xfrm_af2proto(skb_dst(skb)->ops->family);
3391de70830SFlorian Westphal
3401de70830SFlorian Westphal if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
3411de70830SFlorian Westphal dsfield = 0;
3421de70830SFlorian Westphal else
3431de70830SFlorian Westphal dsfield = XFRM_MODE_SKB_CB(skb)->tos;
3441de70830SFlorian Westphal dsfield = INET_ECN_encapsulate(dsfield, XFRM_MODE_SKB_CB(skb)->tos);
3451de70830SFlorian Westphal if (x->props.flags & XFRM_STATE_NOECN)
3461de70830SFlorian Westphal dsfield &= ~INET_ECN_MASK;
3471de70830SFlorian Westphal ipv6_change_dsfield(top_iph, 0, dsfield);
3481de70830SFlorian Westphal top_iph->hop_limit = ip6_dst_hoplimit(xfrm_dst_child(dst));
3491de70830SFlorian Westphal top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
3501de70830SFlorian Westphal top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
3511de70830SFlorian Westphal return 0;
3521de70830SFlorian Westphal }
3531de70830SFlorian Westphal
xfrm6_beet_encap_add(struct xfrm_state * x,struct sk_buff * skb)3541de70830SFlorian Westphal static int xfrm6_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
3551de70830SFlorian Westphal {
3561de70830SFlorian Westphal struct ipv6hdr *top_iph;
3571de70830SFlorian Westphal struct ip_beet_phdr *ph;
3581de70830SFlorian Westphal int optlen, hdr_len;
3591de70830SFlorian Westphal
3601de70830SFlorian Westphal hdr_len = 0;
3611de70830SFlorian Westphal optlen = XFRM_MODE_SKB_CB(skb)->optlen;
3621de70830SFlorian Westphal if (unlikely(optlen))
3631de70830SFlorian Westphal hdr_len += IPV4_BEET_PHMAXLEN - (optlen & 4);
3641de70830SFlorian Westphal
3651de70830SFlorian Westphal skb_set_network_header(skb, -x->props.header_len - hdr_len);
3661de70830SFlorian Westphal if (x->sel.family != AF_INET6)
3671de70830SFlorian Westphal skb->network_header += IPV4_BEET_PHMAXLEN;
3681de70830SFlorian Westphal skb->mac_header = skb->network_header +
3691de70830SFlorian Westphal offsetof(struct ipv6hdr, nexthdr);
3701de70830SFlorian Westphal skb->transport_header = skb->network_header + sizeof(*top_iph);
3711de70830SFlorian Westphal ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdr_len);
3721de70830SFlorian Westphal
3731de70830SFlorian Westphal xfrm6_beet_make_header(skb);
3741de70830SFlorian Westphal
3751de70830SFlorian Westphal top_iph = ipv6_hdr(skb);
3761de70830SFlorian Westphal if (unlikely(optlen)) {
3771de70830SFlorian Westphal if (WARN_ON(optlen < 0))
3781de70830SFlorian Westphal return -EINVAL;
3791de70830SFlorian Westphal
3801de70830SFlorian Westphal ph->padlen = 4 - (optlen & 4);
3811de70830SFlorian Westphal ph->hdrlen = optlen / 8;
3821de70830SFlorian Westphal ph->nexthdr = top_iph->nexthdr;
3831de70830SFlorian Westphal if (ph->padlen)
3841de70830SFlorian Westphal memset(ph + 1, IPOPT_NOP, ph->padlen);
3851de70830SFlorian Westphal
3861de70830SFlorian Westphal top_iph->nexthdr = IPPROTO_BEETPH;
3871de70830SFlorian Westphal }
3881de70830SFlorian Westphal
3891de70830SFlorian Westphal top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
3901de70830SFlorian Westphal top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
3911de70830SFlorian Westphal return 0;
3921de70830SFlorian Westphal }
3931de70830SFlorian Westphal #endif
3941de70830SFlorian Westphal
3951de70830SFlorian Westphal /* Add encapsulation header.
3961de70830SFlorian Westphal *
3971de70830SFlorian Westphal * On exit, the transport header will be set to the start of the
3981de70830SFlorian Westphal * encapsulation header to be filled in by x->type->output and the mac
3991de70830SFlorian Westphal * header will be set to the nextheader (protocol for IPv4) field of the
4001de70830SFlorian Westphal * extension header directly preceding the encapsulation header, or in
4011de70830SFlorian Westphal * its absence, that of the top IP header.
4021de70830SFlorian Westphal * The value of the network header will always point to the top IP header
4031de70830SFlorian Westphal * while skb->data will point to the payload.
4041de70830SFlorian Westphal */
xfrm4_prepare_output(struct xfrm_state * x,struct sk_buff * skb)4050c620e97SFlorian Westphal static int xfrm4_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
4060c620e97SFlorian Westphal {
4070c620e97SFlorian Westphal int err;
4080c620e97SFlorian Westphal
4090c620e97SFlorian Westphal err = xfrm_inner_extract_output(x, skb);
4100c620e97SFlorian Westphal if (err)
4110c620e97SFlorian Westphal return err;
4120c620e97SFlorian Westphal
4130c620e97SFlorian Westphal IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE;
4140c620e97SFlorian Westphal skb->protocol = htons(ETH_P_IP);
4150c620e97SFlorian Westphal
416f4796398SHerbert Xu switch (x->props.mode) {
4171de70830SFlorian Westphal case XFRM_MODE_BEET:
4181de70830SFlorian Westphal return xfrm4_beet_encap_add(x, skb);
4191de70830SFlorian Westphal case XFRM_MODE_TUNNEL:
4201de70830SFlorian Westphal return xfrm4_tunnel_encap_add(x, skb);
4211de70830SFlorian Westphal }
4221de70830SFlorian Westphal
4231de70830SFlorian Westphal WARN_ON_ONCE(1);
4241de70830SFlorian Westphal return -EOPNOTSUPP;
4250c620e97SFlorian Westphal }
4260c620e97SFlorian Westphal
xfrm6_prepare_output(struct xfrm_state * x,struct sk_buff * skb)4270c620e97SFlorian Westphal static int xfrm6_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
4280c620e97SFlorian Westphal {
4290c620e97SFlorian Westphal #if IS_ENABLED(CONFIG_IPV6)
4300c620e97SFlorian Westphal int err;
4310c620e97SFlorian Westphal
4320c620e97SFlorian Westphal err = xfrm_inner_extract_output(x, skb);
4330c620e97SFlorian Westphal if (err)
4340c620e97SFlorian Westphal return err;
4350c620e97SFlorian Westphal
4360c620e97SFlorian Westphal skb->ignore_df = 1;
4370c620e97SFlorian Westphal skb->protocol = htons(ETH_P_IPV6);
4380c620e97SFlorian Westphal
439f4796398SHerbert Xu switch (x->props.mode) {
4401de70830SFlorian Westphal case XFRM_MODE_BEET:
4411de70830SFlorian Westphal return xfrm6_beet_encap_add(x, skb);
4421de70830SFlorian Westphal case XFRM_MODE_TUNNEL:
4431de70830SFlorian Westphal return xfrm6_tunnel_encap_add(x, skb);
4441de70830SFlorian Westphal default:
4450c620e97SFlorian Westphal WARN_ON_ONCE(1);
4460c620e97SFlorian Westphal return -EOPNOTSUPP;
4471de70830SFlorian Westphal }
4480c620e97SFlorian Westphal #endif
4491de70830SFlorian Westphal WARN_ON_ONCE(1);
4501de70830SFlorian Westphal return -EAFNOSUPPORT;
4510c620e97SFlorian Westphal }
4520c620e97SFlorian Westphal
xfrm_outer_mode_output(struct xfrm_state * x,struct sk_buff * skb)4530c620e97SFlorian Westphal static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
4540c620e97SFlorian Westphal {
455f4796398SHerbert Xu switch (x->props.mode) {
4560c620e97SFlorian Westphal case XFRM_MODE_BEET:
4570c620e97SFlorian Westphal case XFRM_MODE_TUNNEL:
458f4796398SHerbert Xu if (x->props.family == AF_INET)
4590c620e97SFlorian Westphal return xfrm4_prepare_output(x, skb);
460f4796398SHerbert Xu if (x->props.family == AF_INET6)
4610c620e97SFlorian Westphal return xfrm6_prepare_output(x, skb);
4620c620e97SFlorian Westphal break;
4630c620e97SFlorian Westphal case XFRM_MODE_TRANSPORT:
464f4796398SHerbert Xu if (x->props.family == AF_INET)
4650c620e97SFlorian Westphal return xfrm4_transport_output(x, skb);
466f4796398SHerbert Xu if (x->props.family == AF_INET6)
4670c620e97SFlorian Westphal return xfrm6_transport_output(x, skb);
4680c620e97SFlorian Westphal break;
4690c620e97SFlorian Westphal case XFRM_MODE_ROUTEOPTIMIZATION:
470f4796398SHerbert Xu if (x->props.family == AF_INET6)
4710c620e97SFlorian Westphal return xfrm6_ro_output(x, skb);
4720c620e97SFlorian Westphal WARN_ON_ONCE(1);
4730c620e97SFlorian Westphal break;
4740c620e97SFlorian Westphal default:
4750c620e97SFlorian Westphal WARN_ON_ONCE(1);
4760c620e97SFlorian Westphal break;
4770c620e97SFlorian Westphal }
4780c620e97SFlorian Westphal
4790c620e97SFlorian Westphal return -EOPNOTSUPP;
4800c620e97SFlorian Westphal }
4810c620e97SFlorian Westphal
4820c620e97SFlorian Westphal #if IS_ENABLED(CONFIG_NET_PKTGEN)
pktgen_xfrm_outer_mode_output(struct xfrm_state * x,struct sk_buff * skb)4830c620e97SFlorian Westphal int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
4840c620e97SFlorian Westphal {
4850c620e97SFlorian Westphal return xfrm_outer_mode_output(x, skb);
4860c620e97SFlorian Westphal }
4870c620e97SFlorian Westphal EXPORT_SYMBOL_GPL(pktgen_xfrm_outer_mode_output);
4880c620e97SFlorian Westphal #endif
4890c620e97SFlorian Westphal
xfrm_output_one(struct sk_buff * skb,int err)490c6581a45SHerbert Xu static int xfrm_output_one(struct sk_buff *skb, int err)
491406ef77cSHerbert Xu {
492adf30907SEric Dumazet struct dst_entry *dst = skb_dst(skb);
493406ef77cSHerbert Xu struct xfrm_state *x = dst->xfrm;
494a6483b79SAlexey Dobriyan struct net *net = xs_net(x);
495406ef77cSHerbert Xu
496f8a70afaSLeon Romanovsky if (err <= 0 || x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
497c6581a45SHerbert Xu goto resume;
498406ef77cSHerbert Xu
499406ef77cSHerbert Xu do {
50026b2072eSSteffen Klassert err = xfrm_skb_check_space(skb);
501b15c4bcdSMasahide NAKAMURA if (err) {
50259c9940eSAlexey Dobriyan XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
503910ef70aSHerbert Xu goto error_nolock;
504b15c4bcdSMasahide NAKAMURA }
505910ef70aSHerbert Xu
5069b42c1f1SSteffen Klassert skb->mark = xfrm_smark_get(skb->mark, x);
507077fbac4SLorenzo Colitti
5080c620e97SFlorian Westphal err = xfrm_outer_mode_output(x, skb);
509b15c4bcdSMasahide NAKAMURA if (err) {
51059c9940eSAlexey Dobriyan XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
511910ef70aSHerbert Xu goto error_nolock;
512b15c4bcdSMasahide NAKAMURA }
513a2deb6d2SHerbert Xu
514406ef77cSHerbert Xu spin_lock_bh(&x->lock);
515bb65a9cbSLi RongQing
516bb65a9cbSLi RongQing if (unlikely(x->km.state != XFRM_STATE_VALID)) {
517bb65a9cbSLi RongQing XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEINVALID);
518497574c7STimo Teräs err = -EINVAL;
519fa8599dbSLi RongQing goto error;
520bb65a9cbSLi RongQing }
521bb65a9cbSLi RongQing
522910ef70aSHerbert Xu err = xfrm_state_check_expire(x);
523b15c4bcdSMasahide NAKAMURA if (err) {
52459c9940eSAlexey Dobriyan XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEEXPIRED);
525406ef77cSHerbert Xu goto error;
526b15c4bcdSMasahide NAKAMURA }
527406ef77cSHerbert Xu
528b5a1d1feSFlorian Westphal err = xfrm_replay_overflow(x, skb);
5299fdc4883SSteffen Klassert if (err) {
53059c9940eSAlexey Dobriyan XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATESEQERROR);
531e1af9f27SPaul Moore goto error;
532e1af9f27SPaul Moore }
533436a0a40SHerbert Xu
534406ef77cSHerbert Xu x->curlft.bytes += skb->len;
535406ef77cSHerbert Xu x->curlft.packets++;
536f7fe25a6SAntony Antony x->lastused = ktime_get_real_seconds();
537406ef77cSHerbert Xu
538406ef77cSHerbert Xu spin_unlock_bh(&x->lock);
539406ef77cSHerbert Xu
5403bc07321SSteffen Klassert skb_dst_force(skb);
5419e143793SSteffen Klassert if (!skb_dst(skb)) {
5429e143793SSteffen Klassert XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
543533555e5SWei Yongjun err = -EHOSTUNREACH;
5449e143793SSteffen Klassert goto error_nolock;
5459e143793SSteffen Klassert }
5463bc07321SSteffen Klassert
547d77e38e6SSteffen Klassert if (xfrm_offload(skb)) {
548d77e38e6SSteffen Klassert x->type_offload->encap(x, skb);
549d77e38e6SSteffen Klassert } else {
55073b9fc49SSteffen Klassert /* Inner headers are invalid now. */
55173b9fc49SSteffen Klassert skb->encapsulation = 0;
55273b9fc49SSteffen Klassert
553b7c6538cSHerbert Xu err = x->type->output(x, skb);
554fcb8c156SHerbert Xu if (err == -EINPROGRESS)
555ebd4687aSJean Sacren goto out;
556d77e38e6SSteffen Klassert }
557c6581a45SHerbert Xu
558c6581a45SHerbert Xu resume:
5590aa64774SMasahide NAKAMURA if (err) {
56059c9940eSAlexey Dobriyan XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR);
561b7c6538cSHerbert Xu goto error_nolock;
5620aa64774SMasahide NAKAMURA }
563b7c6538cSHerbert Xu
5648764ab2cSSteffen Klassert dst = skb_dst_pop(skb);
565adf30907SEric Dumazet if (!dst) {
56659c9940eSAlexey Dobriyan XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
567406ef77cSHerbert Xu err = -EHOSTUNREACH;
568406ef77cSHerbert Xu goto error_nolock;
569406ef77cSHerbert Xu }
570e433430aSSteffen Klassert skb_dst_set(skb, dst);
571406ef77cSHerbert Xu x = dst->xfrm;
572c9500d7bSFlorian Westphal } while (x && !(x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL));
573406ef77cSHerbert Xu
574ebd4687aSJean Sacren return 0;
575406ef77cSHerbert Xu
576406ef77cSHerbert Xu error:
577406ef77cSHerbert Xu spin_unlock_bh(&x->lock);
578862b82c6SHerbert Xu error_nolock:
579862b82c6SHerbert Xu kfree_skb(skb);
580ebd4687aSJean Sacren out:
581ebd4687aSJean Sacren return err;
582862b82c6SHerbert Xu }
583862b82c6SHerbert Xu
xfrm_output_resume(struct sock * sk,struct sk_buff * skb,int err)5849ab1265dSEvan Nimmo int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err)
585862b82c6SHerbert Xu {
58629a26a56SEric W. Biederman struct net *net = xs_net(skb_dst(skb)->xfrm);
587be10de0aSEric W. Biederman
588c6581a45SHerbert Xu while (likely((err = xfrm_output_one(skb, err)) == 0)) {
589895b5c9fSFlorian Westphal nf_reset_ct(skb);
590862b82c6SHerbert Xu
5919ab1265dSEvan Nimmo err = skb_dst(skb)->ops->local_out(net, sk, skb);
592862b82c6SHerbert Xu if (unlikely(err != 1))
593c6581a45SHerbert Xu goto out;
594862b82c6SHerbert Xu
595adf30907SEric Dumazet if (!skb_dst(skb)->xfrm)
5969ab1265dSEvan Nimmo return dst_output(net, sk, skb);
597862b82c6SHerbert Xu
598adf30907SEric Dumazet err = nf_hook(skb_dst(skb)->ops->family,
5999ab1265dSEvan Nimmo NF_INET_POST_ROUTING, net, sk, skb,
600adf30907SEric Dumazet NULL, skb_dst(skb)->dev, xfrm_output2);
601862b82c6SHerbert Xu if (unlikely(err != 1))
602c6581a45SHerbert Xu goto out;
603862b82c6SHerbert Xu }
604862b82c6SHerbert Xu
605c6581a45SHerbert Xu if (err == -EINPROGRESS)
606c6581a45SHerbert Xu err = 0;
607c6581a45SHerbert Xu
608c6581a45SHerbert Xu out:
609862b82c6SHerbert Xu return err;
610862b82c6SHerbert Xu }
611c6581a45SHerbert Xu EXPORT_SYMBOL_GPL(xfrm_output_resume);
612862b82c6SHerbert Xu
xfrm_output2(struct net * net,struct sock * sk,struct sk_buff * skb)6130c4b51f0SEric W. Biederman static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
614c6581a45SHerbert Xu {
6159ab1265dSEvan Nimmo return xfrm_output_resume(sk, skb, 1);
616c6581a45SHerbert Xu }
617c6581a45SHerbert Xu
xfrm_output_gso(struct net * net,struct sock * sk,struct sk_buff * skb)6180c4b51f0SEric W. Biederman static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb)
619862b82c6SHerbert Xu {
620c3b18e0dSJason A. Donenfeld struct sk_buff *segs, *nskb;
621862b82c6SHerbert Xu
622a08e7fd9SCambda Zhu BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_GSO_CB_OFFSET);
623a08e7fd9SCambda Zhu BUILD_BUG_ON(sizeof(*IP6CB(skb)) > SKB_GSO_CB_OFFSET);
624862b82c6SHerbert Xu segs = skb_gso_segment(skb, 0);
625862b82c6SHerbert Xu kfree_skb(skb);
626801678c5SHirofumi Nakagawa if (IS_ERR(segs))
627862b82c6SHerbert Xu return PTR_ERR(segs);
628330966e5SFlorian Westphal if (segs == NULL)
629330966e5SFlorian Westphal return -EINVAL;
630862b82c6SHerbert Xu
631c3b18e0dSJason A. Donenfeld skb_list_walk_safe(segs, segs, nskb) {
632862b82c6SHerbert Xu int err;
633862b82c6SHerbert Xu
634a8305bffSDavid S. Miller skb_mark_not_on_list(segs);
6350c4b51f0SEric W. Biederman err = xfrm_output2(net, sk, segs);
636862b82c6SHerbert Xu
637862b82c6SHerbert Xu if (unlikely(err)) {
63846cfd725SFlorian Westphal kfree_skb_list(nskb);
639862b82c6SHerbert Xu return err;
640862b82c6SHerbert Xu }
641c3b18e0dSJason A. Donenfeld }
642862b82c6SHerbert Xu
643862b82c6SHerbert Xu return 0;
644406ef77cSHerbert Xu }
645c6581a45SHerbert Xu
646fa453523SHuy Nguyen /* For partial checksum offload, the outer header checksum is calculated
647fa453523SHuy Nguyen * by software and the inner header checksum is calculated by hardware.
648fa453523SHuy Nguyen * This requires hardware to know the inner packet type to calculate
649fa453523SHuy Nguyen * the inner header checksum. Save inner ip protocol here to avoid
650fa453523SHuy Nguyen * traversing the packet in the vendor's xmit code.
65145a98ef4SRaed Salem * For IPsec tunnel mode save the ip protocol from the IP header of the
65245a98ef4SRaed Salem * plain text packet. Otherwise If the encap type is IPIP, just save
65345a98ef4SRaed Salem * skb->inner_ipproto in any other case get the ip protocol from the IP
65445a98ef4SRaed Salem * header.
655fa453523SHuy Nguyen */
xfrm_get_inner_ipproto(struct sk_buff * skb,struct xfrm_state * x)65645a98ef4SRaed Salem static void xfrm_get_inner_ipproto(struct sk_buff *skb, struct xfrm_state *x)
657fa453523SHuy Nguyen {
658fa453523SHuy Nguyen struct xfrm_offload *xo = xfrm_offload(skb);
659fa453523SHuy Nguyen const struct ethhdr *eth;
660fa453523SHuy Nguyen
661fa453523SHuy Nguyen if (!xo)
662fa453523SHuy Nguyen return;
663fa453523SHuy Nguyen
66445a98ef4SRaed Salem if (x->outer_mode.encap == XFRM_MODE_TUNNEL) {
66545a98ef4SRaed Salem switch (x->outer_mode.family) {
66645a98ef4SRaed Salem case AF_INET:
66745a98ef4SRaed Salem xo->inner_ipproto = ip_hdr(skb)->protocol;
66845a98ef4SRaed Salem break;
66945a98ef4SRaed Salem case AF_INET6:
67045a98ef4SRaed Salem xo->inner_ipproto = ipv6_hdr(skb)->nexthdr;
67145a98ef4SRaed Salem break;
67245a98ef4SRaed Salem default:
67345a98ef4SRaed Salem break;
67445a98ef4SRaed Salem }
67545a98ef4SRaed Salem
67645a98ef4SRaed Salem return;
67745a98ef4SRaed Salem }
67845a98ef4SRaed Salem
67945a98ef4SRaed Salem /* non-Tunnel Mode */
68045a98ef4SRaed Salem if (!skb->encapsulation)
68145a98ef4SRaed Salem return;
68245a98ef4SRaed Salem
683fa453523SHuy Nguyen if (skb->inner_protocol_type == ENCAP_TYPE_IPPROTO) {
684fa453523SHuy Nguyen xo->inner_ipproto = skb->inner_ipproto;
685fa453523SHuy Nguyen return;
686fa453523SHuy Nguyen }
687fa453523SHuy Nguyen
688fa453523SHuy Nguyen if (skb->inner_protocol_type != ENCAP_TYPE_ETHER)
689fa453523SHuy Nguyen return;
690fa453523SHuy Nguyen
691fa453523SHuy Nguyen eth = (struct ethhdr *)skb_inner_mac_header(skb);
692fa453523SHuy Nguyen
693fa453523SHuy Nguyen switch (ntohs(eth->h_proto)) {
694fa453523SHuy Nguyen case ETH_P_IPV6:
695fa453523SHuy Nguyen xo->inner_ipproto = inner_ipv6_hdr(skb)->nexthdr;
696fa453523SHuy Nguyen break;
697fa453523SHuy Nguyen case ETH_P_IP:
698fa453523SHuy Nguyen xo->inner_ipproto = inner_ip_hdr(skb)->protocol;
699fa453523SHuy Nguyen break;
700fa453523SHuy Nguyen }
701fa453523SHuy Nguyen }
702fa453523SHuy Nguyen
xfrm_output(struct sock * sk,struct sk_buff * skb)7037026b1ddSDavid Miller int xfrm_output(struct sock *sk, struct sk_buff *skb)
704c6581a45SHerbert Xu {
705adf30907SEric Dumazet struct net *net = dev_net(skb_dst(skb)->dev);
706d77e38e6SSteffen Klassert struct xfrm_state *x = skb_dst(skb)->xfrm;
707*52a85947SMike Yu int family;
708c6581a45SHerbert Xu int err;
709c6581a45SHerbert Xu
710*52a85947SMike Yu family = (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) ? x->outer_mode.family
711*52a85947SMike Yu : skb_dst(skb)->ops->family;
712*52a85947SMike Yu
713*52a85947SMike Yu switch (family) {
7142ab6096dSFlorian Westphal case AF_INET:
7152ab6096dSFlorian Westphal memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
7162ab6096dSFlorian Westphal IPCB(skb)->flags |= IPSKB_XFRM_TRANSFORMED;
7172ab6096dSFlorian Westphal break;
7182ab6096dSFlorian Westphal case AF_INET6:
7192ab6096dSFlorian Westphal memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
7202ab6096dSFlorian Westphal
7212ab6096dSFlorian Westphal IP6CB(skb)->flags |= IP6SKB_XFRM_TRANSFORMED;
7222ab6096dSFlorian Westphal break;
7232ab6096dSFlorian Westphal }
7242ab6096dSFlorian Westphal
725f8a70afaSLeon Romanovsky if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) {
726f8a70afaSLeon Romanovsky if (!xfrm_dev_offload_ok(skb, x)) {
727f8a70afaSLeon Romanovsky XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
728f8a70afaSLeon Romanovsky kfree_skb(skb);
729f8a70afaSLeon Romanovsky return -EHOSTUNREACH;
730f8a70afaSLeon Romanovsky }
731f8a70afaSLeon Romanovsky
732f8a70afaSLeon Romanovsky return xfrm_output_resume(sk, skb, 0);
733f8a70afaSLeon Romanovsky }
734f8a70afaSLeon Romanovsky
735d77e38e6SSteffen Klassert secpath_reset(skb);
736d77e38e6SSteffen Klassert
737d77e38e6SSteffen Klassert if (xfrm_dev_offload_ok(skb, x)) {
738d77e38e6SSteffen Klassert struct sec_path *sp;
739d77e38e6SSteffen Klassert
740a84e3f53SFlorian Westphal sp = secpath_set(skb);
741d77e38e6SSteffen Klassert if (!sp) {
742d77e38e6SSteffen Klassert XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
743d77e38e6SSteffen Klassert kfree_skb(skb);
744d77e38e6SSteffen Klassert return -ENOMEM;
745d77e38e6SSteffen Klassert }
746d77e38e6SSteffen Klassert
747d77e38e6SSteffen Klassert sp->olen++;
748a84e3f53SFlorian Westphal sp->xvec[sp->len++] = x;
749d77e38e6SSteffen Klassert xfrm_state_hold(x);
750d77e38e6SSteffen Klassert
75145a98ef4SRaed Salem xfrm_get_inner_ipproto(skb, x);
752fa453523SHuy Nguyen skb->encapsulation = 1;
753fa453523SHuy Nguyen
754d77e38e6SSteffen Klassert if (skb_is_gso(skb)) {
755a204aef9SXin Long if (skb->inner_protocol)
756a204aef9SXin Long return xfrm_output_gso(net, sk, skb);
757d77e38e6SSteffen Klassert
758a204aef9SXin Long skb_shinfo(skb)->gso_type |= SKB_GSO_ESP;
759a204aef9SXin Long goto out;
760d77e38e6SSteffen Klassert }
761d77e38e6SSteffen Klassert
762d77e38e6SSteffen Klassert if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)
763d77e38e6SSteffen Klassert goto out;
764a204aef9SXin Long } else {
765c6581a45SHerbert Xu if (skb_is_gso(skb))
7660c4b51f0SEric W. Biederman return xfrm_output_gso(net, sk, skb);
767a204aef9SXin Long }
768c6581a45SHerbert Xu
769c6581a45SHerbert Xu if (skb->ip_summed == CHECKSUM_PARTIAL) {
770c6581a45SHerbert Xu err = skb_checksum_help(skb);
771c6581a45SHerbert Xu if (err) {
77259c9940eSAlexey Dobriyan XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
773c6581a45SHerbert Xu kfree_skb(skb);
774c6581a45SHerbert Xu return err;
775c6581a45SHerbert Xu }
776c6581a45SHerbert Xu }
777c6581a45SHerbert Xu
778d77e38e6SSteffen Klassert out:
7790c4b51f0SEric W. Biederman return xfrm_output2(net, sk, skb);
780c6581a45SHerbert Xu }
781fc68086cSFabian Frederick EXPORT_SYMBOL_GPL(xfrm_output);
782df9dcb45SKazunori MIYAZAWA
xfrm4_tunnel_check_size(struct sk_buff * skb)7836d64be3dSFlorian Westphal static int xfrm4_tunnel_check_size(struct sk_buff *skb)
7846d64be3dSFlorian Westphal {
7856d64be3dSFlorian Westphal int mtu, ret = 0;
7866d64be3dSFlorian Westphal
7876d64be3dSFlorian Westphal if (IPCB(skb)->flags & IPSKB_XFRM_TUNNEL_SIZE)
7886d64be3dSFlorian Westphal goto out;
7896d64be3dSFlorian Westphal
7906d64be3dSFlorian Westphal if (!(ip_hdr(skb)->frag_off & htons(IP_DF)) || skb->ignore_df)
7916d64be3dSFlorian Westphal goto out;
7926d64be3dSFlorian Westphal
7936d64be3dSFlorian Westphal mtu = dst_mtu(skb_dst(skb));
7946d64be3dSFlorian Westphal if ((!skb_is_gso(skb) && skb->len > mtu) ||
7956d64be3dSFlorian Westphal (skb_is_gso(skb) &&
7966d64be3dSFlorian Westphal !skb_gso_validate_network_len(skb, ip_skb_dst_mtu(skb->sk, skb)))) {
7976d64be3dSFlorian Westphal skb->protocol = htons(ETH_P_IP);
7986d64be3dSFlorian Westphal
7996d64be3dSFlorian Westphal if (skb->sk)
8006d64be3dSFlorian Westphal xfrm_local_error(skb, mtu);
8016d64be3dSFlorian Westphal else
8026d64be3dSFlorian Westphal icmp_send(skb, ICMP_DEST_UNREACH,
8036d64be3dSFlorian Westphal ICMP_FRAG_NEEDED, htonl(mtu));
8046d64be3dSFlorian Westphal ret = -EMSGSIZE;
8056d64be3dSFlorian Westphal }
8066d64be3dSFlorian Westphal out:
8076d64be3dSFlorian Westphal return ret;
8086d64be3dSFlorian Westphal }
8096d64be3dSFlorian Westphal
xfrm4_extract_output(struct xfrm_state * x,struct sk_buff * skb)8106d64be3dSFlorian Westphal static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)
8116d64be3dSFlorian Westphal {
8126d64be3dSFlorian Westphal int err;
8136d64be3dSFlorian Westphal
81468dc022dSXin Long if (x->outer_mode.encap == XFRM_MODE_BEET &&
81568dc022dSXin Long ip_is_fragment(ip_hdr(skb))) {
81668dc022dSXin Long net_warn_ratelimited("BEET mode doesn't support inner IPv4 fragments\n");
81768dc022dSXin Long return -EAFNOSUPPORT;
81868dc022dSXin Long }
81968dc022dSXin Long
8206d64be3dSFlorian Westphal err = xfrm4_tunnel_check_size(skb);
8216d64be3dSFlorian Westphal if (err)
8226d64be3dSFlorian Westphal return err;
8236d64be3dSFlorian Westphal
8246d64be3dSFlorian Westphal XFRM_MODE_SKB_CB(skb)->protocol = ip_hdr(skb)->protocol;
8256d64be3dSFlorian Westphal
8266d64be3dSFlorian Westphal xfrm4_extract_header(skb);
8276d64be3dSFlorian Westphal return 0;
8286d64be3dSFlorian Westphal }
8296d64be3dSFlorian Westphal
830f3075f48SFlorian Westphal #if IS_ENABLED(CONFIG_IPV6)
xfrm6_tunnel_check_size(struct sk_buff * skb)831f3075f48SFlorian Westphal static int xfrm6_tunnel_check_size(struct sk_buff *skb)
832f3075f48SFlorian Westphal {
833f3075f48SFlorian Westphal int mtu, ret = 0;
834f3075f48SFlorian Westphal struct dst_entry *dst = skb_dst(skb);
835f3075f48SFlorian Westphal
836f3075f48SFlorian Westphal if (skb->ignore_df)
837f3075f48SFlorian Westphal goto out;
838f3075f48SFlorian Westphal
839f3075f48SFlorian Westphal mtu = dst_mtu(dst);
840f3075f48SFlorian Westphal if (mtu < IPV6_MIN_MTU)
841f3075f48SFlorian Westphal mtu = IPV6_MIN_MTU;
842f3075f48SFlorian Westphal
843f3075f48SFlorian Westphal if ((!skb_is_gso(skb) && skb->len > mtu) ||
844f3075f48SFlorian Westphal (skb_is_gso(skb) &&
845f3075f48SFlorian Westphal !skb_gso_validate_network_len(skb, ip6_skb_dst_mtu(skb)))) {
846f3075f48SFlorian Westphal skb->dev = dst->dev;
847f3075f48SFlorian Westphal skb->protocol = htons(ETH_P_IPV6);
848f3075f48SFlorian Westphal
849f3075f48SFlorian Westphal if (xfrm6_local_dontfrag(skb->sk))
850f3075f48SFlorian Westphal ipv6_stub->xfrm6_local_rxpmtu(skb, mtu);
851f3075f48SFlorian Westphal else if (skb->sk)
852f3075f48SFlorian Westphal xfrm_local_error(skb, mtu);
853f3075f48SFlorian Westphal else
854f3075f48SFlorian Westphal icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
855f3075f48SFlorian Westphal ret = -EMSGSIZE;
856f3075f48SFlorian Westphal }
857f3075f48SFlorian Westphal out:
858f3075f48SFlorian Westphal return ret;
859f3075f48SFlorian Westphal }
860f3075f48SFlorian Westphal #endif
861f3075f48SFlorian Westphal
xfrm6_extract_output(struct xfrm_state * x,struct sk_buff * skb)862f3075f48SFlorian Westphal static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb)
863f3075f48SFlorian Westphal {
864f3075f48SFlorian Westphal #if IS_ENABLED(CONFIG_IPV6)
865f3075f48SFlorian Westphal int err;
866f3075f48SFlorian Westphal
867f3075f48SFlorian Westphal err = xfrm6_tunnel_check_size(skb);
868f3075f48SFlorian Westphal if (err)
869f3075f48SFlorian Westphal return err;
870f3075f48SFlorian Westphal
871f3075f48SFlorian Westphal XFRM_MODE_SKB_CB(skb)->protocol = ipv6_hdr(skb)->nexthdr;
872f3075f48SFlorian Westphal
873f3075f48SFlorian Westphal xfrm6_extract_header(skb);
874f3075f48SFlorian Westphal return 0;
875f3075f48SFlorian Westphal #else
876f3075f48SFlorian Westphal WARN_ON_ONCE(1);
877f3075f48SFlorian Westphal return -EAFNOSUPPORT;
878f3075f48SFlorian Westphal #endif
879f3075f48SFlorian Westphal }
880f3075f48SFlorian Westphal
xfrm_inner_extract_output(struct xfrm_state * x,struct sk_buff * skb)8810c620e97SFlorian Westphal static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb)
882df9dcb45SKazunori MIYAZAWA {
883f4796398SHerbert Xu switch (skb->protocol) {
884f4796398SHerbert Xu case htons(ETH_P_IP):
8856d64be3dSFlorian Westphal return xfrm4_extract_output(x, skb);
886f4796398SHerbert Xu case htons(ETH_P_IPV6):
887f3075f48SFlorian Westphal return xfrm6_extract_output(x, skb);
8886d64be3dSFlorian Westphal }
889733a5facSFlorian Westphal
890f3075f48SFlorian Westphal return -EAFNOSUPPORT;
891df9dcb45SKazunori MIYAZAWA }
892df9dcb45SKazunori MIYAZAWA
xfrm_local_error(struct sk_buff * skb,int mtu)893628e341fSHannes Frederic Sowa void xfrm_local_error(struct sk_buff *skb, int mtu)
894628e341fSHannes Frederic Sowa {
895844d4874SHannes Frederic Sowa unsigned int proto;
896628e341fSHannes Frederic Sowa struct xfrm_state_afinfo *afinfo;
897628e341fSHannes Frederic Sowa
898844d4874SHannes Frederic Sowa if (skb->protocol == htons(ETH_P_IP))
899844d4874SHannes Frederic Sowa proto = AF_INET;
900f6a23d85SXin Long else if (skb->protocol == htons(ETH_P_IPV6) &&
901f6a23d85SXin Long skb->sk->sk_family == AF_INET6)
902844d4874SHannes Frederic Sowa proto = AF_INET6;
903844d4874SHannes Frederic Sowa else
904844d4874SHannes Frederic Sowa return;
905844d4874SHannes Frederic Sowa
906844d4874SHannes Frederic Sowa afinfo = xfrm_state_get_afinfo(proto);
90746c0ef6eSTaehee Yoo if (afinfo) {
908628e341fSHannes Frederic Sowa afinfo->local_error(skb, mtu);
909af5d27c4SFlorian Westphal rcu_read_unlock();
910628e341fSHannes Frederic Sowa }
91146c0ef6eSTaehee Yoo }
912628e341fSHannes Frederic Sowa EXPORT_SYMBOL_GPL(xfrm_local_error);
913