11ccea77eSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * Copyright (C)2002 USAGI/WIDE Project 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Authors 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Mitsuru KANDA @USAGI : IPv6 Support 81da177e4SLinus Torvalds * Kazunori MIYAZAWA @USAGI : 91da177e4SLinus Torvalds * Kunihiro Ishiguro <kunihiro@ipinfusion.com> 101da177e4SLinus Torvalds * 111da177e4SLinus Torvalds * This file is derived from net/ipv4/esp.c 121da177e4SLinus Torvalds */ 131da177e4SLinus Torvalds 14f3213831SJoe Perches #define pr_fmt(fmt) "IPv6: " fmt 15f3213831SJoe Perches 1638320c70SHerbert Xu #include <crypto/aead.h> 1738320c70SHerbert Xu #include <crypto/authenc.h> 186b7326c8SHerbert Xu #include <linux/err.h> 191da177e4SLinus Torvalds #include <linux/module.h> 201da177e4SLinus Torvalds #include <net/ip.h> 211da177e4SLinus Torvalds #include <net/xfrm.h> 221da177e4SLinus Torvalds #include <net/esp.h> 2372998d8cSAdrian Bunk #include <linux/scatterlist.h> 24a02a6422SHerbert Xu #include <linux/kernel.h> 251da177e4SLinus Torvalds #include <linux/pfkeyv2.h> 261da177e4SLinus Torvalds #include <linux/random.h> 2738320c70SHerbert Xu #include <linux/slab.h> 28b7c6538cSHerbert Xu #include <linux/spinlock.h> 290146dca7SSabrina Dubroca #include <net/ip6_checksum.h> 3081aded24SDavid S. Miller #include <net/ip6_route.h> 311da177e4SLinus Torvalds #include <net/icmp.h> 321da177e4SLinus Torvalds #include <net/ipv6.h> 3314c85021SArnaldo Carvalho de Melo #include <net/protocol.h> 340146dca7SSabrina Dubroca #include <net/udp.h> 351da177e4SLinus Torvalds #include <linux/icmpv6.h> 361da177e4SLinus Torvalds 3703e2a30fSSteffen Klassert #include <linux/highmem.h> 3803e2a30fSSteffen Klassert 3938320c70SHerbert Xu struct esp_skb_cb { 4038320c70SHerbert Xu struct xfrm_skb_cb xfrm; 4138320c70SHerbert Xu void *tmp; 4238320c70SHerbert Xu }; 4338320c70SHerbert Xu 440146dca7SSabrina Dubroca struct esp_output_extra { 450146dca7SSabrina Dubroca __be32 seqhi; 460146dca7SSabrina Dubroca u32 esphoff; 470146dca7SSabrina Dubroca }; 480146dca7SSabrina Dubroca 4938320c70SHerbert Xu #define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0])) 5038320c70SHerbert Xu 5138320c70SHerbert Xu /* 5238320c70SHerbert Xu * Allocate an AEAD request structure with extra space for SG and IV. 5338320c70SHerbert Xu * 54d212a4c2SSteffen Klassert * For alignment considerations the upper 32 bits of the sequence number are 55d212a4c2SSteffen Klassert * placed at the front, if present. Followed by the IV, the request and finally 56d212a4c2SSteffen Klassert * the SG list. 5738320c70SHerbert Xu * 5838320c70SHerbert Xu * TODO: Use spare space in skb for this where possible. 5938320c70SHerbert Xu */ 60d212a4c2SSteffen Klassert static void *esp_alloc_tmp(struct crypto_aead *aead, int nfrags, int seqihlen) 6138320c70SHerbert Xu { 6238320c70SHerbert Xu unsigned int len; 6338320c70SHerbert Xu 64d212a4c2SSteffen Klassert len = seqihlen; 65d212a4c2SSteffen Klassert 66d212a4c2SSteffen Klassert len += crypto_aead_ivsize(aead); 67d212a4c2SSteffen Klassert 6838320c70SHerbert Xu if (len) { 6938320c70SHerbert Xu len += crypto_aead_alignmask(aead) & 7038320c70SHerbert Xu ~(crypto_tfm_ctx_alignment() - 1); 7138320c70SHerbert Xu len = ALIGN(len, crypto_tfm_ctx_alignment()); 7238320c70SHerbert Xu } 7338320c70SHerbert Xu 74000ae7b2SHerbert Xu len += sizeof(struct aead_request) + crypto_aead_reqsize(aead); 7538320c70SHerbert Xu len = ALIGN(len, __alignof__(struct scatterlist)); 7638320c70SHerbert Xu 7738320c70SHerbert Xu len += sizeof(struct scatterlist) * nfrags; 7838320c70SHerbert Xu 7938320c70SHerbert Xu return kmalloc(len, GFP_ATOMIC); 8038320c70SHerbert Xu } 8138320c70SHerbert Xu 820146dca7SSabrina Dubroca static inline void *esp_tmp_extra(void *tmp) 83d212a4c2SSteffen Klassert { 840146dca7SSabrina Dubroca return PTR_ALIGN(tmp, __alignof__(struct esp_output_extra)); 85d212a4c2SSteffen Klassert } 86d212a4c2SSteffen Klassert 87d212a4c2SSteffen Klassert static inline u8 *esp_tmp_iv(struct crypto_aead *aead, void *tmp, int seqhilen) 8838320c70SHerbert Xu { 8938320c70SHerbert Xu return crypto_aead_ivsize(aead) ? 90d212a4c2SSteffen Klassert PTR_ALIGN((u8 *)tmp + seqhilen, 91d212a4c2SSteffen Klassert crypto_aead_alignmask(aead) + 1) : tmp + seqhilen; 9238320c70SHerbert Xu } 9338320c70SHerbert Xu 9438320c70SHerbert Xu static inline struct aead_request *esp_tmp_req(struct crypto_aead *aead, u8 *iv) 9538320c70SHerbert Xu { 9638320c70SHerbert Xu struct aead_request *req; 9738320c70SHerbert Xu 9838320c70SHerbert Xu req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead), 9938320c70SHerbert Xu crypto_tfm_ctx_alignment()); 10038320c70SHerbert Xu aead_request_set_tfm(req, aead); 10138320c70SHerbert Xu return req; 10238320c70SHerbert Xu } 10338320c70SHerbert Xu 10438320c70SHerbert Xu static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead, 10538320c70SHerbert Xu struct aead_request *req) 10638320c70SHerbert Xu { 10738320c70SHerbert Xu return (void *)ALIGN((unsigned long)(req + 1) + 10838320c70SHerbert Xu crypto_aead_reqsize(aead), 10938320c70SHerbert Xu __alignof__(struct scatterlist)); 11038320c70SHerbert Xu } 11138320c70SHerbert Xu 11203e2a30fSSteffen Klassert static void esp_ssg_unref(struct xfrm_state *x, void *tmp) 11303e2a30fSSteffen Klassert { 1140146dca7SSabrina Dubroca struct esp_output_extra *extra = esp_tmp_extra(tmp); 11503e2a30fSSteffen Klassert struct crypto_aead *aead = x->data; 1160146dca7SSabrina Dubroca int extralen = 0; 11703e2a30fSSteffen Klassert u8 *iv; 11803e2a30fSSteffen Klassert struct aead_request *req; 11903e2a30fSSteffen Klassert struct scatterlist *sg; 12003e2a30fSSteffen Klassert 12103e2a30fSSteffen Klassert if (x->props.flags & XFRM_STATE_ESN) 1220146dca7SSabrina Dubroca extralen += sizeof(*extra); 12303e2a30fSSteffen Klassert 1240146dca7SSabrina Dubroca iv = esp_tmp_iv(aead, tmp, extralen); 12503e2a30fSSteffen Klassert req = esp_tmp_req(aead, iv); 12603e2a30fSSteffen Klassert 12703e2a30fSSteffen Klassert /* Unref skb_frag_pages in the src scatterlist if necessary. 12803e2a30fSSteffen Klassert * Skip the first sg which comes from skb->data. 12903e2a30fSSteffen Klassert */ 13003e2a30fSSteffen Klassert if (req->src != req->dst) 13103e2a30fSSteffen Klassert for (sg = sg_next(req->src); sg; sg = sg_next(sg)) 13203e2a30fSSteffen Klassert put_page(sg_page(sg)); 13303e2a30fSSteffen Klassert } 13403e2a30fSSteffen Klassert 1350146dca7SSabrina Dubroca static void esp_output_encap_csum(struct sk_buff *skb) 1360146dca7SSabrina Dubroca { 1370146dca7SSabrina Dubroca /* UDP encap with IPv6 requires a valid checksum */ 1380146dca7SSabrina Dubroca if (*skb_mac_header(skb) == IPPROTO_UDP) { 1390146dca7SSabrina Dubroca struct udphdr *uh = udp_hdr(skb); 1400146dca7SSabrina Dubroca struct ipv6hdr *ip6h = ipv6_hdr(skb); 1410146dca7SSabrina Dubroca int len = ntohs(uh->len); 1420146dca7SSabrina Dubroca unsigned int offset = skb_transport_offset(skb); 1430146dca7SSabrina Dubroca __wsum csum = skb_checksum(skb, offset, skb->len - offset, 0); 1440146dca7SSabrina Dubroca 1450146dca7SSabrina Dubroca uh->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, 1460146dca7SSabrina Dubroca len, IPPROTO_UDP, csum); 1470146dca7SSabrina Dubroca if (uh->check == 0) 1480146dca7SSabrina Dubroca uh->check = CSUM_MANGLED_0; 1490146dca7SSabrina Dubroca } 1500146dca7SSabrina Dubroca } 1510146dca7SSabrina Dubroca 15238320c70SHerbert Xu static void esp_output_done(struct crypto_async_request *base, int err) 15338320c70SHerbert Xu { 15438320c70SHerbert Xu struct sk_buff *skb = base->data; 155f53c7239SSteffen Klassert struct xfrm_offload *xo = xfrm_offload(skb); 15603e2a30fSSteffen Klassert void *tmp; 157f53c7239SSteffen Klassert struct xfrm_state *x; 158f53c7239SSteffen Klassert 1592294be0fSFlorian Westphal if (xo && (xo->flags & XFRM_DEV_RESUME)) { 1602294be0fSFlorian Westphal struct sec_path *sp = skb_sec_path(skb); 1612294be0fSFlorian Westphal 1622294be0fSFlorian Westphal x = sp->xvec[sp->len - 1]; 1632294be0fSFlorian Westphal } else { 164f53c7239SSteffen Klassert x = skb_dst(skb)->xfrm; 1652294be0fSFlorian Westphal } 16638320c70SHerbert Xu 16703e2a30fSSteffen Klassert tmp = ESP_SKB_CB(skb)->tmp; 16803e2a30fSSteffen Klassert esp_ssg_unref(x, tmp); 16903e2a30fSSteffen Klassert kfree(tmp); 170f53c7239SSteffen Klassert 1710146dca7SSabrina Dubroca esp_output_encap_csum(skb); 1720146dca7SSabrina Dubroca 173f53c7239SSteffen Klassert if (xo && (xo->flags & XFRM_DEV_RESUME)) { 174f53c7239SSteffen Klassert if (err) { 175f53c7239SSteffen Klassert XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); 176f53c7239SSteffen Klassert kfree_skb(skb); 177f53c7239SSteffen Klassert return; 178f53c7239SSteffen Klassert } 179f53c7239SSteffen Klassert 180f53c7239SSteffen Klassert skb_push(skb, skb->data - skb_mac_header(skb)); 181f53c7239SSteffen Klassert secpath_reset(skb); 182f53c7239SSteffen Klassert xfrm_dev_resume(skb); 183f53c7239SSteffen Klassert } else { 18438320c70SHerbert Xu xfrm_output_resume(skb, err); 18538320c70SHerbert Xu } 186f53c7239SSteffen Klassert } 18738320c70SHerbert Xu 188000ae7b2SHerbert Xu /* Move ESP header back into place. */ 189000ae7b2SHerbert Xu static void esp_restore_header(struct sk_buff *skb, unsigned int offset) 190000ae7b2SHerbert Xu { 191000ae7b2SHerbert Xu struct ip_esp_hdr *esph = (void *)(skb->data + offset); 192000ae7b2SHerbert Xu void *tmp = ESP_SKB_CB(skb)->tmp; 1930146dca7SSabrina Dubroca __be32 *seqhi = esp_tmp_extra(tmp); 194000ae7b2SHerbert Xu 195000ae7b2SHerbert Xu esph->seq_no = esph->spi; 196000ae7b2SHerbert Xu esph->spi = *seqhi; 197000ae7b2SHerbert Xu } 198000ae7b2SHerbert Xu 199000ae7b2SHerbert Xu static void esp_output_restore_header(struct sk_buff *skb) 200000ae7b2SHerbert Xu { 2010146dca7SSabrina Dubroca void *tmp = ESP_SKB_CB(skb)->tmp; 2020146dca7SSabrina Dubroca struct esp_output_extra *extra = esp_tmp_extra(tmp); 2030146dca7SSabrina Dubroca 2040146dca7SSabrina Dubroca esp_restore_header(skb, skb_transport_offset(skb) + extra->esphoff - 2050146dca7SSabrina Dubroca sizeof(__be32)); 206000ae7b2SHerbert Xu } 207000ae7b2SHerbert Xu 20803e2a30fSSteffen Klassert static struct ip_esp_hdr *esp_output_set_esn(struct sk_buff *skb, 209383d0350SSteffen Klassert struct xfrm_state *x, 21003e2a30fSSteffen Klassert struct ip_esp_hdr *esph, 2110146dca7SSabrina Dubroca struct esp_output_extra *extra) 21203e2a30fSSteffen Klassert { 21303e2a30fSSteffen Klassert /* For ESN we move the header forward by 4 bytes to 21403e2a30fSSteffen Klassert * accomodate the high bits. We will move it back after 21503e2a30fSSteffen Klassert * encryption. 21603e2a30fSSteffen Klassert */ 21703e2a30fSSteffen Klassert if ((x->props.flags & XFRM_STATE_ESN)) { 2180146dca7SSabrina Dubroca __u32 seqhi; 2197862b405SSteffen Klassert struct xfrm_offload *xo = xfrm_offload(skb); 2207862b405SSteffen Klassert 2217862b405SSteffen Klassert if (xo) 2220146dca7SSabrina Dubroca seqhi = xo->seq.hi; 2237862b405SSteffen Klassert else 2240146dca7SSabrina Dubroca seqhi = XFRM_SKB_CB(skb)->seq.output.hi; 2250146dca7SSabrina Dubroca 2260146dca7SSabrina Dubroca extra->esphoff = (unsigned char *)esph - 2270146dca7SSabrina Dubroca skb_transport_header(skb); 2280146dca7SSabrina Dubroca esph = (struct ip_esp_hdr *)((unsigned char *)esph - 4); 2290146dca7SSabrina Dubroca extra->seqhi = esph->spi; 2300146dca7SSabrina Dubroca esph->seq_no = htonl(seqhi); 23103e2a30fSSteffen Klassert } 23203e2a30fSSteffen Klassert 23303e2a30fSSteffen Klassert esph->spi = x->id.spi; 23403e2a30fSSteffen Klassert 23503e2a30fSSteffen Klassert return esph; 23603e2a30fSSteffen Klassert } 23703e2a30fSSteffen Klassert 238000ae7b2SHerbert Xu static void esp_output_done_esn(struct crypto_async_request *base, int err) 239000ae7b2SHerbert Xu { 240000ae7b2SHerbert Xu struct sk_buff *skb = base->data; 241000ae7b2SHerbert Xu 242000ae7b2SHerbert Xu esp_output_restore_header(skb); 243000ae7b2SHerbert Xu esp_output_done(base, err); 244000ae7b2SHerbert Xu } 245000ae7b2SHerbert Xu 2460146dca7SSabrina Dubroca static struct ip_esp_hdr *esp6_output_udp_encap(struct sk_buff *skb, 2470146dca7SSabrina Dubroca int encap_type, 2480146dca7SSabrina Dubroca struct esp_info *esp, 2490146dca7SSabrina Dubroca __be16 sport, 2500146dca7SSabrina Dubroca __be16 dport) 2510146dca7SSabrina Dubroca { 2520146dca7SSabrina Dubroca struct udphdr *uh; 2530146dca7SSabrina Dubroca __be32 *udpdata32; 2540146dca7SSabrina Dubroca unsigned int len; 2550146dca7SSabrina Dubroca 2560146dca7SSabrina Dubroca len = skb->len + esp->tailen - skb_transport_offset(skb); 2570146dca7SSabrina Dubroca if (len > U16_MAX) 2580146dca7SSabrina Dubroca return ERR_PTR(-EMSGSIZE); 2590146dca7SSabrina Dubroca 2600146dca7SSabrina Dubroca uh = (struct udphdr *)esp->esph; 2610146dca7SSabrina Dubroca uh->source = sport; 2620146dca7SSabrina Dubroca uh->dest = dport; 2630146dca7SSabrina Dubroca uh->len = htons(len); 2640146dca7SSabrina Dubroca uh->check = 0; 2650146dca7SSabrina Dubroca 2660146dca7SSabrina Dubroca *skb_mac_header(skb) = IPPROTO_UDP; 2670146dca7SSabrina Dubroca 2680146dca7SSabrina Dubroca if (encap_type == UDP_ENCAP_ESPINUDP_NON_IKE) { 2690146dca7SSabrina Dubroca udpdata32 = (__be32 *)(uh + 1); 2700146dca7SSabrina Dubroca udpdata32[0] = udpdata32[1] = 0; 2710146dca7SSabrina Dubroca return (struct ip_esp_hdr *)(udpdata32 + 2); 2720146dca7SSabrina Dubroca } 2730146dca7SSabrina Dubroca 2740146dca7SSabrina Dubroca return (struct ip_esp_hdr *)(uh + 1); 2750146dca7SSabrina Dubroca } 2760146dca7SSabrina Dubroca 2770146dca7SSabrina Dubroca static int esp6_output_encap(struct xfrm_state *x, struct sk_buff *skb, 2780146dca7SSabrina Dubroca struct esp_info *esp) 2790146dca7SSabrina Dubroca { 2800146dca7SSabrina Dubroca struct xfrm_encap_tmpl *encap = x->encap; 2810146dca7SSabrina Dubroca struct ip_esp_hdr *esph; 2820146dca7SSabrina Dubroca __be16 sport, dport; 2830146dca7SSabrina Dubroca int encap_type; 2840146dca7SSabrina Dubroca 2850146dca7SSabrina Dubroca spin_lock_bh(&x->lock); 2860146dca7SSabrina Dubroca sport = encap->encap_sport; 2870146dca7SSabrina Dubroca dport = encap->encap_dport; 2880146dca7SSabrina Dubroca encap_type = encap->encap_type; 2890146dca7SSabrina Dubroca spin_unlock_bh(&x->lock); 2900146dca7SSabrina Dubroca 2910146dca7SSabrina Dubroca switch (encap_type) { 2920146dca7SSabrina Dubroca default: 2930146dca7SSabrina Dubroca case UDP_ENCAP_ESPINUDP: 2940146dca7SSabrina Dubroca case UDP_ENCAP_ESPINUDP_NON_IKE: 2950146dca7SSabrina Dubroca esph = esp6_output_udp_encap(skb, encap_type, esp, sport, dport); 2960146dca7SSabrina Dubroca break; 2970146dca7SSabrina Dubroca } 2980146dca7SSabrina Dubroca 2990146dca7SSabrina Dubroca if (IS_ERR(esph)) 3000146dca7SSabrina Dubroca return PTR_ERR(esph); 3010146dca7SSabrina Dubroca 3020146dca7SSabrina Dubroca esp->esph = esph; 3030146dca7SSabrina Dubroca 3040146dca7SSabrina Dubroca return 0; 3050146dca7SSabrina Dubroca } 3060146dca7SSabrina Dubroca 307383d0350SSteffen Klassert int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp) 3081da177e4SLinus Torvalds { 30927a884dcSArnaldo Carvalho de Melo u8 *tail; 31003e2a30fSSteffen Klassert u8 *vaddr; 311383d0350SSteffen Klassert int nfrags; 3120146dca7SSabrina Dubroca int esph_offset; 313383d0350SSteffen Klassert struct page *page; 314383d0350SSteffen Klassert struct sk_buff *trailer; 315383d0350SSteffen Klassert int tailen = esp->tailen; 316d212a4c2SSteffen Klassert 3170146dca7SSabrina Dubroca if (x->encap) { 3180146dca7SSabrina Dubroca int err = esp6_output_encap(x, skb, esp); 3190146dca7SSabrina Dubroca 3200146dca7SSabrina Dubroca if (err < 0) 3210146dca7SSabrina Dubroca return err; 3220146dca7SSabrina Dubroca } 3230146dca7SSabrina Dubroca 32403e2a30fSSteffen Klassert if (!skb_cloned(skb)) { 32554ffd790SSteffen Klassert if (tailen <= skb_tailroom(skb)) { 32603e2a30fSSteffen Klassert nfrags = 1; 32703e2a30fSSteffen Klassert trailer = skb; 32803e2a30fSSteffen Klassert tail = skb_tail_pointer(trailer); 32903e2a30fSSteffen Klassert 33003e2a30fSSteffen Klassert goto skip_cow; 33103e2a30fSSteffen Klassert } else if ((skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS) 33203e2a30fSSteffen Klassert && !skb_has_frag_list(skb)) { 33303e2a30fSSteffen Klassert int allocsize; 33403e2a30fSSteffen Klassert struct sock *sk = skb->sk; 33503e2a30fSSteffen Klassert struct page_frag *pfrag = &x->xfrag; 33603e2a30fSSteffen Klassert 337383d0350SSteffen Klassert esp->inplace = false; 338383d0350SSteffen Klassert 33903e2a30fSSteffen Klassert allocsize = ALIGN(tailen, L1_CACHE_BYTES); 34003e2a30fSSteffen Klassert 34103e2a30fSSteffen Klassert spin_lock_bh(&x->lock); 34203e2a30fSSteffen Klassert 34303e2a30fSSteffen Klassert if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { 34403e2a30fSSteffen Klassert spin_unlock_bh(&x->lock); 34503e2a30fSSteffen Klassert goto cow; 34603e2a30fSSteffen Klassert } 34703e2a30fSSteffen Klassert 34803e2a30fSSteffen Klassert page = pfrag->page; 34903e2a30fSSteffen Klassert get_page(page); 35003e2a30fSSteffen Klassert 35103e2a30fSSteffen Klassert vaddr = kmap_atomic(page); 35203e2a30fSSteffen Klassert 35303e2a30fSSteffen Klassert tail = vaddr + pfrag->offset; 35403e2a30fSSteffen Klassert 355383d0350SSteffen Klassert esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto); 35603e2a30fSSteffen Klassert 35703e2a30fSSteffen Klassert kunmap_atomic(vaddr); 35803e2a30fSSteffen Klassert 35903e2a30fSSteffen Klassert nfrags = skb_shinfo(skb)->nr_frags; 36003e2a30fSSteffen Klassert 36103e2a30fSSteffen Klassert __skb_fill_page_desc(skb, nfrags, page, pfrag->offset, 36203e2a30fSSteffen Klassert tailen); 36303e2a30fSSteffen Klassert skb_shinfo(skb)->nr_frags = ++nfrags; 36403e2a30fSSteffen Klassert 36503e2a30fSSteffen Klassert pfrag->offset = pfrag->offset + allocsize; 36636ff0dd3SSteffen Klassert 36736ff0dd3SSteffen Klassert spin_unlock_bh(&x->lock); 36836ff0dd3SSteffen Klassert 36903e2a30fSSteffen Klassert nfrags++; 37003e2a30fSSteffen Klassert 37103e2a30fSSteffen Klassert skb->len += tailen; 37203e2a30fSSteffen Klassert skb->data_len += tailen; 37303e2a30fSSteffen Klassert skb->truesize += tailen; 37409db5124SMartin Willi if (sk && sk_fullsock(sk)) 37514afee4bSReshetova, Elena refcount_add(tailen, &sk->sk_wmem_alloc); 37603e2a30fSSteffen Klassert 377383d0350SSteffen Klassert goto out; 378383d0350SSteffen Klassert } 379383d0350SSteffen Klassert } 38003e2a30fSSteffen Klassert 381383d0350SSteffen Klassert cow: 3820146dca7SSabrina Dubroca esph_offset = (unsigned char *)esp->esph - skb_transport_header(skb); 3830146dca7SSabrina Dubroca 384383d0350SSteffen Klassert nfrags = skb_cow_data(skb, tailen, &trailer); 385383d0350SSteffen Klassert if (nfrags < 0) 386383d0350SSteffen Klassert goto out; 387383d0350SSteffen Klassert tail = skb_tail_pointer(trailer); 3880146dca7SSabrina Dubroca esp->esph = (struct ip_esp_hdr *)(skb_transport_header(skb) + esph_offset); 38903e2a30fSSteffen Klassert 390383d0350SSteffen Klassert skip_cow: 391383d0350SSteffen Klassert esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto); 392383d0350SSteffen Klassert pskb_put(skb, trailer, tailen); 393383d0350SSteffen Klassert 394383d0350SSteffen Klassert out: 395383d0350SSteffen Klassert return nfrags; 396383d0350SSteffen Klassert } 397383d0350SSteffen Klassert EXPORT_SYMBOL_GPL(esp6_output_head); 398383d0350SSteffen Klassert 399383d0350SSteffen Klassert int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp) 400383d0350SSteffen Klassert { 401383d0350SSteffen Klassert u8 *iv; 402383d0350SSteffen Klassert int alen; 403383d0350SSteffen Klassert void *tmp; 404383d0350SSteffen Klassert int ivlen; 405383d0350SSteffen Klassert int assoclen; 4060146dca7SSabrina Dubroca int extralen; 407383d0350SSteffen Klassert struct page *page; 408383d0350SSteffen Klassert struct ip_esp_hdr *esph; 409383d0350SSteffen Klassert struct aead_request *req; 410383d0350SSteffen Klassert struct crypto_aead *aead; 411383d0350SSteffen Klassert struct scatterlist *sg, *dsg; 4120146dca7SSabrina Dubroca struct esp_output_extra *extra; 413383d0350SSteffen Klassert int err = -ENOMEM; 414383d0350SSteffen Klassert 415383d0350SSteffen Klassert assoclen = sizeof(struct ip_esp_hdr); 4160146dca7SSabrina Dubroca extralen = 0; 417383d0350SSteffen Klassert 418383d0350SSteffen Klassert if (x->props.flags & XFRM_STATE_ESN) { 4190146dca7SSabrina Dubroca extralen += sizeof(*extra); 420383d0350SSteffen Klassert assoclen += sizeof(__be32); 421383d0350SSteffen Klassert } 422383d0350SSteffen Klassert 423383d0350SSteffen Klassert aead = x->data; 424383d0350SSteffen Klassert alen = crypto_aead_authsize(aead); 425383d0350SSteffen Klassert ivlen = crypto_aead_ivsize(aead); 426383d0350SSteffen Klassert 4270146dca7SSabrina Dubroca tmp = esp_alloc_tmp(aead, esp->nfrags + 2, extralen); 428e892d2d4SSteffen Klassert if (!tmp) 42903e2a30fSSteffen Klassert goto error; 430383d0350SSteffen Klassert 4310146dca7SSabrina Dubroca extra = esp_tmp_extra(tmp); 4320146dca7SSabrina Dubroca iv = esp_tmp_iv(aead, tmp, extralen); 43303e2a30fSSteffen Klassert req = esp_tmp_req(aead, iv); 43403e2a30fSSteffen Klassert sg = esp_req_sg(aead, req); 43503e2a30fSSteffen Klassert 436383d0350SSteffen Klassert if (esp->inplace) 437383d0350SSteffen Klassert dsg = sg; 438383d0350SSteffen Klassert else 439383d0350SSteffen Klassert dsg = &sg[esp->nfrags]; 44003e2a30fSSteffen Klassert 4410146dca7SSabrina Dubroca esph = esp_output_set_esn(skb, x, esp->esph, extra); 4420146dca7SSabrina Dubroca esp->esph = esph; 443383d0350SSteffen Klassert 444383d0350SSteffen Klassert sg_init_table(sg, esp->nfrags); 4453f297707SJason A. Donenfeld err = skb_to_sgvec(skb, sg, 44603e2a30fSSteffen Klassert (unsigned char *)esph - skb->data, 447383d0350SSteffen Klassert assoclen + ivlen + esp->clen + alen); 4483f297707SJason A. Donenfeld if (unlikely(err < 0)) 449e6194923SSteffen Klassert goto error_free; 450383d0350SSteffen Klassert 451383d0350SSteffen Klassert if (!esp->inplace) { 452383d0350SSteffen Klassert int allocsize; 453383d0350SSteffen Klassert struct page_frag *pfrag = &x->xfrag; 45403e2a30fSSteffen Klassert 45503e2a30fSSteffen Klassert allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES); 45603e2a30fSSteffen Klassert 457383d0350SSteffen Klassert spin_lock_bh(&x->lock); 45803e2a30fSSteffen Klassert if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { 45903e2a30fSSteffen Klassert spin_unlock_bh(&x->lock); 460e6194923SSteffen Klassert goto error_free; 46103e2a30fSSteffen Klassert } 46203e2a30fSSteffen Klassert 46303e2a30fSSteffen Klassert skb_shinfo(skb)->nr_frags = 1; 46403e2a30fSSteffen Klassert 46503e2a30fSSteffen Klassert page = pfrag->page; 46603e2a30fSSteffen Klassert get_page(page); 46703e2a30fSSteffen Klassert /* replace page frags in skb with new page */ 46803e2a30fSSteffen Klassert __skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len); 46903e2a30fSSteffen Klassert pfrag->offset = pfrag->offset + allocsize; 470383d0350SSteffen Klassert spin_unlock_bh(&x->lock); 47103e2a30fSSteffen Klassert 47203e2a30fSSteffen Klassert sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1); 4733f297707SJason A. Donenfeld err = skb_to_sgvec(skb, dsg, 47403e2a30fSSteffen Klassert (unsigned char *)esph - skb->data, 475383d0350SSteffen Klassert assoclen + ivlen + esp->clen + alen); 4763f297707SJason A. Donenfeld if (unlikely(err < 0)) 477e6194923SSteffen Klassert goto error_free; 47803e2a30fSSteffen Klassert } 47903e2a30fSSteffen Klassert 48003e2a30fSSteffen Klassert if ((x->props.flags & XFRM_STATE_ESN)) 48103e2a30fSSteffen Klassert aead_request_set_callback(req, 0, esp_output_done_esn, skb); 48203e2a30fSSteffen Klassert else 48303e2a30fSSteffen Klassert aead_request_set_callback(req, 0, esp_output_done, skb); 48403e2a30fSSteffen Klassert 485383d0350SSteffen Klassert aead_request_set_crypt(req, sg, dsg, ivlen + esp->clen, iv); 486000ae7b2SHerbert Xu aead_request_set_ad(req, assoclen); 4871da177e4SLinus Torvalds 488000ae7b2SHerbert Xu memset(iv, 0, ivlen); 489383d0350SSteffen Klassert memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&esp->seqno + 8 - min(ivlen, 8), 490000ae7b2SHerbert Xu min(ivlen, 8)); 4916b7326c8SHerbert Xu 49238320c70SHerbert Xu ESP_SKB_CB(skb)->tmp = tmp; 493000ae7b2SHerbert Xu err = crypto_aead_encrypt(req); 494000ae7b2SHerbert Xu 495000ae7b2SHerbert Xu switch (err) { 496000ae7b2SHerbert Xu case -EINPROGRESS: 49738320c70SHerbert Xu goto error; 4981da177e4SLinus Torvalds 499068c2e70SGilad Ben-Yossef case -ENOSPC: 50038320c70SHerbert Xu err = NET_XMIT_DROP; 501000ae7b2SHerbert Xu break; 502000ae7b2SHerbert Xu 503000ae7b2SHerbert Xu case 0: 504000ae7b2SHerbert Xu if ((x->props.flags & XFRM_STATE_ESN)) 505000ae7b2SHerbert Xu esp_output_restore_header(skb); 5060146dca7SSabrina Dubroca esp_output_encap_csum(skb); 507000ae7b2SHerbert Xu } 5081da177e4SLinus Torvalds 50903e2a30fSSteffen Klassert if (sg != dsg) 51003e2a30fSSteffen Klassert esp_ssg_unref(x, tmp); 511b7c6538cSHerbert Xu 512e6194923SSteffen Klassert error_free: 513e6194923SSteffen Klassert kfree(tmp); 5141da177e4SLinus Torvalds error: 5151da177e4SLinus Torvalds return err; 5161da177e4SLinus Torvalds } 517383d0350SSteffen Klassert EXPORT_SYMBOL_GPL(esp6_output_tail); 5181da177e4SLinus Torvalds 519383d0350SSteffen Klassert static int esp6_output(struct xfrm_state *x, struct sk_buff *skb) 520383d0350SSteffen Klassert { 521383d0350SSteffen Klassert int alen; 522383d0350SSteffen Klassert int blksize; 523383d0350SSteffen Klassert struct ip_esp_hdr *esph; 524383d0350SSteffen Klassert struct crypto_aead *aead; 525383d0350SSteffen Klassert struct esp_info esp; 526383d0350SSteffen Klassert 527383d0350SSteffen Klassert esp.inplace = true; 528383d0350SSteffen Klassert 529383d0350SSteffen Klassert esp.proto = *skb_mac_header(skb); 530383d0350SSteffen Klassert *skb_mac_header(skb) = IPPROTO_ESP; 531383d0350SSteffen Klassert 532383d0350SSteffen Klassert /* skb is pure payload to encrypt */ 533383d0350SSteffen Klassert 534383d0350SSteffen Klassert aead = x->data; 535383d0350SSteffen Klassert alen = crypto_aead_authsize(aead); 536383d0350SSteffen Klassert 537383d0350SSteffen Klassert esp.tfclen = 0; 538383d0350SSteffen Klassert if (x->tfcpad) { 539383d0350SSteffen Klassert struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb); 540383d0350SSteffen Klassert u32 padto; 541383d0350SSteffen Klassert 542c7b37c76SFlorian Westphal padto = min(x->tfcpad, xfrm_state_mtu(x, dst->child_mtu_cached)); 543383d0350SSteffen Klassert if (skb->len < padto) 544383d0350SSteffen Klassert esp.tfclen = padto - skb->len; 545383d0350SSteffen Klassert } 546383d0350SSteffen Klassert blksize = ALIGN(crypto_aead_blocksize(aead), 4); 547383d0350SSteffen Klassert esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize); 548383d0350SSteffen Klassert esp.plen = esp.clen - skb->len - esp.tfclen; 549383d0350SSteffen Klassert esp.tailen = esp.tfclen + esp.plen + alen; 550383d0350SSteffen Klassert 5510146dca7SSabrina Dubroca esp.esph = ip_esp_hdr(skb); 5520146dca7SSabrina Dubroca 553383d0350SSteffen Klassert esp.nfrags = esp6_output_head(x, skb, &esp); 554383d0350SSteffen Klassert if (esp.nfrags < 0) 555383d0350SSteffen Klassert return esp.nfrags; 556383d0350SSteffen Klassert 5570146dca7SSabrina Dubroca esph = esp.esph; 558383d0350SSteffen Klassert esph->spi = x->id.spi; 559383d0350SSteffen Klassert 560383d0350SSteffen Klassert esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low); 561383d0350SSteffen Klassert esp.seqno = cpu_to_be64(XFRM_SKB_CB(skb)->seq.output.low + 562383d0350SSteffen Klassert ((u64)XFRM_SKB_CB(skb)->seq.output.hi << 32)); 563383d0350SSteffen Klassert 564383d0350SSteffen Klassert skb_push(skb, -skb_network_offset(skb)); 565383d0350SSteffen Klassert 566383d0350SSteffen Klassert return esp6_output_tail(x, skb, &esp); 567383d0350SSteffen Klassert } 568383d0350SSteffen Klassert 56947ebcc0bSYossi Kuperman static inline int esp_remove_trailer(struct sk_buff *skb) 57038320c70SHerbert Xu { 57138320c70SHerbert Xu struct xfrm_state *x = xfrm_input_state(skb); 572d77e38e6SSteffen Klassert struct xfrm_offload *xo = xfrm_offload(skb); 5731c5ad13fSMathias Krause struct crypto_aead *aead = x->data; 57447ebcc0bSYossi Kuperman int alen, hlen, elen; 575e51a6472SIlan Tayari int padlen, trimlen; 576e51a6472SIlan Tayari __wsum csumdiff; 57738320c70SHerbert Xu u8 nexthdr[2]; 57847ebcc0bSYossi Kuperman int ret; 57938320c70SHerbert Xu 58047ebcc0bSYossi Kuperman alen = crypto_aead_authsize(aead); 58147ebcc0bSYossi Kuperman hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead); 58247ebcc0bSYossi Kuperman elen = skb->len - hlen; 58338320c70SHerbert Xu 58447ebcc0bSYossi Kuperman if (xo && (xo->flags & XFRM_ESP_NO_TRAILER)) { 58547ebcc0bSYossi Kuperman ret = xo->proto; 58638320c70SHerbert Xu goto out; 58747ebcc0bSYossi Kuperman } 58838320c70SHerbert Xu 589eee12df5SGustavo A. R. Silva ret = skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2); 590eee12df5SGustavo A. R. Silva BUG_ON(ret); 59138320c70SHerbert Xu 59247ebcc0bSYossi Kuperman ret = -EINVAL; 59338320c70SHerbert Xu padlen = nexthdr[0]; 59438320c70SHerbert Xu if (padlen + 2 + alen >= elen) { 595ba7a46f1SJoe Perches net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n", 596ba7a46f1SJoe Perches padlen + 2, elen - alen); 59738320c70SHerbert Xu goto out; 59838320c70SHerbert Xu } 59938320c70SHerbert Xu 600e51a6472SIlan Tayari trimlen = alen + padlen + 2; 601e51a6472SIlan Tayari if (skb->ip_summed == CHECKSUM_COMPLETE) { 602e51a6472SIlan Tayari csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0); 603e51a6472SIlan Tayari skb->csum = csum_block_sub(skb->csum, csumdiff, 604e51a6472SIlan Tayari skb->len - trimlen); 605e51a6472SIlan Tayari } 606e51a6472SIlan Tayari pskb_trim(skb, skb->len - trimlen); 607e51a6472SIlan Tayari 60847ebcc0bSYossi Kuperman ret = nexthdr[1]; 60947ebcc0bSYossi Kuperman 61047ebcc0bSYossi Kuperman out: 61147ebcc0bSYossi Kuperman return ret; 61247ebcc0bSYossi Kuperman } 61347ebcc0bSYossi Kuperman 61447ebcc0bSYossi Kuperman int esp6_input_done2(struct sk_buff *skb, int err) 61547ebcc0bSYossi Kuperman { 61647ebcc0bSYossi Kuperman struct xfrm_state *x = xfrm_input_state(skb); 61747ebcc0bSYossi Kuperman struct xfrm_offload *xo = xfrm_offload(skb); 61847ebcc0bSYossi Kuperman struct crypto_aead *aead = x->data; 61947ebcc0bSYossi Kuperman int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead); 62047ebcc0bSYossi Kuperman int hdr_len = skb_network_header_len(skb); 62147ebcc0bSYossi Kuperman 62247ebcc0bSYossi Kuperman if (!xo || (xo && !(xo->flags & CRYPTO_DONE))) 62347ebcc0bSYossi Kuperman kfree(ESP_SKB_CB(skb)->tmp); 62447ebcc0bSYossi Kuperman 62547ebcc0bSYossi Kuperman if (unlikely(err)) 62647ebcc0bSYossi Kuperman goto out; 62747ebcc0bSYossi Kuperman 62847ebcc0bSYossi Kuperman err = esp_remove_trailer(skb); 62947ebcc0bSYossi Kuperman if (unlikely(err < 0)) 63047ebcc0bSYossi Kuperman goto out; 63147ebcc0bSYossi Kuperman 6320146dca7SSabrina Dubroca if (x->encap) { 6330146dca7SSabrina Dubroca const struct ipv6hdr *ip6h = ipv6_hdr(skb); 6340146dca7SSabrina Dubroca struct xfrm_encap_tmpl *encap = x->encap; 6350146dca7SSabrina Dubroca struct udphdr *uh = (void *)(skb_network_header(skb) + hdr_len); 6360146dca7SSabrina Dubroca __be16 source; 6370146dca7SSabrina Dubroca 6380146dca7SSabrina Dubroca switch (x->encap->encap_type) { 6390146dca7SSabrina Dubroca case UDP_ENCAP_ESPINUDP: 6400146dca7SSabrina Dubroca case UDP_ENCAP_ESPINUDP_NON_IKE: 6410146dca7SSabrina Dubroca source = uh->source; 6420146dca7SSabrina Dubroca break; 6430146dca7SSabrina Dubroca default: 6440146dca7SSabrina Dubroca WARN_ON_ONCE(1); 6450146dca7SSabrina Dubroca err = -EINVAL; 6460146dca7SSabrina Dubroca goto out; 6470146dca7SSabrina Dubroca } 6480146dca7SSabrina Dubroca 6490146dca7SSabrina Dubroca /* 6500146dca7SSabrina Dubroca * 1) if the NAT-T peer's IP or port changed then 6510146dca7SSabrina Dubroca * advertize the change to the keying daemon. 6520146dca7SSabrina Dubroca * This is an inbound SA, so just compare 6530146dca7SSabrina Dubroca * SRC ports. 6540146dca7SSabrina Dubroca */ 6550146dca7SSabrina Dubroca if (!ipv6_addr_equal(&ip6h->saddr, &x->props.saddr.in6) || 6560146dca7SSabrina Dubroca source != encap->encap_sport) { 6570146dca7SSabrina Dubroca xfrm_address_t ipaddr; 6580146dca7SSabrina Dubroca 6590146dca7SSabrina Dubroca memcpy(&ipaddr.a6, &ip6h->saddr.s6_addr, sizeof(ipaddr.a6)); 6600146dca7SSabrina Dubroca km_new_mapping(x, &ipaddr, source); 6610146dca7SSabrina Dubroca 6620146dca7SSabrina Dubroca /* XXX: perhaps add an extra 6630146dca7SSabrina Dubroca * policy check here, to see 6640146dca7SSabrina Dubroca * if we should allow or 6650146dca7SSabrina Dubroca * reject a packet from a 6660146dca7SSabrina Dubroca * different source 6670146dca7SSabrina Dubroca * address/port. 6680146dca7SSabrina Dubroca */ 6690146dca7SSabrina Dubroca } 6700146dca7SSabrina Dubroca 6710146dca7SSabrina Dubroca /* 6720146dca7SSabrina Dubroca * 2) ignore UDP/TCP checksums in case 6730146dca7SSabrina Dubroca * of NAT-T in Transport Mode, or 6740146dca7SSabrina Dubroca * perform other post-processing fixes 6750146dca7SSabrina Dubroca * as per draft-ietf-ipsec-udp-encaps-06, 6760146dca7SSabrina Dubroca * section 3.1.2 6770146dca7SSabrina Dubroca */ 6780146dca7SSabrina Dubroca if (x->props.mode == XFRM_MODE_TRANSPORT) 6790146dca7SSabrina Dubroca skb->ip_summed = CHECKSUM_UNNECESSARY; 6800146dca7SSabrina Dubroca } 6810146dca7SSabrina Dubroca 68247ebcc0bSYossi Kuperman skb_postpull_rcsum(skb, skb_network_header(skb), 68347ebcc0bSYossi Kuperman skb_network_header_len(skb)); 684e51a6472SIlan Tayari skb_pull_rcsum(skb, hlen); 685a9403f8aSLi RongQing if (x->props.mode == XFRM_MODE_TUNNEL) 686a9403f8aSLi RongQing skb_reset_transport_header(skb); 687a9403f8aSLi RongQing else 68838320c70SHerbert Xu skb_set_transport_header(skb, -hdr_len); 68938320c70SHerbert Xu 69038320c70SHerbert Xu /* RFC4303: Drop dummy packets without any error */ 69138320c70SHerbert Xu if (err == IPPROTO_NONE) 69238320c70SHerbert Xu err = -EINVAL; 69338320c70SHerbert Xu 69438320c70SHerbert Xu out: 69538320c70SHerbert Xu return err; 69638320c70SHerbert Xu } 697383d0350SSteffen Klassert EXPORT_SYMBOL_GPL(esp6_input_done2); 69838320c70SHerbert Xu 69938320c70SHerbert Xu static void esp_input_done(struct crypto_async_request *base, int err) 70038320c70SHerbert Xu { 70138320c70SHerbert Xu struct sk_buff *skb = base->data; 70238320c70SHerbert Xu 703f1fbed0eSSteffen Klassert xfrm_input_resume(skb, esp6_input_done2(skb, err)); 70438320c70SHerbert Xu } 70538320c70SHerbert Xu 706000ae7b2SHerbert Xu static void esp_input_restore_header(struct sk_buff *skb) 707000ae7b2SHerbert Xu { 708000ae7b2SHerbert Xu esp_restore_header(skb, 0); 709000ae7b2SHerbert Xu __skb_pull(skb, 4); 710000ae7b2SHerbert Xu } 711000ae7b2SHerbert Xu 71203e2a30fSSteffen Klassert static void esp_input_set_header(struct sk_buff *skb, __be32 *seqhi) 71303e2a30fSSteffen Klassert { 71403e2a30fSSteffen Klassert struct xfrm_state *x = xfrm_input_state(skb); 71503e2a30fSSteffen Klassert 71603e2a30fSSteffen Klassert /* For ESN we move the header forward by 4 bytes to 71703e2a30fSSteffen Klassert * accomodate the high bits. We will move it back after 71803e2a30fSSteffen Klassert * decryption. 71903e2a30fSSteffen Klassert */ 72003e2a30fSSteffen Klassert if ((x->props.flags & XFRM_STATE_ESN)) { 721d3cc547dSColin Ian King struct ip_esp_hdr *esph = skb_push(skb, 4); 722d3cc547dSColin Ian King 72303e2a30fSSteffen Klassert *seqhi = esph->spi; 72403e2a30fSSteffen Klassert esph->spi = esph->seq_no; 72503e2a30fSSteffen Klassert esph->seq_no = XFRM_SKB_CB(skb)->seq.input.hi; 72603e2a30fSSteffen Klassert } 72703e2a30fSSteffen Klassert } 72803e2a30fSSteffen Klassert 729000ae7b2SHerbert Xu static void esp_input_done_esn(struct crypto_async_request *base, int err) 730000ae7b2SHerbert Xu { 731000ae7b2SHerbert Xu struct sk_buff *skb = base->data; 732000ae7b2SHerbert Xu 733000ae7b2SHerbert Xu esp_input_restore_header(skb); 734000ae7b2SHerbert Xu esp_input_done(base, err); 735000ae7b2SHerbert Xu } 736000ae7b2SHerbert Xu 737e695633eSHerbert Xu static int esp6_input(struct xfrm_state *x, struct sk_buff *skb) 7381da177e4SLinus Torvalds { 7391c5ad13fSMathias Krause struct crypto_aead *aead = x->data; 74038320c70SHerbert Xu struct aead_request *req; 7411da177e4SLinus Torvalds struct sk_buff *trailer; 742000ae7b2SHerbert Xu int ivlen = crypto_aead_ivsize(aead); 7430c05f983SHaishuang Yan int elen = skb->len - sizeof(struct ip_esp_hdr) - ivlen; 7441da177e4SLinus Torvalds int nfrags; 745d212a4c2SSteffen Klassert int assoclen; 746d212a4c2SSteffen Klassert int seqhilen; 7471da177e4SLinus Torvalds int ret = 0; 74838320c70SHerbert Xu void *tmp; 749d212a4c2SSteffen Klassert __be32 *seqhi; 75038320c70SHerbert Xu u8 *iv; 75138320c70SHerbert Xu struct scatterlist *sg; 7521da177e4SLinus Torvalds 7530c05f983SHaishuang Yan if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + ivlen)) { 7541da177e4SLinus Torvalds ret = -EINVAL; 75531a4ab93SHerbert Xu goto out; 7561da177e4SLinus Torvalds } 7571da177e4SLinus Torvalds 75838320c70SHerbert Xu if (elen <= 0) { 7591da177e4SLinus Torvalds ret = -EINVAL; 76031a4ab93SHerbert Xu goto out; 7611da177e4SLinus Torvalds } 7621da177e4SLinus Torvalds 7630c05f983SHaishuang Yan assoclen = sizeof(struct ip_esp_hdr); 764d212a4c2SSteffen Klassert seqhilen = 0; 765d212a4c2SSteffen Klassert 766d212a4c2SSteffen Klassert if (x->props.flags & XFRM_STATE_ESN) { 767d212a4c2SSteffen Klassert seqhilen += sizeof(__be32); 768d212a4c2SSteffen Klassert assoclen += seqhilen; 769d212a4c2SSteffen Klassert } 770d212a4c2SSteffen Klassert 77103e2a30fSSteffen Klassert if (!skb_cloned(skb)) { 77203e2a30fSSteffen Klassert if (!skb_is_nonlinear(skb)) { 77303e2a30fSSteffen Klassert nfrags = 1; 77403e2a30fSSteffen Klassert 77503e2a30fSSteffen Klassert goto skip_cow; 77603e2a30fSSteffen Klassert } else if (!skb_has_frag_list(skb)) { 77703e2a30fSSteffen Klassert nfrags = skb_shinfo(skb)->nr_frags; 77803e2a30fSSteffen Klassert nfrags++; 77903e2a30fSSteffen Klassert 78003e2a30fSSteffen Klassert goto skip_cow; 78103e2a30fSSteffen Klassert } 78203e2a30fSSteffen Klassert } 78303e2a30fSSteffen Klassert 78403e2a30fSSteffen Klassert nfrags = skb_cow_data(skb, 0, &trailer); 78503e2a30fSSteffen Klassert if (nfrags < 0) { 78603e2a30fSSteffen Klassert ret = -EINVAL; 78703e2a30fSSteffen Klassert goto out; 78803e2a30fSSteffen Klassert } 78903e2a30fSSteffen Klassert 79003e2a30fSSteffen Klassert skip_cow: 79103e2a30fSSteffen Klassert ret = -ENOMEM; 792000ae7b2SHerbert Xu tmp = esp_alloc_tmp(aead, nfrags, seqhilen); 79338320c70SHerbert Xu if (!tmp) 79438320c70SHerbert Xu goto out; 79538320c70SHerbert Xu 79638320c70SHerbert Xu ESP_SKB_CB(skb)->tmp = tmp; 7970146dca7SSabrina Dubroca seqhi = esp_tmp_extra(tmp); 798d212a4c2SSteffen Klassert iv = esp_tmp_iv(aead, tmp, seqhilen); 79938320c70SHerbert Xu req = esp_tmp_req(aead, iv); 800000ae7b2SHerbert Xu sg = esp_req_sg(aead, req); 80138320c70SHerbert Xu 80203e2a30fSSteffen Klassert esp_input_set_header(skb, seqhi); 8031da177e4SLinus Torvalds 804ed0e7e0cSDavid S. Miller sg_init_table(sg, nfrags); 8053f297707SJason A. Donenfeld ret = skb_to_sgvec(skb, sg, 0, skb->len); 8067284fdf3SZhen Lei if (unlikely(ret < 0)) { 8077284fdf3SZhen Lei kfree(tmp); 8083f297707SJason A. Donenfeld goto out; 8097284fdf3SZhen Lei } 810d212a4c2SSteffen Klassert 81103e2a30fSSteffen Klassert skb->ip_summed = CHECKSUM_NONE; 81203e2a30fSSteffen Klassert 81303e2a30fSSteffen Klassert if ((x->props.flags & XFRM_STATE_ESN)) 81403e2a30fSSteffen Klassert aead_request_set_callback(req, 0, esp_input_done_esn, skb); 81503e2a30fSSteffen Klassert else 81603e2a30fSSteffen Klassert aead_request_set_callback(req, 0, esp_input_done, skb); 81703e2a30fSSteffen Klassert 818000ae7b2SHerbert Xu aead_request_set_crypt(req, sg, sg, elen + ivlen, iv); 819000ae7b2SHerbert Xu aead_request_set_ad(req, assoclen); 8200ebea8efSHerbert Xu 82138320c70SHerbert Xu ret = crypto_aead_decrypt(req); 82238320c70SHerbert Xu if (ret == -EINPROGRESS) 8236b7326c8SHerbert Xu goto out; 8241da177e4SLinus Torvalds 825000ae7b2SHerbert Xu if ((x->props.flags & XFRM_STATE_ESN)) 826000ae7b2SHerbert Xu esp_input_restore_header(skb); 827000ae7b2SHerbert Xu 828f1fbed0eSSteffen Klassert ret = esp6_input_done2(skb, ret); 8290ebea8efSHerbert Xu 8301da177e4SLinus Torvalds out: 8311da177e4SLinus Torvalds return ret; 8321da177e4SLinus Torvalds } 8331da177e4SLinus Torvalds 834d5860c5cSSteffen Klassert static int esp6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, 835d5fdd6baSBrian Haley u8 type, u8 code, int offset, __be32 info) 8361da177e4SLinus Torvalds { 8374fb236baSAlexey Dobriyan struct net *net = dev_net(skb->dev); 838b71d1d42SEric Dumazet const struct ipv6hdr *iph = (const struct ipv6hdr *)skb->data; 83987bdc48dSHerbert Xu struct ip_esp_hdr *esph = (struct ip_esp_hdr *)(skb->data + offset); 8401da177e4SLinus Torvalds struct xfrm_state *x; 8411da177e4SLinus Torvalds 842b3b2b9e1SSteffen Klassert if (type != ICMPV6_PKT_TOOBIG && 843ec18d9a2SDavid S. Miller type != NDISC_REDIRECT) 844d5860c5cSSteffen Klassert return 0; 8451da177e4SLinus Torvalds 846b71d1d42SEric Dumazet x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr, 847b71d1d42SEric Dumazet esph->spi, IPPROTO_ESP, AF_INET6); 8481da177e4SLinus Torvalds if (!x) 849d5860c5cSSteffen Klassert return 0; 850ec18d9a2SDavid S. Miller 851ec18d9a2SDavid S. Miller if (type == NDISC_REDIRECT) 852e2d118a1SLorenzo Colitti ip6_redirect(skb, net, skb->dev->ifindex, 0, 853e2d118a1SLorenzo Colitti sock_net_uid(net, NULL)); 854ec18d9a2SDavid S. Miller else 855e2d118a1SLorenzo Colitti ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL)); 8561da177e4SLinus Torvalds xfrm_state_put(x); 857d5860c5cSSteffen Klassert 858d5860c5cSSteffen Klassert return 0; 8591da177e4SLinus Torvalds } 8601da177e4SLinus Torvalds 8611da177e4SLinus Torvalds static void esp6_destroy(struct xfrm_state *x) 8621da177e4SLinus Torvalds { 8631c5ad13fSMathias Krause struct crypto_aead *aead = x->data; 8641da177e4SLinus Torvalds 8651c5ad13fSMathias Krause if (!aead) 8661da177e4SLinus Torvalds return; 8671da177e4SLinus Torvalds 8681c5ad13fSMathias Krause crypto_free_aead(aead); 8691da177e4SLinus Torvalds } 8701da177e4SLinus Torvalds 8711a6509d9SHerbert Xu static int esp_init_aead(struct xfrm_state *x) 8721da177e4SLinus Torvalds { 873000ae7b2SHerbert Xu char aead_name[CRYPTO_MAX_ALG_NAME]; 8741a6509d9SHerbert Xu struct crypto_aead *aead; 8751a6509d9SHerbert Xu int err; 8761a6509d9SHerbert Xu 877000ae7b2SHerbert Xu err = -ENAMETOOLONG; 878000ae7b2SHerbert Xu if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", 879000ae7b2SHerbert Xu x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) 880000ae7b2SHerbert Xu goto error; 881000ae7b2SHerbert Xu 882f58869c4SSteffen Klassert aead = crypto_alloc_aead(aead_name, 0, 0); 8831a6509d9SHerbert Xu err = PTR_ERR(aead); 8841a6509d9SHerbert Xu if (IS_ERR(aead)) 8851a6509d9SHerbert Xu goto error; 8861a6509d9SHerbert Xu 8871c5ad13fSMathias Krause x->data = aead; 8881a6509d9SHerbert Xu 8891a6509d9SHerbert Xu err = crypto_aead_setkey(aead, x->aead->alg_key, 8901a6509d9SHerbert Xu (x->aead->alg_key_len + 7) / 8); 8911a6509d9SHerbert Xu if (err) 8921a6509d9SHerbert Xu goto error; 8931a6509d9SHerbert Xu 8941a6509d9SHerbert Xu err = crypto_aead_setauthsize(aead, x->aead->alg_icv_len / 8); 8951a6509d9SHerbert Xu if (err) 8961a6509d9SHerbert Xu goto error; 8971a6509d9SHerbert Xu 8981a6509d9SHerbert Xu error: 8991a6509d9SHerbert Xu return err; 9001a6509d9SHerbert Xu } 9011a6509d9SHerbert Xu 9021a6509d9SHerbert Xu static int esp_init_authenc(struct xfrm_state *x) 9031a6509d9SHerbert Xu { 90438320c70SHerbert Xu struct crypto_aead *aead; 90538320c70SHerbert Xu struct crypto_authenc_key_param *param; 90638320c70SHerbert Xu struct rtattr *rta; 90738320c70SHerbert Xu char *key; 90838320c70SHerbert Xu char *p; 90938320c70SHerbert Xu char authenc_name[CRYPTO_MAX_ALG_NAME]; 91038320c70SHerbert Xu unsigned int keylen; 91138320c70SHerbert Xu int err; 9121da177e4SLinus Torvalds 9131a6509d9SHerbert Xu err = -EINVAL; 91463159f29SIan Morris if (!x->ealg) 9151a6509d9SHerbert Xu goto error; 9161da177e4SLinus Torvalds 9171a6509d9SHerbert Xu err = -ENAMETOOLONG; 918d212a4c2SSteffen Klassert 919d212a4c2SSteffen Klassert if ((x->props.flags & XFRM_STATE_ESN)) { 920d212a4c2SSteffen Klassert if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME, 921000ae7b2SHerbert Xu "%s%sauthencesn(%s,%s)%s", 922000ae7b2SHerbert Xu x->geniv ?: "", x->geniv ? "(" : "", 92338320c70SHerbert Xu x->aalg ? x->aalg->alg_name : "digest_null", 924000ae7b2SHerbert Xu x->ealg->alg_name, 925000ae7b2SHerbert Xu x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) 9261a6509d9SHerbert Xu goto error; 927d212a4c2SSteffen Klassert } else { 928d212a4c2SSteffen Klassert if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME, 929000ae7b2SHerbert Xu "%s%sauthenc(%s,%s)%s", 930000ae7b2SHerbert Xu x->geniv ?: "", x->geniv ? "(" : "", 931d212a4c2SSteffen Klassert x->aalg ? x->aalg->alg_name : "digest_null", 932000ae7b2SHerbert Xu x->ealg->alg_name, 933000ae7b2SHerbert Xu x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) 934d212a4c2SSteffen Klassert goto error; 935d212a4c2SSteffen Klassert } 93638320c70SHerbert Xu 937f58869c4SSteffen Klassert aead = crypto_alloc_aead(authenc_name, 0, 0); 93838320c70SHerbert Xu err = PTR_ERR(aead); 93938320c70SHerbert Xu if (IS_ERR(aead)) 94038320c70SHerbert Xu goto error; 94138320c70SHerbert Xu 9421c5ad13fSMathias Krause x->data = aead; 94338320c70SHerbert Xu 94438320c70SHerbert Xu keylen = (x->aalg ? (x->aalg->alg_key_len + 7) / 8 : 0) + 94538320c70SHerbert Xu (x->ealg->alg_key_len + 7) / 8 + RTA_SPACE(sizeof(*param)); 94638320c70SHerbert Xu err = -ENOMEM; 94738320c70SHerbert Xu key = kmalloc(keylen, GFP_KERNEL); 94838320c70SHerbert Xu if (!key) 94938320c70SHerbert Xu goto error; 95038320c70SHerbert Xu 95138320c70SHerbert Xu p = key; 95238320c70SHerbert Xu rta = (void *)p; 95338320c70SHerbert Xu rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM; 95438320c70SHerbert Xu rta->rta_len = RTA_LENGTH(sizeof(*param)); 95538320c70SHerbert Xu param = RTA_DATA(rta); 95638320c70SHerbert Xu p += RTA_SPACE(sizeof(*param)); 95738320c70SHerbert Xu 9581da177e4SLinus Torvalds if (x->aalg) { 9591da177e4SLinus Torvalds struct xfrm_algo_desc *aalg_desc; 9601da177e4SLinus Torvalds 96138320c70SHerbert Xu memcpy(p, x->aalg->alg_key, (x->aalg->alg_key_len + 7) / 8); 96238320c70SHerbert Xu p += (x->aalg->alg_key_len + 7) / 8; 9631da177e4SLinus Torvalds 9641da177e4SLinus Torvalds aalg_desc = xfrm_aalg_get_byname(x->aalg->alg_name, 0); 9651da177e4SLinus Torvalds BUG_ON(!aalg_desc); 9661da177e4SLinus Torvalds 96738320c70SHerbert Xu err = -EINVAL; 9681da177e4SLinus Torvalds if (aalg_desc->uinfo.auth.icv_fullbits / 8 != 96938320c70SHerbert Xu crypto_aead_authsize(aead)) { 97045083497SJoe Perches pr_info("ESP: %s digestsize %u != %hu\n", 9711da177e4SLinus Torvalds x->aalg->alg_name, 97238320c70SHerbert Xu crypto_aead_authsize(aead), 9731da177e4SLinus Torvalds aalg_desc->uinfo.auth.icv_fullbits / 8); 97438320c70SHerbert Xu goto free_key; 9751da177e4SLinus Torvalds } 9761da177e4SLinus Torvalds 97738320c70SHerbert Xu err = crypto_aead_setauthsize( 9788f8a088cSMartin Willi aead, x->aalg->alg_trunc_len / 8); 97938320c70SHerbert Xu if (err) 98038320c70SHerbert Xu goto free_key; 98138320c70SHerbert Xu } 9821da177e4SLinus Torvalds 98338320c70SHerbert Xu param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8); 98438320c70SHerbert Xu memcpy(p, x->ealg->alg_key, (x->ealg->alg_key_len + 7) / 8); 98538320c70SHerbert Xu 98638320c70SHerbert Xu err = crypto_aead_setkey(aead, key, keylen); 98738320c70SHerbert Xu 98838320c70SHerbert Xu free_key: 98938320c70SHerbert Xu kfree(key); 99038320c70SHerbert Xu 9911a6509d9SHerbert Xu error: 9921a6509d9SHerbert Xu return err; 9931a6509d9SHerbert Xu } 9941a6509d9SHerbert Xu 9951a6509d9SHerbert Xu static int esp6_init_state(struct xfrm_state *x) 9961a6509d9SHerbert Xu { 9971a6509d9SHerbert Xu struct crypto_aead *aead; 9981a6509d9SHerbert Xu u32 align; 9991a6509d9SHerbert Xu int err; 10001a6509d9SHerbert Xu 10011c5ad13fSMathias Krause x->data = NULL; 10021a6509d9SHerbert Xu 10031a6509d9SHerbert Xu if (x->aead) 10041a6509d9SHerbert Xu err = esp_init_aead(x); 10051a6509d9SHerbert Xu else 10061a6509d9SHerbert Xu err = esp_init_authenc(x); 10071a6509d9SHerbert Xu 100838320c70SHerbert Xu if (err) 10091da177e4SLinus Torvalds goto error; 101038320c70SHerbert Xu 10111c5ad13fSMathias Krause aead = x->data; 10121a6509d9SHerbert Xu 101338320c70SHerbert Xu x->props.header_len = sizeof(struct ip_esp_hdr) + 101438320c70SHerbert Xu crypto_aead_ivsize(aead); 1015ca68145fSHerbert Xu switch (x->props.mode) { 1016ca68145fSHerbert Xu case XFRM_MODE_BEET: 1017abf5cdb8SJoakim Koskela if (x->sel.family != AF_INET6) 1018abf5cdb8SJoakim Koskela x->props.header_len += IPV4_BEET_PHMAXLEN + 1019abf5cdb8SJoakim Koskela (sizeof(struct ipv6hdr) - sizeof(struct iphdr)); 1020abf5cdb8SJoakim Koskela break; 1021bcfd09f7SHerbert Xu default: 1022ca68145fSHerbert Xu case XFRM_MODE_TRANSPORT: 1023ca68145fSHerbert Xu break; 1024ca68145fSHerbert Xu case XFRM_MODE_TUNNEL: 10251da177e4SLinus Torvalds x->props.header_len += sizeof(struct ipv6hdr); 1026ea2c47b4SMasahide NAKAMURA break; 1027ca68145fSHerbert Xu } 102838320c70SHerbert Xu 10290146dca7SSabrina Dubroca if (x->encap) { 10300146dca7SSabrina Dubroca struct xfrm_encap_tmpl *encap = x->encap; 10310146dca7SSabrina Dubroca 10320146dca7SSabrina Dubroca switch (encap->encap_type) { 10330146dca7SSabrina Dubroca default: 10340146dca7SSabrina Dubroca err = -EINVAL; 10350146dca7SSabrina Dubroca goto error; 10360146dca7SSabrina Dubroca case UDP_ENCAP_ESPINUDP: 10370146dca7SSabrina Dubroca x->props.header_len += sizeof(struct udphdr); 10380146dca7SSabrina Dubroca break; 10390146dca7SSabrina Dubroca case UDP_ENCAP_ESPINUDP_NON_IKE: 10400146dca7SSabrina Dubroca x->props.header_len += sizeof(struct udphdr) + 2 * sizeof(u32); 10410146dca7SSabrina Dubroca break; 10420146dca7SSabrina Dubroca } 10430146dca7SSabrina Dubroca } 10440146dca7SSabrina Dubroca 104538320c70SHerbert Xu align = ALIGN(crypto_aead_blocksize(aead), 4); 10461c5ad13fSMathias Krause x->props.trailer_len = align + 1 + crypto_aead_authsize(aead); 10471da177e4SLinus Torvalds 10481da177e4SLinus Torvalds error: 104938320c70SHerbert Xu return err; 10501da177e4SLinus Torvalds } 10511da177e4SLinus Torvalds 1052d5860c5cSSteffen Klassert static int esp6_rcv_cb(struct sk_buff *skb, int err) 1053d5860c5cSSteffen Klassert { 1054d5860c5cSSteffen Klassert return 0; 1055d5860c5cSSteffen Klassert } 1056d5860c5cSSteffen Klassert 1057cc24becaSIan Morris static const struct xfrm_type esp6_type = { 10581da177e4SLinus Torvalds .description = "ESP6", 10591da177e4SLinus Torvalds .owner = THIS_MODULE, 10601da177e4SLinus Torvalds .proto = IPPROTO_ESP, 1061436a0a40SHerbert Xu .flags = XFRM_TYPE_REPLAY_PROT, 10621da177e4SLinus Torvalds .init_state = esp6_init_state, 10631da177e4SLinus Torvalds .destructor = esp6_destroy, 10641da177e4SLinus Torvalds .input = esp6_input, 1065aee5adb4SMasahide NAKAMURA .output = esp6_output, 1066aee5adb4SMasahide NAKAMURA .hdr_offset = xfrm6_find_1stfragopt, 10671da177e4SLinus Torvalds }; 10681da177e4SLinus Torvalds 1069d5860c5cSSteffen Klassert static struct xfrm6_protocol esp6_protocol = { 10701da177e4SLinus Torvalds .handler = xfrm6_rcv, 10710146dca7SSabrina Dubroca .input_handler = xfrm_input, 1072d5860c5cSSteffen Klassert .cb_handler = esp6_rcv_cb, 10731da177e4SLinus Torvalds .err_handler = esp6_err, 1074d5860c5cSSteffen Klassert .priority = 0, 10751da177e4SLinus Torvalds }; 10761da177e4SLinus Torvalds 10771da177e4SLinus Torvalds static int __init esp6_init(void) 10781da177e4SLinus Torvalds { 10791da177e4SLinus Torvalds if (xfrm_register_type(&esp6_type, AF_INET6) < 0) { 1080f3213831SJoe Perches pr_info("%s: can't add xfrm type\n", __func__); 10811da177e4SLinus Torvalds return -EAGAIN; 10821da177e4SLinus Torvalds } 1083d5860c5cSSteffen Klassert if (xfrm6_protocol_register(&esp6_protocol, IPPROTO_ESP) < 0) { 1084f3213831SJoe Perches pr_info("%s: can't add protocol\n", __func__); 10851da177e4SLinus Torvalds xfrm_unregister_type(&esp6_type, AF_INET6); 10861da177e4SLinus Torvalds return -EAGAIN; 10871da177e4SLinus Torvalds } 10881da177e4SLinus Torvalds 10891da177e4SLinus Torvalds return 0; 10901da177e4SLinus Torvalds } 10911da177e4SLinus Torvalds 10921da177e4SLinus Torvalds static void __exit esp6_fini(void) 10931da177e4SLinus Torvalds { 1094d5860c5cSSteffen Klassert if (xfrm6_protocol_deregister(&esp6_protocol, IPPROTO_ESP) < 0) 1095f3213831SJoe Perches pr_info("%s: can't remove protocol\n", __func__); 10964f518e80SFlorian Westphal xfrm_unregister_type(&esp6_type, AF_INET6); 10971da177e4SLinus Torvalds } 10981da177e4SLinus Torvalds 10991da177e4SLinus Torvalds module_init(esp6_init); 11001da177e4SLinus Torvalds module_exit(esp6_fini); 11011da177e4SLinus Torvalds 11021da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1103d3d6dd3aSMasahide NAKAMURA MODULE_ALIAS_XFRM_TYPE(AF_INET6, XFRM_PROTO_ESP); 1104