1 /* 2 * IPV6 GSO/GRO offload support 3 * Linux INET implementation 4 * 5 * Copyright (C) 2016 secunet Security Networks AG 6 * Author: Steffen Klassert <steffen.klassert@secunet.com> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms and conditions of the GNU General Public License, 10 * version 2, as published by the Free Software Foundation. 11 * 12 * ESP GRO support 13 */ 14 15 #include <linux/skbuff.h> 16 #include <linux/init.h> 17 #include <net/protocol.h> 18 #include <crypto/aead.h> 19 #include <crypto/authenc.h> 20 #include <linux/err.h> 21 #include <linux/module.h> 22 #include <net/ip.h> 23 #include <net/xfrm.h> 24 #include <net/esp.h> 25 #include <linux/scatterlist.h> 26 #include <linux/kernel.h> 27 #include <linux/slab.h> 28 #include <linux/spinlock.h> 29 #include <net/ip6_route.h> 30 #include <net/ipv6.h> 31 #include <linux/icmpv6.h> 32 33 static __u16 esp6_nexthdr_esp_offset(struct ipv6hdr *ipv6_hdr, int nhlen) 34 { 35 int off = sizeof(struct ipv6hdr); 36 struct ipv6_opt_hdr *exthdr; 37 38 if (likely(ipv6_hdr->nexthdr == NEXTHDR_ESP)) 39 return offsetof(struct ipv6hdr, nexthdr); 40 41 while (off < nhlen) { 42 exthdr = (void *)ipv6_hdr + off; 43 if (exthdr->nexthdr == NEXTHDR_ESP) 44 return off; 45 46 off += ipv6_optlen(exthdr); 47 } 48 49 return 0; 50 } 51 52 static struct sk_buff **esp6_gro_receive(struct sk_buff **head, 53 struct sk_buff *skb) 54 { 55 int offset = skb_gro_offset(skb); 56 struct xfrm_offload *xo; 57 struct xfrm_state *x; 58 __be32 seq; 59 __be32 spi; 60 int nhoff; 61 int err; 62 63 skb_pull(skb, offset); 64 65 if ((err = xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq)) != 0) 66 goto out; 67 68 xo = xfrm_offload(skb); 69 if (!xo || !(xo->flags & CRYPTO_DONE)) { 70 err = secpath_set(skb); 71 if (err) 72 goto out; 73 74 if (skb->sp->len == XFRM_MAX_DEPTH) 75 goto out; 76 77 x = xfrm_state_lookup(dev_net(skb->dev), skb->mark, 78 (xfrm_address_t *)&ipv6_hdr(skb)->daddr, 79 spi, IPPROTO_ESP, AF_INET6); 80 if (!x) 81 goto out; 82 83 skb->sp->xvec[skb->sp->len++] = x; 84 skb->sp->olen++; 85 86 xo = xfrm_offload(skb); 87 if (!xo) { 88 xfrm_state_put(x); 89 goto out; 90 } 91 } 92 93 xo->flags |= XFRM_GRO; 94 95 nhoff = esp6_nexthdr_esp_offset(ipv6_hdr(skb), offset); 96 if (!nhoff) 97 goto out; 98 99 IP6CB(skb)->nhoff = nhoff; 100 XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL; 101 XFRM_SPI_SKB_CB(skb)->family = AF_INET6; 102 XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr); 103 XFRM_SPI_SKB_CB(skb)->seq = seq; 104 105 /* We don't need to handle errors from xfrm_input, it does all 106 * the error handling and frees the resources on error. */ 107 xfrm_input(skb, IPPROTO_ESP, spi, -2); 108 109 return ERR_PTR(-EINPROGRESS); 110 out: 111 skb_push(skb, offset); 112 NAPI_GRO_CB(skb)->same_flow = 0; 113 NAPI_GRO_CB(skb)->flush = 1; 114 115 return NULL; 116 } 117 118 static void esp6_gso_encap(struct xfrm_state *x, struct sk_buff *skb) 119 { 120 struct ip_esp_hdr *esph; 121 struct ipv6hdr *iph = ipv6_hdr(skb); 122 struct xfrm_offload *xo = xfrm_offload(skb); 123 int proto = iph->nexthdr; 124 125 skb_push(skb, -skb_network_offset(skb)); 126 esph = ip_esp_hdr(skb); 127 *skb_mac_header(skb) = IPPROTO_ESP; 128 129 esph->spi = x->id.spi; 130 esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low); 131 132 xo->proto = proto; 133 } 134 135 static struct sk_buff *esp6_gso_segment(struct sk_buff *skb, 136 netdev_features_t features) 137 { 138 struct xfrm_state *x; 139 struct ip_esp_hdr *esph; 140 struct crypto_aead *aead; 141 netdev_features_t esp_features = features; 142 struct xfrm_offload *xo = xfrm_offload(skb); 143 144 if (!xo) 145 return ERR_PTR(-EINVAL); 146 147 x = skb->sp->xvec[skb->sp->len - 1]; 148 aead = x->data; 149 esph = ip_esp_hdr(skb); 150 151 if (esph->spi != x->id.spi) 152 return ERR_PTR(-EINVAL); 153 154 if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead))) 155 return ERR_PTR(-EINVAL); 156 157 __skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)); 158 159 skb->encap_hdr_csum = 1; 160 161 if (!(features & NETIF_F_HW_ESP) || !x->xso.offload_handle || 162 (x->xso.dev != skb->dev)) 163 esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK); 164 165 xo->flags |= XFRM_GSO_SEGMENT; 166 167 return x->outer_mode->gso_segment(x, skb, esp_features); 168 } 169 170 static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb) 171 { 172 struct crypto_aead *aead = x->data; 173 struct xfrm_offload *xo = xfrm_offload(skb); 174 175 if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead))) 176 return -EINVAL; 177 178 if (!(xo->flags & CRYPTO_DONE)) 179 skb->ip_summed = CHECKSUM_NONE; 180 181 return esp6_input_done2(skb, 0); 182 } 183 184 static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) 185 { 186 int len; 187 int err; 188 int alen; 189 int blksize; 190 struct xfrm_offload *xo; 191 struct ip_esp_hdr *esph; 192 struct crypto_aead *aead; 193 struct esp_info esp; 194 bool hw_offload = true; 195 __u32 seq; 196 197 esp.inplace = true; 198 199 xo = xfrm_offload(skb); 200 201 if (!xo) 202 return -EINVAL; 203 204 if (!(features & NETIF_F_HW_ESP) || !x->xso.offload_handle || 205 (x->xso.dev != skb->dev)) { 206 xo->flags |= CRYPTO_FALLBACK; 207 hw_offload = false; 208 } 209 210 esp.proto = xo->proto; 211 212 /* skb is pure payload to encrypt */ 213 214 aead = x->data; 215 alen = crypto_aead_authsize(aead); 216 217 esp.tfclen = 0; 218 /* XXX: Add support for tfc padding here. */ 219 220 blksize = ALIGN(crypto_aead_blocksize(aead), 4); 221 esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize); 222 esp.plen = esp.clen - skb->len - esp.tfclen; 223 esp.tailen = esp.tfclen + esp.plen + alen; 224 225 if (!hw_offload || (hw_offload && !skb_is_gso(skb))) { 226 esp.nfrags = esp6_output_head(x, skb, &esp); 227 if (esp.nfrags < 0) 228 return esp.nfrags; 229 } 230 231 seq = xo->seq.low; 232 233 esph = ip_esp_hdr(skb); 234 esph->spi = x->id.spi; 235 236 skb_push(skb, -skb_network_offset(skb)); 237 238 if (xo->flags & XFRM_GSO_SEGMENT) { 239 esph->seq_no = htonl(seq); 240 241 if (!skb_is_gso(skb)) 242 xo->seq.low++; 243 else 244 xo->seq.low += skb_shinfo(skb)->gso_segs; 245 } 246 247 esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32)); 248 249 len = skb->len - sizeof(struct ipv6hdr); 250 if (len > IPV6_MAXPLEN) 251 len = 0; 252 253 ipv6_hdr(skb)->payload_len = htons(len); 254 255 if (hw_offload) 256 return 0; 257 258 err = esp6_output_tail(x, skb, &esp); 259 if (err) 260 return err; 261 262 secpath_reset(skb); 263 264 return 0; 265 } 266 267 static const struct net_offload esp6_offload = { 268 .callbacks = { 269 .gro_receive = esp6_gro_receive, 270 .gso_segment = esp6_gso_segment, 271 }, 272 }; 273 274 static const struct xfrm_type_offload esp6_type_offload = { 275 .description = "ESP6 OFFLOAD", 276 .owner = THIS_MODULE, 277 .proto = IPPROTO_ESP, 278 .input_tail = esp6_input_tail, 279 .xmit = esp6_xmit, 280 .encap = esp6_gso_encap, 281 }; 282 283 static int __init esp6_offload_init(void) 284 { 285 if (xfrm_register_type_offload(&esp6_type_offload, AF_INET6) < 0) { 286 pr_info("%s: can't add xfrm type offload\n", __func__); 287 return -EAGAIN; 288 } 289 290 return inet6_add_offload(&esp6_offload, IPPROTO_ESP); 291 } 292 293 static void __exit esp6_offload_exit(void) 294 { 295 if (xfrm_unregister_type_offload(&esp6_type_offload, AF_INET6) < 0) 296 pr_info("%s: can't remove xfrm type offload\n", __func__); 297 298 inet6_del_offload(&esp6_offload, IPPROTO_ESP); 299 } 300 301 module_init(esp6_offload_init); 302 module_exit(esp6_offload_exit); 303 MODULE_LICENSE("GPL"); 304 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); 305 MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6, XFRM_PROTO_ESP); 306