1 #include <crypto/aead.h> 2 #include <crypto/authenc.h> 3 #include <linux/err.h> 4 #include <linux/module.h> 5 #include <net/ip.h> 6 #include <net/xfrm.h> 7 #include <net/esp.h> 8 #include <linux/scatterlist.h> 9 #include <linux/kernel.h> 10 #include <linux/pfkeyv2.h> 11 #include <linux/rtnetlink.h> 12 #include <linux/slab.h> 13 #include <linux/spinlock.h> 14 #include <linux/in6.h> 15 #include <net/icmp.h> 16 #include <net/protocol.h> 17 #include <net/udp.h> 18 19 struct esp_skb_cb { 20 struct xfrm_skb_cb xfrm; 21 void *tmp; 22 }; 23 24 #define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0])) 25 26 static u32 esp4_get_mtu(struct xfrm_state *x, int mtu); 27 28 /* 29 * Allocate an AEAD request structure with extra space for SG and IV. 30 * 31 * For alignment considerations the IV is placed at the front, followed 32 * by the request and finally the SG list. 33 * 34 * TODO: Use spare space in skb for this where possible. 35 */ 36 static void *esp_alloc_tmp(struct crypto_aead *aead, int nfrags) 37 { 38 unsigned int len; 39 40 len = crypto_aead_ivsize(aead); 41 if (len) { 42 len += crypto_aead_alignmask(aead) & 43 ~(crypto_tfm_ctx_alignment() - 1); 44 len = ALIGN(len, crypto_tfm_ctx_alignment()); 45 } 46 47 len += sizeof(struct aead_givcrypt_request) + crypto_aead_reqsize(aead); 48 len = ALIGN(len, __alignof__(struct scatterlist)); 49 50 len += sizeof(struct scatterlist) * nfrags; 51 52 return kmalloc(len, GFP_ATOMIC); 53 } 54 55 static inline u8 *esp_tmp_iv(struct crypto_aead *aead, void *tmp) 56 { 57 return crypto_aead_ivsize(aead) ? 58 PTR_ALIGN((u8 *)tmp, crypto_aead_alignmask(aead) + 1) : tmp; 59 } 60 61 static inline struct aead_givcrypt_request *esp_tmp_givreq( 62 struct crypto_aead *aead, u8 *iv) 63 { 64 struct aead_givcrypt_request *req; 65 66 req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead), 67 crypto_tfm_ctx_alignment()); 68 aead_givcrypt_set_tfm(req, aead); 69 return req; 70 } 71 72 static inline struct aead_request *esp_tmp_req(struct crypto_aead *aead, u8 *iv) 73 { 74 struct aead_request *req; 75 76 req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead), 77 crypto_tfm_ctx_alignment()); 78 aead_request_set_tfm(req, aead); 79 return req; 80 } 81 82 static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead, 83 struct aead_request *req) 84 { 85 return (void *)ALIGN((unsigned long)(req + 1) + 86 crypto_aead_reqsize(aead), 87 __alignof__(struct scatterlist)); 88 } 89 90 static inline struct scatterlist *esp_givreq_sg( 91 struct crypto_aead *aead, struct aead_givcrypt_request *req) 92 { 93 return (void *)ALIGN((unsigned long)(req + 1) + 94 crypto_aead_reqsize(aead), 95 __alignof__(struct scatterlist)); 96 } 97 98 static void esp_output_done(struct crypto_async_request *base, int err) 99 { 100 struct sk_buff *skb = base->data; 101 102 kfree(ESP_SKB_CB(skb)->tmp); 103 xfrm_output_resume(skb, err); 104 } 105 106 static int esp_output(struct xfrm_state *x, struct sk_buff *skb) 107 { 108 int err; 109 struct ip_esp_hdr *esph; 110 struct crypto_aead *aead; 111 struct aead_givcrypt_request *req; 112 struct scatterlist *sg; 113 struct scatterlist *asg; 114 struct esp_data *esp; 115 struct sk_buff *trailer; 116 void *tmp; 117 u8 *iv; 118 u8 *tail; 119 int blksize; 120 int clen; 121 int alen; 122 int plen; 123 int tfclen; 124 int nfrags; 125 126 /* skb is pure payload to encrypt */ 127 128 err = -ENOMEM; 129 130 esp = x->data; 131 aead = esp->aead; 132 alen = crypto_aead_authsize(aead); 133 134 tfclen = 0; 135 if (x->tfcpad) { 136 struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb); 137 u32 padto; 138 139 padto = min(x->tfcpad, esp4_get_mtu(x, dst->child_mtu_cached)); 140 if (skb->len < padto) 141 tfclen = padto - skb->len; 142 } 143 blksize = ALIGN(crypto_aead_blocksize(aead), 4); 144 clen = ALIGN(skb->len + 2 + tfclen, blksize); 145 if (esp->padlen) 146 clen = ALIGN(clen, esp->padlen); 147 plen = clen - skb->len - tfclen; 148 149 err = skb_cow_data(skb, tfclen + plen + alen, &trailer); 150 if (err < 0) 151 goto error; 152 nfrags = err; 153 154 tmp = esp_alloc_tmp(aead, nfrags + 1); 155 if (!tmp) 156 goto error; 157 158 iv = esp_tmp_iv(aead, tmp); 159 req = esp_tmp_givreq(aead, iv); 160 asg = esp_givreq_sg(aead, req); 161 sg = asg + 1; 162 163 /* Fill padding... */ 164 tail = skb_tail_pointer(trailer); 165 if (tfclen) { 166 memset(tail, 0, tfclen); 167 tail += tfclen; 168 } 169 do { 170 int i; 171 for (i = 0; i < plen - 2; i++) 172 tail[i] = i + 1; 173 } while (0); 174 tail[plen - 2] = plen - 2; 175 tail[plen - 1] = *skb_mac_header(skb); 176 pskb_put(skb, trailer, clen - skb->len + alen); 177 178 skb_push(skb, -skb_network_offset(skb)); 179 esph = ip_esp_hdr(skb); 180 *skb_mac_header(skb) = IPPROTO_ESP; 181 182 /* this is non-NULL only with UDP Encapsulation */ 183 if (x->encap) { 184 struct xfrm_encap_tmpl *encap = x->encap; 185 struct udphdr *uh; 186 __be32 *udpdata32; 187 __be16 sport, dport; 188 int encap_type; 189 190 spin_lock_bh(&x->lock); 191 sport = encap->encap_sport; 192 dport = encap->encap_dport; 193 encap_type = encap->encap_type; 194 spin_unlock_bh(&x->lock); 195 196 uh = (struct udphdr *)esph; 197 uh->source = sport; 198 uh->dest = dport; 199 uh->len = htons(skb->len - skb_transport_offset(skb)); 200 uh->check = 0; 201 202 switch (encap_type) { 203 default: 204 case UDP_ENCAP_ESPINUDP: 205 esph = (struct ip_esp_hdr *)(uh + 1); 206 break; 207 case UDP_ENCAP_ESPINUDP_NON_IKE: 208 udpdata32 = (__be32 *)(uh + 1); 209 udpdata32[0] = udpdata32[1] = 0; 210 esph = (struct ip_esp_hdr *)(udpdata32 + 2); 211 break; 212 } 213 214 *skb_mac_header(skb) = IPPROTO_UDP; 215 } 216 217 esph->spi = x->id.spi; 218 esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output); 219 220 sg_init_table(sg, nfrags); 221 skb_to_sgvec(skb, sg, 222 esph->enc_data + crypto_aead_ivsize(aead) - skb->data, 223 clen + alen); 224 sg_init_one(asg, esph, sizeof(*esph)); 225 226 aead_givcrypt_set_callback(req, 0, esp_output_done, skb); 227 aead_givcrypt_set_crypt(req, sg, sg, clen, iv); 228 aead_givcrypt_set_assoc(req, asg, sizeof(*esph)); 229 aead_givcrypt_set_giv(req, esph->enc_data, 230 XFRM_SKB_CB(skb)->seq.output); 231 232 ESP_SKB_CB(skb)->tmp = tmp; 233 err = crypto_aead_givencrypt(req); 234 if (err == -EINPROGRESS) 235 goto error; 236 237 if (err == -EBUSY) 238 err = NET_XMIT_DROP; 239 240 kfree(tmp); 241 242 error: 243 return err; 244 } 245 246 static int esp_input_done2(struct sk_buff *skb, int err) 247 { 248 struct iphdr *iph; 249 struct xfrm_state *x = xfrm_input_state(skb); 250 struct esp_data *esp = x->data; 251 struct crypto_aead *aead = esp->aead; 252 int alen = crypto_aead_authsize(aead); 253 int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead); 254 int elen = skb->len - hlen; 255 int ihl; 256 u8 nexthdr[2]; 257 int padlen; 258 259 kfree(ESP_SKB_CB(skb)->tmp); 260 261 if (unlikely(err)) 262 goto out; 263 264 if (skb_copy_bits(skb, skb->len-alen-2, nexthdr, 2)) 265 BUG(); 266 267 err = -EINVAL; 268 padlen = nexthdr[0]; 269 if (padlen + 2 + alen >= elen) 270 goto out; 271 272 /* ... check padding bits here. Silly. :-) */ 273 274 iph = ip_hdr(skb); 275 ihl = iph->ihl * 4; 276 277 if (x->encap) { 278 struct xfrm_encap_tmpl *encap = x->encap; 279 struct udphdr *uh = (void *)(skb_network_header(skb) + ihl); 280 281 /* 282 * 1) if the NAT-T peer's IP or port changed then 283 * advertize the change to the keying daemon. 284 * This is an inbound SA, so just compare 285 * SRC ports. 286 */ 287 if (iph->saddr != x->props.saddr.a4 || 288 uh->source != encap->encap_sport) { 289 xfrm_address_t ipaddr; 290 291 ipaddr.a4 = iph->saddr; 292 km_new_mapping(x, &ipaddr, uh->source); 293 294 /* XXX: perhaps add an extra 295 * policy check here, to see 296 * if we should allow or 297 * reject a packet from a 298 * different source 299 * address/port. 300 */ 301 } 302 303 /* 304 * 2) ignore UDP/TCP checksums in case 305 * of NAT-T in Transport Mode, or 306 * perform other post-processing fixes 307 * as per draft-ietf-ipsec-udp-encaps-06, 308 * section 3.1.2 309 */ 310 if (x->props.mode == XFRM_MODE_TRANSPORT) 311 skb->ip_summed = CHECKSUM_UNNECESSARY; 312 } 313 314 pskb_trim(skb, skb->len - alen - padlen - 2); 315 __skb_pull(skb, hlen); 316 skb_set_transport_header(skb, -ihl); 317 318 err = nexthdr[1]; 319 320 /* RFC4303: Drop dummy packets without any error */ 321 if (err == IPPROTO_NONE) 322 err = -EINVAL; 323 324 out: 325 return err; 326 } 327 328 static void esp_input_done(struct crypto_async_request *base, int err) 329 { 330 struct sk_buff *skb = base->data; 331 332 xfrm_input_resume(skb, esp_input_done2(skb, err)); 333 } 334 335 /* 336 * Note: detecting truncated vs. non-truncated authentication data is very 337 * expensive, so we only support truncated data, which is the recommended 338 * and common case. 339 */ 340 static int esp_input(struct xfrm_state *x, struct sk_buff *skb) 341 { 342 struct ip_esp_hdr *esph; 343 struct esp_data *esp = x->data; 344 struct crypto_aead *aead = esp->aead; 345 struct aead_request *req; 346 struct sk_buff *trailer; 347 int elen = skb->len - sizeof(*esph) - crypto_aead_ivsize(aead); 348 int nfrags; 349 void *tmp; 350 u8 *iv; 351 struct scatterlist *sg; 352 struct scatterlist *asg; 353 int err = -EINVAL; 354 355 if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead))) 356 goto out; 357 358 if (elen <= 0) 359 goto out; 360 361 if ((err = skb_cow_data(skb, 0, &trailer)) < 0) 362 goto out; 363 nfrags = err; 364 365 err = -ENOMEM; 366 tmp = esp_alloc_tmp(aead, nfrags + 1); 367 if (!tmp) 368 goto out; 369 370 ESP_SKB_CB(skb)->tmp = tmp; 371 iv = esp_tmp_iv(aead, tmp); 372 req = esp_tmp_req(aead, iv); 373 asg = esp_req_sg(aead, req); 374 sg = asg + 1; 375 376 skb->ip_summed = CHECKSUM_NONE; 377 378 esph = (struct ip_esp_hdr *)skb->data; 379 380 /* Get ivec. This can be wrong, check against another impls. */ 381 iv = esph->enc_data; 382 383 sg_init_table(sg, nfrags); 384 skb_to_sgvec(skb, sg, sizeof(*esph) + crypto_aead_ivsize(aead), elen); 385 sg_init_one(asg, esph, sizeof(*esph)); 386 387 aead_request_set_callback(req, 0, esp_input_done, skb); 388 aead_request_set_crypt(req, sg, sg, elen, iv); 389 aead_request_set_assoc(req, asg, sizeof(*esph)); 390 391 err = crypto_aead_decrypt(req); 392 if (err == -EINPROGRESS) 393 goto out; 394 395 err = esp_input_done2(skb, err); 396 397 out: 398 return err; 399 } 400 401 static u32 esp4_get_mtu(struct xfrm_state *x, int mtu) 402 { 403 struct esp_data *esp = x->data; 404 u32 blksize = ALIGN(crypto_aead_blocksize(esp->aead), 4); 405 u32 align = max_t(u32, blksize, esp->padlen); 406 u32 rem; 407 408 mtu -= x->props.header_len + crypto_aead_authsize(esp->aead); 409 rem = mtu & (align - 1); 410 mtu &= ~(align - 1); 411 412 switch (x->props.mode) { 413 case XFRM_MODE_TUNNEL: 414 break; 415 default: 416 case XFRM_MODE_TRANSPORT: 417 /* The worst case */ 418 mtu -= blksize - 4; 419 mtu += min_t(u32, blksize - 4, rem); 420 break; 421 case XFRM_MODE_BEET: 422 /* The worst case. */ 423 mtu += min_t(u32, IPV4_BEET_PHMAXLEN, rem); 424 break; 425 } 426 427 return mtu - 2; 428 } 429 430 static void esp4_err(struct sk_buff *skb, u32 info) 431 { 432 struct net *net = dev_net(skb->dev); 433 struct iphdr *iph = (struct iphdr *)skb->data; 434 struct ip_esp_hdr *esph = (struct ip_esp_hdr *)(skb->data+(iph->ihl<<2)); 435 struct xfrm_state *x; 436 437 if (icmp_hdr(skb)->type != ICMP_DEST_UNREACH || 438 icmp_hdr(skb)->code != ICMP_FRAG_NEEDED) 439 return; 440 441 x = xfrm_state_lookup(net, skb->mark, (xfrm_address_t *)&iph->daddr, esph->spi, IPPROTO_ESP, AF_INET); 442 if (!x) 443 return; 444 NETDEBUG(KERN_DEBUG "pmtu discovery on SA ESP/%08x/%08x\n", 445 ntohl(esph->spi), ntohl(iph->daddr)); 446 xfrm_state_put(x); 447 } 448 449 static void esp_destroy(struct xfrm_state *x) 450 { 451 struct esp_data *esp = x->data; 452 453 if (!esp) 454 return; 455 456 crypto_free_aead(esp->aead); 457 kfree(esp); 458 } 459 460 static int esp_init_aead(struct xfrm_state *x) 461 { 462 struct esp_data *esp = x->data; 463 struct crypto_aead *aead; 464 int err; 465 466 aead = crypto_alloc_aead(x->aead->alg_name, 0, 0); 467 err = PTR_ERR(aead); 468 if (IS_ERR(aead)) 469 goto error; 470 471 esp->aead = aead; 472 473 err = crypto_aead_setkey(aead, x->aead->alg_key, 474 (x->aead->alg_key_len + 7) / 8); 475 if (err) 476 goto error; 477 478 err = crypto_aead_setauthsize(aead, x->aead->alg_icv_len / 8); 479 if (err) 480 goto error; 481 482 error: 483 return err; 484 } 485 486 static int esp_init_authenc(struct xfrm_state *x) 487 { 488 struct esp_data *esp = x->data; 489 struct crypto_aead *aead; 490 struct crypto_authenc_key_param *param; 491 struct rtattr *rta; 492 char *key; 493 char *p; 494 char authenc_name[CRYPTO_MAX_ALG_NAME]; 495 unsigned int keylen; 496 int err; 497 498 err = -EINVAL; 499 if (x->ealg == NULL) 500 goto error; 501 502 err = -ENAMETOOLONG; 503 if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME, "authenc(%s,%s)", 504 x->aalg ? x->aalg->alg_name : "digest_null", 505 x->ealg->alg_name) >= CRYPTO_MAX_ALG_NAME) 506 goto error; 507 508 aead = crypto_alloc_aead(authenc_name, 0, 0); 509 err = PTR_ERR(aead); 510 if (IS_ERR(aead)) 511 goto error; 512 513 esp->aead = aead; 514 515 keylen = (x->aalg ? (x->aalg->alg_key_len + 7) / 8 : 0) + 516 (x->ealg->alg_key_len + 7) / 8 + RTA_SPACE(sizeof(*param)); 517 err = -ENOMEM; 518 key = kmalloc(keylen, GFP_KERNEL); 519 if (!key) 520 goto error; 521 522 p = key; 523 rta = (void *)p; 524 rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM; 525 rta->rta_len = RTA_LENGTH(sizeof(*param)); 526 param = RTA_DATA(rta); 527 p += RTA_SPACE(sizeof(*param)); 528 529 if (x->aalg) { 530 struct xfrm_algo_desc *aalg_desc; 531 532 memcpy(p, x->aalg->alg_key, (x->aalg->alg_key_len + 7) / 8); 533 p += (x->aalg->alg_key_len + 7) / 8; 534 535 aalg_desc = xfrm_aalg_get_byname(x->aalg->alg_name, 0); 536 BUG_ON(!aalg_desc); 537 538 err = -EINVAL; 539 if (aalg_desc->uinfo.auth.icv_fullbits/8 != 540 crypto_aead_authsize(aead)) { 541 NETDEBUG(KERN_INFO "ESP: %s digestsize %u != %hu\n", 542 x->aalg->alg_name, 543 crypto_aead_authsize(aead), 544 aalg_desc->uinfo.auth.icv_fullbits/8); 545 goto free_key; 546 } 547 548 err = crypto_aead_setauthsize( 549 aead, x->aalg->alg_trunc_len / 8); 550 if (err) 551 goto free_key; 552 } 553 554 param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8); 555 memcpy(p, x->ealg->alg_key, (x->ealg->alg_key_len + 7) / 8); 556 557 err = crypto_aead_setkey(aead, key, keylen); 558 559 free_key: 560 kfree(key); 561 562 error: 563 return err; 564 } 565 566 static int esp_init_state(struct xfrm_state *x) 567 { 568 struct esp_data *esp; 569 struct crypto_aead *aead; 570 u32 align; 571 int err; 572 573 esp = kzalloc(sizeof(*esp), GFP_KERNEL); 574 if (esp == NULL) 575 return -ENOMEM; 576 577 x->data = esp; 578 579 if (x->aead) 580 err = esp_init_aead(x); 581 else 582 err = esp_init_authenc(x); 583 584 if (err) 585 goto error; 586 587 aead = esp->aead; 588 589 esp->padlen = 0; 590 591 x->props.header_len = sizeof(struct ip_esp_hdr) + 592 crypto_aead_ivsize(aead); 593 if (x->props.mode == XFRM_MODE_TUNNEL) 594 x->props.header_len += sizeof(struct iphdr); 595 else if (x->props.mode == XFRM_MODE_BEET && x->sel.family != AF_INET6) 596 x->props.header_len += IPV4_BEET_PHMAXLEN; 597 if (x->encap) { 598 struct xfrm_encap_tmpl *encap = x->encap; 599 600 switch (encap->encap_type) { 601 default: 602 goto error; 603 case UDP_ENCAP_ESPINUDP: 604 x->props.header_len += sizeof(struct udphdr); 605 break; 606 case UDP_ENCAP_ESPINUDP_NON_IKE: 607 x->props.header_len += sizeof(struct udphdr) + 2 * sizeof(u32); 608 break; 609 } 610 } 611 612 align = ALIGN(crypto_aead_blocksize(aead), 4); 613 if (esp->padlen) 614 align = max_t(u32, align, esp->padlen); 615 x->props.trailer_len = align + 1 + crypto_aead_authsize(esp->aead); 616 617 error: 618 return err; 619 } 620 621 static const struct xfrm_type esp_type = 622 { 623 .description = "ESP4", 624 .owner = THIS_MODULE, 625 .proto = IPPROTO_ESP, 626 .flags = XFRM_TYPE_REPLAY_PROT, 627 .init_state = esp_init_state, 628 .destructor = esp_destroy, 629 .get_mtu = esp4_get_mtu, 630 .input = esp_input, 631 .output = esp_output 632 }; 633 634 static const struct net_protocol esp4_protocol = { 635 .handler = xfrm4_rcv, 636 .err_handler = esp4_err, 637 .no_policy = 1, 638 .netns_ok = 1, 639 }; 640 641 static int __init esp4_init(void) 642 { 643 if (xfrm_register_type(&esp_type, AF_INET) < 0) { 644 printk(KERN_INFO "ip esp init: can't add xfrm type\n"); 645 return -EAGAIN; 646 } 647 if (inet_add_protocol(&esp4_protocol, IPPROTO_ESP) < 0) { 648 printk(KERN_INFO "ip esp init: can't add protocol\n"); 649 xfrm_unregister_type(&esp_type, AF_INET); 650 return -EAGAIN; 651 } 652 return 0; 653 } 654 655 static void __exit esp4_fini(void) 656 { 657 if (inet_del_protocol(&esp4_protocol, IPPROTO_ESP) < 0) 658 printk(KERN_INFO "ip esp close: can't remove protocol\n"); 659 if (xfrm_unregister_type(&esp_type, AF_INET) < 0) 660 printk(KERN_INFO "ip esp close: can't remove xfrm type\n"); 661 } 662 663 module_init(esp4_init); 664 module_exit(esp4_fini); 665 MODULE_LICENSE("GPL"); 666 MODULE_ALIAS_XFRM_TYPE(AF_INET, XFRM_PROTO_ESP); 667