1 /* 2 * GCM: Galois/Counter Mode. 3 * 4 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation. 9 */ 10 11 #include <crypto/gf128mul.h> 12 #include <crypto/internal/aead.h> 13 #include <crypto/internal/skcipher.h> 14 #include <crypto/internal/hash.h> 15 #include <crypto/null.h> 16 #include <crypto/scatterwalk.h> 17 #include <crypto/hash.h> 18 #include "internal.h" 19 #include <linux/completion.h> 20 #include <linux/err.h> 21 #include <linux/init.h> 22 #include <linux/kernel.h> 23 #include <linux/module.h> 24 #include <linux/slab.h> 25 26 struct gcm_instance_ctx { 27 struct crypto_skcipher_spawn ctr; 28 struct crypto_ahash_spawn ghash; 29 }; 30 31 struct crypto_gcm_ctx { 32 struct crypto_ablkcipher *ctr; 33 struct crypto_ahash *ghash; 34 }; 35 36 struct crypto_rfc4106_ctx { 37 struct crypto_aead *child; 38 u8 nonce[4]; 39 }; 40 41 struct crypto_rfc4543_instance_ctx { 42 struct crypto_aead_spawn aead; 43 }; 44 45 struct crypto_rfc4543_ctx { 46 struct crypto_aead *child; 47 struct crypto_blkcipher *null; 48 u8 nonce[4]; 49 }; 50 51 struct crypto_rfc4543_req_ctx { 52 struct aead_request subreq; 53 }; 54 55 struct crypto_gcm_ghash_ctx { 56 unsigned int cryptlen; 57 struct scatterlist *src; 58 int (*complete)(struct aead_request *req, u32 flags); 59 }; 60 61 struct crypto_gcm_req_priv_ctx { 62 u8 iv[16]; 63 u8 auth_tag[16]; 64 u8 iauth_tag[16]; 65 struct scatterlist src[3]; 66 struct scatterlist dst[3]; 67 struct scatterlist sg; 68 struct crypto_gcm_ghash_ctx ghash_ctx; 69 union { 70 struct ahash_request ahreq; 71 struct ablkcipher_request abreq; 72 } u; 73 }; 74 75 struct crypto_gcm_setkey_result { 76 int err; 77 struct completion completion; 78 }; 79 80 static struct { 81 u8 buf[16]; 82 struct scatterlist sg; 83 } *gcm_zeroes; 84 85 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc); 86 87 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( 88 struct aead_request *req) 89 { 90 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); 91 92 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); 93 } 94 95 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) 96 { 97 struct crypto_gcm_setkey_result *result = req->data; 98 99 if (err == -EINPROGRESS) 100 return; 101 102 result->err = err; 103 complete(&result->completion); 104 } 105 106 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, 107 unsigned int keylen) 108 { 109 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 110 struct crypto_ahash *ghash = ctx->ghash; 111 struct crypto_ablkcipher *ctr = ctx->ctr; 112 struct { 113 be128 hash; 114 u8 iv[8]; 115 116 struct crypto_gcm_setkey_result result; 117 118 struct scatterlist sg[1]; 119 struct ablkcipher_request req; 120 } *data; 121 int err; 122 123 crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); 124 crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) & 125 CRYPTO_TFM_REQ_MASK); 126 err = crypto_ablkcipher_setkey(ctr, key, keylen); 127 crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) & 128 CRYPTO_TFM_RES_MASK); 129 if (err) 130 return err; 131 132 data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr), 133 GFP_KERNEL); 134 if (!data) 135 return -ENOMEM; 136 137 init_completion(&data->result.completion); 138 sg_init_one(data->sg, &data->hash, sizeof(data->hash)); 139 ablkcipher_request_set_tfm(&data->req, ctr); 140 ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | 141 CRYPTO_TFM_REQ_MAY_BACKLOG, 142 crypto_gcm_setkey_done, 143 &data->result); 144 ablkcipher_request_set_crypt(&data->req, data->sg, data->sg, 145 sizeof(data->hash), data->iv); 146 147 err = crypto_ablkcipher_encrypt(&data->req); 148 if (err == -EINPROGRESS || err == -EBUSY) { 149 err = wait_for_completion_interruptible( 150 &data->result.completion); 151 if (!err) 152 err = data->result.err; 153 } 154 155 if (err) 156 goto out; 157 158 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); 159 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & 160 CRYPTO_TFM_REQ_MASK); 161 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); 162 crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) & 163 CRYPTO_TFM_RES_MASK); 164 165 out: 166 kzfree(data); 167 return err; 168 } 169 170 static int crypto_gcm_setauthsize(struct crypto_aead *tfm, 171 unsigned int authsize) 172 { 173 switch (authsize) { 174 case 4: 175 case 8: 176 case 12: 177 case 13: 178 case 14: 179 case 15: 180 case 16: 181 break; 182 default: 183 return -EINVAL; 184 } 185 186 return 0; 187 } 188 189 static void crypto_gcm_init_common(struct aead_request *req) 190 { 191 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 192 __be32 counter = cpu_to_be32(1); 193 struct scatterlist *sg; 194 195 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); 196 memcpy(pctx->iv, req->iv, 12); 197 memcpy(pctx->iv + 12, &counter, 4); 198 199 sg_init_table(pctx->src, 3); 200 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); 201 sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); 202 if (sg != pctx->src + 1) 203 scatterwalk_sg_chain(pctx->src, 2, sg); 204 205 if (req->src != req->dst) { 206 sg_init_table(pctx->dst, 3); 207 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); 208 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); 209 if (sg != pctx->dst + 1) 210 scatterwalk_sg_chain(pctx->dst, 2, sg); 211 } 212 } 213 214 static void crypto_gcm_init_crypt(struct aead_request *req, 215 unsigned int cryptlen) 216 { 217 struct crypto_aead *aead = crypto_aead_reqtfm(req); 218 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 219 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 220 struct ablkcipher_request *ablk_req = &pctx->u.abreq; 221 struct scatterlist *dst; 222 223 dst = req->src == req->dst ? pctx->src : pctx->dst; 224 225 ablkcipher_request_set_tfm(ablk_req, ctx->ctr); 226 ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, 227 cryptlen + sizeof(pctx->auth_tag), 228 pctx->iv); 229 } 230 231 static inline unsigned int gcm_remain(unsigned int len) 232 { 233 len &= 0xfU; 234 return len ? 16 - len : 0; 235 } 236 237 static void gcm_hash_len_done(struct crypto_async_request *areq, int err); 238 239 static int gcm_hash_update(struct aead_request *req, 240 crypto_completion_t compl, 241 struct scatterlist *src, 242 unsigned int len, u32 flags) 243 { 244 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 245 struct ahash_request *ahreq = &pctx->u.ahreq; 246 247 ahash_request_set_callback(ahreq, flags, compl, req); 248 ahash_request_set_crypt(ahreq, src, NULL, len); 249 250 return crypto_ahash_update(ahreq); 251 } 252 253 static int gcm_hash_remain(struct aead_request *req, 254 unsigned int remain, 255 crypto_completion_t compl, u32 flags) 256 { 257 return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); 258 } 259 260 static int gcm_hash_len(struct aead_request *req, u32 flags) 261 { 262 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 263 struct ahash_request *ahreq = &pctx->u.ahreq; 264 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 265 u128 lengths; 266 267 lengths.a = cpu_to_be64(req->assoclen * 8); 268 lengths.b = cpu_to_be64(gctx->cryptlen * 8); 269 memcpy(pctx->iauth_tag, &lengths, 16); 270 sg_init_one(&pctx->sg, pctx->iauth_tag, 16); 271 ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); 272 ahash_request_set_crypt(ahreq, &pctx->sg, 273 pctx->iauth_tag, sizeof(lengths)); 274 275 return crypto_ahash_finup(ahreq); 276 } 277 278 static int gcm_hash_len_continue(struct aead_request *req, u32 flags) 279 { 280 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 281 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 282 283 return gctx->complete(req, flags); 284 } 285 286 static void gcm_hash_len_done(struct crypto_async_request *areq, int err) 287 { 288 struct aead_request *req = areq->data; 289 290 if (err) 291 goto out; 292 293 err = gcm_hash_len_continue(req, 0); 294 if (err == -EINPROGRESS) 295 return; 296 297 out: 298 aead_request_complete(req, err); 299 } 300 301 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) 302 { 303 return gcm_hash_len(req, flags) ?: 304 gcm_hash_len_continue(req, flags); 305 } 306 307 static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, 308 int err) 309 { 310 struct aead_request *req = areq->data; 311 312 if (err) 313 goto out; 314 315 err = gcm_hash_crypt_remain_continue(req, 0); 316 if (err == -EINPROGRESS) 317 return; 318 319 out: 320 aead_request_complete(req, err); 321 } 322 323 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) 324 { 325 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 326 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 327 unsigned int remain; 328 329 remain = gcm_remain(gctx->cryptlen); 330 if (remain) 331 return gcm_hash_remain(req, remain, 332 gcm_hash_crypt_remain_done, flags) ?: 333 gcm_hash_crypt_remain_continue(req, flags); 334 335 return gcm_hash_crypt_remain_continue(req, flags); 336 } 337 338 static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err) 339 { 340 struct aead_request *req = areq->data; 341 342 if (err) 343 goto out; 344 345 err = gcm_hash_crypt_continue(req, 0); 346 if (err == -EINPROGRESS) 347 return; 348 349 out: 350 aead_request_complete(req, err); 351 } 352 353 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) 354 { 355 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 356 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 357 358 if (gctx->cryptlen) 359 return gcm_hash_update(req, gcm_hash_crypt_done, 360 gctx->src, gctx->cryptlen, flags) ?: 361 gcm_hash_crypt_continue(req, flags); 362 363 return gcm_hash_crypt_remain_continue(req, flags); 364 } 365 366 static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, 367 int err) 368 { 369 struct aead_request *req = areq->data; 370 371 if (err) 372 goto out; 373 374 err = gcm_hash_assoc_remain_continue(req, 0); 375 if (err == -EINPROGRESS) 376 return; 377 378 out: 379 aead_request_complete(req, err); 380 } 381 382 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) 383 { 384 unsigned int remain; 385 386 remain = gcm_remain(req->assoclen); 387 if (remain) 388 return gcm_hash_remain(req, remain, 389 gcm_hash_assoc_remain_done, flags) ?: 390 gcm_hash_assoc_remain_continue(req, flags); 391 392 return gcm_hash_assoc_remain_continue(req, flags); 393 } 394 395 static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err) 396 { 397 struct aead_request *req = areq->data; 398 399 if (err) 400 goto out; 401 402 err = gcm_hash_assoc_continue(req, 0); 403 if (err == -EINPROGRESS) 404 return; 405 406 out: 407 aead_request_complete(req, err); 408 } 409 410 static int gcm_hash_init_continue(struct aead_request *req, u32 flags) 411 { 412 if (req->assoclen) 413 return gcm_hash_update(req, gcm_hash_assoc_done, 414 req->src, req->assoclen, flags) ?: 415 gcm_hash_assoc_continue(req, flags); 416 417 return gcm_hash_assoc_remain_continue(req, flags); 418 } 419 420 static void gcm_hash_init_done(struct crypto_async_request *areq, int err) 421 { 422 struct aead_request *req = areq->data; 423 424 if (err) 425 goto out; 426 427 err = gcm_hash_init_continue(req, 0); 428 if (err == -EINPROGRESS) 429 return; 430 431 out: 432 aead_request_complete(req, err); 433 } 434 435 static int gcm_hash(struct aead_request *req, u32 flags) 436 { 437 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 438 struct ahash_request *ahreq = &pctx->u.ahreq; 439 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 440 441 ahash_request_set_tfm(ahreq, ctx->ghash); 442 443 ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); 444 return crypto_ahash_init(ahreq) ?: 445 gcm_hash_init_continue(req, flags); 446 } 447 448 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) 449 { 450 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 451 struct crypto_aead *aead = crypto_aead_reqtfm(req); 452 u8 *auth_tag = pctx->auth_tag; 453 454 crypto_xor(auth_tag, pctx->iauth_tag, 16); 455 scatterwalk_map_and_copy(auth_tag, req->dst, 456 req->assoclen + req->cryptlen, 457 crypto_aead_authsize(aead), 1); 458 return 0; 459 } 460 461 static int gcm_encrypt_continue(struct aead_request *req, u32 flags) 462 { 463 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 464 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 465 466 gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); 467 gctx->cryptlen = req->cryptlen; 468 gctx->complete = gcm_enc_copy_hash; 469 470 return gcm_hash(req, flags); 471 } 472 473 static void gcm_encrypt_done(struct crypto_async_request *areq, int err) 474 { 475 struct aead_request *req = areq->data; 476 477 if (err) 478 goto out; 479 480 err = gcm_encrypt_continue(req, 0); 481 if (err == -EINPROGRESS) 482 return; 483 484 out: 485 aead_request_complete(req, err); 486 } 487 488 static int crypto_gcm_encrypt(struct aead_request *req) 489 { 490 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 491 struct ablkcipher_request *abreq = &pctx->u.abreq; 492 u32 flags = aead_request_flags(req); 493 494 crypto_gcm_init_common(req); 495 crypto_gcm_init_crypt(req, req->cryptlen); 496 ablkcipher_request_set_callback(abreq, flags, gcm_encrypt_done, req); 497 498 return crypto_ablkcipher_encrypt(abreq) ?: 499 gcm_encrypt_continue(req, flags); 500 } 501 502 static int crypto_gcm_verify(struct aead_request *req) 503 { 504 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 505 struct crypto_aead *aead = crypto_aead_reqtfm(req); 506 u8 *auth_tag = pctx->auth_tag; 507 u8 *iauth_tag = pctx->iauth_tag; 508 unsigned int authsize = crypto_aead_authsize(aead); 509 unsigned int cryptlen = req->cryptlen - authsize; 510 511 crypto_xor(auth_tag, iauth_tag, 16); 512 scatterwalk_map_and_copy(iauth_tag, req->src, 513 req->assoclen + cryptlen, authsize, 0); 514 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; 515 } 516 517 static void gcm_decrypt_done(struct crypto_async_request *areq, int err) 518 { 519 struct aead_request *req = areq->data; 520 521 if (!err) 522 err = crypto_gcm_verify(req); 523 524 aead_request_complete(req, err); 525 } 526 527 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) 528 { 529 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 530 struct ablkcipher_request *abreq = &pctx->u.abreq; 531 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 532 533 crypto_gcm_init_crypt(req, gctx->cryptlen); 534 ablkcipher_request_set_callback(abreq, flags, gcm_decrypt_done, req); 535 return crypto_ablkcipher_decrypt(abreq) ?: crypto_gcm_verify(req); 536 } 537 538 static int crypto_gcm_decrypt(struct aead_request *req) 539 { 540 struct crypto_aead *aead = crypto_aead_reqtfm(req); 541 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 542 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 543 unsigned int authsize = crypto_aead_authsize(aead); 544 unsigned int cryptlen = req->cryptlen; 545 u32 flags = aead_request_flags(req); 546 547 cryptlen -= authsize; 548 549 crypto_gcm_init_common(req); 550 551 gctx->src = sg_next(pctx->src); 552 gctx->cryptlen = cryptlen; 553 gctx->complete = gcm_dec_hash_continue; 554 555 return gcm_hash(req, flags); 556 } 557 558 static int crypto_gcm_init_tfm(struct crypto_aead *tfm) 559 { 560 struct aead_instance *inst = aead_alg_instance(tfm); 561 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); 562 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 563 struct crypto_ablkcipher *ctr; 564 struct crypto_ahash *ghash; 565 unsigned long align; 566 int err; 567 568 ghash = crypto_spawn_ahash(&ictx->ghash); 569 if (IS_ERR(ghash)) 570 return PTR_ERR(ghash); 571 572 ctr = crypto_spawn_skcipher(&ictx->ctr); 573 err = PTR_ERR(ctr); 574 if (IS_ERR(ctr)) 575 goto err_free_hash; 576 577 ctx->ctr = ctr; 578 ctx->ghash = ghash; 579 580 align = crypto_aead_alignmask(tfm); 581 align &= ~(crypto_tfm_ctx_alignment() - 1); 582 crypto_aead_set_reqsize(tfm, 583 align + offsetof(struct crypto_gcm_req_priv_ctx, u) + 584 max(sizeof(struct ablkcipher_request) + 585 crypto_ablkcipher_reqsize(ctr), 586 sizeof(struct ahash_request) + 587 crypto_ahash_reqsize(ghash))); 588 589 return 0; 590 591 err_free_hash: 592 crypto_free_ahash(ghash); 593 return err; 594 } 595 596 static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) 597 { 598 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 599 600 crypto_free_ahash(ctx->ghash); 601 crypto_free_ablkcipher(ctx->ctr); 602 } 603 604 static int crypto_gcm_create_common(struct crypto_template *tmpl, 605 struct rtattr **tb, 606 const char *full_name, 607 const char *ctr_name, 608 const char *ghash_name) 609 { 610 struct crypto_attr_type *algt; 611 struct aead_instance *inst; 612 struct crypto_alg *ctr; 613 struct crypto_alg *ghash_alg; 614 struct hash_alg_common *ghash; 615 struct gcm_instance_ctx *ctx; 616 int err; 617 618 algt = crypto_get_attr_type(tb); 619 if (IS_ERR(algt)) 620 return PTR_ERR(algt); 621 622 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 623 return -EINVAL; 624 625 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, 626 CRYPTO_ALG_TYPE_HASH, 627 CRYPTO_ALG_TYPE_AHASH_MASK); 628 if (IS_ERR(ghash_alg)) 629 return PTR_ERR(ghash_alg); 630 631 ghash = __crypto_hash_alg_common(ghash_alg); 632 633 err = -ENOMEM; 634 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 635 if (!inst) 636 goto out_put_ghash; 637 638 ctx = aead_instance_ctx(inst); 639 err = crypto_init_ahash_spawn(&ctx->ghash, ghash, 640 aead_crypto_instance(inst)); 641 if (err) 642 goto err_free_inst; 643 644 err = -EINVAL; 645 if (ghash->digestsize != 16) 646 goto err_drop_ghash; 647 648 crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst)); 649 err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, 650 crypto_requires_sync(algt->type, 651 algt->mask)); 652 if (err) 653 goto err_drop_ghash; 654 655 ctr = crypto_skcipher_spawn_alg(&ctx->ctr); 656 657 /* We only support 16-byte blocks. */ 658 if (ctr->cra_ablkcipher.ivsize != 16) 659 goto out_put_ctr; 660 661 /* Not a stream cipher? */ 662 err = -EINVAL; 663 if (ctr->cra_blocksize != 1) 664 goto out_put_ctr; 665 666 err = -ENAMETOOLONG; 667 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 668 "gcm_base(%s,%s)", ctr->cra_driver_name, 669 ghash_alg->cra_driver_name) >= 670 CRYPTO_MAX_ALG_NAME) 671 goto out_put_ctr; 672 673 memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME); 674 675 inst->alg.base.cra_flags = (ghash->base.cra_flags | ctr->cra_flags) & 676 CRYPTO_ALG_ASYNC; 677 inst->alg.base.cra_priority = (ghash->base.cra_priority + 678 ctr->cra_priority) / 2; 679 inst->alg.base.cra_blocksize = 1; 680 inst->alg.base.cra_alignmask = ghash->base.cra_alignmask | 681 ctr->cra_alignmask; 682 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); 683 inst->alg.ivsize = 12; 684 inst->alg.maxauthsize = 16; 685 inst->alg.init = crypto_gcm_init_tfm; 686 inst->alg.exit = crypto_gcm_exit_tfm; 687 inst->alg.setkey = crypto_gcm_setkey; 688 inst->alg.setauthsize = crypto_gcm_setauthsize; 689 inst->alg.encrypt = crypto_gcm_encrypt; 690 inst->alg.decrypt = crypto_gcm_decrypt; 691 692 err = aead_register_instance(tmpl, inst); 693 if (err) 694 goto out_put_ctr; 695 696 out_put_ghash: 697 crypto_mod_put(ghash_alg); 698 return err; 699 700 out_put_ctr: 701 crypto_drop_skcipher(&ctx->ctr); 702 err_drop_ghash: 703 crypto_drop_ahash(&ctx->ghash); 704 err_free_inst: 705 kfree(inst); 706 goto out_put_ghash; 707 } 708 709 static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) 710 { 711 const char *cipher_name; 712 char ctr_name[CRYPTO_MAX_ALG_NAME]; 713 char full_name[CRYPTO_MAX_ALG_NAME]; 714 715 cipher_name = crypto_attr_alg_name(tb[1]); 716 if (IS_ERR(cipher_name)) 717 return PTR_ERR(cipher_name); 718 719 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= 720 CRYPTO_MAX_ALG_NAME) 721 return -ENAMETOOLONG; 722 723 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >= 724 CRYPTO_MAX_ALG_NAME) 725 return -ENAMETOOLONG; 726 727 return crypto_gcm_create_common(tmpl, tb, full_name, 728 ctr_name, "ghash"); 729 } 730 731 static void crypto_gcm_free(struct crypto_instance *inst) 732 { 733 struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst); 734 735 crypto_drop_skcipher(&ctx->ctr); 736 crypto_drop_ahash(&ctx->ghash); 737 kfree(aead_instance(inst)); 738 } 739 740 static struct crypto_template crypto_gcm_tmpl = { 741 .name = "gcm", 742 .create = crypto_gcm_create, 743 .free = crypto_gcm_free, 744 .module = THIS_MODULE, 745 }; 746 747 static int crypto_gcm_base_create(struct crypto_template *tmpl, 748 struct rtattr **tb) 749 { 750 const char *ctr_name; 751 const char *ghash_name; 752 char full_name[CRYPTO_MAX_ALG_NAME]; 753 754 ctr_name = crypto_attr_alg_name(tb[1]); 755 if (IS_ERR(ctr_name)) 756 return PTR_ERR(ctr_name); 757 758 ghash_name = crypto_attr_alg_name(tb[2]); 759 if (IS_ERR(ghash_name)) 760 return PTR_ERR(ghash_name); 761 762 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", 763 ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) 764 return -ENAMETOOLONG; 765 766 return crypto_gcm_create_common(tmpl, tb, full_name, 767 ctr_name, ghash_name); 768 } 769 770 static struct crypto_template crypto_gcm_base_tmpl = { 771 .name = "gcm_base", 772 .create = crypto_gcm_base_create, 773 .free = crypto_gcm_free, 774 .module = THIS_MODULE, 775 }; 776 777 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, 778 unsigned int keylen) 779 { 780 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 781 struct crypto_aead *child = ctx->child; 782 int err; 783 784 if (keylen < 4) 785 return -EINVAL; 786 787 keylen -= 4; 788 memcpy(ctx->nonce, key + keylen, 4); 789 790 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 791 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 792 CRYPTO_TFM_REQ_MASK); 793 err = crypto_aead_setkey(child, key, keylen); 794 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) & 795 CRYPTO_TFM_RES_MASK); 796 797 return err; 798 } 799 800 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, 801 unsigned int authsize) 802 { 803 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 804 805 switch (authsize) { 806 case 8: 807 case 12: 808 case 16: 809 break; 810 default: 811 return -EINVAL; 812 } 813 814 return crypto_aead_setauthsize(ctx->child, authsize); 815 } 816 817 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) 818 { 819 struct aead_request *subreq = aead_request_ctx(req); 820 struct crypto_aead *aead = crypto_aead_reqtfm(req); 821 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); 822 struct crypto_aead *child = ctx->child; 823 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), 824 crypto_aead_alignmask(child) + 1); 825 826 memcpy(iv, ctx->nonce, 4); 827 memcpy(iv + 4, req->iv, 8); 828 829 aead_request_set_tfm(subreq, child); 830 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 831 req->base.data); 832 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv); 833 aead_request_set_ad(subreq, req->assoclen); 834 835 return subreq; 836 } 837 838 static int crypto_rfc4106_encrypt(struct aead_request *req) 839 { 840 req = crypto_rfc4106_crypt(req); 841 842 return crypto_aead_encrypt(req); 843 } 844 845 static int crypto_rfc4106_decrypt(struct aead_request *req) 846 { 847 req = crypto_rfc4106_crypt(req); 848 849 return crypto_aead_decrypt(req); 850 } 851 852 static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm) 853 { 854 struct aead_instance *inst = aead_alg_instance(tfm); 855 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst); 856 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 857 struct crypto_aead *aead; 858 unsigned long align; 859 860 aead = crypto_spawn_aead(spawn); 861 if (IS_ERR(aead)) 862 return PTR_ERR(aead); 863 864 ctx->child = aead; 865 866 align = crypto_aead_alignmask(aead); 867 align &= ~(crypto_tfm_ctx_alignment() - 1); 868 crypto_aead_set_reqsize( 869 tfm, 870 sizeof(struct aead_request) + 871 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 872 align + 12); 873 874 return 0; 875 } 876 877 static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm) 878 { 879 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 880 881 crypto_free_aead(ctx->child); 882 } 883 884 static int crypto_rfc4106_create(struct crypto_template *tmpl, 885 struct rtattr **tb) 886 { 887 struct crypto_attr_type *algt; 888 struct aead_instance *inst; 889 struct crypto_aead_spawn *spawn; 890 struct aead_alg *alg; 891 const char *ccm_name; 892 int err; 893 894 algt = crypto_get_attr_type(tb); 895 if (IS_ERR(algt)) 896 return PTR_ERR(algt); 897 898 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 899 return -EINVAL; 900 901 ccm_name = crypto_attr_alg_name(tb[1]); 902 if (IS_ERR(ccm_name)) 903 return PTR_ERR(ccm_name); 904 905 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 906 if (!inst) 907 return -ENOMEM; 908 909 spawn = aead_instance_ctx(inst); 910 crypto_set_aead_spawn(spawn, aead_crypto_instance(inst)); 911 err = crypto_grab_aead(spawn, ccm_name, 0, 912 crypto_requires_sync(algt->type, algt->mask)); 913 if (err) 914 goto out_free_inst; 915 916 alg = crypto_spawn_aead_alg(spawn); 917 918 err = -EINVAL; 919 920 /* Underlying IV size must be 12. */ 921 if (crypto_aead_alg_ivsize(alg) != 12) 922 goto out_drop_alg; 923 924 /* Not a stream cipher? */ 925 if (alg->base.cra_blocksize != 1) 926 goto out_drop_alg; 927 928 err = -ENAMETOOLONG; 929 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 930 "rfc4106(%s)", alg->base.cra_name) >= 931 CRYPTO_MAX_ALG_NAME || 932 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 933 "rfc4106(%s)", alg->base.cra_driver_name) >= 934 CRYPTO_MAX_ALG_NAME) 935 goto out_drop_alg; 936 937 inst->alg.base.cra_flags |= alg->base.cra_flags & CRYPTO_ALG_ASYNC; 938 inst->alg.base.cra_priority = alg->base.cra_priority; 939 inst->alg.base.cra_blocksize = 1; 940 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 941 942 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); 943 944 inst->alg.ivsize = 8; 945 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 946 947 inst->alg.init = crypto_rfc4106_init_tfm; 948 inst->alg.exit = crypto_rfc4106_exit_tfm; 949 950 inst->alg.setkey = crypto_rfc4106_setkey; 951 inst->alg.setauthsize = crypto_rfc4106_setauthsize; 952 inst->alg.encrypt = crypto_rfc4106_encrypt; 953 inst->alg.decrypt = crypto_rfc4106_decrypt; 954 955 err = aead_register_instance(tmpl, inst); 956 if (err) 957 goto out_drop_alg; 958 959 out: 960 return err; 961 962 out_drop_alg: 963 crypto_drop_aead(spawn); 964 out_free_inst: 965 kfree(inst); 966 goto out; 967 } 968 969 static void crypto_rfc4106_free(struct crypto_instance *inst) 970 { 971 crypto_drop_aead(crypto_instance_ctx(inst)); 972 kfree(aead_instance(inst)); 973 } 974 975 static struct crypto_template crypto_rfc4106_tmpl = { 976 .name = "rfc4106", 977 .create = crypto_rfc4106_create, 978 .free = crypto_rfc4106_free, 979 .module = THIS_MODULE, 980 }; 981 982 static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key, 983 unsigned int keylen) 984 { 985 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 986 struct crypto_aead *child = ctx->child; 987 int err; 988 989 if (keylen < 4) 990 return -EINVAL; 991 992 keylen -= 4; 993 memcpy(ctx->nonce, key + keylen, 4); 994 995 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 996 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 997 CRYPTO_TFM_REQ_MASK); 998 err = crypto_aead_setkey(child, key, keylen); 999 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) & 1000 CRYPTO_TFM_RES_MASK); 1001 1002 return err; 1003 } 1004 1005 static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, 1006 unsigned int authsize) 1007 { 1008 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 1009 1010 if (authsize != 16) 1011 return -EINVAL; 1012 1013 return crypto_aead_setauthsize(ctx->child, authsize); 1014 } 1015 1016 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc) 1017 { 1018 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1019 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 1020 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req); 1021 struct aead_request *subreq = &rctx->subreq; 1022 unsigned int authsize = crypto_aead_authsize(aead); 1023 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), 1024 crypto_aead_alignmask(ctx->child) + 1); 1025 int err; 1026 1027 if (req->src != req->dst) { 1028 err = crypto_rfc4543_copy_src_to_dst(req, enc); 1029 if (err) 1030 return err; 1031 } 1032 1033 memcpy(iv, ctx->nonce, 4); 1034 memcpy(iv + 4, req->iv, 8); 1035 1036 aead_request_set_tfm(subreq, ctx->child); 1037 aead_request_set_callback(subreq, req->base.flags, 1038 req->base.complete, req->base.data); 1039 aead_request_set_crypt(subreq, req->src, req->dst, 1040 enc ? 0 : authsize, iv); 1041 aead_request_set_ad(subreq, req->assoclen + req->cryptlen - 1042 subreq->cryptlen); 1043 1044 return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); 1045 } 1046 1047 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) 1048 { 1049 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1050 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 1051 unsigned int authsize = crypto_aead_authsize(aead); 1052 unsigned int nbytes = req->assoclen + req->cryptlen - 1053 (enc ? 0 : authsize); 1054 struct blkcipher_desc desc = { 1055 .tfm = ctx->null, 1056 }; 1057 1058 return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes); 1059 } 1060 1061 static int crypto_rfc4543_encrypt(struct aead_request *req) 1062 { 1063 return crypto_rfc4543_crypt(req, true); 1064 } 1065 1066 static int crypto_rfc4543_decrypt(struct aead_request *req) 1067 { 1068 return crypto_rfc4543_crypt(req, false); 1069 } 1070 1071 static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) 1072 { 1073 struct aead_instance *inst = aead_alg_instance(tfm); 1074 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst); 1075 struct crypto_aead_spawn *spawn = &ictx->aead; 1076 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 1077 struct crypto_aead *aead; 1078 struct crypto_blkcipher *null; 1079 unsigned long align; 1080 int err = 0; 1081 1082 aead = crypto_spawn_aead(spawn); 1083 if (IS_ERR(aead)) 1084 return PTR_ERR(aead); 1085 1086 null = crypto_get_default_null_skcipher(); 1087 err = PTR_ERR(null); 1088 if (IS_ERR(null)) 1089 goto err_free_aead; 1090 1091 ctx->child = aead; 1092 ctx->null = null; 1093 1094 align = crypto_aead_alignmask(aead); 1095 align &= ~(crypto_tfm_ctx_alignment() - 1); 1096 crypto_aead_set_reqsize( 1097 tfm, 1098 sizeof(struct crypto_rfc4543_req_ctx) + 1099 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 1100 align + 12); 1101 1102 return 0; 1103 1104 err_free_aead: 1105 crypto_free_aead(aead); 1106 return err; 1107 } 1108 1109 static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) 1110 { 1111 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 1112 1113 crypto_free_aead(ctx->child); 1114 crypto_put_default_null_skcipher(); 1115 } 1116 1117 static int crypto_rfc4543_create(struct crypto_template *tmpl, 1118 struct rtattr **tb) 1119 { 1120 struct crypto_attr_type *algt; 1121 struct aead_instance *inst; 1122 struct crypto_aead_spawn *spawn; 1123 struct aead_alg *alg; 1124 struct crypto_rfc4543_instance_ctx *ctx; 1125 const char *ccm_name; 1126 int err; 1127 1128 algt = crypto_get_attr_type(tb); 1129 if (IS_ERR(algt)) 1130 return PTR_ERR(algt); 1131 1132 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 1133 return -EINVAL; 1134 1135 ccm_name = crypto_attr_alg_name(tb[1]); 1136 if (IS_ERR(ccm_name)) 1137 return PTR_ERR(ccm_name); 1138 1139 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 1140 if (!inst) 1141 return -ENOMEM; 1142 1143 ctx = aead_instance_ctx(inst); 1144 spawn = &ctx->aead; 1145 crypto_set_aead_spawn(spawn, aead_crypto_instance(inst)); 1146 err = crypto_grab_aead(spawn, ccm_name, 0, 1147 crypto_requires_sync(algt->type, algt->mask)); 1148 if (err) 1149 goto out_free_inst; 1150 1151 alg = crypto_spawn_aead_alg(spawn); 1152 1153 err = -EINVAL; 1154 1155 /* Underlying IV size must be 12. */ 1156 if (crypto_aead_alg_ivsize(alg) != 12) 1157 goto out_drop_alg; 1158 1159 /* Not a stream cipher? */ 1160 if (alg->base.cra_blocksize != 1) 1161 goto out_drop_alg; 1162 1163 err = -ENAMETOOLONG; 1164 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 1165 "rfc4543(%s)", alg->base.cra_name) >= 1166 CRYPTO_MAX_ALG_NAME || 1167 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 1168 "rfc4543(%s)", alg->base.cra_driver_name) >= 1169 CRYPTO_MAX_ALG_NAME) 1170 goto out_drop_alg; 1171 1172 inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC; 1173 inst->alg.base.cra_priority = alg->base.cra_priority; 1174 inst->alg.base.cra_blocksize = 1; 1175 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 1176 1177 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); 1178 1179 inst->alg.ivsize = 8; 1180 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 1181 1182 inst->alg.init = crypto_rfc4543_init_tfm; 1183 inst->alg.exit = crypto_rfc4543_exit_tfm; 1184 1185 inst->alg.setkey = crypto_rfc4543_setkey; 1186 inst->alg.setauthsize = crypto_rfc4543_setauthsize; 1187 inst->alg.encrypt = crypto_rfc4543_encrypt; 1188 inst->alg.decrypt = crypto_rfc4543_decrypt; 1189 1190 err = aead_register_instance(tmpl, inst); 1191 if (err) 1192 goto out_drop_alg; 1193 1194 out: 1195 return err; 1196 1197 out_drop_alg: 1198 crypto_drop_aead(spawn); 1199 out_free_inst: 1200 kfree(inst); 1201 goto out; 1202 } 1203 1204 static void crypto_rfc4543_free(struct crypto_instance *inst) 1205 { 1206 struct crypto_rfc4543_instance_ctx *ctx = crypto_instance_ctx(inst); 1207 1208 crypto_drop_aead(&ctx->aead); 1209 1210 kfree(aead_instance(inst)); 1211 } 1212 1213 static struct crypto_template crypto_rfc4543_tmpl = { 1214 .name = "rfc4543", 1215 .create = crypto_rfc4543_create, 1216 .free = crypto_rfc4543_free, 1217 .module = THIS_MODULE, 1218 }; 1219 1220 static int __init crypto_gcm_module_init(void) 1221 { 1222 int err; 1223 1224 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL); 1225 if (!gcm_zeroes) 1226 return -ENOMEM; 1227 1228 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); 1229 1230 err = crypto_register_template(&crypto_gcm_base_tmpl); 1231 if (err) 1232 goto out; 1233 1234 err = crypto_register_template(&crypto_gcm_tmpl); 1235 if (err) 1236 goto out_undo_base; 1237 1238 err = crypto_register_template(&crypto_rfc4106_tmpl); 1239 if (err) 1240 goto out_undo_gcm; 1241 1242 err = crypto_register_template(&crypto_rfc4543_tmpl); 1243 if (err) 1244 goto out_undo_rfc4106; 1245 1246 return 0; 1247 1248 out_undo_rfc4106: 1249 crypto_unregister_template(&crypto_rfc4106_tmpl); 1250 out_undo_gcm: 1251 crypto_unregister_template(&crypto_gcm_tmpl); 1252 out_undo_base: 1253 crypto_unregister_template(&crypto_gcm_base_tmpl); 1254 out: 1255 kfree(gcm_zeroes); 1256 return err; 1257 } 1258 1259 static void __exit crypto_gcm_module_exit(void) 1260 { 1261 kfree(gcm_zeroes); 1262 crypto_unregister_template(&crypto_rfc4543_tmpl); 1263 crypto_unregister_template(&crypto_rfc4106_tmpl); 1264 crypto_unregister_template(&crypto_gcm_tmpl); 1265 crypto_unregister_template(&crypto_gcm_base_tmpl); 1266 } 1267 1268 module_init(crypto_gcm_module_init); 1269 module_exit(crypto_gcm_module_exit); 1270 1271 MODULE_LICENSE("GPL"); 1272 MODULE_DESCRIPTION("Galois/Counter Mode"); 1273 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); 1274 MODULE_ALIAS_CRYPTO("gcm_base"); 1275 MODULE_ALIAS_CRYPTO("rfc4106"); 1276 MODULE_ALIAS_CRYPTO("rfc4543"); 1277 MODULE_ALIAS_CRYPTO("gcm"); 1278