1 /* 2 * GCM: Galois/Counter Mode. 3 * 4 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation. 9 */ 10 11 #include <crypto/gf128mul.h> 12 #include <crypto/internal/aead.h> 13 #include <crypto/internal/skcipher.h> 14 #include <crypto/internal/hash.h> 15 #include <crypto/scatterwalk.h> 16 #include <crypto/hash.h> 17 #include "internal.h" 18 #include <linux/completion.h> 19 #include <linux/err.h> 20 #include <linux/init.h> 21 #include <linux/kernel.h> 22 #include <linux/module.h> 23 #include <linux/slab.h> 24 25 struct gcm_instance_ctx { 26 struct crypto_skcipher_spawn ctr; 27 struct crypto_ahash_spawn ghash; 28 }; 29 30 struct crypto_gcm_ctx { 31 struct crypto_ablkcipher *ctr; 32 struct crypto_ahash *ghash; 33 }; 34 35 struct crypto_rfc4106_ctx { 36 struct crypto_aead *child; 37 u8 nonce[4]; 38 }; 39 40 struct crypto_gcm_ghash_ctx { 41 unsigned int cryptlen; 42 struct scatterlist *src; 43 crypto_completion_t complete; 44 }; 45 46 struct crypto_gcm_req_priv_ctx { 47 u8 auth_tag[16]; 48 u8 iauth_tag[16]; 49 struct scatterlist src[2]; 50 struct scatterlist dst[2]; 51 struct crypto_gcm_ghash_ctx ghash_ctx; 52 union { 53 struct ahash_request ahreq; 54 struct ablkcipher_request abreq; 55 } u; 56 }; 57 58 struct crypto_gcm_setkey_result { 59 int err; 60 struct completion completion; 61 }; 62 63 static void *gcm_zeroes; 64 65 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( 66 struct aead_request *req) 67 { 68 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); 69 70 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); 71 } 72 73 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) 74 { 75 struct crypto_gcm_setkey_result *result = req->data; 76 77 if (err == -EINPROGRESS) 78 return; 79 80 result->err = err; 81 complete(&result->completion); 82 } 83 84 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, 85 unsigned int keylen) 86 { 87 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 88 struct crypto_ahash *ghash = ctx->ghash; 89 struct crypto_ablkcipher *ctr = ctx->ctr; 90 struct { 91 be128 hash; 92 u8 iv[8]; 93 94 struct crypto_gcm_setkey_result result; 95 96 struct scatterlist sg[1]; 97 struct ablkcipher_request req; 98 } *data; 99 int err; 100 101 crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); 102 crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) & 103 CRYPTO_TFM_REQ_MASK); 104 105 err = crypto_ablkcipher_setkey(ctr, key, keylen); 106 if (err) 107 return err; 108 109 crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) & 110 CRYPTO_TFM_RES_MASK); 111 112 data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr), 113 GFP_KERNEL); 114 if (!data) 115 return -ENOMEM; 116 117 init_completion(&data->result.completion); 118 sg_init_one(data->sg, &data->hash, sizeof(data->hash)); 119 ablkcipher_request_set_tfm(&data->req, ctr); 120 ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | 121 CRYPTO_TFM_REQ_MAY_BACKLOG, 122 crypto_gcm_setkey_done, 123 &data->result); 124 ablkcipher_request_set_crypt(&data->req, data->sg, data->sg, 125 sizeof(data->hash), data->iv); 126 127 err = crypto_ablkcipher_encrypt(&data->req); 128 if (err == -EINPROGRESS || err == -EBUSY) { 129 err = wait_for_completion_interruptible( 130 &data->result.completion); 131 if (!err) 132 err = data->result.err; 133 } 134 135 if (err) 136 goto out; 137 138 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); 139 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & 140 CRYPTO_TFM_REQ_MASK); 141 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); 142 crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) & 143 CRYPTO_TFM_RES_MASK); 144 145 out: 146 kfree(data); 147 return err; 148 } 149 150 static int crypto_gcm_setauthsize(struct crypto_aead *tfm, 151 unsigned int authsize) 152 { 153 switch (authsize) { 154 case 4: 155 case 8: 156 case 12: 157 case 13: 158 case 14: 159 case 15: 160 case 16: 161 break; 162 default: 163 return -EINVAL; 164 } 165 166 return 0; 167 } 168 169 static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, 170 struct aead_request *req, 171 unsigned int cryptlen) 172 { 173 struct crypto_aead *aead = crypto_aead_reqtfm(req); 174 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 175 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 176 struct scatterlist *dst; 177 __be32 counter = cpu_to_be32(1); 178 179 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); 180 memcpy(req->iv + 12, &counter, 4); 181 182 sg_init_table(pctx->src, 2); 183 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); 184 scatterwalk_sg_chain(pctx->src, 2, req->src); 185 186 dst = pctx->src; 187 if (req->src != req->dst) { 188 sg_init_table(pctx->dst, 2); 189 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); 190 scatterwalk_sg_chain(pctx->dst, 2, req->dst); 191 dst = pctx->dst; 192 } 193 194 ablkcipher_request_set_tfm(ablk_req, ctx->ctr); 195 ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, 196 cryptlen + sizeof(pctx->auth_tag), 197 req->iv); 198 } 199 200 static inline unsigned int gcm_remain(unsigned int len) 201 { 202 len &= 0xfU; 203 return len ? 16 - len : 0; 204 } 205 206 static void gcm_hash_len_done(struct crypto_async_request *areq, int err); 207 static void gcm_hash_final_done(struct crypto_async_request *areq, int err); 208 209 static int gcm_hash_update(struct aead_request *req, 210 struct crypto_gcm_req_priv_ctx *pctx, 211 crypto_completion_t complete, 212 struct scatterlist *src, 213 unsigned int len) 214 { 215 struct ahash_request *ahreq = &pctx->u.ahreq; 216 217 ahash_request_set_callback(ahreq, aead_request_flags(req), 218 complete, req); 219 ahash_request_set_crypt(ahreq, src, NULL, len); 220 221 return crypto_ahash_update(ahreq); 222 } 223 224 static int gcm_hash_remain(struct aead_request *req, 225 struct crypto_gcm_req_priv_ctx *pctx, 226 unsigned int remain, 227 crypto_completion_t complete) 228 { 229 struct ahash_request *ahreq = &pctx->u.ahreq; 230 231 ahash_request_set_callback(ahreq, aead_request_flags(req), 232 complete, req); 233 sg_init_one(pctx->src, gcm_zeroes, remain); 234 ahash_request_set_crypt(ahreq, pctx->src, NULL, remain); 235 236 return crypto_ahash_update(ahreq); 237 } 238 239 static int gcm_hash_len(struct aead_request *req, 240 struct crypto_gcm_req_priv_ctx *pctx) 241 { 242 struct ahash_request *ahreq = &pctx->u.ahreq; 243 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 244 u128 lengths; 245 246 lengths.a = cpu_to_be64(req->assoclen * 8); 247 lengths.b = cpu_to_be64(gctx->cryptlen * 8); 248 memcpy(pctx->iauth_tag, &lengths, 16); 249 sg_init_one(pctx->src, pctx->iauth_tag, 16); 250 ahash_request_set_callback(ahreq, aead_request_flags(req), 251 gcm_hash_len_done, req); 252 ahash_request_set_crypt(ahreq, pctx->src, 253 NULL, sizeof(lengths)); 254 255 return crypto_ahash_update(ahreq); 256 } 257 258 static int gcm_hash_final(struct aead_request *req, 259 struct crypto_gcm_req_priv_ctx *pctx) 260 { 261 struct ahash_request *ahreq = &pctx->u.ahreq; 262 263 ahash_request_set_callback(ahreq, aead_request_flags(req), 264 gcm_hash_final_done, req); 265 ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0); 266 267 return crypto_ahash_final(ahreq); 268 } 269 270 static void gcm_hash_final_done(struct crypto_async_request *areq, 271 int err) 272 { 273 struct aead_request *req = areq->data; 274 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 275 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 276 277 if (!err) 278 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); 279 280 gctx->complete(areq, err); 281 } 282 283 static void gcm_hash_len_done(struct crypto_async_request *areq, 284 int err) 285 { 286 struct aead_request *req = areq->data; 287 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 288 289 if (!err) { 290 err = gcm_hash_final(req, pctx); 291 if (err == -EINPROGRESS || err == -EBUSY) 292 return; 293 } 294 295 gcm_hash_final_done(areq, err); 296 } 297 298 static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, 299 int err) 300 { 301 struct aead_request *req = areq->data; 302 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 303 304 if (!err) { 305 err = gcm_hash_len(req, pctx); 306 if (err == -EINPROGRESS || err == -EBUSY) 307 return; 308 } 309 310 gcm_hash_len_done(areq, err); 311 } 312 313 static void gcm_hash_crypt_done(struct crypto_async_request *areq, 314 int err) 315 { 316 struct aead_request *req = areq->data; 317 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 318 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 319 unsigned int remain; 320 321 if (!err) { 322 remain = gcm_remain(gctx->cryptlen); 323 BUG_ON(!remain); 324 err = gcm_hash_remain(req, pctx, remain, 325 gcm_hash_crypt_remain_done); 326 if (err == -EINPROGRESS || err == -EBUSY) 327 return; 328 } 329 330 gcm_hash_crypt_remain_done(areq, err); 331 } 332 333 static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, 334 int err) 335 { 336 struct aead_request *req = areq->data; 337 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 338 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 339 crypto_completion_t complete; 340 unsigned int remain = 0; 341 342 if (!err && gctx->cryptlen) { 343 remain = gcm_remain(gctx->cryptlen); 344 complete = remain ? gcm_hash_crypt_done : 345 gcm_hash_crypt_remain_done; 346 err = gcm_hash_update(req, pctx, complete, 347 gctx->src, gctx->cryptlen); 348 if (err == -EINPROGRESS || err == -EBUSY) 349 return; 350 } 351 352 if (remain) 353 gcm_hash_crypt_done(areq, err); 354 else 355 gcm_hash_crypt_remain_done(areq, err); 356 } 357 358 static void gcm_hash_assoc_done(struct crypto_async_request *areq, 359 int err) 360 { 361 struct aead_request *req = areq->data; 362 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 363 unsigned int remain; 364 365 if (!err) { 366 remain = gcm_remain(req->assoclen); 367 BUG_ON(!remain); 368 err = gcm_hash_remain(req, pctx, remain, 369 gcm_hash_assoc_remain_done); 370 if (err == -EINPROGRESS || err == -EBUSY) 371 return; 372 } 373 374 gcm_hash_assoc_remain_done(areq, err); 375 } 376 377 static void gcm_hash_init_done(struct crypto_async_request *areq, 378 int err) 379 { 380 struct aead_request *req = areq->data; 381 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 382 crypto_completion_t complete; 383 unsigned int remain = 0; 384 385 if (!err && req->assoclen) { 386 remain = gcm_remain(req->assoclen); 387 complete = remain ? gcm_hash_assoc_done : 388 gcm_hash_assoc_remain_done; 389 err = gcm_hash_update(req, pctx, complete, 390 req->assoc, req->assoclen); 391 if (err == -EINPROGRESS || err == -EBUSY) 392 return; 393 } 394 395 if (remain) 396 gcm_hash_assoc_done(areq, err); 397 else 398 gcm_hash_assoc_remain_done(areq, err); 399 } 400 401 static int gcm_hash(struct aead_request *req, 402 struct crypto_gcm_req_priv_ctx *pctx) 403 { 404 struct ahash_request *ahreq = &pctx->u.ahreq; 405 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 406 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 407 unsigned int remain; 408 crypto_completion_t complete; 409 int err; 410 411 ahash_request_set_tfm(ahreq, ctx->ghash); 412 413 ahash_request_set_callback(ahreq, aead_request_flags(req), 414 gcm_hash_init_done, req); 415 err = crypto_ahash_init(ahreq); 416 if (err) 417 return err; 418 remain = gcm_remain(req->assoclen); 419 complete = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done; 420 err = gcm_hash_update(req, pctx, complete, req->assoc, req->assoclen); 421 if (err) 422 return err; 423 if (remain) { 424 err = gcm_hash_remain(req, pctx, remain, 425 gcm_hash_assoc_remain_done); 426 if (err) 427 return err; 428 } 429 remain = gcm_remain(gctx->cryptlen); 430 complete = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done; 431 err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen); 432 if (err) 433 return err; 434 if (remain) { 435 err = gcm_hash_remain(req, pctx, remain, 436 gcm_hash_crypt_remain_done); 437 if (err) 438 return err; 439 } 440 err = gcm_hash_len(req, pctx); 441 if (err) 442 return err; 443 err = gcm_hash_final(req, pctx); 444 if (err) 445 return err; 446 447 return 0; 448 } 449 450 static void gcm_enc_copy_hash(struct aead_request *req, 451 struct crypto_gcm_req_priv_ctx *pctx) 452 { 453 struct crypto_aead *aead = crypto_aead_reqtfm(req); 454 u8 *auth_tag = pctx->auth_tag; 455 456 scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen, 457 crypto_aead_authsize(aead), 1); 458 } 459 460 static void gcm_enc_hash_done(struct crypto_async_request *areq, 461 int err) 462 { 463 struct aead_request *req = areq->data; 464 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 465 466 if (!err) 467 gcm_enc_copy_hash(req, pctx); 468 469 aead_request_complete(req, err); 470 } 471 472 static void gcm_encrypt_done(struct crypto_async_request *areq, 473 int err) 474 { 475 struct aead_request *req = areq->data; 476 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 477 478 if (!err) { 479 err = gcm_hash(req, pctx); 480 if (err == -EINPROGRESS || err == -EBUSY) 481 return; 482 } 483 484 gcm_enc_hash_done(areq, err); 485 } 486 487 static int crypto_gcm_encrypt(struct aead_request *req) 488 { 489 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 490 struct ablkcipher_request *abreq = &pctx->u.abreq; 491 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 492 int err; 493 494 crypto_gcm_init_crypt(abreq, req, req->cryptlen); 495 ablkcipher_request_set_callback(abreq, aead_request_flags(req), 496 gcm_encrypt_done, req); 497 498 gctx->src = req->dst; 499 gctx->cryptlen = req->cryptlen; 500 gctx->complete = gcm_enc_hash_done; 501 502 err = crypto_ablkcipher_encrypt(abreq); 503 if (err) 504 return err; 505 506 err = gcm_hash(req, pctx); 507 if (err) 508 return err; 509 510 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); 511 gcm_enc_copy_hash(req, pctx); 512 513 return 0; 514 } 515 516 static int crypto_gcm_verify(struct aead_request *req, 517 struct crypto_gcm_req_priv_ctx *pctx) 518 { 519 struct crypto_aead *aead = crypto_aead_reqtfm(req); 520 u8 *auth_tag = pctx->auth_tag; 521 u8 *iauth_tag = pctx->iauth_tag; 522 unsigned int authsize = crypto_aead_authsize(aead); 523 unsigned int cryptlen = req->cryptlen - authsize; 524 525 crypto_xor(auth_tag, iauth_tag, 16); 526 scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0); 527 return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; 528 } 529 530 static void gcm_decrypt_done(struct crypto_async_request *areq, int err) 531 { 532 struct aead_request *req = areq->data; 533 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 534 535 if (!err) 536 err = crypto_gcm_verify(req, pctx); 537 538 aead_request_complete(req, err); 539 } 540 541 static void gcm_dec_hash_done(struct crypto_async_request *areq, int err) 542 { 543 struct aead_request *req = areq->data; 544 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 545 struct ablkcipher_request *abreq = &pctx->u.abreq; 546 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 547 548 if (!err) { 549 ablkcipher_request_set_callback(abreq, aead_request_flags(req), 550 gcm_decrypt_done, req); 551 crypto_gcm_init_crypt(abreq, req, gctx->cryptlen); 552 err = crypto_ablkcipher_decrypt(abreq); 553 if (err == -EINPROGRESS || err == -EBUSY) 554 return; 555 } 556 557 gcm_decrypt_done(areq, err); 558 } 559 560 static int crypto_gcm_decrypt(struct aead_request *req) 561 { 562 struct crypto_aead *aead = crypto_aead_reqtfm(req); 563 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 564 struct ablkcipher_request *abreq = &pctx->u.abreq; 565 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 566 unsigned int authsize = crypto_aead_authsize(aead); 567 unsigned int cryptlen = req->cryptlen; 568 int err; 569 570 if (cryptlen < authsize) 571 return -EINVAL; 572 cryptlen -= authsize; 573 574 gctx->src = req->src; 575 gctx->cryptlen = cryptlen; 576 gctx->complete = gcm_dec_hash_done; 577 578 err = gcm_hash(req, pctx); 579 if (err) 580 return err; 581 582 ablkcipher_request_set_callback(abreq, aead_request_flags(req), 583 gcm_decrypt_done, req); 584 crypto_gcm_init_crypt(abreq, req, cryptlen); 585 err = crypto_ablkcipher_decrypt(abreq); 586 if (err) 587 return err; 588 589 return crypto_gcm_verify(req, pctx); 590 } 591 592 static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) 593 { 594 struct crypto_instance *inst = (void *)tfm->__crt_alg; 595 struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst); 596 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); 597 struct crypto_ablkcipher *ctr; 598 struct crypto_ahash *ghash; 599 unsigned long align; 600 int err; 601 602 ghash = crypto_spawn_ahash(&ictx->ghash); 603 if (IS_ERR(ghash)) 604 return PTR_ERR(ghash); 605 606 ctr = crypto_spawn_skcipher(&ictx->ctr); 607 err = PTR_ERR(ctr); 608 if (IS_ERR(ctr)) 609 goto err_free_hash; 610 611 ctx->ctr = ctr; 612 ctx->ghash = ghash; 613 614 align = crypto_tfm_alg_alignmask(tfm); 615 align &= ~(crypto_tfm_ctx_alignment() - 1); 616 tfm->crt_aead.reqsize = align + 617 offsetof(struct crypto_gcm_req_priv_ctx, u) + 618 max(sizeof(struct ablkcipher_request) + 619 crypto_ablkcipher_reqsize(ctr), 620 sizeof(struct ahash_request) + 621 crypto_ahash_reqsize(ghash)); 622 623 return 0; 624 625 err_free_hash: 626 crypto_free_ahash(ghash); 627 return err; 628 } 629 630 static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm) 631 { 632 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); 633 634 crypto_free_ahash(ctx->ghash); 635 crypto_free_ablkcipher(ctx->ctr); 636 } 637 638 static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, 639 const char *full_name, 640 const char *ctr_name, 641 const char *ghash_name) 642 { 643 struct crypto_attr_type *algt; 644 struct crypto_instance *inst; 645 struct crypto_alg *ctr; 646 struct crypto_alg *ghash_alg; 647 struct ahash_alg *ghash_ahash_alg; 648 struct gcm_instance_ctx *ctx; 649 int err; 650 651 algt = crypto_get_attr_type(tb); 652 err = PTR_ERR(algt); 653 if (IS_ERR(algt)) 654 return ERR_PTR(err); 655 656 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 657 return ERR_PTR(-EINVAL); 658 659 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, 660 CRYPTO_ALG_TYPE_HASH, 661 CRYPTO_ALG_TYPE_AHASH_MASK); 662 err = PTR_ERR(ghash_alg); 663 if (IS_ERR(ghash_alg)) 664 return ERR_PTR(err); 665 666 err = -ENOMEM; 667 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 668 if (!inst) 669 goto out_put_ghash; 670 671 ctx = crypto_instance_ctx(inst); 672 ghash_ahash_alg = container_of(ghash_alg, struct ahash_alg, halg.base); 673 err = crypto_init_ahash_spawn(&ctx->ghash, &ghash_ahash_alg->halg, 674 inst); 675 if (err) 676 goto err_free_inst; 677 678 crypto_set_skcipher_spawn(&ctx->ctr, inst); 679 err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, 680 crypto_requires_sync(algt->type, 681 algt->mask)); 682 if (err) 683 goto err_drop_ghash; 684 685 ctr = crypto_skcipher_spawn_alg(&ctx->ctr); 686 687 /* We only support 16-byte blocks. */ 688 if (ctr->cra_ablkcipher.ivsize != 16) 689 goto out_put_ctr; 690 691 /* Not a stream cipher? */ 692 err = -EINVAL; 693 if (ctr->cra_blocksize != 1) 694 goto out_put_ctr; 695 696 err = -ENAMETOOLONG; 697 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 698 "gcm_base(%s,%s)", ctr->cra_driver_name, 699 ghash_alg->cra_driver_name) >= 700 CRYPTO_MAX_ALG_NAME) 701 goto out_put_ctr; 702 703 memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME); 704 705 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; 706 inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC; 707 inst->alg.cra_priority = ctr->cra_priority; 708 inst->alg.cra_blocksize = 1; 709 inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1); 710 inst->alg.cra_type = &crypto_aead_type; 711 inst->alg.cra_aead.ivsize = 16; 712 inst->alg.cra_aead.maxauthsize = 16; 713 inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx); 714 inst->alg.cra_init = crypto_gcm_init_tfm; 715 inst->alg.cra_exit = crypto_gcm_exit_tfm; 716 inst->alg.cra_aead.setkey = crypto_gcm_setkey; 717 inst->alg.cra_aead.setauthsize = crypto_gcm_setauthsize; 718 inst->alg.cra_aead.encrypt = crypto_gcm_encrypt; 719 inst->alg.cra_aead.decrypt = crypto_gcm_decrypt; 720 721 out: 722 crypto_mod_put(ghash_alg); 723 return inst; 724 725 out_put_ctr: 726 crypto_drop_skcipher(&ctx->ctr); 727 err_drop_ghash: 728 crypto_drop_ahash(&ctx->ghash); 729 err_free_inst: 730 kfree(inst); 731 out_put_ghash: 732 inst = ERR_PTR(err); 733 goto out; 734 } 735 736 static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb) 737 { 738 int err; 739 const char *cipher_name; 740 char ctr_name[CRYPTO_MAX_ALG_NAME]; 741 char full_name[CRYPTO_MAX_ALG_NAME]; 742 743 cipher_name = crypto_attr_alg_name(tb[1]); 744 err = PTR_ERR(cipher_name); 745 if (IS_ERR(cipher_name)) 746 return ERR_PTR(err); 747 748 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= 749 CRYPTO_MAX_ALG_NAME) 750 return ERR_PTR(-ENAMETOOLONG); 751 752 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >= 753 CRYPTO_MAX_ALG_NAME) 754 return ERR_PTR(-ENAMETOOLONG); 755 756 return crypto_gcm_alloc_common(tb, full_name, ctr_name, "ghash"); 757 } 758 759 static void crypto_gcm_free(struct crypto_instance *inst) 760 { 761 struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst); 762 763 crypto_drop_skcipher(&ctx->ctr); 764 crypto_drop_ahash(&ctx->ghash); 765 kfree(inst); 766 } 767 768 static struct crypto_template crypto_gcm_tmpl = { 769 .name = "gcm", 770 .alloc = crypto_gcm_alloc, 771 .free = crypto_gcm_free, 772 .module = THIS_MODULE, 773 }; 774 775 static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) 776 { 777 int err; 778 const char *ctr_name; 779 const char *ghash_name; 780 char full_name[CRYPTO_MAX_ALG_NAME]; 781 782 ctr_name = crypto_attr_alg_name(tb[1]); 783 err = PTR_ERR(ctr_name); 784 if (IS_ERR(ctr_name)) 785 return ERR_PTR(err); 786 787 ghash_name = crypto_attr_alg_name(tb[2]); 788 err = PTR_ERR(ghash_name); 789 if (IS_ERR(ghash_name)) 790 return ERR_PTR(err); 791 792 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", 793 ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) 794 return ERR_PTR(-ENAMETOOLONG); 795 796 return crypto_gcm_alloc_common(tb, full_name, ctr_name, ghash_name); 797 } 798 799 static struct crypto_template crypto_gcm_base_tmpl = { 800 .name = "gcm_base", 801 .alloc = crypto_gcm_base_alloc, 802 .free = crypto_gcm_free, 803 .module = THIS_MODULE, 804 }; 805 806 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, 807 unsigned int keylen) 808 { 809 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 810 struct crypto_aead *child = ctx->child; 811 int err; 812 813 if (keylen < 4) 814 return -EINVAL; 815 816 keylen -= 4; 817 memcpy(ctx->nonce, key + keylen, 4); 818 819 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 820 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 821 CRYPTO_TFM_REQ_MASK); 822 err = crypto_aead_setkey(child, key, keylen); 823 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) & 824 CRYPTO_TFM_RES_MASK); 825 826 return err; 827 } 828 829 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, 830 unsigned int authsize) 831 { 832 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 833 834 switch (authsize) { 835 case 8: 836 case 12: 837 case 16: 838 break; 839 default: 840 return -EINVAL; 841 } 842 843 return crypto_aead_setauthsize(ctx->child, authsize); 844 } 845 846 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) 847 { 848 struct aead_request *subreq = aead_request_ctx(req); 849 struct crypto_aead *aead = crypto_aead_reqtfm(req); 850 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); 851 struct crypto_aead *child = ctx->child; 852 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), 853 crypto_aead_alignmask(child) + 1); 854 855 memcpy(iv, ctx->nonce, 4); 856 memcpy(iv + 4, req->iv, 8); 857 858 aead_request_set_tfm(subreq, child); 859 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 860 req->base.data); 861 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv); 862 aead_request_set_assoc(subreq, req->assoc, req->assoclen); 863 864 return subreq; 865 } 866 867 static int crypto_rfc4106_encrypt(struct aead_request *req) 868 { 869 req = crypto_rfc4106_crypt(req); 870 871 return crypto_aead_encrypt(req); 872 } 873 874 static int crypto_rfc4106_decrypt(struct aead_request *req) 875 { 876 req = crypto_rfc4106_crypt(req); 877 878 return crypto_aead_decrypt(req); 879 } 880 881 static int crypto_rfc4106_init_tfm(struct crypto_tfm *tfm) 882 { 883 struct crypto_instance *inst = (void *)tfm->__crt_alg; 884 struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst); 885 struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm); 886 struct crypto_aead *aead; 887 unsigned long align; 888 889 aead = crypto_spawn_aead(spawn); 890 if (IS_ERR(aead)) 891 return PTR_ERR(aead); 892 893 ctx->child = aead; 894 895 align = crypto_aead_alignmask(aead); 896 align &= ~(crypto_tfm_ctx_alignment() - 1); 897 tfm->crt_aead.reqsize = sizeof(struct aead_request) + 898 ALIGN(crypto_aead_reqsize(aead), 899 crypto_tfm_ctx_alignment()) + 900 align + 16; 901 902 return 0; 903 } 904 905 static void crypto_rfc4106_exit_tfm(struct crypto_tfm *tfm) 906 { 907 struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm); 908 909 crypto_free_aead(ctx->child); 910 } 911 912 static struct crypto_instance *crypto_rfc4106_alloc(struct rtattr **tb) 913 { 914 struct crypto_attr_type *algt; 915 struct crypto_instance *inst; 916 struct crypto_aead_spawn *spawn; 917 struct crypto_alg *alg; 918 const char *ccm_name; 919 int err; 920 921 algt = crypto_get_attr_type(tb); 922 err = PTR_ERR(algt); 923 if (IS_ERR(algt)) 924 return ERR_PTR(err); 925 926 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 927 return ERR_PTR(-EINVAL); 928 929 ccm_name = crypto_attr_alg_name(tb[1]); 930 err = PTR_ERR(ccm_name); 931 if (IS_ERR(ccm_name)) 932 return ERR_PTR(err); 933 934 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 935 if (!inst) 936 return ERR_PTR(-ENOMEM); 937 938 spawn = crypto_instance_ctx(inst); 939 crypto_set_aead_spawn(spawn, inst); 940 err = crypto_grab_aead(spawn, ccm_name, 0, 941 crypto_requires_sync(algt->type, algt->mask)); 942 if (err) 943 goto out_free_inst; 944 945 alg = crypto_aead_spawn_alg(spawn); 946 947 err = -EINVAL; 948 949 /* We only support 16-byte blocks. */ 950 if (alg->cra_aead.ivsize != 16) 951 goto out_drop_alg; 952 953 /* Not a stream cipher? */ 954 if (alg->cra_blocksize != 1) 955 goto out_drop_alg; 956 957 err = -ENAMETOOLONG; 958 if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, 959 "rfc4106(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME || 960 snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 961 "rfc4106(%s)", alg->cra_driver_name) >= 962 CRYPTO_MAX_ALG_NAME) 963 goto out_drop_alg; 964 965 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; 966 inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC; 967 inst->alg.cra_priority = alg->cra_priority; 968 inst->alg.cra_blocksize = 1; 969 inst->alg.cra_alignmask = alg->cra_alignmask; 970 inst->alg.cra_type = &crypto_nivaead_type; 971 972 inst->alg.cra_aead.ivsize = 8; 973 inst->alg.cra_aead.maxauthsize = 16; 974 975 inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); 976 977 inst->alg.cra_init = crypto_rfc4106_init_tfm; 978 inst->alg.cra_exit = crypto_rfc4106_exit_tfm; 979 980 inst->alg.cra_aead.setkey = crypto_rfc4106_setkey; 981 inst->alg.cra_aead.setauthsize = crypto_rfc4106_setauthsize; 982 inst->alg.cra_aead.encrypt = crypto_rfc4106_encrypt; 983 inst->alg.cra_aead.decrypt = crypto_rfc4106_decrypt; 984 985 inst->alg.cra_aead.geniv = "seqiv"; 986 987 out: 988 return inst; 989 990 out_drop_alg: 991 crypto_drop_aead(spawn); 992 out_free_inst: 993 kfree(inst); 994 inst = ERR_PTR(err); 995 goto out; 996 } 997 998 static void crypto_rfc4106_free(struct crypto_instance *inst) 999 { 1000 crypto_drop_spawn(crypto_instance_ctx(inst)); 1001 kfree(inst); 1002 } 1003 1004 static struct crypto_template crypto_rfc4106_tmpl = { 1005 .name = "rfc4106", 1006 .alloc = crypto_rfc4106_alloc, 1007 .free = crypto_rfc4106_free, 1008 .module = THIS_MODULE, 1009 }; 1010 1011 static int __init crypto_gcm_module_init(void) 1012 { 1013 int err; 1014 1015 gcm_zeroes = kzalloc(16, GFP_KERNEL); 1016 if (!gcm_zeroes) 1017 return -ENOMEM; 1018 1019 err = crypto_register_template(&crypto_gcm_base_tmpl); 1020 if (err) 1021 goto out; 1022 1023 err = crypto_register_template(&crypto_gcm_tmpl); 1024 if (err) 1025 goto out_undo_base; 1026 1027 err = crypto_register_template(&crypto_rfc4106_tmpl); 1028 if (err) 1029 goto out_undo_gcm; 1030 1031 return 0; 1032 1033 out_undo_gcm: 1034 crypto_unregister_template(&crypto_gcm_tmpl); 1035 out_undo_base: 1036 crypto_unregister_template(&crypto_gcm_base_tmpl); 1037 out: 1038 kfree(gcm_zeroes); 1039 return err; 1040 } 1041 1042 static void __exit crypto_gcm_module_exit(void) 1043 { 1044 kfree(gcm_zeroes); 1045 crypto_unregister_template(&crypto_rfc4106_tmpl); 1046 crypto_unregister_template(&crypto_gcm_tmpl); 1047 crypto_unregister_template(&crypto_gcm_base_tmpl); 1048 } 1049 1050 module_init(crypto_gcm_module_init); 1051 module_exit(crypto_gcm_module_exit); 1052 1053 MODULE_LICENSE("GPL"); 1054 MODULE_DESCRIPTION("Galois/Counter Mode"); 1055 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); 1056 MODULE_ALIAS("gcm_base"); 1057 MODULE_ALIAS("rfc4106"); 1058