1 /* 2 * GCM: Galois/Counter Mode. 3 * 4 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation. 9 */ 10 11 #include <crypto/gf128mul.h> 12 #include <crypto/internal/aead.h> 13 #include <crypto/internal/skcipher.h> 14 #include <crypto/internal/hash.h> 15 #include <crypto/scatterwalk.h> 16 #include <crypto/hash.h> 17 #include "internal.h" 18 #include <linux/completion.h> 19 #include <linux/err.h> 20 #include <linux/init.h> 21 #include <linux/kernel.h> 22 #include <linux/module.h> 23 #include <linux/slab.h> 24 25 struct gcm_instance_ctx { 26 struct crypto_skcipher_spawn ctr; 27 struct crypto_ahash_spawn ghash; 28 }; 29 30 struct crypto_gcm_ctx { 31 struct crypto_ablkcipher *ctr; 32 struct crypto_ahash *ghash; 33 }; 34 35 struct crypto_rfc4106_ctx { 36 struct crypto_aead *child; 37 u8 nonce[4]; 38 }; 39 40 struct crypto_gcm_ghash_ctx { 41 unsigned int cryptlen; 42 struct scatterlist *src; 43 void (*complete)(struct aead_request *req, int err); 44 }; 45 46 struct crypto_gcm_req_priv_ctx { 47 u8 auth_tag[16]; 48 u8 iauth_tag[16]; 49 struct scatterlist src[2]; 50 struct scatterlist dst[2]; 51 struct crypto_gcm_ghash_ctx ghash_ctx; 52 union { 53 struct ahash_request ahreq; 54 struct ablkcipher_request abreq; 55 } u; 56 }; 57 58 struct crypto_gcm_setkey_result { 59 int err; 60 struct completion completion; 61 }; 62 63 static void *gcm_zeroes; 64 65 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( 66 struct aead_request *req) 67 { 68 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); 69 70 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); 71 } 72 73 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) 74 { 75 struct crypto_gcm_setkey_result *result = req->data; 76 77 if (err == -EINPROGRESS) 78 return; 79 80 result->err = err; 81 complete(&result->completion); 82 } 83 84 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, 85 unsigned int keylen) 86 { 87 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 88 struct crypto_ahash *ghash = ctx->ghash; 89 struct crypto_ablkcipher *ctr = ctx->ctr; 90 struct { 91 be128 hash; 92 u8 iv[8]; 93 94 struct crypto_gcm_setkey_result result; 95 96 struct scatterlist sg[1]; 97 struct ablkcipher_request req; 98 } *data; 99 int err; 100 101 crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); 102 crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) & 103 CRYPTO_TFM_REQ_MASK); 104 105 err = crypto_ablkcipher_setkey(ctr, key, keylen); 106 if (err) 107 return err; 108 109 crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) & 110 CRYPTO_TFM_RES_MASK); 111 112 data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr), 113 GFP_KERNEL); 114 if (!data) 115 return -ENOMEM; 116 117 init_completion(&data->result.completion); 118 sg_init_one(data->sg, &data->hash, sizeof(data->hash)); 119 ablkcipher_request_set_tfm(&data->req, ctr); 120 ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | 121 CRYPTO_TFM_REQ_MAY_BACKLOG, 122 crypto_gcm_setkey_done, 123 &data->result); 124 ablkcipher_request_set_crypt(&data->req, data->sg, data->sg, 125 sizeof(data->hash), data->iv); 126 127 err = crypto_ablkcipher_encrypt(&data->req); 128 if (err == -EINPROGRESS || err == -EBUSY) { 129 err = wait_for_completion_interruptible( 130 &data->result.completion); 131 if (!err) 132 err = data->result.err; 133 } 134 135 if (err) 136 goto out; 137 138 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); 139 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & 140 CRYPTO_TFM_REQ_MASK); 141 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); 142 crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) & 143 CRYPTO_TFM_RES_MASK); 144 145 out: 146 kfree(data); 147 return err; 148 } 149 150 static int crypto_gcm_setauthsize(struct crypto_aead *tfm, 151 unsigned int authsize) 152 { 153 switch (authsize) { 154 case 4: 155 case 8: 156 case 12: 157 case 13: 158 case 14: 159 case 15: 160 case 16: 161 break; 162 default: 163 return -EINVAL; 164 } 165 166 return 0; 167 } 168 169 static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, 170 struct aead_request *req, 171 unsigned int cryptlen) 172 { 173 struct crypto_aead *aead = crypto_aead_reqtfm(req); 174 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 175 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 176 struct scatterlist *dst; 177 __be32 counter = cpu_to_be32(1); 178 179 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); 180 memcpy(req->iv + 12, &counter, 4); 181 182 sg_init_table(pctx->src, 2); 183 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); 184 scatterwalk_sg_chain(pctx->src, 2, req->src); 185 186 dst = pctx->src; 187 if (req->src != req->dst) { 188 sg_init_table(pctx->dst, 2); 189 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); 190 scatterwalk_sg_chain(pctx->dst, 2, req->dst); 191 dst = pctx->dst; 192 } 193 194 ablkcipher_request_set_tfm(ablk_req, ctx->ctr); 195 ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, 196 cryptlen + sizeof(pctx->auth_tag), 197 req->iv); 198 } 199 200 static inline unsigned int gcm_remain(unsigned int len) 201 { 202 len &= 0xfU; 203 return len ? 16 - len : 0; 204 } 205 206 static void gcm_hash_len_done(struct crypto_async_request *areq, int err); 207 static void gcm_hash_final_done(struct crypto_async_request *areq, int err); 208 209 static int gcm_hash_update(struct aead_request *req, 210 struct crypto_gcm_req_priv_ctx *pctx, 211 crypto_completion_t complete, 212 struct scatterlist *src, 213 unsigned int len) 214 { 215 struct ahash_request *ahreq = &pctx->u.ahreq; 216 217 ahash_request_set_callback(ahreq, aead_request_flags(req), 218 complete, req); 219 ahash_request_set_crypt(ahreq, src, NULL, len); 220 221 return crypto_ahash_update(ahreq); 222 } 223 224 static int gcm_hash_remain(struct aead_request *req, 225 struct crypto_gcm_req_priv_ctx *pctx, 226 unsigned int remain, 227 crypto_completion_t complete) 228 { 229 struct ahash_request *ahreq = &pctx->u.ahreq; 230 231 ahash_request_set_callback(ahreq, aead_request_flags(req), 232 complete, req); 233 sg_init_one(pctx->src, gcm_zeroes, remain); 234 ahash_request_set_crypt(ahreq, pctx->src, NULL, remain); 235 236 return crypto_ahash_update(ahreq); 237 } 238 239 static int gcm_hash_len(struct aead_request *req, 240 struct crypto_gcm_req_priv_ctx *pctx) 241 { 242 struct ahash_request *ahreq = &pctx->u.ahreq; 243 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 244 u128 lengths; 245 246 lengths.a = cpu_to_be64(req->assoclen * 8); 247 lengths.b = cpu_to_be64(gctx->cryptlen * 8); 248 memcpy(pctx->iauth_tag, &lengths, 16); 249 sg_init_one(pctx->src, pctx->iauth_tag, 16); 250 ahash_request_set_callback(ahreq, aead_request_flags(req), 251 gcm_hash_len_done, req); 252 ahash_request_set_crypt(ahreq, pctx->src, 253 NULL, sizeof(lengths)); 254 255 return crypto_ahash_update(ahreq); 256 } 257 258 static int gcm_hash_final(struct aead_request *req, 259 struct crypto_gcm_req_priv_ctx *pctx) 260 { 261 struct ahash_request *ahreq = &pctx->u.ahreq; 262 263 ahash_request_set_callback(ahreq, aead_request_flags(req), 264 gcm_hash_final_done, req); 265 ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0); 266 267 return crypto_ahash_final(ahreq); 268 } 269 270 static void __gcm_hash_final_done(struct aead_request *req, int err) 271 { 272 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 273 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 274 275 if (!err) 276 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); 277 278 gctx->complete(req, err); 279 } 280 281 static void gcm_hash_final_done(struct crypto_async_request *areq, int err) 282 { 283 struct aead_request *req = areq->data; 284 285 __gcm_hash_final_done(req, err); 286 } 287 288 static void __gcm_hash_len_done(struct aead_request *req, int err) 289 { 290 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 291 292 if (!err) { 293 err = gcm_hash_final(req, pctx); 294 if (err == -EINPROGRESS || err == -EBUSY) 295 return; 296 } 297 298 __gcm_hash_final_done(req, err); 299 } 300 301 static void gcm_hash_len_done(struct crypto_async_request *areq, int err) 302 { 303 struct aead_request *req = areq->data; 304 305 __gcm_hash_len_done(req, err); 306 } 307 308 static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err) 309 { 310 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 311 312 if (!err) { 313 err = gcm_hash_len(req, pctx); 314 if (err == -EINPROGRESS || err == -EBUSY) 315 return; 316 } 317 318 __gcm_hash_len_done(req, err); 319 } 320 321 static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, 322 int err) 323 { 324 struct aead_request *req = areq->data; 325 326 __gcm_hash_crypt_remain_done(req, err); 327 } 328 329 static void __gcm_hash_crypt_done(struct aead_request *req, int err) 330 { 331 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 332 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 333 unsigned int remain; 334 335 if (!err) { 336 remain = gcm_remain(gctx->cryptlen); 337 BUG_ON(!remain); 338 err = gcm_hash_remain(req, pctx, remain, 339 gcm_hash_crypt_remain_done); 340 if (err == -EINPROGRESS || err == -EBUSY) 341 return; 342 } 343 344 __gcm_hash_crypt_remain_done(req, err); 345 } 346 347 static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err) 348 { 349 struct aead_request *req = areq->data; 350 351 __gcm_hash_crypt_done(req, err); 352 } 353 354 static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err) 355 { 356 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 357 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 358 crypto_completion_t complete; 359 unsigned int remain = 0; 360 361 if (!err && gctx->cryptlen) { 362 remain = gcm_remain(gctx->cryptlen); 363 complete = remain ? gcm_hash_crypt_done : 364 gcm_hash_crypt_remain_done; 365 err = gcm_hash_update(req, pctx, complete, 366 gctx->src, gctx->cryptlen); 367 if (err == -EINPROGRESS || err == -EBUSY) 368 return; 369 } 370 371 if (remain) 372 __gcm_hash_crypt_done(req, err); 373 else 374 __gcm_hash_crypt_remain_done(req, err); 375 } 376 377 static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, 378 int err) 379 { 380 struct aead_request *req = areq->data; 381 382 __gcm_hash_assoc_remain_done(req, err); 383 } 384 385 static void __gcm_hash_assoc_done(struct aead_request *req, int err) 386 { 387 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 388 unsigned int remain; 389 390 if (!err) { 391 remain = gcm_remain(req->assoclen); 392 BUG_ON(!remain); 393 err = gcm_hash_remain(req, pctx, remain, 394 gcm_hash_assoc_remain_done); 395 if (err == -EINPROGRESS || err == -EBUSY) 396 return; 397 } 398 399 __gcm_hash_assoc_remain_done(req, err); 400 } 401 402 static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err) 403 { 404 struct aead_request *req = areq->data; 405 406 __gcm_hash_assoc_done(req, err); 407 } 408 409 static void __gcm_hash_init_done(struct aead_request *req, int err) 410 { 411 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 412 crypto_completion_t complete; 413 unsigned int remain = 0; 414 415 if (!err && req->assoclen) { 416 remain = gcm_remain(req->assoclen); 417 complete = remain ? gcm_hash_assoc_done : 418 gcm_hash_assoc_remain_done; 419 err = gcm_hash_update(req, pctx, complete, 420 req->assoc, req->assoclen); 421 if (err == -EINPROGRESS || err == -EBUSY) 422 return; 423 } 424 425 if (remain) 426 __gcm_hash_assoc_done(req, err); 427 else 428 __gcm_hash_assoc_remain_done(req, err); 429 } 430 431 static void gcm_hash_init_done(struct crypto_async_request *areq, int err) 432 { 433 struct aead_request *req = areq->data; 434 435 __gcm_hash_init_done(req, err); 436 } 437 438 static int gcm_hash(struct aead_request *req, 439 struct crypto_gcm_req_priv_ctx *pctx) 440 { 441 struct ahash_request *ahreq = &pctx->u.ahreq; 442 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 443 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 444 unsigned int remain; 445 crypto_completion_t complete; 446 int err; 447 448 ahash_request_set_tfm(ahreq, ctx->ghash); 449 450 ahash_request_set_callback(ahreq, aead_request_flags(req), 451 gcm_hash_init_done, req); 452 err = crypto_ahash_init(ahreq); 453 if (err) 454 return err; 455 remain = gcm_remain(req->assoclen); 456 complete = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done; 457 err = gcm_hash_update(req, pctx, complete, req->assoc, req->assoclen); 458 if (err) 459 return err; 460 if (remain) { 461 err = gcm_hash_remain(req, pctx, remain, 462 gcm_hash_assoc_remain_done); 463 if (err) 464 return err; 465 } 466 remain = gcm_remain(gctx->cryptlen); 467 complete = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done; 468 err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen); 469 if (err) 470 return err; 471 if (remain) { 472 err = gcm_hash_remain(req, pctx, remain, 473 gcm_hash_crypt_remain_done); 474 if (err) 475 return err; 476 } 477 err = gcm_hash_len(req, pctx); 478 if (err) 479 return err; 480 err = gcm_hash_final(req, pctx); 481 if (err) 482 return err; 483 484 return 0; 485 } 486 487 static void gcm_enc_copy_hash(struct aead_request *req, 488 struct crypto_gcm_req_priv_ctx *pctx) 489 { 490 struct crypto_aead *aead = crypto_aead_reqtfm(req); 491 u8 *auth_tag = pctx->auth_tag; 492 493 scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen, 494 crypto_aead_authsize(aead), 1); 495 } 496 497 static void gcm_enc_hash_done(struct aead_request *req, int err) 498 { 499 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 500 501 if (!err) 502 gcm_enc_copy_hash(req, pctx); 503 504 aead_request_complete(req, err); 505 } 506 507 static void gcm_encrypt_done(struct crypto_async_request *areq, int err) 508 { 509 struct aead_request *req = areq->data; 510 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 511 512 if (!err) { 513 err = gcm_hash(req, pctx); 514 if (err == -EINPROGRESS || err == -EBUSY) 515 return; 516 else if (!err) { 517 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); 518 gcm_enc_copy_hash(req, pctx); 519 } 520 } 521 522 aead_request_complete(req, err); 523 } 524 525 static int crypto_gcm_encrypt(struct aead_request *req) 526 { 527 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 528 struct ablkcipher_request *abreq = &pctx->u.abreq; 529 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 530 int err; 531 532 crypto_gcm_init_crypt(abreq, req, req->cryptlen); 533 ablkcipher_request_set_callback(abreq, aead_request_flags(req), 534 gcm_encrypt_done, req); 535 536 gctx->src = req->dst; 537 gctx->cryptlen = req->cryptlen; 538 gctx->complete = gcm_enc_hash_done; 539 540 err = crypto_ablkcipher_encrypt(abreq); 541 if (err) 542 return err; 543 544 err = gcm_hash(req, pctx); 545 if (err) 546 return err; 547 548 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); 549 gcm_enc_copy_hash(req, pctx); 550 551 return 0; 552 } 553 554 static int crypto_gcm_verify(struct aead_request *req, 555 struct crypto_gcm_req_priv_ctx *pctx) 556 { 557 struct crypto_aead *aead = crypto_aead_reqtfm(req); 558 u8 *auth_tag = pctx->auth_tag; 559 u8 *iauth_tag = pctx->iauth_tag; 560 unsigned int authsize = crypto_aead_authsize(aead); 561 unsigned int cryptlen = req->cryptlen - authsize; 562 563 crypto_xor(auth_tag, iauth_tag, 16); 564 scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0); 565 return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; 566 } 567 568 static void gcm_decrypt_done(struct crypto_async_request *areq, int err) 569 { 570 struct aead_request *req = areq->data; 571 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 572 573 if (!err) 574 err = crypto_gcm_verify(req, pctx); 575 576 aead_request_complete(req, err); 577 } 578 579 static void gcm_dec_hash_done(struct aead_request *req, int err) 580 { 581 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 582 struct ablkcipher_request *abreq = &pctx->u.abreq; 583 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 584 585 if (!err) { 586 ablkcipher_request_set_callback(abreq, aead_request_flags(req), 587 gcm_decrypt_done, req); 588 crypto_gcm_init_crypt(abreq, req, gctx->cryptlen); 589 err = crypto_ablkcipher_decrypt(abreq); 590 if (err == -EINPROGRESS || err == -EBUSY) 591 return; 592 else if (!err) 593 err = crypto_gcm_verify(req, pctx); 594 } 595 596 aead_request_complete(req, err); 597 } 598 599 static int crypto_gcm_decrypt(struct aead_request *req) 600 { 601 struct crypto_aead *aead = crypto_aead_reqtfm(req); 602 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 603 struct ablkcipher_request *abreq = &pctx->u.abreq; 604 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 605 unsigned int authsize = crypto_aead_authsize(aead); 606 unsigned int cryptlen = req->cryptlen; 607 int err; 608 609 if (cryptlen < authsize) 610 return -EINVAL; 611 cryptlen -= authsize; 612 613 gctx->src = req->src; 614 gctx->cryptlen = cryptlen; 615 gctx->complete = gcm_dec_hash_done; 616 617 err = gcm_hash(req, pctx); 618 if (err) 619 return err; 620 621 ablkcipher_request_set_callback(abreq, aead_request_flags(req), 622 gcm_decrypt_done, req); 623 crypto_gcm_init_crypt(abreq, req, cryptlen); 624 err = crypto_ablkcipher_decrypt(abreq); 625 if (err) 626 return err; 627 628 return crypto_gcm_verify(req, pctx); 629 } 630 631 static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) 632 { 633 struct crypto_instance *inst = (void *)tfm->__crt_alg; 634 struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst); 635 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); 636 struct crypto_ablkcipher *ctr; 637 struct crypto_ahash *ghash; 638 unsigned long align; 639 int err; 640 641 ghash = crypto_spawn_ahash(&ictx->ghash); 642 if (IS_ERR(ghash)) 643 return PTR_ERR(ghash); 644 645 ctr = crypto_spawn_skcipher(&ictx->ctr); 646 err = PTR_ERR(ctr); 647 if (IS_ERR(ctr)) 648 goto err_free_hash; 649 650 ctx->ctr = ctr; 651 ctx->ghash = ghash; 652 653 align = crypto_tfm_alg_alignmask(tfm); 654 align &= ~(crypto_tfm_ctx_alignment() - 1); 655 tfm->crt_aead.reqsize = align + 656 offsetof(struct crypto_gcm_req_priv_ctx, u) + 657 max(sizeof(struct ablkcipher_request) + 658 crypto_ablkcipher_reqsize(ctr), 659 sizeof(struct ahash_request) + 660 crypto_ahash_reqsize(ghash)); 661 662 return 0; 663 664 err_free_hash: 665 crypto_free_ahash(ghash); 666 return err; 667 } 668 669 static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm) 670 { 671 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); 672 673 crypto_free_ahash(ctx->ghash); 674 crypto_free_ablkcipher(ctx->ctr); 675 } 676 677 static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, 678 const char *full_name, 679 const char *ctr_name, 680 const char *ghash_name) 681 { 682 struct crypto_attr_type *algt; 683 struct crypto_instance *inst; 684 struct crypto_alg *ctr; 685 struct crypto_alg *ghash_alg; 686 struct ahash_alg *ghash_ahash_alg; 687 struct gcm_instance_ctx *ctx; 688 int err; 689 690 algt = crypto_get_attr_type(tb); 691 err = PTR_ERR(algt); 692 if (IS_ERR(algt)) 693 return ERR_PTR(err); 694 695 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 696 return ERR_PTR(-EINVAL); 697 698 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, 699 CRYPTO_ALG_TYPE_HASH, 700 CRYPTO_ALG_TYPE_AHASH_MASK); 701 err = PTR_ERR(ghash_alg); 702 if (IS_ERR(ghash_alg)) 703 return ERR_PTR(err); 704 705 err = -ENOMEM; 706 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 707 if (!inst) 708 goto out_put_ghash; 709 710 ctx = crypto_instance_ctx(inst); 711 ghash_ahash_alg = container_of(ghash_alg, struct ahash_alg, halg.base); 712 err = crypto_init_ahash_spawn(&ctx->ghash, &ghash_ahash_alg->halg, 713 inst); 714 if (err) 715 goto err_free_inst; 716 717 crypto_set_skcipher_spawn(&ctx->ctr, inst); 718 err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, 719 crypto_requires_sync(algt->type, 720 algt->mask)); 721 if (err) 722 goto err_drop_ghash; 723 724 ctr = crypto_skcipher_spawn_alg(&ctx->ctr); 725 726 /* We only support 16-byte blocks. */ 727 if (ctr->cra_ablkcipher.ivsize != 16) 728 goto out_put_ctr; 729 730 /* Not a stream cipher? */ 731 err = -EINVAL; 732 if (ctr->cra_blocksize != 1) 733 goto out_put_ctr; 734 735 err = -ENAMETOOLONG; 736 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 737 "gcm_base(%s,%s)", ctr->cra_driver_name, 738 ghash_alg->cra_driver_name) >= 739 CRYPTO_MAX_ALG_NAME) 740 goto out_put_ctr; 741 742 memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME); 743 744 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; 745 inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC; 746 inst->alg.cra_priority = ctr->cra_priority; 747 inst->alg.cra_blocksize = 1; 748 inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1); 749 inst->alg.cra_type = &crypto_aead_type; 750 inst->alg.cra_aead.ivsize = 16; 751 inst->alg.cra_aead.maxauthsize = 16; 752 inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx); 753 inst->alg.cra_init = crypto_gcm_init_tfm; 754 inst->alg.cra_exit = crypto_gcm_exit_tfm; 755 inst->alg.cra_aead.setkey = crypto_gcm_setkey; 756 inst->alg.cra_aead.setauthsize = crypto_gcm_setauthsize; 757 inst->alg.cra_aead.encrypt = crypto_gcm_encrypt; 758 inst->alg.cra_aead.decrypt = crypto_gcm_decrypt; 759 760 out: 761 crypto_mod_put(ghash_alg); 762 return inst; 763 764 out_put_ctr: 765 crypto_drop_skcipher(&ctx->ctr); 766 err_drop_ghash: 767 crypto_drop_ahash(&ctx->ghash); 768 err_free_inst: 769 kfree(inst); 770 out_put_ghash: 771 inst = ERR_PTR(err); 772 goto out; 773 } 774 775 static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb) 776 { 777 int err; 778 const char *cipher_name; 779 char ctr_name[CRYPTO_MAX_ALG_NAME]; 780 char full_name[CRYPTO_MAX_ALG_NAME]; 781 782 cipher_name = crypto_attr_alg_name(tb[1]); 783 err = PTR_ERR(cipher_name); 784 if (IS_ERR(cipher_name)) 785 return ERR_PTR(err); 786 787 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= 788 CRYPTO_MAX_ALG_NAME) 789 return ERR_PTR(-ENAMETOOLONG); 790 791 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >= 792 CRYPTO_MAX_ALG_NAME) 793 return ERR_PTR(-ENAMETOOLONG); 794 795 return crypto_gcm_alloc_common(tb, full_name, ctr_name, "ghash"); 796 } 797 798 static void crypto_gcm_free(struct crypto_instance *inst) 799 { 800 struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst); 801 802 crypto_drop_skcipher(&ctx->ctr); 803 crypto_drop_ahash(&ctx->ghash); 804 kfree(inst); 805 } 806 807 static struct crypto_template crypto_gcm_tmpl = { 808 .name = "gcm", 809 .alloc = crypto_gcm_alloc, 810 .free = crypto_gcm_free, 811 .module = THIS_MODULE, 812 }; 813 814 static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) 815 { 816 int err; 817 const char *ctr_name; 818 const char *ghash_name; 819 char full_name[CRYPTO_MAX_ALG_NAME]; 820 821 ctr_name = crypto_attr_alg_name(tb[1]); 822 err = PTR_ERR(ctr_name); 823 if (IS_ERR(ctr_name)) 824 return ERR_PTR(err); 825 826 ghash_name = crypto_attr_alg_name(tb[2]); 827 err = PTR_ERR(ghash_name); 828 if (IS_ERR(ghash_name)) 829 return ERR_PTR(err); 830 831 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", 832 ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) 833 return ERR_PTR(-ENAMETOOLONG); 834 835 return crypto_gcm_alloc_common(tb, full_name, ctr_name, ghash_name); 836 } 837 838 static struct crypto_template crypto_gcm_base_tmpl = { 839 .name = "gcm_base", 840 .alloc = crypto_gcm_base_alloc, 841 .free = crypto_gcm_free, 842 .module = THIS_MODULE, 843 }; 844 845 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, 846 unsigned int keylen) 847 { 848 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 849 struct crypto_aead *child = ctx->child; 850 int err; 851 852 if (keylen < 4) 853 return -EINVAL; 854 855 keylen -= 4; 856 memcpy(ctx->nonce, key + keylen, 4); 857 858 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 859 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 860 CRYPTO_TFM_REQ_MASK); 861 err = crypto_aead_setkey(child, key, keylen); 862 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) & 863 CRYPTO_TFM_RES_MASK); 864 865 return err; 866 } 867 868 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, 869 unsigned int authsize) 870 { 871 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 872 873 switch (authsize) { 874 case 8: 875 case 12: 876 case 16: 877 break; 878 default: 879 return -EINVAL; 880 } 881 882 return crypto_aead_setauthsize(ctx->child, authsize); 883 } 884 885 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) 886 { 887 struct aead_request *subreq = aead_request_ctx(req); 888 struct crypto_aead *aead = crypto_aead_reqtfm(req); 889 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); 890 struct crypto_aead *child = ctx->child; 891 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), 892 crypto_aead_alignmask(child) + 1); 893 894 memcpy(iv, ctx->nonce, 4); 895 memcpy(iv + 4, req->iv, 8); 896 897 aead_request_set_tfm(subreq, child); 898 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 899 req->base.data); 900 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv); 901 aead_request_set_assoc(subreq, req->assoc, req->assoclen); 902 903 return subreq; 904 } 905 906 static int crypto_rfc4106_encrypt(struct aead_request *req) 907 { 908 req = crypto_rfc4106_crypt(req); 909 910 return crypto_aead_encrypt(req); 911 } 912 913 static int crypto_rfc4106_decrypt(struct aead_request *req) 914 { 915 req = crypto_rfc4106_crypt(req); 916 917 return crypto_aead_decrypt(req); 918 } 919 920 static int crypto_rfc4106_init_tfm(struct crypto_tfm *tfm) 921 { 922 struct crypto_instance *inst = (void *)tfm->__crt_alg; 923 struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst); 924 struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm); 925 struct crypto_aead *aead; 926 unsigned long align; 927 928 aead = crypto_spawn_aead(spawn); 929 if (IS_ERR(aead)) 930 return PTR_ERR(aead); 931 932 ctx->child = aead; 933 934 align = crypto_aead_alignmask(aead); 935 align &= ~(crypto_tfm_ctx_alignment() - 1); 936 tfm->crt_aead.reqsize = sizeof(struct aead_request) + 937 ALIGN(crypto_aead_reqsize(aead), 938 crypto_tfm_ctx_alignment()) + 939 align + 16; 940 941 return 0; 942 } 943 944 static void crypto_rfc4106_exit_tfm(struct crypto_tfm *tfm) 945 { 946 struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm); 947 948 crypto_free_aead(ctx->child); 949 } 950 951 static struct crypto_instance *crypto_rfc4106_alloc(struct rtattr **tb) 952 { 953 struct crypto_attr_type *algt; 954 struct crypto_instance *inst; 955 struct crypto_aead_spawn *spawn; 956 struct crypto_alg *alg; 957 const char *ccm_name; 958 int err; 959 960 algt = crypto_get_attr_type(tb); 961 err = PTR_ERR(algt); 962 if (IS_ERR(algt)) 963 return ERR_PTR(err); 964 965 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 966 return ERR_PTR(-EINVAL); 967 968 ccm_name = crypto_attr_alg_name(tb[1]); 969 err = PTR_ERR(ccm_name); 970 if (IS_ERR(ccm_name)) 971 return ERR_PTR(err); 972 973 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 974 if (!inst) 975 return ERR_PTR(-ENOMEM); 976 977 spawn = crypto_instance_ctx(inst); 978 crypto_set_aead_spawn(spawn, inst); 979 err = crypto_grab_aead(spawn, ccm_name, 0, 980 crypto_requires_sync(algt->type, algt->mask)); 981 if (err) 982 goto out_free_inst; 983 984 alg = crypto_aead_spawn_alg(spawn); 985 986 err = -EINVAL; 987 988 /* We only support 16-byte blocks. */ 989 if (alg->cra_aead.ivsize != 16) 990 goto out_drop_alg; 991 992 /* Not a stream cipher? */ 993 if (alg->cra_blocksize != 1) 994 goto out_drop_alg; 995 996 err = -ENAMETOOLONG; 997 if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, 998 "rfc4106(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME || 999 snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 1000 "rfc4106(%s)", alg->cra_driver_name) >= 1001 CRYPTO_MAX_ALG_NAME) 1002 goto out_drop_alg; 1003 1004 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; 1005 inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC; 1006 inst->alg.cra_priority = alg->cra_priority; 1007 inst->alg.cra_blocksize = 1; 1008 inst->alg.cra_alignmask = alg->cra_alignmask; 1009 inst->alg.cra_type = &crypto_nivaead_type; 1010 1011 inst->alg.cra_aead.ivsize = 8; 1012 inst->alg.cra_aead.maxauthsize = 16; 1013 1014 inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); 1015 1016 inst->alg.cra_init = crypto_rfc4106_init_tfm; 1017 inst->alg.cra_exit = crypto_rfc4106_exit_tfm; 1018 1019 inst->alg.cra_aead.setkey = crypto_rfc4106_setkey; 1020 inst->alg.cra_aead.setauthsize = crypto_rfc4106_setauthsize; 1021 inst->alg.cra_aead.encrypt = crypto_rfc4106_encrypt; 1022 inst->alg.cra_aead.decrypt = crypto_rfc4106_decrypt; 1023 1024 inst->alg.cra_aead.geniv = "seqiv"; 1025 1026 out: 1027 return inst; 1028 1029 out_drop_alg: 1030 crypto_drop_aead(spawn); 1031 out_free_inst: 1032 kfree(inst); 1033 inst = ERR_PTR(err); 1034 goto out; 1035 } 1036 1037 static void crypto_rfc4106_free(struct crypto_instance *inst) 1038 { 1039 crypto_drop_spawn(crypto_instance_ctx(inst)); 1040 kfree(inst); 1041 } 1042 1043 static struct crypto_template crypto_rfc4106_tmpl = { 1044 .name = "rfc4106", 1045 .alloc = crypto_rfc4106_alloc, 1046 .free = crypto_rfc4106_free, 1047 .module = THIS_MODULE, 1048 }; 1049 1050 static int __init crypto_gcm_module_init(void) 1051 { 1052 int err; 1053 1054 gcm_zeroes = kzalloc(16, GFP_KERNEL); 1055 if (!gcm_zeroes) 1056 return -ENOMEM; 1057 1058 err = crypto_register_template(&crypto_gcm_base_tmpl); 1059 if (err) 1060 goto out; 1061 1062 err = crypto_register_template(&crypto_gcm_tmpl); 1063 if (err) 1064 goto out_undo_base; 1065 1066 err = crypto_register_template(&crypto_rfc4106_tmpl); 1067 if (err) 1068 goto out_undo_gcm; 1069 1070 return 0; 1071 1072 out_undo_gcm: 1073 crypto_unregister_template(&crypto_gcm_tmpl); 1074 out_undo_base: 1075 crypto_unregister_template(&crypto_gcm_base_tmpl); 1076 out: 1077 kfree(gcm_zeroes); 1078 return err; 1079 } 1080 1081 static void __exit crypto_gcm_module_exit(void) 1082 { 1083 kfree(gcm_zeroes); 1084 crypto_unregister_template(&crypto_rfc4106_tmpl); 1085 crypto_unregister_template(&crypto_gcm_tmpl); 1086 crypto_unregister_template(&crypto_gcm_base_tmpl); 1087 } 1088 1089 module_init(crypto_gcm_module_init); 1090 module_exit(crypto_gcm_module_exit); 1091 1092 MODULE_LICENSE("GPL"); 1093 MODULE_DESCRIPTION("Galois/Counter Mode"); 1094 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); 1095 MODULE_ALIAS("gcm_base"); 1096 MODULE_ALIAS("rfc4106"); 1097