1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GCM: Galois/Counter Mode. 4 * 5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 6 */ 7 8 #include <crypto/gf128mul.h> 9 #include <crypto/internal/aead.h> 10 #include <crypto/internal/skcipher.h> 11 #include <crypto/internal/hash.h> 12 #include <crypto/null.h> 13 #include <crypto/scatterwalk.h> 14 #include <crypto/gcm.h> 15 #include <crypto/hash.h> 16 #include "internal.h" 17 #include <linux/err.h> 18 #include <linux/init.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/slab.h> 22 23 struct gcm_instance_ctx { 24 struct crypto_skcipher_spawn ctr; 25 struct crypto_ahash_spawn ghash; 26 }; 27 28 struct crypto_gcm_ctx { 29 struct crypto_skcipher *ctr; 30 struct crypto_ahash *ghash; 31 }; 32 33 struct crypto_rfc4106_ctx { 34 struct crypto_aead *child; 35 u8 nonce[4]; 36 }; 37 38 struct crypto_rfc4106_req_ctx { 39 struct scatterlist src[3]; 40 struct scatterlist dst[3]; 41 struct aead_request subreq; 42 }; 43 44 struct crypto_rfc4543_instance_ctx { 45 struct crypto_aead_spawn aead; 46 }; 47 48 struct crypto_rfc4543_ctx { 49 struct crypto_aead *child; 50 struct crypto_sync_skcipher *null; 51 u8 nonce[4]; 52 }; 53 54 struct crypto_rfc4543_req_ctx { 55 struct aead_request subreq; 56 }; 57 58 struct crypto_gcm_ghash_ctx { 59 unsigned int cryptlen; 60 struct scatterlist *src; 61 int (*complete)(struct aead_request *req, u32 flags); 62 }; 63 64 struct crypto_gcm_req_priv_ctx { 65 u8 iv[16]; 66 u8 auth_tag[16]; 67 u8 iauth_tag[16]; 68 struct scatterlist src[3]; 69 struct scatterlist dst[3]; 70 struct scatterlist sg; 71 struct crypto_gcm_ghash_ctx ghash_ctx; 72 union { 73 struct ahash_request ahreq; 74 struct skcipher_request skreq; 75 } u; 76 }; 77 78 static struct { 79 u8 buf[16]; 80 struct scatterlist sg; 81 } *gcm_zeroes; 82 83 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc); 84 85 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( 86 struct aead_request *req) 87 { 88 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); 89 90 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); 91 } 92 93 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, 94 unsigned int keylen) 95 { 96 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 97 struct crypto_ahash *ghash = ctx->ghash; 98 struct crypto_skcipher *ctr = ctx->ctr; 99 struct { 100 be128 hash; 101 u8 iv[16]; 102 103 struct crypto_wait wait; 104 105 struct scatterlist sg[1]; 106 struct skcipher_request req; 107 } *data; 108 int err; 109 110 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); 111 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & 112 CRYPTO_TFM_REQ_MASK); 113 err = crypto_skcipher_setkey(ctr, key, keylen); 114 crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) & 115 CRYPTO_TFM_RES_MASK); 116 if (err) 117 return err; 118 119 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr), 120 GFP_KERNEL); 121 if (!data) 122 return -ENOMEM; 123 124 crypto_init_wait(&data->wait); 125 sg_init_one(data->sg, &data->hash, sizeof(data->hash)); 126 skcipher_request_set_tfm(&data->req, ctr); 127 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | 128 CRYPTO_TFM_REQ_MAY_BACKLOG, 129 crypto_req_done, 130 &data->wait); 131 skcipher_request_set_crypt(&data->req, data->sg, data->sg, 132 sizeof(data->hash), data->iv); 133 134 err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), 135 &data->wait); 136 137 if (err) 138 goto out; 139 140 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); 141 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & 142 CRYPTO_TFM_REQ_MASK); 143 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); 144 crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) & 145 CRYPTO_TFM_RES_MASK); 146 147 out: 148 kzfree(data); 149 return err; 150 } 151 152 static int crypto_gcm_setauthsize(struct crypto_aead *tfm, 153 unsigned int authsize) 154 { 155 switch (authsize) { 156 case 4: 157 case 8: 158 case 12: 159 case 13: 160 case 14: 161 case 15: 162 case 16: 163 break; 164 default: 165 return -EINVAL; 166 } 167 168 return 0; 169 } 170 171 static void crypto_gcm_init_common(struct aead_request *req) 172 { 173 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 174 __be32 counter = cpu_to_be32(1); 175 struct scatterlist *sg; 176 177 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); 178 memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE); 179 memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4); 180 181 sg_init_table(pctx->src, 3); 182 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); 183 sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); 184 if (sg != pctx->src + 1) 185 sg_chain(pctx->src, 2, sg); 186 187 if (req->src != req->dst) { 188 sg_init_table(pctx->dst, 3); 189 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); 190 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); 191 if (sg != pctx->dst + 1) 192 sg_chain(pctx->dst, 2, sg); 193 } 194 } 195 196 static void crypto_gcm_init_crypt(struct aead_request *req, 197 unsigned int cryptlen) 198 { 199 struct crypto_aead *aead = crypto_aead_reqtfm(req); 200 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); 201 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 202 struct skcipher_request *skreq = &pctx->u.skreq; 203 struct scatterlist *dst; 204 205 dst = req->src == req->dst ? pctx->src : pctx->dst; 206 207 skcipher_request_set_tfm(skreq, ctx->ctr); 208 skcipher_request_set_crypt(skreq, pctx->src, dst, 209 cryptlen + sizeof(pctx->auth_tag), 210 pctx->iv); 211 } 212 213 static inline unsigned int gcm_remain(unsigned int len) 214 { 215 len &= 0xfU; 216 return len ? 16 - len : 0; 217 } 218 219 static void gcm_hash_len_done(struct crypto_async_request *areq, int err); 220 221 static int gcm_hash_update(struct aead_request *req, 222 crypto_completion_t compl, 223 struct scatterlist *src, 224 unsigned int len, u32 flags) 225 { 226 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 227 struct ahash_request *ahreq = &pctx->u.ahreq; 228 229 ahash_request_set_callback(ahreq, flags, compl, req); 230 ahash_request_set_crypt(ahreq, src, NULL, len); 231 232 return crypto_ahash_update(ahreq); 233 } 234 235 static int gcm_hash_remain(struct aead_request *req, 236 unsigned int remain, 237 crypto_completion_t compl, u32 flags) 238 { 239 return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); 240 } 241 242 static int gcm_hash_len(struct aead_request *req, u32 flags) 243 { 244 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 245 struct ahash_request *ahreq = &pctx->u.ahreq; 246 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 247 be128 lengths; 248 249 lengths.a = cpu_to_be64(req->assoclen * 8); 250 lengths.b = cpu_to_be64(gctx->cryptlen * 8); 251 memcpy(pctx->iauth_tag, &lengths, 16); 252 sg_init_one(&pctx->sg, pctx->iauth_tag, 16); 253 ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); 254 ahash_request_set_crypt(ahreq, &pctx->sg, 255 pctx->iauth_tag, sizeof(lengths)); 256 257 return crypto_ahash_finup(ahreq); 258 } 259 260 static int gcm_hash_len_continue(struct aead_request *req, u32 flags) 261 { 262 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 263 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 264 265 return gctx->complete(req, flags); 266 } 267 268 static void gcm_hash_len_done(struct crypto_async_request *areq, int err) 269 { 270 struct aead_request *req = areq->data; 271 272 if (err) 273 goto out; 274 275 err = gcm_hash_len_continue(req, 0); 276 if (err == -EINPROGRESS) 277 return; 278 279 out: 280 aead_request_complete(req, err); 281 } 282 283 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) 284 { 285 return gcm_hash_len(req, flags) ?: 286 gcm_hash_len_continue(req, flags); 287 } 288 289 static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, 290 int err) 291 { 292 struct aead_request *req = areq->data; 293 294 if (err) 295 goto out; 296 297 err = gcm_hash_crypt_remain_continue(req, 0); 298 if (err == -EINPROGRESS) 299 return; 300 301 out: 302 aead_request_complete(req, err); 303 } 304 305 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) 306 { 307 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 308 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 309 unsigned int remain; 310 311 remain = gcm_remain(gctx->cryptlen); 312 if (remain) 313 return gcm_hash_remain(req, remain, 314 gcm_hash_crypt_remain_done, flags) ?: 315 gcm_hash_crypt_remain_continue(req, flags); 316 317 return gcm_hash_crypt_remain_continue(req, flags); 318 } 319 320 static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err) 321 { 322 struct aead_request *req = areq->data; 323 324 if (err) 325 goto out; 326 327 err = gcm_hash_crypt_continue(req, 0); 328 if (err == -EINPROGRESS) 329 return; 330 331 out: 332 aead_request_complete(req, err); 333 } 334 335 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) 336 { 337 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 338 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 339 340 if (gctx->cryptlen) 341 return gcm_hash_update(req, gcm_hash_crypt_done, 342 gctx->src, gctx->cryptlen, flags) ?: 343 gcm_hash_crypt_continue(req, flags); 344 345 return gcm_hash_crypt_remain_continue(req, flags); 346 } 347 348 static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, 349 int err) 350 { 351 struct aead_request *req = areq->data; 352 353 if (err) 354 goto out; 355 356 err = gcm_hash_assoc_remain_continue(req, 0); 357 if (err == -EINPROGRESS) 358 return; 359 360 out: 361 aead_request_complete(req, err); 362 } 363 364 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) 365 { 366 unsigned int remain; 367 368 remain = gcm_remain(req->assoclen); 369 if (remain) 370 return gcm_hash_remain(req, remain, 371 gcm_hash_assoc_remain_done, flags) ?: 372 gcm_hash_assoc_remain_continue(req, flags); 373 374 return gcm_hash_assoc_remain_continue(req, flags); 375 } 376 377 static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err) 378 { 379 struct aead_request *req = areq->data; 380 381 if (err) 382 goto out; 383 384 err = gcm_hash_assoc_continue(req, 0); 385 if (err == -EINPROGRESS) 386 return; 387 388 out: 389 aead_request_complete(req, err); 390 } 391 392 static int gcm_hash_init_continue(struct aead_request *req, u32 flags) 393 { 394 if (req->assoclen) 395 return gcm_hash_update(req, gcm_hash_assoc_done, 396 req->src, req->assoclen, flags) ?: 397 gcm_hash_assoc_continue(req, flags); 398 399 return gcm_hash_assoc_remain_continue(req, flags); 400 } 401 402 static void gcm_hash_init_done(struct crypto_async_request *areq, int err) 403 { 404 struct aead_request *req = areq->data; 405 406 if (err) 407 goto out; 408 409 err = gcm_hash_init_continue(req, 0); 410 if (err == -EINPROGRESS) 411 return; 412 413 out: 414 aead_request_complete(req, err); 415 } 416 417 static int gcm_hash(struct aead_request *req, u32 flags) 418 { 419 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 420 struct ahash_request *ahreq = &pctx->u.ahreq; 421 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 422 423 ahash_request_set_tfm(ahreq, ctx->ghash); 424 425 ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); 426 return crypto_ahash_init(ahreq) ?: 427 gcm_hash_init_continue(req, flags); 428 } 429 430 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) 431 { 432 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 433 struct crypto_aead *aead = crypto_aead_reqtfm(req); 434 u8 *auth_tag = pctx->auth_tag; 435 436 crypto_xor(auth_tag, pctx->iauth_tag, 16); 437 scatterwalk_map_and_copy(auth_tag, req->dst, 438 req->assoclen + req->cryptlen, 439 crypto_aead_authsize(aead), 1); 440 return 0; 441 } 442 443 static int gcm_encrypt_continue(struct aead_request *req, u32 flags) 444 { 445 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 446 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 447 448 gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); 449 gctx->cryptlen = req->cryptlen; 450 gctx->complete = gcm_enc_copy_hash; 451 452 return gcm_hash(req, flags); 453 } 454 455 static void gcm_encrypt_done(struct crypto_async_request *areq, int err) 456 { 457 struct aead_request *req = areq->data; 458 459 if (err) 460 goto out; 461 462 err = gcm_encrypt_continue(req, 0); 463 if (err == -EINPROGRESS) 464 return; 465 466 out: 467 aead_request_complete(req, err); 468 } 469 470 static int crypto_gcm_encrypt(struct aead_request *req) 471 { 472 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 473 struct skcipher_request *skreq = &pctx->u.skreq; 474 u32 flags = aead_request_flags(req); 475 476 crypto_gcm_init_common(req); 477 crypto_gcm_init_crypt(req, req->cryptlen); 478 skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); 479 480 return crypto_skcipher_encrypt(skreq) ?: 481 gcm_encrypt_continue(req, flags); 482 } 483 484 static int crypto_gcm_verify(struct aead_request *req) 485 { 486 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 487 struct crypto_aead *aead = crypto_aead_reqtfm(req); 488 u8 *auth_tag = pctx->auth_tag; 489 u8 *iauth_tag = pctx->iauth_tag; 490 unsigned int authsize = crypto_aead_authsize(aead); 491 unsigned int cryptlen = req->cryptlen - authsize; 492 493 crypto_xor(auth_tag, iauth_tag, 16); 494 scatterwalk_map_and_copy(iauth_tag, req->src, 495 req->assoclen + cryptlen, authsize, 0); 496 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; 497 } 498 499 static void gcm_decrypt_done(struct crypto_async_request *areq, int err) 500 { 501 struct aead_request *req = areq->data; 502 503 if (!err) 504 err = crypto_gcm_verify(req); 505 506 aead_request_complete(req, err); 507 } 508 509 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) 510 { 511 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 512 struct skcipher_request *skreq = &pctx->u.skreq; 513 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 514 515 crypto_gcm_init_crypt(req, gctx->cryptlen); 516 skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); 517 return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); 518 } 519 520 static int crypto_gcm_decrypt(struct aead_request *req) 521 { 522 struct crypto_aead *aead = crypto_aead_reqtfm(req); 523 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 524 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 525 unsigned int authsize = crypto_aead_authsize(aead); 526 unsigned int cryptlen = req->cryptlen; 527 u32 flags = aead_request_flags(req); 528 529 cryptlen -= authsize; 530 531 crypto_gcm_init_common(req); 532 533 gctx->src = sg_next(pctx->src); 534 gctx->cryptlen = cryptlen; 535 gctx->complete = gcm_dec_hash_continue; 536 537 return gcm_hash(req, flags); 538 } 539 540 static int crypto_gcm_init_tfm(struct crypto_aead *tfm) 541 { 542 struct aead_instance *inst = aead_alg_instance(tfm); 543 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); 544 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 545 struct crypto_skcipher *ctr; 546 struct crypto_ahash *ghash; 547 unsigned long align; 548 int err; 549 550 ghash = crypto_spawn_ahash(&ictx->ghash); 551 if (IS_ERR(ghash)) 552 return PTR_ERR(ghash); 553 554 ctr = crypto_spawn_skcipher(&ictx->ctr); 555 err = PTR_ERR(ctr); 556 if (IS_ERR(ctr)) 557 goto err_free_hash; 558 559 ctx->ctr = ctr; 560 ctx->ghash = ghash; 561 562 align = crypto_aead_alignmask(tfm); 563 align &= ~(crypto_tfm_ctx_alignment() - 1); 564 crypto_aead_set_reqsize(tfm, 565 align + offsetof(struct crypto_gcm_req_priv_ctx, u) + 566 max(sizeof(struct skcipher_request) + 567 crypto_skcipher_reqsize(ctr), 568 sizeof(struct ahash_request) + 569 crypto_ahash_reqsize(ghash))); 570 571 return 0; 572 573 err_free_hash: 574 crypto_free_ahash(ghash); 575 return err; 576 } 577 578 static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) 579 { 580 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); 581 582 crypto_free_ahash(ctx->ghash); 583 crypto_free_skcipher(ctx->ctr); 584 } 585 586 static void crypto_gcm_free(struct aead_instance *inst) 587 { 588 struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); 589 590 crypto_drop_skcipher(&ctx->ctr); 591 crypto_drop_ahash(&ctx->ghash); 592 kfree(inst); 593 } 594 595 static int crypto_gcm_create_common(struct crypto_template *tmpl, 596 struct rtattr **tb, 597 const char *ctr_name, 598 const char *ghash_name) 599 { 600 struct crypto_attr_type *algt; 601 struct aead_instance *inst; 602 struct skcipher_alg *ctr; 603 struct crypto_alg *ghash_alg; 604 struct hash_alg_common *ghash; 605 struct gcm_instance_ctx *ctx; 606 int err; 607 608 algt = crypto_get_attr_type(tb); 609 if (IS_ERR(algt)) 610 return PTR_ERR(algt); 611 612 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 613 return -EINVAL; 614 615 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, 616 CRYPTO_ALG_TYPE_HASH, 617 CRYPTO_ALG_TYPE_AHASH_MASK | 618 crypto_requires_sync(algt->type, 619 algt->mask)); 620 if (IS_ERR(ghash_alg)) 621 return PTR_ERR(ghash_alg); 622 623 ghash = __crypto_hash_alg_common(ghash_alg); 624 625 err = -ENOMEM; 626 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 627 if (!inst) 628 goto out_put_ghash; 629 630 ctx = aead_instance_ctx(inst); 631 err = crypto_init_ahash_spawn(&ctx->ghash, ghash, 632 aead_crypto_instance(inst)); 633 if (err) 634 goto err_free_inst; 635 636 err = -EINVAL; 637 if (strcmp(ghash->base.cra_name, "ghash") != 0 || 638 ghash->digestsize != 16) 639 goto err_drop_ghash; 640 641 crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst)); 642 err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, 643 crypto_requires_sync(algt->type, 644 algt->mask)); 645 if (err) 646 goto err_drop_ghash; 647 648 ctr = crypto_spawn_skcipher_alg(&ctx->ctr); 649 650 /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ 651 err = -EINVAL; 652 if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || 653 crypto_skcipher_alg_ivsize(ctr) != 16 || 654 ctr->base.cra_blocksize != 1) 655 goto out_put_ctr; 656 657 err = -ENAMETOOLONG; 658 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 659 "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) 660 goto out_put_ctr; 661 662 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 663 "gcm_base(%s,%s)", ctr->base.cra_driver_name, 664 ghash_alg->cra_driver_name) >= 665 CRYPTO_MAX_ALG_NAME) 666 goto out_put_ctr; 667 668 inst->alg.base.cra_flags = (ghash->base.cra_flags | 669 ctr->base.cra_flags) & CRYPTO_ALG_ASYNC; 670 inst->alg.base.cra_priority = (ghash->base.cra_priority + 671 ctr->base.cra_priority) / 2; 672 inst->alg.base.cra_blocksize = 1; 673 inst->alg.base.cra_alignmask = ghash->base.cra_alignmask | 674 ctr->base.cra_alignmask; 675 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); 676 inst->alg.ivsize = GCM_AES_IV_SIZE; 677 inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr); 678 inst->alg.maxauthsize = 16; 679 inst->alg.init = crypto_gcm_init_tfm; 680 inst->alg.exit = crypto_gcm_exit_tfm; 681 inst->alg.setkey = crypto_gcm_setkey; 682 inst->alg.setauthsize = crypto_gcm_setauthsize; 683 inst->alg.encrypt = crypto_gcm_encrypt; 684 inst->alg.decrypt = crypto_gcm_decrypt; 685 686 inst->free = crypto_gcm_free; 687 688 err = aead_register_instance(tmpl, inst); 689 if (err) 690 goto out_put_ctr; 691 692 out_put_ghash: 693 crypto_mod_put(ghash_alg); 694 return err; 695 696 out_put_ctr: 697 crypto_drop_skcipher(&ctx->ctr); 698 err_drop_ghash: 699 crypto_drop_ahash(&ctx->ghash); 700 err_free_inst: 701 kfree(inst); 702 goto out_put_ghash; 703 } 704 705 static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) 706 { 707 const char *cipher_name; 708 char ctr_name[CRYPTO_MAX_ALG_NAME]; 709 710 cipher_name = crypto_attr_alg_name(tb[1]); 711 if (IS_ERR(cipher_name)) 712 return PTR_ERR(cipher_name); 713 714 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= 715 CRYPTO_MAX_ALG_NAME) 716 return -ENAMETOOLONG; 717 718 return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); 719 } 720 721 static int crypto_gcm_base_create(struct crypto_template *tmpl, 722 struct rtattr **tb) 723 { 724 const char *ctr_name; 725 const char *ghash_name; 726 727 ctr_name = crypto_attr_alg_name(tb[1]); 728 if (IS_ERR(ctr_name)) 729 return PTR_ERR(ctr_name); 730 731 ghash_name = crypto_attr_alg_name(tb[2]); 732 if (IS_ERR(ghash_name)) 733 return PTR_ERR(ghash_name); 734 735 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); 736 } 737 738 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, 739 unsigned int keylen) 740 { 741 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 742 struct crypto_aead *child = ctx->child; 743 int err; 744 745 if (keylen < 4) 746 return -EINVAL; 747 748 keylen -= 4; 749 memcpy(ctx->nonce, key + keylen, 4); 750 751 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 752 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 753 CRYPTO_TFM_REQ_MASK); 754 err = crypto_aead_setkey(child, key, keylen); 755 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) & 756 CRYPTO_TFM_RES_MASK); 757 758 return err; 759 } 760 761 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, 762 unsigned int authsize) 763 { 764 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); 765 766 switch (authsize) { 767 case 8: 768 case 12: 769 case 16: 770 break; 771 default: 772 return -EINVAL; 773 } 774 775 return crypto_aead_setauthsize(ctx->child, authsize); 776 } 777 778 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) 779 { 780 struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req); 781 struct crypto_aead *aead = crypto_aead_reqtfm(req); 782 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); 783 struct aead_request *subreq = &rctx->subreq; 784 struct crypto_aead *child = ctx->child; 785 struct scatterlist *sg; 786 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), 787 crypto_aead_alignmask(child) + 1); 788 789 scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0); 790 791 memcpy(iv, ctx->nonce, 4); 792 memcpy(iv + 4, req->iv, 8); 793 794 sg_init_table(rctx->src, 3); 795 sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8); 796 sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); 797 if (sg != rctx->src + 1) 798 sg_chain(rctx->src, 2, sg); 799 800 if (req->src != req->dst) { 801 sg_init_table(rctx->dst, 3); 802 sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8); 803 sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); 804 if (sg != rctx->dst + 1) 805 sg_chain(rctx->dst, 2, sg); 806 } 807 808 aead_request_set_tfm(subreq, child); 809 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 810 req->base.data); 811 aead_request_set_crypt(subreq, rctx->src, 812 req->src == req->dst ? rctx->src : rctx->dst, 813 req->cryptlen, iv); 814 aead_request_set_ad(subreq, req->assoclen - 8); 815 816 return subreq; 817 } 818 819 static int crypto_rfc4106_encrypt(struct aead_request *req) 820 { 821 if (req->assoclen != 16 && req->assoclen != 20) 822 return -EINVAL; 823 824 req = crypto_rfc4106_crypt(req); 825 826 return crypto_aead_encrypt(req); 827 } 828 829 static int crypto_rfc4106_decrypt(struct aead_request *req) 830 { 831 if (req->assoclen != 16 && req->assoclen != 20) 832 return -EINVAL; 833 834 req = crypto_rfc4106_crypt(req); 835 836 return crypto_aead_decrypt(req); 837 } 838 839 static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm) 840 { 841 struct aead_instance *inst = aead_alg_instance(tfm); 842 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst); 843 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 844 struct crypto_aead *aead; 845 unsigned long align; 846 847 aead = crypto_spawn_aead(spawn); 848 if (IS_ERR(aead)) 849 return PTR_ERR(aead); 850 851 ctx->child = aead; 852 853 align = crypto_aead_alignmask(aead); 854 align &= ~(crypto_tfm_ctx_alignment() - 1); 855 crypto_aead_set_reqsize( 856 tfm, 857 sizeof(struct crypto_rfc4106_req_ctx) + 858 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 859 align + 24); 860 861 return 0; 862 } 863 864 static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm) 865 { 866 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); 867 868 crypto_free_aead(ctx->child); 869 } 870 871 static void crypto_rfc4106_free(struct aead_instance *inst) 872 { 873 crypto_drop_aead(aead_instance_ctx(inst)); 874 kfree(inst); 875 } 876 877 static int crypto_rfc4106_create(struct crypto_template *tmpl, 878 struct rtattr **tb) 879 { 880 struct crypto_attr_type *algt; 881 struct aead_instance *inst; 882 struct crypto_aead_spawn *spawn; 883 struct aead_alg *alg; 884 const char *ccm_name; 885 int err; 886 887 algt = crypto_get_attr_type(tb); 888 if (IS_ERR(algt)) 889 return PTR_ERR(algt); 890 891 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 892 return -EINVAL; 893 894 ccm_name = crypto_attr_alg_name(tb[1]); 895 if (IS_ERR(ccm_name)) 896 return PTR_ERR(ccm_name); 897 898 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 899 if (!inst) 900 return -ENOMEM; 901 902 spawn = aead_instance_ctx(inst); 903 crypto_set_aead_spawn(spawn, aead_crypto_instance(inst)); 904 err = crypto_grab_aead(spawn, ccm_name, 0, 905 crypto_requires_sync(algt->type, algt->mask)); 906 if (err) 907 goto out_free_inst; 908 909 alg = crypto_spawn_aead_alg(spawn); 910 911 err = -EINVAL; 912 913 /* Underlying IV size must be 12. */ 914 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) 915 goto out_drop_alg; 916 917 /* Not a stream cipher? */ 918 if (alg->base.cra_blocksize != 1) 919 goto out_drop_alg; 920 921 err = -ENAMETOOLONG; 922 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 923 "rfc4106(%s)", alg->base.cra_name) >= 924 CRYPTO_MAX_ALG_NAME || 925 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 926 "rfc4106(%s)", alg->base.cra_driver_name) >= 927 CRYPTO_MAX_ALG_NAME) 928 goto out_drop_alg; 929 930 inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC; 931 inst->alg.base.cra_priority = alg->base.cra_priority; 932 inst->alg.base.cra_blocksize = 1; 933 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 934 935 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); 936 937 inst->alg.ivsize = GCM_RFC4106_IV_SIZE; 938 inst->alg.chunksize = crypto_aead_alg_chunksize(alg); 939 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 940 941 inst->alg.init = crypto_rfc4106_init_tfm; 942 inst->alg.exit = crypto_rfc4106_exit_tfm; 943 944 inst->alg.setkey = crypto_rfc4106_setkey; 945 inst->alg.setauthsize = crypto_rfc4106_setauthsize; 946 inst->alg.encrypt = crypto_rfc4106_encrypt; 947 inst->alg.decrypt = crypto_rfc4106_decrypt; 948 949 inst->free = crypto_rfc4106_free; 950 951 err = aead_register_instance(tmpl, inst); 952 if (err) 953 goto out_drop_alg; 954 955 out: 956 return err; 957 958 out_drop_alg: 959 crypto_drop_aead(spawn); 960 out_free_inst: 961 kfree(inst); 962 goto out; 963 } 964 965 static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key, 966 unsigned int keylen) 967 { 968 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 969 struct crypto_aead *child = ctx->child; 970 int err; 971 972 if (keylen < 4) 973 return -EINVAL; 974 975 keylen -= 4; 976 memcpy(ctx->nonce, key + keylen, 4); 977 978 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 979 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & 980 CRYPTO_TFM_REQ_MASK); 981 err = crypto_aead_setkey(child, key, keylen); 982 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) & 983 CRYPTO_TFM_RES_MASK); 984 985 return err; 986 } 987 988 static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, 989 unsigned int authsize) 990 { 991 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); 992 993 if (authsize != 16) 994 return -EINVAL; 995 996 return crypto_aead_setauthsize(ctx->child, authsize); 997 } 998 999 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc) 1000 { 1001 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1002 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 1003 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req); 1004 struct aead_request *subreq = &rctx->subreq; 1005 unsigned int authsize = crypto_aead_authsize(aead); 1006 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), 1007 crypto_aead_alignmask(ctx->child) + 1); 1008 int err; 1009 1010 if (req->src != req->dst) { 1011 err = crypto_rfc4543_copy_src_to_dst(req, enc); 1012 if (err) 1013 return err; 1014 } 1015 1016 memcpy(iv, ctx->nonce, 4); 1017 memcpy(iv + 4, req->iv, 8); 1018 1019 aead_request_set_tfm(subreq, ctx->child); 1020 aead_request_set_callback(subreq, req->base.flags, 1021 req->base.complete, req->base.data); 1022 aead_request_set_crypt(subreq, req->src, req->dst, 1023 enc ? 0 : authsize, iv); 1024 aead_request_set_ad(subreq, req->assoclen + req->cryptlen - 1025 subreq->cryptlen); 1026 1027 return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); 1028 } 1029 1030 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) 1031 { 1032 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1033 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); 1034 unsigned int authsize = crypto_aead_authsize(aead); 1035 unsigned int nbytes = req->assoclen + req->cryptlen - 1036 (enc ? 0 : authsize); 1037 SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null); 1038 1039 skcipher_request_set_sync_tfm(nreq, ctx->null); 1040 skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL); 1041 skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL); 1042 1043 return crypto_skcipher_encrypt(nreq); 1044 } 1045 1046 static int crypto_rfc4543_encrypt(struct aead_request *req) 1047 { 1048 return crypto_rfc4543_crypt(req, true); 1049 } 1050 1051 static int crypto_rfc4543_decrypt(struct aead_request *req) 1052 { 1053 return crypto_rfc4543_crypt(req, false); 1054 } 1055 1056 static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) 1057 { 1058 struct aead_instance *inst = aead_alg_instance(tfm); 1059 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst); 1060 struct crypto_aead_spawn *spawn = &ictx->aead; 1061 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 1062 struct crypto_aead *aead; 1063 struct crypto_sync_skcipher *null; 1064 unsigned long align; 1065 int err = 0; 1066 1067 aead = crypto_spawn_aead(spawn); 1068 if (IS_ERR(aead)) 1069 return PTR_ERR(aead); 1070 1071 null = crypto_get_default_null_skcipher(); 1072 err = PTR_ERR(null); 1073 if (IS_ERR(null)) 1074 goto err_free_aead; 1075 1076 ctx->child = aead; 1077 ctx->null = null; 1078 1079 align = crypto_aead_alignmask(aead); 1080 align &= ~(crypto_tfm_ctx_alignment() - 1); 1081 crypto_aead_set_reqsize( 1082 tfm, 1083 sizeof(struct crypto_rfc4543_req_ctx) + 1084 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + 1085 align + GCM_AES_IV_SIZE); 1086 1087 return 0; 1088 1089 err_free_aead: 1090 crypto_free_aead(aead); 1091 return err; 1092 } 1093 1094 static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) 1095 { 1096 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); 1097 1098 crypto_free_aead(ctx->child); 1099 crypto_put_default_null_skcipher(); 1100 } 1101 1102 static void crypto_rfc4543_free(struct aead_instance *inst) 1103 { 1104 struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst); 1105 1106 crypto_drop_aead(&ctx->aead); 1107 1108 kfree(inst); 1109 } 1110 1111 static int crypto_rfc4543_create(struct crypto_template *tmpl, 1112 struct rtattr **tb) 1113 { 1114 struct crypto_attr_type *algt; 1115 struct aead_instance *inst; 1116 struct crypto_aead_spawn *spawn; 1117 struct aead_alg *alg; 1118 struct crypto_rfc4543_instance_ctx *ctx; 1119 const char *ccm_name; 1120 int err; 1121 1122 algt = crypto_get_attr_type(tb); 1123 if (IS_ERR(algt)) 1124 return PTR_ERR(algt); 1125 1126 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 1127 return -EINVAL; 1128 1129 ccm_name = crypto_attr_alg_name(tb[1]); 1130 if (IS_ERR(ccm_name)) 1131 return PTR_ERR(ccm_name); 1132 1133 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 1134 if (!inst) 1135 return -ENOMEM; 1136 1137 ctx = aead_instance_ctx(inst); 1138 spawn = &ctx->aead; 1139 crypto_set_aead_spawn(spawn, aead_crypto_instance(inst)); 1140 err = crypto_grab_aead(spawn, ccm_name, 0, 1141 crypto_requires_sync(algt->type, algt->mask)); 1142 if (err) 1143 goto out_free_inst; 1144 1145 alg = crypto_spawn_aead_alg(spawn); 1146 1147 err = -EINVAL; 1148 1149 /* Underlying IV size must be 12. */ 1150 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) 1151 goto out_drop_alg; 1152 1153 /* Not a stream cipher? */ 1154 if (alg->base.cra_blocksize != 1) 1155 goto out_drop_alg; 1156 1157 err = -ENAMETOOLONG; 1158 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 1159 "rfc4543(%s)", alg->base.cra_name) >= 1160 CRYPTO_MAX_ALG_NAME || 1161 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, 1162 "rfc4543(%s)", alg->base.cra_driver_name) >= 1163 CRYPTO_MAX_ALG_NAME) 1164 goto out_drop_alg; 1165 1166 inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC; 1167 inst->alg.base.cra_priority = alg->base.cra_priority; 1168 inst->alg.base.cra_blocksize = 1; 1169 inst->alg.base.cra_alignmask = alg->base.cra_alignmask; 1170 1171 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); 1172 1173 inst->alg.ivsize = GCM_RFC4543_IV_SIZE; 1174 inst->alg.chunksize = crypto_aead_alg_chunksize(alg); 1175 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 1176 1177 inst->alg.init = crypto_rfc4543_init_tfm; 1178 inst->alg.exit = crypto_rfc4543_exit_tfm; 1179 1180 inst->alg.setkey = crypto_rfc4543_setkey; 1181 inst->alg.setauthsize = crypto_rfc4543_setauthsize; 1182 inst->alg.encrypt = crypto_rfc4543_encrypt; 1183 inst->alg.decrypt = crypto_rfc4543_decrypt; 1184 1185 inst->free = crypto_rfc4543_free, 1186 1187 err = aead_register_instance(tmpl, inst); 1188 if (err) 1189 goto out_drop_alg; 1190 1191 out: 1192 return err; 1193 1194 out_drop_alg: 1195 crypto_drop_aead(spawn); 1196 out_free_inst: 1197 kfree(inst); 1198 goto out; 1199 } 1200 1201 static struct crypto_template crypto_gcm_tmpls[] = { 1202 { 1203 .name = "gcm_base", 1204 .create = crypto_gcm_base_create, 1205 .module = THIS_MODULE, 1206 }, { 1207 .name = "gcm", 1208 .create = crypto_gcm_create, 1209 .module = THIS_MODULE, 1210 }, { 1211 .name = "rfc4106", 1212 .create = crypto_rfc4106_create, 1213 .module = THIS_MODULE, 1214 }, { 1215 .name = "rfc4543", 1216 .create = crypto_rfc4543_create, 1217 .module = THIS_MODULE, 1218 }, 1219 }; 1220 1221 static int __init crypto_gcm_module_init(void) 1222 { 1223 int err; 1224 1225 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL); 1226 if (!gcm_zeroes) 1227 return -ENOMEM; 1228 1229 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); 1230 1231 err = crypto_register_templates(crypto_gcm_tmpls, 1232 ARRAY_SIZE(crypto_gcm_tmpls)); 1233 if (err) 1234 kfree(gcm_zeroes); 1235 1236 return err; 1237 } 1238 1239 static void __exit crypto_gcm_module_exit(void) 1240 { 1241 kfree(gcm_zeroes); 1242 crypto_unregister_templates(crypto_gcm_tmpls, 1243 ARRAY_SIZE(crypto_gcm_tmpls)); 1244 } 1245 1246 subsys_initcall(crypto_gcm_module_init); 1247 module_exit(crypto_gcm_module_exit); 1248 1249 MODULE_LICENSE("GPL"); 1250 MODULE_DESCRIPTION("Galois/Counter Mode"); 1251 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); 1252 MODULE_ALIAS_CRYPTO("gcm_base"); 1253 MODULE_ALIAS_CRYPTO("rfc4106"); 1254 MODULE_ALIAS_CRYPTO("rfc4543"); 1255 MODULE_ALIAS_CRYPTO("gcm"); 1256