1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. 4 * 5 * Copyright (C) 2015 - 2018 Linaro Ltd. 6 * Copyright (C) 2023 Google LLC. 7 */ 8 9 #include <asm/hwcap.h> 10 #include <asm/neon.h> 11 #include <asm/simd.h> 12 #include <asm/unaligned.h> 13 #include <crypto/aes.h> 14 #include <crypto/gcm.h> 15 #include <crypto/b128ops.h> 16 #include <crypto/cryptd.h> 17 #include <crypto/internal/aead.h> 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/simd.h> 20 #include <crypto/internal/skcipher.h> 21 #include <crypto/gf128mul.h> 22 #include <crypto/scatterwalk.h> 23 #include <linux/cpufeature.h> 24 #include <linux/crypto.h> 25 #include <linux/jump_label.h> 26 #include <linux/module.h> 27 28 MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions"); 29 MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); 30 MODULE_LICENSE("GPL"); 31 MODULE_ALIAS_CRYPTO("ghash"); 32 MODULE_ALIAS_CRYPTO("gcm(aes)"); 33 MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); 34 35 #define GHASH_BLOCK_SIZE 16 36 #define GHASH_DIGEST_SIZE 16 37 38 #define RFC4106_NONCE_SIZE 4 39 40 struct ghash_key { 41 be128 k; 42 u64 h[][2]; 43 }; 44 45 struct gcm_key { 46 u64 h[4][2]; 47 u32 rk[AES_MAX_KEYLENGTH_U32]; 48 int rounds; 49 u8 nonce[]; // for RFC4106 nonce 50 }; 51 52 struct ghash_desc_ctx { 53 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; 54 u8 buf[GHASH_BLOCK_SIZE]; 55 u32 count; 56 }; 57 58 struct ghash_async_ctx { 59 struct cryptd_ahash *cryptd_tfm; 60 }; 61 62 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 63 u64 const h[][2], const char *head); 64 65 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 66 u64 const h[][2], const char *head); 67 68 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64); 69 70 static int ghash_init(struct shash_desc *desc) 71 { 72 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 73 74 *ctx = (struct ghash_desc_ctx){}; 75 return 0; 76 } 77 78 static void ghash_do_update(int blocks, u64 dg[], const char *src, 79 struct ghash_key *key, const char *head) 80 { 81 if (likely(crypto_simd_usable())) { 82 kernel_neon_begin(); 83 if (static_branch_likely(&use_p64)) 84 pmull_ghash_update_p64(blocks, dg, src, key->h, head); 85 else 86 pmull_ghash_update_p8(blocks, dg, src, key->h, head); 87 kernel_neon_end(); 88 } else { 89 be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) }; 90 91 do { 92 const u8 *in = src; 93 94 if (head) { 95 in = head; 96 blocks++; 97 head = NULL; 98 } else { 99 src += GHASH_BLOCK_SIZE; 100 } 101 102 crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE); 103 gf128mul_lle(&dst, &key->k); 104 } while (--blocks); 105 106 dg[0] = be64_to_cpu(dst.b); 107 dg[1] = be64_to_cpu(dst.a); 108 } 109 } 110 111 static int ghash_update(struct shash_desc *desc, const u8 *src, 112 unsigned int len) 113 { 114 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 115 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 116 117 ctx->count += len; 118 119 if ((partial + len) >= GHASH_BLOCK_SIZE) { 120 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 121 int blocks; 122 123 if (partial) { 124 int p = GHASH_BLOCK_SIZE - partial; 125 126 memcpy(ctx->buf + partial, src, p); 127 src += p; 128 len -= p; 129 } 130 131 blocks = len / GHASH_BLOCK_SIZE; 132 len %= GHASH_BLOCK_SIZE; 133 134 ghash_do_update(blocks, ctx->digest, src, key, 135 partial ? ctx->buf : NULL); 136 src += blocks * GHASH_BLOCK_SIZE; 137 partial = 0; 138 } 139 if (len) 140 memcpy(ctx->buf + partial, src, len); 141 return 0; 142 } 143 144 static int ghash_final(struct shash_desc *desc, u8 *dst) 145 { 146 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 147 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 148 149 if (partial) { 150 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 151 152 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); 153 ghash_do_update(1, ctx->digest, ctx->buf, key, NULL); 154 } 155 put_unaligned_be64(ctx->digest[1], dst); 156 put_unaligned_be64(ctx->digest[0], dst + 8); 157 158 *ctx = (struct ghash_desc_ctx){}; 159 return 0; 160 } 161 162 static void ghash_reflect(u64 h[], const be128 *k) 163 { 164 u64 carry = be64_to_cpu(k->a) >> 63; 165 166 h[0] = (be64_to_cpu(k->b) << 1) | carry; 167 h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63); 168 169 if (carry) 170 h[1] ^= 0xc200000000000000UL; 171 } 172 173 static int ghash_setkey(struct crypto_shash *tfm, 174 const u8 *inkey, unsigned int keylen) 175 { 176 struct ghash_key *key = crypto_shash_ctx(tfm); 177 178 if (keylen != GHASH_BLOCK_SIZE) 179 return -EINVAL; 180 181 /* needed for the fallback */ 182 memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); 183 ghash_reflect(key->h[0], &key->k); 184 185 if (static_branch_likely(&use_p64)) { 186 be128 h = key->k; 187 188 gf128mul_lle(&h, &key->k); 189 ghash_reflect(key->h[1], &h); 190 191 gf128mul_lle(&h, &key->k); 192 ghash_reflect(key->h[2], &h); 193 194 gf128mul_lle(&h, &key->k); 195 ghash_reflect(key->h[3], &h); 196 } 197 return 0; 198 } 199 200 static struct shash_alg ghash_alg = { 201 .digestsize = GHASH_DIGEST_SIZE, 202 .init = ghash_init, 203 .update = ghash_update, 204 .final = ghash_final, 205 .setkey = ghash_setkey, 206 .descsize = sizeof(struct ghash_desc_ctx), 207 208 .base.cra_name = "ghash", 209 .base.cra_driver_name = "ghash-ce-sync", 210 .base.cra_priority = 300 - 1, 211 .base.cra_blocksize = GHASH_BLOCK_SIZE, 212 .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), 213 .base.cra_module = THIS_MODULE, 214 }; 215 216 static int ghash_async_init(struct ahash_request *req) 217 { 218 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 219 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 220 struct ahash_request *cryptd_req = ahash_request_ctx(req); 221 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 222 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 223 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 224 225 desc->tfm = child; 226 return crypto_shash_init(desc); 227 } 228 229 static int ghash_async_update(struct ahash_request *req) 230 { 231 struct ahash_request *cryptd_req = ahash_request_ctx(req); 232 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 233 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 234 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 235 236 if (!crypto_simd_usable() || 237 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 238 memcpy(cryptd_req, req, sizeof(*req)); 239 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 240 return crypto_ahash_update(cryptd_req); 241 } else { 242 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 243 return shash_ahash_update(req, desc); 244 } 245 } 246 247 static int ghash_async_final(struct ahash_request *req) 248 { 249 struct ahash_request *cryptd_req = ahash_request_ctx(req); 250 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 251 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 252 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 253 254 if (!crypto_simd_usable() || 255 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 256 memcpy(cryptd_req, req, sizeof(*req)); 257 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 258 return crypto_ahash_final(cryptd_req); 259 } else { 260 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 261 return crypto_shash_final(desc, req->result); 262 } 263 } 264 265 static int ghash_async_digest(struct ahash_request *req) 266 { 267 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 268 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 269 struct ahash_request *cryptd_req = ahash_request_ctx(req); 270 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 271 272 if (!crypto_simd_usable() || 273 (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 274 memcpy(cryptd_req, req, sizeof(*req)); 275 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 276 return crypto_ahash_digest(cryptd_req); 277 } else { 278 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 279 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 280 281 desc->tfm = child; 282 return shash_ahash_digest(req, desc); 283 } 284 } 285 286 static int ghash_async_import(struct ahash_request *req, const void *in) 287 { 288 struct ahash_request *cryptd_req = ahash_request_ctx(req); 289 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 290 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 291 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 292 293 desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm); 294 295 return crypto_shash_import(desc, in); 296 } 297 298 static int ghash_async_export(struct ahash_request *req, void *out) 299 { 300 struct ahash_request *cryptd_req = ahash_request_ctx(req); 301 struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 302 303 return crypto_shash_export(desc, out); 304 } 305 306 static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, 307 unsigned int keylen) 308 { 309 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 310 struct crypto_ahash *child = &ctx->cryptd_tfm->base; 311 312 crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 313 crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) 314 & CRYPTO_TFM_REQ_MASK); 315 return crypto_ahash_setkey(child, key, keylen); 316 } 317 318 static int ghash_async_init_tfm(struct crypto_tfm *tfm) 319 { 320 struct cryptd_ahash *cryptd_tfm; 321 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 322 323 cryptd_tfm = cryptd_alloc_ahash("ghash-ce-sync", 0, 0); 324 if (IS_ERR(cryptd_tfm)) 325 return PTR_ERR(cryptd_tfm); 326 ctx->cryptd_tfm = cryptd_tfm; 327 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 328 sizeof(struct ahash_request) + 329 crypto_ahash_reqsize(&cryptd_tfm->base)); 330 331 return 0; 332 } 333 334 static void ghash_async_exit_tfm(struct crypto_tfm *tfm) 335 { 336 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 337 338 cryptd_free_ahash(ctx->cryptd_tfm); 339 } 340 341 static struct ahash_alg ghash_async_alg = { 342 .init = ghash_async_init, 343 .update = ghash_async_update, 344 .final = ghash_async_final, 345 .setkey = ghash_async_setkey, 346 .digest = ghash_async_digest, 347 .import = ghash_async_import, 348 .export = ghash_async_export, 349 .halg.digestsize = GHASH_DIGEST_SIZE, 350 .halg.statesize = sizeof(struct ghash_desc_ctx), 351 .halg.base = { 352 .cra_name = "ghash", 353 .cra_driver_name = "ghash-ce", 354 .cra_priority = 300, 355 .cra_flags = CRYPTO_ALG_ASYNC, 356 .cra_blocksize = GHASH_BLOCK_SIZE, 357 .cra_ctxsize = sizeof(struct ghash_async_ctx), 358 .cra_module = THIS_MODULE, 359 .cra_init = ghash_async_init_tfm, 360 .cra_exit = ghash_async_exit_tfm, 361 }, 362 }; 363 364 365 void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src, 366 struct gcm_key const *k, char *dst, 367 const char *iv, int rounds, u32 counter); 368 369 void pmull_gcm_enc_final(int blocks, u64 dg[], char *tag, 370 struct gcm_key const *k, char *head, 371 const char *iv, int rounds, u32 counter); 372 373 void pmull_gcm_decrypt(int bytes, u64 dg[], const char *src, 374 struct gcm_key const *k, char *dst, 375 const char *iv, int rounds, u32 counter); 376 377 int pmull_gcm_dec_final(int bytes, u64 dg[], char *tag, 378 struct gcm_key const *k, char *head, 379 const char *iv, int rounds, u32 counter, 380 const char *otag, int authsize); 381 382 static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey, 383 unsigned int keylen) 384 { 385 struct gcm_key *ctx = crypto_aead_ctx(tfm); 386 struct crypto_aes_ctx aes_ctx; 387 be128 h, k; 388 int ret; 389 390 ret = aes_expandkey(&aes_ctx, inkey, keylen); 391 if (ret) 392 return -EINVAL; 393 394 aes_encrypt(&aes_ctx, (u8 *)&k, (u8[AES_BLOCK_SIZE]){}); 395 396 memcpy(ctx->rk, aes_ctx.key_enc, sizeof(ctx->rk)); 397 ctx->rounds = 6 + keylen / 4; 398 399 memzero_explicit(&aes_ctx, sizeof(aes_ctx)); 400 401 ghash_reflect(ctx->h[0], &k); 402 403 h = k; 404 gf128mul_lle(&h, &k); 405 ghash_reflect(ctx->h[1], &h); 406 407 gf128mul_lle(&h, &k); 408 ghash_reflect(ctx->h[2], &h); 409 410 gf128mul_lle(&h, &k); 411 ghash_reflect(ctx->h[3], &h); 412 413 return 0; 414 } 415 416 static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 417 { 418 return crypto_gcm_check_authsize(authsize); 419 } 420 421 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], 422 int *buf_count, struct gcm_key *ctx) 423 { 424 if (*buf_count > 0) { 425 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count); 426 427 memcpy(&buf[*buf_count], src, buf_added); 428 429 *buf_count += buf_added; 430 src += buf_added; 431 count -= buf_added; 432 } 433 434 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) { 435 int blocks = count / GHASH_BLOCK_SIZE; 436 437 pmull_ghash_update_p64(blocks, dg, src, ctx->h, 438 *buf_count ? buf : NULL); 439 440 src += blocks * GHASH_BLOCK_SIZE; 441 count %= GHASH_BLOCK_SIZE; 442 *buf_count = 0; 443 } 444 445 if (count > 0) { 446 memcpy(buf, src, count); 447 *buf_count = count; 448 } 449 } 450 451 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) 452 { 453 struct crypto_aead *aead = crypto_aead_reqtfm(req); 454 struct gcm_key *ctx = crypto_aead_ctx(aead); 455 u8 buf[GHASH_BLOCK_SIZE]; 456 struct scatter_walk walk; 457 int buf_count = 0; 458 459 scatterwalk_start(&walk, req->src); 460 461 do { 462 u32 n = scatterwalk_clamp(&walk, len); 463 u8 *p; 464 465 if (!n) { 466 scatterwalk_start(&walk, sg_next(walk.sg)); 467 n = scatterwalk_clamp(&walk, len); 468 } 469 470 p = scatterwalk_map(&walk); 471 gcm_update_mac(dg, p, n, buf, &buf_count, ctx); 472 scatterwalk_unmap(p); 473 474 if (unlikely(len / SZ_4K > (len - n) / SZ_4K)) { 475 kernel_neon_end(); 476 kernel_neon_begin(); 477 } 478 479 len -= n; 480 scatterwalk_advance(&walk, n); 481 scatterwalk_done(&walk, 0, len); 482 } while (len); 483 484 if (buf_count) { 485 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); 486 pmull_ghash_update_p64(1, dg, buf, ctx->h, NULL); 487 } 488 } 489 490 static int gcm_encrypt(struct aead_request *req, const u8 *iv, u32 assoclen) 491 { 492 struct crypto_aead *aead = crypto_aead_reqtfm(req); 493 struct gcm_key *ctx = crypto_aead_ctx(aead); 494 struct skcipher_walk walk; 495 u8 buf[AES_BLOCK_SIZE]; 496 u32 counter = 2; 497 u64 dg[2] = {}; 498 be128 lengths; 499 const u8 *src; 500 u8 *tag, *dst; 501 int tail, err; 502 503 if (WARN_ON_ONCE(!may_use_simd())) 504 return -EBUSY; 505 506 err = skcipher_walk_aead_encrypt(&walk, req, false); 507 508 kernel_neon_begin(); 509 510 if (assoclen) 511 gcm_calculate_auth_mac(req, dg, assoclen); 512 513 src = walk.src.virt.addr; 514 dst = walk.dst.virt.addr; 515 516 while (walk.nbytes >= AES_BLOCK_SIZE) { 517 int nblocks = walk.nbytes / AES_BLOCK_SIZE; 518 519 pmull_gcm_encrypt(nblocks, dg, src, ctx, dst, iv, 520 ctx->rounds, counter); 521 counter += nblocks; 522 523 if (walk.nbytes == walk.total) { 524 src += nblocks * AES_BLOCK_SIZE; 525 dst += nblocks * AES_BLOCK_SIZE; 526 break; 527 } 528 529 kernel_neon_end(); 530 531 err = skcipher_walk_done(&walk, 532 walk.nbytes % AES_BLOCK_SIZE); 533 if (err) 534 return err; 535 536 src = walk.src.virt.addr; 537 dst = walk.dst.virt.addr; 538 539 kernel_neon_begin(); 540 } 541 542 543 lengths.a = cpu_to_be64(assoclen * 8); 544 lengths.b = cpu_to_be64(req->cryptlen * 8); 545 546 tag = (u8 *)&lengths; 547 tail = walk.nbytes % AES_BLOCK_SIZE; 548 549 /* 550 * Bounce via a buffer unless we are encrypting in place and src/dst 551 * are not pointing to the start of the walk buffer. In that case, we 552 * can do a NEON load/xor/store sequence in place as long as we move 553 * the plain/ciphertext and keystream to the start of the register. If 554 * not, do a memcpy() to the end of the buffer so we can reuse the same 555 * logic. 556 */ 557 if (unlikely(tail && (tail == walk.nbytes || src != dst))) 558 src = memcpy(buf + sizeof(buf) - tail, src, tail); 559 560 pmull_gcm_enc_final(tail, dg, tag, ctx, (u8 *)src, iv, 561 ctx->rounds, counter); 562 kernel_neon_end(); 563 564 if (unlikely(tail && src != dst)) 565 memcpy(dst, src, tail); 566 567 if (walk.nbytes) { 568 err = skcipher_walk_done(&walk, 0); 569 if (err) 570 return err; 571 } 572 573 /* copy authtag to end of dst */ 574 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen, 575 crypto_aead_authsize(aead), 1); 576 577 return 0; 578 } 579 580 static int gcm_decrypt(struct aead_request *req, const u8 *iv, u32 assoclen) 581 { 582 struct crypto_aead *aead = crypto_aead_reqtfm(req); 583 struct gcm_key *ctx = crypto_aead_ctx(aead); 584 int authsize = crypto_aead_authsize(aead); 585 struct skcipher_walk walk; 586 u8 otag[AES_BLOCK_SIZE]; 587 u8 buf[AES_BLOCK_SIZE]; 588 u32 counter = 2; 589 u64 dg[2] = {}; 590 be128 lengths; 591 const u8 *src; 592 u8 *tag, *dst; 593 int tail, err, ret; 594 595 if (WARN_ON_ONCE(!may_use_simd())) 596 return -EBUSY; 597 598 scatterwalk_map_and_copy(otag, req->src, 599 req->assoclen + req->cryptlen - authsize, 600 authsize, 0); 601 602 err = skcipher_walk_aead_decrypt(&walk, req, false); 603 604 kernel_neon_begin(); 605 606 if (assoclen) 607 gcm_calculate_auth_mac(req, dg, assoclen); 608 609 src = walk.src.virt.addr; 610 dst = walk.dst.virt.addr; 611 612 while (walk.nbytes >= AES_BLOCK_SIZE) { 613 int nblocks = walk.nbytes / AES_BLOCK_SIZE; 614 615 pmull_gcm_decrypt(nblocks, dg, src, ctx, dst, iv, 616 ctx->rounds, counter); 617 counter += nblocks; 618 619 if (walk.nbytes == walk.total) { 620 src += nblocks * AES_BLOCK_SIZE; 621 dst += nblocks * AES_BLOCK_SIZE; 622 break; 623 } 624 625 kernel_neon_end(); 626 627 err = skcipher_walk_done(&walk, 628 walk.nbytes % AES_BLOCK_SIZE); 629 if (err) 630 return err; 631 632 src = walk.src.virt.addr; 633 dst = walk.dst.virt.addr; 634 635 kernel_neon_begin(); 636 } 637 638 lengths.a = cpu_to_be64(assoclen * 8); 639 lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8); 640 641 tag = (u8 *)&lengths; 642 tail = walk.nbytes % AES_BLOCK_SIZE; 643 644 if (unlikely(tail && (tail == walk.nbytes || src != dst))) 645 src = memcpy(buf + sizeof(buf) - tail, src, tail); 646 647 ret = pmull_gcm_dec_final(tail, dg, tag, ctx, (u8 *)src, iv, 648 ctx->rounds, counter, otag, authsize); 649 kernel_neon_end(); 650 651 if (unlikely(tail && src != dst)) 652 memcpy(dst, src, tail); 653 654 if (walk.nbytes) { 655 err = skcipher_walk_done(&walk, 0); 656 if (err) 657 return err; 658 } 659 660 return ret ? -EBADMSG : 0; 661 } 662 663 static int gcm_aes_encrypt(struct aead_request *req) 664 { 665 return gcm_encrypt(req, req->iv, req->assoclen); 666 } 667 668 static int gcm_aes_decrypt(struct aead_request *req) 669 { 670 return gcm_decrypt(req, req->iv, req->assoclen); 671 } 672 673 static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, 674 unsigned int keylen) 675 { 676 struct gcm_key *ctx = crypto_aead_ctx(tfm); 677 int err; 678 679 keylen -= RFC4106_NONCE_SIZE; 680 err = gcm_aes_setkey(tfm, inkey, keylen); 681 if (err) 682 return err; 683 684 memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); 685 return 0; 686 } 687 688 static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 689 { 690 return crypto_rfc4106_check_authsize(authsize); 691 } 692 693 static int rfc4106_encrypt(struct aead_request *req) 694 { 695 struct crypto_aead *aead = crypto_aead_reqtfm(req); 696 struct gcm_key *ctx = crypto_aead_ctx(aead); 697 u8 iv[GCM_AES_IV_SIZE]; 698 699 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 700 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 701 702 return crypto_ipsec_check_assoclen(req->assoclen) ?: 703 gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 704 } 705 706 static int rfc4106_decrypt(struct aead_request *req) 707 { 708 struct crypto_aead *aead = crypto_aead_reqtfm(req); 709 struct gcm_key *ctx = crypto_aead_ctx(aead); 710 u8 iv[GCM_AES_IV_SIZE]; 711 712 memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); 713 memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); 714 715 return crypto_ipsec_check_assoclen(req->assoclen) ?: 716 gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE); 717 } 718 719 static struct aead_alg gcm_aes_algs[] = {{ 720 .ivsize = GCM_AES_IV_SIZE, 721 .chunksize = AES_BLOCK_SIZE, 722 .maxauthsize = AES_BLOCK_SIZE, 723 .setkey = gcm_aes_setkey, 724 .setauthsize = gcm_aes_setauthsize, 725 .encrypt = gcm_aes_encrypt, 726 .decrypt = gcm_aes_decrypt, 727 728 .base.cra_name = "gcm(aes)", 729 .base.cra_driver_name = "gcm-aes-ce", 730 .base.cra_priority = 400, 731 .base.cra_blocksize = 1, 732 .base.cra_ctxsize = sizeof(struct gcm_key), 733 .base.cra_module = THIS_MODULE, 734 }, { 735 .ivsize = GCM_RFC4106_IV_SIZE, 736 .chunksize = AES_BLOCK_SIZE, 737 .maxauthsize = AES_BLOCK_SIZE, 738 .setkey = rfc4106_setkey, 739 .setauthsize = rfc4106_setauthsize, 740 .encrypt = rfc4106_encrypt, 741 .decrypt = rfc4106_decrypt, 742 743 .base.cra_name = "rfc4106(gcm(aes))", 744 .base.cra_driver_name = "rfc4106-gcm-aes-ce", 745 .base.cra_priority = 400, 746 .base.cra_blocksize = 1, 747 .base.cra_ctxsize = sizeof(struct gcm_key) + RFC4106_NONCE_SIZE, 748 .base.cra_module = THIS_MODULE, 749 }}; 750 751 static int __init ghash_ce_mod_init(void) 752 { 753 int err; 754 755 if (!(elf_hwcap & HWCAP_NEON)) 756 return -ENODEV; 757 758 if (elf_hwcap2 & HWCAP2_PMULL) { 759 err = crypto_register_aeads(gcm_aes_algs, 760 ARRAY_SIZE(gcm_aes_algs)); 761 if (err) 762 return err; 763 ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]); 764 static_branch_enable(&use_p64); 765 } 766 767 err = crypto_register_shash(&ghash_alg); 768 if (err) 769 goto err_aead; 770 err = crypto_register_ahash(&ghash_async_alg); 771 if (err) 772 goto err_shash; 773 774 return 0; 775 776 err_shash: 777 crypto_unregister_shash(&ghash_alg); 778 err_aead: 779 if (elf_hwcap2 & HWCAP2_PMULL) 780 crypto_unregister_aeads(gcm_aes_algs, 781 ARRAY_SIZE(gcm_aes_algs)); 782 return err; 783 } 784 785 static void __exit ghash_ce_mod_exit(void) 786 { 787 crypto_unregister_ahash(&ghash_async_alg); 788 crypto_unregister_shash(&ghash_alg); 789 if (elf_hwcap2 & HWCAP2_PMULL) 790 crypto_unregister_aeads(gcm_aes_algs, 791 ARRAY_SIZE(gcm_aes_algs)); 792 } 793 794 module_init(ghash_ce_mod_init); 795 module_exit(ghash_ce_mod_exit); 796