1 /* 2 * Accelerated GHASH implementation with ARMv8 PMULL instructions. 3 * 4 * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation. 9 */ 10 11 #include <asm/neon.h> 12 #include <asm/simd.h> 13 #include <asm/unaligned.h> 14 #include <crypto/aes.h> 15 #include <crypto/algapi.h> 16 #include <crypto/b128ops.h> 17 #include <crypto/gf128mul.h> 18 #include <crypto/internal/aead.h> 19 #include <crypto/internal/hash.h> 20 #include <crypto/internal/simd.h> 21 #include <crypto/internal/skcipher.h> 22 #include <crypto/scatterwalk.h> 23 #include <linux/cpufeature.h> 24 #include <linux/crypto.h> 25 #include <linux/module.h> 26 27 MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions"); 28 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 29 MODULE_LICENSE("GPL v2"); 30 MODULE_ALIAS_CRYPTO("ghash"); 31 32 #define GHASH_BLOCK_SIZE 16 33 #define GHASH_DIGEST_SIZE 16 34 #define GCM_IV_SIZE 12 35 36 struct ghash_key { 37 u64 h[2]; 38 u64 h2[2]; 39 u64 h3[2]; 40 u64 h4[2]; 41 42 be128 k; 43 }; 44 45 struct ghash_desc_ctx { 46 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; 47 u8 buf[GHASH_BLOCK_SIZE]; 48 u32 count; 49 }; 50 51 struct gcm_aes_ctx { 52 struct crypto_aes_ctx aes_key; 53 struct ghash_key ghash_key; 54 }; 55 56 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 57 struct ghash_key const *k, 58 const char *head); 59 60 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 61 struct ghash_key const *k, 62 const char *head); 63 64 asmlinkage void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[], 65 const u8 src[], struct ghash_key const *k, 66 u8 ctr[], u32 const rk[], int rounds, 67 u8 ks[]); 68 69 asmlinkage void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[], 70 const u8 src[], struct ghash_key const *k, 71 u8 ctr[], u32 const rk[], int rounds); 72 73 asmlinkage void pmull_gcm_encrypt_block(u8 dst[], u8 const src[], 74 u32 const rk[], int rounds); 75 76 asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds); 77 78 static int ghash_init(struct shash_desc *desc) 79 { 80 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 81 82 *ctx = (struct ghash_desc_ctx){}; 83 return 0; 84 } 85 86 static void ghash_do_update(int blocks, u64 dg[], const char *src, 87 struct ghash_key *key, const char *head, 88 void (*simd_update)(int blocks, u64 dg[], 89 const char *src, 90 struct ghash_key const *k, 91 const char *head)) 92 { 93 if (likely(crypto_simd_usable())) { 94 kernel_neon_begin(); 95 simd_update(blocks, dg, src, key, head); 96 kernel_neon_end(); 97 } else { 98 be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) }; 99 100 do { 101 const u8 *in = src; 102 103 if (head) { 104 in = head; 105 blocks++; 106 head = NULL; 107 } else { 108 src += GHASH_BLOCK_SIZE; 109 } 110 111 crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE); 112 gf128mul_lle(&dst, &key->k); 113 } while (--blocks); 114 115 dg[0] = be64_to_cpu(dst.b); 116 dg[1] = be64_to_cpu(dst.a); 117 } 118 } 119 120 /* avoid hogging the CPU for too long */ 121 #define MAX_BLOCKS (SZ_64K / GHASH_BLOCK_SIZE) 122 123 static int __ghash_update(struct shash_desc *desc, const u8 *src, 124 unsigned int len, 125 void (*simd_update)(int blocks, u64 dg[], 126 const char *src, 127 struct ghash_key const *k, 128 const char *head)) 129 { 130 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 131 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 132 133 ctx->count += len; 134 135 if ((partial + len) >= GHASH_BLOCK_SIZE) { 136 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 137 int blocks; 138 139 if (partial) { 140 int p = GHASH_BLOCK_SIZE - partial; 141 142 memcpy(ctx->buf + partial, src, p); 143 src += p; 144 len -= p; 145 } 146 147 blocks = len / GHASH_BLOCK_SIZE; 148 len %= GHASH_BLOCK_SIZE; 149 150 do { 151 int chunk = min(blocks, MAX_BLOCKS); 152 153 ghash_do_update(chunk, ctx->digest, src, key, 154 partial ? ctx->buf : NULL, 155 simd_update); 156 157 blocks -= chunk; 158 src += chunk * GHASH_BLOCK_SIZE; 159 partial = 0; 160 } while (unlikely(blocks > 0)); 161 } 162 if (len) 163 memcpy(ctx->buf + partial, src, len); 164 return 0; 165 } 166 167 static int ghash_update_p8(struct shash_desc *desc, const u8 *src, 168 unsigned int len) 169 { 170 return __ghash_update(desc, src, len, pmull_ghash_update_p8); 171 } 172 173 static int ghash_update_p64(struct shash_desc *desc, const u8 *src, 174 unsigned int len) 175 { 176 return __ghash_update(desc, src, len, pmull_ghash_update_p64); 177 } 178 179 static int ghash_final_p8(struct shash_desc *desc, u8 *dst) 180 { 181 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 182 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 183 184 if (partial) { 185 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 186 187 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); 188 189 ghash_do_update(1, ctx->digest, ctx->buf, key, NULL, 190 pmull_ghash_update_p8); 191 } 192 put_unaligned_be64(ctx->digest[1], dst); 193 put_unaligned_be64(ctx->digest[0], dst + 8); 194 195 *ctx = (struct ghash_desc_ctx){}; 196 return 0; 197 } 198 199 static int ghash_final_p64(struct shash_desc *desc, u8 *dst) 200 { 201 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 202 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 203 204 if (partial) { 205 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 206 207 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); 208 209 ghash_do_update(1, ctx->digest, ctx->buf, key, NULL, 210 pmull_ghash_update_p64); 211 } 212 put_unaligned_be64(ctx->digest[1], dst); 213 put_unaligned_be64(ctx->digest[0], dst + 8); 214 215 *ctx = (struct ghash_desc_ctx){}; 216 return 0; 217 } 218 219 static void ghash_reflect(u64 h[], const be128 *k) 220 { 221 u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0; 222 223 h[0] = (be64_to_cpu(k->b) << 1) | carry; 224 h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63); 225 226 if (carry) 227 h[1] ^= 0xc200000000000000UL; 228 } 229 230 static int __ghash_setkey(struct ghash_key *key, 231 const u8 *inkey, unsigned int keylen) 232 { 233 be128 h; 234 235 /* needed for the fallback */ 236 memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); 237 238 ghash_reflect(key->h, &key->k); 239 240 h = key->k; 241 gf128mul_lle(&h, &key->k); 242 ghash_reflect(key->h2, &h); 243 244 gf128mul_lle(&h, &key->k); 245 ghash_reflect(key->h3, &h); 246 247 gf128mul_lle(&h, &key->k); 248 ghash_reflect(key->h4, &h); 249 250 return 0; 251 } 252 253 static int ghash_setkey(struct crypto_shash *tfm, 254 const u8 *inkey, unsigned int keylen) 255 { 256 struct ghash_key *key = crypto_shash_ctx(tfm); 257 258 if (keylen != GHASH_BLOCK_SIZE) { 259 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 260 return -EINVAL; 261 } 262 263 return __ghash_setkey(key, inkey, keylen); 264 } 265 266 static struct shash_alg ghash_alg[] = {{ 267 .base.cra_name = "ghash", 268 .base.cra_driver_name = "ghash-neon", 269 .base.cra_priority = 100, 270 .base.cra_blocksize = GHASH_BLOCK_SIZE, 271 .base.cra_ctxsize = sizeof(struct ghash_key), 272 .base.cra_module = THIS_MODULE, 273 274 .digestsize = GHASH_DIGEST_SIZE, 275 .init = ghash_init, 276 .update = ghash_update_p8, 277 .final = ghash_final_p8, 278 .setkey = ghash_setkey, 279 .descsize = sizeof(struct ghash_desc_ctx), 280 }, { 281 .base.cra_name = "ghash", 282 .base.cra_driver_name = "ghash-ce", 283 .base.cra_priority = 200, 284 .base.cra_blocksize = GHASH_BLOCK_SIZE, 285 .base.cra_ctxsize = sizeof(struct ghash_key), 286 .base.cra_module = THIS_MODULE, 287 288 .digestsize = GHASH_DIGEST_SIZE, 289 .init = ghash_init, 290 .update = ghash_update_p64, 291 .final = ghash_final_p64, 292 .setkey = ghash_setkey, 293 .descsize = sizeof(struct ghash_desc_ctx), 294 }}; 295 296 static int num_rounds(struct crypto_aes_ctx *ctx) 297 { 298 /* 299 * # of rounds specified by AES: 300 * 128 bit key 10 rounds 301 * 192 bit key 12 rounds 302 * 256 bit key 14 rounds 303 * => n byte key => 6 + (n/4) rounds 304 */ 305 return 6 + ctx->key_length / 4; 306 } 307 308 static int gcm_setkey(struct crypto_aead *tfm, const u8 *inkey, 309 unsigned int keylen) 310 { 311 struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm); 312 u8 key[GHASH_BLOCK_SIZE]; 313 int ret; 314 315 ret = crypto_aes_expand_key(&ctx->aes_key, inkey, keylen); 316 if (ret) { 317 tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 318 return -EINVAL; 319 } 320 321 __aes_arm64_encrypt(ctx->aes_key.key_enc, key, (u8[AES_BLOCK_SIZE]){}, 322 num_rounds(&ctx->aes_key)); 323 324 return __ghash_setkey(&ctx->ghash_key, key, sizeof(be128)); 325 } 326 327 static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 328 { 329 switch (authsize) { 330 case 4: 331 case 8: 332 case 12 ... 16: 333 break; 334 default: 335 return -EINVAL; 336 } 337 return 0; 338 } 339 340 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], 341 int *buf_count, struct gcm_aes_ctx *ctx) 342 { 343 if (*buf_count > 0) { 344 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count); 345 346 memcpy(&buf[*buf_count], src, buf_added); 347 348 *buf_count += buf_added; 349 src += buf_added; 350 count -= buf_added; 351 } 352 353 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) { 354 int blocks = count / GHASH_BLOCK_SIZE; 355 356 ghash_do_update(blocks, dg, src, &ctx->ghash_key, 357 *buf_count ? buf : NULL, 358 pmull_ghash_update_p64); 359 360 src += blocks * GHASH_BLOCK_SIZE; 361 count %= GHASH_BLOCK_SIZE; 362 *buf_count = 0; 363 } 364 365 if (count > 0) { 366 memcpy(buf, src, count); 367 *buf_count = count; 368 } 369 } 370 371 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[]) 372 { 373 struct crypto_aead *aead = crypto_aead_reqtfm(req); 374 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 375 u8 buf[GHASH_BLOCK_SIZE]; 376 struct scatter_walk walk; 377 u32 len = req->assoclen; 378 int buf_count = 0; 379 380 scatterwalk_start(&walk, req->src); 381 382 do { 383 u32 n = scatterwalk_clamp(&walk, len); 384 u8 *p; 385 386 if (!n) { 387 scatterwalk_start(&walk, sg_next(walk.sg)); 388 n = scatterwalk_clamp(&walk, len); 389 } 390 p = scatterwalk_map(&walk); 391 392 gcm_update_mac(dg, p, n, buf, &buf_count, ctx); 393 len -= n; 394 395 scatterwalk_unmap(p); 396 scatterwalk_advance(&walk, n); 397 scatterwalk_done(&walk, 0, len); 398 } while (len); 399 400 if (buf_count) { 401 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); 402 ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL, 403 pmull_ghash_update_p64); 404 } 405 } 406 407 static void gcm_final(struct aead_request *req, struct gcm_aes_ctx *ctx, 408 u64 dg[], u8 tag[], int cryptlen) 409 { 410 u8 mac[AES_BLOCK_SIZE]; 411 u128 lengths; 412 413 lengths.a = cpu_to_be64(req->assoclen * 8); 414 lengths.b = cpu_to_be64(cryptlen * 8); 415 416 ghash_do_update(1, dg, (void *)&lengths, &ctx->ghash_key, NULL, 417 pmull_ghash_update_p64); 418 419 put_unaligned_be64(dg[1], mac); 420 put_unaligned_be64(dg[0], mac + 8); 421 422 crypto_xor(tag, mac, AES_BLOCK_SIZE); 423 } 424 425 static int gcm_encrypt(struct aead_request *req) 426 { 427 struct crypto_aead *aead = crypto_aead_reqtfm(req); 428 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 429 struct skcipher_walk walk; 430 u8 iv[AES_BLOCK_SIZE]; 431 u8 ks[2 * AES_BLOCK_SIZE]; 432 u8 tag[AES_BLOCK_SIZE]; 433 u64 dg[2] = {}; 434 int nrounds = num_rounds(&ctx->aes_key); 435 int err; 436 437 if (req->assoclen) 438 gcm_calculate_auth_mac(req, dg); 439 440 memcpy(iv, req->iv, GCM_IV_SIZE); 441 put_unaligned_be32(1, iv + GCM_IV_SIZE); 442 443 err = skcipher_walk_aead_encrypt(&walk, req, false); 444 445 if (likely(crypto_simd_usable() && walk.total >= 2 * AES_BLOCK_SIZE)) { 446 u32 const *rk = NULL; 447 448 kernel_neon_begin(); 449 pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc, nrounds); 450 put_unaligned_be32(2, iv + GCM_IV_SIZE); 451 pmull_gcm_encrypt_block(ks, iv, NULL, nrounds); 452 put_unaligned_be32(3, iv + GCM_IV_SIZE); 453 pmull_gcm_encrypt_block(ks + AES_BLOCK_SIZE, iv, NULL, nrounds); 454 put_unaligned_be32(4, iv + GCM_IV_SIZE); 455 456 do { 457 int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2; 458 459 if (rk) 460 kernel_neon_begin(); 461 462 pmull_gcm_encrypt(blocks, dg, walk.dst.virt.addr, 463 walk.src.virt.addr, &ctx->ghash_key, 464 iv, rk, nrounds, ks); 465 kernel_neon_end(); 466 467 err = skcipher_walk_done(&walk, 468 walk.nbytes % (2 * AES_BLOCK_SIZE)); 469 470 rk = ctx->aes_key.key_enc; 471 } while (walk.nbytes >= 2 * AES_BLOCK_SIZE); 472 } else { 473 __aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, nrounds); 474 put_unaligned_be32(2, iv + GCM_IV_SIZE); 475 476 while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) { 477 const int blocks = 478 walk.nbytes / (2 * AES_BLOCK_SIZE) * 2; 479 u8 *dst = walk.dst.virt.addr; 480 u8 *src = walk.src.virt.addr; 481 int remaining = blocks; 482 483 do { 484 __aes_arm64_encrypt(ctx->aes_key.key_enc, 485 ks, iv, nrounds); 486 crypto_xor_cpy(dst, src, ks, AES_BLOCK_SIZE); 487 crypto_inc(iv, AES_BLOCK_SIZE); 488 489 dst += AES_BLOCK_SIZE; 490 src += AES_BLOCK_SIZE; 491 } while (--remaining > 0); 492 493 ghash_do_update(blocks, dg, 494 walk.dst.virt.addr, &ctx->ghash_key, 495 NULL, pmull_ghash_update_p64); 496 497 err = skcipher_walk_done(&walk, 498 walk.nbytes % (2 * AES_BLOCK_SIZE)); 499 } 500 if (walk.nbytes) { 501 __aes_arm64_encrypt(ctx->aes_key.key_enc, ks, iv, 502 nrounds); 503 if (walk.nbytes > AES_BLOCK_SIZE) { 504 crypto_inc(iv, AES_BLOCK_SIZE); 505 __aes_arm64_encrypt(ctx->aes_key.key_enc, 506 ks + AES_BLOCK_SIZE, iv, 507 nrounds); 508 } 509 } 510 } 511 512 /* handle the tail */ 513 if (walk.nbytes) { 514 u8 buf[GHASH_BLOCK_SIZE]; 515 unsigned int nbytes = walk.nbytes; 516 u8 *dst = walk.dst.virt.addr; 517 u8 *head = NULL; 518 519 crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, ks, 520 walk.nbytes); 521 522 if (walk.nbytes > GHASH_BLOCK_SIZE) { 523 head = dst; 524 dst += GHASH_BLOCK_SIZE; 525 nbytes %= GHASH_BLOCK_SIZE; 526 } 527 528 memcpy(buf, dst, nbytes); 529 memset(buf + nbytes, 0, GHASH_BLOCK_SIZE - nbytes); 530 ghash_do_update(!!nbytes, dg, buf, &ctx->ghash_key, head, 531 pmull_ghash_update_p64); 532 533 err = skcipher_walk_done(&walk, 0); 534 } 535 536 if (err) 537 return err; 538 539 gcm_final(req, ctx, dg, tag, req->cryptlen); 540 541 /* copy authtag to end of dst */ 542 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen, 543 crypto_aead_authsize(aead), 1); 544 545 return 0; 546 } 547 548 static int gcm_decrypt(struct aead_request *req) 549 { 550 struct crypto_aead *aead = crypto_aead_reqtfm(req); 551 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 552 unsigned int authsize = crypto_aead_authsize(aead); 553 struct skcipher_walk walk; 554 u8 iv[2 * AES_BLOCK_SIZE]; 555 u8 tag[AES_BLOCK_SIZE]; 556 u8 buf[2 * GHASH_BLOCK_SIZE]; 557 u64 dg[2] = {}; 558 int nrounds = num_rounds(&ctx->aes_key); 559 int err; 560 561 if (req->assoclen) 562 gcm_calculate_auth_mac(req, dg); 563 564 memcpy(iv, req->iv, GCM_IV_SIZE); 565 put_unaligned_be32(1, iv + GCM_IV_SIZE); 566 567 err = skcipher_walk_aead_decrypt(&walk, req, false); 568 569 if (likely(crypto_simd_usable() && walk.total >= 2 * AES_BLOCK_SIZE)) { 570 u32 const *rk = NULL; 571 572 kernel_neon_begin(); 573 pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc, nrounds); 574 put_unaligned_be32(2, iv + GCM_IV_SIZE); 575 576 do { 577 int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2; 578 int rem = walk.total - blocks * AES_BLOCK_SIZE; 579 580 if (rk) 581 kernel_neon_begin(); 582 583 pmull_gcm_decrypt(blocks, dg, walk.dst.virt.addr, 584 walk.src.virt.addr, &ctx->ghash_key, 585 iv, rk, nrounds); 586 587 /* check if this is the final iteration of the loop */ 588 if (rem < (2 * AES_BLOCK_SIZE)) { 589 u8 *iv2 = iv + AES_BLOCK_SIZE; 590 591 if (rem > AES_BLOCK_SIZE) { 592 memcpy(iv2, iv, AES_BLOCK_SIZE); 593 crypto_inc(iv2, AES_BLOCK_SIZE); 594 } 595 596 pmull_gcm_encrypt_block(iv, iv, NULL, nrounds); 597 598 if (rem > AES_BLOCK_SIZE) 599 pmull_gcm_encrypt_block(iv2, iv2, NULL, 600 nrounds); 601 } 602 603 kernel_neon_end(); 604 605 err = skcipher_walk_done(&walk, 606 walk.nbytes % (2 * AES_BLOCK_SIZE)); 607 608 rk = ctx->aes_key.key_enc; 609 } while (walk.nbytes >= 2 * AES_BLOCK_SIZE); 610 } else { 611 __aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, nrounds); 612 put_unaligned_be32(2, iv + GCM_IV_SIZE); 613 614 while (walk.nbytes >= (2 * AES_BLOCK_SIZE)) { 615 int blocks = walk.nbytes / (2 * AES_BLOCK_SIZE) * 2; 616 u8 *dst = walk.dst.virt.addr; 617 u8 *src = walk.src.virt.addr; 618 619 ghash_do_update(blocks, dg, walk.src.virt.addr, 620 &ctx->ghash_key, NULL, 621 pmull_ghash_update_p64); 622 623 do { 624 __aes_arm64_encrypt(ctx->aes_key.key_enc, 625 buf, iv, nrounds); 626 crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE); 627 crypto_inc(iv, AES_BLOCK_SIZE); 628 629 dst += AES_BLOCK_SIZE; 630 src += AES_BLOCK_SIZE; 631 } while (--blocks > 0); 632 633 err = skcipher_walk_done(&walk, 634 walk.nbytes % (2 * AES_BLOCK_SIZE)); 635 } 636 if (walk.nbytes) { 637 if (walk.nbytes > AES_BLOCK_SIZE) { 638 u8 *iv2 = iv + AES_BLOCK_SIZE; 639 640 memcpy(iv2, iv, AES_BLOCK_SIZE); 641 crypto_inc(iv2, AES_BLOCK_SIZE); 642 643 __aes_arm64_encrypt(ctx->aes_key.key_enc, iv2, 644 iv2, nrounds); 645 } 646 __aes_arm64_encrypt(ctx->aes_key.key_enc, iv, iv, 647 nrounds); 648 } 649 } 650 651 /* handle the tail */ 652 if (walk.nbytes) { 653 const u8 *src = walk.src.virt.addr; 654 const u8 *head = NULL; 655 unsigned int nbytes = walk.nbytes; 656 657 if (walk.nbytes > GHASH_BLOCK_SIZE) { 658 head = src; 659 src += GHASH_BLOCK_SIZE; 660 nbytes %= GHASH_BLOCK_SIZE; 661 } 662 663 memcpy(buf, src, nbytes); 664 memset(buf + nbytes, 0, GHASH_BLOCK_SIZE - nbytes); 665 ghash_do_update(!!nbytes, dg, buf, &ctx->ghash_key, head, 666 pmull_ghash_update_p64); 667 668 crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, iv, 669 walk.nbytes); 670 671 err = skcipher_walk_done(&walk, 0); 672 } 673 674 if (err) 675 return err; 676 677 gcm_final(req, ctx, dg, tag, req->cryptlen - authsize); 678 679 /* compare calculated auth tag with the stored one */ 680 scatterwalk_map_and_copy(buf, req->src, 681 req->assoclen + req->cryptlen - authsize, 682 authsize, 0); 683 684 if (crypto_memneq(tag, buf, authsize)) 685 return -EBADMSG; 686 return 0; 687 } 688 689 static struct aead_alg gcm_aes_alg = { 690 .ivsize = GCM_IV_SIZE, 691 .chunksize = 2 * AES_BLOCK_SIZE, 692 .maxauthsize = AES_BLOCK_SIZE, 693 .setkey = gcm_setkey, 694 .setauthsize = gcm_setauthsize, 695 .encrypt = gcm_encrypt, 696 .decrypt = gcm_decrypt, 697 698 .base.cra_name = "gcm(aes)", 699 .base.cra_driver_name = "gcm-aes-ce", 700 .base.cra_priority = 300, 701 .base.cra_blocksize = 1, 702 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx), 703 .base.cra_module = THIS_MODULE, 704 }; 705 706 static int __init ghash_ce_mod_init(void) 707 { 708 int ret; 709 710 if (!cpu_have_named_feature(ASIMD)) 711 return -ENODEV; 712 713 if (cpu_have_named_feature(PMULL)) 714 ret = crypto_register_shashes(ghash_alg, 715 ARRAY_SIZE(ghash_alg)); 716 else 717 /* only register the first array element */ 718 ret = crypto_register_shash(ghash_alg); 719 720 if (ret) 721 return ret; 722 723 if (cpu_have_named_feature(PMULL)) { 724 ret = crypto_register_aead(&gcm_aes_alg); 725 if (ret) 726 crypto_unregister_shashes(ghash_alg, 727 ARRAY_SIZE(ghash_alg)); 728 } 729 return ret; 730 } 731 732 static void __exit ghash_ce_mod_exit(void) 733 { 734 if (cpu_have_named_feature(PMULL)) 735 crypto_unregister_shashes(ghash_alg, ARRAY_SIZE(ghash_alg)); 736 else 737 crypto_unregister_shash(ghash_alg); 738 crypto_unregister_aead(&gcm_aes_alg); 739 } 740 741 static const struct cpu_feature ghash_cpu_feature[] = { 742 { cpu_feature(PMULL) }, { } 743 }; 744 MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature); 745 746 module_init(ghash_ce_mod_init); 747 module_exit(ghash_ce_mod_exit); 748