1 /* 2 * Accelerated GHASH implementation with ARMv8 PMULL instructions. 3 * 4 * Copyright (C) 2014 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation. 9 */ 10 11 #include <asm/neon.h> 12 #include <asm/simd.h> 13 #include <asm/unaligned.h> 14 #include <crypto/aes.h> 15 #include <crypto/algapi.h> 16 #include <crypto/b128ops.h> 17 #include <crypto/gf128mul.h> 18 #include <crypto/internal/aead.h> 19 #include <crypto/internal/hash.h> 20 #include <crypto/internal/skcipher.h> 21 #include <crypto/scatterwalk.h> 22 #include <linux/cpufeature.h> 23 #include <linux/crypto.h> 24 #include <linux/module.h> 25 26 MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions"); 27 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 28 MODULE_LICENSE("GPL v2"); 29 MODULE_ALIAS_CRYPTO("ghash"); 30 31 #define GHASH_BLOCK_SIZE 16 32 #define GHASH_DIGEST_SIZE 16 33 #define GCM_IV_SIZE 12 34 35 struct ghash_key { 36 u64 a; 37 u64 b; 38 be128 k; 39 }; 40 41 struct ghash_desc_ctx { 42 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; 43 u8 buf[GHASH_BLOCK_SIZE]; 44 u32 count; 45 }; 46 47 struct gcm_aes_ctx { 48 struct crypto_aes_ctx aes_key; 49 struct ghash_key ghash_key; 50 }; 51 52 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 53 struct ghash_key const *k, 54 const char *head); 55 56 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 57 struct ghash_key const *k, 58 const char *head); 59 60 static void (*pmull_ghash_update)(int blocks, u64 dg[], const char *src, 61 struct ghash_key const *k, 62 const char *head); 63 64 asmlinkage void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[], 65 const u8 src[], struct ghash_key const *k, 66 u8 ctr[], u32 const rk[], int rounds, 67 u8 ks[]); 68 69 asmlinkage void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[], 70 const u8 src[], struct ghash_key const *k, 71 u8 ctr[], u32 const rk[], int rounds); 72 73 asmlinkage void pmull_gcm_encrypt_block(u8 dst[], u8 const src[], 74 u32 const rk[], int rounds); 75 76 asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds); 77 78 static int ghash_init(struct shash_desc *desc) 79 { 80 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 81 82 *ctx = (struct ghash_desc_ctx){}; 83 return 0; 84 } 85 86 static void ghash_do_update(int blocks, u64 dg[], const char *src, 87 struct ghash_key *key, const char *head) 88 { 89 if (likely(may_use_simd())) { 90 kernel_neon_begin(); 91 pmull_ghash_update(blocks, dg, src, key, head); 92 kernel_neon_end(); 93 } else { 94 be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) }; 95 96 do { 97 const u8 *in = src; 98 99 if (head) { 100 in = head; 101 blocks++; 102 head = NULL; 103 } else { 104 src += GHASH_BLOCK_SIZE; 105 } 106 107 crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE); 108 gf128mul_lle(&dst, &key->k); 109 } while (--blocks); 110 111 dg[0] = be64_to_cpu(dst.b); 112 dg[1] = be64_to_cpu(dst.a); 113 } 114 } 115 116 static int ghash_update(struct shash_desc *desc, const u8 *src, 117 unsigned int len) 118 { 119 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 120 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 121 122 ctx->count += len; 123 124 if ((partial + len) >= GHASH_BLOCK_SIZE) { 125 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 126 int blocks; 127 128 if (partial) { 129 int p = GHASH_BLOCK_SIZE - partial; 130 131 memcpy(ctx->buf + partial, src, p); 132 src += p; 133 len -= p; 134 } 135 136 blocks = len / GHASH_BLOCK_SIZE; 137 len %= GHASH_BLOCK_SIZE; 138 139 ghash_do_update(blocks, ctx->digest, src, key, 140 partial ? ctx->buf : NULL); 141 142 src += blocks * GHASH_BLOCK_SIZE; 143 partial = 0; 144 } 145 if (len) 146 memcpy(ctx->buf + partial, src, len); 147 return 0; 148 } 149 150 static int ghash_final(struct shash_desc *desc, u8 *dst) 151 { 152 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 153 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 154 155 if (partial) { 156 struct ghash_key *key = crypto_shash_ctx(desc->tfm); 157 158 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); 159 160 ghash_do_update(1, ctx->digest, ctx->buf, key, NULL); 161 } 162 put_unaligned_be64(ctx->digest[1], dst); 163 put_unaligned_be64(ctx->digest[0], dst + 8); 164 165 *ctx = (struct ghash_desc_ctx){}; 166 return 0; 167 } 168 169 static int __ghash_setkey(struct ghash_key *key, 170 const u8 *inkey, unsigned int keylen) 171 { 172 u64 a, b; 173 174 /* needed for the fallback */ 175 memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); 176 177 /* perform multiplication by 'x' in GF(2^128) */ 178 b = get_unaligned_be64(inkey); 179 a = get_unaligned_be64(inkey + 8); 180 181 key->a = (a << 1) | (b >> 63); 182 key->b = (b << 1) | (a >> 63); 183 184 if (b >> 63) 185 key->b ^= 0xc200000000000000UL; 186 187 return 0; 188 } 189 190 static int ghash_setkey(struct crypto_shash *tfm, 191 const u8 *inkey, unsigned int keylen) 192 { 193 struct ghash_key *key = crypto_shash_ctx(tfm); 194 195 if (keylen != GHASH_BLOCK_SIZE) { 196 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 197 return -EINVAL; 198 } 199 200 return __ghash_setkey(key, inkey, keylen); 201 } 202 203 static struct shash_alg ghash_alg = { 204 .base.cra_name = "ghash", 205 .base.cra_driver_name = "ghash-ce", 206 .base.cra_priority = 200, 207 .base.cra_blocksize = GHASH_BLOCK_SIZE, 208 .base.cra_ctxsize = sizeof(struct ghash_key), 209 .base.cra_module = THIS_MODULE, 210 211 .digestsize = GHASH_DIGEST_SIZE, 212 .init = ghash_init, 213 .update = ghash_update, 214 .final = ghash_final, 215 .setkey = ghash_setkey, 216 .descsize = sizeof(struct ghash_desc_ctx), 217 }; 218 219 static int num_rounds(struct crypto_aes_ctx *ctx) 220 { 221 /* 222 * # of rounds specified by AES: 223 * 128 bit key 10 rounds 224 * 192 bit key 12 rounds 225 * 256 bit key 14 rounds 226 * => n byte key => 6 + (n/4) rounds 227 */ 228 return 6 + ctx->key_length / 4; 229 } 230 231 static int gcm_setkey(struct crypto_aead *tfm, const u8 *inkey, 232 unsigned int keylen) 233 { 234 struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm); 235 u8 key[GHASH_BLOCK_SIZE]; 236 int ret; 237 238 ret = crypto_aes_expand_key(&ctx->aes_key, inkey, keylen); 239 if (ret) { 240 tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 241 return -EINVAL; 242 } 243 244 __aes_arm64_encrypt(ctx->aes_key.key_enc, key, (u8[AES_BLOCK_SIZE]){}, 245 num_rounds(&ctx->aes_key)); 246 247 return __ghash_setkey(&ctx->ghash_key, key, sizeof(key)); 248 } 249 250 static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 251 { 252 switch (authsize) { 253 case 4: 254 case 8: 255 case 12 ... 16: 256 break; 257 default: 258 return -EINVAL; 259 } 260 return 0; 261 } 262 263 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], 264 int *buf_count, struct gcm_aes_ctx *ctx) 265 { 266 if (*buf_count > 0) { 267 int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count); 268 269 memcpy(&buf[*buf_count], src, buf_added); 270 271 *buf_count += buf_added; 272 src += buf_added; 273 count -= buf_added; 274 } 275 276 if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) { 277 int blocks = count / GHASH_BLOCK_SIZE; 278 279 ghash_do_update(blocks, dg, src, &ctx->ghash_key, 280 *buf_count ? buf : NULL); 281 282 src += blocks * GHASH_BLOCK_SIZE; 283 count %= GHASH_BLOCK_SIZE; 284 *buf_count = 0; 285 } 286 287 if (count > 0) { 288 memcpy(buf, src, count); 289 *buf_count = count; 290 } 291 } 292 293 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[]) 294 { 295 struct crypto_aead *aead = crypto_aead_reqtfm(req); 296 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 297 u8 buf[GHASH_BLOCK_SIZE]; 298 struct scatter_walk walk; 299 u32 len = req->assoclen; 300 int buf_count = 0; 301 302 scatterwalk_start(&walk, req->src); 303 304 do { 305 u32 n = scatterwalk_clamp(&walk, len); 306 u8 *p; 307 308 if (!n) { 309 scatterwalk_start(&walk, sg_next(walk.sg)); 310 n = scatterwalk_clamp(&walk, len); 311 } 312 p = scatterwalk_map(&walk); 313 314 gcm_update_mac(dg, p, n, buf, &buf_count, ctx); 315 len -= n; 316 317 scatterwalk_unmap(p); 318 scatterwalk_advance(&walk, n); 319 scatterwalk_done(&walk, 0, len); 320 } while (len); 321 322 if (buf_count) { 323 memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); 324 ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL); 325 } 326 } 327 328 static void gcm_final(struct aead_request *req, struct gcm_aes_ctx *ctx, 329 u64 dg[], u8 tag[], int cryptlen) 330 { 331 u8 mac[AES_BLOCK_SIZE]; 332 u128 lengths; 333 334 lengths.a = cpu_to_be64(req->assoclen * 8); 335 lengths.b = cpu_to_be64(cryptlen * 8); 336 337 ghash_do_update(1, dg, (void *)&lengths, &ctx->ghash_key, NULL); 338 339 put_unaligned_be64(dg[1], mac); 340 put_unaligned_be64(dg[0], mac + 8); 341 342 crypto_xor(tag, mac, AES_BLOCK_SIZE); 343 } 344 345 static int gcm_encrypt(struct aead_request *req) 346 { 347 struct crypto_aead *aead = crypto_aead_reqtfm(req); 348 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 349 struct skcipher_walk walk; 350 u8 iv[AES_BLOCK_SIZE]; 351 u8 ks[AES_BLOCK_SIZE]; 352 u8 tag[AES_BLOCK_SIZE]; 353 u64 dg[2] = {}; 354 int err; 355 356 if (req->assoclen) 357 gcm_calculate_auth_mac(req, dg); 358 359 memcpy(iv, req->iv, GCM_IV_SIZE); 360 put_unaligned_be32(1, iv + GCM_IV_SIZE); 361 362 if (likely(may_use_simd())) { 363 kernel_neon_begin(); 364 365 pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc, 366 num_rounds(&ctx->aes_key)); 367 put_unaligned_be32(2, iv + GCM_IV_SIZE); 368 pmull_gcm_encrypt_block(ks, iv, NULL, 369 num_rounds(&ctx->aes_key)); 370 put_unaligned_be32(3, iv + GCM_IV_SIZE); 371 kernel_neon_end(); 372 373 err = skcipher_walk_aead_encrypt(&walk, req, false); 374 375 while (walk.nbytes >= AES_BLOCK_SIZE) { 376 int blocks = walk.nbytes / AES_BLOCK_SIZE; 377 378 kernel_neon_begin(); 379 pmull_gcm_encrypt(blocks, dg, walk.dst.virt.addr, 380 walk.src.virt.addr, &ctx->ghash_key, 381 iv, ctx->aes_key.key_enc, 382 num_rounds(&ctx->aes_key), ks); 383 kernel_neon_end(); 384 385 err = skcipher_walk_done(&walk, 386 walk.nbytes % AES_BLOCK_SIZE); 387 } 388 } else { 389 __aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, 390 num_rounds(&ctx->aes_key)); 391 put_unaligned_be32(2, iv + GCM_IV_SIZE); 392 393 err = skcipher_walk_aead_encrypt(&walk, req, false); 394 395 while (walk.nbytes >= AES_BLOCK_SIZE) { 396 int blocks = walk.nbytes / AES_BLOCK_SIZE; 397 u8 *dst = walk.dst.virt.addr; 398 u8 *src = walk.src.virt.addr; 399 400 do { 401 __aes_arm64_encrypt(ctx->aes_key.key_enc, 402 ks, iv, 403 num_rounds(&ctx->aes_key)); 404 crypto_xor_cpy(dst, src, ks, AES_BLOCK_SIZE); 405 crypto_inc(iv, AES_BLOCK_SIZE); 406 407 dst += AES_BLOCK_SIZE; 408 src += AES_BLOCK_SIZE; 409 } while (--blocks > 0); 410 411 ghash_do_update(walk.nbytes / AES_BLOCK_SIZE, dg, 412 walk.dst.virt.addr, &ctx->ghash_key, 413 NULL); 414 415 err = skcipher_walk_done(&walk, 416 walk.nbytes % AES_BLOCK_SIZE); 417 } 418 if (walk.nbytes) 419 __aes_arm64_encrypt(ctx->aes_key.key_enc, ks, iv, 420 num_rounds(&ctx->aes_key)); 421 } 422 423 /* handle the tail */ 424 if (walk.nbytes) { 425 u8 buf[GHASH_BLOCK_SIZE]; 426 427 crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, ks, 428 walk.nbytes); 429 430 memcpy(buf, walk.dst.virt.addr, walk.nbytes); 431 memset(buf + walk.nbytes, 0, GHASH_BLOCK_SIZE - walk.nbytes); 432 ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL); 433 434 err = skcipher_walk_done(&walk, 0); 435 } 436 437 if (err) 438 return err; 439 440 gcm_final(req, ctx, dg, tag, req->cryptlen); 441 442 /* copy authtag to end of dst */ 443 scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen, 444 crypto_aead_authsize(aead), 1); 445 446 return 0; 447 } 448 449 static int gcm_decrypt(struct aead_request *req) 450 { 451 struct crypto_aead *aead = crypto_aead_reqtfm(req); 452 struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead); 453 unsigned int authsize = crypto_aead_authsize(aead); 454 struct skcipher_walk walk; 455 u8 iv[AES_BLOCK_SIZE]; 456 u8 tag[AES_BLOCK_SIZE]; 457 u8 buf[GHASH_BLOCK_SIZE]; 458 u64 dg[2] = {}; 459 int err; 460 461 if (req->assoclen) 462 gcm_calculate_auth_mac(req, dg); 463 464 memcpy(iv, req->iv, GCM_IV_SIZE); 465 put_unaligned_be32(1, iv + GCM_IV_SIZE); 466 467 if (likely(may_use_simd())) { 468 kernel_neon_begin(); 469 470 pmull_gcm_encrypt_block(tag, iv, ctx->aes_key.key_enc, 471 num_rounds(&ctx->aes_key)); 472 put_unaligned_be32(2, iv + GCM_IV_SIZE); 473 kernel_neon_end(); 474 475 err = skcipher_walk_aead_decrypt(&walk, req, false); 476 477 while (walk.nbytes >= AES_BLOCK_SIZE) { 478 int blocks = walk.nbytes / AES_BLOCK_SIZE; 479 480 kernel_neon_begin(); 481 pmull_gcm_decrypt(blocks, dg, walk.dst.virt.addr, 482 walk.src.virt.addr, &ctx->ghash_key, 483 iv, ctx->aes_key.key_enc, 484 num_rounds(&ctx->aes_key)); 485 kernel_neon_end(); 486 487 err = skcipher_walk_done(&walk, 488 walk.nbytes % AES_BLOCK_SIZE); 489 } 490 if (walk.nbytes) 491 pmull_gcm_encrypt_block(iv, iv, NULL, 492 num_rounds(&ctx->aes_key)); 493 } else { 494 __aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, 495 num_rounds(&ctx->aes_key)); 496 put_unaligned_be32(2, iv + GCM_IV_SIZE); 497 498 err = skcipher_walk_aead_decrypt(&walk, req, false); 499 500 while (walk.nbytes >= AES_BLOCK_SIZE) { 501 int blocks = walk.nbytes / AES_BLOCK_SIZE; 502 u8 *dst = walk.dst.virt.addr; 503 u8 *src = walk.src.virt.addr; 504 505 ghash_do_update(blocks, dg, walk.src.virt.addr, 506 &ctx->ghash_key, NULL); 507 508 do { 509 __aes_arm64_encrypt(ctx->aes_key.key_enc, 510 buf, iv, 511 num_rounds(&ctx->aes_key)); 512 crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE); 513 crypto_inc(iv, AES_BLOCK_SIZE); 514 515 dst += AES_BLOCK_SIZE; 516 src += AES_BLOCK_SIZE; 517 } while (--blocks > 0); 518 519 err = skcipher_walk_done(&walk, 520 walk.nbytes % AES_BLOCK_SIZE); 521 } 522 if (walk.nbytes) 523 __aes_arm64_encrypt(ctx->aes_key.key_enc, iv, iv, 524 num_rounds(&ctx->aes_key)); 525 } 526 527 /* handle the tail */ 528 if (walk.nbytes) { 529 memcpy(buf, walk.src.virt.addr, walk.nbytes); 530 memset(buf + walk.nbytes, 0, GHASH_BLOCK_SIZE - walk.nbytes); 531 ghash_do_update(1, dg, buf, &ctx->ghash_key, NULL); 532 533 crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, iv, 534 walk.nbytes); 535 536 err = skcipher_walk_done(&walk, 0); 537 } 538 539 if (err) 540 return err; 541 542 gcm_final(req, ctx, dg, tag, req->cryptlen - authsize); 543 544 /* compare calculated auth tag with the stored one */ 545 scatterwalk_map_and_copy(buf, req->src, 546 req->assoclen + req->cryptlen - authsize, 547 authsize, 0); 548 549 if (crypto_memneq(tag, buf, authsize)) 550 return -EBADMSG; 551 return 0; 552 } 553 554 static struct aead_alg gcm_aes_alg = { 555 .ivsize = GCM_IV_SIZE, 556 .chunksize = AES_BLOCK_SIZE, 557 .maxauthsize = AES_BLOCK_SIZE, 558 .setkey = gcm_setkey, 559 .setauthsize = gcm_setauthsize, 560 .encrypt = gcm_encrypt, 561 .decrypt = gcm_decrypt, 562 563 .base.cra_name = "gcm(aes)", 564 .base.cra_driver_name = "gcm-aes-ce", 565 .base.cra_priority = 300, 566 .base.cra_blocksize = 1, 567 .base.cra_ctxsize = sizeof(struct gcm_aes_ctx), 568 .base.cra_module = THIS_MODULE, 569 }; 570 571 static int __init ghash_ce_mod_init(void) 572 { 573 int ret; 574 575 if (!(elf_hwcap & HWCAP_ASIMD)) 576 return -ENODEV; 577 578 if (elf_hwcap & HWCAP_PMULL) 579 pmull_ghash_update = pmull_ghash_update_p64; 580 581 else 582 pmull_ghash_update = pmull_ghash_update_p8; 583 584 ret = crypto_register_shash(&ghash_alg); 585 if (ret) 586 return ret; 587 588 if (elf_hwcap & HWCAP_PMULL) { 589 ret = crypto_register_aead(&gcm_aes_alg); 590 if (ret) 591 crypto_unregister_shash(&ghash_alg); 592 } 593 return ret; 594 } 595 596 static void __exit ghash_ce_mod_exit(void) 597 { 598 crypto_unregister_shash(&ghash_alg); 599 crypto_unregister_aead(&gcm_aes_alg); 600 } 601 602 static const struct cpu_feature ghash_cpu_feature[] = { 603 { cpu_feature(PMULL) }, { } 604 }; 605 MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature); 606 607 module_init(ghash_ce_mod_init); 608 module_exit(ghash_ce_mod_exit); 609