1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES 4 * 5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/neon.h> 9 #include <asm/hwcap.h> 10 #include <asm/simd.h> 11 #include <crypto/aes.h> 12 #include <crypto/ctr.h> 13 #include <crypto/sha.h> 14 #include <crypto/internal/hash.h> 15 #include <crypto/internal/simd.h> 16 #include <crypto/internal/skcipher.h> 17 #include <crypto/scatterwalk.h> 18 #include <linux/module.h> 19 #include <linux/cpufeature.h> 20 #include <crypto/xts.h> 21 22 #include "aes-ce-setkey.h" 23 24 #ifdef USE_V8_CRYPTO_EXTENSIONS 25 #define MODE "ce" 26 #define PRIO 300 27 #define aes_expandkey ce_aes_expandkey 28 #define aes_ecb_encrypt ce_aes_ecb_encrypt 29 #define aes_ecb_decrypt ce_aes_ecb_decrypt 30 #define aes_cbc_encrypt ce_aes_cbc_encrypt 31 #define aes_cbc_decrypt ce_aes_cbc_decrypt 32 #define aes_cbc_cts_encrypt ce_aes_cbc_cts_encrypt 33 #define aes_cbc_cts_decrypt ce_aes_cbc_cts_decrypt 34 #define aes_essiv_cbc_encrypt ce_aes_essiv_cbc_encrypt 35 #define aes_essiv_cbc_decrypt ce_aes_essiv_cbc_decrypt 36 #define aes_ctr_encrypt ce_aes_ctr_encrypt 37 #define aes_xts_encrypt ce_aes_xts_encrypt 38 #define aes_xts_decrypt ce_aes_xts_decrypt 39 #define aes_mac_update ce_aes_mac_update 40 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 41 #else 42 #define MODE "neon" 43 #define PRIO 200 44 #define aes_ecb_encrypt neon_aes_ecb_encrypt 45 #define aes_ecb_decrypt neon_aes_ecb_decrypt 46 #define aes_cbc_encrypt neon_aes_cbc_encrypt 47 #define aes_cbc_decrypt neon_aes_cbc_decrypt 48 #define aes_cbc_cts_encrypt neon_aes_cbc_cts_encrypt 49 #define aes_cbc_cts_decrypt neon_aes_cbc_cts_decrypt 50 #define aes_essiv_cbc_encrypt neon_aes_essiv_cbc_encrypt 51 #define aes_essiv_cbc_decrypt neon_aes_essiv_cbc_decrypt 52 #define aes_ctr_encrypt neon_aes_ctr_encrypt 53 #define aes_xts_encrypt neon_aes_xts_encrypt 54 #define aes_xts_decrypt neon_aes_xts_decrypt 55 #define aes_mac_update neon_aes_mac_update 56 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); 57 #endif 58 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !defined(CONFIG_CRYPTO_AES_ARM64_BS) 59 MODULE_ALIAS_CRYPTO("ecb(aes)"); 60 MODULE_ALIAS_CRYPTO("cbc(aes)"); 61 MODULE_ALIAS_CRYPTO("ctr(aes)"); 62 MODULE_ALIAS_CRYPTO("xts(aes)"); 63 #endif 64 MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); 65 MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)"); 66 MODULE_ALIAS_CRYPTO("cmac(aes)"); 67 MODULE_ALIAS_CRYPTO("xcbc(aes)"); 68 MODULE_ALIAS_CRYPTO("cbcmac(aes)"); 69 70 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 71 MODULE_LICENSE("GPL v2"); 72 73 /* defined in aes-modes.S */ 74 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], 75 int rounds, int blocks); 76 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], 77 int rounds, int blocks); 78 79 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], 80 int rounds, int blocks, u8 iv[]); 81 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], 82 int rounds, int blocks, u8 iv[]); 83 84 asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], 85 int rounds, int bytes, u8 const iv[]); 86 asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], 87 int rounds, int bytes, u8 const iv[]); 88 89 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], 90 int rounds, int blocks, u8 ctr[]); 91 92 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], 93 int rounds, int bytes, u32 const rk2[], u8 iv[], 94 int first); 95 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], 96 int rounds, int bytes, u32 const rk2[], u8 iv[], 97 int first); 98 99 asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], 100 int rounds, int blocks, u8 iv[], 101 u32 const rk2[]); 102 asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], 103 int rounds, int blocks, u8 iv[], 104 u32 const rk2[]); 105 106 asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds, 107 int blocks, u8 dg[], int enc_before, 108 int enc_after); 109 110 struct crypto_aes_xts_ctx { 111 struct crypto_aes_ctx key1; 112 struct crypto_aes_ctx __aligned(8) key2; 113 }; 114 115 struct crypto_aes_essiv_cbc_ctx { 116 struct crypto_aes_ctx key1; 117 struct crypto_aes_ctx __aligned(8) key2; 118 struct crypto_shash *hash; 119 }; 120 121 struct mac_tfm_ctx { 122 struct crypto_aes_ctx key; 123 u8 __aligned(8) consts[]; 124 }; 125 126 struct mac_desc_ctx { 127 unsigned int len; 128 u8 dg[AES_BLOCK_SIZE]; 129 }; 130 131 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 132 unsigned int key_len) 133 { 134 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 135 int ret; 136 137 ret = aes_expandkey(ctx, in_key, key_len); 138 if (ret) 139 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 140 141 return ret; 142 } 143 144 static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm, 145 const u8 *in_key, unsigned int key_len) 146 { 147 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 148 int ret; 149 150 ret = xts_verify_key(tfm, in_key, key_len); 151 if (ret) 152 return ret; 153 154 ret = aes_expandkey(&ctx->key1, in_key, key_len / 2); 155 if (!ret) 156 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2], 157 key_len / 2); 158 if (!ret) 159 return 0; 160 161 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 162 return -EINVAL; 163 } 164 165 static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm, 166 const u8 *in_key, 167 unsigned int key_len) 168 { 169 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 170 SHASH_DESC_ON_STACK(desc, ctx->hash); 171 u8 digest[SHA256_DIGEST_SIZE]; 172 int ret; 173 174 ret = aes_expandkey(&ctx->key1, in_key, key_len); 175 if (ret) 176 goto out; 177 178 desc->tfm = ctx->hash; 179 crypto_shash_digest(desc, in_key, key_len, digest); 180 181 ret = aes_expandkey(&ctx->key2, digest, sizeof(digest)); 182 if (ret) 183 goto out; 184 185 return 0; 186 out: 187 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 188 return -EINVAL; 189 } 190 191 static int __maybe_unused ecb_encrypt(struct skcipher_request *req) 192 { 193 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 194 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 195 int err, rounds = 6 + ctx->key_length / 4; 196 struct skcipher_walk walk; 197 unsigned int blocks; 198 199 err = skcipher_walk_virt(&walk, req, false); 200 201 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 202 kernel_neon_begin(); 203 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 204 ctx->key_enc, rounds, blocks); 205 kernel_neon_end(); 206 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 207 } 208 return err; 209 } 210 211 static int __maybe_unused ecb_decrypt(struct skcipher_request *req) 212 { 213 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 214 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 215 int err, rounds = 6 + ctx->key_length / 4; 216 struct skcipher_walk walk; 217 unsigned int blocks; 218 219 err = skcipher_walk_virt(&walk, req, false); 220 221 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 222 kernel_neon_begin(); 223 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 224 ctx->key_dec, rounds, blocks); 225 kernel_neon_end(); 226 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 227 } 228 return err; 229 } 230 231 static int cbc_encrypt_walk(struct skcipher_request *req, 232 struct skcipher_walk *walk) 233 { 234 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 235 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 236 int err = 0, rounds = 6 + ctx->key_length / 4; 237 unsigned int blocks; 238 239 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { 240 kernel_neon_begin(); 241 aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr, 242 ctx->key_enc, rounds, blocks, walk->iv); 243 kernel_neon_end(); 244 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); 245 } 246 return err; 247 } 248 249 static int __maybe_unused cbc_encrypt(struct skcipher_request *req) 250 { 251 struct skcipher_walk walk; 252 int err; 253 254 err = skcipher_walk_virt(&walk, req, false); 255 if (err) 256 return err; 257 return cbc_encrypt_walk(req, &walk); 258 } 259 260 static int cbc_decrypt_walk(struct skcipher_request *req, 261 struct skcipher_walk *walk) 262 { 263 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 264 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 265 int err = 0, rounds = 6 + ctx->key_length / 4; 266 unsigned int blocks; 267 268 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { 269 kernel_neon_begin(); 270 aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr, 271 ctx->key_dec, rounds, blocks, walk->iv); 272 kernel_neon_end(); 273 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); 274 } 275 return err; 276 } 277 278 static int __maybe_unused cbc_decrypt(struct skcipher_request *req) 279 { 280 struct skcipher_walk walk; 281 int err; 282 283 err = skcipher_walk_virt(&walk, req, false); 284 if (err) 285 return err; 286 return cbc_decrypt_walk(req, &walk); 287 } 288 289 static int cts_cbc_encrypt(struct skcipher_request *req) 290 { 291 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 292 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 293 int err, rounds = 6 + ctx->key_length / 4; 294 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 295 struct scatterlist *src = req->src, *dst = req->dst; 296 struct scatterlist sg_src[2], sg_dst[2]; 297 struct skcipher_request subreq; 298 struct skcipher_walk walk; 299 300 skcipher_request_set_tfm(&subreq, tfm); 301 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 302 NULL, NULL); 303 304 if (req->cryptlen <= AES_BLOCK_SIZE) { 305 if (req->cryptlen < AES_BLOCK_SIZE) 306 return -EINVAL; 307 cbc_blocks = 1; 308 } 309 310 if (cbc_blocks > 0) { 311 skcipher_request_set_crypt(&subreq, req->src, req->dst, 312 cbc_blocks * AES_BLOCK_SIZE, 313 req->iv); 314 315 err = skcipher_walk_virt(&walk, &subreq, false) ?: 316 cbc_encrypt_walk(&subreq, &walk); 317 if (err) 318 return err; 319 320 if (req->cryptlen == AES_BLOCK_SIZE) 321 return 0; 322 323 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 324 if (req->dst != req->src) 325 dst = scatterwalk_ffwd(sg_dst, req->dst, 326 subreq.cryptlen); 327 } 328 329 /* handle ciphertext stealing */ 330 skcipher_request_set_crypt(&subreq, src, dst, 331 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 332 req->iv); 333 334 err = skcipher_walk_virt(&walk, &subreq, false); 335 if (err) 336 return err; 337 338 kernel_neon_begin(); 339 aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 340 ctx->key_enc, rounds, walk.nbytes, walk.iv); 341 kernel_neon_end(); 342 343 return skcipher_walk_done(&walk, 0); 344 } 345 346 static int cts_cbc_decrypt(struct skcipher_request *req) 347 { 348 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 349 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 350 int err, rounds = 6 + ctx->key_length / 4; 351 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 352 struct scatterlist *src = req->src, *dst = req->dst; 353 struct scatterlist sg_src[2], sg_dst[2]; 354 struct skcipher_request subreq; 355 struct skcipher_walk walk; 356 357 skcipher_request_set_tfm(&subreq, tfm); 358 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 359 NULL, NULL); 360 361 if (req->cryptlen <= AES_BLOCK_SIZE) { 362 if (req->cryptlen < AES_BLOCK_SIZE) 363 return -EINVAL; 364 cbc_blocks = 1; 365 } 366 367 if (cbc_blocks > 0) { 368 skcipher_request_set_crypt(&subreq, req->src, req->dst, 369 cbc_blocks * AES_BLOCK_SIZE, 370 req->iv); 371 372 err = skcipher_walk_virt(&walk, &subreq, false) ?: 373 cbc_decrypt_walk(&subreq, &walk); 374 if (err) 375 return err; 376 377 if (req->cryptlen == AES_BLOCK_SIZE) 378 return 0; 379 380 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 381 if (req->dst != req->src) 382 dst = scatterwalk_ffwd(sg_dst, req->dst, 383 subreq.cryptlen); 384 } 385 386 /* handle ciphertext stealing */ 387 skcipher_request_set_crypt(&subreq, src, dst, 388 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 389 req->iv); 390 391 err = skcipher_walk_virt(&walk, &subreq, false); 392 if (err) 393 return err; 394 395 kernel_neon_begin(); 396 aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 397 ctx->key_dec, rounds, walk.nbytes, walk.iv); 398 kernel_neon_end(); 399 400 return skcipher_walk_done(&walk, 0); 401 } 402 403 static int __maybe_unused essiv_cbc_init_tfm(struct crypto_skcipher *tfm) 404 { 405 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 406 407 ctx->hash = crypto_alloc_shash("sha256", 0, 0); 408 409 return PTR_ERR_OR_ZERO(ctx->hash); 410 } 411 412 static void __maybe_unused essiv_cbc_exit_tfm(struct crypto_skcipher *tfm) 413 { 414 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 415 416 crypto_free_shash(ctx->hash); 417 } 418 419 static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req) 420 { 421 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 422 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 423 int err, rounds = 6 + ctx->key1.key_length / 4; 424 struct skcipher_walk walk; 425 unsigned int blocks; 426 427 err = skcipher_walk_virt(&walk, req, false); 428 429 blocks = walk.nbytes / AES_BLOCK_SIZE; 430 if (blocks) { 431 kernel_neon_begin(); 432 aes_essiv_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 433 ctx->key1.key_enc, rounds, blocks, 434 req->iv, ctx->key2.key_enc); 435 kernel_neon_end(); 436 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 437 } 438 return err ?: cbc_encrypt_walk(req, &walk); 439 } 440 441 static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req) 442 { 443 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 444 struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 445 int err, rounds = 6 + ctx->key1.key_length / 4; 446 struct skcipher_walk walk; 447 unsigned int blocks; 448 449 err = skcipher_walk_virt(&walk, req, false); 450 451 blocks = walk.nbytes / AES_BLOCK_SIZE; 452 if (blocks) { 453 kernel_neon_begin(); 454 aes_essiv_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 455 ctx->key1.key_dec, rounds, blocks, 456 req->iv, ctx->key2.key_enc); 457 kernel_neon_end(); 458 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 459 } 460 return err ?: cbc_decrypt_walk(req, &walk); 461 } 462 463 static int ctr_encrypt(struct skcipher_request *req) 464 { 465 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 466 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 467 int err, rounds = 6 + ctx->key_length / 4; 468 struct skcipher_walk walk; 469 int blocks; 470 471 err = skcipher_walk_virt(&walk, req, false); 472 473 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 474 kernel_neon_begin(); 475 aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 476 ctx->key_enc, rounds, blocks, walk.iv); 477 kernel_neon_end(); 478 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 479 } 480 if (walk.nbytes) { 481 u8 __aligned(8) tail[AES_BLOCK_SIZE]; 482 unsigned int nbytes = walk.nbytes; 483 u8 *tdst = walk.dst.virt.addr; 484 u8 *tsrc = walk.src.virt.addr; 485 486 /* 487 * Tell aes_ctr_encrypt() to process a tail block. 488 */ 489 blocks = -1; 490 491 kernel_neon_begin(); 492 aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds, 493 blocks, walk.iv); 494 kernel_neon_end(); 495 crypto_xor_cpy(tdst, tsrc, tail, nbytes); 496 err = skcipher_walk_done(&walk, 0); 497 } 498 499 return err; 500 } 501 502 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) 503 { 504 const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 505 unsigned long flags; 506 507 /* 508 * Temporarily disable interrupts to avoid races where 509 * cachelines are evicted when the CPU is interrupted 510 * to do something else. 511 */ 512 local_irq_save(flags); 513 aes_encrypt(ctx, dst, src); 514 local_irq_restore(flags); 515 } 516 517 static int __maybe_unused ctr_encrypt_sync(struct skcipher_request *req) 518 { 519 if (!crypto_simd_usable()) 520 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); 521 522 return ctr_encrypt(req); 523 } 524 525 static int __maybe_unused xts_encrypt(struct skcipher_request *req) 526 { 527 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 528 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 529 int err, first, rounds = 6 + ctx->key1.key_length / 4; 530 int tail = req->cryptlen % AES_BLOCK_SIZE; 531 struct scatterlist sg_src[2], sg_dst[2]; 532 struct skcipher_request subreq; 533 struct scatterlist *src, *dst; 534 struct skcipher_walk walk; 535 536 if (req->cryptlen < AES_BLOCK_SIZE) 537 return -EINVAL; 538 539 err = skcipher_walk_virt(&walk, req, false); 540 541 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 542 int xts_blocks = DIV_ROUND_UP(req->cryptlen, 543 AES_BLOCK_SIZE) - 2; 544 545 skcipher_walk_abort(&walk); 546 547 skcipher_request_set_tfm(&subreq, tfm); 548 skcipher_request_set_callback(&subreq, 549 skcipher_request_flags(req), 550 NULL, NULL); 551 skcipher_request_set_crypt(&subreq, req->src, req->dst, 552 xts_blocks * AES_BLOCK_SIZE, 553 req->iv); 554 req = &subreq; 555 err = skcipher_walk_virt(&walk, req, false); 556 } else { 557 tail = 0; 558 } 559 560 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) { 561 int nbytes = walk.nbytes; 562 563 if (walk.nbytes < walk.total) 564 nbytes &= ~(AES_BLOCK_SIZE - 1); 565 566 kernel_neon_begin(); 567 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 568 ctx->key1.key_enc, rounds, nbytes, 569 ctx->key2.key_enc, walk.iv, first); 570 kernel_neon_end(); 571 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 572 } 573 574 if (err || likely(!tail)) 575 return err; 576 577 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 578 if (req->dst != req->src) 579 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 580 581 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 582 req->iv); 583 584 err = skcipher_walk_virt(&walk, &subreq, false); 585 if (err) 586 return err; 587 588 kernel_neon_begin(); 589 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 590 ctx->key1.key_enc, rounds, walk.nbytes, 591 ctx->key2.key_enc, walk.iv, first); 592 kernel_neon_end(); 593 594 return skcipher_walk_done(&walk, 0); 595 } 596 597 static int __maybe_unused xts_decrypt(struct skcipher_request *req) 598 { 599 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 600 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 601 int err, first, rounds = 6 + ctx->key1.key_length / 4; 602 int tail = req->cryptlen % AES_BLOCK_SIZE; 603 struct scatterlist sg_src[2], sg_dst[2]; 604 struct skcipher_request subreq; 605 struct scatterlist *src, *dst; 606 struct skcipher_walk walk; 607 608 if (req->cryptlen < AES_BLOCK_SIZE) 609 return -EINVAL; 610 611 err = skcipher_walk_virt(&walk, req, false); 612 613 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 614 int xts_blocks = DIV_ROUND_UP(req->cryptlen, 615 AES_BLOCK_SIZE) - 2; 616 617 skcipher_walk_abort(&walk); 618 619 skcipher_request_set_tfm(&subreq, tfm); 620 skcipher_request_set_callback(&subreq, 621 skcipher_request_flags(req), 622 NULL, NULL); 623 skcipher_request_set_crypt(&subreq, req->src, req->dst, 624 xts_blocks * AES_BLOCK_SIZE, 625 req->iv); 626 req = &subreq; 627 err = skcipher_walk_virt(&walk, req, false); 628 } else { 629 tail = 0; 630 } 631 632 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) { 633 int nbytes = walk.nbytes; 634 635 if (walk.nbytes < walk.total) 636 nbytes &= ~(AES_BLOCK_SIZE - 1); 637 638 kernel_neon_begin(); 639 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 640 ctx->key1.key_dec, rounds, nbytes, 641 ctx->key2.key_enc, walk.iv, first); 642 kernel_neon_end(); 643 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 644 } 645 646 if (err || likely(!tail)) 647 return err; 648 649 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 650 if (req->dst != req->src) 651 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 652 653 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 654 req->iv); 655 656 err = skcipher_walk_virt(&walk, &subreq, false); 657 if (err) 658 return err; 659 660 661 kernel_neon_begin(); 662 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 663 ctx->key1.key_dec, rounds, walk.nbytes, 664 ctx->key2.key_enc, walk.iv, first); 665 kernel_neon_end(); 666 667 return skcipher_walk_done(&walk, 0); 668 } 669 670 static struct skcipher_alg aes_algs[] = { { 671 #if defined(USE_V8_CRYPTO_EXTENSIONS) || !defined(CONFIG_CRYPTO_AES_ARM64_BS) 672 .base = { 673 .cra_name = "__ecb(aes)", 674 .cra_driver_name = "__ecb-aes-" MODE, 675 .cra_priority = PRIO, 676 .cra_flags = CRYPTO_ALG_INTERNAL, 677 .cra_blocksize = AES_BLOCK_SIZE, 678 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 679 .cra_module = THIS_MODULE, 680 }, 681 .min_keysize = AES_MIN_KEY_SIZE, 682 .max_keysize = AES_MAX_KEY_SIZE, 683 .setkey = skcipher_aes_setkey, 684 .encrypt = ecb_encrypt, 685 .decrypt = ecb_decrypt, 686 }, { 687 .base = { 688 .cra_name = "__cbc(aes)", 689 .cra_driver_name = "__cbc-aes-" MODE, 690 .cra_priority = PRIO, 691 .cra_flags = CRYPTO_ALG_INTERNAL, 692 .cra_blocksize = AES_BLOCK_SIZE, 693 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 694 .cra_module = THIS_MODULE, 695 }, 696 .min_keysize = AES_MIN_KEY_SIZE, 697 .max_keysize = AES_MAX_KEY_SIZE, 698 .ivsize = AES_BLOCK_SIZE, 699 .setkey = skcipher_aes_setkey, 700 .encrypt = cbc_encrypt, 701 .decrypt = cbc_decrypt, 702 }, { 703 .base = { 704 .cra_name = "__ctr(aes)", 705 .cra_driver_name = "__ctr-aes-" MODE, 706 .cra_priority = PRIO, 707 .cra_flags = CRYPTO_ALG_INTERNAL, 708 .cra_blocksize = 1, 709 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 710 .cra_module = THIS_MODULE, 711 }, 712 .min_keysize = AES_MIN_KEY_SIZE, 713 .max_keysize = AES_MAX_KEY_SIZE, 714 .ivsize = AES_BLOCK_SIZE, 715 .chunksize = AES_BLOCK_SIZE, 716 .setkey = skcipher_aes_setkey, 717 .encrypt = ctr_encrypt, 718 .decrypt = ctr_encrypt, 719 }, { 720 .base = { 721 .cra_name = "ctr(aes)", 722 .cra_driver_name = "ctr-aes-" MODE, 723 .cra_priority = PRIO - 1, 724 .cra_blocksize = 1, 725 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 726 .cra_module = THIS_MODULE, 727 }, 728 .min_keysize = AES_MIN_KEY_SIZE, 729 .max_keysize = AES_MAX_KEY_SIZE, 730 .ivsize = AES_BLOCK_SIZE, 731 .chunksize = AES_BLOCK_SIZE, 732 .setkey = skcipher_aes_setkey, 733 .encrypt = ctr_encrypt_sync, 734 .decrypt = ctr_encrypt_sync, 735 }, { 736 .base = { 737 .cra_name = "__xts(aes)", 738 .cra_driver_name = "__xts-aes-" MODE, 739 .cra_priority = PRIO, 740 .cra_flags = CRYPTO_ALG_INTERNAL, 741 .cra_blocksize = AES_BLOCK_SIZE, 742 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 743 .cra_module = THIS_MODULE, 744 }, 745 .min_keysize = 2 * AES_MIN_KEY_SIZE, 746 .max_keysize = 2 * AES_MAX_KEY_SIZE, 747 .ivsize = AES_BLOCK_SIZE, 748 .walksize = 2 * AES_BLOCK_SIZE, 749 .setkey = xts_set_key, 750 .encrypt = xts_encrypt, 751 .decrypt = xts_decrypt, 752 }, { 753 #endif 754 .base = { 755 .cra_name = "__cts(cbc(aes))", 756 .cra_driver_name = "__cts-cbc-aes-" MODE, 757 .cra_priority = PRIO, 758 .cra_flags = CRYPTO_ALG_INTERNAL, 759 .cra_blocksize = AES_BLOCK_SIZE, 760 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 761 .cra_module = THIS_MODULE, 762 }, 763 .min_keysize = AES_MIN_KEY_SIZE, 764 .max_keysize = AES_MAX_KEY_SIZE, 765 .ivsize = AES_BLOCK_SIZE, 766 .walksize = 2 * AES_BLOCK_SIZE, 767 .setkey = skcipher_aes_setkey, 768 .encrypt = cts_cbc_encrypt, 769 .decrypt = cts_cbc_decrypt, 770 }, { 771 .base = { 772 .cra_name = "__essiv(cbc(aes),sha256)", 773 .cra_driver_name = "__essiv-cbc-aes-sha256-" MODE, 774 .cra_priority = PRIO + 1, 775 .cra_flags = CRYPTO_ALG_INTERNAL, 776 .cra_blocksize = AES_BLOCK_SIZE, 777 .cra_ctxsize = sizeof(struct crypto_aes_essiv_cbc_ctx), 778 .cra_module = THIS_MODULE, 779 }, 780 .min_keysize = AES_MIN_KEY_SIZE, 781 .max_keysize = AES_MAX_KEY_SIZE, 782 .ivsize = AES_BLOCK_SIZE, 783 .setkey = essiv_cbc_set_key, 784 .encrypt = essiv_cbc_encrypt, 785 .decrypt = essiv_cbc_decrypt, 786 .init = essiv_cbc_init_tfm, 787 .exit = essiv_cbc_exit_tfm, 788 } }; 789 790 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key, 791 unsigned int key_len) 792 { 793 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 794 int err; 795 796 err = aes_expandkey(&ctx->key, in_key, key_len); 797 if (err) 798 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 799 800 return err; 801 } 802 803 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x) 804 { 805 u64 a = be64_to_cpu(x->a); 806 u64 b = be64_to_cpu(x->b); 807 808 y->a = cpu_to_be64((a << 1) | (b >> 63)); 809 y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); 810 } 811 812 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key, 813 unsigned int key_len) 814 { 815 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 816 be128 *consts = (be128 *)ctx->consts; 817 int rounds = 6 + key_len / 4; 818 int err; 819 820 err = cbcmac_setkey(tfm, in_key, key_len); 821 if (err) 822 return err; 823 824 /* encrypt the zero vector */ 825 kernel_neon_begin(); 826 aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc, 827 rounds, 1); 828 kernel_neon_end(); 829 830 cmac_gf128_mul_by_x(consts, consts); 831 cmac_gf128_mul_by_x(consts + 1, consts); 832 833 return 0; 834 } 835 836 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key, 837 unsigned int key_len) 838 { 839 static u8 const ks[3][AES_BLOCK_SIZE] = { 840 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 841 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 842 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 843 }; 844 845 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 846 int rounds = 6 + key_len / 4; 847 u8 key[AES_BLOCK_SIZE]; 848 int err; 849 850 err = cbcmac_setkey(tfm, in_key, key_len); 851 if (err) 852 return err; 853 854 kernel_neon_begin(); 855 aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1); 856 aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2); 857 kernel_neon_end(); 858 859 return cbcmac_setkey(tfm, key, sizeof(key)); 860 } 861 862 static int mac_init(struct shash_desc *desc) 863 { 864 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 865 866 memset(ctx->dg, 0, AES_BLOCK_SIZE); 867 ctx->len = 0; 868 869 return 0; 870 } 871 872 static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, 873 u8 dg[], int enc_before, int enc_after) 874 { 875 int rounds = 6 + ctx->key_length / 4; 876 877 if (crypto_simd_usable()) { 878 kernel_neon_begin(); 879 aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before, 880 enc_after); 881 kernel_neon_end(); 882 } else { 883 if (enc_before) 884 aes_encrypt(ctx, dg, dg); 885 886 while (blocks--) { 887 crypto_xor(dg, in, AES_BLOCK_SIZE); 888 in += AES_BLOCK_SIZE; 889 890 if (blocks || enc_after) 891 aes_encrypt(ctx, dg, dg); 892 } 893 } 894 } 895 896 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) 897 { 898 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 899 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 900 901 while (len > 0) { 902 unsigned int l; 903 904 if ((ctx->len % AES_BLOCK_SIZE) == 0 && 905 (ctx->len + len) > AES_BLOCK_SIZE) { 906 907 int blocks = len / AES_BLOCK_SIZE; 908 909 len %= AES_BLOCK_SIZE; 910 911 mac_do_update(&tctx->key, p, blocks, ctx->dg, 912 (ctx->len != 0), (len != 0)); 913 914 p += blocks * AES_BLOCK_SIZE; 915 916 if (!len) { 917 ctx->len = AES_BLOCK_SIZE; 918 break; 919 } 920 ctx->len = 0; 921 } 922 923 l = min(len, AES_BLOCK_SIZE - ctx->len); 924 925 if (l <= AES_BLOCK_SIZE) { 926 crypto_xor(ctx->dg + ctx->len, p, l); 927 ctx->len += l; 928 len -= l; 929 p += l; 930 } 931 } 932 933 return 0; 934 } 935 936 static int cbcmac_final(struct shash_desc *desc, u8 *out) 937 { 938 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 939 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 940 941 mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0); 942 943 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 944 945 return 0; 946 } 947 948 static int cmac_final(struct shash_desc *desc, u8 *out) 949 { 950 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 951 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 952 u8 *consts = tctx->consts; 953 954 if (ctx->len != AES_BLOCK_SIZE) { 955 ctx->dg[ctx->len] ^= 0x80; 956 consts += AES_BLOCK_SIZE; 957 } 958 959 mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1); 960 961 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 962 963 return 0; 964 } 965 966 static struct shash_alg mac_algs[] = { { 967 .base.cra_name = "cmac(aes)", 968 .base.cra_driver_name = "cmac-aes-" MODE, 969 .base.cra_priority = PRIO, 970 .base.cra_blocksize = AES_BLOCK_SIZE, 971 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 972 2 * AES_BLOCK_SIZE, 973 .base.cra_module = THIS_MODULE, 974 975 .digestsize = AES_BLOCK_SIZE, 976 .init = mac_init, 977 .update = mac_update, 978 .final = cmac_final, 979 .setkey = cmac_setkey, 980 .descsize = sizeof(struct mac_desc_ctx), 981 }, { 982 .base.cra_name = "xcbc(aes)", 983 .base.cra_driver_name = "xcbc-aes-" MODE, 984 .base.cra_priority = PRIO, 985 .base.cra_blocksize = AES_BLOCK_SIZE, 986 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 987 2 * AES_BLOCK_SIZE, 988 .base.cra_module = THIS_MODULE, 989 990 .digestsize = AES_BLOCK_SIZE, 991 .init = mac_init, 992 .update = mac_update, 993 .final = cmac_final, 994 .setkey = xcbc_setkey, 995 .descsize = sizeof(struct mac_desc_ctx), 996 }, { 997 .base.cra_name = "cbcmac(aes)", 998 .base.cra_driver_name = "cbcmac-aes-" MODE, 999 .base.cra_priority = PRIO, 1000 .base.cra_blocksize = 1, 1001 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), 1002 .base.cra_module = THIS_MODULE, 1003 1004 .digestsize = AES_BLOCK_SIZE, 1005 .init = mac_init, 1006 .update = mac_update, 1007 .final = cbcmac_final, 1008 .setkey = cbcmac_setkey, 1009 .descsize = sizeof(struct mac_desc_ctx), 1010 } }; 1011 1012 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 1013 1014 static void aes_exit(void) 1015 { 1016 int i; 1017 1018 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) 1019 if (aes_simd_algs[i]) 1020 simd_skcipher_free(aes_simd_algs[i]); 1021 1022 crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 1023 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 1024 } 1025 1026 static int __init aes_init(void) 1027 { 1028 struct simd_skcipher_alg *simd; 1029 const char *basename; 1030 const char *algname; 1031 const char *drvname; 1032 int err; 1033 int i; 1034 1035 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 1036 if (err) 1037 return err; 1038 1039 err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 1040 if (err) 1041 goto unregister_ciphers; 1042 1043 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 1044 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) 1045 continue; 1046 1047 algname = aes_algs[i].base.cra_name + 2; 1048 drvname = aes_algs[i].base.cra_driver_name + 2; 1049 basename = aes_algs[i].base.cra_driver_name; 1050 simd = simd_skcipher_create_compat(algname, drvname, basename); 1051 err = PTR_ERR(simd); 1052 if (IS_ERR(simd)) 1053 goto unregister_simds; 1054 1055 aes_simd_algs[i] = simd; 1056 } 1057 1058 return 0; 1059 1060 unregister_simds: 1061 aes_exit(); 1062 return err; 1063 unregister_ciphers: 1064 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 1065 return err; 1066 } 1067 1068 #ifdef USE_V8_CRYPTO_EXTENSIONS 1069 module_cpu_feature_match(AES, aes_init); 1070 #else 1071 module_init(aes_init); 1072 EXPORT_SYMBOL(neon_aes_ecb_encrypt); 1073 EXPORT_SYMBOL(neon_aes_cbc_encrypt); 1074 EXPORT_SYMBOL(neon_aes_xts_encrypt); 1075 EXPORT_SYMBOL(neon_aes_xts_decrypt); 1076 #endif 1077 module_exit(aes_exit); 1078