1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES 4 * 5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/neon.h> 9 #include <asm/hwcap.h> 10 #include <asm/simd.h> 11 #include <crypto/aes.h> 12 #include <crypto/ctr.h> 13 #include <crypto/internal/hash.h> 14 #include <crypto/internal/simd.h> 15 #include <crypto/internal/skcipher.h> 16 #include <crypto/scatterwalk.h> 17 #include <linux/module.h> 18 #include <linux/cpufeature.h> 19 #include <crypto/xts.h> 20 21 #include "aes-ce-setkey.h" 22 23 #ifdef USE_V8_CRYPTO_EXTENSIONS 24 #define MODE "ce" 25 #define PRIO 300 26 #define aes_expandkey ce_aes_expandkey 27 #define aes_ecb_encrypt ce_aes_ecb_encrypt 28 #define aes_ecb_decrypt ce_aes_ecb_decrypt 29 #define aes_cbc_encrypt ce_aes_cbc_encrypt 30 #define aes_cbc_decrypt ce_aes_cbc_decrypt 31 #define aes_cbc_cts_encrypt ce_aes_cbc_cts_encrypt 32 #define aes_cbc_cts_decrypt ce_aes_cbc_cts_decrypt 33 #define aes_ctr_encrypt ce_aes_ctr_encrypt 34 #define aes_xts_encrypt ce_aes_xts_encrypt 35 #define aes_xts_decrypt ce_aes_xts_decrypt 36 #define aes_mac_update ce_aes_mac_update 37 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 38 #else 39 #define MODE "neon" 40 #define PRIO 200 41 #define aes_ecb_encrypt neon_aes_ecb_encrypt 42 #define aes_ecb_decrypt neon_aes_ecb_decrypt 43 #define aes_cbc_encrypt neon_aes_cbc_encrypt 44 #define aes_cbc_decrypt neon_aes_cbc_decrypt 45 #define aes_cbc_cts_encrypt neon_aes_cbc_cts_encrypt 46 #define aes_cbc_cts_decrypt neon_aes_cbc_cts_decrypt 47 #define aes_ctr_encrypt neon_aes_ctr_encrypt 48 #define aes_xts_encrypt neon_aes_xts_encrypt 49 #define aes_xts_decrypt neon_aes_xts_decrypt 50 #define aes_mac_update neon_aes_mac_update 51 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); 52 MODULE_ALIAS_CRYPTO("ecb(aes)"); 53 MODULE_ALIAS_CRYPTO("cbc(aes)"); 54 MODULE_ALIAS_CRYPTO("ctr(aes)"); 55 MODULE_ALIAS_CRYPTO("xts(aes)"); 56 MODULE_ALIAS_CRYPTO("cmac(aes)"); 57 MODULE_ALIAS_CRYPTO("xcbc(aes)"); 58 MODULE_ALIAS_CRYPTO("cbcmac(aes)"); 59 #endif 60 61 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 62 MODULE_LICENSE("GPL v2"); 63 64 /* defined in aes-modes.S */ 65 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], 66 int rounds, int blocks); 67 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], 68 int rounds, int blocks); 69 70 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], 71 int rounds, int blocks, u8 iv[]); 72 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], 73 int rounds, int blocks, u8 iv[]); 74 75 asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], 76 int rounds, int bytes, u8 const iv[]); 77 asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], 78 int rounds, int bytes, u8 const iv[]); 79 80 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], 81 int rounds, int blocks, u8 ctr[]); 82 83 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], 84 int rounds, int blocks, u32 const rk2[], u8 iv[], 85 int first); 86 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], 87 int rounds, int blocks, u32 const rk2[], u8 iv[], 88 int first); 89 90 asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds, 91 int blocks, u8 dg[], int enc_before, 92 int enc_after); 93 94 struct cts_cbc_req_ctx { 95 struct scatterlist sg_src[2]; 96 struct scatterlist sg_dst[2]; 97 struct skcipher_request subreq; 98 }; 99 100 struct crypto_aes_xts_ctx { 101 struct crypto_aes_ctx key1; 102 struct crypto_aes_ctx __aligned(8) key2; 103 }; 104 105 struct mac_tfm_ctx { 106 struct crypto_aes_ctx key; 107 u8 __aligned(8) consts[]; 108 }; 109 110 struct mac_desc_ctx { 111 unsigned int len; 112 u8 dg[AES_BLOCK_SIZE]; 113 }; 114 115 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 116 unsigned int key_len) 117 { 118 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 119 int ret; 120 121 ret = aes_expandkey(ctx, in_key, key_len); 122 if (ret) 123 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 124 125 return ret; 126 } 127 128 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 129 unsigned int key_len) 130 { 131 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 132 int ret; 133 134 ret = xts_verify_key(tfm, in_key, key_len); 135 if (ret) 136 return ret; 137 138 ret = aes_expandkey(&ctx->key1, in_key, key_len / 2); 139 if (!ret) 140 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2], 141 key_len / 2); 142 if (!ret) 143 return 0; 144 145 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 146 return -EINVAL; 147 } 148 149 static int ecb_encrypt(struct skcipher_request *req) 150 { 151 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 152 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 153 int err, rounds = 6 + ctx->key_length / 4; 154 struct skcipher_walk walk; 155 unsigned int blocks; 156 157 err = skcipher_walk_virt(&walk, req, false); 158 159 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 160 kernel_neon_begin(); 161 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 162 ctx->key_enc, rounds, blocks); 163 kernel_neon_end(); 164 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 165 } 166 return err; 167 } 168 169 static int ecb_decrypt(struct skcipher_request *req) 170 { 171 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 172 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 173 int err, rounds = 6 + ctx->key_length / 4; 174 struct skcipher_walk walk; 175 unsigned int blocks; 176 177 err = skcipher_walk_virt(&walk, req, false); 178 179 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 180 kernel_neon_begin(); 181 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 182 ctx->key_dec, rounds, blocks); 183 kernel_neon_end(); 184 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 185 } 186 return err; 187 } 188 189 static int cbc_encrypt(struct skcipher_request *req) 190 { 191 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 192 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 193 int err, rounds = 6 + ctx->key_length / 4; 194 struct skcipher_walk walk; 195 unsigned int blocks; 196 197 err = skcipher_walk_virt(&walk, req, false); 198 199 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 200 kernel_neon_begin(); 201 aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 202 ctx->key_enc, rounds, blocks, walk.iv); 203 kernel_neon_end(); 204 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 205 } 206 return err; 207 } 208 209 static int cbc_decrypt(struct skcipher_request *req) 210 { 211 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 212 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 213 int err, rounds = 6 + ctx->key_length / 4; 214 struct skcipher_walk walk; 215 unsigned int blocks; 216 217 err = skcipher_walk_virt(&walk, req, false); 218 219 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 220 kernel_neon_begin(); 221 aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 222 ctx->key_dec, rounds, blocks, walk.iv); 223 kernel_neon_end(); 224 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 225 } 226 return err; 227 } 228 229 static int cts_cbc_init_tfm(struct crypto_skcipher *tfm) 230 { 231 crypto_skcipher_set_reqsize(tfm, sizeof(struct cts_cbc_req_ctx)); 232 return 0; 233 } 234 235 static int cts_cbc_encrypt(struct skcipher_request *req) 236 { 237 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 238 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 239 struct cts_cbc_req_ctx *rctx = skcipher_request_ctx(req); 240 int err, rounds = 6 + ctx->key_length / 4; 241 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 242 struct scatterlist *src = req->src, *dst = req->dst; 243 struct skcipher_walk walk; 244 245 skcipher_request_set_tfm(&rctx->subreq, tfm); 246 247 if (req->cryptlen <= AES_BLOCK_SIZE) { 248 if (req->cryptlen < AES_BLOCK_SIZE) 249 return -EINVAL; 250 cbc_blocks = 1; 251 } 252 253 if (cbc_blocks > 0) { 254 unsigned int blocks; 255 256 skcipher_request_set_crypt(&rctx->subreq, req->src, req->dst, 257 cbc_blocks * AES_BLOCK_SIZE, 258 req->iv); 259 260 err = skcipher_walk_virt(&walk, &rctx->subreq, false); 261 262 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 263 kernel_neon_begin(); 264 aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 265 ctx->key_enc, rounds, blocks, walk.iv); 266 kernel_neon_end(); 267 err = skcipher_walk_done(&walk, 268 walk.nbytes % AES_BLOCK_SIZE); 269 } 270 if (err) 271 return err; 272 273 if (req->cryptlen == AES_BLOCK_SIZE) 274 return 0; 275 276 dst = src = scatterwalk_ffwd(rctx->sg_src, req->src, 277 rctx->subreq.cryptlen); 278 if (req->dst != req->src) 279 dst = scatterwalk_ffwd(rctx->sg_dst, req->dst, 280 rctx->subreq.cryptlen); 281 } 282 283 /* handle ciphertext stealing */ 284 skcipher_request_set_crypt(&rctx->subreq, src, dst, 285 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 286 req->iv); 287 288 err = skcipher_walk_virt(&walk, &rctx->subreq, false); 289 if (err) 290 return err; 291 292 kernel_neon_begin(); 293 aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 294 ctx->key_enc, rounds, walk.nbytes, walk.iv); 295 kernel_neon_end(); 296 297 return skcipher_walk_done(&walk, 0); 298 } 299 300 static int cts_cbc_decrypt(struct skcipher_request *req) 301 { 302 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 303 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 304 struct cts_cbc_req_ctx *rctx = skcipher_request_ctx(req); 305 int err, rounds = 6 + ctx->key_length / 4; 306 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 307 struct scatterlist *src = req->src, *dst = req->dst; 308 struct skcipher_walk walk; 309 310 skcipher_request_set_tfm(&rctx->subreq, tfm); 311 312 if (req->cryptlen <= AES_BLOCK_SIZE) { 313 if (req->cryptlen < AES_BLOCK_SIZE) 314 return -EINVAL; 315 cbc_blocks = 1; 316 } 317 318 if (cbc_blocks > 0) { 319 unsigned int blocks; 320 321 skcipher_request_set_crypt(&rctx->subreq, req->src, req->dst, 322 cbc_blocks * AES_BLOCK_SIZE, 323 req->iv); 324 325 err = skcipher_walk_virt(&walk, &rctx->subreq, false); 326 327 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 328 kernel_neon_begin(); 329 aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 330 ctx->key_dec, rounds, blocks, walk.iv); 331 kernel_neon_end(); 332 err = skcipher_walk_done(&walk, 333 walk.nbytes % AES_BLOCK_SIZE); 334 } 335 if (err) 336 return err; 337 338 if (req->cryptlen == AES_BLOCK_SIZE) 339 return 0; 340 341 dst = src = scatterwalk_ffwd(rctx->sg_src, req->src, 342 rctx->subreq.cryptlen); 343 if (req->dst != req->src) 344 dst = scatterwalk_ffwd(rctx->sg_dst, req->dst, 345 rctx->subreq.cryptlen); 346 } 347 348 /* handle ciphertext stealing */ 349 skcipher_request_set_crypt(&rctx->subreq, src, dst, 350 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 351 req->iv); 352 353 err = skcipher_walk_virt(&walk, &rctx->subreq, false); 354 if (err) 355 return err; 356 357 kernel_neon_begin(); 358 aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 359 ctx->key_dec, rounds, walk.nbytes, walk.iv); 360 kernel_neon_end(); 361 362 return skcipher_walk_done(&walk, 0); 363 } 364 365 static int ctr_encrypt(struct skcipher_request *req) 366 { 367 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 368 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 369 int err, rounds = 6 + ctx->key_length / 4; 370 struct skcipher_walk walk; 371 int blocks; 372 373 err = skcipher_walk_virt(&walk, req, false); 374 375 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 376 kernel_neon_begin(); 377 aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 378 ctx->key_enc, rounds, blocks, walk.iv); 379 kernel_neon_end(); 380 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 381 } 382 if (walk.nbytes) { 383 u8 __aligned(8) tail[AES_BLOCK_SIZE]; 384 unsigned int nbytes = walk.nbytes; 385 u8 *tdst = walk.dst.virt.addr; 386 u8 *tsrc = walk.src.virt.addr; 387 388 /* 389 * Tell aes_ctr_encrypt() to process a tail block. 390 */ 391 blocks = -1; 392 393 kernel_neon_begin(); 394 aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds, 395 blocks, walk.iv); 396 kernel_neon_end(); 397 crypto_xor_cpy(tdst, tsrc, tail, nbytes); 398 err = skcipher_walk_done(&walk, 0); 399 } 400 401 return err; 402 } 403 404 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) 405 { 406 const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 407 unsigned long flags; 408 409 /* 410 * Temporarily disable interrupts to avoid races where 411 * cachelines are evicted when the CPU is interrupted 412 * to do something else. 413 */ 414 local_irq_save(flags); 415 aes_encrypt(ctx, dst, src); 416 local_irq_restore(flags); 417 } 418 419 static int ctr_encrypt_sync(struct skcipher_request *req) 420 { 421 if (!crypto_simd_usable()) 422 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); 423 424 return ctr_encrypt(req); 425 } 426 427 static int xts_encrypt(struct skcipher_request *req) 428 { 429 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 430 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 431 int err, first, rounds = 6 + ctx->key1.key_length / 4; 432 struct skcipher_walk walk; 433 unsigned int blocks; 434 435 err = skcipher_walk_virt(&walk, req, false); 436 437 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 438 kernel_neon_begin(); 439 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 440 ctx->key1.key_enc, rounds, blocks, 441 ctx->key2.key_enc, walk.iv, first); 442 kernel_neon_end(); 443 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 444 } 445 446 return err; 447 } 448 449 static int xts_decrypt(struct skcipher_request *req) 450 { 451 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 452 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 453 int err, first, rounds = 6 + ctx->key1.key_length / 4; 454 struct skcipher_walk walk; 455 unsigned int blocks; 456 457 err = skcipher_walk_virt(&walk, req, false); 458 459 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 460 kernel_neon_begin(); 461 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 462 ctx->key1.key_dec, rounds, blocks, 463 ctx->key2.key_enc, walk.iv, first); 464 kernel_neon_end(); 465 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 466 } 467 468 return err; 469 } 470 471 static struct skcipher_alg aes_algs[] = { { 472 .base = { 473 .cra_name = "__ecb(aes)", 474 .cra_driver_name = "__ecb-aes-" MODE, 475 .cra_priority = PRIO, 476 .cra_flags = CRYPTO_ALG_INTERNAL, 477 .cra_blocksize = AES_BLOCK_SIZE, 478 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 479 .cra_module = THIS_MODULE, 480 }, 481 .min_keysize = AES_MIN_KEY_SIZE, 482 .max_keysize = AES_MAX_KEY_SIZE, 483 .setkey = skcipher_aes_setkey, 484 .encrypt = ecb_encrypt, 485 .decrypt = ecb_decrypt, 486 }, { 487 .base = { 488 .cra_name = "__cbc(aes)", 489 .cra_driver_name = "__cbc-aes-" MODE, 490 .cra_priority = PRIO, 491 .cra_flags = CRYPTO_ALG_INTERNAL, 492 .cra_blocksize = AES_BLOCK_SIZE, 493 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 494 .cra_module = THIS_MODULE, 495 }, 496 .min_keysize = AES_MIN_KEY_SIZE, 497 .max_keysize = AES_MAX_KEY_SIZE, 498 .ivsize = AES_BLOCK_SIZE, 499 .setkey = skcipher_aes_setkey, 500 .encrypt = cbc_encrypt, 501 .decrypt = cbc_decrypt, 502 }, { 503 .base = { 504 .cra_name = "__cts(cbc(aes))", 505 .cra_driver_name = "__cts-cbc-aes-" MODE, 506 .cra_priority = PRIO, 507 .cra_flags = CRYPTO_ALG_INTERNAL, 508 .cra_blocksize = AES_BLOCK_SIZE, 509 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 510 .cra_module = THIS_MODULE, 511 }, 512 .min_keysize = AES_MIN_KEY_SIZE, 513 .max_keysize = AES_MAX_KEY_SIZE, 514 .ivsize = AES_BLOCK_SIZE, 515 .walksize = 2 * AES_BLOCK_SIZE, 516 .setkey = skcipher_aes_setkey, 517 .encrypt = cts_cbc_encrypt, 518 .decrypt = cts_cbc_decrypt, 519 .init = cts_cbc_init_tfm, 520 }, { 521 .base = { 522 .cra_name = "__ctr(aes)", 523 .cra_driver_name = "__ctr-aes-" MODE, 524 .cra_priority = PRIO, 525 .cra_flags = CRYPTO_ALG_INTERNAL, 526 .cra_blocksize = 1, 527 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 528 .cra_module = THIS_MODULE, 529 }, 530 .min_keysize = AES_MIN_KEY_SIZE, 531 .max_keysize = AES_MAX_KEY_SIZE, 532 .ivsize = AES_BLOCK_SIZE, 533 .chunksize = AES_BLOCK_SIZE, 534 .setkey = skcipher_aes_setkey, 535 .encrypt = ctr_encrypt, 536 .decrypt = ctr_encrypt, 537 }, { 538 .base = { 539 .cra_name = "ctr(aes)", 540 .cra_driver_name = "ctr-aes-" MODE, 541 .cra_priority = PRIO - 1, 542 .cra_blocksize = 1, 543 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 544 .cra_module = THIS_MODULE, 545 }, 546 .min_keysize = AES_MIN_KEY_SIZE, 547 .max_keysize = AES_MAX_KEY_SIZE, 548 .ivsize = AES_BLOCK_SIZE, 549 .chunksize = AES_BLOCK_SIZE, 550 .setkey = skcipher_aes_setkey, 551 .encrypt = ctr_encrypt_sync, 552 .decrypt = ctr_encrypt_sync, 553 }, { 554 .base = { 555 .cra_name = "__xts(aes)", 556 .cra_driver_name = "__xts-aes-" MODE, 557 .cra_priority = PRIO, 558 .cra_flags = CRYPTO_ALG_INTERNAL, 559 .cra_blocksize = AES_BLOCK_SIZE, 560 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 561 .cra_module = THIS_MODULE, 562 }, 563 .min_keysize = 2 * AES_MIN_KEY_SIZE, 564 .max_keysize = 2 * AES_MAX_KEY_SIZE, 565 .ivsize = AES_BLOCK_SIZE, 566 .setkey = xts_set_key, 567 .encrypt = xts_encrypt, 568 .decrypt = xts_decrypt, 569 } }; 570 571 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key, 572 unsigned int key_len) 573 { 574 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 575 int err; 576 577 err = aes_expandkey(&ctx->key, in_key, key_len); 578 if (err) 579 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 580 581 return err; 582 } 583 584 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x) 585 { 586 u64 a = be64_to_cpu(x->a); 587 u64 b = be64_to_cpu(x->b); 588 589 y->a = cpu_to_be64((a << 1) | (b >> 63)); 590 y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); 591 } 592 593 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key, 594 unsigned int key_len) 595 { 596 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 597 be128 *consts = (be128 *)ctx->consts; 598 int rounds = 6 + key_len / 4; 599 int err; 600 601 err = cbcmac_setkey(tfm, in_key, key_len); 602 if (err) 603 return err; 604 605 /* encrypt the zero vector */ 606 kernel_neon_begin(); 607 aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc, 608 rounds, 1); 609 kernel_neon_end(); 610 611 cmac_gf128_mul_by_x(consts, consts); 612 cmac_gf128_mul_by_x(consts + 1, consts); 613 614 return 0; 615 } 616 617 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key, 618 unsigned int key_len) 619 { 620 static u8 const ks[3][AES_BLOCK_SIZE] = { 621 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, 622 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, 623 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, 624 }; 625 626 struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); 627 int rounds = 6 + key_len / 4; 628 u8 key[AES_BLOCK_SIZE]; 629 int err; 630 631 err = cbcmac_setkey(tfm, in_key, key_len); 632 if (err) 633 return err; 634 635 kernel_neon_begin(); 636 aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1); 637 aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2); 638 kernel_neon_end(); 639 640 return cbcmac_setkey(tfm, key, sizeof(key)); 641 } 642 643 static int mac_init(struct shash_desc *desc) 644 { 645 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 646 647 memset(ctx->dg, 0, AES_BLOCK_SIZE); 648 ctx->len = 0; 649 650 return 0; 651 } 652 653 static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, 654 u8 dg[], int enc_before, int enc_after) 655 { 656 int rounds = 6 + ctx->key_length / 4; 657 658 if (crypto_simd_usable()) { 659 kernel_neon_begin(); 660 aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before, 661 enc_after); 662 kernel_neon_end(); 663 } else { 664 if (enc_before) 665 aes_encrypt(ctx, dg, dg); 666 667 while (blocks--) { 668 crypto_xor(dg, in, AES_BLOCK_SIZE); 669 in += AES_BLOCK_SIZE; 670 671 if (blocks || enc_after) 672 aes_encrypt(ctx, dg, dg); 673 } 674 } 675 } 676 677 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) 678 { 679 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 680 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 681 682 while (len > 0) { 683 unsigned int l; 684 685 if ((ctx->len % AES_BLOCK_SIZE) == 0 && 686 (ctx->len + len) > AES_BLOCK_SIZE) { 687 688 int blocks = len / AES_BLOCK_SIZE; 689 690 len %= AES_BLOCK_SIZE; 691 692 mac_do_update(&tctx->key, p, blocks, ctx->dg, 693 (ctx->len != 0), (len != 0)); 694 695 p += blocks * AES_BLOCK_SIZE; 696 697 if (!len) { 698 ctx->len = AES_BLOCK_SIZE; 699 break; 700 } 701 ctx->len = 0; 702 } 703 704 l = min(len, AES_BLOCK_SIZE - ctx->len); 705 706 if (l <= AES_BLOCK_SIZE) { 707 crypto_xor(ctx->dg + ctx->len, p, l); 708 ctx->len += l; 709 len -= l; 710 p += l; 711 } 712 } 713 714 return 0; 715 } 716 717 static int cbcmac_final(struct shash_desc *desc, u8 *out) 718 { 719 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 720 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 721 722 mac_do_update(&tctx->key, NULL, 0, ctx->dg, (ctx->len != 0), 0); 723 724 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 725 726 return 0; 727 } 728 729 static int cmac_final(struct shash_desc *desc, u8 *out) 730 { 731 struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 732 struct mac_desc_ctx *ctx = shash_desc_ctx(desc); 733 u8 *consts = tctx->consts; 734 735 if (ctx->len != AES_BLOCK_SIZE) { 736 ctx->dg[ctx->len] ^= 0x80; 737 consts += AES_BLOCK_SIZE; 738 } 739 740 mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1); 741 742 memcpy(out, ctx->dg, AES_BLOCK_SIZE); 743 744 return 0; 745 } 746 747 static struct shash_alg mac_algs[] = { { 748 .base.cra_name = "cmac(aes)", 749 .base.cra_driver_name = "cmac-aes-" MODE, 750 .base.cra_priority = PRIO, 751 .base.cra_blocksize = AES_BLOCK_SIZE, 752 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 753 2 * AES_BLOCK_SIZE, 754 .base.cra_module = THIS_MODULE, 755 756 .digestsize = AES_BLOCK_SIZE, 757 .init = mac_init, 758 .update = mac_update, 759 .final = cmac_final, 760 .setkey = cmac_setkey, 761 .descsize = sizeof(struct mac_desc_ctx), 762 }, { 763 .base.cra_name = "xcbc(aes)", 764 .base.cra_driver_name = "xcbc-aes-" MODE, 765 .base.cra_priority = PRIO, 766 .base.cra_blocksize = AES_BLOCK_SIZE, 767 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + 768 2 * AES_BLOCK_SIZE, 769 .base.cra_module = THIS_MODULE, 770 771 .digestsize = AES_BLOCK_SIZE, 772 .init = mac_init, 773 .update = mac_update, 774 .final = cmac_final, 775 .setkey = xcbc_setkey, 776 .descsize = sizeof(struct mac_desc_ctx), 777 }, { 778 .base.cra_name = "cbcmac(aes)", 779 .base.cra_driver_name = "cbcmac-aes-" MODE, 780 .base.cra_priority = PRIO, 781 .base.cra_blocksize = 1, 782 .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), 783 .base.cra_module = THIS_MODULE, 784 785 .digestsize = AES_BLOCK_SIZE, 786 .init = mac_init, 787 .update = mac_update, 788 .final = cbcmac_final, 789 .setkey = cbcmac_setkey, 790 .descsize = sizeof(struct mac_desc_ctx), 791 } }; 792 793 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 794 795 static void aes_exit(void) 796 { 797 int i; 798 799 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) 800 if (aes_simd_algs[i]) 801 simd_skcipher_free(aes_simd_algs[i]); 802 803 crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 804 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 805 } 806 807 static int __init aes_init(void) 808 { 809 struct simd_skcipher_alg *simd; 810 const char *basename; 811 const char *algname; 812 const char *drvname; 813 int err; 814 int i; 815 816 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 817 if (err) 818 return err; 819 820 err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); 821 if (err) 822 goto unregister_ciphers; 823 824 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 825 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) 826 continue; 827 828 algname = aes_algs[i].base.cra_name + 2; 829 drvname = aes_algs[i].base.cra_driver_name + 2; 830 basename = aes_algs[i].base.cra_driver_name; 831 simd = simd_skcipher_create_compat(algname, drvname, basename); 832 err = PTR_ERR(simd); 833 if (IS_ERR(simd)) 834 goto unregister_simds; 835 836 aes_simd_algs[i] = simd; 837 } 838 839 return 0; 840 841 unregister_simds: 842 aes_exit(); 843 return err; 844 unregister_ciphers: 845 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 846 return err; 847 } 848 849 #ifdef USE_V8_CRYPTO_EXTENSIONS 850 module_cpu_feature_match(AES, aes_init); 851 #else 852 module_init(aes_init); 853 EXPORT_SYMBOL(neon_aes_ecb_encrypt); 854 EXPORT_SYMBOL(neon_aes_cbc_encrypt); 855 #endif 856 module_exit(aes_exit); 857