1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * aes-ce-glue.c - wrapper code for ARMv8 AES 4 * 5 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/hwcap.h> 9 #include <asm/neon.h> 10 #include <asm/simd.h> 11 #include <asm/unaligned.h> 12 #include <crypto/aes.h> 13 #include <crypto/ctr.h> 14 #include <crypto/internal/simd.h> 15 #include <crypto/internal/skcipher.h> 16 #include <crypto/scatterwalk.h> 17 #include <linux/cpufeature.h> 18 #include <linux/module.h> 19 #include <crypto/xts.h> 20 21 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 22 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 23 MODULE_LICENSE("GPL v2"); 24 25 /* defined in aes-ce-core.S */ 26 asmlinkage u32 ce_aes_sub(u32 input); 27 asmlinkage void ce_aes_invert(void *dst, void *src); 28 29 asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], 30 int rounds, int blocks); 31 asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], 32 int rounds, int blocks); 33 34 asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], 35 int rounds, int blocks, u8 iv[]); 36 asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], 37 int rounds, int blocks, u8 iv[]); 38 asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], 39 int rounds, int bytes, u8 const iv[]); 40 asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], 41 int rounds, int bytes, u8 const iv[]); 42 43 asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], 44 int rounds, int blocks, u8 ctr[]); 45 46 asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], 47 int rounds, int bytes, u8 iv[], 48 u32 const rk2[], int first); 49 asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], 50 int rounds, int bytes, u8 iv[], 51 u32 const rk2[], int first); 52 53 struct aes_block { 54 u8 b[AES_BLOCK_SIZE]; 55 }; 56 57 static int num_rounds(struct crypto_aes_ctx *ctx) 58 { 59 /* 60 * # of rounds specified by AES: 61 * 128 bit key 10 rounds 62 * 192 bit key 12 rounds 63 * 256 bit key 14 rounds 64 * => n byte key => 6 + (n/4) rounds 65 */ 66 return 6 + ctx->key_length / 4; 67 } 68 69 static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 70 unsigned int key_len) 71 { 72 /* 73 * The AES key schedule round constants 74 */ 75 static u8 const rcon[] = { 76 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 77 }; 78 79 u32 kwords = key_len / sizeof(u32); 80 struct aes_block *key_enc, *key_dec; 81 int i, j; 82 83 if (key_len != AES_KEYSIZE_128 && 84 key_len != AES_KEYSIZE_192 && 85 key_len != AES_KEYSIZE_256) 86 return -EINVAL; 87 88 ctx->key_length = key_len; 89 for (i = 0; i < kwords; i++) 90 ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32)); 91 92 kernel_neon_begin(); 93 for (i = 0; i < sizeof(rcon); i++) { 94 u32 *rki = ctx->key_enc + (i * kwords); 95 u32 *rko = rki + kwords; 96 97 rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8); 98 rko[0] = rko[0] ^ rki[0] ^ rcon[i]; 99 rko[1] = rko[0] ^ rki[1]; 100 rko[2] = rko[1] ^ rki[2]; 101 rko[3] = rko[2] ^ rki[3]; 102 103 if (key_len == AES_KEYSIZE_192) { 104 if (i >= 7) 105 break; 106 rko[4] = rko[3] ^ rki[4]; 107 rko[5] = rko[4] ^ rki[5]; 108 } else if (key_len == AES_KEYSIZE_256) { 109 if (i >= 6) 110 break; 111 rko[4] = ce_aes_sub(rko[3]) ^ rki[4]; 112 rko[5] = rko[4] ^ rki[5]; 113 rko[6] = rko[5] ^ rki[6]; 114 rko[7] = rko[6] ^ rki[7]; 115 } 116 } 117 118 /* 119 * Generate the decryption keys for the Equivalent Inverse Cipher. 120 * This involves reversing the order of the round keys, and applying 121 * the Inverse Mix Columns transformation on all but the first and 122 * the last one. 123 */ 124 key_enc = (struct aes_block *)ctx->key_enc; 125 key_dec = (struct aes_block *)ctx->key_dec; 126 j = num_rounds(ctx); 127 128 key_dec[0] = key_enc[j]; 129 for (i = 1, j--; j > 0; i++, j--) 130 ce_aes_invert(key_dec + i, key_enc + j); 131 key_dec[i] = key_enc[0]; 132 133 kernel_neon_end(); 134 return 0; 135 } 136 137 static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 138 unsigned int key_len) 139 { 140 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 141 142 return ce_aes_expandkey(ctx, in_key, key_len); 143 } 144 145 struct crypto_aes_xts_ctx { 146 struct crypto_aes_ctx key1; 147 struct crypto_aes_ctx __aligned(8) key2; 148 }; 149 150 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 151 unsigned int key_len) 152 { 153 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 154 int ret; 155 156 ret = xts_verify_key(tfm, in_key, key_len); 157 if (ret) 158 return ret; 159 160 ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2); 161 if (!ret) 162 ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2], 163 key_len / 2); 164 return ret; 165 } 166 167 static int ecb_encrypt(struct skcipher_request *req) 168 { 169 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 170 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 171 struct skcipher_walk walk; 172 unsigned int blocks; 173 int err; 174 175 err = skcipher_walk_virt(&walk, req, false); 176 177 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 178 kernel_neon_begin(); 179 ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 180 ctx->key_enc, num_rounds(ctx), blocks); 181 kernel_neon_end(); 182 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 183 } 184 return err; 185 } 186 187 static int ecb_decrypt(struct skcipher_request *req) 188 { 189 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 190 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 191 struct skcipher_walk walk; 192 unsigned int blocks; 193 int err; 194 195 err = skcipher_walk_virt(&walk, req, false); 196 197 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 198 kernel_neon_begin(); 199 ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 200 ctx->key_dec, num_rounds(ctx), blocks); 201 kernel_neon_end(); 202 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 203 } 204 return err; 205 } 206 207 static int cbc_encrypt_walk(struct skcipher_request *req, 208 struct skcipher_walk *walk) 209 { 210 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 211 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 212 unsigned int blocks; 213 int err = 0; 214 215 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { 216 kernel_neon_begin(); 217 ce_aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr, 218 ctx->key_enc, num_rounds(ctx), blocks, 219 walk->iv); 220 kernel_neon_end(); 221 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); 222 } 223 return err; 224 } 225 226 static int cbc_encrypt(struct skcipher_request *req) 227 { 228 struct skcipher_walk walk; 229 int err; 230 231 err = skcipher_walk_virt(&walk, req, false); 232 if (err) 233 return err; 234 return cbc_encrypt_walk(req, &walk); 235 } 236 237 static int cbc_decrypt_walk(struct skcipher_request *req, 238 struct skcipher_walk *walk) 239 { 240 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 241 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 242 unsigned int blocks; 243 int err = 0; 244 245 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) { 246 kernel_neon_begin(); 247 ce_aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr, 248 ctx->key_dec, num_rounds(ctx), blocks, 249 walk->iv); 250 kernel_neon_end(); 251 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE); 252 } 253 return err; 254 } 255 256 static int cbc_decrypt(struct skcipher_request *req) 257 { 258 struct skcipher_walk walk; 259 int err; 260 261 err = skcipher_walk_virt(&walk, req, false); 262 if (err) 263 return err; 264 return cbc_decrypt_walk(req, &walk); 265 } 266 267 static int cts_cbc_encrypt(struct skcipher_request *req) 268 { 269 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 270 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 271 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 272 struct scatterlist *src = req->src, *dst = req->dst; 273 struct scatterlist sg_src[2], sg_dst[2]; 274 struct skcipher_request subreq; 275 struct skcipher_walk walk; 276 int err; 277 278 skcipher_request_set_tfm(&subreq, tfm); 279 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 280 NULL, NULL); 281 282 if (req->cryptlen <= AES_BLOCK_SIZE) { 283 if (req->cryptlen < AES_BLOCK_SIZE) 284 return -EINVAL; 285 cbc_blocks = 1; 286 } 287 288 if (cbc_blocks > 0) { 289 skcipher_request_set_crypt(&subreq, req->src, req->dst, 290 cbc_blocks * AES_BLOCK_SIZE, 291 req->iv); 292 293 err = skcipher_walk_virt(&walk, &subreq, false) ?: 294 cbc_encrypt_walk(&subreq, &walk); 295 if (err) 296 return err; 297 298 if (req->cryptlen == AES_BLOCK_SIZE) 299 return 0; 300 301 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 302 if (req->dst != req->src) 303 dst = scatterwalk_ffwd(sg_dst, req->dst, 304 subreq.cryptlen); 305 } 306 307 /* handle ciphertext stealing */ 308 skcipher_request_set_crypt(&subreq, src, dst, 309 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 310 req->iv); 311 312 err = skcipher_walk_virt(&walk, &subreq, false); 313 if (err) 314 return err; 315 316 kernel_neon_begin(); 317 ce_aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 318 ctx->key_enc, num_rounds(ctx), walk.nbytes, 319 walk.iv); 320 kernel_neon_end(); 321 322 return skcipher_walk_done(&walk, 0); 323 } 324 325 static int cts_cbc_decrypt(struct skcipher_request *req) 326 { 327 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 328 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 329 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 330 struct scatterlist *src = req->src, *dst = req->dst; 331 struct scatterlist sg_src[2], sg_dst[2]; 332 struct skcipher_request subreq; 333 struct skcipher_walk walk; 334 int err; 335 336 skcipher_request_set_tfm(&subreq, tfm); 337 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 338 NULL, NULL); 339 340 if (req->cryptlen <= AES_BLOCK_SIZE) { 341 if (req->cryptlen < AES_BLOCK_SIZE) 342 return -EINVAL; 343 cbc_blocks = 1; 344 } 345 346 if (cbc_blocks > 0) { 347 skcipher_request_set_crypt(&subreq, req->src, req->dst, 348 cbc_blocks * AES_BLOCK_SIZE, 349 req->iv); 350 351 err = skcipher_walk_virt(&walk, &subreq, false) ?: 352 cbc_decrypt_walk(&subreq, &walk); 353 if (err) 354 return err; 355 356 if (req->cryptlen == AES_BLOCK_SIZE) 357 return 0; 358 359 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 360 if (req->dst != req->src) 361 dst = scatterwalk_ffwd(sg_dst, req->dst, 362 subreq.cryptlen); 363 } 364 365 /* handle ciphertext stealing */ 366 skcipher_request_set_crypt(&subreq, src, dst, 367 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 368 req->iv); 369 370 err = skcipher_walk_virt(&walk, &subreq, false); 371 if (err) 372 return err; 373 374 kernel_neon_begin(); 375 ce_aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 376 ctx->key_dec, num_rounds(ctx), walk.nbytes, 377 walk.iv); 378 kernel_neon_end(); 379 380 return skcipher_walk_done(&walk, 0); 381 } 382 383 static int ctr_encrypt(struct skcipher_request *req) 384 { 385 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 386 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 387 struct skcipher_walk walk; 388 int err, blocks; 389 390 err = skcipher_walk_virt(&walk, req, false); 391 392 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 393 kernel_neon_begin(); 394 ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 395 ctx->key_enc, num_rounds(ctx), blocks, 396 walk.iv); 397 kernel_neon_end(); 398 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 399 } 400 if (walk.nbytes) { 401 u8 __aligned(8) tail[AES_BLOCK_SIZE]; 402 unsigned int nbytes = walk.nbytes; 403 u8 *tdst = walk.dst.virt.addr; 404 u8 *tsrc = walk.src.virt.addr; 405 406 /* 407 * Tell aes_ctr_encrypt() to process a tail block. 408 */ 409 blocks = -1; 410 411 kernel_neon_begin(); 412 ce_aes_ctr_encrypt(tail, NULL, ctx->key_enc, num_rounds(ctx), 413 blocks, walk.iv); 414 kernel_neon_end(); 415 crypto_xor_cpy(tdst, tsrc, tail, nbytes); 416 err = skcipher_walk_done(&walk, 0); 417 } 418 return err; 419 } 420 421 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) 422 { 423 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 424 unsigned long flags; 425 426 /* 427 * Temporarily disable interrupts to avoid races where 428 * cachelines are evicted when the CPU is interrupted 429 * to do something else. 430 */ 431 local_irq_save(flags); 432 aes_encrypt(ctx, dst, src); 433 local_irq_restore(flags); 434 } 435 436 static int ctr_encrypt_sync(struct skcipher_request *req) 437 { 438 if (!crypto_simd_usable()) 439 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); 440 441 return ctr_encrypt(req); 442 } 443 444 static int xts_encrypt(struct skcipher_request *req) 445 { 446 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 447 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 448 int err, first, rounds = num_rounds(&ctx->key1); 449 int tail = req->cryptlen % AES_BLOCK_SIZE; 450 struct scatterlist sg_src[2], sg_dst[2]; 451 struct skcipher_request subreq; 452 struct scatterlist *src, *dst; 453 struct skcipher_walk walk; 454 455 if (req->cryptlen < AES_BLOCK_SIZE) 456 return -EINVAL; 457 458 err = skcipher_walk_virt(&walk, req, false); 459 460 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 461 int xts_blocks = DIV_ROUND_UP(req->cryptlen, 462 AES_BLOCK_SIZE) - 2; 463 464 skcipher_walk_abort(&walk); 465 466 skcipher_request_set_tfm(&subreq, tfm); 467 skcipher_request_set_callback(&subreq, 468 skcipher_request_flags(req), 469 NULL, NULL); 470 skcipher_request_set_crypt(&subreq, req->src, req->dst, 471 xts_blocks * AES_BLOCK_SIZE, 472 req->iv); 473 req = &subreq; 474 err = skcipher_walk_virt(&walk, req, false); 475 } else { 476 tail = 0; 477 } 478 479 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) { 480 int nbytes = walk.nbytes; 481 482 if (walk.nbytes < walk.total) 483 nbytes &= ~(AES_BLOCK_SIZE - 1); 484 485 kernel_neon_begin(); 486 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 487 ctx->key1.key_enc, rounds, nbytes, walk.iv, 488 ctx->key2.key_enc, first); 489 kernel_neon_end(); 490 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 491 } 492 493 if (err || likely(!tail)) 494 return err; 495 496 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 497 if (req->dst != req->src) 498 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 499 500 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 501 req->iv); 502 503 err = skcipher_walk_virt(&walk, req, false); 504 if (err) 505 return err; 506 507 kernel_neon_begin(); 508 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 509 ctx->key1.key_enc, rounds, walk.nbytes, walk.iv, 510 ctx->key2.key_enc, first); 511 kernel_neon_end(); 512 513 return skcipher_walk_done(&walk, 0); 514 } 515 516 static int xts_decrypt(struct skcipher_request *req) 517 { 518 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 519 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 520 int err, first, rounds = num_rounds(&ctx->key1); 521 int tail = req->cryptlen % AES_BLOCK_SIZE; 522 struct scatterlist sg_src[2], sg_dst[2]; 523 struct skcipher_request subreq; 524 struct scatterlist *src, *dst; 525 struct skcipher_walk walk; 526 527 if (req->cryptlen < AES_BLOCK_SIZE) 528 return -EINVAL; 529 530 err = skcipher_walk_virt(&walk, req, false); 531 532 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 533 int xts_blocks = DIV_ROUND_UP(req->cryptlen, 534 AES_BLOCK_SIZE) - 2; 535 536 skcipher_walk_abort(&walk); 537 538 skcipher_request_set_tfm(&subreq, tfm); 539 skcipher_request_set_callback(&subreq, 540 skcipher_request_flags(req), 541 NULL, NULL); 542 skcipher_request_set_crypt(&subreq, req->src, req->dst, 543 xts_blocks * AES_BLOCK_SIZE, 544 req->iv); 545 req = &subreq; 546 err = skcipher_walk_virt(&walk, req, false); 547 } else { 548 tail = 0; 549 } 550 551 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) { 552 int nbytes = walk.nbytes; 553 554 if (walk.nbytes < walk.total) 555 nbytes &= ~(AES_BLOCK_SIZE - 1); 556 557 kernel_neon_begin(); 558 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 559 ctx->key1.key_dec, rounds, nbytes, walk.iv, 560 ctx->key2.key_enc, first); 561 kernel_neon_end(); 562 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 563 } 564 565 if (err || likely(!tail)) 566 return err; 567 568 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 569 if (req->dst != req->src) 570 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 571 572 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 573 req->iv); 574 575 err = skcipher_walk_virt(&walk, req, false); 576 if (err) 577 return err; 578 579 kernel_neon_begin(); 580 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 581 ctx->key1.key_dec, rounds, walk.nbytes, walk.iv, 582 ctx->key2.key_enc, first); 583 kernel_neon_end(); 584 585 return skcipher_walk_done(&walk, 0); 586 } 587 588 static struct skcipher_alg aes_algs[] = { { 589 .base.cra_name = "__ecb(aes)", 590 .base.cra_driver_name = "__ecb-aes-ce", 591 .base.cra_priority = 300, 592 .base.cra_flags = CRYPTO_ALG_INTERNAL, 593 .base.cra_blocksize = AES_BLOCK_SIZE, 594 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), 595 .base.cra_module = THIS_MODULE, 596 597 .min_keysize = AES_MIN_KEY_SIZE, 598 .max_keysize = AES_MAX_KEY_SIZE, 599 .setkey = ce_aes_setkey, 600 .encrypt = ecb_encrypt, 601 .decrypt = ecb_decrypt, 602 }, { 603 .base.cra_name = "__cbc(aes)", 604 .base.cra_driver_name = "__cbc-aes-ce", 605 .base.cra_priority = 300, 606 .base.cra_flags = CRYPTO_ALG_INTERNAL, 607 .base.cra_blocksize = AES_BLOCK_SIZE, 608 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), 609 .base.cra_module = THIS_MODULE, 610 611 .min_keysize = AES_MIN_KEY_SIZE, 612 .max_keysize = AES_MAX_KEY_SIZE, 613 .ivsize = AES_BLOCK_SIZE, 614 .setkey = ce_aes_setkey, 615 .encrypt = cbc_encrypt, 616 .decrypt = cbc_decrypt, 617 }, { 618 .base.cra_name = "__cts(cbc(aes))", 619 .base.cra_driver_name = "__cts-cbc-aes-ce", 620 .base.cra_priority = 300, 621 .base.cra_flags = CRYPTO_ALG_INTERNAL, 622 .base.cra_blocksize = AES_BLOCK_SIZE, 623 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), 624 .base.cra_module = THIS_MODULE, 625 626 .min_keysize = AES_MIN_KEY_SIZE, 627 .max_keysize = AES_MAX_KEY_SIZE, 628 .ivsize = AES_BLOCK_SIZE, 629 .walksize = 2 * AES_BLOCK_SIZE, 630 .setkey = ce_aes_setkey, 631 .encrypt = cts_cbc_encrypt, 632 .decrypt = cts_cbc_decrypt, 633 }, { 634 .base.cra_name = "__ctr(aes)", 635 .base.cra_driver_name = "__ctr-aes-ce", 636 .base.cra_priority = 300, 637 .base.cra_flags = CRYPTO_ALG_INTERNAL, 638 .base.cra_blocksize = 1, 639 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), 640 .base.cra_module = THIS_MODULE, 641 642 .min_keysize = AES_MIN_KEY_SIZE, 643 .max_keysize = AES_MAX_KEY_SIZE, 644 .ivsize = AES_BLOCK_SIZE, 645 .chunksize = AES_BLOCK_SIZE, 646 .setkey = ce_aes_setkey, 647 .encrypt = ctr_encrypt, 648 .decrypt = ctr_encrypt, 649 }, { 650 .base.cra_name = "ctr(aes)", 651 .base.cra_driver_name = "ctr-aes-ce-sync", 652 .base.cra_priority = 300 - 1, 653 .base.cra_blocksize = 1, 654 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx), 655 .base.cra_module = THIS_MODULE, 656 657 .min_keysize = AES_MIN_KEY_SIZE, 658 .max_keysize = AES_MAX_KEY_SIZE, 659 .ivsize = AES_BLOCK_SIZE, 660 .chunksize = AES_BLOCK_SIZE, 661 .setkey = ce_aes_setkey, 662 .encrypt = ctr_encrypt_sync, 663 .decrypt = ctr_encrypt_sync, 664 }, { 665 .base.cra_name = "__xts(aes)", 666 .base.cra_driver_name = "__xts-aes-ce", 667 .base.cra_priority = 300, 668 .base.cra_flags = CRYPTO_ALG_INTERNAL, 669 .base.cra_blocksize = AES_BLOCK_SIZE, 670 .base.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 671 .base.cra_module = THIS_MODULE, 672 673 .min_keysize = 2 * AES_MIN_KEY_SIZE, 674 .max_keysize = 2 * AES_MAX_KEY_SIZE, 675 .ivsize = AES_BLOCK_SIZE, 676 .walksize = 2 * AES_BLOCK_SIZE, 677 .setkey = xts_set_key, 678 .encrypt = xts_encrypt, 679 .decrypt = xts_decrypt, 680 } }; 681 682 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 683 684 static void aes_exit(void) 685 { 686 int i; 687 688 for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++) 689 simd_skcipher_free(aes_simd_algs[i]); 690 691 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 692 } 693 694 static int __init aes_init(void) 695 { 696 struct simd_skcipher_alg *simd; 697 const char *basename; 698 const char *algname; 699 const char *drvname; 700 int err; 701 int i; 702 703 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 704 if (err) 705 return err; 706 707 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 708 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) 709 continue; 710 711 algname = aes_algs[i].base.cra_name + 2; 712 drvname = aes_algs[i].base.cra_driver_name + 2; 713 basename = aes_algs[i].base.cra_driver_name; 714 simd = simd_skcipher_create_compat(aes_algs + i, algname, drvname, basename); 715 err = PTR_ERR(simd); 716 if (IS_ERR(simd)) 717 goto unregister_simds; 718 719 aes_simd_algs[i] = simd; 720 } 721 722 return 0; 723 724 unregister_simds: 725 aes_exit(); 726 return err; 727 } 728 729 module_cpu_feature_match(AES, aes_init); 730 module_exit(aes_exit); 731