1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Bit sliced AES using NEON instructions 4 * 5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/neon.h> 9 #include <asm/simd.h> 10 #include <crypto/aes.h> 11 #include <crypto/ctr.h> 12 #include <crypto/internal/simd.h> 13 #include <crypto/internal/skcipher.h> 14 #include <crypto/xts.h> 15 #include <linux/module.h> 16 17 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 18 MODULE_LICENSE("GPL v2"); 19 20 MODULE_ALIAS_CRYPTO("ecb(aes)"); 21 MODULE_ALIAS_CRYPTO("cbc(aes)"); 22 MODULE_ALIAS_CRYPTO("ctr(aes)"); 23 MODULE_ALIAS_CRYPTO("xts(aes)"); 24 25 asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds); 26 27 asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], 28 int rounds, int blocks); 29 asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], 30 int rounds, int blocks); 31 32 asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], 33 int rounds, int blocks, u8 iv[]); 34 35 asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], 36 int rounds, int blocks, u8 iv[], u8 final[]); 37 38 asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], 39 int rounds, int blocks, u8 iv[]); 40 asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], 41 int rounds, int blocks, u8 iv[]); 42 43 /* borrowed from aes-neon-blk.ko */ 44 asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], 45 int rounds, int blocks); 46 asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], 47 int rounds, int blocks, u8 iv[]); 48 49 struct aesbs_ctx { 50 u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32]; 51 int rounds; 52 } __aligned(AES_BLOCK_SIZE); 53 54 struct aesbs_cbc_ctx { 55 struct aesbs_ctx key; 56 u32 enc[AES_MAX_KEYLENGTH_U32]; 57 }; 58 59 struct aesbs_ctr_ctx { 60 struct aesbs_ctx key; /* must be first member */ 61 struct crypto_aes_ctx fallback; 62 }; 63 64 struct aesbs_xts_ctx { 65 struct aesbs_ctx key; 66 u32 twkey[AES_MAX_KEYLENGTH_U32]; 67 }; 68 69 static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 70 unsigned int key_len) 71 { 72 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 73 struct crypto_aes_ctx rk; 74 int err; 75 76 err = aes_expandkey(&rk, in_key, key_len); 77 if (err) 78 return err; 79 80 ctx->rounds = 6 + key_len / 4; 81 82 kernel_neon_begin(); 83 aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds); 84 kernel_neon_end(); 85 86 return 0; 87 } 88 89 static int __ecb_crypt(struct skcipher_request *req, 90 void (*fn)(u8 out[], u8 const in[], u8 const rk[], 91 int rounds, int blocks)) 92 { 93 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 94 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 95 struct skcipher_walk walk; 96 int err; 97 98 err = skcipher_walk_virt(&walk, req, false); 99 100 while (walk.nbytes >= AES_BLOCK_SIZE) { 101 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 102 103 if (walk.nbytes < walk.total) 104 blocks = round_down(blocks, 105 walk.stride / AES_BLOCK_SIZE); 106 107 kernel_neon_begin(); 108 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk, 109 ctx->rounds, blocks); 110 kernel_neon_end(); 111 err = skcipher_walk_done(&walk, 112 walk.nbytes - blocks * AES_BLOCK_SIZE); 113 } 114 115 return err; 116 } 117 118 static int ecb_encrypt(struct skcipher_request *req) 119 { 120 return __ecb_crypt(req, aesbs_ecb_encrypt); 121 } 122 123 static int ecb_decrypt(struct skcipher_request *req) 124 { 125 return __ecb_crypt(req, aesbs_ecb_decrypt); 126 } 127 128 static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 129 unsigned int key_len) 130 { 131 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 132 struct crypto_aes_ctx rk; 133 int err; 134 135 err = aes_expandkey(&rk, in_key, key_len); 136 if (err) 137 return err; 138 139 ctx->key.rounds = 6 + key_len / 4; 140 141 memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc)); 142 143 kernel_neon_begin(); 144 aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds); 145 kernel_neon_end(); 146 147 return 0; 148 } 149 150 static int cbc_encrypt(struct skcipher_request *req) 151 { 152 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 153 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 154 struct skcipher_walk walk; 155 int err; 156 157 err = skcipher_walk_virt(&walk, req, false); 158 159 while (walk.nbytes >= AES_BLOCK_SIZE) { 160 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 161 162 /* fall back to the non-bitsliced NEON implementation */ 163 kernel_neon_begin(); 164 neon_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 165 ctx->enc, ctx->key.rounds, blocks, 166 walk.iv); 167 kernel_neon_end(); 168 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 169 } 170 return err; 171 } 172 173 static int cbc_decrypt(struct skcipher_request *req) 174 { 175 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 176 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 177 struct skcipher_walk walk; 178 int err; 179 180 err = skcipher_walk_virt(&walk, req, false); 181 182 while (walk.nbytes >= AES_BLOCK_SIZE) { 183 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 184 185 if (walk.nbytes < walk.total) 186 blocks = round_down(blocks, 187 walk.stride / AES_BLOCK_SIZE); 188 189 kernel_neon_begin(); 190 aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 191 ctx->key.rk, ctx->key.rounds, blocks, 192 walk.iv); 193 kernel_neon_end(); 194 err = skcipher_walk_done(&walk, 195 walk.nbytes - blocks * AES_BLOCK_SIZE); 196 } 197 198 return err; 199 } 200 201 static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key, 202 unsigned int key_len) 203 { 204 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); 205 int err; 206 207 err = aes_expandkey(&ctx->fallback, in_key, key_len); 208 if (err) 209 return err; 210 211 ctx->key.rounds = 6 + key_len / 4; 212 213 kernel_neon_begin(); 214 aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds); 215 kernel_neon_end(); 216 217 return 0; 218 } 219 220 static int ctr_encrypt(struct skcipher_request *req) 221 { 222 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 223 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 224 struct skcipher_walk walk; 225 u8 buf[AES_BLOCK_SIZE]; 226 int err; 227 228 err = skcipher_walk_virt(&walk, req, false); 229 230 while (walk.nbytes > 0) { 231 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 232 u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL; 233 234 if (walk.nbytes < walk.total) { 235 blocks = round_down(blocks, 236 walk.stride / AES_BLOCK_SIZE); 237 final = NULL; 238 } 239 240 kernel_neon_begin(); 241 aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 242 ctx->rk, ctx->rounds, blocks, walk.iv, final); 243 kernel_neon_end(); 244 245 if (final) { 246 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; 247 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; 248 249 crypto_xor_cpy(dst, src, final, 250 walk.total % AES_BLOCK_SIZE); 251 252 err = skcipher_walk_done(&walk, 0); 253 break; 254 } 255 err = skcipher_walk_done(&walk, 256 walk.nbytes - blocks * AES_BLOCK_SIZE); 257 } 258 return err; 259 } 260 261 static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 262 unsigned int key_len) 263 { 264 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 265 struct crypto_aes_ctx rk; 266 int err; 267 268 err = xts_verify_key(tfm, in_key, key_len); 269 if (err) 270 return err; 271 272 key_len /= 2; 273 err = aes_expandkey(&rk, in_key + key_len, key_len); 274 if (err) 275 return err; 276 277 memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey)); 278 279 return aesbs_setkey(tfm, in_key, key_len); 280 } 281 282 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) 283 { 284 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); 285 unsigned long flags; 286 287 /* 288 * Temporarily disable interrupts to avoid races where 289 * cachelines are evicted when the CPU is interrupted 290 * to do something else. 291 */ 292 local_irq_save(flags); 293 aes_encrypt(&ctx->fallback, dst, src); 294 local_irq_restore(flags); 295 } 296 297 static int ctr_encrypt_sync(struct skcipher_request *req) 298 { 299 if (!crypto_simd_usable()) 300 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); 301 302 return ctr_encrypt(req); 303 } 304 305 static int __xts_crypt(struct skcipher_request *req, 306 void (*fn)(u8 out[], u8 const in[], u8 const rk[], 307 int rounds, int blocks, u8 iv[])) 308 { 309 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 310 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 311 struct skcipher_walk walk; 312 int err; 313 314 err = skcipher_walk_virt(&walk, req, false); 315 if (err) 316 return err; 317 318 kernel_neon_begin(); 319 neon_aes_ecb_encrypt(walk.iv, walk.iv, ctx->twkey, ctx->key.rounds, 1); 320 kernel_neon_end(); 321 322 while (walk.nbytes >= AES_BLOCK_SIZE) { 323 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 324 325 if (walk.nbytes < walk.total) 326 blocks = round_down(blocks, 327 walk.stride / AES_BLOCK_SIZE); 328 329 kernel_neon_begin(); 330 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk, 331 ctx->key.rounds, blocks, walk.iv); 332 kernel_neon_end(); 333 err = skcipher_walk_done(&walk, 334 walk.nbytes - blocks * AES_BLOCK_SIZE); 335 } 336 return err; 337 } 338 339 static int xts_encrypt(struct skcipher_request *req) 340 { 341 return __xts_crypt(req, aesbs_xts_encrypt); 342 } 343 344 static int xts_decrypt(struct skcipher_request *req) 345 { 346 return __xts_crypt(req, aesbs_xts_decrypt); 347 } 348 349 static struct skcipher_alg aes_algs[] = { { 350 .base.cra_name = "__ecb(aes)", 351 .base.cra_driver_name = "__ecb-aes-neonbs", 352 .base.cra_priority = 250, 353 .base.cra_blocksize = AES_BLOCK_SIZE, 354 .base.cra_ctxsize = sizeof(struct aesbs_ctx), 355 .base.cra_module = THIS_MODULE, 356 .base.cra_flags = CRYPTO_ALG_INTERNAL, 357 358 .min_keysize = AES_MIN_KEY_SIZE, 359 .max_keysize = AES_MAX_KEY_SIZE, 360 .walksize = 8 * AES_BLOCK_SIZE, 361 .setkey = aesbs_setkey, 362 .encrypt = ecb_encrypt, 363 .decrypt = ecb_decrypt, 364 }, { 365 .base.cra_name = "__cbc(aes)", 366 .base.cra_driver_name = "__cbc-aes-neonbs", 367 .base.cra_priority = 250, 368 .base.cra_blocksize = AES_BLOCK_SIZE, 369 .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), 370 .base.cra_module = THIS_MODULE, 371 .base.cra_flags = CRYPTO_ALG_INTERNAL, 372 373 .min_keysize = AES_MIN_KEY_SIZE, 374 .max_keysize = AES_MAX_KEY_SIZE, 375 .walksize = 8 * AES_BLOCK_SIZE, 376 .ivsize = AES_BLOCK_SIZE, 377 .setkey = aesbs_cbc_setkey, 378 .encrypt = cbc_encrypt, 379 .decrypt = cbc_decrypt, 380 }, { 381 .base.cra_name = "__ctr(aes)", 382 .base.cra_driver_name = "__ctr-aes-neonbs", 383 .base.cra_priority = 250, 384 .base.cra_blocksize = 1, 385 .base.cra_ctxsize = sizeof(struct aesbs_ctx), 386 .base.cra_module = THIS_MODULE, 387 .base.cra_flags = CRYPTO_ALG_INTERNAL, 388 389 .min_keysize = AES_MIN_KEY_SIZE, 390 .max_keysize = AES_MAX_KEY_SIZE, 391 .chunksize = AES_BLOCK_SIZE, 392 .walksize = 8 * AES_BLOCK_SIZE, 393 .ivsize = AES_BLOCK_SIZE, 394 .setkey = aesbs_setkey, 395 .encrypt = ctr_encrypt, 396 .decrypt = ctr_encrypt, 397 }, { 398 .base.cra_name = "ctr(aes)", 399 .base.cra_driver_name = "ctr-aes-neonbs", 400 .base.cra_priority = 250 - 1, 401 .base.cra_blocksize = 1, 402 .base.cra_ctxsize = sizeof(struct aesbs_ctr_ctx), 403 .base.cra_module = THIS_MODULE, 404 405 .min_keysize = AES_MIN_KEY_SIZE, 406 .max_keysize = AES_MAX_KEY_SIZE, 407 .chunksize = AES_BLOCK_SIZE, 408 .walksize = 8 * AES_BLOCK_SIZE, 409 .ivsize = AES_BLOCK_SIZE, 410 .setkey = aesbs_ctr_setkey_sync, 411 .encrypt = ctr_encrypt_sync, 412 .decrypt = ctr_encrypt_sync, 413 }, { 414 .base.cra_name = "__xts(aes)", 415 .base.cra_driver_name = "__xts-aes-neonbs", 416 .base.cra_priority = 250, 417 .base.cra_blocksize = AES_BLOCK_SIZE, 418 .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), 419 .base.cra_module = THIS_MODULE, 420 .base.cra_flags = CRYPTO_ALG_INTERNAL, 421 422 .min_keysize = 2 * AES_MIN_KEY_SIZE, 423 .max_keysize = 2 * AES_MAX_KEY_SIZE, 424 .walksize = 8 * AES_BLOCK_SIZE, 425 .ivsize = AES_BLOCK_SIZE, 426 .setkey = aesbs_xts_setkey, 427 .encrypt = xts_encrypt, 428 .decrypt = xts_decrypt, 429 } }; 430 431 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 432 433 static void aes_exit(void) 434 { 435 int i; 436 437 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) 438 if (aes_simd_algs[i]) 439 simd_skcipher_free(aes_simd_algs[i]); 440 441 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 442 } 443 444 static int __init aes_init(void) 445 { 446 struct simd_skcipher_alg *simd; 447 const char *basename; 448 const char *algname; 449 const char *drvname; 450 int err; 451 int i; 452 453 if (!cpu_have_named_feature(ASIMD)) 454 return -ENODEV; 455 456 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 457 if (err) 458 return err; 459 460 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 461 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) 462 continue; 463 464 algname = aes_algs[i].base.cra_name + 2; 465 drvname = aes_algs[i].base.cra_driver_name + 2; 466 basename = aes_algs[i].base.cra_driver_name; 467 simd = simd_skcipher_create_compat(algname, drvname, basename); 468 err = PTR_ERR(simd); 469 if (IS_ERR(simd)) 470 goto unregister_simds; 471 472 aes_simd_algs[i] = simd; 473 } 474 return 0; 475 476 unregister_simds: 477 aes_exit(); 478 return err; 479 } 480 481 module_init(aes_init); 482 module_exit(aes_exit); 483