1 /* 2 * Bit sliced AES using NEON instructions 3 * 4 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <asm/neon.h> 12 #include <crypto/aes.h> 13 #include <crypto/cbc.h> 14 #include <crypto/internal/simd.h> 15 #include <crypto/internal/skcipher.h> 16 #include <crypto/xts.h> 17 #include <linux/module.h> 18 19 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 20 MODULE_LICENSE("GPL v2"); 21 22 MODULE_ALIAS_CRYPTO("ecb(aes)"); 23 MODULE_ALIAS_CRYPTO("cbc(aes)"); 24 MODULE_ALIAS_CRYPTO("ctr(aes)"); 25 MODULE_ALIAS_CRYPTO("xts(aes)"); 26 27 asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds); 28 29 asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], 30 int rounds, int blocks); 31 asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], 32 int rounds, int blocks); 33 34 asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], 35 int rounds, int blocks, u8 iv[]); 36 37 asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], 38 int rounds, int blocks, u8 ctr[], u8 final[]); 39 40 asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], 41 int rounds, int blocks, u8 iv[]); 42 asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], 43 int rounds, int blocks, u8 iv[]); 44 45 struct aesbs_ctx { 46 int rounds; 47 u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32] __aligned(AES_BLOCK_SIZE); 48 }; 49 50 struct aesbs_cbc_ctx { 51 struct aesbs_ctx key; 52 struct crypto_cipher *enc_tfm; 53 }; 54 55 struct aesbs_xts_ctx { 56 struct aesbs_ctx key; 57 struct crypto_cipher *tweak_tfm; 58 }; 59 60 static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 61 unsigned int key_len) 62 { 63 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 64 struct crypto_aes_ctx rk; 65 int err; 66 67 err = crypto_aes_expand_key(&rk, in_key, key_len); 68 if (err) 69 return err; 70 71 ctx->rounds = 6 + key_len / 4; 72 73 kernel_neon_begin(); 74 aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds); 75 kernel_neon_end(); 76 77 return 0; 78 } 79 80 static int __ecb_crypt(struct skcipher_request *req, 81 void (*fn)(u8 out[], u8 const in[], u8 const rk[], 82 int rounds, int blocks)) 83 { 84 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 85 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 86 struct skcipher_walk walk; 87 int err; 88 89 err = skcipher_walk_virt(&walk, req, true); 90 91 kernel_neon_begin(); 92 while (walk.nbytes >= AES_BLOCK_SIZE) { 93 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 94 95 if (walk.nbytes < walk.total) 96 blocks = round_down(blocks, 97 walk.stride / AES_BLOCK_SIZE); 98 99 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk, 100 ctx->rounds, blocks); 101 err = skcipher_walk_done(&walk, 102 walk.nbytes - blocks * AES_BLOCK_SIZE); 103 } 104 kernel_neon_end(); 105 106 return err; 107 } 108 109 static int ecb_encrypt(struct skcipher_request *req) 110 { 111 return __ecb_crypt(req, aesbs_ecb_encrypt); 112 } 113 114 static int ecb_decrypt(struct skcipher_request *req) 115 { 116 return __ecb_crypt(req, aesbs_ecb_decrypt); 117 } 118 119 static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 120 unsigned int key_len) 121 { 122 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 123 struct crypto_aes_ctx rk; 124 int err; 125 126 err = crypto_aes_expand_key(&rk, in_key, key_len); 127 if (err) 128 return err; 129 130 ctx->key.rounds = 6 + key_len / 4; 131 132 kernel_neon_begin(); 133 aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds); 134 kernel_neon_end(); 135 136 return crypto_cipher_setkey(ctx->enc_tfm, in_key, key_len); 137 } 138 139 static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) 140 { 141 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 142 143 crypto_cipher_encrypt_one(ctx->enc_tfm, dst, src); 144 } 145 146 static int cbc_encrypt(struct skcipher_request *req) 147 { 148 return crypto_cbc_encrypt_walk(req, cbc_encrypt_one); 149 } 150 151 static int cbc_decrypt(struct skcipher_request *req) 152 { 153 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 154 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 155 struct skcipher_walk walk; 156 int err; 157 158 err = skcipher_walk_virt(&walk, req, true); 159 160 kernel_neon_begin(); 161 while (walk.nbytes >= AES_BLOCK_SIZE) { 162 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 163 164 if (walk.nbytes < walk.total) 165 blocks = round_down(blocks, 166 walk.stride / AES_BLOCK_SIZE); 167 168 aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 169 ctx->key.rk, ctx->key.rounds, blocks, 170 walk.iv); 171 err = skcipher_walk_done(&walk, 172 walk.nbytes - blocks * AES_BLOCK_SIZE); 173 } 174 kernel_neon_end(); 175 176 return err; 177 } 178 179 static int cbc_init(struct crypto_tfm *tfm) 180 { 181 struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); 182 183 ctx->enc_tfm = crypto_alloc_cipher("aes", 0, 0); 184 if (IS_ERR(ctx->enc_tfm)) 185 return PTR_ERR(ctx->enc_tfm); 186 return 0; 187 } 188 189 static void cbc_exit(struct crypto_tfm *tfm) 190 { 191 struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); 192 193 crypto_free_cipher(ctx->enc_tfm); 194 } 195 196 static int ctr_encrypt(struct skcipher_request *req) 197 { 198 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 199 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 200 struct skcipher_walk walk; 201 u8 buf[AES_BLOCK_SIZE]; 202 int err; 203 204 err = skcipher_walk_virt(&walk, req, true); 205 206 kernel_neon_begin(); 207 while (walk.nbytes > 0) { 208 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 209 u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL; 210 211 if (walk.nbytes < walk.total) { 212 blocks = round_down(blocks, 213 walk.stride / AES_BLOCK_SIZE); 214 final = NULL; 215 } 216 217 aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 218 ctx->rk, ctx->rounds, blocks, walk.iv, final); 219 220 if (final) { 221 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; 222 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; 223 224 if (dst != src) 225 memcpy(dst, src, walk.total % AES_BLOCK_SIZE); 226 crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE); 227 228 err = skcipher_walk_done(&walk, 0); 229 break; 230 } 231 err = skcipher_walk_done(&walk, 232 walk.nbytes - blocks * AES_BLOCK_SIZE); 233 } 234 kernel_neon_end(); 235 236 return err; 237 } 238 239 static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 240 unsigned int key_len) 241 { 242 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 243 int err; 244 245 err = xts_verify_key(tfm, in_key, key_len); 246 if (err) 247 return err; 248 249 key_len /= 2; 250 err = crypto_cipher_setkey(ctx->tweak_tfm, in_key + key_len, key_len); 251 if (err) 252 return err; 253 254 return aesbs_setkey(tfm, in_key, key_len); 255 } 256 257 static int xts_init(struct crypto_tfm *tfm) 258 { 259 struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); 260 261 ctx->tweak_tfm = crypto_alloc_cipher("aes", 0, 0); 262 if (IS_ERR(ctx->tweak_tfm)) 263 return PTR_ERR(ctx->tweak_tfm); 264 return 0; 265 } 266 267 static void xts_exit(struct crypto_tfm *tfm) 268 { 269 struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); 270 271 crypto_free_cipher(ctx->tweak_tfm); 272 } 273 274 static int __xts_crypt(struct skcipher_request *req, 275 void (*fn)(u8 out[], u8 const in[], u8 const rk[], 276 int rounds, int blocks, u8 iv[])) 277 { 278 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 279 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 280 struct skcipher_walk walk; 281 int err; 282 283 err = skcipher_walk_virt(&walk, req, true); 284 285 crypto_cipher_encrypt_one(ctx->tweak_tfm, walk.iv, walk.iv); 286 287 kernel_neon_begin(); 288 while (walk.nbytes >= AES_BLOCK_SIZE) { 289 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 290 291 if (walk.nbytes < walk.total) 292 blocks = round_down(blocks, 293 walk.stride / AES_BLOCK_SIZE); 294 295 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk, 296 ctx->key.rounds, blocks, walk.iv); 297 err = skcipher_walk_done(&walk, 298 walk.nbytes - blocks * AES_BLOCK_SIZE); 299 } 300 kernel_neon_end(); 301 302 return err; 303 } 304 305 static int xts_encrypt(struct skcipher_request *req) 306 { 307 return __xts_crypt(req, aesbs_xts_encrypt); 308 } 309 310 static int xts_decrypt(struct skcipher_request *req) 311 { 312 return __xts_crypt(req, aesbs_xts_decrypt); 313 } 314 315 static struct skcipher_alg aes_algs[] = { { 316 .base.cra_name = "__ecb(aes)", 317 .base.cra_driver_name = "__ecb-aes-neonbs", 318 .base.cra_priority = 250, 319 .base.cra_blocksize = AES_BLOCK_SIZE, 320 .base.cra_ctxsize = sizeof(struct aesbs_ctx), 321 .base.cra_module = THIS_MODULE, 322 .base.cra_flags = CRYPTO_ALG_INTERNAL, 323 324 .min_keysize = AES_MIN_KEY_SIZE, 325 .max_keysize = AES_MAX_KEY_SIZE, 326 .walksize = 8 * AES_BLOCK_SIZE, 327 .setkey = aesbs_setkey, 328 .encrypt = ecb_encrypt, 329 .decrypt = ecb_decrypt, 330 }, { 331 .base.cra_name = "__cbc(aes)", 332 .base.cra_driver_name = "__cbc-aes-neonbs", 333 .base.cra_priority = 250, 334 .base.cra_blocksize = AES_BLOCK_SIZE, 335 .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), 336 .base.cra_module = THIS_MODULE, 337 .base.cra_flags = CRYPTO_ALG_INTERNAL, 338 .base.cra_init = cbc_init, 339 .base.cra_exit = cbc_exit, 340 341 .min_keysize = AES_MIN_KEY_SIZE, 342 .max_keysize = AES_MAX_KEY_SIZE, 343 .walksize = 8 * AES_BLOCK_SIZE, 344 .ivsize = AES_BLOCK_SIZE, 345 .setkey = aesbs_cbc_setkey, 346 .encrypt = cbc_encrypt, 347 .decrypt = cbc_decrypt, 348 }, { 349 .base.cra_name = "__ctr(aes)", 350 .base.cra_driver_name = "__ctr-aes-neonbs", 351 .base.cra_priority = 250, 352 .base.cra_blocksize = 1, 353 .base.cra_ctxsize = sizeof(struct aesbs_ctx), 354 .base.cra_module = THIS_MODULE, 355 .base.cra_flags = CRYPTO_ALG_INTERNAL, 356 357 .min_keysize = AES_MIN_KEY_SIZE, 358 .max_keysize = AES_MAX_KEY_SIZE, 359 .chunksize = AES_BLOCK_SIZE, 360 .walksize = 8 * AES_BLOCK_SIZE, 361 .ivsize = AES_BLOCK_SIZE, 362 .setkey = aesbs_setkey, 363 .encrypt = ctr_encrypt, 364 .decrypt = ctr_encrypt, 365 }, { 366 .base.cra_name = "__xts(aes)", 367 .base.cra_driver_name = "__xts-aes-neonbs", 368 .base.cra_priority = 250, 369 .base.cra_blocksize = AES_BLOCK_SIZE, 370 .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), 371 .base.cra_module = THIS_MODULE, 372 .base.cra_flags = CRYPTO_ALG_INTERNAL, 373 .base.cra_init = xts_init, 374 .base.cra_exit = xts_exit, 375 376 .min_keysize = 2 * AES_MIN_KEY_SIZE, 377 .max_keysize = 2 * AES_MAX_KEY_SIZE, 378 .walksize = 8 * AES_BLOCK_SIZE, 379 .ivsize = AES_BLOCK_SIZE, 380 .setkey = aesbs_xts_setkey, 381 .encrypt = xts_encrypt, 382 .decrypt = xts_decrypt, 383 } }; 384 385 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 386 387 static void aes_exit(void) 388 { 389 int i; 390 391 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) 392 if (aes_simd_algs[i]) 393 simd_skcipher_free(aes_simd_algs[i]); 394 395 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 396 } 397 398 static int __init aes_init(void) 399 { 400 struct simd_skcipher_alg *simd; 401 const char *basename; 402 const char *algname; 403 const char *drvname; 404 int err; 405 int i; 406 407 if (!(elf_hwcap & HWCAP_NEON)) 408 return -ENODEV; 409 410 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 411 if (err) 412 return err; 413 414 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 415 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) 416 continue; 417 418 algname = aes_algs[i].base.cra_name + 2; 419 drvname = aes_algs[i].base.cra_driver_name + 2; 420 basename = aes_algs[i].base.cra_driver_name; 421 simd = simd_skcipher_create_compat(algname, drvname, basename); 422 err = PTR_ERR(simd); 423 if (IS_ERR(simd)) 424 goto unregister_simds; 425 426 aes_simd_algs[i] = simd; 427 } 428 return 0; 429 430 unregister_simds: 431 aes_exit(); 432 return err; 433 } 434 435 late_initcall(aes_init); 436 module_exit(aes_exit); 437