1 /* 2 * Bit sliced AES using NEON instructions 3 * 4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <asm/neon.h> 12 #include <crypto/aes.h> 13 #include <crypto/internal/simd.h> 14 #include <crypto/internal/skcipher.h> 15 #include <crypto/xts.h> 16 #include <linux/module.h> 17 18 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 19 MODULE_LICENSE("GPL v2"); 20 21 MODULE_ALIAS_CRYPTO("ecb(aes)"); 22 MODULE_ALIAS_CRYPTO("cbc(aes)"); 23 MODULE_ALIAS_CRYPTO("ctr(aes)"); 24 MODULE_ALIAS_CRYPTO("xts(aes)"); 25 26 asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds); 27 28 asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], 29 int rounds, int blocks); 30 asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], 31 int rounds, int blocks); 32 33 asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], 34 int rounds, int blocks, u8 iv[]); 35 36 asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], 37 int rounds, int blocks, u8 iv[], u8 final[]); 38 39 asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], 40 int rounds, int blocks, u8 iv[]); 41 asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], 42 int rounds, int blocks, u8 iv[]); 43 44 /* borrowed from aes-neon-blk.ko */ 45 asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], 46 int rounds, int blocks, int first); 47 asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], 48 int rounds, int blocks, u8 iv[], 49 int first); 50 51 struct aesbs_ctx { 52 u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32]; 53 int rounds; 54 } __aligned(AES_BLOCK_SIZE); 55 56 struct aesbs_cbc_ctx { 57 struct aesbs_ctx key; 58 u32 enc[AES_MAX_KEYLENGTH_U32]; 59 }; 60 61 struct aesbs_xts_ctx { 62 struct aesbs_ctx key; 63 u32 twkey[AES_MAX_KEYLENGTH_U32]; 64 }; 65 66 static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 67 unsigned int key_len) 68 { 69 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 70 struct crypto_aes_ctx rk; 71 int err; 72 73 err = crypto_aes_expand_key(&rk, in_key, key_len); 74 if (err) 75 return err; 76 77 ctx->rounds = 6 + key_len / 4; 78 79 kernel_neon_begin(); 80 aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds); 81 kernel_neon_end(); 82 83 return 0; 84 } 85 86 static int __ecb_crypt(struct skcipher_request *req, 87 void (*fn)(u8 out[], u8 const in[], u8 const rk[], 88 int rounds, int blocks)) 89 { 90 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 91 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 92 struct skcipher_walk walk; 93 int err; 94 95 err = skcipher_walk_virt(&walk, req, true); 96 97 kernel_neon_begin(); 98 while (walk.nbytes >= AES_BLOCK_SIZE) { 99 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 100 101 if (walk.nbytes < walk.total) 102 blocks = round_down(blocks, 103 walk.stride / AES_BLOCK_SIZE); 104 105 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk, 106 ctx->rounds, blocks); 107 err = skcipher_walk_done(&walk, 108 walk.nbytes - blocks * AES_BLOCK_SIZE); 109 } 110 kernel_neon_end(); 111 112 return err; 113 } 114 115 static int ecb_encrypt(struct skcipher_request *req) 116 { 117 return __ecb_crypt(req, aesbs_ecb_encrypt); 118 } 119 120 static int ecb_decrypt(struct skcipher_request *req) 121 { 122 return __ecb_crypt(req, aesbs_ecb_decrypt); 123 } 124 125 static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 126 unsigned int key_len) 127 { 128 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 129 struct crypto_aes_ctx rk; 130 int err; 131 132 err = crypto_aes_expand_key(&rk, in_key, key_len); 133 if (err) 134 return err; 135 136 ctx->key.rounds = 6 + key_len / 4; 137 138 memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc)); 139 140 kernel_neon_begin(); 141 aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds); 142 kernel_neon_end(); 143 144 return 0; 145 } 146 147 static int cbc_encrypt(struct skcipher_request *req) 148 { 149 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 150 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 151 struct skcipher_walk walk; 152 int err, first = 1; 153 154 err = skcipher_walk_virt(&walk, req, true); 155 156 kernel_neon_begin(); 157 while (walk.nbytes >= AES_BLOCK_SIZE) { 158 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 159 160 /* fall back to the non-bitsliced NEON implementation */ 161 neon_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 162 ctx->enc, ctx->key.rounds, blocks, walk.iv, 163 first); 164 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 165 first = 0; 166 } 167 kernel_neon_end(); 168 return err; 169 } 170 171 static int cbc_decrypt(struct skcipher_request *req) 172 { 173 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 174 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 175 struct skcipher_walk walk; 176 int err; 177 178 err = skcipher_walk_virt(&walk, req, true); 179 180 kernel_neon_begin(); 181 while (walk.nbytes >= AES_BLOCK_SIZE) { 182 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 183 184 if (walk.nbytes < walk.total) 185 blocks = round_down(blocks, 186 walk.stride / AES_BLOCK_SIZE); 187 188 aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 189 ctx->key.rk, ctx->key.rounds, blocks, 190 walk.iv); 191 err = skcipher_walk_done(&walk, 192 walk.nbytes - blocks * AES_BLOCK_SIZE); 193 } 194 kernel_neon_end(); 195 196 return err; 197 } 198 199 static int ctr_encrypt(struct skcipher_request *req) 200 { 201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 202 struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm); 203 struct skcipher_walk walk; 204 u8 buf[AES_BLOCK_SIZE]; 205 int err; 206 207 err = skcipher_walk_virt(&walk, req, true); 208 209 kernel_neon_begin(); 210 while (walk.nbytes > 0) { 211 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 212 u8 *final = (walk.total % AES_BLOCK_SIZE) ? buf : NULL; 213 214 if (walk.nbytes < walk.total) { 215 blocks = round_down(blocks, 216 walk.stride / AES_BLOCK_SIZE); 217 final = NULL; 218 } 219 220 aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 221 ctx->rk, ctx->rounds, blocks, walk.iv, final); 222 223 if (final) { 224 u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; 225 u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; 226 227 if (dst != src) 228 memcpy(dst, src, walk.total % AES_BLOCK_SIZE); 229 crypto_xor(dst, final, walk.total % AES_BLOCK_SIZE); 230 231 err = skcipher_walk_done(&walk, 0); 232 break; 233 } 234 err = skcipher_walk_done(&walk, 235 walk.nbytes - blocks * AES_BLOCK_SIZE); 236 } 237 kernel_neon_end(); 238 239 return err; 240 } 241 242 static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 243 unsigned int key_len) 244 { 245 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 246 struct crypto_aes_ctx rk; 247 int err; 248 249 err = xts_verify_key(tfm, in_key, key_len); 250 if (err) 251 return err; 252 253 key_len /= 2; 254 err = crypto_aes_expand_key(&rk, in_key + key_len, key_len); 255 if (err) 256 return err; 257 258 memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey)); 259 260 return aesbs_setkey(tfm, in_key, key_len); 261 } 262 263 static int __xts_crypt(struct skcipher_request *req, 264 void (*fn)(u8 out[], u8 const in[], u8 const rk[], 265 int rounds, int blocks, u8 iv[])) 266 { 267 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 268 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 269 struct skcipher_walk walk; 270 int err; 271 272 err = skcipher_walk_virt(&walk, req, true); 273 274 kernel_neon_begin(); 275 276 neon_aes_ecb_encrypt(walk.iv, walk.iv, ctx->twkey, 277 ctx->key.rounds, 1, 1); 278 279 while (walk.nbytes >= AES_BLOCK_SIZE) { 280 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; 281 282 if (walk.nbytes < walk.total) 283 blocks = round_down(blocks, 284 walk.stride / AES_BLOCK_SIZE); 285 286 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk, 287 ctx->key.rounds, blocks, walk.iv); 288 err = skcipher_walk_done(&walk, 289 walk.nbytes - blocks * AES_BLOCK_SIZE); 290 } 291 kernel_neon_end(); 292 293 return err; 294 } 295 296 static int xts_encrypt(struct skcipher_request *req) 297 { 298 return __xts_crypt(req, aesbs_xts_encrypt); 299 } 300 301 static int xts_decrypt(struct skcipher_request *req) 302 { 303 return __xts_crypt(req, aesbs_xts_decrypt); 304 } 305 306 static struct skcipher_alg aes_algs[] = { { 307 .base.cra_name = "__ecb(aes)", 308 .base.cra_driver_name = "__ecb-aes-neonbs", 309 .base.cra_priority = 250, 310 .base.cra_blocksize = AES_BLOCK_SIZE, 311 .base.cra_ctxsize = sizeof(struct aesbs_ctx), 312 .base.cra_module = THIS_MODULE, 313 .base.cra_flags = CRYPTO_ALG_INTERNAL, 314 315 .min_keysize = AES_MIN_KEY_SIZE, 316 .max_keysize = AES_MAX_KEY_SIZE, 317 .walksize = 8 * AES_BLOCK_SIZE, 318 .setkey = aesbs_setkey, 319 .encrypt = ecb_encrypt, 320 .decrypt = ecb_decrypt, 321 }, { 322 .base.cra_name = "__cbc(aes)", 323 .base.cra_driver_name = "__cbc-aes-neonbs", 324 .base.cra_priority = 250, 325 .base.cra_blocksize = AES_BLOCK_SIZE, 326 .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), 327 .base.cra_module = THIS_MODULE, 328 .base.cra_flags = CRYPTO_ALG_INTERNAL, 329 330 .min_keysize = AES_MIN_KEY_SIZE, 331 .max_keysize = AES_MAX_KEY_SIZE, 332 .walksize = 8 * AES_BLOCK_SIZE, 333 .ivsize = AES_BLOCK_SIZE, 334 .setkey = aesbs_cbc_setkey, 335 .encrypt = cbc_encrypt, 336 .decrypt = cbc_decrypt, 337 }, { 338 .base.cra_name = "__ctr(aes)", 339 .base.cra_driver_name = "__ctr-aes-neonbs", 340 .base.cra_priority = 250, 341 .base.cra_blocksize = 1, 342 .base.cra_ctxsize = sizeof(struct aesbs_ctx), 343 .base.cra_module = THIS_MODULE, 344 .base.cra_flags = CRYPTO_ALG_INTERNAL, 345 346 .min_keysize = AES_MIN_KEY_SIZE, 347 .max_keysize = AES_MAX_KEY_SIZE, 348 .chunksize = AES_BLOCK_SIZE, 349 .walksize = 8 * AES_BLOCK_SIZE, 350 .ivsize = AES_BLOCK_SIZE, 351 .setkey = aesbs_setkey, 352 .encrypt = ctr_encrypt, 353 .decrypt = ctr_encrypt, 354 }, { 355 .base.cra_name = "ctr(aes)", 356 .base.cra_driver_name = "ctr-aes-neonbs", 357 .base.cra_priority = 250 - 1, 358 .base.cra_blocksize = 1, 359 .base.cra_ctxsize = sizeof(struct aesbs_ctx), 360 .base.cra_module = THIS_MODULE, 361 362 .min_keysize = AES_MIN_KEY_SIZE, 363 .max_keysize = AES_MAX_KEY_SIZE, 364 .chunksize = AES_BLOCK_SIZE, 365 .walksize = 8 * AES_BLOCK_SIZE, 366 .ivsize = AES_BLOCK_SIZE, 367 .setkey = aesbs_setkey, 368 .encrypt = ctr_encrypt, 369 .decrypt = ctr_encrypt, 370 }, { 371 .base.cra_name = "__xts(aes)", 372 .base.cra_driver_name = "__xts-aes-neonbs", 373 .base.cra_priority = 250, 374 .base.cra_blocksize = AES_BLOCK_SIZE, 375 .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), 376 .base.cra_module = THIS_MODULE, 377 .base.cra_flags = CRYPTO_ALG_INTERNAL, 378 379 .min_keysize = 2 * AES_MIN_KEY_SIZE, 380 .max_keysize = 2 * AES_MAX_KEY_SIZE, 381 .walksize = 8 * AES_BLOCK_SIZE, 382 .ivsize = AES_BLOCK_SIZE, 383 .setkey = aesbs_xts_setkey, 384 .encrypt = xts_encrypt, 385 .decrypt = xts_decrypt, 386 } }; 387 388 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 389 390 static void aes_exit(void) 391 { 392 int i; 393 394 for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) 395 if (aes_simd_algs[i]) 396 simd_skcipher_free(aes_simd_algs[i]); 397 398 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 399 } 400 401 static int __init aes_init(void) 402 { 403 struct simd_skcipher_alg *simd; 404 const char *basename; 405 const char *algname; 406 const char *drvname; 407 int err; 408 int i; 409 410 if (!(elf_hwcap & HWCAP_ASIMD)) 411 return -ENODEV; 412 413 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 414 if (err) 415 return err; 416 417 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 418 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) 419 continue; 420 421 algname = aes_algs[i].base.cra_name + 2; 422 drvname = aes_algs[i].base.cra_driver_name + 2; 423 basename = aes_algs[i].base.cra_driver_name; 424 simd = simd_skcipher_create_compat(algname, drvname, basename); 425 err = PTR_ERR(simd); 426 if (IS_ERR(simd)) 427 goto unregister_simds; 428 429 aes_simd_algs[i] = simd; 430 } 431 return 0; 432 433 unregister_simds: 434 aes_exit(); 435 return err; 436 } 437 438 module_init(aes_init); 439 module_exit(aes_exit); 440