1 /* 2 * aes-ce-glue.c - wrapper code for ARMv8 AES 3 * 4 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <asm/hwcap.h> 12 #include <asm/neon.h> 13 #include <asm/hwcap.h> 14 #include <crypto/aes.h> 15 #include <crypto/internal/simd.h> 16 #include <crypto/internal/skcipher.h> 17 #include <linux/module.h> 18 #include <crypto/xts.h> 19 20 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 21 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 22 MODULE_LICENSE("GPL v2"); 23 24 /* defined in aes-ce-core.S */ 25 asmlinkage u32 ce_aes_sub(u32 input); 26 asmlinkage void ce_aes_invert(void *dst, void *src); 27 28 asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], 29 int rounds, int blocks); 30 asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], 31 int rounds, int blocks); 32 33 asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], 34 int rounds, int blocks, u8 iv[]); 35 asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], 36 int rounds, int blocks, u8 iv[]); 37 38 asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], 39 int rounds, int blocks, u8 ctr[]); 40 41 asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], 42 int rounds, int blocks, u8 iv[], 43 u8 const rk2[], int first); 44 asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], 45 int rounds, int blocks, u8 iv[], 46 u8 const rk2[], int first); 47 48 struct aes_block { 49 u8 b[AES_BLOCK_SIZE]; 50 }; 51 52 static int num_rounds(struct crypto_aes_ctx *ctx) 53 { 54 /* 55 * # of rounds specified by AES: 56 * 128 bit key 10 rounds 57 * 192 bit key 12 rounds 58 * 256 bit key 14 rounds 59 * => n byte key => 6 + (n/4) rounds 60 */ 61 return 6 + ctx->key_length / 4; 62 } 63 64 static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 65 unsigned int key_len) 66 { 67 /* 68 * The AES key schedule round constants 69 */ 70 static u8 const rcon[] = { 71 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 72 }; 73 74 u32 kwords = key_len / sizeof(u32); 75 struct aes_block *key_enc, *key_dec; 76 int i, j; 77 78 if (key_len != AES_KEYSIZE_128 && 79 key_len != AES_KEYSIZE_192 && 80 key_len != AES_KEYSIZE_256) 81 return -EINVAL; 82 83 memcpy(ctx->key_enc, in_key, key_len); 84 ctx->key_length = key_len; 85 86 kernel_neon_begin(); 87 for (i = 0; i < sizeof(rcon); i++) { 88 u32 *rki = ctx->key_enc + (i * kwords); 89 u32 *rko = rki + kwords; 90 91 #ifndef CONFIG_CPU_BIG_ENDIAN 92 rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8); 93 rko[0] = rko[0] ^ rki[0] ^ rcon[i]; 94 #else 95 rko[0] = rol32(ce_aes_sub(rki[kwords - 1]), 8); 96 rko[0] = rko[0] ^ rki[0] ^ (rcon[i] << 24); 97 #endif 98 rko[1] = rko[0] ^ rki[1]; 99 rko[2] = rko[1] ^ rki[2]; 100 rko[3] = rko[2] ^ rki[3]; 101 102 if (key_len == AES_KEYSIZE_192) { 103 if (i >= 7) 104 break; 105 rko[4] = rko[3] ^ rki[4]; 106 rko[5] = rko[4] ^ rki[5]; 107 } else if (key_len == AES_KEYSIZE_256) { 108 if (i >= 6) 109 break; 110 rko[4] = ce_aes_sub(rko[3]) ^ rki[4]; 111 rko[5] = rko[4] ^ rki[5]; 112 rko[6] = rko[5] ^ rki[6]; 113 rko[7] = rko[6] ^ rki[7]; 114 } 115 } 116 117 /* 118 * Generate the decryption keys for the Equivalent Inverse Cipher. 119 * This involves reversing the order of the round keys, and applying 120 * the Inverse Mix Columns transformation on all but the first and 121 * the last one. 122 */ 123 key_enc = (struct aes_block *)ctx->key_enc; 124 key_dec = (struct aes_block *)ctx->key_dec; 125 j = num_rounds(ctx); 126 127 key_dec[0] = key_enc[j]; 128 for (i = 1, j--; j > 0; i++, j--) 129 ce_aes_invert(key_dec + i, key_enc + j); 130 key_dec[i] = key_enc[0]; 131 132 kernel_neon_end(); 133 return 0; 134 } 135 136 static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 137 unsigned int key_len) 138 { 139 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 140 int ret; 141 142 ret = ce_aes_expandkey(ctx, in_key, key_len); 143 if (!ret) 144 return 0; 145 146 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 147 return -EINVAL; 148 } 149 150 struct crypto_aes_xts_ctx { 151 struct crypto_aes_ctx key1; 152 struct crypto_aes_ctx __aligned(8) key2; 153 }; 154 155 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 156 unsigned int key_len) 157 { 158 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 159 int ret; 160 161 ret = xts_verify_key(tfm, in_key, key_len); 162 if (ret) 163 return ret; 164 165 ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2); 166 if (!ret) 167 ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2], 168 key_len / 2); 169 if (!ret) 170 return 0; 171 172 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 173 return -EINVAL; 174 } 175 176 static int ecb_encrypt(struct skcipher_request *req) 177 { 178 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 179 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 180 struct skcipher_walk walk; 181 unsigned int blocks; 182 int err; 183 184 err = skcipher_walk_virt(&walk, req, true); 185 186 kernel_neon_begin(); 187 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 188 ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 189 (u8 *)ctx->key_enc, num_rounds(ctx), blocks); 190 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 191 } 192 kernel_neon_end(); 193 return err; 194 } 195 196 static int ecb_decrypt(struct skcipher_request *req) 197 { 198 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 199 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 200 struct skcipher_walk walk; 201 unsigned int blocks; 202 int err; 203 204 err = skcipher_walk_virt(&walk, req, true); 205 206 kernel_neon_begin(); 207 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 208 ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 209 (u8 *)ctx->key_dec, num_rounds(ctx), blocks); 210 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 211 } 212 kernel_neon_end(); 213 return err; 214 } 215 216 static int cbc_encrypt(struct skcipher_request *req) 217 { 218 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 219 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 220 struct skcipher_walk walk; 221 unsigned int blocks; 222 int err; 223 224 err = skcipher_walk_virt(&walk, req, true); 225 226 kernel_neon_begin(); 227 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 228 ce_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 229 (u8 *)ctx->key_enc, num_rounds(ctx), blocks, 230 walk.iv); 231 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 232 } 233 kernel_neon_end(); 234 return err; 235 } 236 237 static int cbc_decrypt(struct skcipher_request *req) 238 { 239 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 240 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 241 struct skcipher_walk walk; 242 unsigned int blocks; 243 int err; 244 245 err = skcipher_walk_virt(&walk, req, true); 246 247 kernel_neon_begin(); 248 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 249 ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 250 (u8 *)ctx->key_dec, num_rounds(ctx), blocks, 251 walk.iv); 252 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 253 } 254 kernel_neon_end(); 255 return err; 256 } 257 258 static int ctr_encrypt(struct skcipher_request *req) 259 { 260 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 261 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 262 struct skcipher_walk walk; 263 int err, blocks; 264 265 err = skcipher_walk_virt(&walk, req, true); 266 267 kernel_neon_begin(); 268 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 269 ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 270 (u8 *)ctx->key_enc, num_rounds(ctx), blocks, 271 walk.iv); 272 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 273 } 274 if (walk.nbytes) { 275 u8 __aligned(8) tail[AES_BLOCK_SIZE]; 276 unsigned int nbytes = walk.nbytes; 277 u8 *tdst = walk.dst.virt.addr; 278 u8 *tsrc = walk.src.virt.addr; 279 280 /* 281 * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need 282 * to tell aes_ctr_encrypt() to only read half a block. 283 */ 284 blocks = (nbytes <= 8) ? -1 : 1; 285 286 ce_aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, 287 num_rounds(ctx), blocks, walk.iv); 288 memcpy(tdst, tail, nbytes); 289 err = skcipher_walk_done(&walk, 0); 290 } 291 kernel_neon_end(); 292 293 return err; 294 } 295 296 static int xts_encrypt(struct skcipher_request *req) 297 { 298 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 299 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 300 int err, first, rounds = num_rounds(&ctx->key1); 301 struct skcipher_walk walk; 302 unsigned int blocks; 303 304 err = skcipher_walk_virt(&walk, req, true); 305 306 kernel_neon_begin(); 307 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 308 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 309 (u8 *)ctx->key1.key_enc, rounds, blocks, 310 walk.iv, (u8 *)ctx->key2.key_enc, first); 311 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 312 } 313 kernel_neon_end(); 314 315 return err; 316 } 317 318 static int xts_decrypt(struct skcipher_request *req) 319 { 320 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 321 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 322 int err, first, rounds = num_rounds(&ctx->key1); 323 struct skcipher_walk walk; 324 unsigned int blocks; 325 326 err = skcipher_walk_virt(&walk, req, true); 327 328 kernel_neon_begin(); 329 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 330 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 331 (u8 *)ctx->key1.key_dec, rounds, blocks, 332 walk.iv, (u8 *)ctx->key2.key_enc, first); 333 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 334 } 335 kernel_neon_end(); 336 337 return err; 338 } 339 340 static struct skcipher_alg aes_algs[] = { { 341 .base = { 342 .cra_name = "__ecb(aes)", 343 .cra_driver_name = "__ecb-aes-ce", 344 .cra_priority = 300, 345 .cra_flags = CRYPTO_ALG_INTERNAL, 346 .cra_blocksize = AES_BLOCK_SIZE, 347 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 348 .cra_alignmask = 7, 349 .cra_module = THIS_MODULE, 350 }, 351 .min_keysize = AES_MIN_KEY_SIZE, 352 .max_keysize = AES_MAX_KEY_SIZE, 353 .setkey = ce_aes_setkey, 354 .encrypt = ecb_encrypt, 355 .decrypt = ecb_decrypt, 356 }, { 357 .base = { 358 .cra_name = "__cbc(aes)", 359 .cra_driver_name = "__cbc-aes-ce", 360 .cra_priority = 300, 361 .cra_flags = CRYPTO_ALG_INTERNAL, 362 .cra_blocksize = AES_BLOCK_SIZE, 363 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 364 .cra_alignmask = 7, 365 .cra_module = THIS_MODULE, 366 }, 367 .min_keysize = AES_MIN_KEY_SIZE, 368 .max_keysize = AES_MAX_KEY_SIZE, 369 .ivsize = AES_BLOCK_SIZE, 370 .setkey = ce_aes_setkey, 371 .encrypt = cbc_encrypt, 372 .decrypt = cbc_decrypt, 373 }, { 374 .base = { 375 .cra_name = "__ctr(aes)", 376 .cra_driver_name = "__ctr-aes-ce", 377 .cra_priority = 300, 378 .cra_flags = CRYPTO_ALG_INTERNAL, 379 .cra_blocksize = 1, 380 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 381 .cra_alignmask = 7, 382 .cra_module = THIS_MODULE, 383 }, 384 .min_keysize = AES_MIN_KEY_SIZE, 385 .max_keysize = AES_MAX_KEY_SIZE, 386 .ivsize = AES_BLOCK_SIZE, 387 .chunksize = AES_BLOCK_SIZE, 388 .setkey = ce_aes_setkey, 389 .encrypt = ctr_encrypt, 390 .decrypt = ctr_encrypt, 391 }, { 392 .base = { 393 .cra_name = "__xts(aes)", 394 .cra_driver_name = "__xts-aes-ce", 395 .cra_priority = 300, 396 .cra_flags = CRYPTO_ALG_INTERNAL, 397 .cra_blocksize = AES_BLOCK_SIZE, 398 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 399 .cra_alignmask = 7, 400 .cra_module = THIS_MODULE, 401 }, 402 .min_keysize = 2 * AES_MIN_KEY_SIZE, 403 .max_keysize = 2 * AES_MAX_KEY_SIZE, 404 .ivsize = AES_BLOCK_SIZE, 405 .setkey = xts_set_key, 406 .encrypt = xts_encrypt, 407 .decrypt = xts_decrypt, 408 } }; 409 410 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 411 412 static void aes_exit(void) 413 { 414 int i; 415 416 for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++) 417 simd_skcipher_free(aes_simd_algs[i]); 418 419 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 420 } 421 422 static int __init aes_init(void) 423 { 424 struct simd_skcipher_alg *simd; 425 const char *basename; 426 const char *algname; 427 const char *drvname; 428 int err; 429 int i; 430 431 if (!(elf_hwcap2 & HWCAP2_AES)) 432 return -ENODEV; 433 434 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 435 if (err) 436 return err; 437 438 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 439 algname = aes_algs[i].base.cra_name + 2; 440 drvname = aes_algs[i].base.cra_driver_name + 2; 441 basename = aes_algs[i].base.cra_driver_name; 442 simd = simd_skcipher_create_compat(algname, drvname, basename); 443 err = PTR_ERR(simd); 444 if (IS_ERR(simd)) 445 goto unregister_simds; 446 447 aes_simd_algs[i] = simd; 448 } 449 450 return 0; 451 452 unregister_simds: 453 aes_exit(); 454 return err; 455 } 456 457 module_init(aes_init); 458 module_exit(aes_exit); 459