1 /* 2 * aes-ce-glue.c - wrapper code for ARMv8 AES 3 * 4 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <asm/hwcap.h> 12 #include <asm/neon.h> 13 #include <asm/hwcap.h> 14 #include <crypto/aes.h> 15 #include <crypto/internal/simd.h> 16 #include <crypto/internal/skcipher.h> 17 #include <linux/cpufeature.h> 18 #include <linux/module.h> 19 #include <crypto/xts.h> 20 21 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 22 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 23 MODULE_LICENSE("GPL v2"); 24 25 /* defined in aes-ce-core.S */ 26 asmlinkage u32 ce_aes_sub(u32 input); 27 asmlinkage void ce_aes_invert(void *dst, void *src); 28 29 asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], 30 int rounds, int blocks); 31 asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], 32 int rounds, int blocks); 33 34 asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], 35 int rounds, int blocks, u8 iv[]); 36 asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], 37 int rounds, int blocks, u8 iv[]); 38 39 asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], 40 int rounds, int blocks, u8 ctr[]); 41 42 asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], 43 int rounds, int blocks, u8 iv[], 44 u8 const rk2[], int first); 45 asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], 46 int rounds, int blocks, u8 iv[], 47 u8 const rk2[], int first); 48 49 struct aes_block { 50 u8 b[AES_BLOCK_SIZE]; 51 }; 52 53 static int num_rounds(struct crypto_aes_ctx *ctx) 54 { 55 /* 56 * # of rounds specified by AES: 57 * 128 bit key 10 rounds 58 * 192 bit key 12 rounds 59 * 256 bit key 14 rounds 60 * => n byte key => 6 + (n/4) rounds 61 */ 62 return 6 + ctx->key_length / 4; 63 } 64 65 static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, 66 unsigned int key_len) 67 { 68 /* 69 * The AES key schedule round constants 70 */ 71 static u8 const rcon[] = { 72 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 73 }; 74 75 u32 kwords = key_len / sizeof(u32); 76 struct aes_block *key_enc, *key_dec; 77 int i, j; 78 79 if (key_len != AES_KEYSIZE_128 && 80 key_len != AES_KEYSIZE_192 && 81 key_len != AES_KEYSIZE_256) 82 return -EINVAL; 83 84 memcpy(ctx->key_enc, in_key, key_len); 85 ctx->key_length = key_len; 86 87 kernel_neon_begin(); 88 for (i = 0; i < sizeof(rcon); i++) { 89 u32 *rki = ctx->key_enc + (i * kwords); 90 u32 *rko = rki + kwords; 91 92 #ifndef CONFIG_CPU_BIG_ENDIAN 93 rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8); 94 rko[0] = rko[0] ^ rki[0] ^ rcon[i]; 95 #else 96 rko[0] = rol32(ce_aes_sub(rki[kwords - 1]), 8); 97 rko[0] = rko[0] ^ rki[0] ^ (rcon[i] << 24); 98 #endif 99 rko[1] = rko[0] ^ rki[1]; 100 rko[2] = rko[1] ^ rki[2]; 101 rko[3] = rko[2] ^ rki[3]; 102 103 if (key_len == AES_KEYSIZE_192) { 104 if (i >= 7) 105 break; 106 rko[4] = rko[3] ^ rki[4]; 107 rko[5] = rko[4] ^ rki[5]; 108 } else if (key_len == AES_KEYSIZE_256) { 109 if (i >= 6) 110 break; 111 rko[4] = ce_aes_sub(rko[3]) ^ rki[4]; 112 rko[5] = rko[4] ^ rki[5]; 113 rko[6] = rko[5] ^ rki[6]; 114 rko[7] = rko[6] ^ rki[7]; 115 } 116 } 117 118 /* 119 * Generate the decryption keys for the Equivalent Inverse Cipher. 120 * This involves reversing the order of the round keys, and applying 121 * the Inverse Mix Columns transformation on all but the first and 122 * the last one. 123 */ 124 key_enc = (struct aes_block *)ctx->key_enc; 125 key_dec = (struct aes_block *)ctx->key_dec; 126 j = num_rounds(ctx); 127 128 key_dec[0] = key_enc[j]; 129 for (i = 1, j--; j > 0; i++, j--) 130 ce_aes_invert(key_dec + i, key_enc + j); 131 key_dec[i] = key_enc[0]; 132 133 kernel_neon_end(); 134 return 0; 135 } 136 137 static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 138 unsigned int key_len) 139 { 140 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 141 int ret; 142 143 ret = ce_aes_expandkey(ctx, in_key, key_len); 144 if (!ret) 145 return 0; 146 147 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 148 return -EINVAL; 149 } 150 151 struct crypto_aes_xts_ctx { 152 struct crypto_aes_ctx key1; 153 struct crypto_aes_ctx __aligned(8) key2; 154 }; 155 156 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key, 157 unsigned int key_len) 158 { 159 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 160 int ret; 161 162 ret = xts_verify_key(tfm, in_key, key_len); 163 if (ret) 164 return ret; 165 166 ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2); 167 if (!ret) 168 ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2], 169 key_len / 2); 170 if (!ret) 171 return 0; 172 173 crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 174 return -EINVAL; 175 } 176 177 static int ecb_encrypt(struct skcipher_request *req) 178 { 179 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 180 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 181 struct skcipher_walk walk; 182 unsigned int blocks; 183 int err; 184 185 err = skcipher_walk_virt(&walk, req, true); 186 187 kernel_neon_begin(); 188 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 189 ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 190 (u8 *)ctx->key_enc, num_rounds(ctx), blocks); 191 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 192 } 193 kernel_neon_end(); 194 return err; 195 } 196 197 static int ecb_decrypt(struct skcipher_request *req) 198 { 199 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 200 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 201 struct skcipher_walk walk; 202 unsigned int blocks; 203 int err; 204 205 err = skcipher_walk_virt(&walk, req, true); 206 207 kernel_neon_begin(); 208 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 209 ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 210 (u8 *)ctx->key_dec, num_rounds(ctx), blocks); 211 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 212 } 213 kernel_neon_end(); 214 return err; 215 } 216 217 static int cbc_encrypt(struct skcipher_request *req) 218 { 219 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 220 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 221 struct skcipher_walk walk; 222 unsigned int blocks; 223 int err; 224 225 err = skcipher_walk_virt(&walk, req, true); 226 227 kernel_neon_begin(); 228 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 229 ce_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 230 (u8 *)ctx->key_enc, num_rounds(ctx), blocks, 231 walk.iv); 232 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 233 } 234 kernel_neon_end(); 235 return err; 236 } 237 238 static int cbc_decrypt(struct skcipher_request *req) 239 { 240 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 241 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 242 struct skcipher_walk walk; 243 unsigned int blocks; 244 int err; 245 246 err = skcipher_walk_virt(&walk, req, true); 247 248 kernel_neon_begin(); 249 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 250 ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 251 (u8 *)ctx->key_dec, num_rounds(ctx), blocks, 252 walk.iv); 253 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 254 } 255 kernel_neon_end(); 256 return err; 257 } 258 259 static int ctr_encrypt(struct skcipher_request *req) 260 { 261 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 262 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 263 struct skcipher_walk walk; 264 int err, blocks; 265 266 err = skcipher_walk_virt(&walk, req, true); 267 268 kernel_neon_begin(); 269 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 270 ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 271 (u8 *)ctx->key_enc, num_rounds(ctx), blocks, 272 walk.iv); 273 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 274 } 275 if (walk.nbytes) { 276 u8 __aligned(8) tail[AES_BLOCK_SIZE]; 277 unsigned int nbytes = walk.nbytes; 278 u8 *tdst = walk.dst.virt.addr; 279 u8 *tsrc = walk.src.virt.addr; 280 281 /* 282 * Tell aes_ctr_encrypt() to process a tail block. 283 */ 284 blocks = -1; 285 286 ce_aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, 287 num_rounds(ctx), blocks, walk.iv); 288 if (tdst != tsrc) 289 memcpy(tdst, tsrc, nbytes); 290 crypto_xor(tdst, tail, nbytes); 291 err = skcipher_walk_done(&walk, 0); 292 } 293 kernel_neon_end(); 294 295 return err; 296 } 297 298 static int xts_encrypt(struct skcipher_request *req) 299 { 300 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 301 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 302 int err, first, rounds = num_rounds(&ctx->key1); 303 struct skcipher_walk walk; 304 unsigned int blocks; 305 306 err = skcipher_walk_virt(&walk, req, true); 307 308 kernel_neon_begin(); 309 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 310 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 311 (u8 *)ctx->key1.key_enc, rounds, blocks, 312 walk.iv, (u8 *)ctx->key2.key_enc, first); 313 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 314 } 315 kernel_neon_end(); 316 317 return err; 318 } 319 320 static int xts_decrypt(struct skcipher_request *req) 321 { 322 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 323 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 324 int err, first, rounds = num_rounds(&ctx->key1); 325 struct skcipher_walk walk; 326 unsigned int blocks; 327 328 err = skcipher_walk_virt(&walk, req, true); 329 330 kernel_neon_begin(); 331 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 332 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 333 (u8 *)ctx->key1.key_dec, rounds, blocks, 334 walk.iv, (u8 *)ctx->key2.key_enc, first); 335 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); 336 } 337 kernel_neon_end(); 338 339 return err; 340 } 341 342 static struct skcipher_alg aes_algs[] = { { 343 .base = { 344 .cra_name = "__ecb(aes)", 345 .cra_driver_name = "__ecb-aes-ce", 346 .cra_priority = 300, 347 .cra_flags = CRYPTO_ALG_INTERNAL, 348 .cra_blocksize = AES_BLOCK_SIZE, 349 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 350 .cra_module = THIS_MODULE, 351 }, 352 .min_keysize = AES_MIN_KEY_SIZE, 353 .max_keysize = AES_MAX_KEY_SIZE, 354 .setkey = ce_aes_setkey, 355 .encrypt = ecb_encrypt, 356 .decrypt = ecb_decrypt, 357 }, { 358 .base = { 359 .cra_name = "__cbc(aes)", 360 .cra_driver_name = "__cbc-aes-ce", 361 .cra_priority = 300, 362 .cra_flags = CRYPTO_ALG_INTERNAL, 363 .cra_blocksize = AES_BLOCK_SIZE, 364 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 365 .cra_module = THIS_MODULE, 366 }, 367 .min_keysize = AES_MIN_KEY_SIZE, 368 .max_keysize = AES_MAX_KEY_SIZE, 369 .ivsize = AES_BLOCK_SIZE, 370 .setkey = ce_aes_setkey, 371 .encrypt = cbc_encrypt, 372 .decrypt = cbc_decrypt, 373 }, { 374 .base = { 375 .cra_name = "__ctr(aes)", 376 .cra_driver_name = "__ctr-aes-ce", 377 .cra_priority = 300, 378 .cra_flags = CRYPTO_ALG_INTERNAL, 379 .cra_blocksize = 1, 380 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 381 .cra_module = THIS_MODULE, 382 }, 383 .min_keysize = AES_MIN_KEY_SIZE, 384 .max_keysize = AES_MAX_KEY_SIZE, 385 .ivsize = AES_BLOCK_SIZE, 386 .chunksize = AES_BLOCK_SIZE, 387 .setkey = ce_aes_setkey, 388 .encrypt = ctr_encrypt, 389 .decrypt = ctr_encrypt, 390 }, { 391 .base = { 392 .cra_name = "__xts(aes)", 393 .cra_driver_name = "__xts-aes-ce", 394 .cra_priority = 300, 395 .cra_flags = CRYPTO_ALG_INTERNAL, 396 .cra_blocksize = AES_BLOCK_SIZE, 397 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 398 .cra_module = THIS_MODULE, 399 }, 400 .min_keysize = 2 * AES_MIN_KEY_SIZE, 401 .max_keysize = 2 * AES_MAX_KEY_SIZE, 402 .ivsize = AES_BLOCK_SIZE, 403 .setkey = xts_set_key, 404 .encrypt = xts_encrypt, 405 .decrypt = xts_decrypt, 406 } }; 407 408 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; 409 410 static void aes_exit(void) 411 { 412 int i; 413 414 for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++) 415 simd_skcipher_free(aes_simd_algs[i]); 416 417 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 418 } 419 420 static int __init aes_init(void) 421 { 422 struct simd_skcipher_alg *simd; 423 const char *basename; 424 const char *algname; 425 const char *drvname; 426 int err; 427 int i; 428 429 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 430 if (err) 431 return err; 432 433 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 434 algname = aes_algs[i].base.cra_name + 2; 435 drvname = aes_algs[i].base.cra_driver_name + 2; 436 basename = aes_algs[i].base.cra_driver_name; 437 simd = simd_skcipher_create_compat(algname, drvname, basename); 438 err = PTR_ERR(simd); 439 if (IS_ERR(simd)) 440 goto unregister_simds; 441 442 aes_simd_algs[i] = simd; 443 } 444 445 return 0; 446 447 unregister_simds: 448 aes_exit(); 449 return err; 450 } 451 452 module_cpu_feature_match(AES, aes_init); 453 module_exit(aes_exit); 454