1 /* 2 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES 3 * 4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <asm/neon.h> 12 #include <asm/hwcap.h> 13 #include <crypto/aes.h> 14 #include <crypto/ablk_helper.h> 15 #include <crypto/algapi.h> 16 #include <linux/module.h> 17 #include <linux/cpufeature.h> 18 #include <crypto/xts.h> 19 20 #include "aes-ce-setkey.h" 21 22 #ifdef USE_V8_CRYPTO_EXTENSIONS 23 #define MODE "ce" 24 #define PRIO 300 25 #define aes_setkey ce_aes_setkey 26 #define aes_expandkey ce_aes_expandkey 27 #define aes_ecb_encrypt ce_aes_ecb_encrypt 28 #define aes_ecb_decrypt ce_aes_ecb_decrypt 29 #define aes_cbc_encrypt ce_aes_cbc_encrypt 30 #define aes_cbc_decrypt ce_aes_cbc_decrypt 31 #define aes_ctr_encrypt ce_aes_ctr_encrypt 32 #define aes_xts_encrypt ce_aes_xts_encrypt 33 #define aes_xts_decrypt ce_aes_xts_decrypt 34 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); 35 #else 36 #define MODE "neon" 37 #define PRIO 200 38 #define aes_setkey crypto_aes_set_key 39 #define aes_expandkey crypto_aes_expand_key 40 #define aes_ecb_encrypt neon_aes_ecb_encrypt 41 #define aes_ecb_decrypt neon_aes_ecb_decrypt 42 #define aes_cbc_encrypt neon_aes_cbc_encrypt 43 #define aes_cbc_decrypt neon_aes_cbc_decrypt 44 #define aes_ctr_encrypt neon_aes_ctr_encrypt 45 #define aes_xts_encrypt neon_aes_xts_encrypt 46 #define aes_xts_decrypt neon_aes_xts_decrypt 47 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); 48 MODULE_ALIAS_CRYPTO("ecb(aes)"); 49 MODULE_ALIAS_CRYPTO("cbc(aes)"); 50 MODULE_ALIAS_CRYPTO("ctr(aes)"); 51 MODULE_ALIAS_CRYPTO("xts(aes)"); 52 #endif 53 54 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 55 MODULE_LICENSE("GPL v2"); 56 57 /* defined in aes-modes.S */ 58 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], 59 int rounds, int blocks, int first); 60 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], 61 int rounds, int blocks, int first); 62 63 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], 64 int rounds, int blocks, u8 iv[], int first); 65 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], 66 int rounds, int blocks, u8 iv[], int first); 67 68 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], 69 int rounds, int blocks, u8 ctr[], int first); 70 71 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], 72 int rounds, int blocks, u8 const rk2[], u8 iv[], 73 int first); 74 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], 75 int rounds, int blocks, u8 const rk2[], u8 iv[], 76 int first); 77 78 struct crypto_aes_xts_ctx { 79 struct crypto_aes_ctx key1; 80 struct crypto_aes_ctx __aligned(8) key2; 81 }; 82 83 static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key, 84 unsigned int key_len) 85 { 86 struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); 87 int ret; 88 89 ret = xts_check_key(tfm, in_key, key_len); 90 if (ret) 91 return ret; 92 93 ret = aes_expandkey(&ctx->key1, in_key, key_len / 2); 94 if (!ret) 95 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2], 96 key_len / 2); 97 if (!ret) 98 return 0; 99 100 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 101 return -EINVAL; 102 } 103 104 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 105 struct scatterlist *src, unsigned int nbytes) 106 { 107 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 108 int err, first, rounds = 6 + ctx->key_length / 4; 109 struct blkcipher_walk walk; 110 unsigned int blocks; 111 112 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 113 blkcipher_walk_init(&walk, dst, src, nbytes); 114 err = blkcipher_walk_virt(desc, &walk); 115 116 kernel_neon_begin(); 117 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 118 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 119 (u8 *)ctx->key_enc, rounds, blocks, first); 120 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); 121 } 122 kernel_neon_end(); 123 return err; 124 } 125 126 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 127 struct scatterlist *src, unsigned int nbytes) 128 { 129 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 130 int err, first, rounds = 6 + ctx->key_length / 4; 131 struct blkcipher_walk walk; 132 unsigned int blocks; 133 134 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 135 blkcipher_walk_init(&walk, dst, src, nbytes); 136 err = blkcipher_walk_virt(desc, &walk); 137 138 kernel_neon_begin(); 139 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 140 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 141 (u8 *)ctx->key_dec, rounds, blocks, first); 142 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); 143 } 144 kernel_neon_end(); 145 return err; 146 } 147 148 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 149 struct scatterlist *src, unsigned int nbytes) 150 { 151 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 152 int err, first, rounds = 6 + ctx->key_length / 4; 153 struct blkcipher_walk walk; 154 unsigned int blocks; 155 156 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 157 blkcipher_walk_init(&walk, dst, src, nbytes); 158 err = blkcipher_walk_virt(desc, &walk); 159 160 kernel_neon_begin(); 161 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 162 aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 163 (u8 *)ctx->key_enc, rounds, blocks, walk.iv, 164 first); 165 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); 166 } 167 kernel_neon_end(); 168 return err; 169 } 170 171 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 172 struct scatterlist *src, unsigned int nbytes) 173 { 174 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 175 int err, first, rounds = 6 + ctx->key_length / 4; 176 struct blkcipher_walk walk; 177 unsigned int blocks; 178 179 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 180 blkcipher_walk_init(&walk, dst, src, nbytes); 181 err = blkcipher_walk_virt(desc, &walk); 182 183 kernel_neon_begin(); 184 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 185 aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 186 (u8 *)ctx->key_dec, rounds, blocks, walk.iv, 187 first); 188 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); 189 } 190 kernel_neon_end(); 191 return err; 192 } 193 194 static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 195 struct scatterlist *src, unsigned int nbytes) 196 { 197 struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 198 int err, first, rounds = 6 + ctx->key_length / 4; 199 struct blkcipher_walk walk; 200 int blocks; 201 202 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 203 blkcipher_walk_init(&walk, dst, src, nbytes); 204 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 205 206 first = 1; 207 kernel_neon_begin(); 208 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { 209 aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 210 (u8 *)ctx->key_enc, rounds, blocks, walk.iv, 211 first); 212 first = 0; 213 nbytes -= blocks * AES_BLOCK_SIZE; 214 if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE) 215 break; 216 err = blkcipher_walk_done(desc, &walk, 217 walk.nbytes % AES_BLOCK_SIZE); 218 } 219 if (walk.nbytes % AES_BLOCK_SIZE) { 220 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; 221 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; 222 u8 __aligned(8) tail[AES_BLOCK_SIZE]; 223 224 /* 225 * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need 226 * to tell aes_ctr_encrypt() to only read half a block. 227 */ 228 blocks = (nbytes <= 8) ? -1 : 1; 229 230 aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds, 231 blocks, walk.iv, first); 232 memcpy(tdst, tail, nbytes); 233 err = blkcipher_walk_done(desc, &walk, 0); 234 } 235 kernel_neon_end(); 236 237 return err; 238 } 239 240 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 241 struct scatterlist *src, unsigned int nbytes) 242 { 243 struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 244 int err, first, rounds = 6 + ctx->key1.key_length / 4; 245 struct blkcipher_walk walk; 246 unsigned int blocks; 247 248 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 249 blkcipher_walk_init(&walk, dst, src, nbytes); 250 err = blkcipher_walk_virt(desc, &walk); 251 252 kernel_neon_begin(); 253 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 254 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, 255 (u8 *)ctx->key1.key_enc, rounds, blocks, 256 (u8 *)ctx->key2.key_enc, walk.iv, first); 257 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); 258 } 259 kernel_neon_end(); 260 261 return err; 262 } 263 264 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 265 struct scatterlist *src, unsigned int nbytes) 266 { 267 struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 268 int err, first, rounds = 6 + ctx->key1.key_length / 4; 269 struct blkcipher_walk walk; 270 unsigned int blocks; 271 272 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 273 blkcipher_walk_init(&walk, dst, src, nbytes); 274 err = blkcipher_walk_virt(desc, &walk); 275 276 kernel_neon_begin(); 277 for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { 278 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, 279 (u8 *)ctx->key1.key_dec, rounds, blocks, 280 (u8 *)ctx->key2.key_enc, walk.iv, first); 281 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); 282 } 283 kernel_neon_end(); 284 285 return err; 286 } 287 288 static struct crypto_alg aes_algs[] = { { 289 .cra_name = "__ecb-aes-" MODE, 290 .cra_driver_name = "__driver-ecb-aes-" MODE, 291 .cra_priority = 0, 292 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 293 CRYPTO_ALG_INTERNAL, 294 .cra_blocksize = AES_BLOCK_SIZE, 295 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 296 .cra_alignmask = 7, 297 .cra_type = &crypto_blkcipher_type, 298 .cra_module = THIS_MODULE, 299 .cra_blkcipher = { 300 .min_keysize = AES_MIN_KEY_SIZE, 301 .max_keysize = AES_MAX_KEY_SIZE, 302 .ivsize = 0, 303 .setkey = aes_setkey, 304 .encrypt = ecb_encrypt, 305 .decrypt = ecb_decrypt, 306 }, 307 }, { 308 .cra_name = "__cbc-aes-" MODE, 309 .cra_driver_name = "__driver-cbc-aes-" MODE, 310 .cra_priority = 0, 311 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 312 CRYPTO_ALG_INTERNAL, 313 .cra_blocksize = AES_BLOCK_SIZE, 314 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 315 .cra_alignmask = 7, 316 .cra_type = &crypto_blkcipher_type, 317 .cra_module = THIS_MODULE, 318 .cra_blkcipher = { 319 .min_keysize = AES_MIN_KEY_SIZE, 320 .max_keysize = AES_MAX_KEY_SIZE, 321 .ivsize = AES_BLOCK_SIZE, 322 .setkey = aes_setkey, 323 .encrypt = cbc_encrypt, 324 .decrypt = cbc_decrypt, 325 }, 326 }, { 327 .cra_name = "__ctr-aes-" MODE, 328 .cra_driver_name = "__driver-ctr-aes-" MODE, 329 .cra_priority = 0, 330 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 331 CRYPTO_ALG_INTERNAL, 332 .cra_blocksize = 1, 333 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 334 .cra_alignmask = 7, 335 .cra_type = &crypto_blkcipher_type, 336 .cra_module = THIS_MODULE, 337 .cra_blkcipher = { 338 .min_keysize = AES_MIN_KEY_SIZE, 339 .max_keysize = AES_MAX_KEY_SIZE, 340 .ivsize = AES_BLOCK_SIZE, 341 .setkey = aes_setkey, 342 .encrypt = ctr_encrypt, 343 .decrypt = ctr_encrypt, 344 }, 345 }, { 346 .cra_name = "__xts-aes-" MODE, 347 .cra_driver_name = "__driver-xts-aes-" MODE, 348 .cra_priority = 0, 349 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 350 CRYPTO_ALG_INTERNAL, 351 .cra_blocksize = AES_BLOCK_SIZE, 352 .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), 353 .cra_alignmask = 7, 354 .cra_type = &crypto_blkcipher_type, 355 .cra_module = THIS_MODULE, 356 .cra_blkcipher = { 357 .min_keysize = 2 * AES_MIN_KEY_SIZE, 358 .max_keysize = 2 * AES_MAX_KEY_SIZE, 359 .ivsize = AES_BLOCK_SIZE, 360 .setkey = xts_set_key, 361 .encrypt = xts_encrypt, 362 .decrypt = xts_decrypt, 363 }, 364 }, { 365 .cra_name = "ecb(aes)", 366 .cra_driver_name = "ecb-aes-" MODE, 367 .cra_priority = PRIO, 368 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 369 .cra_blocksize = AES_BLOCK_SIZE, 370 .cra_ctxsize = sizeof(struct async_helper_ctx), 371 .cra_alignmask = 7, 372 .cra_type = &crypto_ablkcipher_type, 373 .cra_module = THIS_MODULE, 374 .cra_init = ablk_init, 375 .cra_exit = ablk_exit, 376 .cra_ablkcipher = { 377 .min_keysize = AES_MIN_KEY_SIZE, 378 .max_keysize = AES_MAX_KEY_SIZE, 379 .ivsize = 0, 380 .setkey = ablk_set_key, 381 .encrypt = ablk_encrypt, 382 .decrypt = ablk_decrypt, 383 } 384 }, { 385 .cra_name = "cbc(aes)", 386 .cra_driver_name = "cbc-aes-" MODE, 387 .cra_priority = PRIO, 388 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 389 .cra_blocksize = AES_BLOCK_SIZE, 390 .cra_ctxsize = sizeof(struct async_helper_ctx), 391 .cra_alignmask = 7, 392 .cra_type = &crypto_ablkcipher_type, 393 .cra_module = THIS_MODULE, 394 .cra_init = ablk_init, 395 .cra_exit = ablk_exit, 396 .cra_ablkcipher = { 397 .min_keysize = AES_MIN_KEY_SIZE, 398 .max_keysize = AES_MAX_KEY_SIZE, 399 .ivsize = AES_BLOCK_SIZE, 400 .setkey = ablk_set_key, 401 .encrypt = ablk_encrypt, 402 .decrypt = ablk_decrypt, 403 } 404 }, { 405 .cra_name = "ctr(aes)", 406 .cra_driver_name = "ctr-aes-" MODE, 407 .cra_priority = PRIO, 408 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 409 .cra_blocksize = 1, 410 .cra_ctxsize = sizeof(struct async_helper_ctx), 411 .cra_alignmask = 7, 412 .cra_type = &crypto_ablkcipher_type, 413 .cra_module = THIS_MODULE, 414 .cra_init = ablk_init, 415 .cra_exit = ablk_exit, 416 .cra_ablkcipher = { 417 .min_keysize = AES_MIN_KEY_SIZE, 418 .max_keysize = AES_MAX_KEY_SIZE, 419 .ivsize = AES_BLOCK_SIZE, 420 .setkey = ablk_set_key, 421 .encrypt = ablk_encrypt, 422 .decrypt = ablk_decrypt, 423 } 424 }, { 425 .cra_name = "xts(aes)", 426 .cra_driver_name = "xts-aes-" MODE, 427 .cra_priority = PRIO, 428 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 429 .cra_blocksize = AES_BLOCK_SIZE, 430 .cra_ctxsize = sizeof(struct async_helper_ctx), 431 .cra_alignmask = 7, 432 .cra_type = &crypto_ablkcipher_type, 433 .cra_module = THIS_MODULE, 434 .cra_init = ablk_init, 435 .cra_exit = ablk_exit, 436 .cra_ablkcipher = { 437 .min_keysize = 2 * AES_MIN_KEY_SIZE, 438 .max_keysize = 2 * AES_MAX_KEY_SIZE, 439 .ivsize = AES_BLOCK_SIZE, 440 .setkey = ablk_set_key, 441 .encrypt = ablk_encrypt, 442 .decrypt = ablk_decrypt, 443 } 444 } }; 445 446 static int __init aes_init(void) 447 { 448 return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs)); 449 } 450 451 static void __exit aes_exit(void) 452 { 453 crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs)); 454 } 455 456 #ifdef USE_V8_CRYPTO_EXTENSIONS 457 module_cpu_feature_match(AES, aes_init); 458 #else 459 module_init(aes_init); 460 #endif 461 module_exit(aes_exit); 462