1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue code for AES implementation for SPE instructions (PPC) 4 * 5 * Based on generic implementation. The assembler module takes care 6 * about the SPE registers so it can run from interrupt context. 7 * 8 * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> 9 */ 10 11 #include <crypto/aes.h> 12 #include <linux/module.h> 13 #include <linux/init.h> 14 #include <linux/types.h> 15 #include <linux/errno.h> 16 #include <linux/crypto.h> 17 #include <asm/byteorder.h> 18 #include <asm/switch_to.h> 19 #include <crypto/algapi.h> 20 #include <crypto/internal/skcipher.h> 21 #include <crypto/xts.h> 22 #include <crypto/gf128mul.h> 23 #include <crypto/scatterwalk.h> 24 25 /* 26 * MAX_BYTES defines the number of bytes that are allowed to be processed 27 * between preempt_disable() and preempt_enable(). e500 cores can issue two 28 * instructions per clock cycle using one 32/64 bit unit (SU1) and one 32 29 * bit unit (SU2). One of these can be a memory access that is executed via 30 * a single load and store unit (LSU). XTS-AES-256 takes ~780 operations per 31 * 16 byte block block or 25 cycles per byte. Thus 768 bytes of input data 32 * will need an estimated maximum of 20,000 cycles. Headroom for cache misses 33 * included. Even with the low end model clocked at 667 MHz this equals to a 34 * critical time window of less than 30us. The value has been chosen to 35 * process a 512 byte disk block in one or a large 1400 bytes IPsec network 36 * packet in two runs. 37 * 38 */ 39 #define MAX_BYTES 768 40 41 struct ppc_aes_ctx { 42 u32 key_enc[AES_MAX_KEYLENGTH_U32]; 43 u32 key_dec[AES_MAX_KEYLENGTH_U32]; 44 u32 rounds; 45 }; 46 47 struct ppc_xts_ctx { 48 u32 key_enc[AES_MAX_KEYLENGTH_U32]; 49 u32 key_dec[AES_MAX_KEYLENGTH_U32]; 50 u32 key_twk[AES_MAX_KEYLENGTH_U32]; 51 u32 rounds; 52 }; 53 54 extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds); 55 extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds); 56 extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, 57 u32 bytes); 58 extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, 59 u32 bytes); 60 extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, 61 u32 bytes, u8 *iv); 62 extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, 63 u32 bytes, u8 *iv); 64 extern void ppc_crypt_ctr (u8 *out, const u8 *in, u32 *key_enc, u32 rounds, 65 u32 bytes, u8 *iv); 66 extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, 67 u32 bytes, u8 *iv, u32 *key_twk); 68 extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, 69 u32 bytes, u8 *iv, u32 *key_twk); 70 71 extern void ppc_expand_key_128(u32 *key_enc, const u8 *key); 72 extern void ppc_expand_key_192(u32 *key_enc, const u8 *key); 73 extern void ppc_expand_key_256(u32 *key_enc, const u8 *key); 74 75 extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc, 76 unsigned int key_len); 77 78 static void spe_begin(void) 79 { 80 /* disable preemption and save users SPE registers if required */ 81 preempt_disable(); 82 enable_kernel_spe(); 83 } 84 85 static void spe_end(void) 86 { 87 disable_kernel_spe(); 88 /* reenable preemption */ 89 preempt_enable(); 90 } 91 92 static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, 93 unsigned int key_len) 94 { 95 struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); 96 97 switch (key_len) { 98 case AES_KEYSIZE_128: 99 ctx->rounds = 4; 100 ppc_expand_key_128(ctx->key_enc, in_key); 101 break; 102 case AES_KEYSIZE_192: 103 ctx->rounds = 5; 104 ppc_expand_key_192(ctx->key_enc, in_key); 105 break; 106 case AES_KEYSIZE_256: 107 ctx->rounds = 6; 108 ppc_expand_key_256(ctx->key_enc, in_key); 109 break; 110 default: 111 return -EINVAL; 112 } 113 114 ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len); 115 116 return 0; 117 } 118 119 static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm, 120 const u8 *in_key, unsigned int key_len) 121 { 122 return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len); 123 } 124 125 static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, 126 unsigned int key_len) 127 { 128 struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 129 int err; 130 131 err = xts_verify_key(tfm, in_key, key_len); 132 if (err) 133 return err; 134 135 key_len >>= 1; 136 137 switch (key_len) { 138 case AES_KEYSIZE_128: 139 ctx->rounds = 4; 140 ppc_expand_key_128(ctx->key_enc, in_key); 141 ppc_expand_key_128(ctx->key_twk, in_key + AES_KEYSIZE_128); 142 break; 143 case AES_KEYSIZE_192: 144 ctx->rounds = 5; 145 ppc_expand_key_192(ctx->key_enc, in_key); 146 ppc_expand_key_192(ctx->key_twk, in_key + AES_KEYSIZE_192); 147 break; 148 case AES_KEYSIZE_256: 149 ctx->rounds = 6; 150 ppc_expand_key_256(ctx->key_enc, in_key); 151 ppc_expand_key_256(ctx->key_twk, in_key + AES_KEYSIZE_256); 152 break; 153 default: 154 return -EINVAL; 155 } 156 157 ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len); 158 159 return 0; 160 } 161 162 static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 163 { 164 struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); 165 166 spe_begin(); 167 ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds); 168 spe_end(); 169 } 170 171 static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 172 { 173 struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); 174 175 spe_begin(); 176 ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds); 177 spe_end(); 178 } 179 180 static int ppc_ecb_crypt(struct skcipher_request *req, bool enc) 181 { 182 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 183 struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 184 struct skcipher_walk walk; 185 unsigned int nbytes; 186 int err; 187 188 err = skcipher_walk_virt(&walk, req, false); 189 190 while ((nbytes = walk.nbytes) != 0) { 191 nbytes = min_t(unsigned int, nbytes, MAX_BYTES); 192 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 193 194 spe_begin(); 195 if (enc) 196 ppc_encrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr, 197 ctx->key_enc, ctx->rounds, nbytes); 198 else 199 ppc_decrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr, 200 ctx->key_dec, ctx->rounds, nbytes); 201 spe_end(); 202 203 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 204 } 205 206 return err; 207 } 208 209 static int ppc_ecb_encrypt(struct skcipher_request *req) 210 { 211 return ppc_ecb_crypt(req, true); 212 } 213 214 static int ppc_ecb_decrypt(struct skcipher_request *req) 215 { 216 return ppc_ecb_crypt(req, false); 217 } 218 219 static int ppc_cbc_crypt(struct skcipher_request *req, bool enc) 220 { 221 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 222 struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 223 struct skcipher_walk walk; 224 unsigned int nbytes; 225 int err; 226 227 err = skcipher_walk_virt(&walk, req, false); 228 229 while ((nbytes = walk.nbytes) != 0) { 230 nbytes = min_t(unsigned int, nbytes, MAX_BYTES); 231 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 232 233 spe_begin(); 234 if (enc) 235 ppc_encrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr, 236 ctx->key_enc, ctx->rounds, nbytes, 237 walk.iv); 238 else 239 ppc_decrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr, 240 ctx->key_dec, ctx->rounds, nbytes, 241 walk.iv); 242 spe_end(); 243 244 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 245 } 246 247 return err; 248 } 249 250 static int ppc_cbc_encrypt(struct skcipher_request *req) 251 { 252 return ppc_cbc_crypt(req, true); 253 } 254 255 static int ppc_cbc_decrypt(struct skcipher_request *req) 256 { 257 return ppc_cbc_crypt(req, false); 258 } 259 260 static int ppc_ctr_crypt(struct skcipher_request *req) 261 { 262 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 263 struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 264 struct skcipher_walk walk; 265 unsigned int nbytes; 266 int err; 267 268 err = skcipher_walk_virt(&walk, req, false); 269 270 while ((nbytes = walk.nbytes) != 0) { 271 nbytes = min_t(unsigned int, nbytes, MAX_BYTES); 272 if (nbytes < walk.total) 273 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 274 275 spe_begin(); 276 ppc_crypt_ctr(walk.dst.virt.addr, walk.src.virt.addr, 277 ctx->key_enc, ctx->rounds, nbytes, walk.iv); 278 spe_end(); 279 280 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 281 } 282 283 return err; 284 } 285 286 static int ppc_xts_crypt(struct skcipher_request *req, bool enc) 287 { 288 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 289 struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 290 struct skcipher_walk walk; 291 unsigned int nbytes; 292 int err; 293 u32 *twk; 294 295 err = skcipher_walk_virt(&walk, req, false); 296 twk = ctx->key_twk; 297 298 while ((nbytes = walk.nbytes) != 0) { 299 nbytes = min_t(unsigned int, nbytes, MAX_BYTES); 300 nbytes = round_down(nbytes, AES_BLOCK_SIZE); 301 302 spe_begin(); 303 if (enc) 304 ppc_encrypt_xts(walk.dst.virt.addr, walk.src.virt.addr, 305 ctx->key_enc, ctx->rounds, nbytes, 306 walk.iv, twk); 307 else 308 ppc_decrypt_xts(walk.dst.virt.addr, walk.src.virt.addr, 309 ctx->key_dec, ctx->rounds, nbytes, 310 walk.iv, twk); 311 spe_end(); 312 313 twk = NULL; 314 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 315 } 316 317 return err; 318 } 319 320 static int ppc_xts_encrypt(struct skcipher_request *req) 321 { 322 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 323 struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 324 int tail = req->cryptlen % AES_BLOCK_SIZE; 325 int offset = req->cryptlen - tail - AES_BLOCK_SIZE; 326 struct skcipher_request subreq; 327 u8 b[2][AES_BLOCK_SIZE]; 328 int err; 329 330 if (req->cryptlen < AES_BLOCK_SIZE) 331 return -EINVAL; 332 333 if (tail) { 334 subreq = *req; 335 skcipher_request_set_crypt(&subreq, req->src, req->dst, 336 req->cryptlen - tail, req->iv); 337 req = &subreq; 338 } 339 340 err = ppc_xts_crypt(req, true); 341 if (err || !tail) 342 return err; 343 344 scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE, 0); 345 memcpy(b[1], b[0], tail); 346 scatterwalk_map_and_copy(b[0], req->src, offset + AES_BLOCK_SIZE, tail, 0); 347 348 spe_begin(); 349 ppc_encrypt_xts(b[0], b[0], ctx->key_enc, ctx->rounds, AES_BLOCK_SIZE, 350 req->iv, NULL); 351 spe_end(); 352 353 scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1); 354 355 return 0; 356 } 357 358 static int ppc_xts_decrypt(struct skcipher_request *req) 359 { 360 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 361 struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 362 int tail = req->cryptlen % AES_BLOCK_SIZE; 363 int offset = req->cryptlen - tail - AES_BLOCK_SIZE; 364 struct skcipher_request subreq; 365 u8 b[3][AES_BLOCK_SIZE]; 366 le128 twk; 367 int err; 368 369 if (req->cryptlen < AES_BLOCK_SIZE) 370 return -EINVAL; 371 372 if (tail) { 373 subreq = *req; 374 skcipher_request_set_crypt(&subreq, req->src, req->dst, 375 offset, req->iv); 376 req = &subreq; 377 } 378 379 err = ppc_xts_crypt(req, false); 380 if (err || !tail) 381 return err; 382 383 scatterwalk_map_and_copy(b[1], req->src, offset, AES_BLOCK_SIZE + tail, 0); 384 385 spe_begin(); 386 if (!offset) 387 ppc_encrypt_ecb(req->iv, req->iv, ctx->key_twk, ctx->rounds, 388 AES_BLOCK_SIZE); 389 390 gf128mul_x_ble(&twk, (le128 *)req->iv); 391 392 ppc_decrypt_xts(b[1], b[1], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE, 393 (u8 *)&twk, NULL); 394 memcpy(b[0], b[2], tail); 395 memcpy(b[0] + tail, b[1] + tail, AES_BLOCK_SIZE - tail); 396 ppc_decrypt_xts(b[0], b[0], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE, 397 req->iv, NULL); 398 spe_end(); 399 400 scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1); 401 402 return 0; 403 } 404 405 /* 406 * Algorithm definitions. Disabling alignment (cra_alignmask=0) was chosen 407 * because the e500 platform can handle unaligned reads/writes very efficently. 408 * This improves IPsec thoughput by another few percent. Additionally we assume 409 * that AES context is always aligned to at least 8 bytes because it is created 410 * with kmalloc() in the crypto infrastructure 411 */ 412 413 static struct crypto_alg aes_cipher_alg = { 414 .cra_name = "aes", 415 .cra_driver_name = "aes-ppc-spe", 416 .cra_priority = 300, 417 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 418 .cra_blocksize = AES_BLOCK_SIZE, 419 .cra_ctxsize = sizeof(struct ppc_aes_ctx), 420 .cra_alignmask = 0, 421 .cra_module = THIS_MODULE, 422 .cra_u = { 423 .cipher = { 424 .cia_min_keysize = AES_MIN_KEY_SIZE, 425 .cia_max_keysize = AES_MAX_KEY_SIZE, 426 .cia_setkey = ppc_aes_setkey, 427 .cia_encrypt = ppc_aes_encrypt, 428 .cia_decrypt = ppc_aes_decrypt 429 } 430 } 431 }; 432 433 static struct skcipher_alg aes_skcipher_algs[] = { 434 { 435 .base.cra_name = "ecb(aes)", 436 .base.cra_driver_name = "ecb-ppc-spe", 437 .base.cra_priority = 300, 438 .base.cra_blocksize = AES_BLOCK_SIZE, 439 .base.cra_ctxsize = sizeof(struct ppc_aes_ctx), 440 .base.cra_module = THIS_MODULE, 441 .min_keysize = AES_MIN_KEY_SIZE, 442 .max_keysize = AES_MAX_KEY_SIZE, 443 .setkey = ppc_aes_setkey_skcipher, 444 .encrypt = ppc_ecb_encrypt, 445 .decrypt = ppc_ecb_decrypt, 446 }, { 447 .base.cra_name = "cbc(aes)", 448 .base.cra_driver_name = "cbc-ppc-spe", 449 .base.cra_priority = 300, 450 .base.cra_blocksize = AES_BLOCK_SIZE, 451 .base.cra_ctxsize = sizeof(struct ppc_aes_ctx), 452 .base.cra_module = THIS_MODULE, 453 .min_keysize = AES_MIN_KEY_SIZE, 454 .max_keysize = AES_MAX_KEY_SIZE, 455 .ivsize = AES_BLOCK_SIZE, 456 .setkey = ppc_aes_setkey_skcipher, 457 .encrypt = ppc_cbc_encrypt, 458 .decrypt = ppc_cbc_decrypt, 459 }, { 460 .base.cra_name = "ctr(aes)", 461 .base.cra_driver_name = "ctr-ppc-spe", 462 .base.cra_priority = 300, 463 .base.cra_blocksize = 1, 464 .base.cra_ctxsize = sizeof(struct ppc_aes_ctx), 465 .base.cra_module = THIS_MODULE, 466 .min_keysize = AES_MIN_KEY_SIZE, 467 .max_keysize = AES_MAX_KEY_SIZE, 468 .ivsize = AES_BLOCK_SIZE, 469 .setkey = ppc_aes_setkey_skcipher, 470 .encrypt = ppc_ctr_crypt, 471 .decrypt = ppc_ctr_crypt, 472 .chunksize = AES_BLOCK_SIZE, 473 }, { 474 .base.cra_name = "xts(aes)", 475 .base.cra_driver_name = "xts-ppc-spe", 476 .base.cra_priority = 300, 477 .base.cra_blocksize = AES_BLOCK_SIZE, 478 .base.cra_ctxsize = sizeof(struct ppc_xts_ctx), 479 .base.cra_module = THIS_MODULE, 480 .min_keysize = AES_MIN_KEY_SIZE * 2, 481 .max_keysize = AES_MAX_KEY_SIZE * 2, 482 .ivsize = AES_BLOCK_SIZE, 483 .setkey = ppc_xts_setkey, 484 .encrypt = ppc_xts_encrypt, 485 .decrypt = ppc_xts_decrypt, 486 } 487 }; 488 489 static int __init ppc_aes_mod_init(void) 490 { 491 int err; 492 493 err = crypto_register_alg(&aes_cipher_alg); 494 if (err) 495 return err; 496 497 err = crypto_register_skciphers(aes_skcipher_algs, 498 ARRAY_SIZE(aes_skcipher_algs)); 499 if (err) 500 crypto_unregister_alg(&aes_cipher_alg); 501 return err; 502 } 503 504 static void __exit ppc_aes_mod_fini(void) 505 { 506 crypto_unregister_alg(&aes_cipher_alg); 507 crypto_unregister_skciphers(aes_skcipher_algs, 508 ARRAY_SIZE(aes_skcipher_algs)); 509 } 510 511 module_init(ppc_aes_mod_init); 512 module_exit(ppc_aes_mod_fini); 513 514 MODULE_LICENSE("GPL"); 515 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS, SPE optimized"); 516 517 MODULE_ALIAS_CRYPTO("aes"); 518 MODULE_ALIAS_CRYPTO("ecb(aes)"); 519 MODULE_ALIAS_CRYPTO("cbc(aes)"); 520 MODULE_ALIAS_CRYPTO("ctr(aes)"); 521 MODULE_ALIAS_CRYPTO("xts(aes)"); 522 MODULE_ALIAS_CRYPTO("aes-ppc-spe"); 523