1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Glue code for DES encryption optimized for sparc64 crypto opcodes. 3 * 4 * Copyright (C) 2012 David S. Miller <davem@davemloft.net> 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/crypto.h> 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/mm.h> 13 #include <linux/types.h> 14 #include <crypto/algapi.h> 15 #include <crypto/internal/des.h> 16 #include <crypto/internal/skcipher.h> 17 18 #include <asm/fpumacro.h> 19 #include <asm/pstate.h> 20 #include <asm/elf.h> 21 22 #include "opcodes.h" 23 24 struct des_sparc64_ctx { 25 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; 26 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; 27 }; 28 29 struct des3_ede_sparc64_ctx { 30 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 31 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 32 }; 33 34 static void encrypt_to_decrypt(u64 *d, const u64 *e) 35 { 36 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1; 37 int i; 38 39 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++) 40 *d++ = *s--; 41 } 42 43 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key); 44 45 static int des_set_key(struct crypto_tfm *tfm, const u8 *key, 46 unsigned int keylen) 47 { 48 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 49 int err; 50 51 /* Even though we have special instructions for key expansion, 52 * we call des_verify_key() so that we don't have to write our own 53 * weak key detection code. 54 */ 55 err = crypto_des_verify_key(tfm, key); 56 if (err) 57 return err; 58 59 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]); 60 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]); 61 62 return 0; 63 } 64 65 static int des_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key, 66 unsigned int keylen) 67 { 68 return des_set_key(crypto_skcipher_tfm(tfm), key, keylen); 69 } 70 71 extern void des_sparc64_crypt(const u64 *key, const u64 *input, 72 u64 *output); 73 74 static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 75 { 76 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 77 const u64 *K = ctx->encrypt_expkey; 78 79 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 80 } 81 82 static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 83 { 84 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 85 const u64 *K = ctx->decrypt_expkey; 86 87 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 88 } 89 90 extern void des_sparc64_load_keys(const u64 *key); 91 92 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output, 93 unsigned int len); 94 95 static int __ecb_crypt(struct skcipher_request *req, bool encrypt) 96 { 97 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 98 const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); 99 struct skcipher_walk walk; 100 unsigned int nbytes; 101 int err; 102 103 err = skcipher_walk_virt(&walk, req, true); 104 if (err) 105 return err; 106 107 if (encrypt) 108 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 109 else 110 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 111 while ((nbytes = walk.nbytes) != 0) { 112 des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr, 113 round_down(nbytes, DES_BLOCK_SIZE)); 114 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); 115 } 116 fprs_write(0); 117 return err; 118 } 119 120 static int ecb_encrypt(struct skcipher_request *req) 121 { 122 return __ecb_crypt(req, true); 123 } 124 125 static int ecb_decrypt(struct skcipher_request *req) 126 { 127 return __ecb_crypt(req, false); 128 } 129 130 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output, 131 unsigned int len, u64 *iv); 132 133 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, 134 unsigned int len, u64 *iv); 135 136 static int __cbc_crypt(struct skcipher_request *req, bool encrypt) 137 { 138 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 139 const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); 140 struct skcipher_walk walk; 141 unsigned int nbytes; 142 int err; 143 144 err = skcipher_walk_virt(&walk, req, true); 145 if (err) 146 return err; 147 148 if (encrypt) 149 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 150 else 151 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 152 while ((nbytes = walk.nbytes) != 0) { 153 if (encrypt) 154 des_sparc64_cbc_encrypt(walk.src.virt.addr, 155 walk.dst.virt.addr, 156 round_down(nbytes, 157 DES_BLOCK_SIZE), 158 walk.iv); 159 else 160 des_sparc64_cbc_decrypt(walk.src.virt.addr, 161 walk.dst.virt.addr, 162 round_down(nbytes, 163 DES_BLOCK_SIZE), 164 walk.iv); 165 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); 166 } 167 fprs_write(0); 168 return err; 169 } 170 171 static int cbc_encrypt(struct skcipher_request *req) 172 { 173 return __cbc_crypt(req, true); 174 } 175 176 static int cbc_decrypt(struct skcipher_request *req) 177 { 178 return __cbc_crypt(req, false); 179 } 180 181 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, 182 unsigned int keylen) 183 { 184 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 185 u64 k1[DES_EXPKEY_WORDS / 2]; 186 u64 k2[DES_EXPKEY_WORDS / 2]; 187 u64 k3[DES_EXPKEY_WORDS / 2]; 188 int err; 189 190 err = crypto_des3_ede_verify_key(tfm, key); 191 if (err) 192 return err; 193 194 des_sparc64_key_expand((const u32 *)key, k1); 195 key += DES_KEY_SIZE; 196 des_sparc64_key_expand((const u32 *)key, k2); 197 key += DES_KEY_SIZE; 198 des_sparc64_key_expand((const u32 *)key, k3); 199 200 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1)); 201 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]); 202 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 203 &k3[0], sizeof(k3)); 204 205 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]); 206 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2], 207 &k2[0], sizeof(k2)); 208 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 209 &k1[0]); 210 211 return 0; 212 } 213 214 static int des3_ede_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key, 215 unsigned int keylen) 216 { 217 return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen); 218 } 219 220 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input, 221 u64 *output); 222 223 static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 224 { 225 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 226 const u64 *K = ctx->encrypt_expkey; 227 228 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 229 } 230 231 static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 232 { 233 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 234 const u64 *K = ctx->decrypt_expkey; 235 236 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 237 } 238 239 extern void des3_ede_sparc64_load_keys(const u64 *key); 240 241 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input, 242 u64 *output, unsigned int len); 243 244 static int __ecb3_crypt(struct skcipher_request *req, bool encrypt) 245 { 246 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 247 const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); 248 struct skcipher_walk walk; 249 const u64 *K; 250 unsigned int nbytes; 251 int err; 252 253 err = skcipher_walk_virt(&walk, req, true); 254 if (err) 255 return err; 256 257 if (encrypt) 258 K = &ctx->encrypt_expkey[0]; 259 else 260 K = &ctx->decrypt_expkey[0]; 261 des3_ede_sparc64_load_keys(K); 262 while ((nbytes = walk.nbytes) != 0) { 263 des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr, 264 walk.dst.virt.addr, 265 round_down(nbytes, DES_BLOCK_SIZE)); 266 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); 267 } 268 fprs_write(0); 269 return err; 270 } 271 272 static int ecb3_encrypt(struct skcipher_request *req) 273 { 274 return __ecb3_crypt(req, true); 275 } 276 277 static int ecb3_decrypt(struct skcipher_request *req) 278 { 279 return __ecb3_crypt(req, false); 280 } 281 282 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input, 283 u64 *output, unsigned int len, 284 u64 *iv); 285 286 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input, 287 u64 *output, unsigned int len, 288 u64 *iv); 289 290 static int __cbc3_crypt(struct skcipher_request *req, bool encrypt) 291 { 292 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 293 const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); 294 struct skcipher_walk walk; 295 const u64 *K; 296 unsigned int nbytes; 297 int err; 298 299 err = skcipher_walk_virt(&walk, req, true); 300 if (err) 301 return err; 302 303 if (encrypt) 304 K = &ctx->encrypt_expkey[0]; 305 else 306 K = &ctx->decrypt_expkey[0]; 307 des3_ede_sparc64_load_keys(K); 308 while ((nbytes = walk.nbytes) != 0) { 309 if (encrypt) 310 des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr, 311 walk.dst.virt.addr, 312 round_down(nbytes, 313 DES_BLOCK_SIZE), 314 walk.iv); 315 else 316 des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr, 317 walk.dst.virt.addr, 318 round_down(nbytes, 319 DES_BLOCK_SIZE), 320 walk.iv); 321 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); 322 } 323 fprs_write(0); 324 return err; 325 } 326 327 static int cbc3_encrypt(struct skcipher_request *req) 328 { 329 return __cbc3_crypt(req, true); 330 } 331 332 static int cbc3_decrypt(struct skcipher_request *req) 333 { 334 return __cbc3_crypt(req, false); 335 } 336 337 static struct crypto_alg cipher_algs[] = { 338 { 339 .cra_name = "des", 340 .cra_driver_name = "des-sparc64", 341 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 342 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 343 .cra_blocksize = DES_BLOCK_SIZE, 344 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 345 .cra_alignmask = 7, 346 .cra_module = THIS_MODULE, 347 .cra_u = { 348 .cipher = { 349 .cia_min_keysize = DES_KEY_SIZE, 350 .cia_max_keysize = DES_KEY_SIZE, 351 .cia_setkey = des_set_key, 352 .cia_encrypt = sparc_des_encrypt, 353 .cia_decrypt = sparc_des_decrypt 354 } 355 } 356 }, { 357 .cra_name = "des3_ede", 358 .cra_driver_name = "des3_ede-sparc64", 359 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 360 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 361 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 362 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 363 .cra_alignmask = 7, 364 .cra_module = THIS_MODULE, 365 .cra_u = { 366 .cipher = { 367 .cia_min_keysize = DES3_EDE_KEY_SIZE, 368 .cia_max_keysize = DES3_EDE_KEY_SIZE, 369 .cia_setkey = des3_ede_set_key, 370 .cia_encrypt = sparc_des3_ede_encrypt, 371 .cia_decrypt = sparc_des3_ede_decrypt 372 } 373 } 374 } 375 }; 376 377 static struct skcipher_alg skcipher_algs[] = { 378 { 379 .base.cra_name = "ecb(des)", 380 .base.cra_driver_name = "ecb-des-sparc64", 381 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, 382 .base.cra_blocksize = DES_BLOCK_SIZE, 383 .base.cra_ctxsize = sizeof(struct des_sparc64_ctx), 384 .base.cra_alignmask = 7, 385 .base.cra_module = THIS_MODULE, 386 .min_keysize = DES_KEY_SIZE, 387 .max_keysize = DES_KEY_SIZE, 388 .setkey = des_set_key_skcipher, 389 .encrypt = ecb_encrypt, 390 .decrypt = ecb_decrypt, 391 }, { 392 .base.cra_name = "cbc(des)", 393 .base.cra_driver_name = "cbc-des-sparc64", 394 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, 395 .base.cra_blocksize = DES_BLOCK_SIZE, 396 .base.cra_ctxsize = sizeof(struct des_sparc64_ctx), 397 .base.cra_alignmask = 7, 398 .base.cra_module = THIS_MODULE, 399 .min_keysize = DES_KEY_SIZE, 400 .max_keysize = DES_KEY_SIZE, 401 .ivsize = DES_BLOCK_SIZE, 402 .setkey = des_set_key_skcipher, 403 .encrypt = cbc_encrypt, 404 .decrypt = cbc_decrypt, 405 }, { 406 .base.cra_name = "ecb(des3_ede)", 407 .base.cra_driver_name = "ecb-des3_ede-sparc64", 408 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, 409 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 410 .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 411 .base.cra_alignmask = 7, 412 .base.cra_module = THIS_MODULE, 413 .min_keysize = DES3_EDE_KEY_SIZE, 414 .max_keysize = DES3_EDE_KEY_SIZE, 415 .setkey = des3_ede_set_key_skcipher, 416 .encrypt = ecb3_encrypt, 417 .decrypt = ecb3_decrypt, 418 }, { 419 .base.cra_name = "cbc(des3_ede)", 420 .base.cra_driver_name = "cbc-des3_ede-sparc64", 421 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, 422 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 423 .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 424 .base.cra_alignmask = 7, 425 .base.cra_module = THIS_MODULE, 426 .min_keysize = DES3_EDE_KEY_SIZE, 427 .max_keysize = DES3_EDE_KEY_SIZE, 428 .ivsize = DES3_EDE_BLOCK_SIZE, 429 .setkey = des3_ede_set_key_skcipher, 430 .encrypt = cbc3_encrypt, 431 .decrypt = cbc3_decrypt, 432 } 433 }; 434 435 static bool __init sparc64_has_des_opcode(void) 436 { 437 unsigned long cfr; 438 439 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 440 return false; 441 442 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 443 if (!(cfr & CFR_DES)) 444 return false; 445 446 return true; 447 } 448 449 static int __init des_sparc64_mod_init(void) 450 { 451 int err; 452 453 if (!sparc64_has_des_opcode()) { 454 pr_info("sparc64 des opcodes not available.\n"); 455 return -ENODEV; 456 } 457 pr_info("Using sparc64 des opcodes optimized DES implementation\n"); 458 err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); 459 if (err) 460 return err; 461 err = crypto_register_skciphers(skcipher_algs, 462 ARRAY_SIZE(skcipher_algs)); 463 if (err) 464 crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); 465 return err; 466 } 467 468 static void __exit des_sparc64_mod_fini(void) 469 { 470 crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); 471 crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); 472 } 473 474 module_init(des_sparc64_mod_init); 475 module_exit(des_sparc64_mod_fini); 476 477 MODULE_LICENSE("GPL"); 478 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); 479 480 MODULE_ALIAS_CRYPTO("des"); 481 MODULE_ALIAS_CRYPTO("des3_ede"); 482 483 #include "crop_devid.c" 484