1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Glue code for DES encryption optimized for sparc64 crypto opcodes. 3 * 4 * Copyright (C) 2012 David S. Miller <davem@davemloft.net> 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/crypto.h> 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/mm.h> 13 #include <linux/types.h> 14 #include <crypto/algapi.h> 15 #include <crypto/des.h> 16 17 #include <asm/fpumacro.h> 18 #include <asm/pstate.h> 19 #include <asm/elf.h> 20 21 #include "opcodes.h" 22 23 struct des_sparc64_ctx { 24 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; 25 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; 26 }; 27 28 struct des3_ede_sparc64_ctx { 29 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 30 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 31 }; 32 33 static void encrypt_to_decrypt(u64 *d, const u64 *e) 34 { 35 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1; 36 int i; 37 38 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++) 39 *d++ = *s--; 40 } 41 42 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key); 43 44 static int des_set_key(struct crypto_tfm *tfm, const u8 *key, 45 unsigned int keylen) 46 { 47 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 48 u32 *flags = &tfm->crt_flags; 49 u32 tmp[DES_EXPKEY_WORDS]; 50 int ret; 51 52 /* Even though we have special instructions for key expansion, 53 * we call des_ekey() so that we don't have to write our own 54 * weak key detection code. 55 */ 56 ret = des_ekey(tmp, key); 57 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { 58 *flags |= CRYPTO_TFM_RES_WEAK_KEY; 59 return -EINVAL; 60 } 61 62 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]); 63 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]); 64 65 return 0; 66 } 67 68 extern void des_sparc64_crypt(const u64 *key, const u64 *input, 69 u64 *output); 70 71 static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 72 { 73 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 74 const u64 *K = ctx->encrypt_expkey; 75 76 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 77 } 78 79 static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 80 { 81 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 82 const u64 *K = ctx->decrypt_expkey; 83 84 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 85 } 86 87 extern void des_sparc64_load_keys(const u64 *key); 88 89 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output, 90 unsigned int len); 91 92 #define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1)) 93 94 static int __ecb_crypt(struct blkcipher_desc *desc, 95 struct scatterlist *dst, struct scatterlist *src, 96 unsigned int nbytes, bool encrypt) 97 { 98 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 99 struct blkcipher_walk walk; 100 int err; 101 102 blkcipher_walk_init(&walk, dst, src, nbytes); 103 err = blkcipher_walk_virt(desc, &walk); 104 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 105 106 if (encrypt) 107 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 108 else 109 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 110 while ((nbytes = walk.nbytes)) { 111 unsigned int block_len = nbytes & DES_BLOCK_MASK; 112 113 if (likely(block_len)) { 114 des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr, 115 (u64 *) walk.dst.virt.addr, 116 block_len); 117 } 118 nbytes &= DES_BLOCK_SIZE - 1; 119 err = blkcipher_walk_done(desc, &walk, nbytes); 120 } 121 fprs_write(0); 122 return err; 123 } 124 125 static int ecb_encrypt(struct blkcipher_desc *desc, 126 struct scatterlist *dst, struct scatterlist *src, 127 unsigned int nbytes) 128 { 129 return __ecb_crypt(desc, dst, src, nbytes, true); 130 } 131 132 static int ecb_decrypt(struct blkcipher_desc *desc, 133 struct scatterlist *dst, struct scatterlist *src, 134 unsigned int nbytes) 135 { 136 return __ecb_crypt(desc, dst, src, nbytes, false); 137 } 138 139 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output, 140 unsigned int len, u64 *iv); 141 142 static int cbc_encrypt(struct blkcipher_desc *desc, 143 struct scatterlist *dst, struct scatterlist *src, 144 unsigned int nbytes) 145 { 146 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 147 struct blkcipher_walk walk; 148 int err; 149 150 blkcipher_walk_init(&walk, dst, src, nbytes); 151 err = blkcipher_walk_virt(desc, &walk); 152 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 153 154 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 155 while ((nbytes = walk.nbytes)) { 156 unsigned int block_len = nbytes & DES_BLOCK_MASK; 157 158 if (likely(block_len)) { 159 des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr, 160 (u64 *) walk.dst.virt.addr, 161 block_len, (u64 *) walk.iv); 162 } 163 nbytes &= DES_BLOCK_SIZE - 1; 164 err = blkcipher_walk_done(desc, &walk, nbytes); 165 } 166 fprs_write(0); 167 return err; 168 } 169 170 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, 171 unsigned int len, u64 *iv); 172 173 static int cbc_decrypt(struct blkcipher_desc *desc, 174 struct scatterlist *dst, struct scatterlist *src, 175 unsigned int nbytes) 176 { 177 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 178 struct blkcipher_walk walk; 179 int err; 180 181 blkcipher_walk_init(&walk, dst, src, nbytes); 182 err = blkcipher_walk_virt(desc, &walk); 183 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 184 185 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 186 while ((nbytes = walk.nbytes)) { 187 unsigned int block_len = nbytes & DES_BLOCK_MASK; 188 189 if (likely(block_len)) { 190 des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr, 191 (u64 *) walk.dst.virt.addr, 192 block_len, (u64 *) walk.iv); 193 } 194 nbytes &= DES_BLOCK_SIZE - 1; 195 err = blkcipher_walk_done(desc, &walk, nbytes); 196 } 197 fprs_write(0); 198 return err; 199 } 200 201 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, 202 unsigned int keylen) 203 { 204 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 205 u32 *flags = &tfm->crt_flags; 206 u64 k1[DES_EXPKEY_WORDS / 2]; 207 u64 k2[DES_EXPKEY_WORDS / 2]; 208 u64 k3[DES_EXPKEY_WORDS / 2]; 209 int err; 210 211 err = __des3_verify_key(flags, key); 212 if (unlikely(err)) 213 return err; 214 215 des_sparc64_key_expand((const u32 *)key, k1); 216 key += DES_KEY_SIZE; 217 des_sparc64_key_expand((const u32 *)key, k2); 218 key += DES_KEY_SIZE; 219 des_sparc64_key_expand((const u32 *)key, k3); 220 221 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1)); 222 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]); 223 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 224 &k3[0], sizeof(k3)); 225 226 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]); 227 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2], 228 &k2[0], sizeof(k2)); 229 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 230 &k1[0]); 231 232 return 0; 233 } 234 235 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input, 236 u64 *output); 237 238 static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 239 { 240 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 241 const u64 *K = ctx->encrypt_expkey; 242 243 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 244 } 245 246 static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 247 { 248 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 249 const u64 *K = ctx->decrypt_expkey; 250 251 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 252 } 253 254 extern void des3_ede_sparc64_load_keys(const u64 *key); 255 256 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input, 257 u64 *output, unsigned int len); 258 259 static int __ecb3_crypt(struct blkcipher_desc *desc, 260 struct scatterlist *dst, struct scatterlist *src, 261 unsigned int nbytes, bool encrypt) 262 { 263 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 264 struct blkcipher_walk walk; 265 const u64 *K; 266 int err; 267 268 blkcipher_walk_init(&walk, dst, src, nbytes); 269 err = blkcipher_walk_virt(desc, &walk); 270 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 271 272 if (encrypt) 273 K = &ctx->encrypt_expkey[0]; 274 else 275 K = &ctx->decrypt_expkey[0]; 276 des3_ede_sparc64_load_keys(K); 277 while ((nbytes = walk.nbytes)) { 278 unsigned int block_len = nbytes & DES_BLOCK_MASK; 279 280 if (likely(block_len)) { 281 const u64 *src64 = (const u64 *)walk.src.virt.addr; 282 des3_ede_sparc64_ecb_crypt(K, src64, 283 (u64 *) walk.dst.virt.addr, 284 block_len); 285 } 286 nbytes &= DES_BLOCK_SIZE - 1; 287 err = blkcipher_walk_done(desc, &walk, nbytes); 288 } 289 fprs_write(0); 290 return err; 291 } 292 293 static int ecb3_encrypt(struct blkcipher_desc *desc, 294 struct scatterlist *dst, struct scatterlist *src, 295 unsigned int nbytes) 296 { 297 return __ecb3_crypt(desc, dst, src, nbytes, true); 298 } 299 300 static int ecb3_decrypt(struct blkcipher_desc *desc, 301 struct scatterlist *dst, struct scatterlist *src, 302 unsigned int nbytes) 303 { 304 return __ecb3_crypt(desc, dst, src, nbytes, false); 305 } 306 307 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input, 308 u64 *output, unsigned int len, 309 u64 *iv); 310 311 static int cbc3_encrypt(struct blkcipher_desc *desc, 312 struct scatterlist *dst, struct scatterlist *src, 313 unsigned int nbytes) 314 { 315 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 316 struct blkcipher_walk walk; 317 const u64 *K; 318 int err; 319 320 blkcipher_walk_init(&walk, dst, src, nbytes); 321 err = blkcipher_walk_virt(desc, &walk); 322 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 323 324 K = &ctx->encrypt_expkey[0]; 325 des3_ede_sparc64_load_keys(K); 326 while ((nbytes = walk.nbytes)) { 327 unsigned int block_len = nbytes & DES_BLOCK_MASK; 328 329 if (likely(block_len)) { 330 const u64 *src64 = (const u64 *)walk.src.virt.addr; 331 des3_ede_sparc64_cbc_encrypt(K, src64, 332 (u64 *) walk.dst.virt.addr, 333 block_len, 334 (u64 *) walk.iv); 335 } 336 nbytes &= DES_BLOCK_SIZE - 1; 337 err = blkcipher_walk_done(desc, &walk, nbytes); 338 } 339 fprs_write(0); 340 return err; 341 } 342 343 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input, 344 u64 *output, unsigned int len, 345 u64 *iv); 346 347 static int cbc3_decrypt(struct blkcipher_desc *desc, 348 struct scatterlist *dst, struct scatterlist *src, 349 unsigned int nbytes) 350 { 351 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 352 struct blkcipher_walk walk; 353 const u64 *K; 354 int err; 355 356 blkcipher_walk_init(&walk, dst, src, nbytes); 357 err = blkcipher_walk_virt(desc, &walk); 358 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 359 360 K = &ctx->decrypt_expkey[0]; 361 des3_ede_sparc64_load_keys(K); 362 while ((nbytes = walk.nbytes)) { 363 unsigned int block_len = nbytes & DES_BLOCK_MASK; 364 365 if (likely(block_len)) { 366 const u64 *src64 = (const u64 *)walk.src.virt.addr; 367 des3_ede_sparc64_cbc_decrypt(K, src64, 368 (u64 *) walk.dst.virt.addr, 369 block_len, 370 (u64 *) walk.iv); 371 } 372 nbytes &= DES_BLOCK_SIZE - 1; 373 err = blkcipher_walk_done(desc, &walk, nbytes); 374 } 375 fprs_write(0); 376 return err; 377 } 378 379 static struct crypto_alg algs[] = { { 380 .cra_name = "des", 381 .cra_driver_name = "des-sparc64", 382 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 383 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 384 .cra_blocksize = DES_BLOCK_SIZE, 385 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 386 .cra_alignmask = 7, 387 .cra_module = THIS_MODULE, 388 .cra_u = { 389 .cipher = { 390 .cia_min_keysize = DES_KEY_SIZE, 391 .cia_max_keysize = DES_KEY_SIZE, 392 .cia_setkey = des_set_key, 393 .cia_encrypt = des_encrypt, 394 .cia_decrypt = des_decrypt 395 } 396 } 397 }, { 398 .cra_name = "ecb(des)", 399 .cra_driver_name = "ecb-des-sparc64", 400 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 401 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 402 .cra_blocksize = DES_BLOCK_SIZE, 403 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 404 .cra_alignmask = 7, 405 .cra_type = &crypto_blkcipher_type, 406 .cra_module = THIS_MODULE, 407 .cra_u = { 408 .blkcipher = { 409 .min_keysize = DES_KEY_SIZE, 410 .max_keysize = DES_KEY_SIZE, 411 .setkey = des_set_key, 412 .encrypt = ecb_encrypt, 413 .decrypt = ecb_decrypt, 414 }, 415 }, 416 }, { 417 .cra_name = "cbc(des)", 418 .cra_driver_name = "cbc-des-sparc64", 419 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 420 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 421 .cra_blocksize = DES_BLOCK_SIZE, 422 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 423 .cra_alignmask = 7, 424 .cra_type = &crypto_blkcipher_type, 425 .cra_module = THIS_MODULE, 426 .cra_u = { 427 .blkcipher = { 428 .min_keysize = DES_KEY_SIZE, 429 .max_keysize = DES_KEY_SIZE, 430 .ivsize = DES_BLOCK_SIZE, 431 .setkey = des_set_key, 432 .encrypt = cbc_encrypt, 433 .decrypt = cbc_decrypt, 434 }, 435 }, 436 }, { 437 .cra_name = "des3_ede", 438 .cra_driver_name = "des3_ede-sparc64", 439 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 440 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 441 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 442 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 443 .cra_alignmask = 7, 444 .cra_module = THIS_MODULE, 445 .cra_u = { 446 .cipher = { 447 .cia_min_keysize = DES3_EDE_KEY_SIZE, 448 .cia_max_keysize = DES3_EDE_KEY_SIZE, 449 .cia_setkey = des3_ede_set_key, 450 .cia_encrypt = des3_ede_encrypt, 451 .cia_decrypt = des3_ede_decrypt 452 } 453 } 454 }, { 455 .cra_name = "ecb(des3_ede)", 456 .cra_driver_name = "ecb-des3_ede-sparc64", 457 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 458 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 459 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 460 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 461 .cra_alignmask = 7, 462 .cra_type = &crypto_blkcipher_type, 463 .cra_module = THIS_MODULE, 464 .cra_u = { 465 .blkcipher = { 466 .min_keysize = DES3_EDE_KEY_SIZE, 467 .max_keysize = DES3_EDE_KEY_SIZE, 468 .setkey = des3_ede_set_key, 469 .encrypt = ecb3_encrypt, 470 .decrypt = ecb3_decrypt, 471 }, 472 }, 473 }, { 474 .cra_name = "cbc(des3_ede)", 475 .cra_driver_name = "cbc-des3_ede-sparc64", 476 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 477 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 478 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 479 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 480 .cra_alignmask = 7, 481 .cra_type = &crypto_blkcipher_type, 482 .cra_module = THIS_MODULE, 483 .cra_u = { 484 .blkcipher = { 485 .min_keysize = DES3_EDE_KEY_SIZE, 486 .max_keysize = DES3_EDE_KEY_SIZE, 487 .ivsize = DES3_EDE_BLOCK_SIZE, 488 .setkey = des3_ede_set_key, 489 .encrypt = cbc3_encrypt, 490 .decrypt = cbc3_decrypt, 491 }, 492 }, 493 } }; 494 495 static bool __init sparc64_has_des_opcode(void) 496 { 497 unsigned long cfr; 498 499 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 500 return false; 501 502 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 503 if (!(cfr & CFR_DES)) 504 return false; 505 506 return true; 507 } 508 509 static int __init des_sparc64_mod_init(void) 510 { 511 if (sparc64_has_des_opcode()) { 512 pr_info("Using sparc64 des opcodes optimized DES implementation\n"); 513 return crypto_register_algs(algs, ARRAY_SIZE(algs)); 514 } 515 pr_info("sparc64 des opcodes not available.\n"); 516 return -ENODEV; 517 } 518 519 static void __exit des_sparc64_mod_fini(void) 520 { 521 crypto_unregister_algs(algs, ARRAY_SIZE(algs)); 522 } 523 524 module_init(des_sparc64_mod_init); 525 module_exit(des_sparc64_mod_fini); 526 527 MODULE_LICENSE("GPL"); 528 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); 529 530 MODULE_ALIAS_CRYPTO("des"); 531 MODULE_ALIAS_CRYPTO("des3_ede"); 532 533 #include "crop_devid.c" 534