1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Glue code for DES encryption optimized for sparc64 crypto opcodes. 3 * 4 * Copyright (C) 2012 David S. Miller <davem@davemloft.net> 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/crypto.h> 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/mm.h> 13 #include <linux/types.h> 14 #include <crypto/algapi.h> 15 #include <crypto/internal/des.h> 16 17 #include <asm/fpumacro.h> 18 #include <asm/pstate.h> 19 #include <asm/elf.h> 20 21 #include "opcodes.h" 22 23 struct des_sparc64_ctx { 24 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; 25 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; 26 }; 27 28 struct des3_ede_sparc64_ctx { 29 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 30 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 31 }; 32 33 static void encrypt_to_decrypt(u64 *d, const u64 *e) 34 { 35 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1; 36 int i; 37 38 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++) 39 *d++ = *s--; 40 } 41 42 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key); 43 44 static int des_set_key(struct crypto_tfm *tfm, const u8 *key, 45 unsigned int keylen) 46 { 47 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 48 int err; 49 50 /* Even though we have special instructions for key expansion, 51 * we call des_verify_key() so that we don't have to write our own 52 * weak key detection code. 53 */ 54 err = crypto_des_verify_key(tfm, key); 55 if (err) 56 return err; 57 58 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]); 59 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]); 60 61 return 0; 62 } 63 64 extern void des_sparc64_crypt(const u64 *key, const u64 *input, 65 u64 *output); 66 67 static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 68 { 69 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 70 const u64 *K = ctx->encrypt_expkey; 71 72 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 73 } 74 75 static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 76 { 77 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 78 const u64 *K = ctx->decrypt_expkey; 79 80 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 81 } 82 83 extern void des_sparc64_load_keys(const u64 *key); 84 85 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output, 86 unsigned int len); 87 88 #define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1)) 89 90 static int __ecb_crypt(struct blkcipher_desc *desc, 91 struct scatterlist *dst, struct scatterlist *src, 92 unsigned int nbytes, bool encrypt) 93 { 94 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 95 struct blkcipher_walk walk; 96 int err; 97 98 blkcipher_walk_init(&walk, dst, src, nbytes); 99 err = blkcipher_walk_virt(desc, &walk); 100 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 101 102 if (encrypt) 103 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 104 else 105 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 106 while ((nbytes = walk.nbytes)) { 107 unsigned int block_len = nbytes & DES_BLOCK_MASK; 108 109 if (likely(block_len)) { 110 des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr, 111 (u64 *) walk.dst.virt.addr, 112 block_len); 113 } 114 nbytes &= DES_BLOCK_SIZE - 1; 115 err = blkcipher_walk_done(desc, &walk, nbytes); 116 } 117 fprs_write(0); 118 return err; 119 } 120 121 static int ecb_encrypt(struct blkcipher_desc *desc, 122 struct scatterlist *dst, struct scatterlist *src, 123 unsigned int nbytes) 124 { 125 return __ecb_crypt(desc, dst, src, nbytes, true); 126 } 127 128 static int ecb_decrypt(struct blkcipher_desc *desc, 129 struct scatterlist *dst, struct scatterlist *src, 130 unsigned int nbytes) 131 { 132 return __ecb_crypt(desc, dst, src, nbytes, false); 133 } 134 135 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output, 136 unsigned int len, u64 *iv); 137 138 static int cbc_encrypt(struct blkcipher_desc *desc, 139 struct scatterlist *dst, struct scatterlist *src, 140 unsigned int nbytes) 141 { 142 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 143 struct blkcipher_walk walk; 144 int err; 145 146 blkcipher_walk_init(&walk, dst, src, nbytes); 147 err = blkcipher_walk_virt(desc, &walk); 148 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 149 150 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 151 while ((nbytes = walk.nbytes)) { 152 unsigned int block_len = nbytes & DES_BLOCK_MASK; 153 154 if (likely(block_len)) { 155 des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr, 156 (u64 *) walk.dst.virt.addr, 157 block_len, (u64 *) walk.iv); 158 } 159 nbytes &= DES_BLOCK_SIZE - 1; 160 err = blkcipher_walk_done(desc, &walk, nbytes); 161 } 162 fprs_write(0); 163 return err; 164 } 165 166 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, 167 unsigned int len, u64 *iv); 168 169 static int cbc_decrypt(struct blkcipher_desc *desc, 170 struct scatterlist *dst, struct scatterlist *src, 171 unsigned int nbytes) 172 { 173 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 174 struct blkcipher_walk walk; 175 int err; 176 177 blkcipher_walk_init(&walk, dst, src, nbytes); 178 err = blkcipher_walk_virt(desc, &walk); 179 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 180 181 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 182 while ((nbytes = walk.nbytes)) { 183 unsigned int block_len = nbytes & DES_BLOCK_MASK; 184 185 if (likely(block_len)) { 186 des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr, 187 (u64 *) walk.dst.virt.addr, 188 block_len, (u64 *) walk.iv); 189 } 190 nbytes &= DES_BLOCK_SIZE - 1; 191 err = blkcipher_walk_done(desc, &walk, nbytes); 192 } 193 fprs_write(0); 194 return err; 195 } 196 197 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, 198 unsigned int keylen) 199 { 200 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 201 u64 k1[DES_EXPKEY_WORDS / 2]; 202 u64 k2[DES_EXPKEY_WORDS / 2]; 203 u64 k3[DES_EXPKEY_WORDS / 2]; 204 int err; 205 206 err = crypto_des3_ede_verify_key(tfm, key); 207 if (err) 208 return err; 209 210 des_sparc64_key_expand((const u32 *)key, k1); 211 key += DES_KEY_SIZE; 212 des_sparc64_key_expand((const u32 *)key, k2); 213 key += DES_KEY_SIZE; 214 des_sparc64_key_expand((const u32 *)key, k3); 215 216 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1)); 217 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]); 218 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 219 &k3[0], sizeof(k3)); 220 221 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]); 222 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2], 223 &k2[0], sizeof(k2)); 224 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 225 &k1[0]); 226 227 return 0; 228 } 229 230 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input, 231 u64 *output); 232 233 static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 234 { 235 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 236 const u64 *K = ctx->encrypt_expkey; 237 238 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 239 } 240 241 static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 242 { 243 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 244 const u64 *K = ctx->decrypt_expkey; 245 246 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 247 } 248 249 extern void des3_ede_sparc64_load_keys(const u64 *key); 250 251 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input, 252 u64 *output, unsigned int len); 253 254 static int __ecb3_crypt(struct blkcipher_desc *desc, 255 struct scatterlist *dst, struct scatterlist *src, 256 unsigned int nbytes, bool encrypt) 257 { 258 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 259 struct blkcipher_walk walk; 260 const u64 *K; 261 int err; 262 263 blkcipher_walk_init(&walk, dst, src, nbytes); 264 err = blkcipher_walk_virt(desc, &walk); 265 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 266 267 if (encrypt) 268 K = &ctx->encrypt_expkey[0]; 269 else 270 K = &ctx->decrypt_expkey[0]; 271 des3_ede_sparc64_load_keys(K); 272 while ((nbytes = walk.nbytes)) { 273 unsigned int block_len = nbytes & DES_BLOCK_MASK; 274 275 if (likely(block_len)) { 276 const u64 *src64 = (const u64 *)walk.src.virt.addr; 277 des3_ede_sparc64_ecb_crypt(K, src64, 278 (u64 *) walk.dst.virt.addr, 279 block_len); 280 } 281 nbytes &= DES_BLOCK_SIZE - 1; 282 err = blkcipher_walk_done(desc, &walk, nbytes); 283 } 284 fprs_write(0); 285 return err; 286 } 287 288 static int ecb3_encrypt(struct blkcipher_desc *desc, 289 struct scatterlist *dst, struct scatterlist *src, 290 unsigned int nbytes) 291 { 292 return __ecb3_crypt(desc, dst, src, nbytes, true); 293 } 294 295 static int ecb3_decrypt(struct blkcipher_desc *desc, 296 struct scatterlist *dst, struct scatterlist *src, 297 unsigned int nbytes) 298 { 299 return __ecb3_crypt(desc, dst, src, nbytes, false); 300 } 301 302 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input, 303 u64 *output, unsigned int len, 304 u64 *iv); 305 306 static int cbc3_encrypt(struct blkcipher_desc *desc, 307 struct scatterlist *dst, struct scatterlist *src, 308 unsigned int nbytes) 309 { 310 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 311 struct blkcipher_walk walk; 312 const u64 *K; 313 int err; 314 315 blkcipher_walk_init(&walk, dst, src, nbytes); 316 err = blkcipher_walk_virt(desc, &walk); 317 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 318 319 K = &ctx->encrypt_expkey[0]; 320 des3_ede_sparc64_load_keys(K); 321 while ((nbytes = walk.nbytes)) { 322 unsigned int block_len = nbytes & DES_BLOCK_MASK; 323 324 if (likely(block_len)) { 325 const u64 *src64 = (const u64 *)walk.src.virt.addr; 326 des3_ede_sparc64_cbc_encrypt(K, src64, 327 (u64 *) walk.dst.virt.addr, 328 block_len, 329 (u64 *) walk.iv); 330 } 331 nbytes &= DES_BLOCK_SIZE - 1; 332 err = blkcipher_walk_done(desc, &walk, nbytes); 333 } 334 fprs_write(0); 335 return err; 336 } 337 338 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input, 339 u64 *output, unsigned int len, 340 u64 *iv); 341 342 static int cbc3_decrypt(struct blkcipher_desc *desc, 343 struct scatterlist *dst, struct scatterlist *src, 344 unsigned int nbytes) 345 { 346 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 347 struct blkcipher_walk walk; 348 const u64 *K; 349 int err; 350 351 blkcipher_walk_init(&walk, dst, src, nbytes); 352 err = blkcipher_walk_virt(desc, &walk); 353 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 354 355 K = &ctx->decrypt_expkey[0]; 356 des3_ede_sparc64_load_keys(K); 357 while ((nbytes = walk.nbytes)) { 358 unsigned int block_len = nbytes & DES_BLOCK_MASK; 359 360 if (likely(block_len)) { 361 const u64 *src64 = (const u64 *)walk.src.virt.addr; 362 des3_ede_sparc64_cbc_decrypt(K, src64, 363 (u64 *) walk.dst.virt.addr, 364 block_len, 365 (u64 *) walk.iv); 366 } 367 nbytes &= DES_BLOCK_SIZE - 1; 368 err = blkcipher_walk_done(desc, &walk, nbytes); 369 } 370 fprs_write(0); 371 return err; 372 } 373 374 static struct crypto_alg algs[] = { { 375 .cra_name = "des", 376 .cra_driver_name = "des-sparc64", 377 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 378 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 379 .cra_blocksize = DES_BLOCK_SIZE, 380 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 381 .cra_alignmask = 7, 382 .cra_module = THIS_MODULE, 383 .cra_u = { 384 .cipher = { 385 .cia_min_keysize = DES_KEY_SIZE, 386 .cia_max_keysize = DES_KEY_SIZE, 387 .cia_setkey = des_set_key, 388 .cia_encrypt = sparc_des_encrypt, 389 .cia_decrypt = sparc_des_decrypt 390 } 391 } 392 }, { 393 .cra_name = "ecb(des)", 394 .cra_driver_name = "ecb-des-sparc64", 395 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 396 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 397 .cra_blocksize = DES_BLOCK_SIZE, 398 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 399 .cra_alignmask = 7, 400 .cra_type = &crypto_blkcipher_type, 401 .cra_module = THIS_MODULE, 402 .cra_u = { 403 .blkcipher = { 404 .min_keysize = DES_KEY_SIZE, 405 .max_keysize = DES_KEY_SIZE, 406 .setkey = des_set_key, 407 .encrypt = ecb_encrypt, 408 .decrypt = ecb_decrypt, 409 }, 410 }, 411 }, { 412 .cra_name = "cbc(des)", 413 .cra_driver_name = "cbc-des-sparc64", 414 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 415 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 416 .cra_blocksize = DES_BLOCK_SIZE, 417 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 418 .cra_alignmask = 7, 419 .cra_type = &crypto_blkcipher_type, 420 .cra_module = THIS_MODULE, 421 .cra_u = { 422 .blkcipher = { 423 .min_keysize = DES_KEY_SIZE, 424 .max_keysize = DES_KEY_SIZE, 425 .ivsize = DES_BLOCK_SIZE, 426 .setkey = des_set_key, 427 .encrypt = cbc_encrypt, 428 .decrypt = cbc_decrypt, 429 }, 430 }, 431 }, { 432 .cra_name = "des3_ede", 433 .cra_driver_name = "des3_ede-sparc64", 434 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 435 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 436 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 437 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 438 .cra_alignmask = 7, 439 .cra_module = THIS_MODULE, 440 .cra_u = { 441 .cipher = { 442 .cia_min_keysize = DES3_EDE_KEY_SIZE, 443 .cia_max_keysize = DES3_EDE_KEY_SIZE, 444 .cia_setkey = des3_ede_set_key, 445 .cia_encrypt = sparc_des3_ede_encrypt, 446 .cia_decrypt = sparc_des3_ede_decrypt 447 } 448 } 449 }, { 450 .cra_name = "ecb(des3_ede)", 451 .cra_driver_name = "ecb-des3_ede-sparc64", 452 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 453 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 454 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 455 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 456 .cra_alignmask = 7, 457 .cra_type = &crypto_blkcipher_type, 458 .cra_module = THIS_MODULE, 459 .cra_u = { 460 .blkcipher = { 461 .min_keysize = DES3_EDE_KEY_SIZE, 462 .max_keysize = DES3_EDE_KEY_SIZE, 463 .setkey = des3_ede_set_key, 464 .encrypt = ecb3_encrypt, 465 .decrypt = ecb3_decrypt, 466 }, 467 }, 468 }, { 469 .cra_name = "cbc(des3_ede)", 470 .cra_driver_name = "cbc-des3_ede-sparc64", 471 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 472 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 473 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 474 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 475 .cra_alignmask = 7, 476 .cra_type = &crypto_blkcipher_type, 477 .cra_module = THIS_MODULE, 478 .cra_u = { 479 .blkcipher = { 480 .min_keysize = DES3_EDE_KEY_SIZE, 481 .max_keysize = DES3_EDE_KEY_SIZE, 482 .ivsize = DES3_EDE_BLOCK_SIZE, 483 .setkey = des3_ede_set_key, 484 .encrypt = cbc3_encrypt, 485 .decrypt = cbc3_decrypt, 486 }, 487 }, 488 } }; 489 490 static bool __init sparc64_has_des_opcode(void) 491 { 492 unsigned long cfr; 493 494 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 495 return false; 496 497 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 498 if (!(cfr & CFR_DES)) 499 return false; 500 501 return true; 502 } 503 504 static int __init des_sparc64_mod_init(void) 505 { 506 if (sparc64_has_des_opcode()) { 507 pr_info("Using sparc64 des opcodes optimized DES implementation\n"); 508 return crypto_register_algs(algs, ARRAY_SIZE(algs)); 509 } 510 pr_info("sparc64 des opcodes not available.\n"); 511 return -ENODEV; 512 } 513 514 static void __exit des_sparc64_mod_fini(void) 515 { 516 crypto_unregister_algs(algs, ARRAY_SIZE(algs)); 517 } 518 519 module_init(des_sparc64_mod_init); 520 module_exit(des_sparc64_mod_fini); 521 522 MODULE_LICENSE("GPL"); 523 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); 524 525 MODULE_ALIAS_CRYPTO("des"); 526 MODULE_ALIAS_CRYPTO("des3_ede"); 527 528 #include "crop_devid.c" 529