1 /* Glue code for DES encryption optimized for sparc64 crypto opcodes. 2 * 3 * Copyright (C) 2012 David S. Miller <davem@davemloft.net> 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/crypto.h> 9 #include <linux/init.h> 10 #include <linux/module.h> 11 #include <linux/mm.h> 12 #include <linux/types.h> 13 #include <crypto/algapi.h> 14 #include <crypto/des.h> 15 16 #include <asm/fpumacro.h> 17 #include <asm/pstate.h> 18 #include <asm/elf.h> 19 20 #include "opcodes.h" 21 22 struct des_sparc64_ctx { 23 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; 24 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; 25 }; 26 27 struct des3_ede_sparc64_ctx { 28 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 29 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 30 }; 31 32 static void encrypt_to_decrypt(u64 *d, const u64 *e) 33 { 34 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1; 35 int i; 36 37 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++) 38 *d++ = *s--; 39 } 40 41 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key); 42 43 static int des_set_key(struct crypto_tfm *tfm, const u8 *key, 44 unsigned int keylen) 45 { 46 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 47 u32 *flags = &tfm->crt_flags; 48 u32 tmp[DES_EXPKEY_WORDS]; 49 int ret; 50 51 /* Even though we have special instructions for key expansion, 52 * we call des_ekey() so that we don't have to write our own 53 * weak key detection code. 54 */ 55 ret = des_ekey(tmp, key); 56 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) { 57 *flags |= CRYPTO_TFM_RES_WEAK_KEY; 58 return -EINVAL; 59 } 60 61 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]); 62 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]); 63 64 return 0; 65 } 66 67 extern void des_sparc64_crypt(const u64 *key, const u64 *input, 68 u64 *output); 69 70 static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 71 { 72 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 73 const u64 *K = ctx->encrypt_expkey; 74 75 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 76 } 77 78 static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 79 { 80 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 81 const u64 *K = ctx->decrypt_expkey; 82 83 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 84 } 85 86 extern void des_sparc64_load_keys(const u64 *key); 87 88 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output, 89 unsigned int len); 90 91 #define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1)) 92 93 static int __ecb_crypt(struct blkcipher_desc *desc, 94 struct scatterlist *dst, struct scatterlist *src, 95 unsigned int nbytes, bool encrypt) 96 { 97 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 98 struct blkcipher_walk walk; 99 int err; 100 101 blkcipher_walk_init(&walk, dst, src, nbytes); 102 err = blkcipher_walk_virt(desc, &walk); 103 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 104 105 if (encrypt) 106 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 107 else 108 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 109 while ((nbytes = walk.nbytes)) { 110 unsigned int block_len = nbytes & DES_BLOCK_MASK; 111 112 if (likely(block_len)) { 113 des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr, 114 (u64 *) walk.dst.virt.addr, 115 block_len); 116 } 117 nbytes &= DES_BLOCK_SIZE - 1; 118 err = blkcipher_walk_done(desc, &walk, nbytes); 119 } 120 fprs_write(0); 121 return err; 122 } 123 124 static int ecb_encrypt(struct blkcipher_desc *desc, 125 struct scatterlist *dst, struct scatterlist *src, 126 unsigned int nbytes) 127 { 128 return __ecb_crypt(desc, dst, src, nbytes, true); 129 } 130 131 static int ecb_decrypt(struct blkcipher_desc *desc, 132 struct scatterlist *dst, struct scatterlist *src, 133 unsigned int nbytes) 134 { 135 return __ecb_crypt(desc, dst, src, nbytes, false); 136 } 137 138 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output, 139 unsigned int len, u64 *iv); 140 141 static int cbc_encrypt(struct blkcipher_desc *desc, 142 struct scatterlist *dst, struct scatterlist *src, 143 unsigned int nbytes) 144 { 145 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 146 struct blkcipher_walk walk; 147 int err; 148 149 blkcipher_walk_init(&walk, dst, src, nbytes); 150 err = blkcipher_walk_virt(desc, &walk); 151 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 152 153 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 154 while ((nbytes = walk.nbytes)) { 155 unsigned int block_len = nbytes & DES_BLOCK_MASK; 156 157 if (likely(block_len)) { 158 des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr, 159 (u64 *) walk.dst.virt.addr, 160 block_len, (u64 *) walk.iv); 161 } 162 nbytes &= DES_BLOCK_SIZE - 1; 163 err = blkcipher_walk_done(desc, &walk, nbytes); 164 } 165 fprs_write(0); 166 return err; 167 } 168 169 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, 170 unsigned int len, u64 *iv); 171 172 static int cbc_decrypt(struct blkcipher_desc *desc, 173 struct scatterlist *dst, struct scatterlist *src, 174 unsigned int nbytes) 175 { 176 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 177 struct blkcipher_walk walk; 178 int err; 179 180 blkcipher_walk_init(&walk, dst, src, nbytes); 181 err = blkcipher_walk_virt(desc, &walk); 182 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 183 184 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 185 while ((nbytes = walk.nbytes)) { 186 unsigned int block_len = nbytes & DES_BLOCK_MASK; 187 188 if (likely(block_len)) { 189 des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr, 190 (u64 *) walk.dst.virt.addr, 191 block_len, (u64 *) walk.iv); 192 } 193 nbytes &= DES_BLOCK_SIZE - 1; 194 err = blkcipher_walk_done(desc, &walk, nbytes); 195 } 196 fprs_write(0); 197 return err; 198 } 199 200 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, 201 unsigned int keylen) 202 { 203 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 204 u32 *flags = &tfm->crt_flags; 205 u64 k1[DES_EXPKEY_WORDS / 2]; 206 u64 k2[DES_EXPKEY_WORDS / 2]; 207 u64 k3[DES_EXPKEY_WORDS / 2]; 208 int err; 209 210 err = __des3_verify_key(flags, key); 211 if (unlikely(err)) 212 return err; 213 214 des_sparc64_key_expand((const u32 *)key, k1); 215 key += DES_KEY_SIZE; 216 des_sparc64_key_expand((const u32 *)key, k2); 217 key += DES_KEY_SIZE; 218 des_sparc64_key_expand((const u32 *)key, k3); 219 220 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1)); 221 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]); 222 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 223 &k3[0], sizeof(k3)); 224 225 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]); 226 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2], 227 &k2[0], sizeof(k2)); 228 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 229 &k1[0]); 230 231 return 0; 232 } 233 234 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input, 235 u64 *output); 236 237 static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 238 { 239 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 240 const u64 *K = ctx->encrypt_expkey; 241 242 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 243 } 244 245 static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 246 { 247 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 248 const u64 *K = ctx->decrypt_expkey; 249 250 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 251 } 252 253 extern void des3_ede_sparc64_load_keys(const u64 *key); 254 255 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input, 256 u64 *output, unsigned int len); 257 258 static int __ecb3_crypt(struct blkcipher_desc *desc, 259 struct scatterlist *dst, struct scatterlist *src, 260 unsigned int nbytes, bool encrypt) 261 { 262 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 263 struct blkcipher_walk walk; 264 const u64 *K; 265 int err; 266 267 blkcipher_walk_init(&walk, dst, src, nbytes); 268 err = blkcipher_walk_virt(desc, &walk); 269 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 270 271 if (encrypt) 272 K = &ctx->encrypt_expkey[0]; 273 else 274 K = &ctx->decrypt_expkey[0]; 275 des3_ede_sparc64_load_keys(K); 276 while ((nbytes = walk.nbytes)) { 277 unsigned int block_len = nbytes & DES_BLOCK_MASK; 278 279 if (likely(block_len)) { 280 const u64 *src64 = (const u64 *)walk.src.virt.addr; 281 des3_ede_sparc64_ecb_crypt(K, src64, 282 (u64 *) walk.dst.virt.addr, 283 block_len); 284 } 285 nbytes &= DES_BLOCK_SIZE - 1; 286 err = blkcipher_walk_done(desc, &walk, nbytes); 287 } 288 fprs_write(0); 289 return err; 290 } 291 292 static int ecb3_encrypt(struct blkcipher_desc *desc, 293 struct scatterlist *dst, struct scatterlist *src, 294 unsigned int nbytes) 295 { 296 return __ecb3_crypt(desc, dst, src, nbytes, true); 297 } 298 299 static int ecb3_decrypt(struct blkcipher_desc *desc, 300 struct scatterlist *dst, struct scatterlist *src, 301 unsigned int nbytes) 302 { 303 return __ecb3_crypt(desc, dst, src, nbytes, false); 304 } 305 306 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input, 307 u64 *output, unsigned int len, 308 u64 *iv); 309 310 static int cbc3_encrypt(struct blkcipher_desc *desc, 311 struct scatterlist *dst, struct scatterlist *src, 312 unsigned int nbytes) 313 { 314 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 315 struct blkcipher_walk walk; 316 const u64 *K; 317 int err; 318 319 blkcipher_walk_init(&walk, dst, src, nbytes); 320 err = blkcipher_walk_virt(desc, &walk); 321 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 322 323 K = &ctx->encrypt_expkey[0]; 324 des3_ede_sparc64_load_keys(K); 325 while ((nbytes = walk.nbytes)) { 326 unsigned int block_len = nbytes & DES_BLOCK_MASK; 327 328 if (likely(block_len)) { 329 const u64 *src64 = (const u64 *)walk.src.virt.addr; 330 des3_ede_sparc64_cbc_encrypt(K, src64, 331 (u64 *) walk.dst.virt.addr, 332 block_len, 333 (u64 *) walk.iv); 334 } 335 nbytes &= DES_BLOCK_SIZE - 1; 336 err = blkcipher_walk_done(desc, &walk, nbytes); 337 } 338 fprs_write(0); 339 return err; 340 } 341 342 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input, 343 u64 *output, unsigned int len, 344 u64 *iv); 345 346 static int cbc3_decrypt(struct blkcipher_desc *desc, 347 struct scatterlist *dst, struct scatterlist *src, 348 unsigned int nbytes) 349 { 350 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 351 struct blkcipher_walk walk; 352 const u64 *K; 353 int err; 354 355 blkcipher_walk_init(&walk, dst, src, nbytes); 356 err = blkcipher_walk_virt(desc, &walk); 357 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 358 359 K = &ctx->decrypt_expkey[0]; 360 des3_ede_sparc64_load_keys(K); 361 while ((nbytes = walk.nbytes)) { 362 unsigned int block_len = nbytes & DES_BLOCK_MASK; 363 364 if (likely(block_len)) { 365 const u64 *src64 = (const u64 *)walk.src.virt.addr; 366 des3_ede_sparc64_cbc_decrypt(K, src64, 367 (u64 *) walk.dst.virt.addr, 368 block_len, 369 (u64 *) walk.iv); 370 } 371 nbytes &= DES_BLOCK_SIZE - 1; 372 err = blkcipher_walk_done(desc, &walk, nbytes); 373 } 374 fprs_write(0); 375 return err; 376 } 377 378 static struct crypto_alg algs[] = { { 379 .cra_name = "des", 380 .cra_driver_name = "des-sparc64", 381 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 382 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 383 .cra_blocksize = DES_BLOCK_SIZE, 384 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 385 .cra_alignmask = 7, 386 .cra_module = THIS_MODULE, 387 .cra_u = { 388 .cipher = { 389 .cia_min_keysize = DES_KEY_SIZE, 390 .cia_max_keysize = DES_KEY_SIZE, 391 .cia_setkey = des_set_key, 392 .cia_encrypt = des_encrypt, 393 .cia_decrypt = des_decrypt 394 } 395 } 396 }, { 397 .cra_name = "ecb(des)", 398 .cra_driver_name = "ecb-des-sparc64", 399 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 400 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 401 .cra_blocksize = DES_BLOCK_SIZE, 402 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 403 .cra_alignmask = 7, 404 .cra_type = &crypto_blkcipher_type, 405 .cra_module = THIS_MODULE, 406 .cra_u = { 407 .blkcipher = { 408 .min_keysize = DES_KEY_SIZE, 409 .max_keysize = DES_KEY_SIZE, 410 .setkey = des_set_key, 411 .encrypt = ecb_encrypt, 412 .decrypt = ecb_decrypt, 413 }, 414 }, 415 }, { 416 .cra_name = "cbc(des)", 417 .cra_driver_name = "cbc-des-sparc64", 418 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 419 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 420 .cra_blocksize = DES_BLOCK_SIZE, 421 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 422 .cra_alignmask = 7, 423 .cra_type = &crypto_blkcipher_type, 424 .cra_module = THIS_MODULE, 425 .cra_u = { 426 .blkcipher = { 427 .min_keysize = DES_KEY_SIZE, 428 .max_keysize = DES_KEY_SIZE, 429 .ivsize = DES_BLOCK_SIZE, 430 .setkey = des_set_key, 431 .encrypt = cbc_encrypt, 432 .decrypt = cbc_decrypt, 433 }, 434 }, 435 }, { 436 .cra_name = "des3_ede", 437 .cra_driver_name = "des3_ede-sparc64", 438 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 439 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 440 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 441 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 442 .cra_alignmask = 7, 443 .cra_module = THIS_MODULE, 444 .cra_u = { 445 .cipher = { 446 .cia_min_keysize = DES3_EDE_KEY_SIZE, 447 .cia_max_keysize = DES3_EDE_KEY_SIZE, 448 .cia_setkey = des3_ede_set_key, 449 .cia_encrypt = des3_ede_encrypt, 450 .cia_decrypt = des3_ede_decrypt 451 } 452 } 453 }, { 454 .cra_name = "ecb(des3_ede)", 455 .cra_driver_name = "ecb-des3_ede-sparc64", 456 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 457 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 458 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 459 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 460 .cra_alignmask = 7, 461 .cra_type = &crypto_blkcipher_type, 462 .cra_module = THIS_MODULE, 463 .cra_u = { 464 .blkcipher = { 465 .min_keysize = DES3_EDE_KEY_SIZE, 466 .max_keysize = DES3_EDE_KEY_SIZE, 467 .setkey = des3_ede_set_key, 468 .encrypt = ecb3_encrypt, 469 .decrypt = ecb3_decrypt, 470 }, 471 }, 472 }, { 473 .cra_name = "cbc(des3_ede)", 474 .cra_driver_name = "cbc-des3_ede-sparc64", 475 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 476 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 477 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 478 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 479 .cra_alignmask = 7, 480 .cra_type = &crypto_blkcipher_type, 481 .cra_module = THIS_MODULE, 482 .cra_u = { 483 .blkcipher = { 484 .min_keysize = DES3_EDE_KEY_SIZE, 485 .max_keysize = DES3_EDE_KEY_SIZE, 486 .ivsize = DES3_EDE_BLOCK_SIZE, 487 .setkey = des3_ede_set_key, 488 .encrypt = cbc3_encrypt, 489 .decrypt = cbc3_decrypt, 490 }, 491 }, 492 } }; 493 494 static bool __init sparc64_has_des_opcode(void) 495 { 496 unsigned long cfr; 497 498 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 499 return false; 500 501 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 502 if (!(cfr & CFR_DES)) 503 return false; 504 505 return true; 506 } 507 508 static int __init des_sparc64_mod_init(void) 509 { 510 if (sparc64_has_des_opcode()) { 511 pr_info("Using sparc64 des opcodes optimized DES implementation\n"); 512 return crypto_register_algs(algs, ARRAY_SIZE(algs)); 513 } 514 pr_info("sparc64 des opcodes not available.\n"); 515 return -ENODEV; 516 } 517 518 static void __exit des_sparc64_mod_fini(void) 519 { 520 crypto_unregister_algs(algs, ARRAY_SIZE(algs)); 521 } 522 523 module_init(des_sparc64_mod_init); 524 module_exit(des_sparc64_mod_fini); 525 526 MODULE_LICENSE("GPL"); 527 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); 528 529 MODULE_ALIAS_CRYPTO("des"); 530 MODULE_ALIAS_CRYPTO("des3_ede"); 531 532 #include "crop_devid.c" 533