1 /* Glue code for DES encryption optimized for sparc64 crypto opcodes. 2 * 3 * Copyright (C) 2012 David S. Miller <davem@davemloft.net> 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/crypto.h> 9 #include <linux/init.h> 10 #include <linux/module.h> 11 #include <linux/mm.h> 12 #include <linux/types.h> 13 #include <crypto/algapi.h> 14 #include <crypto/des.h> 15 16 #include <asm/fpumacro.h> 17 #include <asm/pstate.h> 18 #include <asm/elf.h> 19 20 #include "opcodes.h" 21 22 struct des_sparc64_ctx { 23 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; 24 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; 25 }; 26 27 struct des3_ede_sparc64_ctx { 28 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 29 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; 30 }; 31 32 static void encrypt_to_decrypt(u64 *d, const u64 *e) 33 { 34 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1; 35 int i; 36 37 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++) 38 *d++ = *s--; 39 } 40 41 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key); 42 43 static int des_set_key(struct crypto_tfm *tfm, const u8 *key, 44 unsigned int keylen) 45 { 46 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 47 u32 *flags = &tfm->crt_flags; 48 u32 tmp[DES_EXPKEY_WORDS]; 49 int ret; 50 51 /* Even though we have special instructions for key expansion, 52 * we call des_ekey() so that we don't have to write our own 53 * weak key detection code. 54 */ 55 ret = des_ekey(tmp, key); 56 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 57 *flags |= CRYPTO_TFM_RES_WEAK_KEY; 58 return -EINVAL; 59 } 60 61 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]); 62 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]); 63 64 return 0; 65 } 66 67 extern void des_sparc64_crypt(const u64 *key, const u64 *input, 68 u64 *output); 69 70 static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 71 { 72 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 73 const u64 *K = ctx->encrypt_expkey; 74 75 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 76 } 77 78 static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 79 { 80 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 81 const u64 *K = ctx->decrypt_expkey; 82 83 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 84 } 85 86 extern void des_sparc64_load_keys(const u64 *key); 87 88 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output, 89 unsigned int len); 90 91 #define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1)) 92 93 static int __ecb_crypt(struct blkcipher_desc *desc, 94 struct scatterlist *dst, struct scatterlist *src, 95 unsigned int nbytes, bool encrypt) 96 { 97 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 98 struct blkcipher_walk walk; 99 int err; 100 101 blkcipher_walk_init(&walk, dst, src, nbytes); 102 err = blkcipher_walk_virt(desc, &walk); 103 104 if (encrypt) 105 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 106 else 107 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 108 while ((nbytes = walk.nbytes)) { 109 unsigned int block_len = nbytes & DES_BLOCK_MASK; 110 111 if (likely(block_len)) { 112 des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr, 113 (u64 *) walk.dst.virt.addr, 114 block_len); 115 } 116 nbytes &= DES_BLOCK_SIZE - 1; 117 err = blkcipher_walk_done(desc, &walk, nbytes); 118 } 119 fprs_write(0); 120 return err; 121 } 122 123 static int ecb_encrypt(struct blkcipher_desc *desc, 124 struct scatterlist *dst, struct scatterlist *src, 125 unsigned int nbytes) 126 { 127 return __ecb_crypt(desc, dst, src, nbytes, true); 128 } 129 130 static int ecb_decrypt(struct blkcipher_desc *desc, 131 struct scatterlist *dst, struct scatterlist *src, 132 unsigned int nbytes) 133 { 134 return __ecb_crypt(desc, dst, src, nbytes, false); 135 } 136 137 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output, 138 unsigned int len, u64 *iv); 139 140 static int cbc_encrypt(struct blkcipher_desc *desc, 141 struct scatterlist *dst, struct scatterlist *src, 142 unsigned int nbytes) 143 { 144 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 145 struct blkcipher_walk walk; 146 int err; 147 148 blkcipher_walk_init(&walk, dst, src, nbytes); 149 err = blkcipher_walk_virt(desc, &walk); 150 151 des_sparc64_load_keys(&ctx->encrypt_expkey[0]); 152 while ((nbytes = walk.nbytes)) { 153 unsigned int block_len = nbytes & DES_BLOCK_MASK; 154 155 if (likely(block_len)) { 156 des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr, 157 (u64 *) walk.dst.virt.addr, 158 block_len, (u64 *) walk.iv); 159 } 160 nbytes &= DES_BLOCK_SIZE - 1; 161 err = blkcipher_walk_done(desc, &walk, nbytes); 162 } 163 fprs_write(0); 164 return err; 165 } 166 167 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, 168 unsigned int len, u64 *iv); 169 170 static int cbc_decrypt(struct blkcipher_desc *desc, 171 struct scatterlist *dst, struct scatterlist *src, 172 unsigned int nbytes) 173 { 174 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 175 struct blkcipher_walk walk; 176 int err; 177 178 blkcipher_walk_init(&walk, dst, src, nbytes); 179 err = blkcipher_walk_virt(desc, &walk); 180 181 des_sparc64_load_keys(&ctx->decrypt_expkey[0]); 182 while ((nbytes = walk.nbytes)) { 183 unsigned int block_len = nbytes & DES_BLOCK_MASK; 184 185 if (likely(block_len)) { 186 des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr, 187 (u64 *) walk.dst.virt.addr, 188 block_len, (u64 *) walk.iv); 189 } 190 nbytes &= DES_BLOCK_SIZE - 1; 191 err = blkcipher_walk_done(desc, &walk, nbytes); 192 } 193 fprs_write(0); 194 return err; 195 } 196 197 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, 198 unsigned int keylen) 199 { 200 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); 201 const u32 *K = (const u32 *)key; 202 u32 *flags = &tfm->crt_flags; 203 u64 k1[DES_EXPKEY_WORDS / 2]; 204 u64 k2[DES_EXPKEY_WORDS / 2]; 205 u64 k3[DES_EXPKEY_WORDS / 2]; 206 207 if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || 208 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && 209 (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { 210 *flags |= CRYPTO_TFM_RES_WEAK_KEY; 211 return -EINVAL; 212 } 213 214 des_sparc64_key_expand((const u32 *)key, k1); 215 key += DES_KEY_SIZE; 216 des_sparc64_key_expand((const u32 *)key, k2); 217 key += DES_KEY_SIZE; 218 des_sparc64_key_expand((const u32 *)key, k3); 219 220 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1)); 221 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]); 222 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 223 &k3[0], sizeof(k3)); 224 225 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]); 226 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2], 227 &k2[0], sizeof(k2)); 228 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], 229 &k1[0]); 230 231 return 0; 232 } 233 234 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input, 235 u64 *output); 236 237 static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 238 { 239 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 240 const u64 *K = ctx->encrypt_expkey; 241 242 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 243 } 244 245 static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 246 { 247 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); 248 const u64 *K = ctx->decrypt_expkey; 249 250 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); 251 } 252 253 extern void des3_ede_sparc64_load_keys(const u64 *key); 254 255 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input, 256 u64 *output, unsigned int len); 257 258 static int __ecb3_crypt(struct blkcipher_desc *desc, 259 struct scatterlist *dst, struct scatterlist *src, 260 unsigned int nbytes, bool encrypt) 261 { 262 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 263 struct blkcipher_walk walk; 264 const u64 *K; 265 int err; 266 267 blkcipher_walk_init(&walk, dst, src, nbytes); 268 err = blkcipher_walk_virt(desc, &walk); 269 270 if (encrypt) 271 K = &ctx->encrypt_expkey[0]; 272 else 273 K = &ctx->decrypt_expkey[0]; 274 des3_ede_sparc64_load_keys(K); 275 while ((nbytes = walk.nbytes)) { 276 unsigned int block_len = nbytes & DES_BLOCK_MASK; 277 278 if (likely(block_len)) { 279 const u64 *src64 = (const u64 *)walk.src.virt.addr; 280 des3_ede_sparc64_ecb_crypt(K, src64, 281 (u64 *) walk.dst.virt.addr, 282 block_len); 283 } 284 nbytes &= DES_BLOCK_SIZE - 1; 285 err = blkcipher_walk_done(desc, &walk, nbytes); 286 } 287 fprs_write(0); 288 return err; 289 } 290 291 static int ecb3_encrypt(struct blkcipher_desc *desc, 292 struct scatterlist *dst, struct scatterlist *src, 293 unsigned int nbytes) 294 { 295 return __ecb3_crypt(desc, dst, src, nbytes, true); 296 } 297 298 static int ecb3_decrypt(struct blkcipher_desc *desc, 299 struct scatterlist *dst, struct scatterlist *src, 300 unsigned int nbytes) 301 { 302 return __ecb3_crypt(desc, dst, src, nbytes, false); 303 } 304 305 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input, 306 u64 *output, unsigned int len, 307 u64 *iv); 308 309 static int cbc3_encrypt(struct blkcipher_desc *desc, 310 struct scatterlist *dst, struct scatterlist *src, 311 unsigned int nbytes) 312 { 313 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 314 struct blkcipher_walk walk; 315 const u64 *K; 316 int err; 317 318 blkcipher_walk_init(&walk, dst, src, nbytes); 319 err = blkcipher_walk_virt(desc, &walk); 320 321 K = &ctx->encrypt_expkey[0]; 322 des3_ede_sparc64_load_keys(K); 323 while ((nbytes = walk.nbytes)) { 324 unsigned int block_len = nbytes & DES_BLOCK_MASK; 325 326 if (likely(block_len)) { 327 const u64 *src64 = (const u64 *)walk.src.virt.addr; 328 des3_ede_sparc64_cbc_encrypt(K, src64, 329 (u64 *) walk.dst.virt.addr, 330 block_len, 331 (u64 *) walk.iv); 332 } 333 nbytes &= DES_BLOCK_SIZE - 1; 334 err = blkcipher_walk_done(desc, &walk, nbytes); 335 } 336 fprs_write(0); 337 return err; 338 } 339 340 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input, 341 u64 *output, unsigned int len, 342 u64 *iv); 343 344 static int cbc3_decrypt(struct blkcipher_desc *desc, 345 struct scatterlist *dst, struct scatterlist *src, 346 unsigned int nbytes) 347 { 348 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 349 struct blkcipher_walk walk; 350 const u64 *K; 351 int err; 352 353 blkcipher_walk_init(&walk, dst, src, nbytes); 354 err = blkcipher_walk_virt(desc, &walk); 355 356 K = &ctx->decrypt_expkey[0]; 357 des3_ede_sparc64_load_keys(K); 358 while ((nbytes = walk.nbytes)) { 359 unsigned int block_len = nbytes & DES_BLOCK_MASK; 360 361 if (likely(block_len)) { 362 const u64 *src64 = (const u64 *)walk.src.virt.addr; 363 des3_ede_sparc64_cbc_decrypt(K, src64, 364 (u64 *) walk.dst.virt.addr, 365 block_len, 366 (u64 *) walk.iv); 367 } 368 nbytes &= DES_BLOCK_SIZE - 1; 369 err = blkcipher_walk_done(desc, &walk, nbytes); 370 } 371 fprs_write(0); 372 return err; 373 } 374 375 static struct crypto_alg algs[] = { { 376 .cra_name = "des", 377 .cra_driver_name = "des-sparc64", 378 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 379 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 380 .cra_blocksize = DES_BLOCK_SIZE, 381 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 382 .cra_alignmask = 7, 383 .cra_module = THIS_MODULE, 384 .cra_u = { 385 .cipher = { 386 .cia_min_keysize = DES_KEY_SIZE, 387 .cia_max_keysize = DES_KEY_SIZE, 388 .cia_setkey = des_set_key, 389 .cia_encrypt = des_encrypt, 390 .cia_decrypt = des_decrypt 391 } 392 } 393 }, { 394 .cra_name = "ecb(des)", 395 .cra_driver_name = "ecb-des-sparc64", 396 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 397 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 398 .cra_blocksize = DES_BLOCK_SIZE, 399 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 400 .cra_alignmask = 7, 401 .cra_type = &crypto_blkcipher_type, 402 .cra_module = THIS_MODULE, 403 .cra_u = { 404 .blkcipher = { 405 .min_keysize = DES_KEY_SIZE, 406 .max_keysize = DES_KEY_SIZE, 407 .setkey = des_set_key, 408 .encrypt = ecb_encrypt, 409 .decrypt = ecb_decrypt, 410 }, 411 }, 412 }, { 413 .cra_name = "cbc(des)", 414 .cra_driver_name = "cbc-des-sparc64", 415 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 416 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 417 .cra_blocksize = DES_BLOCK_SIZE, 418 .cra_ctxsize = sizeof(struct des_sparc64_ctx), 419 .cra_alignmask = 7, 420 .cra_type = &crypto_blkcipher_type, 421 .cra_module = THIS_MODULE, 422 .cra_u = { 423 .blkcipher = { 424 .min_keysize = DES_KEY_SIZE, 425 .max_keysize = DES_KEY_SIZE, 426 .setkey = des_set_key, 427 .encrypt = cbc_encrypt, 428 .decrypt = cbc_decrypt, 429 }, 430 }, 431 }, { 432 .cra_name = "des3_ede", 433 .cra_driver_name = "des3_ede-sparc64", 434 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 435 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 436 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 437 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 438 .cra_alignmask = 7, 439 .cra_module = THIS_MODULE, 440 .cra_u = { 441 .cipher = { 442 .cia_min_keysize = DES3_EDE_KEY_SIZE, 443 .cia_max_keysize = DES3_EDE_KEY_SIZE, 444 .cia_setkey = des3_ede_set_key, 445 .cia_encrypt = des3_ede_encrypt, 446 .cia_decrypt = des3_ede_decrypt 447 } 448 } 449 }, { 450 .cra_name = "ecb(des3_ede)", 451 .cra_driver_name = "ecb-des3_ede-sparc64", 452 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 453 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 454 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 455 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 456 .cra_alignmask = 7, 457 .cra_type = &crypto_blkcipher_type, 458 .cra_module = THIS_MODULE, 459 .cra_u = { 460 .blkcipher = { 461 .min_keysize = DES3_EDE_KEY_SIZE, 462 .max_keysize = DES3_EDE_KEY_SIZE, 463 .setkey = des3_ede_set_key, 464 .encrypt = ecb3_encrypt, 465 .decrypt = ecb3_decrypt, 466 }, 467 }, 468 }, { 469 .cra_name = "cbc(des3_ede)", 470 .cra_driver_name = "cbc-des3_ede-sparc64", 471 .cra_priority = SPARC_CR_OPCODE_PRIORITY, 472 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 473 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 474 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), 475 .cra_alignmask = 7, 476 .cra_type = &crypto_blkcipher_type, 477 .cra_module = THIS_MODULE, 478 .cra_u = { 479 .blkcipher = { 480 .min_keysize = DES3_EDE_KEY_SIZE, 481 .max_keysize = DES3_EDE_KEY_SIZE, 482 .setkey = des3_ede_set_key, 483 .encrypt = cbc3_encrypt, 484 .decrypt = cbc3_decrypt, 485 }, 486 }, 487 } }; 488 489 static bool __init sparc64_has_des_opcode(void) 490 { 491 unsigned long cfr; 492 493 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 494 return false; 495 496 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 497 if (!(cfr & CFR_DES)) 498 return false; 499 500 return true; 501 } 502 503 static int __init des_sparc64_mod_init(void) 504 { 505 int i; 506 507 for (i = 0; i < ARRAY_SIZE(algs); i++) 508 INIT_LIST_HEAD(&algs[i].cra_list); 509 510 if (sparc64_has_des_opcode()) { 511 pr_info("Using sparc64 des opcodes optimized DES implementation\n"); 512 return crypto_register_algs(algs, ARRAY_SIZE(algs)); 513 } 514 pr_info("sparc64 des opcodes not available.\n"); 515 return -ENODEV; 516 } 517 518 static void __exit des_sparc64_mod_fini(void) 519 { 520 crypto_unregister_algs(algs, ARRAY_SIZE(algs)); 521 } 522 523 module_init(des_sparc64_mod_init); 524 module_exit(des_sparc64_mod_fini); 525 526 MODULE_LICENSE("GPL"); 527 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); 528 529 MODULE_ALIAS("des"); 530 531 #include "crop_devid.c" 532