1 /* 2 * Glue Code for assembler optimized version of 3DES 3 * 4 * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 5 * 6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 8 * CTR part based on code (crypto/ctr.c) by: 9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 */ 22 23 #include <asm/processor.h> 24 #include <crypto/des.h> 25 #include <linux/crypto.h> 26 #include <linux/init.h> 27 #include <linux/module.h> 28 #include <linux/types.h> 29 #include <crypto/algapi.h> 30 31 struct des3_ede_x86_ctx { 32 u32 enc_expkey[DES3_EDE_EXPKEY_WORDS]; 33 u32 dec_expkey[DES3_EDE_EXPKEY_WORDS]; 34 }; 35 36 /* regular block cipher functions */ 37 asmlinkage void des3_ede_x86_64_crypt_blk(const u32 *expkey, u8 *dst, 38 const u8 *src); 39 40 /* 3-way parallel cipher functions */ 41 asmlinkage void des3_ede_x86_64_crypt_blk_3way(const u32 *expkey, u8 *dst, 42 const u8 *src); 43 44 static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, 45 const u8 *src) 46 { 47 u32 *enc_ctx = ctx->enc_expkey; 48 49 des3_ede_x86_64_crypt_blk(enc_ctx, dst, src); 50 } 51 52 static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, 53 const u8 *src) 54 { 55 u32 *dec_ctx = ctx->dec_expkey; 56 57 des3_ede_x86_64_crypt_blk(dec_ctx, dst, src); 58 } 59 60 static inline void des3_ede_enc_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, 61 const u8 *src) 62 { 63 u32 *enc_ctx = ctx->enc_expkey; 64 65 des3_ede_x86_64_crypt_blk_3way(enc_ctx, dst, src); 66 } 67 68 static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, 69 const u8 *src) 70 { 71 u32 *dec_ctx = ctx->dec_expkey; 72 73 des3_ede_x86_64_crypt_blk_3way(dec_ctx, dst, src); 74 } 75 76 static void des3_ede_x86_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 77 { 78 des3_ede_enc_blk(crypto_tfm_ctx(tfm), dst, src); 79 } 80 81 static void des3_ede_x86_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 82 { 83 des3_ede_dec_blk(crypto_tfm_ctx(tfm), dst, src); 84 } 85 86 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, 87 const u32 *expkey) 88 { 89 unsigned int bsize = DES3_EDE_BLOCK_SIZE; 90 unsigned int nbytes; 91 int err; 92 93 err = blkcipher_walk_virt(desc, walk); 94 95 while ((nbytes = walk->nbytes)) { 96 u8 *wsrc = walk->src.virt.addr; 97 u8 *wdst = walk->dst.virt.addr; 98 99 /* Process four block batch */ 100 if (nbytes >= bsize * 3) { 101 do { 102 des3_ede_x86_64_crypt_blk_3way(expkey, wdst, 103 wsrc); 104 105 wsrc += bsize * 3; 106 wdst += bsize * 3; 107 nbytes -= bsize * 3; 108 } while (nbytes >= bsize * 3); 109 110 if (nbytes < bsize) 111 goto done; 112 } 113 114 /* Handle leftovers */ 115 do { 116 des3_ede_x86_64_crypt_blk(expkey, wdst, wsrc); 117 118 wsrc += bsize; 119 wdst += bsize; 120 nbytes -= bsize; 121 } while (nbytes >= bsize); 122 123 done: 124 err = blkcipher_walk_done(desc, walk, nbytes); 125 } 126 127 return err; 128 } 129 130 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 131 struct scatterlist *src, unsigned int nbytes) 132 { 133 struct des3_ede_x86_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 134 struct blkcipher_walk walk; 135 136 blkcipher_walk_init(&walk, dst, src, nbytes); 137 return ecb_crypt(desc, &walk, ctx->enc_expkey); 138 } 139 140 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 141 struct scatterlist *src, unsigned int nbytes) 142 { 143 struct des3_ede_x86_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 144 struct blkcipher_walk walk; 145 146 blkcipher_walk_init(&walk, dst, src, nbytes); 147 return ecb_crypt(desc, &walk, ctx->dec_expkey); 148 } 149 150 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 151 struct blkcipher_walk *walk) 152 { 153 struct des3_ede_x86_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 154 unsigned int bsize = DES3_EDE_BLOCK_SIZE; 155 unsigned int nbytes = walk->nbytes; 156 u64 *src = (u64 *)walk->src.virt.addr; 157 u64 *dst = (u64 *)walk->dst.virt.addr; 158 u64 *iv = (u64 *)walk->iv; 159 160 do { 161 *dst = *src ^ *iv; 162 des3_ede_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 163 iv = dst; 164 165 src += 1; 166 dst += 1; 167 nbytes -= bsize; 168 } while (nbytes >= bsize); 169 170 *(u64 *)walk->iv = *iv; 171 return nbytes; 172 } 173 174 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 175 struct scatterlist *src, unsigned int nbytes) 176 { 177 struct blkcipher_walk walk; 178 int err; 179 180 blkcipher_walk_init(&walk, dst, src, nbytes); 181 err = blkcipher_walk_virt(desc, &walk); 182 183 while ((nbytes = walk.nbytes)) { 184 nbytes = __cbc_encrypt(desc, &walk); 185 err = blkcipher_walk_done(desc, &walk, nbytes); 186 } 187 188 return err; 189 } 190 191 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 192 struct blkcipher_walk *walk) 193 { 194 struct des3_ede_x86_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 195 unsigned int bsize = DES3_EDE_BLOCK_SIZE; 196 unsigned int nbytes = walk->nbytes; 197 u64 *src = (u64 *)walk->src.virt.addr; 198 u64 *dst = (u64 *)walk->dst.virt.addr; 199 u64 ivs[3 - 1]; 200 u64 last_iv; 201 202 /* Start of the last block. */ 203 src += nbytes / bsize - 1; 204 dst += nbytes / bsize - 1; 205 206 last_iv = *src; 207 208 /* Process four block batch */ 209 if (nbytes >= bsize * 3) { 210 do { 211 nbytes -= bsize * 3 - bsize; 212 src -= 3 - 1; 213 dst -= 3 - 1; 214 215 ivs[0] = src[0]; 216 ivs[1] = src[1]; 217 218 des3_ede_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); 219 220 dst[1] ^= ivs[0]; 221 dst[2] ^= ivs[1]; 222 223 nbytes -= bsize; 224 if (nbytes < bsize) 225 goto done; 226 227 *dst ^= *(src - 1); 228 src -= 1; 229 dst -= 1; 230 } while (nbytes >= bsize * 3); 231 } 232 233 /* Handle leftovers */ 234 for (;;) { 235 des3_ede_dec_blk(ctx, (u8 *)dst, (u8 *)src); 236 237 nbytes -= bsize; 238 if (nbytes < bsize) 239 break; 240 241 *dst ^= *(src - 1); 242 src -= 1; 243 dst -= 1; 244 } 245 246 done: 247 *dst ^= *(u64 *)walk->iv; 248 *(u64 *)walk->iv = last_iv; 249 250 return nbytes; 251 } 252 253 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 254 struct scatterlist *src, unsigned int nbytes) 255 { 256 struct blkcipher_walk walk; 257 int err; 258 259 blkcipher_walk_init(&walk, dst, src, nbytes); 260 err = blkcipher_walk_virt(desc, &walk); 261 262 while ((nbytes = walk.nbytes)) { 263 nbytes = __cbc_decrypt(desc, &walk); 264 err = blkcipher_walk_done(desc, &walk, nbytes); 265 } 266 267 return err; 268 } 269 270 static void ctr_crypt_final(struct des3_ede_x86_ctx *ctx, 271 struct blkcipher_walk *walk) 272 { 273 u8 *ctrblk = walk->iv; 274 u8 keystream[DES3_EDE_BLOCK_SIZE]; 275 u8 *src = walk->src.virt.addr; 276 u8 *dst = walk->dst.virt.addr; 277 unsigned int nbytes = walk->nbytes; 278 279 des3_ede_enc_blk(ctx, keystream, ctrblk); 280 crypto_xor_cpy(dst, keystream, src, nbytes); 281 282 crypto_inc(ctrblk, DES3_EDE_BLOCK_SIZE); 283 } 284 285 static unsigned int __ctr_crypt(struct blkcipher_desc *desc, 286 struct blkcipher_walk *walk) 287 { 288 struct des3_ede_x86_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 289 unsigned int bsize = DES3_EDE_BLOCK_SIZE; 290 unsigned int nbytes = walk->nbytes; 291 __be64 *src = (__be64 *)walk->src.virt.addr; 292 __be64 *dst = (__be64 *)walk->dst.virt.addr; 293 u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); 294 __be64 ctrblocks[3]; 295 296 /* Process four block batch */ 297 if (nbytes >= bsize * 3) { 298 do { 299 /* create ctrblks for parallel encrypt */ 300 ctrblocks[0] = cpu_to_be64(ctrblk++); 301 ctrblocks[1] = cpu_to_be64(ctrblk++); 302 ctrblocks[2] = cpu_to_be64(ctrblk++); 303 304 des3_ede_enc_blk_3way(ctx, (u8 *)ctrblocks, 305 (u8 *)ctrblocks); 306 307 dst[0] = src[0] ^ ctrblocks[0]; 308 dst[1] = src[1] ^ ctrblocks[1]; 309 dst[2] = src[2] ^ ctrblocks[2]; 310 311 src += 3; 312 dst += 3; 313 } while ((nbytes -= bsize * 3) >= bsize * 3); 314 315 if (nbytes < bsize) 316 goto done; 317 } 318 319 /* Handle leftovers */ 320 do { 321 ctrblocks[0] = cpu_to_be64(ctrblk++); 322 323 des3_ede_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); 324 325 dst[0] = src[0] ^ ctrblocks[0]; 326 327 src += 1; 328 dst += 1; 329 } while ((nbytes -= bsize) >= bsize); 330 331 done: 332 *(__be64 *)walk->iv = cpu_to_be64(ctrblk); 333 return nbytes; 334 } 335 336 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 337 struct scatterlist *src, unsigned int nbytes) 338 { 339 struct blkcipher_walk walk; 340 int err; 341 342 blkcipher_walk_init(&walk, dst, src, nbytes); 343 err = blkcipher_walk_virt_block(desc, &walk, DES3_EDE_BLOCK_SIZE); 344 345 while ((nbytes = walk.nbytes) >= DES3_EDE_BLOCK_SIZE) { 346 nbytes = __ctr_crypt(desc, &walk); 347 err = blkcipher_walk_done(desc, &walk, nbytes); 348 } 349 350 if (walk.nbytes) { 351 ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk); 352 err = blkcipher_walk_done(desc, &walk, 0); 353 } 354 355 return err; 356 } 357 358 static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key, 359 unsigned int keylen) 360 { 361 struct des3_ede_x86_ctx *ctx = crypto_tfm_ctx(tfm); 362 u32 i, j, tmp; 363 int err; 364 365 /* Generate encryption context using generic implementation. */ 366 err = __des3_ede_setkey(ctx->enc_expkey, &tfm->crt_flags, key, keylen); 367 if (err < 0) 368 return err; 369 370 /* Fix encryption context for this implementation and form decryption 371 * context. */ 372 j = DES3_EDE_EXPKEY_WORDS - 2; 373 for (i = 0; i < DES3_EDE_EXPKEY_WORDS; i += 2, j -= 2) { 374 tmp = ror32(ctx->enc_expkey[i + 1], 4); 375 ctx->enc_expkey[i + 1] = tmp; 376 377 ctx->dec_expkey[j + 0] = ctx->enc_expkey[i + 0]; 378 ctx->dec_expkey[j + 1] = tmp; 379 } 380 381 return 0; 382 } 383 384 static struct crypto_alg des3_ede_algs[4] = { { 385 .cra_name = "des3_ede", 386 .cra_driver_name = "des3_ede-asm", 387 .cra_priority = 200, 388 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 389 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 390 .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 391 .cra_alignmask = 0, 392 .cra_module = THIS_MODULE, 393 .cra_u = { 394 .cipher = { 395 .cia_min_keysize = DES3_EDE_KEY_SIZE, 396 .cia_max_keysize = DES3_EDE_KEY_SIZE, 397 .cia_setkey = des3_ede_x86_setkey, 398 .cia_encrypt = des3_ede_x86_encrypt, 399 .cia_decrypt = des3_ede_x86_decrypt, 400 } 401 } 402 }, { 403 .cra_name = "ecb(des3_ede)", 404 .cra_driver_name = "ecb-des3_ede-asm", 405 .cra_priority = 300, 406 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 407 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 408 .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 409 .cra_alignmask = 0, 410 .cra_type = &crypto_blkcipher_type, 411 .cra_module = THIS_MODULE, 412 .cra_u = { 413 .blkcipher = { 414 .min_keysize = DES3_EDE_KEY_SIZE, 415 .max_keysize = DES3_EDE_KEY_SIZE, 416 .setkey = des3_ede_x86_setkey, 417 .encrypt = ecb_encrypt, 418 .decrypt = ecb_decrypt, 419 }, 420 }, 421 }, { 422 .cra_name = "cbc(des3_ede)", 423 .cra_driver_name = "cbc-des3_ede-asm", 424 .cra_priority = 300, 425 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 426 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 427 .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 428 .cra_alignmask = 0, 429 .cra_type = &crypto_blkcipher_type, 430 .cra_module = THIS_MODULE, 431 .cra_u = { 432 .blkcipher = { 433 .min_keysize = DES3_EDE_KEY_SIZE, 434 .max_keysize = DES3_EDE_KEY_SIZE, 435 .ivsize = DES3_EDE_BLOCK_SIZE, 436 .setkey = des3_ede_x86_setkey, 437 .encrypt = cbc_encrypt, 438 .decrypt = cbc_decrypt, 439 }, 440 }, 441 }, { 442 .cra_name = "ctr(des3_ede)", 443 .cra_driver_name = "ctr-des3_ede-asm", 444 .cra_priority = 300, 445 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 446 .cra_blocksize = 1, 447 .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 448 .cra_alignmask = 0, 449 .cra_type = &crypto_blkcipher_type, 450 .cra_module = THIS_MODULE, 451 .cra_u = { 452 .blkcipher = { 453 .min_keysize = DES3_EDE_KEY_SIZE, 454 .max_keysize = DES3_EDE_KEY_SIZE, 455 .ivsize = DES3_EDE_BLOCK_SIZE, 456 .setkey = des3_ede_x86_setkey, 457 .encrypt = ctr_crypt, 458 .decrypt = ctr_crypt, 459 }, 460 }, 461 } }; 462 463 static bool is_blacklisted_cpu(void) 464 { 465 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 466 return false; 467 468 if (boot_cpu_data.x86 == 0x0f) { 469 /* 470 * On Pentium 4, des3_ede-x86_64 is slower than generic C 471 * implementation because use of 64bit rotates (which are really 472 * slow on P4). Therefore blacklist P4s. 473 */ 474 return true; 475 } 476 477 return false; 478 } 479 480 static int force; 481 module_param(force, int, 0); 482 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 483 484 static int __init des3_ede_x86_init(void) 485 { 486 if (!force && is_blacklisted_cpu()) { 487 pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n"); 488 return -ENODEV; 489 } 490 491 return crypto_register_algs(des3_ede_algs, ARRAY_SIZE(des3_ede_algs)); 492 } 493 494 static void __exit des3_ede_x86_fini(void) 495 { 496 crypto_unregister_algs(des3_ede_algs, ARRAY_SIZE(des3_ede_algs)); 497 } 498 499 module_init(des3_ede_x86_init); 500 module_exit(des3_ede_x86_fini); 501 502 MODULE_LICENSE("GPL"); 503 MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized"); 504 MODULE_ALIAS_CRYPTO("des3_ede"); 505 MODULE_ALIAS_CRYPTO("des3_ede-asm"); 506 MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>"); 507