1 /* 2 * Glue Code for assembler optimized version of Blowfish 3 * 4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 5 * 6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 8 * CTR part based on code (crypto/ctr.c) by: 9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; if not, write to the Free Software 23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 24 * USA 25 * 26 */ 27 28 #include <asm/processor.h> 29 #include <crypto/blowfish.h> 30 #include <linux/crypto.h> 31 #include <linux/init.h> 32 #include <linux/module.h> 33 #include <linux/types.h> 34 #include <crypto/algapi.h> 35 36 /* regular block cipher functions */ 37 asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src, 38 bool xor); 39 asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src); 40 41 /* 4-way parallel cipher functions */ 42 asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, 43 const u8 *src, bool xor); 44 asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst, 45 const u8 *src); 46 47 static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src) 48 { 49 __blowfish_enc_blk(ctx, dst, src, false); 50 } 51 52 static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst, 53 const u8 *src) 54 { 55 __blowfish_enc_blk(ctx, dst, src, true); 56 } 57 58 static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, 59 const u8 *src) 60 { 61 __blowfish_enc_blk_4way(ctx, dst, src, false); 62 } 63 64 static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst, 65 const u8 *src) 66 { 67 __blowfish_enc_blk_4way(ctx, dst, src, true); 68 } 69 70 static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 71 { 72 blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src); 73 } 74 75 static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 76 { 77 blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src); 78 } 79 80 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, 81 void (*fn)(struct bf_ctx *, u8 *, const u8 *), 82 void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *)) 83 { 84 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 85 unsigned int bsize = BF_BLOCK_SIZE; 86 unsigned int nbytes; 87 int err; 88 89 err = blkcipher_walk_virt(desc, walk); 90 91 while ((nbytes = walk->nbytes)) { 92 u8 *wsrc = walk->src.virt.addr; 93 u8 *wdst = walk->dst.virt.addr; 94 95 /* Process four block batch */ 96 if (nbytes >= bsize * 4) { 97 do { 98 fn_4way(ctx, wdst, wsrc); 99 100 wsrc += bsize * 4; 101 wdst += bsize * 4; 102 nbytes -= bsize * 4; 103 } while (nbytes >= bsize * 4); 104 105 if (nbytes < bsize) 106 goto done; 107 } 108 109 /* Handle leftovers */ 110 do { 111 fn(ctx, wdst, wsrc); 112 113 wsrc += bsize; 114 wdst += bsize; 115 nbytes -= bsize; 116 } while (nbytes >= bsize); 117 118 done: 119 err = blkcipher_walk_done(desc, walk, nbytes); 120 } 121 122 return err; 123 } 124 125 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 126 struct scatterlist *src, unsigned int nbytes) 127 { 128 struct blkcipher_walk walk; 129 130 blkcipher_walk_init(&walk, dst, src, nbytes); 131 return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way); 132 } 133 134 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 135 struct scatterlist *src, unsigned int nbytes) 136 { 137 struct blkcipher_walk walk; 138 139 blkcipher_walk_init(&walk, dst, src, nbytes); 140 return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way); 141 } 142 143 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 144 struct blkcipher_walk *walk) 145 { 146 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 147 unsigned int bsize = BF_BLOCK_SIZE; 148 unsigned int nbytes = walk->nbytes; 149 u64 *src = (u64 *)walk->src.virt.addr; 150 u64 *dst = (u64 *)walk->dst.virt.addr; 151 u64 *iv = (u64 *)walk->iv; 152 153 do { 154 *dst = *src ^ *iv; 155 blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 156 iv = dst; 157 158 src += 1; 159 dst += 1; 160 nbytes -= bsize; 161 } while (nbytes >= bsize); 162 163 *(u64 *)walk->iv = *iv; 164 return nbytes; 165 } 166 167 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 168 struct scatterlist *src, unsigned int nbytes) 169 { 170 struct blkcipher_walk walk; 171 int err; 172 173 blkcipher_walk_init(&walk, dst, src, nbytes); 174 err = blkcipher_walk_virt(desc, &walk); 175 176 while ((nbytes = walk.nbytes)) { 177 nbytes = __cbc_encrypt(desc, &walk); 178 err = blkcipher_walk_done(desc, &walk, nbytes); 179 } 180 181 return err; 182 } 183 184 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 185 struct blkcipher_walk *walk) 186 { 187 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 188 unsigned int bsize = BF_BLOCK_SIZE; 189 unsigned int nbytes = walk->nbytes; 190 u64 *src = (u64 *)walk->src.virt.addr; 191 u64 *dst = (u64 *)walk->dst.virt.addr; 192 u64 ivs[4 - 1]; 193 u64 last_iv; 194 195 /* Start of the last block. */ 196 src += nbytes / bsize - 1; 197 dst += nbytes / bsize - 1; 198 199 last_iv = *src; 200 201 /* Process four block batch */ 202 if (nbytes >= bsize * 4) { 203 do { 204 nbytes -= bsize * 4 - bsize; 205 src -= 4 - 1; 206 dst -= 4 - 1; 207 208 ivs[0] = src[0]; 209 ivs[1] = src[1]; 210 ivs[2] = src[2]; 211 212 blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src); 213 214 dst[1] ^= ivs[0]; 215 dst[2] ^= ivs[1]; 216 dst[3] ^= ivs[2]; 217 218 nbytes -= bsize; 219 if (nbytes < bsize) 220 goto done; 221 222 *dst ^= *(src - 1); 223 src -= 1; 224 dst -= 1; 225 } while (nbytes >= bsize * 4); 226 } 227 228 /* Handle leftovers */ 229 for (;;) { 230 blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src); 231 232 nbytes -= bsize; 233 if (nbytes < bsize) 234 break; 235 236 *dst ^= *(src - 1); 237 src -= 1; 238 dst -= 1; 239 } 240 241 done: 242 *dst ^= *(u64 *)walk->iv; 243 *(u64 *)walk->iv = last_iv; 244 245 return nbytes; 246 } 247 248 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 249 struct scatterlist *src, unsigned int nbytes) 250 { 251 struct blkcipher_walk walk; 252 int err; 253 254 blkcipher_walk_init(&walk, dst, src, nbytes); 255 err = blkcipher_walk_virt(desc, &walk); 256 257 while ((nbytes = walk.nbytes)) { 258 nbytes = __cbc_decrypt(desc, &walk); 259 err = blkcipher_walk_done(desc, &walk, nbytes); 260 } 261 262 return err; 263 } 264 265 static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk) 266 { 267 u8 *ctrblk = walk->iv; 268 u8 keystream[BF_BLOCK_SIZE]; 269 u8 *src = walk->src.virt.addr; 270 u8 *dst = walk->dst.virt.addr; 271 unsigned int nbytes = walk->nbytes; 272 273 blowfish_enc_blk(ctx, keystream, ctrblk); 274 crypto_xor_cpy(dst, keystream, src, nbytes); 275 276 crypto_inc(ctrblk, BF_BLOCK_SIZE); 277 } 278 279 static unsigned int __ctr_crypt(struct blkcipher_desc *desc, 280 struct blkcipher_walk *walk) 281 { 282 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 283 unsigned int bsize = BF_BLOCK_SIZE; 284 unsigned int nbytes = walk->nbytes; 285 u64 *src = (u64 *)walk->src.virt.addr; 286 u64 *dst = (u64 *)walk->dst.virt.addr; 287 u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); 288 __be64 ctrblocks[4]; 289 290 /* Process four block batch */ 291 if (nbytes >= bsize * 4) { 292 do { 293 if (dst != src) { 294 dst[0] = src[0]; 295 dst[1] = src[1]; 296 dst[2] = src[2]; 297 dst[3] = src[3]; 298 } 299 300 /* create ctrblks for parallel encrypt */ 301 ctrblocks[0] = cpu_to_be64(ctrblk++); 302 ctrblocks[1] = cpu_to_be64(ctrblk++); 303 ctrblocks[2] = cpu_to_be64(ctrblk++); 304 ctrblocks[3] = cpu_to_be64(ctrblk++); 305 306 blowfish_enc_blk_xor_4way(ctx, (u8 *)dst, 307 (u8 *)ctrblocks); 308 309 src += 4; 310 dst += 4; 311 } while ((nbytes -= bsize * 4) >= bsize * 4); 312 313 if (nbytes < bsize) 314 goto done; 315 } 316 317 /* Handle leftovers */ 318 do { 319 if (dst != src) 320 *dst = *src; 321 322 ctrblocks[0] = cpu_to_be64(ctrblk++); 323 324 blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks); 325 326 src += 1; 327 dst += 1; 328 } while ((nbytes -= bsize) >= bsize); 329 330 done: 331 *(__be64 *)walk->iv = cpu_to_be64(ctrblk); 332 return nbytes; 333 } 334 335 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 336 struct scatterlist *src, unsigned int nbytes) 337 { 338 struct blkcipher_walk walk; 339 int err; 340 341 blkcipher_walk_init(&walk, dst, src, nbytes); 342 err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE); 343 344 while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) { 345 nbytes = __ctr_crypt(desc, &walk); 346 err = blkcipher_walk_done(desc, &walk, nbytes); 347 } 348 349 if (walk.nbytes) { 350 ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk); 351 err = blkcipher_walk_done(desc, &walk, 0); 352 } 353 354 return err; 355 } 356 357 static struct crypto_alg bf_algs[4] = { { 358 .cra_name = "blowfish", 359 .cra_driver_name = "blowfish-asm", 360 .cra_priority = 200, 361 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 362 .cra_blocksize = BF_BLOCK_SIZE, 363 .cra_ctxsize = sizeof(struct bf_ctx), 364 .cra_alignmask = 0, 365 .cra_module = THIS_MODULE, 366 .cra_u = { 367 .cipher = { 368 .cia_min_keysize = BF_MIN_KEY_SIZE, 369 .cia_max_keysize = BF_MAX_KEY_SIZE, 370 .cia_setkey = blowfish_setkey, 371 .cia_encrypt = blowfish_encrypt, 372 .cia_decrypt = blowfish_decrypt, 373 } 374 } 375 }, { 376 .cra_name = "ecb(blowfish)", 377 .cra_driver_name = "ecb-blowfish-asm", 378 .cra_priority = 300, 379 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 380 .cra_blocksize = BF_BLOCK_SIZE, 381 .cra_ctxsize = sizeof(struct bf_ctx), 382 .cra_alignmask = 0, 383 .cra_type = &crypto_blkcipher_type, 384 .cra_module = THIS_MODULE, 385 .cra_u = { 386 .blkcipher = { 387 .min_keysize = BF_MIN_KEY_SIZE, 388 .max_keysize = BF_MAX_KEY_SIZE, 389 .setkey = blowfish_setkey, 390 .encrypt = ecb_encrypt, 391 .decrypt = ecb_decrypt, 392 }, 393 }, 394 }, { 395 .cra_name = "cbc(blowfish)", 396 .cra_driver_name = "cbc-blowfish-asm", 397 .cra_priority = 300, 398 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 399 .cra_blocksize = BF_BLOCK_SIZE, 400 .cra_ctxsize = sizeof(struct bf_ctx), 401 .cra_alignmask = 0, 402 .cra_type = &crypto_blkcipher_type, 403 .cra_module = THIS_MODULE, 404 .cra_u = { 405 .blkcipher = { 406 .min_keysize = BF_MIN_KEY_SIZE, 407 .max_keysize = BF_MAX_KEY_SIZE, 408 .ivsize = BF_BLOCK_SIZE, 409 .setkey = blowfish_setkey, 410 .encrypt = cbc_encrypt, 411 .decrypt = cbc_decrypt, 412 }, 413 }, 414 }, { 415 .cra_name = "ctr(blowfish)", 416 .cra_driver_name = "ctr-blowfish-asm", 417 .cra_priority = 300, 418 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 419 .cra_blocksize = 1, 420 .cra_ctxsize = sizeof(struct bf_ctx), 421 .cra_alignmask = 0, 422 .cra_type = &crypto_blkcipher_type, 423 .cra_module = THIS_MODULE, 424 .cra_u = { 425 .blkcipher = { 426 .min_keysize = BF_MIN_KEY_SIZE, 427 .max_keysize = BF_MAX_KEY_SIZE, 428 .ivsize = BF_BLOCK_SIZE, 429 .setkey = blowfish_setkey, 430 .encrypt = ctr_crypt, 431 .decrypt = ctr_crypt, 432 }, 433 }, 434 } }; 435 436 static bool is_blacklisted_cpu(void) 437 { 438 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 439 return false; 440 441 if (boot_cpu_data.x86 == 0x0f) { 442 /* 443 * On Pentium 4, blowfish-x86_64 is slower than generic C 444 * implementation because use of 64bit rotates (which are really 445 * slow on P4). Therefore blacklist P4s. 446 */ 447 return true; 448 } 449 450 return false; 451 } 452 453 static int force; 454 module_param(force, int, 0); 455 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 456 457 static int __init init(void) 458 { 459 if (!force && is_blacklisted_cpu()) { 460 printk(KERN_INFO 461 "blowfish-x86_64: performance on this CPU " 462 "would be suboptimal: disabling " 463 "blowfish-x86_64.\n"); 464 return -ENODEV; 465 } 466 467 return crypto_register_algs(bf_algs, ARRAY_SIZE(bf_algs)); 468 } 469 470 static void __exit fini(void) 471 { 472 crypto_unregister_algs(bf_algs, ARRAY_SIZE(bf_algs)); 473 } 474 475 module_init(init); 476 module_exit(fini); 477 478 MODULE_LICENSE("GPL"); 479 MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized"); 480 MODULE_ALIAS_CRYPTO("blowfish"); 481 MODULE_ALIAS_CRYPTO("blowfish-asm"); 482