1 /* 2 * Glue Code for assembler optimized version of Blowfish 3 * 4 * Copyright © 2011-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 5 * 6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 8 * CTR part based on code (crypto/ctr.c) by: 9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; if not, write to the Free Software 23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 24 * USA 25 * 26 */ 27 28 #include <asm/processor.h> 29 #include <crypto/blowfish.h> 30 #include <linux/crypto.h> 31 #include <linux/init.h> 32 #include <linux/module.h> 33 #include <linux/types.h> 34 #include <crypto/algapi.h> 35 #include <asm/crypto/blowfish.h> 36 37 /* regular block cipher functions */ 38 asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src, 39 bool xor); 40 EXPORT_SYMBOL_GPL(__blowfish_enc_blk); 41 42 asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src); 43 EXPORT_SYMBOL_GPL(blowfish_dec_blk); 44 45 /* 4-way parallel cipher functions */ 46 asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, 47 const u8 *src, bool xor); 48 EXPORT_SYMBOL_GPL(__blowfish_enc_blk_4way); 49 50 asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst, 51 const u8 *src); 52 EXPORT_SYMBOL_GPL(blowfish_dec_blk_4way); 53 54 static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 55 { 56 blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src); 57 } 58 59 static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 60 { 61 blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src); 62 } 63 64 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, 65 void (*fn)(struct bf_ctx *, u8 *, const u8 *), 66 void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *)) 67 { 68 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 69 unsigned int bsize = BF_BLOCK_SIZE; 70 unsigned int nbytes; 71 int err; 72 73 err = blkcipher_walk_virt(desc, walk); 74 75 while ((nbytes = walk->nbytes)) { 76 u8 *wsrc = walk->src.virt.addr; 77 u8 *wdst = walk->dst.virt.addr; 78 79 /* Process four block batch */ 80 if (nbytes >= bsize * 4) { 81 do { 82 fn_4way(ctx, wdst, wsrc); 83 84 wsrc += bsize * 4; 85 wdst += bsize * 4; 86 nbytes -= bsize * 4; 87 } while (nbytes >= bsize * 4); 88 89 if (nbytes < bsize) 90 goto done; 91 } 92 93 /* Handle leftovers */ 94 do { 95 fn(ctx, wdst, wsrc); 96 97 wsrc += bsize; 98 wdst += bsize; 99 nbytes -= bsize; 100 } while (nbytes >= bsize); 101 102 done: 103 err = blkcipher_walk_done(desc, walk, nbytes); 104 } 105 106 return err; 107 } 108 109 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 110 struct scatterlist *src, unsigned int nbytes) 111 { 112 struct blkcipher_walk walk; 113 114 blkcipher_walk_init(&walk, dst, src, nbytes); 115 return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way); 116 } 117 118 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 119 struct scatterlist *src, unsigned int nbytes) 120 { 121 struct blkcipher_walk walk; 122 123 blkcipher_walk_init(&walk, dst, src, nbytes); 124 return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way); 125 } 126 127 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 128 struct blkcipher_walk *walk) 129 { 130 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 131 unsigned int bsize = BF_BLOCK_SIZE; 132 unsigned int nbytes = walk->nbytes; 133 u64 *src = (u64 *)walk->src.virt.addr; 134 u64 *dst = (u64 *)walk->dst.virt.addr; 135 u64 *iv = (u64 *)walk->iv; 136 137 do { 138 *dst = *src ^ *iv; 139 blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 140 iv = dst; 141 142 src += 1; 143 dst += 1; 144 nbytes -= bsize; 145 } while (nbytes >= bsize); 146 147 *(u64 *)walk->iv = *iv; 148 return nbytes; 149 } 150 151 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 152 struct scatterlist *src, unsigned int nbytes) 153 { 154 struct blkcipher_walk walk; 155 int err; 156 157 blkcipher_walk_init(&walk, dst, src, nbytes); 158 err = blkcipher_walk_virt(desc, &walk); 159 160 while ((nbytes = walk.nbytes)) { 161 nbytes = __cbc_encrypt(desc, &walk); 162 err = blkcipher_walk_done(desc, &walk, nbytes); 163 } 164 165 return err; 166 } 167 168 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 169 struct blkcipher_walk *walk) 170 { 171 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 172 unsigned int bsize = BF_BLOCK_SIZE; 173 unsigned int nbytes = walk->nbytes; 174 u64 *src = (u64 *)walk->src.virt.addr; 175 u64 *dst = (u64 *)walk->dst.virt.addr; 176 u64 ivs[4 - 1]; 177 u64 last_iv; 178 179 /* Start of the last block. */ 180 src += nbytes / bsize - 1; 181 dst += nbytes / bsize - 1; 182 183 last_iv = *src; 184 185 /* Process four block batch */ 186 if (nbytes >= bsize * 4) { 187 do { 188 nbytes -= bsize * 4 - bsize; 189 src -= 4 - 1; 190 dst -= 4 - 1; 191 192 ivs[0] = src[0]; 193 ivs[1] = src[1]; 194 ivs[2] = src[2]; 195 196 blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src); 197 198 dst[1] ^= ivs[0]; 199 dst[2] ^= ivs[1]; 200 dst[3] ^= ivs[2]; 201 202 nbytes -= bsize; 203 if (nbytes < bsize) 204 goto done; 205 206 *dst ^= *(src - 1); 207 src -= 1; 208 dst -= 1; 209 } while (nbytes >= bsize * 4); 210 211 if (nbytes < bsize) 212 goto done; 213 } 214 215 /* Handle leftovers */ 216 for (;;) { 217 blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src); 218 219 nbytes -= bsize; 220 if (nbytes < bsize) 221 break; 222 223 *dst ^= *(src - 1); 224 src -= 1; 225 dst -= 1; 226 } 227 228 done: 229 *dst ^= *(u64 *)walk->iv; 230 *(u64 *)walk->iv = last_iv; 231 232 return nbytes; 233 } 234 235 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 236 struct scatterlist *src, unsigned int nbytes) 237 { 238 struct blkcipher_walk walk; 239 int err; 240 241 blkcipher_walk_init(&walk, dst, src, nbytes); 242 err = blkcipher_walk_virt(desc, &walk); 243 244 while ((nbytes = walk.nbytes)) { 245 nbytes = __cbc_decrypt(desc, &walk); 246 err = blkcipher_walk_done(desc, &walk, nbytes); 247 } 248 249 return err; 250 } 251 252 static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk) 253 { 254 u8 *ctrblk = walk->iv; 255 u8 keystream[BF_BLOCK_SIZE]; 256 u8 *src = walk->src.virt.addr; 257 u8 *dst = walk->dst.virt.addr; 258 unsigned int nbytes = walk->nbytes; 259 260 blowfish_enc_blk(ctx, keystream, ctrblk); 261 crypto_xor(keystream, src, nbytes); 262 memcpy(dst, keystream, nbytes); 263 264 crypto_inc(ctrblk, BF_BLOCK_SIZE); 265 } 266 267 static unsigned int __ctr_crypt(struct blkcipher_desc *desc, 268 struct blkcipher_walk *walk) 269 { 270 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 271 unsigned int bsize = BF_BLOCK_SIZE; 272 unsigned int nbytes = walk->nbytes; 273 u64 *src = (u64 *)walk->src.virt.addr; 274 u64 *dst = (u64 *)walk->dst.virt.addr; 275 u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); 276 __be64 ctrblocks[4]; 277 278 /* Process four block batch */ 279 if (nbytes >= bsize * 4) { 280 do { 281 if (dst != src) { 282 dst[0] = src[0]; 283 dst[1] = src[1]; 284 dst[2] = src[2]; 285 dst[3] = src[3]; 286 } 287 288 /* create ctrblks for parallel encrypt */ 289 ctrblocks[0] = cpu_to_be64(ctrblk++); 290 ctrblocks[1] = cpu_to_be64(ctrblk++); 291 ctrblocks[2] = cpu_to_be64(ctrblk++); 292 ctrblocks[3] = cpu_to_be64(ctrblk++); 293 294 blowfish_enc_blk_xor_4way(ctx, (u8 *)dst, 295 (u8 *)ctrblocks); 296 297 src += 4; 298 dst += 4; 299 } while ((nbytes -= bsize * 4) >= bsize * 4); 300 301 if (nbytes < bsize) 302 goto done; 303 } 304 305 /* Handle leftovers */ 306 do { 307 if (dst != src) 308 *dst = *src; 309 310 ctrblocks[0] = cpu_to_be64(ctrblk++); 311 312 blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks); 313 314 src += 1; 315 dst += 1; 316 } while ((nbytes -= bsize) >= bsize); 317 318 done: 319 *(__be64 *)walk->iv = cpu_to_be64(ctrblk); 320 return nbytes; 321 } 322 323 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 324 struct scatterlist *src, unsigned int nbytes) 325 { 326 struct blkcipher_walk walk; 327 int err; 328 329 blkcipher_walk_init(&walk, dst, src, nbytes); 330 err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE); 331 332 while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) { 333 nbytes = __ctr_crypt(desc, &walk); 334 err = blkcipher_walk_done(desc, &walk, nbytes); 335 } 336 337 if (walk.nbytes) { 338 ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk); 339 err = blkcipher_walk_done(desc, &walk, 0); 340 } 341 342 return err; 343 } 344 345 static struct crypto_alg bf_algs[4] = { { 346 .cra_name = "blowfish", 347 .cra_driver_name = "blowfish-asm", 348 .cra_priority = 200, 349 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 350 .cra_blocksize = BF_BLOCK_SIZE, 351 .cra_ctxsize = sizeof(struct bf_ctx), 352 .cra_alignmask = 0, 353 .cra_module = THIS_MODULE, 354 .cra_u = { 355 .cipher = { 356 .cia_min_keysize = BF_MIN_KEY_SIZE, 357 .cia_max_keysize = BF_MAX_KEY_SIZE, 358 .cia_setkey = blowfish_setkey, 359 .cia_encrypt = blowfish_encrypt, 360 .cia_decrypt = blowfish_decrypt, 361 } 362 } 363 }, { 364 .cra_name = "ecb(blowfish)", 365 .cra_driver_name = "ecb-blowfish-asm", 366 .cra_priority = 300, 367 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 368 .cra_blocksize = BF_BLOCK_SIZE, 369 .cra_ctxsize = sizeof(struct bf_ctx), 370 .cra_alignmask = 0, 371 .cra_type = &crypto_blkcipher_type, 372 .cra_module = THIS_MODULE, 373 .cra_u = { 374 .blkcipher = { 375 .min_keysize = BF_MIN_KEY_SIZE, 376 .max_keysize = BF_MAX_KEY_SIZE, 377 .setkey = blowfish_setkey, 378 .encrypt = ecb_encrypt, 379 .decrypt = ecb_decrypt, 380 }, 381 }, 382 }, { 383 .cra_name = "cbc(blowfish)", 384 .cra_driver_name = "cbc-blowfish-asm", 385 .cra_priority = 300, 386 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 387 .cra_blocksize = BF_BLOCK_SIZE, 388 .cra_ctxsize = sizeof(struct bf_ctx), 389 .cra_alignmask = 0, 390 .cra_type = &crypto_blkcipher_type, 391 .cra_module = THIS_MODULE, 392 .cra_u = { 393 .blkcipher = { 394 .min_keysize = BF_MIN_KEY_SIZE, 395 .max_keysize = BF_MAX_KEY_SIZE, 396 .ivsize = BF_BLOCK_SIZE, 397 .setkey = blowfish_setkey, 398 .encrypt = cbc_encrypt, 399 .decrypt = cbc_decrypt, 400 }, 401 }, 402 }, { 403 .cra_name = "ctr(blowfish)", 404 .cra_driver_name = "ctr-blowfish-asm", 405 .cra_priority = 300, 406 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 407 .cra_blocksize = 1, 408 .cra_ctxsize = sizeof(struct bf_ctx), 409 .cra_alignmask = 0, 410 .cra_type = &crypto_blkcipher_type, 411 .cra_module = THIS_MODULE, 412 .cra_u = { 413 .blkcipher = { 414 .min_keysize = BF_MIN_KEY_SIZE, 415 .max_keysize = BF_MAX_KEY_SIZE, 416 .ivsize = BF_BLOCK_SIZE, 417 .setkey = blowfish_setkey, 418 .encrypt = ctr_crypt, 419 .decrypt = ctr_crypt, 420 }, 421 }, 422 } }; 423 424 static bool is_blacklisted_cpu(void) 425 { 426 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 427 return false; 428 429 if (boot_cpu_data.x86 == 0x0f) { 430 /* 431 * On Pentium 4, blowfish-x86_64 is slower than generic C 432 * implementation because use of 64bit rotates (which are really 433 * slow on P4). Therefore blacklist P4s. 434 */ 435 return true; 436 } 437 438 return false; 439 } 440 441 static int force; 442 module_param(force, int, 0); 443 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 444 445 static int __init init(void) 446 { 447 if (!force && is_blacklisted_cpu()) { 448 printk(KERN_INFO 449 "blowfish-x86_64: performance on this CPU " 450 "would be suboptimal: disabling " 451 "blowfish-x86_64.\n"); 452 return -ENODEV; 453 } 454 455 return crypto_register_algs(bf_algs, ARRAY_SIZE(bf_algs)); 456 } 457 458 static void __exit fini(void) 459 { 460 crypto_unregister_algs(bf_algs, ARRAY_SIZE(bf_algs)); 461 } 462 463 module_init(init); 464 module_exit(fini); 465 466 MODULE_LICENSE("GPL"); 467 MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized"); 468 MODULE_ALIAS("blowfish"); 469 MODULE_ALIAS("blowfish-asm"); 470