1 /* 2 * Glue Code for assembler optimized version of Blowfish 3 * 4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 5 * 6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 8 * CTR part based on code (crypto/ctr.c) by: 9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2 of the License, or 14 * (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; if not, write to the Free Software 23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 24 * USA 25 * 26 */ 27 28 #include <crypto/algapi.h> 29 #include <crypto/blowfish.h> 30 #include <crypto/internal/skcipher.h> 31 #include <linux/crypto.h> 32 #include <linux/init.h> 33 #include <linux/module.h> 34 #include <linux/types.h> 35 36 /* regular block cipher functions */ 37 asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src, 38 bool xor); 39 asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src); 40 41 /* 4-way parallel cipher functions */ 42 asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, 43 const u8 *src, bool xor); 44 asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst, 45 const u8 *src); 46 47 static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src) 48 { 49 __blowfish_enc_blk(ctx, dst, src, false); 50 } 51 52 static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst, 53 const u8 *src) 54 { 55 __blowfish_enc_blk(ctx, dst, src, true); 56 } 57 58 static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst, 59 const u8 *src) 60 { 61 __blowfish_enc_blk_4way(ctx, dst, src, false); 62 } 63 64 static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst, 65 const u8 *src) 66 { 67 __blowfish_enc_blk_4way(ctx, dst, src, true); 68 } 69 70 static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 71 { 72 blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src); 73 } 74 75 static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 76 { 77 blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src); 78 } 79 80 static int blowfish_setkey_skcipher(struct crypto_skcipher *tfm, 81 const u8 *key, unsigned int keylen) 82 { 83 return blowfish_setkey(&tfm->base, key, keylen); 84 } 85 86 static int ecb_crypt(struct skcipher_request *req, 87 void (*fn)(struct bf_ctx *, u8 *, const u8 *), 88 void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *)) 89 { 90 unsigned int bsize = BF_BLOCK_SIZE; 91 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 92 struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); 93 struct skcipher_walk walk; 94 unsigned int nbytes; 95 int err; 96 97 err = skcipher_walk_virt(&walk, req, false); 98 99 while ((nbytes = walk.nbytes)) { 100 u8 *wsrc = walk.src.virt.addr; 101 u8 *wdst = walk.dst.virt.addr; 102 103 /* Process four block batch */ 104 if (nbytes >= bsize * 4) { 105 do { 106 fn_4way(ctx, wdst, wsrc); 107 108 wsrc += bsize * 4; 109 wdst += bsize * 4; 110 nbytes -= bsize * 4; 111 } while (nbytes >= bsize * 4); 112 113 if (nbytes < bsize) 114 goto done; 115 } 116 117 /* Handle leftovers */ 118 do { 119 fn(ctx, wdst, wsrc); 120 121 wsrc += bsize; 122 wdst += bsize; 123 nbytes -= bsize; 124 } while (nbytes >= bsize); 125 126 done: 127 err = skcipher_walk_done(&walk, nbytes); 128 } 129 130 return err; 131 } 132 133 static int ecb_encrypt(struct skcipher_request *req) 134 { 135 return ecb_crypt(req, blowfish_enc_blk, blowfish_enc_blk_4way); 136 } 137 138 static int ecb_decrypt(struct skcipher_request *req) 139 { 140 return ecb_crypt(req, blowfish_dec_blk, blowfish_dec_blk_4way); 141 } 142 143 static unsigned int __cbc_encrypt(struct bf_ctx *ctx, 144 struct skcipher_walk *walk) 145 { 146 unsigned int bsize = BF_BLOCK_SIZE; 147 unsigned int nbytes = walk->nbytes; 148 u64 *src = (u64 *)walk->src.virt.addr; 149 u64 *dst = (u64 *)walk->dst.virt.addr; 150 u64 *iv = (u64 *)walk->iv; 151 152 do { 153 *dst = *src ^ *iv; 154 blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 155 iv = dst; 156 157 src += 1; 158 dst += 1; 159 nbytes -= bsize; 160 } while (nbytes >= bsize); 161 162 *(u64 *)walk->iv = *iv; 163 return nbytes; 164 } 165 166 static int cbc_encrypt(struct skcipher_request *req) 167 { 168 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 169 struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); 170 struct skcipher_walk walk; 171 unsigned int nbytes; 172 int err; 173 174 err = skcipher_walk_virt(&walk, req, false); 175 176 while ((nbytes = walk.nbytes)) { 177 nbytes = __cbc_encrypt(ctx, &walk); 178 err = skcipher_walk_done(&walk, nbytes); 179 } 180 181 return err; 182 } 183 184 static unsigned int __cbc_decrypt(struct bf_ctx *ctx, 185 struct skcipher_walk *walk) 186 { 187 unsigned int bsize = BF_BLOCK_SIZE; 188 unsigned int nbytes = walk->nbytes; 189 u64 *src = (u64 *)walk->src.virt.addr; 190 u64 *dst = (u64 *)walk->dst.virt.addr; 191 u64 ivs[4 - 1]; 192 u64 last_iv; 193 194 /* Start of the last block. */ 195 src += nbytes / bsize - 1; 196 dst += nbytes / bsize - 1; 197 198 last_iv = *src; 199 200 /* Process four block batch */ 201 if (nbytes >= bsize * 4) { 202 do { 203 nbytes -= bsize * 4 - bsize; 204 src -= 4 - 1; 205 dst -= 4 - 1; 206 207 ivs[0] = src[0]; 208 ivs[1] = src[1]; 209 ivs[2] = src[2]; 210 211 blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src); 212 213 dst[1] ^= ivs[0]; 214 dst[2] ^= ivs[1]; 215 dst[3] ^= ivs[2]; 216 217 nbytes -= bsize; 218 if (nbytes < bsize) 219 goto done; 220 221 *dst ^= *(src - 1); 222 src -= 1; 223 dst -= 1; 224 } while (nbytes >= bsize * 4); 225 } 226 227 /* Handle leftovers */ 228 for (;;) { 229 blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src); 230 231 nbytes -= bsize; 232 if (nbytes < bsize) 233 break; 234 235 *dst ^= *(src - 1); 236 src -= 1; 237 dst -= 1; 238 } 239 240 done: 241 *dst ^= *(u64 *)walk->iv; 242 *(u64 *)walk->iv = last_iv; 243 244 return nbytes; 245 } 246 247 static int cbc_decrypt(struct skcipher_request *req) 248 { 249 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 250 struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); 251 struct skcipher_walk walk; 252 unsigned int nbytes; 253 int err; 254 255 err = skcipher_walk_virt(&walk, req, false); 256 257 while ((nbytes = walk.nbytes)) { 258 nbytes = __cbc_decrypt(ctx, &walk); 259 err = skcipher_walk_done(&walk, nbytes); 260 } 261 262 return err; 263 } 264 265 static void ctr_crypt_final(struct bf_ctx *ctx, struct skcipher_walk *walk) 266 { 267 u8 *ctrblk = walk->iv; 268 u8 keystream[BF_BLOCK_SIZE]; 269 u8 *src = walk->src.virt.addr; 270 u8 *dst = walk->dst.virt.addr; 271 unsigned int nbytes = walk->nbytes; 272 273 blowfish_enc_blk(ctx, keystream, ctrblk); 274 crypto_xor_cpy(dst, keystream, src, nbytes); 275 276 crypto_inc(ctrblk, BF_BLOCK_SIZE); 277 } 278 279 static unsigned int __ctr_crypt(struct bf_ctx *ctx, struct skcipher_walk *walk) 280 { 281 unsigned int bsize = BF_BLOCK_SIZE; 282 unsigned int nbytes = walk->nbytes; 283 u64 *src = (u64 *)walk->src.virt.addr; 284 u64 *dst = (u64 *)walk->dst.virt.addr; 285 u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); 286 __be64 ctrblocks[4]; 287 288 /* Process four block batch */ 289 if (nbytes >= bsize * 4) { 290 do { 291 if (dst != src) { 292 dst[0] = src[0]; 293 dst[1] = src[1]; 294 dst[2] = src[2]; 295 dst[3] = src[3]; 296 } 297 298 /* create ctrblks for parallel encrypt */ 299 ctrblocks[0] = cpu_to_be64(ctrblk++); 300 ctrblocks[1] = cpu_to_be64(ctrblk++); 301 ctrblocks[2] = cpu_to_be64(ctrblk++); 302 ctrblocks[3] = cpu_to_be64(ctrblk++); 303 304 blowfish_enc_blk_xor_4way(ctx, (u8 *)dst, 305 (u8 *)ctrblocks); 306 307 src += 4; 308 dst += 4; 309 } while ((nbytes -= bsize * 4) >= bsize * 4); 310 311 if (nbytes < bsize) 312 goto done; 313 } 314 315 /* Handle leftovers */ 316 do { 317 if (dst != src) 318 *dst = *src; 319 320 ctrblocks[0] = cpu_to_be64(ctrblk++); 321 322 blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks); 323 324 src += 1; 325 dst += 1; 326 } while ((nbytes -= bsize) >= bsize); 327 328 done: 329 *(__be64 *)walk->iv = cpu_to_be64(ctrblk); 330 return nbytes; 331 } 332 333 static int ctr_crypt(struct skcipher_request *req) 334 { 335 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 336 struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); 337 struct skcipher_walk walk; 338 unsigned int nbytes; 339 int err; 340 341 err = skcipher_walk_virt(&walk, req, false); 342 343 while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) { 344 nbytes = __ctr_crypt(ctx, &walk); 345 err = skcipher_walk_done(&walk, nbytes); 346 } 347 348 if (nbytes) { 349 ctr_crypt_final(ctx, &walk); 350 err = skcipher_walk_done(&walk, 0); 351 } 352 353 return err; 354 } 355 356 static struct crypto_alg bf_cipher_alg = { 357 .cra_name = "blowfish", 358 .cra_driver_name = "blowfish-asm", 359 .cra_priority = 200, 360 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 361 .cra_blocksize = BF_BLOCK_SIZE, 362 .cra_ctxsize = sizeof(struct bf_ctx), 363 .cra_alignmask = 0, 364 .cra_module = THIS_MODULE, 365 .cra_u = { 366 .cipher = { 367 .cia_min_keysize = BF_MIN_KEY_SIZE, 368 .cia_max_keysize = BF_MAX_KEY_SIZE, 369 .cia_setkey = blowfish_setkey, 370 .cia_encrypt = blowfish_encrypt, 371 .cia_decrypt = blowfish_decrypt, 372 } 373 } 374 }; 375 376 static struct skcipher_alg bf_skcipher_algs[] = { 377 { 378 .base.cra_name = "ecb(blowfish)", 379 .base.cra_driver_name = "ecb-blowfish-asm", 380 .base.cra_priority = 300, 381 .base.cra_blocksize = BF_BLOCK_SIZE, 382 .base.cra_ctxsize = sizeof(struct bf_ctx), 383 .base.cra_module = THIS_MODULE, 384 .min_keysize = BF_MIN_KEY_SIZE, 385 .max_keysize = BF_MAX_KEY_SIZE, 386 .setkey = blowfish_setkey_skcipher, 387 .encrypt = ecb_encrypt, 388 .decrypt = ecb_decrypt, 389 }, { 390 .base.cra_name = "cbc(blowfish)", 391 .base.cra_driver_name = "cbc-blowfish-asm", 392 .base.cra_priority = 300, 393 .base.cra_blocksize = BF_BLOCK_SIZE, 394 .base.cra_ctxsize = sizeof(struct bf_ctx), 395 .base.cra_module = THIS_MODULE, 396 .min_keysize = BF_MIN_KEY_SIZE, 397 .max_keysize = BF_MAX_KEY_SIZE, 398 .ivsize = BF_BLOCK_SIZE, 399 .setkey = blowfish_setkey_skcipher, 400 .encrypt = cbc_encrypt, 401 .decrypt = cbc_decrypt, 402 }, { 403 .base.cra_name = "ctr(blowfish)", 404 .base.cra_driver_name = "ctr-blowfish-asm", 405 .base.cra_priority = 300, 406 .base.cra_blocksize = 1, 407 .base.cra_ctxsize = sizeof(struct bf_ctx), 408 .base.cra_module = THIS_MODULE, 409 .min_keysize = BF_MIN_KEY_SIZE, 410 .max_keysize = BF_MAX_KEY_SIZE, 411 .ivsize = BF_BLOCK_SIZE, 412 .chunksize = BF_BLOCK_SIZE, 413 .setkey = blowfish_setkey_skcipher, 414 .encrypt = ctr_crypt, 415 .decrypt = ctr_crypt, 416 }, 417 }; 418 419 static bool is_blacklisted_cpu(void) 420 { 421 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 422 return false; 423 424 if (boot_cpu_data.x86 == 0x0f) { 425 /* 426 * On Pentium 4, blowfish-x86_64 is slower than generic C 427 * implementation because use of 64bit rotates (which are really 428 * slow on P4). Therefore blacklist P4s. 429 */ 430 return true; 431 } 432 433 return false; 434 } 435 436 static int force; 437 module_param(force, int, 0); 438 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 439 440 static int __init init(void) 441 { 442 int err; 443 444 if (!force && is_blacklisted_cpu()) { 445 printk(KERN_INFO 446 "blowfish-x86_64: performance on this CPU " 447 "would be suboptimal: disabling " 448 "blowfish-x86_64.\n"); 449 return -ENODEV; 450 } 451 452 err = crypto_register_alg(&bf_cipher_alg); 453 if (err) 454 return err; 455 456 err = crypto_register_skciphers(bf_skcipher_algs, 457 ARRAY_SIZE(bf_skcipher_algs)); 458 if (err) 459 crypto_unregister_alg(&bf_cipher_alg); 460 461 return err; 462 } 463 464 static void __exit fini(void) 465 { 466 crypto_unregister_alg(&bf_cipher_alg); 467 crypto_unregister_skciphers(bf_skcipher_algs, 468 ARRAY_SIZE(bf_skcipher_algs)); 469 } 470 471 module_init(init); 472 module_exit(fini); 473 474 MODULE_LICENSE("GPL"); 475 MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized"); 476 MODULE_ALIAS_CRYPTO("blowfish"); 477 MODULE_ALIAS_CRYPTO("blowfish-asm"); 478