1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for assembler optimized version of 3DES 4 * 5 * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 * 7 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 8 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 9 */ 10 11 #include <crypto/algapi.h> 12 #include <crypto/des.h> 13 #include <crypto/internal/skcipher.h> 14 #include <linux/crypto.h> 15 #include <linux/init.h> 16 #include <linux/module.h> 17 #include <linux/types.h> 18 19 struct des3_ede_x86_ctx { 20 struct des3_ede_ctx enc; 21 struct des3_ede_ctx dec; 22 }; 23 24 /* regular block cipher functions */ 25 asmlinkage void des3_ede_x86_64_crypt_blk(const u32 *expkey, u8 *dst, 26 const u8 *src); 27 28 /* 3-way parallel cipher functions */ 29 asmlinkage void des3_ede_x86_64_crypt_blk_3way(const u32 *expkey, u8 *dst, 30 const u8 *src); 31 32 static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, 33 const u8 *src) 34 { 35 u32 *enc_ctx = ctx->enc.expkey; 36 37 des3_ede_x86_64_crypt_blk(enc_ctx, dst, src); 38 } 39 40 static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, 41 const u8 *src) 42 { 43 u32 *dec_ctx = ctx->dec.expkey; 44 45 des3_ede_x86_64_crypt_blk(dec_ctx, dst, src); 46 } 47 48 static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, 49 const u8 *src) 50 { 51 u32 *dec_ctx = ctx->dec.expkey; 52 53 des3_ede_x86_64_crypt_blk_3way(dec_ctx, dst, src); 54 } 55 56 static void des3_ede_x86_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 57 { 58 des3_ede_enc_blk(crypto_tfm_ctx(tfm), dst, src); 59 } 60 61 static void des3_ede_x86_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 62 { 63 des3_ede_dec_blk(crypto_tfm_ctx(tfm), dst, src); 64 } 65 66 static int ecb_crypt(struct skcipher_request *req, const u32 *expkey) 67 { 68 const unsigned int bsize = DES3_EDE_BLOCK_SIZE; 69 struct skcipher_walk walk; 70 unsigned int nbytes; 71 int err; 72 73 err = skcipher_walk_virt(&walk, req, false); 74 75 while ((nbytes = walk.nbytes)) { 76 u8 *wsrc = walk.src.virt.addr; 77 u8 *wdst = walk.dst.virt.addr; 78 79 /* Process four block batch */ 80 if (nbytes >= bsize * 3) { 81 do { 82 des3_ede_x86_64_crypt_blk_3way(expkey, wdst, 83 wsrc); 84 85 wsrc += bsize * 3; 86 wdst += bsize * 3; 87 nbytes -= bsize * 3; 88 } while (nbytes >= bsize * 3); 89 90 if (nbytes < bsize) 91 goto done; 92 } 93 94 /* Handle leftovers */ 95 do { 96 des3_ede_x86_64_crypt_blk(expkey, wdst, wsrc); 97 98 wsrc += bsize; 99 wdst += bsize; 100 nbytes -= bsize; 101 } while (nbytes >= bsize); 102 103 done: 104 err = skcipher_walk_done(&walk, nbytes); 105 } 106 107 return err; 108 } 109 110 static int ecb_encrypt(struct skcipher_request *req) 111 { 112 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 113 struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 114 115 return ecb_crypt(req, ctx->enc.expkey); 116 } 117 118 static int ecb_decrypt(struct skcipher_request *req) 119 { 120 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 121 struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 122 123 return ecb_crypt(req, ctx->dec.expkey); 124 } 125 126 static unsigned int __cbc_encrypt(struct des3_ede_x86_ctx *ctx, 127 struct skcipher_walk *walk) 128 { 129 unsigned int bsize = DES3_EDE_BLOCK_SIZE; 130 unsigned int nbytes = walk->nbytes; 131 u64 *src = (u64 *)walk->src.virt.addr; 132 u64 *dst = (u64 *)walk->dst.virt.addr; 133 u64 *iv = (u64 *)walk->iv; 134 135 do { 136 *dst = *src ^ *iv; 137 des3_ede_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 138 iv = dst; 139 140 src += 1; 141 dst += 1; 142 nbytes -= bsize; 143 } while (nbytes >= bsize); 144 145 *(u64 *)walk->iv = *iv; 146 return nbytes; 147 } 148 149 static int cbc_encrypt(struct skcipher_request *req) 150 { 151 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 152 struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 153 struct skcipher_walk walk; 154 unsigned int nbytes; 155 int err; 156 157 err = skcipher_walk_virt(&walk, req, false); 158 159 while (walk.nbytes) { 160 nbytes = __cbc_encrypt(ctx, &walk); 161 err = skcipher_walk_done(&walk, nbytes); 162 } 163 164 return err; 165 } 166 167 static unsigned int __cbc_decrypt(struct des3_ede_x86_ctx *ctx, 168 struct skcipher_walk *walk) 169 { 170 unsigned int bsize = DES3_EDE_BLOCK_SIZE; 171 unsigned int nbytes = walk->nbytes; 172 u64 *src = (u64 *)walk->src.virt.addr; 173 u64 *dst = (u64 *)walk->dst.virt.addr; 174 u64 ivs[3 - 1]; 175 u64 last_iv; 176 177 /* Start of the last block. */ 178 src += nbytes / bsize - 1; 179 dst += nbytes / bsize - 1; 180 181 last_iv = *src; 182 183 /* Process four block batch */ 184 if (nbytes >= bsize * 3) { 185 do { 186 nbytes -= bsize * 3 - bsize; 187 src -= 3 - 1; 188 dst -= 3 - 1; 189 190 ivs[0] = src[0]; 191 ivs[1] = src[1]; 192 193 des3_ede_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); 194 195 dst[1] ^= ivs[0]; 196 dst[2] ^= ivs[1]; 197 198 nbytes -= bsize; 199 if (nbytes < bsize) 200 goto done; 201 202 *dst ^= *(src - 1); 203 src -= 1; 204 dst -= 1; 205 } while (nbytes >= bsize * 3); 206 } 207 208 /* Handle leftovers */ 209 for (;;) { 210 des3_ede_dec_blk(ctx, (u8 *)dst, (u8 *)src); 211 212 nbytes -= bsize; 213 if (nbytes < bsize) 214 break; 215 216 *dst ^= *(src - 1); 217 src -= 1; 218 dst -= 1; 219 } 220 221 done: 222 *dst ^= *(u64 *)walk->iv; 223 *(u64 *)walk->iv = last_iv; 224 225 return nbytes; 226 } 227 228 static int cbc_decrypt(struct skcipher_request *req) 229 { 230 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 231 struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 232 struct skcipher_walk walk; 233 unsigned int nbytes; 234 int err; 235 236 err = skcipher_walk_virt(&walk, req, false); 237 238 while (walk.nbytes) { 239 nbytes = __cbc_decrypt(ctx, &walk); 240 err = skcipher_walk_done(&walk, nbytes); 241 } 242 243 return err; 244 } 245 246 static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key, 247 unsigned int keylen) 248 { 249 struct des3_ede_x86_ctx *ctx = crypto_tfm_ctx(tfm); 250 u32 i, j, tmp; 251 int err; 252 253 err = des3_ede_expand_key(&ctx->enc, key, keylen); 254 if (err == -ENOKEY) { 255 if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) 256 err = -EINVAL; 257 else 258 err = 0; 259 } 260 261 if (err) { 262 memset(ctx, 0, sizeof(*ctx)); 263 return err; 264 } 265 266 /* Fix encryption context for this implementation and form decryption 267 * context. */ 268 j = DES3_EDE_EXPKEY_WORDS - 2; 269 for (i = 0; i < DES3_EDE_EXPKEY_WORDS; i += 2, j -= 2) { 270 tmp = ror32(ctx->enc.expkey[i + 1], 4); 271 ctx->enc.expkey[i + 1] = tmp; 272 273 ctx->dec.expkey[j + 0] = ctx->enc.expkey[i + 0]; 274 ctx->dec.expkey[j + 1] = tmp; 275 } 276 277 return 0; 278 } 279 280 static int des3_ede_x86_setkey_skcipher(struct crypto_skcipher *tfm, 281 const u8 *key, 282 unsigned int keylen) 283 { 284 return des3_ede_x86_setkey(&tfm->base, key, keylen); 285 } 286 287 static struct crypto_alg des3_ede_cipher = { 288 .cra_name = "des3_ede", 289 .cra_driver_name = "des3_ede-asm", 290 .cra_priority = 200, 291 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 292 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 293 .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 294 .cra_alignmask = 0, 295 .cra_module = THIS_MODULE, 296 .cra_u = { 297 .cipher = { 298 .cia_min_keysize = DES3_EDE_KEY_SIZE, 299 .cia_max_keysize = DES3_EDE_KEY_SIZE, 300 .cia_setkey = des3_ede_x86_setkey, 301 .cia_encrypt = des3_ede_x86_encrypt, 302 .cia_decrypt = des3_ede_x86_decrypt, 303 } 304 } 305 }; 306 307 static struct skcipher_alg des3_ede_skciphers[] = { 308 { 309 .base.cra_name = "ecb(des3_ede)", 310 .base.cra_driver_name = "ecb-des3_ede-asm", 311 .base.cra_priority = 300, 312 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 313 .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 314 .base.cra_module = THIS_MODULE, 315 .min_keysize = DES3_EDE_KEY_SIZE, 316 .max_keysize = DES3_EDE_KEY_SIZE, 317 .setkey = des3_ede_x86_setkey_skcipher, 318 .encrypt = ecb_encrypt, 319 .decrypt = ecb_decrypt, 320 }, { 321 .base.cra_name = "cbc(des3_ede)", 322 .base.cra_driver_name = "cbc-des3_ede-asm", 323 .base.cra_priority = 300, 324 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 325 .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 326 .base.cra_module = THIS_MODULE, 327 .min_keysize = DES3_EDE_KEY_SIZE, 328 .max_keysize = DES3_EDE_KEY_SIZE, 329 .ivsize = DES3_EDE_BLOCK_SIZE, 330 .setkey = des3_ede_x86_setkey_skcipher, 331 .encrypt = cbc_encrypt, 332 .decrypt = cbc_decrypt, 333 } 334 }; 335 336 static bool is_blacklisted_cpu(void) 337 { 338 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 339 return false; 340 341 if (boot_cpu_data.x86 == 0x0f) { 342 /* 343 * On Pentium 4, des3_ede-x86_64 is slower than generic C 344 * implementation because use of 64bit rotates (which are really 345 * slow on P4). Therefore blacklist P4s. 346 */ 347 return true; 348 } 349 350 return false; 351 } 352 353 static int force; 354 module_param(force, int, 0); 355 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 356 357 static int __init des3_ede_x86_init(void) 358 { 359 int err; 360 361 if (!force && is_blacklisted_cpu()) { 362 pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n"); 363 return -ENODEV; 364 } 365 366 err = crypto_register_alg(&des3_ede_cipher); 367 if (err) 368 return err; 369 370 err = crypto_register_skciphers(des3_ede_skciphers, 371 ARRAY_SIZE(des3_ede_skciphers)); 372 if (err) 373 crypto_unregister_alg(&des3_ede_cipher); 374 375 return err; 376 } 377 378 static void __exit des3_ede_x86_fini(void) 379 { 380 crypto_unregister_alg(&des3_ede_cipher); 381 crypto_unregister_skciphers(des3_ede_skciphers, 382 ARRAY_SIZE(des3_ede_skciphers)); 383 } 384 385 module_init(des3_ede_x86_init); 386 module_exit(des3_ede_x86_fini); 387 388 MODULE_LICENSE("GPL"); 389 MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized"); 390 MODULE_ALIAS_CRYPTO("des3_ede"); 391 MODULE_ALIAS_CRYPTO("des3_ede-asm"); 392 MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>"); 393