1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for assembler optimized version of 3DES 4 * 5 * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 * 7 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: 8 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 9 */ 10 11 #include <crypto/algapi.h> 12 #include <crypto/des.h> 13 #include <crypto/internal/skcipher.h> 14 #include <linux/crypto.h> 15 #include <linux/init.h> 16 #include <linux/module.h> 17 #include <linux/types.h> 18 19 struct des3_ede_x86_ctx { 20 struct des3_ede_ctx enc; 21 struct des3_ede_ctx dec; 22 }; 23 24 /* regular block cipher functions */ 25 asmlinkage void des3_ede_x86_64_crypt_blk(const u32 *expkey, u8 *dst, 26 const u8 *src); 27 28 /* 3-way parallel cipher functions */ 29 asmlinkage void des3_ede_x86_64_crypt_blk_3way(const u32 *expkey, u8 *dst, 30 const u8 *src); 31 32 static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, 33 const u8 *src) 34 { 35 u32 *enc_ctx = ctx->enc.expkey; 36 37 des3_ede_x86_64_crypt_blk(enc_ctx, dst, src); 38 } 39 40 static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, 41 const u8 *src) 42 { 43 u32 *dec_ctx = ctx->dec.expkey; 44 45 des3_ede_x86_64_crypt_blk(dec_ctx, dst, src); 46 } 47 48 static inline void des3_ede_enc_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, 49 const u8 *src) 50 { 51 u32 *enc_ctx = ctx->enc.expkey; 52 53 des3_ede_x86_64_crypt_blk_3way(enc_ctx, dst, src); 54 } 55 56 static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, 57 const u8 *src) 58 { 59 u32 *dec_ctx = ctx->dec.expkey; 60 61 des3_ede_x86_64_crypt_blk_3way(dec_ctx, dst, src); 62 } 63 64 static void des3_ede_x86_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 65 { 66 des3_ede_enc_blk(crypto_tfm_ctx(tfm), dst, src); 67 } 68 69 static void des3_ede_x86_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 70 { 71 des3_ede_dec_blk(crypto_tfm_ctx(tfm), dst, src); 72 } 73 74 static int ecb_crypt(struct skcipher_request *req, const u32 *expkey) 75 { 76 const unsigned int bsize = DES3_EDE_BLOCK_SIZE; 77 struct skcipher_walk walk; 78 unsigned int nbytes; 79 int err; 80 81 err = skcipher_walk_virt(&walk, req, false); 82 83 while ((nbytes = walk.nbytes)) { 84 u8 *wsrc = walk.src.virt.addr; 85 u8 *wdst = walk.dst.virt.addr; 86 87 /* Process four block batch */ 88 if (nbytes >= bsize * 3) { 89 do { 90 des3_ede_x86_64_crypt_blk_3way(expkey, wdst, 91 wsrc); 92 93 wsrc += bsize * 3; 94 wdst += bsize * 3; 95 nbytes -= bsize * 3; 96 } while (nbytes >= bsize * 3); 97 98 if (nbytes < bsize) 99 goto done; 100 } 101 102 /* Handle leftovers */ 103 do { 104 des3_ede_x86_64_crypt_blk(expkey, wdst, wsrc); 105 106 wsrc += bsize; 107 wdst += bsize; 108 nbytes -= bsize; 109 } while (nbytes >= bsize); 110 111 done: 112 err = skcipher_walk_done(&walk, nbytes); 113 } 114 115 return err; 116 } 117 118 static int ecb_encrypt(struct skcipher_request *req) 119 { 120 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 121 struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 122 123 return ecb_crypt(req, ctx->enc.expkey); 124 } 125 126 static int ecb_decrypt(struct skcipher_request *req) 127 { 128 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 129 struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 130 131 return ecb_crypt(req, ctx->dec.expkey); 132 } 133 134 static unsigned int __cbc_encrypt(struct des3_ede_x86_ctx *ctx, 135 struct skcipher_walk *walk) 136 { 137 unsigned int bsize = DES3_EDE_BLOCK_SIZE; 138 unsigned int nbytes = walk->nbytes; 139 u64 *src = (u64 *)walk->src.virt.addr; 140 u64 *dst = (u64 *)walk->dst.virt.addr; 141 u64 *iv = (u64 *)walk->iv; 142 143 do { 144 *dst = *src ^ *iv; 145 des3_ede_enc_blk(ctx, (u8 *)dst, (u8 *)dst); 146 iv = dst; 147 148 src += 1; 149 dst += 1; 150 nbytes -= bsize; 151 } while (nbytes >= bsize); 152 153 *(u64 *)walk->iv = *iv; 154 return nbytes; 155 } 156 157 static int cbc_encrypt(struct skcipher_request *req) 158 { 159 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 160 struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 161 struct skcipher_walk walk; 162 unsigned int nbytes; 163 int err; 164 165 err = skcipher_walk_virt(&walk, req, false); 166 167 while ((nbytes = walk.nbytes)) { 168 nbytes = __cbc_encrypt(ctx, &walk); 169 err = skcipher_walk_done(&walk, nbytes); 170 } 171 172 return err; 173 } 174 175 static unsigned int __cbc_decrypt(struct des3_ede_x86_ctx *ctx, 176 struct skcipher_walk *walk) 177 { 178 unsigned int bsize = DES3_EDE_BLOCK_SIZE; 179 unsigned int nbytes = walk->nbytes; 180 u64 *src = (u64 *)walk->src.virt.addr; 181 u64 *dst = (u64 *)walk->dst.virt.addr; 182 u64 ivs[3 - 1]; 183 u64 last_iv; 184 185 /* Start of the last block. */ 186 src += nbytes / bsize - 1; 187 dst += nbytes / bsize - 1; 188 189 last_iv = *src; 190 191 /* Process four block batch */ 192 if (nbytes >= bsize * 3) { 193 do { 194 nbytes -= bsize * 3 - bsize; 195 src -= 3 - 1; 196 dst -= 3 - 1; 197 198 ivs[0] = src[0]; 199 ivs[1] = src[1]; 200 201 des3_ede_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); 202 203 dst[1] ^= ivs[0]; 204 dst[2] ^= ivs[1]; 205 206 nbytes -= bsize; 207 if (nbytes < bsize) 208 goto done; 209 210 *dst ^= *(src - 1); 211 src -= 1; 212 dst -= 1; 213 } while (nbytes >= bsize * 3); 214 } 215 216 /* Handle leftovers */ 217 for (;;) { 218 des3_ede_dec_blk(ctx, (u8 *)dst, (u8 *)src); 219 220 nbytes -= bsize; 221 if (nbytes < bsize) 222 break; 223 224 *dst ^= *(src - 1); 225 src -= 1; 226 dst -= 1; 227 } 228 229 done: 230 *dst ^= *(u64 *)walk->iv; 231 *(u64 *)walk->iv = last_iv; 232 233 return nbytes; 234 } 235 236 static int cbc_decrypt(struct skcipher_request *req) 237 { 238 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 239 struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); 240 struct skcipher_walk walk; 241 unsigned int nbytes; 242 int err; 243 244 err = skcipher_walk_virt(&walk, req, false); 245 246 while ((nbytes = walk.nbytes)) { 247 nbytes = __cbc_decrypt(ctx, &walk); 248 err = skcipher_walk_done(&walk, nbytes); 249 } 250 251 return err; 252 } 253 254 static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key, 255 unsigned int keylen) 256 { 257 struct des3_ede_x86_ctx *ctx = crypto_tfm_ctx(tfm); 258 u32 i, j, tmp; 259 int err; 260 261 err = des3_ede_expand_key(&ctx->enc, key, keylen); 262 if (err == -ENOKEY) { 263 if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) 264 err = -EINVAL; 265 else 266 err = 0; 267 } 268 269 if (err) { 270 memset(ctx, 0, sizeof(*ctx)); 271 return err; 272 } 273 274 /* Fix encryption context for this implementation and form decryption 275 * context. */ 276 j = DES3_EDE_EXPKEY_WORDS - 2; 277 for (i = 0; i < DES3_EDE_EXPKEY_WORDS; i += 2, j -= 2) { 278 tmp = ror32(ctx->enc.expkey[i + 1], 4); 279 ctx->enc.expkey[i + 1] = tmp; 280 281 ctx->dec.expkey[j + 0] = ctx->enc.expkey[i + 0]; 282 ctx->dec.expkey[j + 1] = tmp; 283 } 284 285 return 0; 286 } 287 288 static int des3_ede_x86_setkey_skcipher(struct crypto_skcipher *tfm, 289 const u8 *key, 290 unsigned int keylen) 291 { 292 return des3_ede_x86_setkey(&tfm->base, key, keylen); 293 } 294 295 static struct crypto_alg des3_ede_cipher = { 296 .cra_name = "des3_ede", 297 .cra_driver_name = "des3_ede-asm", 298 .cra_priority = 200, 299 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 300 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 301 .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 302 .cra_alignmask = 0, 303 .cra_module = THIS_MODULE, 304 .cra_u = { 305 .cipher = { 306 .cia_min_keysize = DES3_EDE_KEY_SIZE, 307 .cia_max_keysize = DES3_EDE_KEY_SIZE, 308 .cia_setkey = des3_ede_x86_setkey, 309 .cia_encrypt = des3_ede_x86_encrypt, 310 .cia_decrypt = des3_ede_x86_decrypt, 311 } 312 } 313 }; 314 315 static struct skcipher_alg des3_ede_skciphers[] = { 316 { 317 .base.cra_name = "ecb(des3_ede)", 318 .base.cra_driver_name = "ecb-des3_ede-asm", 319 .base.cra_priority = 300, 320 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 321 .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 322 .base.cra_module = THIS_MODULE, 323 .min_keysize = DES3_EDE_KEY_SIZE, 324 .max_keysize = DES3_EDE_KEY_SIZE, 325 .setkey = des3_ede_x86_setkey_skcipher, 326 .encrypt = ecb_encrypt, 327 .decrypt = ecb_decrypt, 328 }, { 329 .base.cra_name = "cbc(des3_ede)", 330 .base.cra_driver_name = "cbc-des3_ede-asm", 331 .base.cra_priority = 300, 332 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, 333 .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 334 .base.cra_module = THIS_MODULE, 335 .min_keysize = DES3_EDE_KEY_SIZE, 336 .max_keysize = DES3_EDE_KEY_SIZE, 337 .ivsize = DES3_EDE_BLOCK_SIZE, 338 .setkey = des3_ede_x86_setkey_skcipher, 339 .encrypt = cbc_encrypt, 340 .decrypt = cbc_decrypt, 341 } 342 }; 343 344 static bool is_blacklisted_cpu(void) 345 { 346 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) 347 return false; 348 349 if (boot_cpu_data.x86 == 0x0f) { 350 /* 351 * On Pentium 4, des3_ede-x86_64 is slower than generic C 352 * implementation because use of 64bit rotates (which are really 353 * slow on P4). Therefore blacklist P4s. 354 */ 355 return true; 356 } 357 358 return false; 359 } 360 361 static int force; 362 module_param(force, int, 0); 363 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist"); 364 365 static int __init des3_ede_x86_init(void) 366 { 367 int err; 368 369 if (!force && is_blacklisted_cpu()) { 370 pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n"); 371 return -ENODEV; 372 } 373 374 err = crypto_register_alg(&des3_ede_cipher); 375 if (err) 376 return err; 377 378 err = crypto_register_skciphers(des3_ede_skciphers, 379 ARRAY_SIZE(des3_ede_skciphers)); 380 if (err) 381 crypto_unregister_alg(&des3_ede_cipher); 382 383 return err; 384 } 385 386 static void __exit des3_ede_x86_fini(void) 387 { 388 crypto_unregister_alg(&des3_ede_cipher); 389 crypto_unregister_skciphers(des3_ede_skciphers, 390 ARRAY_SIZE(des3_ede_skciphers)); 391 } 392 393 module_init(des3_ede_x86_init); 394 module_exit(des3_ede_x86_fini); 395 396 MODULE_LICENSE("GPL"); 397 MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized"); 398 MODULE_ALIAS_CRYPTO("des3_ede"); 399 MODULE_ALIAS_CRYPTO("des3_ede-asm"); 400 MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>"); 401