1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for AVX assembler version of Twofish Cipher 4 * 5 * Copyright (C) 2012 Johannes Goetzfried 6 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 7 * 8 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/types.h> 13 #include <linux/crypto.h> 14 #include <linux/err.h> 15 #include <crypto/algapi.h> 16 #include <crypto/internal/simd.h> 17 #include <crypto/twofish.h> 18 #include <crypto/xts.h> 19 #include <asm/crypto/glue_helper.h> 20 #include <asm/crypto/twofish.h> 21 22 #define TWOFISH_PARALLEL_BLOCKS 8 23 24 /* 8-way parallel cipher functions */ 25 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst, 26 const u8 *src); 27 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst, 28 const u8 *src); 29 30 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst, 31 const u8 *src); 32 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst, 33 const u8 *src, le128 *iv); 34 35 asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst, 36 const u8 *src, le128 *iv); 37 asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst, 38 const u8 *src, le128 *iv); 39 40 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, 41 const u8 *key, unsigned int keylen) 42 { 43 return twofish_setkey(&tfm->base, key, keylen); 44 } 45 46 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 47 const u8 *src) 48 { 49 __twofish_enc_blk_3way(ctx, dst, src, false); 50 } 51 52 static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 53 { 54 glue_xts_crypt_128bit_one(ctx, dst, src, iv, 55 GLUE_FUNC_CAST(twofish_enc_blk)); 56 } 57 58 static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 59 { 60 glue_xts_crypt_128bit_one(ctx, dst, src, iv, 61 GLUE_FUNC_CAST(twofish_dec_blk)); 62 } 63 64 struct twofish_xts_ctx { 65 struct twofish_ctx tweak_ctx; 66 struct twofish_ctx crypt_ctx; 67 }; 68 69 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, 70 unsigned int keylen) 71 { 72 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 73 u32 *flags = &tfm->base.crt_flags; 74 int err; 75 76 err = xts_verify_key(tfm, key, keylen); 77 if (err) 78 return err; 79 80 /* first half of xts-key is for crypt */ 81 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); 82 if (err) 83 return err; 84 85 /* second half of xts-key is for tweak */ 86 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, 87 flags); 88 } 89 90 static const struct common_glue_ctx twofish_enc = { 91 .num_funcs = 3, 92 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 93 94 .funcs = { { 95 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 96 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) } 97 }, { 98 .num_blocks = 3, 99 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } 100 }, { 101 .num_blocks = 1, 102 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } 103 } } 104 }; 105 106 static const struct common_glue_ctx twofish_ctr = { 107 .num_funcs = 3, 108 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 109 110 .funcs = { { 111 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 112 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) } 113 }, { 114 .num_blocks = 3, 115 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) } 116 }, { 117 .num_blocks = 1, 118 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) } 119 } } 120 }; 121 122 static const struct common_glue_ctx twofish_enc_xts = { 123 .num_funcs = 2, 124 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 125 126 .funcs = { { 127 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 128 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) } 129 }, { 130 .num_blocks = 1, 131 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) } 132 } } 133 }; 134 135 static const struct common_glue_ctx twofish_dec = { 136 .num_funcs = 3, 137 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 138 139 .funcs = { { 140 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 141 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) } 142 }, { 143 .num_blocks = 3, 144 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } 145 }, { 146 .num_blocks = 1, 147 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } 148 } } 149 }; 150 151 static const struct common_glue_ctx twofish_dec_cbc = { 152 .num_funcs = 3, 153 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 154 155 .funcs = { { 156 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 157 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) } 158 }, { 159 .num_blocks = 3, 160 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } 161 }, { 162 .num_blocks = 1, 163 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } 164 } } 165 }; 166 167 static const struct common_glue_ctx twofish_dec_xts = { 168 .num_funcs = 2, 169 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 170 171 .funcs = { { 172 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 173 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) } 174 }, { 175 .num_blocks = 1, 176 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) } 177 } } 178 }; 179 180 static int ecb_encrypt(struct skcipher_request *req) 181 { 182 return glue_ecb_req_128bit(&twofish_enc, req); 183 } 184 185 static int ecb_decrypt(struct skcipher_request *req) 186 { 187 return glue_ecb_req_128bit(&twofish_dec, req); 188 } 189 190 static int cbc_encrypt(struct skcipher_request *req) 191 { 192 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk), 193 req); 194 } 195 196 static int cbc_decrypt(struct skcipher_request *req) 197 { 198 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); 199 } 200 201 static int ctr_crypt(struct skcipher_request *req) 202 { 203 return glue_ctr_req_128bit(&twofish_ctr, req); 204 } 205 206 static int xts_encrypt(struct skcipher_request *req) 207 { 208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 209 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 210 211 return glue_xts_req_128bit(&twofish_enc_xts, req, 212 XTS_TWEAK_CAST(twofish_enc_blk), 213 &ctx->tweak_ctx, &ctx->crypt_ctx, false); 214 } 215 216 static int xts_decrypt(struct skcipher_request *req) 217 { 218 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 219 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 220 221 return glue_xts_req_128bit(&twofish_dec_xts, req, 222 XTS_TWEAK_CAST(twofish_enc_blk), 223 &ctx->tweak_ctx, &ctx->crypt_ctx, true); 224 } 225 226 static struct skcipher_alg twofish_algs[] = { 227 { 228 .base.cra_name = "__ecb(twofish)", 229 .base.cra_driver_name = "__ecb-twofish-avx", 230 .base.cra_priority = 400, 231 .base.cra_flags = CRYPTO_ALG_INTERNAL, 232 .base.cra_blocksize = TF_BLOCK_SIZE, 233 .base.cra_ctxsize = sizeof(struct twofish_ctx), 234 .base.cra_module = THIS_MODULE, 235 .min_keysize = TF_MIN_KEY_SIZE, 236 .max_keysize = TF_MAX_KEY_SIZE, 237 .setkey = twofish_setkey_skcipher, 238 .encrypt = ecb_encrypt, 239 .decrypt = ecb_decrypt, 240 }, { 241 .base.cra_name = "__cbc(twofish)", 242 .base.cra_driver_name = "__cbc-twofish-avx", 243 .base.cra_priority = 400, 244 .base.cra_flags = CRYPTO_ALG_INTERNAL, 245 .base.cra_blocksize = TF_BLOCK_SIZE, 246 .base.cra_ctxsize = sizeof(struct twofish_ctx), 247 .base.cra_module = THIS_MODULE, 248 .min_keysize = TF_MIN_KEY_SIZE, 249 .max_keysize = TF_MAX_KEY_SIZE, 250 .ivsize = TF_BLOCK_SIZE, 251 .setkey = twofish_setkey_skcipher, 252 .encrypt = cbc_encrypt, 253 .decrypt = cbc_decrypt, 254 }, { 255 .base.cra_name = "__ctr(twofish)", 256 .base.cra_driver_name = "__ctr-twofish-avx", 257 .base.cra_priority = 400, 258 .base.cra_flags = CRYPTO_ALG_INTERNAL, 259 .base.cra_blocksize = 1, 260 .base.cra_ctxsize = sizeof(struct twofish_ctx), 261 .base.cra_module = THIS_MODULE, 262 .min_keysize = TF_MIN_KEY_SIZE, 263 .max_keysize = TF_MAX_KEY_SIZE, 264 .ivsize = TF_BLOCK_SIZE, 265 .chunksize = TF_BLOCK_SIZE, 266 .setkey = twofish_setkey_skcipher, 267 .encrypt = ctr_crypt, 268 .decrypt = ctr_crypt, 269 }, { 270 .base.cra_name = "__xts(twofish)", 271 .base.cra_driver_name = "__xts-twofish-avx", 272 .base.cra_priority = 400, 273 .base.cra_flags = CRYPTO_ALG_INTERNAL, 274 .base.cra_blocksize = TF_BLOCK_SIZE, 275 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), 276 .base.cra_module = THIS_MODULE, 277 .min_keysize = 2 * TF_MIN_KEY_SIZE, 278 .max_keysize = 2 * TF_MAX_KEY_SIZE, 279 .ivsize = TF_BLOCK_SIZE, 280 .setkey = xts_twofish_setkey, 281 .encrypt = xts_encrypt, 282 .decrypt = xts_decrypt, 283 }, 284 }; 285 286 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; 287 288 static int __init twofish_init(void) 289 { 290 const char *feature_name; 291 292 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) { 293 pr_info("CPU feature '%s' is not supported.\n", feature_name); 294 return -ENODEV; 295 } 296 297 return simd_register_skciphers_compat(twofish_algs, 298 ARRAY_SIZE(twofish_algs), 299 twofish_simd_algs); 300 } 301 302 static void __exit twofish_exit(void) 303 { 304 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), 305 twofish_simd_algs); 306 } 307 308 module_init(twofish_init); 309 module_exit(twofish_exit); 310 311 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); 312 MODULE_LICENSE("GPL"); 313 MODULE_ALIAS_CRYPTO("twofish"); 314