1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for AVX assembler version of Twofish Cipher 4 * 5 * Copyright (C) 2012 Johannes Goetzfried 6 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 7 * 8 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 9 */ 10 11 #include <linux/module.h> 12 #include <linux/types.h> 13 #include <linux/crypto.h> 14 #include <linux/err.h> 15 #include <crypto/algapi.h> 16 #include <crypto/internal/simd.h> 17 #include <crypto/twofish.h> 18 #include <crypto/xts.h> 19 #include <asm/crypto/glue_helper.h> 20 #include <asm/crypto/twofish.h> 21 22 #define TWOFISH_PARALLEL_BLOCKS 8 23 24 /* 8-way parallel cipher functions */ 25 asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); 26 asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); 27 28 asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); 29 asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src, 30 le128 *iv); 31 32 asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, 33 le128 *iv); 34 asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, 35 le128 *iv); 36 37 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, 38 const u8 *key, unsigned int keylen) 39 { 40 return twofish_setkey(&tfm->base, key, keylen); 41 } 42 43 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) 44 { 45 __twofish_enc_blk_3way(ctx, dst, src, false); 46 } 47 48 static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) 49 { 50 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk); 51 } 52 53 static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) 54 { 55 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk); 56 } 57 58 struct twofish_xts_ctx { 59 struct twofish_ctx tweak_ctx; 60 struct twofish_ctx crypt_ctx; 61 }; 62 63 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, 64 unsigned int keylen) 65 { 66 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 67 u32 *flags = &tfm->base.crt_flags; 68 int err; 69 70 err = xts_verify_key(tfm, key, keylen); 71 if (err) 72 return err; 73 74 /* first half of xts-key is for crypt */ 75 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); 76 if (err) 77 return err; 78 79 /* second half of xts-key is for tweak */ 80 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, 81 flags); 82 } 83 84 static const struct common_glue_ctx twofish_enc = { 85 .num_funcs = 3, 86 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 87 88 .funcs = { { 89 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 90 .fn_u = { .ecb = twofish_ecb_enc_8way } 91 }, { 92 .num_blocks = 3, 93 .fn_u = { .ecb = twofish_enc_blk_3way } 94 }, { 95 .num_blocks = 1, 96 .fn_u = { .ecb = twofish_enc_blk } 97 } } 98 }; 99 100 static const struct common_glue_ctx twofish_ctr = { 101 .num_funcs = 3, 102 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 103 104 .funcs = { { 105 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 106 .fn_u = { .ctr = twofish_ctr_8way } 107 }, { 108 .num_blocks = 3, 109 .fn_u = { .ctr = twofish_enc_blk_ctr_3way } 110 }, { 111 .num_blocks = 1, 112 .fn_u = { .ctr = twofish_enc_blk_ctr } 113 } } 114 }; 115 116 static const struct common_glue_ctx twofish_enc_xts = { 117 .num_funcs = 2, 118 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 119 120 .funcs = { { 121 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 122 .fn_u = { .xts = twofish_xts_enc_8way } 123 }, { 124 .num_blocks = 1, 125 .fn_u = { .xts = twofish_xts_enc } 126 } } 127 }; 128 129 static const struct common_glue_ctx twofish_dec = { 130 .num_funcs = 3, 131 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 132 133 .funcs = { { 134 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 135 .fn_u = { .ecb = twofish_ecb_dec_8way } 136 }, { 137 .num_blocks = 3, 138 .fn_u = { .ecb = twofish_dec_blk_3way } 139 }, { 140 .num_blocks = 1, 141 .fn_u = { .ecb = twofish_dec_blk } 142 } } 143 }; 144 145 static const struct common_glue_ctx twofish_dec_cbc = { 146 .num_funcs = 3, 147 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 148 149 .funcs = { { 150 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 151 .fn_u = { .cbc = twofish_cbc_dec_8way } 152 }, { 153 .num_blocks = 3, 154 .fn_u = { .cbc = twofish_dec_blk_cbc_3way } 155 }, { 156 .num_blocks = 1, 157 .fn_u = { .cbc = twofish_dec_blk } 158 } } 159 }; 160 161 static const struct common_glue_ctx twofish_dec_xts = { 162 .num_funcs = 2, 163 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 164 165 .funcs = { { 166 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 167 .fn_u = { .xts = twofish_xts_dec_8way } 168 }, { 169 .num_blocks = 1, 170 .fn_u = { .xts = twofish_xts_dec } 171 } } 172 }; 173 174 static int ecb_encrypt(struct skcipher_request *req) 175 { 176 return glue_ecb_req_128bit(&twofish_enc, req); 177 } 178 179 static int ecb_decrypt(struct skcipher_request *req) 180 { 181 return glue_ecb_req_128bit(&twofish_dec, req); 182 } 183 184 static int cbc_encrypt(struct skcipher_request *req) 185 { 186 return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req); 187 } 188 189 static int cbc_decrypt(struct skcipher_request *req) 190 { 191 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); 192 } 193 194 static int ctr_crypt(struct skcipher_request *req) 195 { 196 return glue_ctr_req_128bit(&twofish_ctr, req); 197 } 198 199 static int xts_encrypt(struct skcipher_request *req) 200 { 201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 202 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 203 204 return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk, 205 &ctx->tweak_ctx, &ctx->crypt_ctx, false); 206 } 207 208 static int xts_decrypt(struct skcipher_request *req) 209 { 210 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 211 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 212 213 return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk, 214 &ctx->tweak_ctx, &ctx->crypt_ctx, true); 215 } 216 217 static struct skcipher_alg twofish_algs[] = { 218 { 219 .base.cra_name = "__ecb(twofish)", 220 .base.cra_driver_name = "__ecb-twofish-avx", 221 .base.cra_priority = 400, 222 .base.cra_flags = CRYPTO_ALG_INTERNAL, 223 .base.cra_blocksize = TF_BLOCK_SIZE, 224 .base.cra_ctxsize = sizeof(struct twofish_ctx), 225 .base.cra_module = THIS_MODULE, 226 .min_keysize = TF_MIN_KEY_SIZE, 227 .max_keysize = TF_MAX_KEY_SIZE, 228 .setkey = twofish_setkey_skcipher, 229 .encrypt = ecb_encrypt, 230 .decrypt = ecb_decrypt, 231 }, { 232 .base.cra_name = "__cbc(twofish)", 233 .base.cra_driver_name = "__cbc-twofish-avx", 234 .base.cra_priority = 400, 235 .base.cra_flags = CRYPTO_ALG_INTERNAL, 236 .base.cra_blocksize = TF_BLOCK_SIZE, 237 .base.cra_ctxsize = sizeof(struct twofish_ctx), 238 .base.cra_module = THIS_MODULE, 239 .min_keysize = TF_MIN_KEY_SIZE, 240 .max_keysize = TF_MAX_KEY_SIZE, 241 .ivsize = TF_BLOCK_SIZE, 242 .setkey = twofish_setkey_skcipher, 243 .encrypt = cbc_encrypt, 244 .decrypt = cbc_decrypt, 245 }, { 246 .base.cra_name = "__ctr(twofish)", 247 .base.cra_driver_name = "__ctr-twofish-avx", 248 .base.cra_priority = 400, 249 .base.cra_flags = CRYPTO_ALG_INTERNAL, 250 .base.cra_blocksize = 1, 251 .base.cra_ctxsize = sizeof(struct twofish_ctx), 252 .base.cra_module = THIS_MODULE, 253 .min_keysize = TF_MIN_KEY_SIZE, 254 .max_keysize = TF_MAX_KEY_SIZE, 255 .ivsize = TF_BLOCK_SIZE, 256 .chunksize = TF_BLOCK_SIZE, 257 .setkey = twofish_setkey_skcipher, 258 .encrypt = ctr_crypt, 259 .decrypt = ctr_crypt, 260 }, { 261 .base.cra_name = "__xts(twofish)", 262 .base.cra_driver_name = "__xts-twofish-avx", 263 .base.cra_priority = 400, 264 .base.cra_flags = CRYPTO_ALG_INTERNAL, 265 .base.cra_blocksize = TF_BLOCK_SIZE, 266 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), 267 .base.cra_module = THIS_MODULE, 268 .min_keysize = 2 * TF_MIN_KEY_SIZE, 269 .max_keysize = 2 * TF_MAX_KEY_SIZE, 270 .ivsize = TF_BLOCK_SIZE, 271 .setkey = xts_twofish_setkey, 272 .encrypt = xts_encrypt, 273 .decrypt = xts_decrypt, 274 }, 275 }; 276 277 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; 278 279 static int __init twofish_init(void) 280 { 281 const char *feature_name; 282 283 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) { 284 pr_info("CPU feature '%s' is not supported.\n", feature_name); 285 return -ENODEV; 286 } 287 288 return simd_register_skciphers_compat(twofish_algs, 289 ARRAY_SIZE(twofish_algs), 290 twofish_simd_algs); 291 } 292 293 static void __exit twofish_exit(void) 294 { 295 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), 296 twofish_simd_algs); 297 } 298 299 module_init(twofish_init); 300 module_exit(twofish_exit); 301 302 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); 303 MODULE_LICENSE("GPL"); 304 MODULE_ALIAS_CRYPTO("twofish"); 305