1 /* 2 * Glue Code for AVX assembler version of Twofish Cipher 3 * 4 * Copyright (C) 2012 Johannes Goetzfried 5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 6 * 7 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published by 11 * the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software 21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 22 * USA 23 * 24 */ 25 26 #include <linux/module.h> 27 #include <linux/types.h> 28 #include <linux/crypto.h> 29 #include <linux/err.h> 30 #include <crypto/algapi.h> 31 #include <crypto/internal/simd.h> 32 #include <crypto/twofish.h> 33 #include <crypto/xts.h> 34 #include <asm/crypto/glue_helper.h> 35 #include <asm/crypto/twofish.h> 36 37 #define TWOFISH_PARALLEL_BLOCKS 8 38 39 /* 8-way parallel cipher functions */ 40 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst, 41 const u8 *src); 42 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst, 43 const u8 *src); 44 45 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst, 46 const u8 *src); 47 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst, 48 const u8 *src, le128 *iv); 49 50 asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst, 51 const u8 *src, le128 *iv); 52 asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst, 53 const u8 *src, le128 *iv); 54 55 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, 56 const u8 *key, unsigned int keylen) 57 { 58 return twofish_setkey(&tfm->base, key, keylen); 59 } 60 61 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 62 const u8 *src) 63 { 64 __twofish_enc_blk_3way(ctx, dst, src, false); 65 } 66 67 static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 68 { 69 glue_xts_crypt_128bit_one(ctx, dst, src, iv, 70 GLUE_FUNC_CAST(twofish_enc_blk)); 71 } 72 73 static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 74 { 75 glue_xts_crypt_128bit_one(ctx, dst, src, iv, 76 GLUE_FUNC_CAST(twofish_dec_blk)); 77 } 78 79 struct twofish_xts_ctx { 80 struct twofish_ctx tweak_ctx; 81 struct twofish_ctx crypt_ctx; 82 }; 83 84 static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, 85 unsigned int keylen) 86 { 87 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 88 u32 *flags = &tfm->base.crt_flags; 89 int err; 90 91 err = xts_verify_key(tfm, key, keylen); 92 if (err) 93 return err; 94 95 /* first half of xts-key is for crypt */ 96 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); 97 if (err) 98 return err; 99 100 /* second half of xts-key is for tweak */ 101 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, 102 flags); 103 } 104 105 static const struct common_glue_ctx twofish_enc = { 106 .num_funcs = 3, 107 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 108 109 .funcs = { { 110 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 111 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) } 112 }, { 113 .num_blocks = 3, 114 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } 115 }, { 116 .num_blocks = 1, 117 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } 118 } } 119 }; 120 121 static const struct common_glue_ctx twofish_ctr = { 122 .num_funcs = 3, 123 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 124 125 .funcs = { { 126 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 127 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) } 128 }, { 129 .num_blocks = 3, 130 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) } 131 }, { 132 .num_blocks = 1, 133 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) } 134 } } 135 }; 136 137 static const struct common_glue_ctx twofish_enc_xts = { 138 .num_funcs = 2, 139 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 140 141 .funcs = { { 142 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 143 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) } 144 }, { 145 .num_blocks = 1, 146 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) } 147 } } 148 }; 149 150 static const struct common_glue_ctx twofish_dec = { 151 .num_funcs = 3, 152 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 153 154 .funcs = { { 155 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 156 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) } 157 }, { 158 .num_blocks = 3, 159 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } 160 }, { 161 .num_blocks = 1, 162 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } 163 } } 164 }; 165 166 static const struct common_glue_ctx twofish_dec_cbc = { 167 .num_funcs = 3, 168 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 169 170 .funcs = { { 171 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 172 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) } 173 }, { 174 .num_blocks = 3, 175 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } 176 }, { 177 .num_blocks = 1, 178 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } 179 } } 180 }; 181 182 static const struct common_glue_ctx twofish_dec_xts = { 183 .num_funcs = 2, 184 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 185 186 .funcs = { { 187 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 188 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) } 189 }, { 190 .num_blocks = 1, 191 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) } 192 } } 193 }; 194 195 static int ecb_encrypt(struct skcipher_request *req) 196 { 197 return glue_ecb_req_128bit(&twofish_enc, req); 198 } 199 200 static int ecb_decrypt(struct skcipher_request *req) 201 { 202 return glue_ecb_req_128bit(&twofish_dec, req); 203 } 204 205 static int cbc_encrypt(struct skcipher_request *req) 206 { 207 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk), 208 req); 209 } 210 211 static int cbc_decrypt(struct skcipher_request *req) 212 { 213 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); 214 } 215 216 static int ctr_crypt(struct skcipher_request *req) 217 { 218 return glue_ctr_req_128bit(&twofish_ctr, req); 219 } 220 221 static int xts_encrypt(struct skcipher_request *req) 222 { 223 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 224 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 225 226 return glue_xts_req_128bit(&twofish_enc_xts, req, 227 XTS_TWEAK_CAST(twofish_enc_blk), 228 &ctx->tweak_ctx, &ctx->crypt_ctx); 229 } 230 231 static int xts_decrypt(struct skcipher_request *req) 232 { 233 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 234 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 235 236 return glue_xts_req_128bit(&twofish_dec_xts, req, 237 XTS_TWEAK_CAST(twofish_enc_blk), 238 &ctx->tweak_ctx, &ctx->crypt_ctx); 239 } 240 241 static struct skcipher_alg twofish_algs[] = { 242 { 243 .base.cra_name = "__ecb(twofish)", 244 .base.cra_driver_name = "__ecb-twofish-avx", 245 .base.cra_priority = 400, 246 .base.cra_flags = CRYPTO_ALG_INTERNAL, 247 .base.cra_blocksize = TF_BLOCK_SIZE, 248 .base.cra_ctxsize = sizeof(struct twofish_ctx), 249 .base.cra_module = THIS_MODULE, 250 .min_keysize = TF_MIN_KEY_SIZE, 251 .max_keysize = TF_MAX_KEY_SIZE, 252 .setkey = twofish_setkey_skcipher, 253 .encrypt = ecb_encrypt, 254 .decrypt = ecb_decrypt, 255 }, { 256 .base.cra_name = "__cbc(twofish)", 257 .base.cra_driver_name = "__cbc-twofish-avx", 258 .base.cra_priority = 400, 259 .base.cra_flags = CRYPTO_ALG_INTERNAL, 260 .base.cra_blocksize = TF_BLOCK_SIZE, 261 .base.cra_ctxsize = sizeof(struct twofish_ctx), 262 .base.cra_module = THIS_MODULE, 263 .min_keysize = TF_MIN_KEY_SIZE, 264 .max_keysize = TF_MAX_KEY_SIZE, 265 .ivsize = TF_BLOCK_SIZE, 266 .setkey = twofish_setkey_skcipher, 267 .encrypt = cbc_encrypt, 268 .decrypt = cbc_decrypt, 269 }, { 270 .base.cra_name = "__ctr(twofish)", 271 .base.cra_driver_name = "__ctr-twofish-avx", 272 .base.cra_priority = 400, 273 .base.cra_flags = CRYPTO_ALG_INTERNAL, 274 .base.cra_blocksize = 1, 275 .base.cra_ctxsize = sizeof(struct twofish_ctx), 276 .base.cra_module = THIS_MODULE, 277 .min_keysize = TF_MIN_KEY_SIZE, 278 .max_keysize = TF_MAX_KEY_SIZE, 279 .ivsize = TF_BLOCK_SIZE, 280 .chunksize = TF_BLOCK_SIZE, 281 .setkey = twofish_setkey_skcipher, 282 .encrypt = ctr_crypt, 283 .decrypt = ctr_crypt, 284 }, { 285 .base.cra_name = "__xts(twofish)", 286 .base.cra_driver_name = "__xts-twofish-avx", 287 .base.cra_priority = 400, 288 .base.cra_flags = CRYPTO_ALG_INTERNAL, 289 .base.cra_blocksize = TF_BLOCK_SIZE, 290 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), 291 .base.cra_module = THIS_MODULE, 292 .min_keysize = 2 * TF_MIN_KEY_SIZE, 293 .max_keysize = 2 * TF_MAX_KEY_SIZE, 294 .ivsize = TF_BLOCK_SIZE, 295 .setkey = xts_twofish_setkey, 296 .encrypt = xts_encrypt, 297 .decrypt = xts_decrypt, 298 }, 299 }; 300 301 static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)]; 302 303 static int __init twofish_init(void) 304 { 305 const char *feature_name; 306 307 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) { 308 pr_info("CPU feature '%s' is not supported.\n", feature_name); 309 return -ENODEV; 310 } 311 312 return simd_register_skciphers_compat(twofish_algs, 313 ARRAY_SIZE(twofish_algs), 314 twofish_simd_algs); 315 } 316 317 static void __exit twofish_exit(void) 318 { 319 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs), 320 twofish_simd_algs); 321 } 322 323 module_init(twofish_init); 324 module_exit(twofish_exit); 325 326 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); 327 MODULE_LICENSE("GPL"); 328 MODULE_ALIAS_CRYPTO("twofish"); 329