1 /* 2 * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia 3 * 4 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 * 11 */ 12 13 #include <asm/crypto/camellia.h> 14 #include <asm/crypto/glue_helper.h> 15 #include <crypto/algapi.h> 16 #include <crypto/internal/simd.h> 17 #include <crypto/xts.h> 18 #include <linux/crypto.h> 19 #include <linux/err.h> 20 #include <linux/module.h> 21 #include <linux/types.h> 22 23 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 24 25 /* 16-way parallel cipher functions (avx/aes-ni) */ 26 asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst, 27 const u8 *src); 28 EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way); 29 30 asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst, 31 const u8 *src); 32 EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way); 33 34 asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst, 35 const u8 *src); 36 EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way); 37 38 asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst, 39 const u8 *src, le128 *iv); 40 EXPORT_SYMBOL_GPL(camellia_ctr_16way); 41 42 asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst, 43 const u8 *src, le128 *iv); 44 EXPORT_SYMBOL_GPL(camellia_xts_enc_16way); 45 46 asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst, 47 const u8 *src, le128 *iv); 48 EXPORT_SYMBOL_GPL(camellia_xts_dec_16way); 49 50 void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 51 { 52 glue_xts_crypt_128bit_one(ctx, dst, src, iv, 53 GLUE_FUNC_CAST(camellia_enc_blk)); 54 } 55 EXPORT_SYMBOL_GPL(camellia_xts_enc); 56 57 void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 58 { 59 glue_xts_crypt_128bit_one(ctx, dst, src, iv, 60 GLUE_FUNC_CAST(camellia_dec_blk)); 61 } 62 EXPORT_SYMBOL_GPL(camellia_xts_dec); 63 64 static const struct common_glue_ctx camellia_enc = { 65 .num_funcs = 3, 66 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 67 68 .funcs = { { 69 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 70 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) } 71 }, { 72 .num_blocks = 2, 73 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) } 74 }, { 75 .num_blocks = 1, 76 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) } 77 } } 78 }; 79 80 static const struct common_glue_ctx camellia_ctr = { 81 .num_funcs = 3, 82 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 83 84 .funcs = { { 85 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 86 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) } 87 }, { 88 .num_blocks = 2, 89 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) } 90 }, { 91 .num_blocks = 1, 92 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) } 93 } } 94 }; 95 96 static const struct common_glue_ctx camellia_enc_xts = { 97 .num_funcs = 2, 98 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 99 100 .funcs = { { 101 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 102 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) } 103 }, { 104 .num_blocks = 1, 105 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) } 106 } } 107 }; 108 109 static const struct common_glue_ctx camellia_dec = { 110 .num_funcs = 3, 111 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 112 113 .funcs = { { 114 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 115 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) } 116 }, { 117 .num_blocks = 2, 118 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) } 119 }, { 120 .num_blocks = 1, 121 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) } 122 } } 123 }; 124 125 static const struct common_glue_ctx camellia_dec_cbc = { 126 .num_funcs = 3, 127 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 128 129 .funcs = { { 130 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 131 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) } 132 }, { 133 .num_blocks = 2, 134 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) } 135 }, { 136 .num_blocks = 1, 137 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) } 138 } } 139 }; 140 141 static const struct common_glue_ctx camellia_dec_xts = { 142 .num_funcs = 2, 143 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 144 145 .funcs = { { 146 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 147 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) } 148 }, { 149 .num_blocks = 1, 150 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) } 151 } } 152 }; 153 154 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, 155 unsigned int keylen) 156 { 157 return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen, 158 &tfm->base.crt_flags); 159 } 160 161 static int ecb_encrypt(struct skcipher_request *req) 162 { 163 return glue_ecb_req_128bit(&camellia_enc, req); 164 } 165 166 static int ecb_decrypt(struct skcipher_request *req) 167 { 168 return glue_ecb_req_128bit(&camellia_dec, req); 169 } 170 171 static int cbc_encrypt(struct skcipher_request *req) 172 { 173 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(camellia_enc_blk), 174 req); 175 } 176 177 static int cbc_decrypt(struct skcipher_request *req) 178 { 179 return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); 180 } 181 182 static int ctr_crypt(struct skcipher_request *req) 183 { 184 return glue_ctr_req_128bit(&camellia_ctr, req); 185 } 186 187 int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, 188 unsigned int keylen) 189 { 190 struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 191 u32 *flags = &tfm->base.crt_flags; 192 int err; 193 194 err = xts_verify_key(tfm, key, keylen); 195 if (err) 196 return err; 197 198 /* first half of xts-key is for crypt */ 199 err = __camellia_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); 200 if (err) 201 return err; 202 203 /* second half of xts-key is for tweak */ 204 return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, 205 flags); 206 } 207 EXPORT_SYMBOL_GPL(xts_camellia_setkey); 208 209 static int xts_encrypt(struct skcipher_request *req) 210 { 211 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 212 struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 213 214 return glue_xts_req_128bit(&camellia_enc_xts, req, 215 XTS_TWEAK_CAST(camellia_enc_blk), 216 &ctx->tweak_ctx, &ctx->crypt_ctx); 217 } 218 219 static int xts_decrypt(struct skcipher_request *req) 220 { 221 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 222 struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 223 224 return glue_xts_req_128bit(&camellia_dec_xts, req, 225 XTS_TWEAK_CAST(camellia_enc_blk), 226 &ctx->tweak_ctx, &ctx->crypt_ctx); 227 } 228 229 static struct skcipher_alg camellia_algs[] = { 230 { 231 .base.cra_name = "__ecb(camellia)", 232 .base.cra_driver_name = "__ecb-camellia-aesni", 233 .base.cra_priority = 400, 234 .base.cra_flags = CRYPTO_ALG_INTERNAL, 235 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 236 .base.cra_ctxsize = sizeof(struct camellia_ctx), 237 .base.cra_module = THIS_MODULE, 238 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 239 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 240 .setkey = camellia_setkey, 241 .encrypt = ecb_encrypt, 242 .decrypt = ecb_decrypt, 243 }, { 244 .base.cra_name = "__cbc(camellia)", 245 .base.cra_driver_name = "__cbc-camellia-aesni", 246 .base.cra_priority = 400, 247 .base.cra_flags = CRYPTO_ALG_INTERNAL, 248 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 249 .base.cra_ctxsize = sizeof(struct camellia_ctx), 250 .base.cra_module = THIS_MODULE, 251 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 252 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 253 .ivsize = CAMELLIA_BLOCK_SIZE, 254 .setkey = camellia_setkey, 255 .encrypt = cbc_encrypt, 256 .decrypt = cbc_decrypt, 257 }, { 258 .base.cra_name = "__ctr(camellia)", 259 .base.cra_driver_name = "__ctr-camellia-aesni", 260 .base.cra_priority = 400, 261 .base.cra_flags = CRYPTO_ALG_INTERNAL, 262 .base.cra_blocksize = 1, 263 .base.cra_ctxsize = sizeof(struct camellia_ctx), 264 .base.cra_module = THIS_MODULE, 265 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 266 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 267 .ivsize = CAMELLIA_BLOCK_SIZE, 268 .chunksize = CAMELLIA_BLOCK_SIZE, 269 .setkey = camellia_setkey, 270 .encrypt = ctr_crypt, 271 .decrypt = ctr_crypt, 272 }, { 273 .base.cra_name = "__xts(camellia)", 274 .base.cra_driver_name = "__xts-camellia-aesni", 275 .base.cra_priority = 400, 276 .base.cra_flags = CRYPTO_ALG_INTERNAL, 277 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 278 .base.cra_ctxsize = sizeof(struct camellia_xts_ctx), 279 .base.cra_module = THIS_MODULE, 280 .min_keysize = 2 * CAMELLIA_MIN_KEY_SIZE, 281 .max_keysize = 2 * CAMELLIA_MAX_KEY_SIZE, 282 .ivsize = CAMELLIA_BLOCK_SIZE, 283 .setkey = xts_camellia_setkey, 284 .encrypt = xts_encrypt, 285 .decrypt = xts_decrypt, 286 }, 287 }; 288 289 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; 290 291 static int __init camellia_aesni_init(void) 292 { 293 const char *feature_name; 294 295 if (!boot_cpu_has(X86_FEATURE_AVX) || 296 !boot_cpu_has(X86_FEATURE_AES) || 297 !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 298 pr_info("AVX or AES-NI instructions are not detected.\n"); 299 return -ENODEV; 300 } 301 302 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 303 &feature_name)) { 304 pr_info("CPU feature '%s' is not supported.\n", feature_name); 305 return -ENODEV; 306 } 307 308 return simd_register_skciphers_compat(camellia_algs, 309 ARRAY_SIZE(camellia_algs), 310 camellia_simd_algs); 311 } 312 313 static void __exit camellia_aesni_fini(void) 314 { 315 simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), 316 camellia_simd_algs); 317 } 318 319 module_init(camellia_aesni_init); 320 module_exit(camellia_aesni_fini); 321 322 MODULE_LICENSE("GPL"); 323 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized"); 324 MODULE_ALIAS_CRYPTO("camellia"); 325 MODULE_ALIAS_CRYPTO("camellia-asm"); 326