1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia 4 * 5 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 */ 7 8 #include <asm/crypto/camellia.h> 9 #include <asm/crypto/glue_helper.h> 10 #include <crypto/algapi.h> 11 #include <crypto/internal/simd.h> 12 #include <crypto/xts.h> 13 #include <linux/crypto.h> 14 #include <linux/err.h> 15 #include <linux/module.h> 16 #include <linux/types.h> 17 18 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 19 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32 20 21 /* 32-way AVX2/AES-NI parallel cipher functions */ 22 asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src); 23 asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src); 24 25 asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src); 26 asmlinkage void camellia_ctr_32way(const void *ctx, u8 *dst, const u8 *src, 27 le128 *iv); 28 29 asmlinkage void camellia_xts_enc_32way(const void *ctx, u8 *dst, const u8 *src, 30 le128 *iv); 31 asmlinkage void camellia_xts_dec_32way(const void *ctx, u8 *dst, const u8 *src, 32 le128 *iv); 33 34 static const struct common_glue_ctx camellia_enc = { 35 .num_funcs = 4, 36 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 37 38 .funcs = { { 39 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 40 .fn_u = { .ecb = camellia_ecb_enc_32way } 41 }, { 42 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 43 .fn_u = { .ecb = camellia_ecb_enc_16way } 44 }, { 45 .num_blocks = 2, 46 .fn_u = { .ecb = camellia_enc_blk_2way } 47 }, { 48 .num_blocks = 1, 49 .fn_u = { .ecb = camellia_enc_blk } 50 } } 51 }; 52 53 static const struct common_glue_ctx camellia_ctr = { 54 .num_funcs = 4, 55 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 56 57 .funcs = { { 58 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 59 .fn_u = { .ctr = camellia_ctr_32way } 60 }, { 61 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 62 .fn_u = { .ctr = camellia_ctr_16way } 63 }, { 64 .num_blocks = 2, 65 .fn_u = { .ctr = camellia_crypt_ctr_2way } 66 }, { 67 .num_blocks = 1, 68 .fn_u = { .ctr = camellia_crypt_ctr } 69 } } 70 }; 71 72 static const struct common_glue_ctx camellia_enc_xts = { 73 .num_funcs = 3, 74 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 75 76 .funcs = { { 77 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 78 .fn_u = { .xts = camellia_xts_enc_32way } 79 }, { 80 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 81 .fn_u = { .xts = camellia_xts_enc_16way } 82 }, { 83 .num_blocks = 1, 84 .fn_u = { .xts = camellia_xts_enc } 85 } } 86 }; 87 88 static const struct common_glue_ctx camellia_dec = { 89 .num_funcs = 4, 90 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 91 92 .funcs = { { 93 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 94 .fn_u = { .ecb = camellia_ecb_dec_32way } 95 }, { 96 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 97 .fn_u = { .ecb = camellia_ecb_dec_16way } 98 }, { 99 .num_blocks = 2, 100 .fn_u = { .ecb = camellia_dec_blk_2way } 101 }, { 102 .num_blocks = 1, 103 .fn_u = { .ecb = camellia_dec_blk } 104 } } 105 }; 106 107 static const struct common_glue_ctx camellia_dec_cbc = { 108 .num_funcs = 4, 109 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 110 111 .funcs = { { 112 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 113 .fn_u = { .cbc = camellia_cbc_dec_32way } 114 }, { 115 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 116 .fn_u = { .cbc = camellia_cbc_dec_16way } 117 }, { 118 .num_blocks = 2, 119 .fn_u = { .cbc = camellia_decrypt_cbc_2way } 120 }, { 121 .num_blocks = 1, 122 .fn_u = { .cbc = camellia_dec_blk } 123 } } 124 }; 125 126 static const struct common_glue_ctx camellia_dec_xts = { 127 .num_funcs = 3, 128 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 129 130 .funcs = { { 131 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, 132 .fn_u = { .xts = camellia_xts_dec_32way } 133 }, { 134 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 135 .fn_u = { .xts = camellia_xts_dec_16way } 136 }, { 137 .num_blocks = 1, 138 .fn_u = { .xts = camellia_xts_dec } 139 } } 140 }; 141 142 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, 143 unsigned int keylen) 144 { 145 return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen, 146 &tfm->base.crt_flags); 147 } 148 149 static int ecb_encrypt(struct skcipher_request *req) 150 { 151 return glue_ecb_req_128bit(&camellia_enc, req); 152 } 153 154 static int ecb_decrypt(struct skcipher_request *req) 155 { 156 return glue_ecb_req_128bit(&camellia_dec, req); 157 } 158 159 static int cbc_encrypt(struct skcipher_request *req) 160 { 161 return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req); 162 } 163 164 static int cbc_decrypt(struct skcipher_request *req) 165 { 166 return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); 167 } 168 169 static int ctr_crypt(struct skcipher_request *req) 170 { 171 return glue_ctr_req_128bit(&camellia_ctr, req); 172 } 173 174 static int xts_encrypt(struct skcipher_request *req) 175 { 176 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 177 struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 178 179 return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk, 180 &ctx->tweak_ctx, &ctx->crypt_ctx, false); 181 } 182 183 static int xts_decrypt(struct skcipher_request *req) 184 { 185 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 186 struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 187 188 return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk, 189 &ctx->tweak_ctx, &ctx->crypt_ctx, true); 190 } 191 192 static struct skcipher_alg camellia_algs[] = { 193 { 194 .base.cra_name = "__ecb(camellia)", 195 .base.cra_driver_name = "__ecb-camellia-aesni-avx2", 196 .base.cra_priority = 500, 197 .base.cra_flags = CRYPTO_ALG_INTERNAL, 198 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 199 .base.cra_ctxsize = sizeof(struct camellia_ctx), 200 .base.cra_module = THIS_MODULE, 201 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 202 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 203 .setkey = camellia_setkey, 204 .encrypt = ecb_encrypt, 205 .decrypt = ecb_decrypt, 206 }, { 207 .base.cra_name = "__cbc(camellia)", 208 .base.cra_driver_name = "__cbc-camellia-aesni-avx2", 209 .base.cra_priority = 500, 210 .base.cra_flags = CRYPTO_ALG_INTERNAL, 211 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 212 .base.cra_ctxsize = sizeof(struct camellia_ctx), 213 .base.cra_module = THIS_MODULE, 214 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 215 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 216 .ivsize = CAMELLIA_BLOCK_SIZE, 217 .setkey = camellia_setkey, 218 .encrypt = cbc_encrypt, 219 .decrypt = cbc_decrypt, 220 }, { 221 .base.cra_name = "__ctr(camellia)", 222 .base.cra_driver_name = "__ctr-camellia-aesni-avx2", 223 .base.cra_priority = 500, 224 .base.cra_flags = CRYPTO_ALG_INTERNAL, 225 .base.cra_blocksize = 1, 226 .base.cra_ctxsize = sizeof(struct camellia_ctx), 227 .base.cra_module = THIS_MODULE, 228 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 229 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 230 .ivsize = CAMELLIA_BLOCK_SIZE, 231 .chunksize = CAMELLIA_BLOCK_SIZE, 232 .setkey = camellia_setkey, 233 .encrypt = ctr_crypt, 234 .decrypt = ctr_crypt, 235 }, { 236 .base.cra_name = "__xts(camellia)", 237 .base.cra_driver_name = "__xts-camellia-aesni-avx2", 238 .base.cra_priority = 500, 239 .base.cra_flags = CRYPTO_ALG_INTERNAL, 240 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 241 .base.cra_ctxsize = sizeof(struct camellia_xts_ctx), 242 .base.cra_module = THIS_MODULE, 243 .min_keysize = 2 * CAMELLIA_MIN_KEY_SIZE, 244 .max_keysize = 2 * CAMELLIA_MAX_KEY_SIZE, 245 .ivsize = CAMELLIA_BLOCK_SIZE, 246 .setkey = xts_camellia_setkey, 247 .encrypt = xts_encrypt, 248 .decrypt = xts_decrypt, 249 }, 250 }; 251 252 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; 253 254 static int __init camellia_aesni_init(void) 255 { 256 const char *feature_name; 257 258 if (!boot_cpu_has(X86_FEATURE_AVX) || 259 !boot_cpu_has(X86_FEATURE_AVX2) || 260 !boot_cpu_has(X86_FEATURE_AES) || 261 !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 262 pr_info("AVX2 or AES-NI instructions are not detected.\n"); 263 return -ENODEV; 264 } 265 266 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 267 &feature_name)) { 268 pr_info("CPU feature '%s' is not supported.\n", feature_name); 269 return -ENODEV; 270 } 271 272 return simd_register_skciphers_compat(camellia_algs, 273 ARRAY_SIZE(camellia_algs), 274 camellia_simd_algs); 275 } 276 277 static void __exit camellia_aesni_fini(void) 278 { 279 simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), 280 camellia_simd_algs); 281 } 282 283 module_init(camellia_aesni_init); 284 module_exit(camellia_aesni_fini); 285 286 MODULE_LICENSE("GPL"); 287 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized"); 288 MODULE_ALIAS_CRYPTO("camellia"); 289 MODULE_ALIAS_CRYPTO("camellia-asm"); 290