1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia 4 * 5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 6 */ 7 8 #include <asm/crypto/camellia.h> 9 #include <asm/crypto/glue_helper.h> 10 #include <crypto/algapi.h> 11 #include <crypto/internal/simd.h> 12 #include <linux/crypto.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 #include <linux/types.h> 16 17 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 18 19 /* 16-way parallel cipher functions (avx/aes-ni) */ 20 asmlinkage void camellia_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src); 21 EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way); 22 23 asmlinkage void camellia_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src); 24 EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way); 25 26 asmlinkage void camellia_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); 27 EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way); 28 29 asmlinkage void camellia_ctr_16way(const void *ctx, u8 *dst, const u8 *src, 30 le128 *iv); 31 EXPORT_SYMBOL_GPL(camellia_ctr_16way); 32 33 static const struct common_glue_ctx camellia_enc = { 34 .num_funcs = 3, 35 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 36 37 .funcs = { { 38 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 39 .fn_u = { .ecb = camellia_ecb_enc_16way } 40 }, { 41 .num_blocks = 2, 42 .fn_u = { .ecb = camellia_enc_blk_2way } 43 }, { 44 .num_blocks = 1, 45 .fn_u = { .ecb = camellia_enc_blk } 46 } } 47 }; 48 49 static const struct common_glue_ctx camellia_ctr = { 50 .num_funcs = 3, 51 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 52 53 .funcs = { { 54 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 55 .fn_u = { .ctr = camellia_ctr_16way } 56 }, { 57 .num_blocks = 2, 58 .fn_u = { .ctr = camellia_crypt_ctr_2way } 59 }, { 60 .num_blocks = 1, 61 .fn_u = { .ctr = camellia_crypt_ctr } 62 } } 63 }; 64 65 static const struct common_glue_ctx camellia_dec = { 66 .num_funcs = 3, 67 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 68 69 .funcs = { { 70 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 71 .fn_u = { .ecb = camellia_ecb_dec_16way } 72 }, { 73 .num_blocks = 2, 74 .fn_u = { .ecb = camellia_dec_blk_2way } 75 }, { 76 .num_blocks = 1, 77 .fn_u = { .ecb = camellia_dec_blk } 78 } } 79 }; 80 81 static const struct common_glue_ctx camellia_dec_cbc = { 82 .num_funcs = 3, 83 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, 84 85 .funcs = { { 86 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, 87 .fn_u = { .cbc = camellia_cbc_dec_16way } 88 }, { 89 .num_blocks = 2, 90 .fn_u = { .cbc = camellia_decrypt_cbc_2way } 91 }, { 92 .num_blocks = 1, 93 .fn_u = { .cbc = camellia_dec_blk } 94 } } 95 }; 96 97 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, 98 unsigned int keylen) 99 { 100 return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen); 101 } 102 103 static int ecb_encrypt(struct skcipher_request *req) 104 { 105 return glue_ecb_req_128bit(&camellia_enc, req); 106 } 107 108 static int ecb_decrypt(struct skcipher_request *req) 109 { 110 return glue_ecb_req_128bit(&camellia_dec, req); 111 } 112 113 static int cbc_encrypt(struct skcipher_request *req) 114 { 115 return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req); 116 } 117 118 static int cbc_decrypt(struct skcipher_request *req) 119 { 120 return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); 121 } 122 123 static int ctr_crypt(struct skcipher_request *req) 124 { 125 return glue_ctr_req_128bit(&camellia_ctr, req); 126 } 127 128 static struct skcipher_alg camellia_algs[] = { 129 { 130 .base.cra_name = "__ecb(camellia)", 131 .base.cra_driver_name = "__ecb-camellia-aesni", 132 .base.cra_priority = 400, 133 .base.cra_flags = CRYPTO_ALG_INTERNAL, 134 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 135 .base.cra_ctxsize = sizeof(struct camellia_ctx), 136 .base.cra_module = THIS_MODULE, 137 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 138 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 139 .setkey = camellia_setkey, 140 .encrypt = ecb_encrypt, 141 .decrypt = ecb_decrypt, 142 }, { 143 .base.cra_name = "__cbc(camellia)", 144 .base.cra_driver_name = "__cbc-camellia-aesni", 145 .base.cra_priority = 400, 146 .base.cra_flags = CRYPTO_ALG_INTERNAL, 147 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, 148 .base.cra_ctxsize = sizeof(struct camellia_ctx), 149 .base.cra_module = THIS_MODULE, 150 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 151 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 152 .ivsize = CAMELLIA_BLOCK_SIZE, 153 .setkey = camellia_setkey, 154 .encrypt = cbc_encrypt, 155 .decrypt = cbc_decrypt, 156 }, { 157 .base.cra_name = "__ctr(camellia)", 158 .base.cra_driver_name = "__ctr-camellia-aesni", 159 .base.cra_priority = 400, 160 .base.cra_flags = CRYPTO_ALG_INTERNAL, 161 .base.cra_blocksize = 1, 162 .base.cra_ctxsize = sizeof(struct camellia_ctx), 163 .base.cra_module = THIS_MODULE, 164 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 165 .max_keysize = CAMELLIA_MAX_KEY_SIZE, 166 .ivsize = CAMELLIA_BLOCK_SIZE, 167 .chunksize = CAMELLIA_BLOCK_SIZE, 168 .setkey = camellia_setkey, 169 .encrypt = ctr_crypt, 170 .decrypt = ctr_crypt, 171 } 172 }; 173 174 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; 175 176 static int __init camellia_aesni_init(void) 177 { 178 const char *feature_name; 179 180 if (!boot_cpu_has(X86_FEATURE_AVX) || 181 !boot_cpu_has(X86_FEATURE_AES) || 182 !boot_cpu_has(X86_FEATURE_OSXSAVE)) { 183 pr_info("AVX or AES-NI instructions are not detected.\n"); 184 return -ENODEV; 185 } 186 187 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, 188 &feature_name)) { 189 pr_info("CPU feature '%s' is not supported.\n", feature_name); 190 return -ENODEV; 191 } 192 193 return simd_register_skciphers_compat(camellia_algs, 194 ARRAY_SIZE(camellia_algs), 195 camellia_simd_algs); 196 } 197 198 static void __exit camellia_aesni_fini(void) 199 { 200 simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs), 201 camellia_simd_algs); 202 } 203 204 module_init(camellia_aesni_init); 205 module_exit(camellia_aesni_fini); 206 207 MODULE_LICENSE("GPL"); 208 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized"); 209 MODULE_ALIAS_CRYPTO("camellia"); 210 MODULE_ALIAS_CRYPTO("camellia-asm"); 211