1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
4  *
5  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
6  */
7 
8 #include <linux/module.h>
9 #include <linux/types.h>
10 #include <linux/crypto.h>
11 #include <linux/err.h>
12 #include <crypto/algapi.h>
13 #include <crypto/internal/simd.h>
14 #include <crypto/serpent.h>
15 #include <crypto/xts.h>
16 #include <asm/crypto/glue_helper.h>
17 #include <asm/crypto/serpent-avx.h>
18 
19 #define SERPENT_AVX2_PARALLEL_BLOCKS 16
20 
21 /* 16-way AVX2 parallel cipher functions */
22 asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
23 				      const u8 *src);
24 asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
25 				      const u8 *src);
26 asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);
27 
28 asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
29 				  le128 *iv);
30 asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
31 				      const u8 *src, le128 *iv);
32 asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
33 				      const u8 *src, le128 *iv);
34 
35 static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
36 				   const u8 *key, unsigned int keylen)
37 {
38 	return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
39 }
40 
41 static const struct common_glue_ctx serpent_enc = {
42 	.num_funcs = 3,
43 	.fpu_blocks_limit = 8,
44 
45 	.funcs = { {
46 		.num_blocks = 16,
47 		.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
48 	}, {
49 		.num_blocks = 8,
50 		.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
51 	}, {
52 		.num_blocks = 1,
53 		.fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
54 	} }
55 };
56 
57 static const struct common_glue_ctx serpent_ctr = {
58 	.num_funcs = 3,
59 	.fpu_blocks_limit = 8,
60 
61 	.funcs = { {
62 		.num_blocks = 16,
63 		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
64 	},  {
65 		.num_blocks = 8,
66 		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
67 	}, {
68 		.num_blocks = 1,
69 		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
70 	} }
71 };
72 
73 static const struct common_glue_ctx serpent_enc_xts = {
74 	.num_funcs = 3,
75 	.fpu_blocks_limit = 8,
76 
77 	.funcs = { {
78 		.num_blocks = 16,
79 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
80 	}, {
81 		.num_blocks = 8,
82 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
83 	}, {
84 		.num_blocks = 1,
85 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
86 	} }
87 };
88 
89 static const struct common_glue_ctx serpent_dec = {
90 	.num_funcs = 3,
91 	.fpu_blocks_limit = 8,
92 
93 	.funcs = { {
94 		.num_blocks = 16,
95 		.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
96 	}, {
97 		.num_blocks = 8,
98 		.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
99 	}, {
100 		.num_blocks = 1,
101 		.fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
102 	} }
103 };
104 
105 static const struct common_glue_ctx serpent_dec_cbc = {
106 	.num_funcs = 3,
107 	.fpu_blocks_limit = 8,
108 
109 	.funcs = { {
110 		.num_blocks = 16,
111 		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
112 	}, {
113 		.num_blocks = 8,
114 		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
115 	}, {
116 		.num_blocks = 1,
117 		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
118 	} }
119 };
120 
121 static const struct common_glue_ctx serpent_dec_xts = {
122 	.num_funcs = 3,
123 	.fpu_blocks_limit = 8,
124 
125 	.funcs = { {
126 		.num_blocks = 16,
127 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
128 	}, {
129 		.num_blocks = 8,
130 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
131 	}, {
132 		.num_blocks = 1,
133 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
134 	} }
135 };
136 
137 static int ecb_encrypt(struct skcipher_request *req)
138 {
139 	return glue_ecb_req_128bit(&serpent_enc, req);
140 }
141 
142 static int ecb_decrypt(struct skcipher_request *req)
143 {
144 	return glue_ecb_req_128bit(&serpent_dec, req);
145 }
146 
147 static int cbc_encrypt(struct skcipher_request *req)
148 {
149 	return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt),
150 					   req);
151 }
152 
153 static int cbc_decrypt(struct skcipher_request *req)
154 {
155 	return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
156 }
157 
158 static int ctr_crypt(struct skcipher_request *req)
159 {
160 	return glue_ctr_req_128bit(&serpent_ctr, req);
161 }
162 
163 static int xts_encrypt(struct skcipher_request *req)
164 {
165 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
166 	struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
167 
168 	return glue_xts_req_128bit(&serpent_enc_xts, req,
169 				   XTS_TWEAK_CAST(__serpent_encrypt),
170 				   &ctx->tweak_ctx, &ctx->crypt_ctx, false);
171 }
172 
173 static int xts_decrypt(struct skcipher_request *req)
174 {
175 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
176 	struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
177 
178 	return glue_xts_req_128bit(&serpent_dec_xts, req,
179 				   XTS_TWEAK_CAST(__serpent_encrypt),
180 				   &ctx->tweak_ctx, &ctx->crypt_ctx, true);
181 }
182 
183 static struct skcipher_alg serpent_algs[] = {
184 	{
185 		.base.cra_name		= "__ecb(serpent)",
186 		.base.cra_driver_name	= "__ecb-serpent-avx2",
187 		.base.cra_priority	= 600,
188 		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
189 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
190 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
191 		.base.cra_module	= THIS_MODULE,
192 		.min_keysize		= SERPENT_MIN_KEY_SIZE,
193 		.max_keysize		= SERPENT_MAX_KEY_SIZE,
194 		.setkey			= serpent_setkey_skcipher,
195 		.encrypt		= ecb_encrypt,
196 		.decrypt		= ecb_decrypt,
197 	}, {
198 		.base.cra_name		= "__cbc(serpent)",
199 		.base.cra_driver_name	= "__cbc-serpent-avx2",
200 		.base.cra_priority	= 600,
201 		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
202 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
203 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
204 		.base.cra_module	= THIS_MODULE,
205 		.min_keysize		= SERPENT_MIN_KEY_SIZE,
206 		.max_keysize		= SERPENT_MAX_KEY_SIZE,
207 		.ivsize			= SERPENT_BLOCK_SIZE,
208 		.setkey			= serpent_setkey_skcipher,
209 		.encrypt		= cbc_encrypt,
210 		.decrypt		= cbc_decrypt,
211 	}, {
212 		.base.cra_name		= "__ctr(serpent)",
213 		.base.cra_driver_name	= "__ctr-serpent-avx2",
214 		.base.cra_priority	= 600,
215 		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
216 		.base.cra_blocksize	= 1,
217 		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
218 		.base.cra_module	= THIS_MODULE,
219 		.min_keysize		= SERPENT_MIN_KEY_SIZE,
220 		.max_keysize		= SERPENT_MAX_KEY_SIZE,
221 		.ivsize			= SERPENT_BLOCK_SIZE,
222 		.chunksize		= SERPENT_BLOCK_SIZE,
223 		.setkey			= serpent_setkey_skcipher,
224 		.encrypt		= ctr_crypt,
225 		.decrypt		= ctr_crypt,
226 	}, {
227 		.base.cra_name		= "__xts(serpent)",
228 		.base.cra_driver_name	= "__xts-serpent-avx2",
229 		.base.cra_priority	= 600,
230 		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
231 		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
232 		.base.cra_ctxsize	= sizeof(struct serpent_xts_ctx),
233 		.base.cra_module	= THIS_MODULE,
234 		.min_keysize		= 2 * SERPENT_MIN_KEY_SIZE,
235 		.max_keysize		= 2 * SERPENT_MAX_KEY_SIZE,
236 		.ivsize			= SERPENT_BLOCK_SIZE,
237 		.setkey			= xts_serpent_setkey,
238 		.encrypt		= xts_encrypt,
239 		.decrypt		= xts_decrypt,
240 	},
241 };
242 
243 static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
244 
245 static int __init init(void)
246 {
247 	const char *feature_name;
248 
249 	if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
250 		pr_info("AVX2 instructions are not detected.\n");
251 		return -ENODEV;
252 	}
253 	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
254 				&feature_name)) {
255 		pr_info("CPU feature '%s' is not supported.\n", feature_name);
256 		return -ENODEV;
257 	}
258 
259 	return simd_register_skciphers_compat(serpent_algs,
260 					      ARRAY_SIZE(serpent_algs),
261 					      serpent_simd_algs);
262 }
263 
264 static void __exit fini(void)
265 {
266 	simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
267 				  serpent_simd_algs);
268 }
269 
270 module_init(init);
271 module_exit(fini);
272 
273 MODULE_LICENSE("GPL");
274 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
275 MODULE_ALIAS_CRYPTO("serpent");
276 MODULE_ALIAS_CRYPTO("serpent-asm");
277